docs: add full-text search (#2735)
### Change Type - [x] `patch` — Bug fix ### Test Plan 1. Make sure search (AI and regular) still works as expected. ### Release Notes - Docs: Add full-text search.
This commit is contained in:
parent
538734782c
commit
b50cda0a6e
7 changed files with 64 additions and 47 deletions
|
@ -154,6 +154,10 @@ export async function GET(req: NextRequest) {
|
|||
.filter((a) => a.score > bottomScore)
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.sort((a, b) => (b.type === 'heading' ? -1 : 1) - (a.type === 'heading' ? -1 : 1))
|
||||
results[section as keyof Data['results']] = results[section as keyof Data['results']].slice(
|
||||
0,
|
||||
10
|
||||
)
|
||||
})
|
||||
|
||||
return new Response(
|
||||
|
|
|
@ -14,10 +14,6 @@ type Data = {
|
|||
|
||||
const BANNED_HEADINGS = ['new', 'constructor', 'properties', 'example', 'methods']
|
||||
|
||||
function scoreResultBasedOnLengthSimilarity(title: string, query: string) {
|
||||
return 1 - Math.min(1, Math.max(0, Math.abs(title.length - query.length) / 12))
|
||||
}
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
const { searchParams } = new URL(req.url)
|
||||
const query = searchParams.get('q')?.toLowerCase()
|
||||
|
@ -37,22 +33,17 @@ export async function GET(req: NextRequest) {
|
|||
try {
|
||||
const results: Data['results'] = structuredClone(SEARCH_RESULTS)
|
||||
const db = await getDb()
|
||||
const queryWithoutSpaces = query.replace(/\s/g, '')
|
||||
const searchForArticle = await db.db.prepare(
|
||||
`
|
||||
SELECT id, title, sectionId, categoryId, content
|
||||
FROM articles
|
||||
WHERE title LIKE '%' || ? || '%'
|
||||
OR description LIKE '%' || ? || '%'
|
||||
OR content LIKE '%' || ? || '%'
|
||||
OR keywords LIKE '%' || ? || '%';
|
||||
SELECT id, title, sectionId, categoryId, content
|
||||
FROM ftsArticles
|
||||
WHERE ftsArticles MATCH ?
|
||||
ORDER BY bm25(ftsArticles, 1000.0)
|
||||
`,
|
||||
queryWithoutSpaces,
|
||||
queryWithoutSpaces,
|
||||
queryWithoutSpaces
|
||||
query
|
||||
)
|
||||
|
||||
await searchForArticle.all(query).then(async (queryResults) => {
|
||||
await searchForArticle.all().then(async (queryResults) => {
|
||||
for (const article of queryResults) {
|
||||
const section = await db.getSection(article.sectionId)
|
||||
const category = await db.getCategory(article.categoryId)
|
||||
|
@ -67,7 +58,7 @@ export async function GET(req: NextRequest) {
|
|||
url: isUncategorized
|
||||
? `${section.id}/${article.id}`
|
||||
: `${section.id}/${category.id}/${article.id}`,
|
||||
score: scoreResultBasedOnLengthSimilarity(article.title, query),
|
||||
score: 0,
|
||||
})
|
||||
}
|
||||
})
|
||||
|
@ -75,15 +66,14 @@ export async function GET(req: NextRequest) {
|
|||
const searchForArticleHeadings = await db.db.prepare(
|
||||
`
|
||||
SELECT id, title, articleId, slug
|
||||
FROM headings
|
||||
WHERE title LIKE '%' || ? || '%'
|
||||
OR slug LIKE '%' || ? || '%'
|
||||
FROM ftsHeadings
|
||||
WHERE ftsHeadings MATCH ?
|
||||
ORDER BY bm25(ftsHeadings, 1000.0)
|
||||
`,
|
||||
queryWithoutSpaces,
|
||||
queryWithoutSpaces
|
||||
query
|
||||
)
|
||||
|
||||
await searchForArticleHeadings.all(queryWithoutSpaces).then(async (queryResults) => {
|
||||
await searchForArticleHeadings.all().then(async (queryResults) => {
|
||||
for (const heading of queryResults) {
|
||||
if (BANNED_HEADINGS.some((h) => heading.slug.endsWith(h))) continue
|
||||
|
||||
|
@ -104,14 +94,17 @@ export async function GET(req: NextRequest) {
|
|||
url: isUncategorized
|
||||
? `${section.id}/${article.id}#${heading.slug}`
|
||||
: `${section.id}/${category.id}/${article.id}#${heading.slug}`,
|
||||
score: scoreResultBasedOnLengthSimilarity(article.title, query),
|
||||
score: 0,
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
Object.keys(results).forEach((section: string) =>
|
||||
results[section as keyof Data['results']].sort((a, b) => b.score - a.score)
|
||||
)
|
||||
Object.keys(results).forEach((section: string) => {
|
||||
results[section as keyof Data['results']] = results[section as keyof Data['results']].slice(
|
||||
0,
|
||||
20
|
||||
)
|
||||
})
|
||||
|
||||
results.articles.sort(
|
||||
(a, b) => (b.type === 'heading' ? -1 : 1) - (a.type === 'heading' ? -1 : 1)
|
||||
|
|
|
@ -98,21 +98,14 @@ const Autocomplete = forwardRef(function Autocomplete(
|
|||
<Icon className="autocomplete__icon" icon="search" small />
|
||||
)}
|
||||
|
||||
<Combobox
|
||||
autoSelect
|
||||
placeholder="Search…"
|
||||
ref={ref}
|
||||
className="autocomplete__input"
|
||||
value={value}
|
||||
/>
|
||||
<Combobox placeholder="Search…" ref={ref} className="autocomplete__input" value={value} />
|
||||
|
||||
{value && <ComboboxCancel className="autocomplete__cancel" />}
|
||||
|
||||
{value && (
|
||||
<ComboboxPopover className="autocomplete__popover">
|
||||
{value && options.length !== 0 && (
|
||||
<ComboboxPopover sameWidth className="autocomplete__popover">
|
||||
{customUI}
|
||||
{options.length === 0 && <span className="autocomplete__empty">No results found.</span>}
|
||||
{options.length !== 0 && renderedGroups}
|
||||
{renderedGroups}
|
||||
</ComboboxPopover>
|
||||
)}
|
||||
</div>
|
||||
|
|
|
@ -39,17 +39,22 @@ export function Search() {
|
|||
const res = await fetch(endPoint)
|
||||
if (res.ok) {
|
||||
const json = await res.json()
|
||||
const topExamples = json.results.examples.slice(0, 5)
|
||||
const topArticles = json.results.articles.slice(0, 10)
|
||||
const topAPI = json.results.apiDocs.slice(0, 10)
|
||||
const topExamples = json.results.examples.slice(0, 10)
|
||||
const topAPI = json.results.apiDocs.slice(0, 20)
|
||||
const allResults = topExamples.concat(topArticles).concat(topAPI)
|
||||
setSearchResults(
|
||||
allResults.map((result: SearchResult) => ({
|
||||
label: result.title,
|
||||
value: result.url,
|
||||
group: result.sectionType,
|
||||
}))
|
||||
)
|
||||
|
||||
if (allResults.length) {
|
||||
setSearchResults(
|
||||
allResults.map((result: SearchResult) => ({
|
||||
label: result.title,
|
||||
value: result.url,
|
||||
group: result.sectionType,
|
||||
}))
|
||||
)
|
||||
} else {
|
||||
setSearchResults([{ label: 'No results found.', value: '#', group: 'docs' }])
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
|
@ -66,6 +71,7 @@ export function Search() {
|
|||
}
|
||||
|
||||
const handleSearchTypeChange = () => {
|
||||
setSearchResults([])
|
||||
setSearchType(searchType === SEARCH_TYPE.AI ? SEARCH_TYPE.NORMAL : SEARCH_TYPE.AI)
|
||||
handleInputChange(query)
|
||||
}
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
import { addAuthors } from '@/utils/addAuthors'
|
||||
import { addContentToDb } from '@/utils/addContent'
|
||||
import { addContentToDb, addFTS } from '@/utils/addContent'
|
||||
import { autoLinkDocs } from '@/utils/autoLinkDocs'
|
||||
import { nicelog } from '@/utils/nicelog'
|
||||
import { connect } from './connect'
|
||||
|
@ -23,6 +23,9 @@ export async function refreshContent(opts = {} as { silent: boolean }) {
|
|||
if (!opts.silent) nicelog('◦ Generating / adding API content to db...')
|
||||
await addContentToDb(db, await generateApiContent())
|
||||
|
||||
if (!opts.silent) nicelog('◦ Adding full-text search...')
|
||||
await addFTS(db)
|
||||
|
||||
if (!opts.silent) nicelog('◦ Fixing links to API docs...')
|
||||
await autoLinkDocs(db)
|
||||
|
||||
|
|
|
@ -1068,6 +1068,8 @@ body {
|
|||
}
|
||||
|
||||
.sidebar__section__links {
|
||||
position: relative;
|
||||
z-index: 1;
|
||||
display: flex;
|
||||
justify-content: space-around;
|
||||
padding: 16px 0;
|
||||
|
|
|
@ -119,6 +119,22 @@ export async function addContentToDb(
|
|||
}
|
||||
}
|
||||
|
||||
export async function addFTS(db: Database<sqlite3.Database, sqlite3.Statement>) {
|
||||
await db.run(`DROP TABLE IF EXISTS ftsArticles`)
|
||||
await db.run(
|
||||
`CREATE VIRTUAL TABLE ftsArticles USING fts5(title, content, description, keywords, id, sectionId, categoryId, tokenize="trigram")`
|
||||
)
|
||||
await db.run(
|
||||
`INSERT INTO ftsArticles SELECT title, content, description, keywords, id, sectionId, categoryId FROM articles;`
|
||||
)
|
||||
|
||||
await db.run(`DROP TABLE IF EXISTS ftsHeadings`)
|
||||
await db.run(
|
||||
`CREATE VIRTUAL TABLE ftsHeadings USING fts5(title, slug, id, articleId, tokenize="trigram")`
|
||||
)
|
||||
await db.run(`INSERT INTO ftsHeadings SELECT title, slug, id, articleId FROM headings;`)
|
||||
}
|
||||
|
||||
const slugs = new GithubSlugger()
|
||||
|
||||
const MATCH_HEADINGS = /(?:^|\n)(#{1,6})\s+(.+?)(?=\n|$)/g
|
||||
|
|
Loading…
Reference in a new issue