docs: add full-text search (#2735)

### Change Type

- [x] `patch` — Bug fix

### Test Plan

1. Make sure search (AI and regular) still works as expected.

### Release Notes

- Docs: Add full-text search.
This commit is contained in:
Mime Čuvalo 2024-02-06 09:49:31 +00:00 committed by GitHub
parent 538734782c
commit b50cda0a6e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 64 additions and 47 deletions

View file

@ -154,6 +154,10 @@ export async function GET(req: NextRequest) {
.filter((a) => a.score > bottomScore)
.sort((a, b) => b.score - a.score)
.sort((a, b) => (b.type === 'heading' ? -1 : 1) - (a.type === 'heading' ? -1 : 1))
results[section as keyof Data['results']] = results[section as keyof Data['results']].slice(
0,
10
)
})
return new Response(

View file

@ -14,10 +14,6 @@ type Data = {
const BANNED_HEADINGS = ['new', 'constructor', 'properties', 'example', 'methods']
function scoreResultBasedOnLengthSimilarity(title: string, query: string) {
return 1 - Math.min(1, Math.max(0, Math.abs(title.length - query.length) / 12))
}
export async function GET(req: NextRequest) {
const { searchParams } = new URL(req.url)
const query = searchParams.get('q')?.toLowerCase()
@ -37,22 +33,17 @@ export async function GET(req: NextRequest) {
try {
const results: Data['results'] = structuredClone(SEARCH_RESULTS)
const db = await getDb()
const queryWithoutSpaces = query.replace(/\s/g, '')
const searchForArticle = await db.db.prepare(
`
SELECT id, title, sectionId, categoryId, content
FROM articles
WHERE title LIKE '%' || ? || '%'
OR description LIKE '%' || ? || '%'
OR content LIKE '%' || ? || '%'
OR keywords LIKE '%' || ? || '%';
SELECT id, title, sectionId, categoryId, content
FROM ftsArticles
WHERE ftsArticles MATCH ?
ORDER BY bm25(ftsArticles, 1000.0)
`,
queryWithoutSpaces,
queryWithoutSpaces,
queryWithoutSpaces
query
)
await searchForArticle.all(query).then(async (queryResults) => {
await searchForArticle.all().then(async (queryResults) => {
for (const article of queryResults) {
const section = await db.getSection(article.sectionId)
const category = await db.getCategory(article.categoryId)
@ -67,7 +58,7 @@ export async function GET(req: NextRequest) {
url: isUncategorized
? `${section.id}/${article.id}`
: `${section.id}/${category.id}/${article.id}`,
score: scoreResultBasedOnLengthSimilarity(article.title, query),
score: 0,
})
}
})
@ -75,15 +66,14 @@ export async function GET(req: NextRequest) {
const searchForArticleHeadings = await db.db.prepare(
`
SELECT id, title, articleId, slug
FROM headings
WHERE title LIKE '%' || ? || '%'
OR slug LIKE '%' || ? || '%'
FROM ftsHeadings
WHERE ftsHeadings MATCH ?
ORDER BY bm25(ftsHeadings, 1000.0)
`,
queryWithoutSpaces,
queryWithoutSpaces
query
)
await searchForArticleHeadings.all(queryWithoutSpaces).then(async (queryResults) => {
await searchForArticleHeadings.all().then(async (queryResults) => {
for (const heading of queryResults) {
if (BANNED_HEADINGS.some((h) => heading.slug.endsWith(h))) continue
@ -104,14 +94,17 @@ export async function GET(req: NextRequest) {
url: isUncategorized
? `${section.id}/${article.id}#${heading.slug}`
: `${section.id}/${category.id}/${article.id}#${heading.slug}`,
score: scoreResultBasedOnLengthSimilarity(article.title, query),
score: 0,
})
}
})
Object.keys(results).forEach((section: string) =>
results[section as keyof Data['results']].sort((a, b) => b.score - a.score)
)
Object.keys(results).forEach((section: string) => {
results[section as keyof Data['results']] = results[section as keyof Data['results']].slice(
0,
20
)
})
results.articles.sort(
(a, b) => (b.type === 'heading' ? -1 : 1) - (a.type === 'heading' ? -1 : 1)

View file

@ -98,21 +98,14 @@ const Autocomplete = forwardRef(function Autocomplete(
<Icon className="autocomplete__icon" icon="search" small />
)}
<Combobox
autoSelect
placeholder="Search…"
ref={ref}
className="autocomplete__input"
value={value}
/>
<Combobox placeholder="Search…" ref={ref} className="autocomplete__input" value={value} />
{value && <ComboboxCancel className="autocomplete__cancel" />}
{value && (
<ComboboxPopover className="autocomplete__popover">
{value && options.length !== 0 && (
<ComboboxPopover sameWidth className="autocomplete__popover">
{customUI}
{options.length === 0 && <span className="autocomplete__empty">No results found.</span>}
{options.length !== 0 && renderedGroups}
{renderedGroups}
</ComboboxPopover>
)}
</div>

View file

@ -39,17 +39,22 @@ export function Search() {
const res = await fetch(endPoint)
if (res.ok) {
const json = await res.json()
const topExamples = json.results.examples.slice(0, 5)
const topArticles = json.results.articles.slice(0, 10)
const topAPI = json.results.apiDocs.slice(0, 10)
const topExamples = json.results.examples.slice(0, 10)
const topAPI = json.results.apiDocs.slice(0, 20)
const allResults = topExamples.concat(topArticles).concat(topAPI)
setSearchResults(
allResults.map((result: SearchResult) => ({
label: result.title,
value: result.url,
group: result.sectionType,
}))
)
if (allResults.length) {
setSearchResults(
allResults.map((result: SearchResult) => ({
label: result.title,
value: result.url,
group: result.sectionType,
}))
)
} else {
setSearchResults([{ label: 'No results found.', value: '#', group: 'docs' }])
}
}
} catch (err) {
console.error(err)
@ -66,6 +71,7 @@ export function Search() {
}
const handleSearchTypeChange = () => {
setSearchResults([])
setSearchType(searchType === SEARCH_TYPE.AI ? SEARCH_TYPE.NORMAL : SEARCH_TYPE.AI)
handleInputChange(query)
}

View file

@ -1,5 +1,5 @@
import { addAuthors } from '@/utils/addAuthors'
import { addContentToDb } from '@/utils/addContent'
import { addContentToDb, addFTS } from '@/utils/addContent'
import { autoLinkDocs } from '@/utils/autoLinkDocs'
import { nicelog } from '@/utils/nicelog'
import { connect } from './connect'
@ -23,6 +23,9 @@ export async function refreshContent(opts = {} as { silent: boolean }) {
if (!opts.silent) nicelog('◦ Generating / adding API content to db...')
await addContentToDb(db, await generateApiContent())
if (!opts.silent) nicelog('◦ Adding full-text search...')
await addFTS(db)
if (!opts.silent) nicelog('◦ Fixing links to API docs...')
await autoLinkDocs(db)

View file

@ -1068,6 +1068,8 @@ body {
}
.sidebar__section__links {
position: relative;
z-index: 1;
display: flex;
justify-content: space-around;
padding: 16px 0;

View file

@ -119,6 +119,22 @@ export async function addContentToDb(
}
}
export async function addFTS(db: Database<sqlite3.Database, sqlite3.Statement>) {
await db.run(`DROP TABLE IF EXISTS ftsArticles`)
await db.run(
`CREATE VIRTUAL TABLE ftsArticles USING fts5(title, content, description, keywords, id, sectionId, categoryId, tokenize="trigram")`
)
await db.run(
`INSERT INTO ftsArticles SELECT title, content, description, keywords, id, sectionId, categoryId FROM articles;`
)
await db.run(`DROP TABLE IF EXISTS ftsHeadings`)
await db.run(
`CREATE VIRTUAL TABLE ftsHeadings USING fts5(title, slug, id, articleId, tokenize="trigram")`
)
await db.run(`INSERT INTO ftsHeadings SELECT title, slug, id, articleId FROM headings;`)
}
const slugs = new GithubSlugger()
const MATCH_HEADINGS = /(?:^|\n)(#{1,6})\s+(.+?)(?=\n|$)/g