docs: add full-text search (#2735)

### Change Type

- [x] `patch` — Bug fix

### Test Plan

1. Make sure search (AI and regular) still works as expected.

### Release Notes

- Docs: Add full-text search.

apps/docs/app/api/ai/route.ts

@@ -154,6 +154,10 @@ export async function GET(req: NextRequest) {
 				.filter((a) => a.score > bottomScore)
 				.sort((a, b) => b.score - a.score)
 				.sort((a, b) => (b.type === 'heading' ? -1 : 1) - (a.type === 'heading' ? -1 : 1))
+			results[section as keyof Data['results']] = results[section as keyof Data['results']].slice(
+				0,
+				10
+			)
 		})
 
 		return new Response(

apps/docs/app/api/search/route.ts

@@ -14,10 +14,6 @@ type Data = {
 
 const BANNED_HEADINGS = ['new', 'constructor', 'properties', 'example', 'methods']
 
-function scoreResultBasedOnLengthSimilarity(title: string, query: string) {
-	return 1 - Math.min(1, Math.max(0, Math.abs(title.length - query.length) / 12))
-}
-
 export async function GET(req: NextRequest) {
 	const { searchParams } = new URL(req.url)
 	const query = searchParams.get('q')?.toLowerCase()
@@ -37,22 +33,17 @@ export async function GET(req: NextRequest) {
 	try {
 		const results: Data['results'] = structuredClone(SEARCH_RESULTS)
 		const db = await getDb()
-		const queryWithoutSpaces = query.replace(/\s/g, '')
 		const searchForArticle = await db.db.prepare(
 			`
 	SELECT id, title, sectionId, categoryId, content
-	FROM articles 
-	WHERE title LIKE '%' || ? || '%'
-		OR description LIKE '%' || ? || '%'
-		OR content LIKE '%' || ? || '%'
-		OR keywords LIKE '%' || ? || '%';
+	FROM ftsArticles
+	WHERE ftsArticles MATCH ?
+	ORDER BY bm25(ftsArticles, 1000.0)
 `,
-			queryWithoutSpaces,
-			queryWithoutSpaces,
-			queryWithoutSpaces
+			query
 		)
 
-		await searchForArticle.all(query).then(async (queryResults) => {
+		await searchForArticle.all().then(async (queryResults) => {
 			for (const article of queryResults) {
 				const section = await db.getSection(article.sectionId)
 				const category = await db.getCategory(article.categoryId)
@@ -67,7 +58,7 @@ export async function GET(req: NextRequest) {
 					url: isUncategorized
 						? `${section.id}/${article.id}`
 						: `${section.id}/${category.id}/${article.id}`,
-					score: scoreResultBasedOnLengthSimilarity(article.title, query),
+					score: 0,
 				})
 			}
 		})
@@ -75,15 +66,14 @@ export async function GET(req: NextRequest) {
 		const searchForArticleHeadings = await db.db.prepare(
 			`
 	SELECT id, title, articleId, slug
-	FROM headings 
-	WHERE title LIKE '%' || ? || '%'
-		OR slug LIKE '%' || ? || '%'
+	FROM ftsHeadings
+	WHERE ftsHeadings MATCH ?
+	ORDER BY bm25(ftsHeadings, 1000.0)
 `,
-			queryWithoutSpaces,
-			queryWithoutSpaces
+			query
 		)
 
-		await searchForArticleHeadings.all(queryWithoutSpaces).then(async (queryResults) => {
+		await searchForArticleHeadings.all().then(async (queryResults) => {
 			for (const heading of queryResults) {
 				if (BANNED_HEADINGS.some((h) => heading.slug.endsWith(h))) continue
 
@@ -104,14 +94,17 @@ export async function GET(req: NextRequest) {
 					url: isUncategorized
 						? `${section.id}/${article.id}#${heading.slug}`
 						: `${section.id}/${category.id}/${article.id}#${heading.slug}`,
-					score: scoreResultBasedOnLengthSimilarity(article.title, query),
+					score: 0,
 				})
 			}
 		})
 
-		Object.keys(results).forEach((section: string) =>
-			results[section as keyof Data['results']].sort((a, b) => b.score - a.score)
+		Object.keys(results).forEach((section: string) => {
+			results[section as keyof Data['results']] = results[section as keyof Data['results']].slice(
+				0,
+				20
 			)
+		})
 
 		results.articles.sort(
 			(a, b) => (b.type === 'heading' ? -1 : 1) - (a.type === 'heading' ? -1 : 1)

apps/docs/components/Autocomplete.tsx

@@ -98,21 +98,14 @@ const Autocomplete = forwardRef(function Autocomplete(
 					<Icon className="autocomplete__icon" icon="search" small />
 				)}
 
-				<Combobox
-					autoSelect
-					placeholder="Search…"
-					ref={ref}
-					className="autocomplete__input"
-					value={value}
-				/>
+				<Combobox placeholder="Search…" ref={ref} className="autocomplete__input" value={value} />
 
 				{value && <ComboboxCancel className="autocomplete__cancel" />}
 
-				{value && (
-					<ComboboxPopover className="autocomplete__popover">
+				{value && options.length !== 0 && (
+					<ComboboxPopover sameWidth className="autocomplete__popover">
 						{customUI}
-						{options.length === 0 && <span className="autocomplete__empty">No results found.</span>}
-						{options.length !== 0 && renderedGroups}
+						{renderedGroups}
 					</ComboboxPopover>
 				)}
 			</div>

apps/docs/components/Search.tsx

@@ -39,10 +39,12 @@ export function Search() {
 				const res = await fetch(endPoint)
 				if (res.ok) {
 					const json = await res.json()
+					const topExamples = json.results.examples.slice(0, 5)
 					const topArticles = json.results.articles.slice(0, 10)
-					const topAPI = json.results.apiDocs.slice(0, 10)
-					const topExamples = json.results.examples.slice(0, 10)
+					const topAPI = json.results.apiDocs.slice(0, 20)
 					const allResults = topExamples.concat(topArticles).concat(topAPI)
+
+					if (allResults.length) {
 						setSearchResults(
 							allResults.map((result: SearchResult) => ({
 								label: result.title,
@@ -50,6 +52,9 @@ export function Search() {
 								group: result.sectionType,
 							}))
 						)
+					} else {
+						setSearchResults([{ label: 'No results found.', value: '#', group: 'docs' }])
+					}
 				}
 			} catch (err) {
 				console.error(err)
@@ -66,6 +71,7 @@ export function Search() {
 	}
 
 	const handleSearchTypeChange = () => {
+		setSearchResults([])
 		setSearchType(searchType === SEARCH_TYPE.AI ? SEARCH_TYPE.NORMAL : SEARCH_TYPE.AI)
 		handleInputChange(query)
 	}

apps/docs/scripts/functions/refreshContent.ts

@@ -1,5 +1,5 @@
 import { addAuthors } from '@/utils/addAuthors'
-import { addContentToDb } from '@/utils/addContent'
+import { addContentToDb, addFTS } from '@/utils/addContent'
 import { autoLinkDocs } from '@/utils/autoLinkDocs'
 import { nicelog } from '@/utils/nicelog'
 import { connect } from './connect'
@@ -23,6 +23,9 @@ export async function refreshContent(opts = {} as { silent: boolean }) {
 	if (!opts.silent) nicelog('◦ Generating / adding API content to db...')
 	await addContentToDb(db, await generateApiContent())
 
+	if (!opts.silent) nicelog('◦ Adding full-text search...')
+	await addFTS(db)
+
 	if (!opts.silent) nicelog('◦ Fixing links to API docs...')
 	await autoLinkDocs(db)
 

apps/docs/styles/globals.css

@@ -1068,6 +1068,8 @@ body {
 	}
 
 	.sidebar__section__links {
+		position: relative;
+		z-index: 1;
 		display: flex;
 		justify-content: space-around;
 		padding: 16px 0;

apps/docs/utils/addContent.ts

@@ -119,6 +119,22 @@ export async function addContentToDb(
 	}
 }
 
+export async function addFTS(db: Database<sqlite3.Database, sqlite3.Statement>) {
+	await db.run(`DROP TABLE IF EXISTS ftsArticles`)
+	await db.run(
+		`CREATE VIRTUAL TABLE ftsArticles USING fts5(title, content, description, keywords, id, sectionId, categoryId, tokenize="trigram")`
+	)
+	await db.run(
+		`INSERT INTO ftsArticles SELECT title, content, description, keywords, id, sectionId, categoryId FROM articles;`
+	)
+
+	await db.run(`DROP TABLE IF EXISTS ftsHeadings`)
+	await db.run(
+		`CREATE VIRTUAL TABLE ftsHeadings USING fts5(title, slug, id, articleId, tokenize="trigram")`
+	)
+	await db.run(`INSERT INTO ftsHeadings SELECT title, slug, id, articleId FROM headings;`)
+}
+
 const slugs = new GithubSlugger()
 
 const MATCH_HEADINGS = /(?:^|\n)(#{1,6})\s+(.+?)(?=\n|$)/g