Add docs (#2470)

This PR adds the docs app back into the tldraw monorepo.

## Deploying

We'll want to update our deploy script to update the SOURCE_SHA to the
newest release sha... and then deploy the docs pulling api.json files
from that release. We _could_ update the docs on every push to main, but
we don't have to unless something has changed. Right now there's no
automated deployments from this repo.

## Side effects

To make this one work, I needed to update the lock file. This might be
ok (new year new lock file), and everything builds as expected, though
we may want to spend some time with our scripts to be sure that things
are all good.

I also updated our prettier installation, which decided to add trailing
commas to every generic type. Which is, I suppose, [correct
behavior](https://github.com/prettier/prettier-vscode/issues/955)? But
that caused diffs in every file, which is unfortunate.

### Change Type

- [x] `internal` — Any other changes that don't affect the published
package[^2]

apps/docs/scripts/functions/checkBrokenLinks.ts

@@ -0,0 +1,29 @@
+import { nicelog } from '@/utils/nicelog'
+import blc from 'broken-link-checker'
+
+const IGNORED_URLS = ['https://twitter.com/tldraw', 'https://tldraw.com']
+
+export async function checkBrokenLinks() {
+	nicelog('Checking broken links...')
+	const checked = new Set<string>()
+	const checker = new blc.SiteChecker(
+		{
+			filterLevel: 1,
+		},
+		{
+			link(result) {
+				if (IGNORED_URLS.includes(result.url.original)) return
+				if (checked.has(result.url.resolved)) return
+				// nicelog('Checking', result.url.resolved.replace('http://localhost:3001', ''))
+				if (result.broken) {
+					nicelog(`BROKEN: ${result.url.resolved} on page ${result.base.resolved}`)
+				}
+				checked.add(result.url.resolved)
+			},
+			end() {
+				nicelog('done')
+			},
+		}
+	)
+	checker.enqueue('http://localhost:3001/docs/assets', null)
+}

apps/docs/scripts/functions/connect.ts

@@ -0,0 +1,91 @@
+import path from 'path'
+import { open } from 'sqlite'
+import sqlite3 from 'sqlite3'
+
+export async function connect(opts = {} as { reset?: boolean }) {
+	const db = await open({
+		filename: path.join(process.cwd(), 'content.db'),
+		driver: sqlite3.Database,
+	})
+
+	if (opts.reset) {
+		// Create the authors table if it does not exist
+
+		await db.run(`DROP TABLE IF EXISTS authors`)
+		await db.run(`CREATE TABLE IF NOT EXISTS authors (
+			id TEXT PRIMARY_KEY,
+			name TEXT NOT NULL,
+			email TEXT NOT NULL,
+			twitter TEXT NOT NULL,
+			image TEXT NOT NULL
+		)`)
+
+		// Create the sections table if it does not exist
+
+		await db.run(`DROP TABLE IF EXISTS sections`)
+		await db.run(`CREATE TABLE sections (
+			id TEXT PRIMARY KEY,
+			idx INTEGER NOT NULL,
+			title TEXT NOT NULL,
+			path TEXT NOT NULL,
+			description TEXT,
+			sidebar_behavior TEXT
+		)`)
+
+		// Create the categories table if it does not exist
+
+		await db.run(`DROP TABLE IF EXISTS categories`)
+		await db.run(`CREATE TABLE IF NOT EXISTS categories (
+			id TEXT PRIMARY KEY,
+			title TEXT NOT NULL,
+			description TEXT,
+			sectionId TEXT NOT NULL,
+			sectionIndex INTEGER NOT NULL,
+			path TEXT,
+			FOREIGN KEY (id) REFERENCES sections(id)
+		)`)
+
+		// Create the articles table if it does not exist
+
+		// drop the table if it exists
+
+		await db.run(`DROP TABLE IF EXISTS articles`)
+		await db.run(`CREATE TABLE IF NOT EXISTS articles (
+			id TEXT PRIMARY KEY,
+			groupIndex INTEGER NOT NULL,
+			categoryIndex INTEGER NOT NULL,
+			sectionIndex INTEGER NOT NULL,
+			groupId TEXT,
+			categoryId TEXT NOT NULL,
+			sectionId TEXT NOT NULL,
+			authorId TEXT NOT NULL,
+			title TEXT NOT NULL,
+			description TEXT,
+			hero TEXT,
+			status TEXT NOT NULL,
+			date TEXT,
+			sourceUrl TEXT,
+			keywords TEXT,
+			content TEXT NOT NULL,
+			path TEXT,
+			FOREIGN KEY (authorId) REFERENCES authors(id),
+			FOREIGN KEY (sectionId) REFERENCES sections(id),
+			FOREIGN KEY (categoryId) REFERENCES categories(id)
+		)`)
+
+		await db.run(`DROP TABLE IF EXISTS headings`)
+		await db.run(`CREATE TABLE IF NOT EXISTS headings (
+			id INTEGER PRIMARY KEY AUTOINCREMENT,
+			idx INTEGER NOT NULL,
+			articleId TEXT NOT NULL,
+			level INTEGER NOT NULL,
+			title TEXT NOT NULL,
+			slug TEXT NOT NULL,
+			isCode BOOL NOT NULL,
+			path TEXT NOT NULL,
+			FOREIGN KEY (articleId) REFERENCES articles(id)
+		)`)
+	}
+
+	return db
+}

apps/docs/scripts/functions/createApiMarkdown.ts

@@ -0,0 +1,93 @@
+import { InputSection } from '@/types/content-types'
+import { nicelog } from '@/utils/nicelog'
+import { ApiModel } from '@microsoft/api-extractor-model'
+import fs from 'fs'
+import path from 'path'
+import { CONTENT_DIR, getSlug } from '../utils'
+import { getApiMarkdown } from './getApiMarkdown'
+
+export async function createApiMarkdown() {
+	const apiInputSection: InputSection = {
+		id: 'gen' as string,
+		title: 'API Reference',
+		description: "Reference for the tldraw package's APIs (generated).",
+		categories: [],
+		sidebar_behavior: 'show-title',
+	}
+
+	const addedCategories = new Set<string>()
+
+	const INPUT_DIR = path.join(process.cwd(), 'api')
+	const OUTPUT_DIR = path.join(CONTENT_DIR, 'gen')
+
+	if (fs.existsSync(OUTPUT_DIR)) {
+		fs.rmSync(OUTPUT_DIR, { recursive: true })
+	}
+
+	fs.mkdirSync(OUTPUT_DIR)
+
+	const model = new ApiModel()
+	const packageModels = []
+
+	// get all files in the INPUT_DIR
+	const files = fs.readdirSync(INPUT_DIR)
+	for (const file of files) {
+		// get the file path
+		const filePath = path.join(INPUT_DIR, file)
+
+		// parse the file
+		const apiModel = model.loadPackage(filePath)
+
+		// add the parsed file to the packageModels array
+		packageModels.push(apiModel)
+	}
+
+	await Promise.allSettled(
+		packageModels.map(async (packageModel) => {
+			const categoryName = packageModel.name.replace(`@tldraw/`, '')
+
+			if (!addedCategories.has(categoryName)) {
+				apiInputSection.categories!.push({
+					id: categoryName,
+					title: packageModel.name,
+					description: '',
+					groups: [
+						'Namespace',
+						'Class',
+						'Function',
+						'Variable',
+						'Enum',
+						'Interface',
+						'TypeAlias',
+					].map((title) => ({
+						id: title,
+						path: null,
+					})),
+				})
+				addedCategories.add(categoryName)
+			}
+
+			const entrypoint = packageModel.entryPoints[0]
+
+			for (let j = 0; j < entrypoint.members.length; j++) {
+				const item = entrypoint.members[j]
+
+				const result = await getApiMarkdown(categoryName, item, j)
+				const outputFileName = `${getSlug(item)}.mdx`
+				nicelog(`✎ ${outputFileName}`)
+				fs.writeFileSync(path.join(OUTPUT_DIR, outputFileName), result.markdown)
+			}
+		})
+	)
+
+	// Add the API section to the sections.json file
+
+	const sectionsJsonPath = path.join(CONTENT_DIR, 'sections.json')
+	const sectionsJson = JSON.parse(fs.readFileSync(sectionsJsonPath, 'utf8')) as InputSection[]
+	sectionsJson.splice(
+		sectionsJson.findIndex((s) => s.id === 'gen'),
+		1
+	)
+	sectionsJson.push(apiInputSection)
+	fs.writeFileSync(sectionsJsonPath, JSON.stringify(sectionsJson, null, 2))
+}

apps/docs/scripts/functions/fetchApiSource.ts

@@ -0,0 +1,52 @@
+import fs from 'fs'
+import { Octokit } from 'octokit'
+import path from 'path'
+import { TLDRAW_PACKAGES_TO_INCLUDE_IN_DOCS } from './package-list'
+
+const octokit = new Octokit({
+	auth: process.env.GITHUB_ACCESS_TOKEN,
+})
+
+const { log: nicelog } = console
+
+export async function fetchApiSource() {
+	try {
+		const API_DIRECTORY = path.join(process.cwd(), 'api')
+
+		if (fs.existsSync(API_DIRECTORY)) {
+			fs.rmSync(API_DIRECTORY, { recursive: true })
+		}
+
+		fs.mkdirSync(API_DIRECTORY)
+
+		for (const folderName of TLDRAW_PACKAGES_TO_INCLUDE_IN_DOCS) {
+			const filePath = path.join(API_DIRECTORY, folderName + '.api.json')
+
+			nicelog(`• Fetching API for ${folderName}...`)
+
+			const res = await octokit.request('GET /repos/{owner}/{repo}/contents/{path}', {
+				owner: 'tldraw',
+				repo: 'tldraw',
+				path: `packages/${folderName}/api/api.json`,
+				branch: process.env.SOURCE_SHA || 'main',
+				headers: {
+					'X-GitHub-Api-Version': '2022-11-28',
+					accept: 'application/vnd.github.VERSION.raw',
+				},
+			})
+
+			if (res.status === 200) {
+				nicelog(`• Writing ${filePath}...`)
+				fs.writeFileSync(filePath, (res as any).data)
+			} else {
+				throw Error(`x Could not get API for ${folderName}.`)
+			}
+		}
+
+		nicelog('✔ Complete!')
+	} catch (error) {
+		nicelog(`x Could not generate site content.`)
+
+		throw error
+	}
+}

apps/docs/scripts/functions/fetchReleases.ts

@@ -0,0 +1,73 @@
+import fs from 'fs'
+import { Octokit } from 'octokit'
+import path from 'path'
+
+const octokit = new Octokit({
+	auth: process.env.GITHUB_ACCESS_TOKEN,
+})
+
+const { log: nicelog } = console
+
+export async function fetchReleases() {
+	try {
+		const RELEASES_DIRECTORY = path.join(process.cwd(), 'content', 'releases')
+
+		if (fs.existsSync(RELEASES_DIRECTORY)) {
+			fs.rmSync(RELEASES_DIRECTORY, { recursive: true })
+		}
+
+		fs.mkdirSync(RELEASES_DIRECTORY)
+
+		const res = await octokit.rest.repos.listReleases({
+			owner: 'tldraw',
+			repo: 'tldraw',
+		})
+
+		if (res.status === 200) {
+			nicelog(`• Writing releases...`)
+			res.data
+				.filter((release) => !release.draft && release.tag_name.startsWith('v2.0.0'))
+				.forEach((release, i) => {
+					const date = (
+						release.published_at ? new Date(release.published_at) : new Date()
+					).toLocaleDateString('en-US', {
+						year: 'numeric',
+						month: 'numeric',
+						day: 'numeric',
+					})
+
+					let m = ''
+
+					m += `---\n`
+					m += `title: ${release.tag_name}\n`
+					m += `description: Examples\n`
+					m += `author: tldraw\n`
+					m += `date: ${date}\n`
+					m += `order: ${i}\n`
+					m += `status: published\n`
+					m += `---\n\n`
+					m += `[View on GitHub](${release.html_url})\n\n`
+
+					const body = (release.body ?? '')
+						.replaceAll(/### Release Notes\n/g, '')
+						.replaceAll(/\[([^\]]+)\]$/g, '$1')
+						.replace(/<image (.*)">/g, '<image $1" />')
+						.replace(/<([^>]+)\/?>(?=\s|$)/g, '`<$1>`')
+						.replace(/`<image(.*) \/>`/g, '<image$1 />')
+						.replace(/`<img(.*) \/>`/g, '<img$1 />')
+						.replace(/\/\/>/g, '/>')
+
+					m += body
+
+					const filePath = path.join(RELEASES_DIRECTORY, `${release.tag_name}.mdx`)
+					fs.writeFileSync(filePath, m)
+				})
+		} else {
+			throw Error(`x Could not get releases for @tldraw/tldraw.`)
+		}
+	} catch (error) {
+		nicelog(`x Could not generate release content.`)
+
+		throw error
+	}
+}

apps/docs/scripts/functions/generateApiContent.ts

@@ -0,0 +1,25 @@
+import path from 'path'
+import { Articles, GeneratedContent, InputSection } from '../../types/content-types'
+import { generateSection } from './generateSection'
+
+const { log: nicelog } = console
+
+export async function generateApiContent(): Promise<GeneratedContent> {
+	const articles: Articles = {}
+	const CONTENT_DIRECTORY = path.join(process.cwd(), 'content')
+	// eslint-disable-next-line @typescript-eslint/no-var-requires
+	const sections = require(path.join(CONTENT_DIRECTORY, 'sections.json')) as InputSection[]
+
+	try {
+		const inputApiSection = sections.find((s) => s.id === 'gen')
+		if (!inputApiSection) throw new Error(`Could not find section with id 'gen'`)
+		const outputApiSection = generateSection(inputApiSection, articles, 999999) // always at the end!
+		const contentComplete = { sections: [outputApiSection], articles }
+
+		return contentComplete
+	} catch (error) {
+		nicelog(`x Could not generate API content`)
+
+		throw error
+	}
+}

apps/docs/scripts/functions/generateContent.ts

@@ -0,0 +1,21 @@
+import { GeneratedContent, InputSection } from '@/types/content-types'
+import path from 'path'
+import { generateSection } from './generateSection'
+
+export async function generateContent() {
+	const CONTENT_DIRECTORY = path.join(process.cwd(), 'content')
+	// eslint-disable-next-line @typescript-eslint/no-var-requires
+	const sections = require(path.join(CONTENT_DIRECTORY, 'sections.json')) as InputSection[]
+
+	const result: GeneratedContent = {
+		articles: {},
+		sections: [],
+	}
+
+	for (let i = 0; i < sections.length; i++) {
+		if (sections[i].id === 'gen') continue
+		result.sections.push(generateSection(sections[i], result.articles, i))
+	}
+
+	return result
+}

apps/docs/scripts/functions/generateSection.ts

@@ -0,0 +1,179 @@
+import fs from 'fs'
+import matter from 'gray-matter'
+import path from 'path'
+import {
+	Article,
+	ArticleStatus,
+	Articles,
+	Category,
+	InputSection,
+	Section,
+} from '../../types/content-types'
+import { CONTENT_DIR } from '../utils'
+
+export function generateSection(section: InputSection, articles: Articles, index: number): Section {
+	// Uncategorized articles
+	const _ucg: Article[] = []
+
+	// A temporary table of articles mapped to categories
+	const _categoryArticles: Record<string, Article[]> = Object.fromEntries(
+		section.categories.map((category) => [category.id, []])
+	)
+
+	// The file directory for this section
+	const dir = path.join(CONTENT_DIR, section.id)
+	const files = fs.readdirSync(dir, { withFileTypes: false })
+
+	const isGenerated = section.id === 'gen'
+
+	for (const file of files) {
+		const filename = file.toString()
+		const fileContent = fs.readFileSync(path.join(dir, filename)).toString()
+		const extension = path.extname(filename)
+		const articleId = filename.replace(extension, '')
+
+		const parsed = matter({ content: fileContent }, {})
+
+		// If we're in prod and the article isn't published, skip it
+		if (process.env.NODE_ENV !== 'development' && parsed.data.status !== 'published') {
+			continue
+		}
+
+		// If a category was provided but that category was not found in the section, throw an error
+		const category =
+			parsed.data.category && section.categories.find((c) => c.id === parsed.data.category)
+
+		// By default, the category is ucg (uncategorized, with the section id in the id)
+		const { category: categoryId = section.id + '_ucg', author = 'api' } = parsed.data
+
+		const isUncategorized = categoryId === section.id + '_ucg'
+		const isIndex = articleId === section.id
+
+		const article: Article = {
+			id: articleId,
+			type: 'article',
+			sectionIndex: 0,
+			groupIndex: -1,
+			groupId: parsed.data.group ?? null,
+			categoryIndex: parsed.data.order ?? -1,
+			sectionId: section.id,
+			author,
+			categoryId,
+			status: parsed.data.status ?? ArticleStatus.Draft,
+			title: parsed.data.title ?? 'Untitled article',
+			description: parsed.data.description,
+			hero: parsed.data.hero ?? null,
+			date: parsed.data.date ? new Date(parsed.data.date).toISOString() : null,
+			keywords: parsed.data.keywords ?? [],
+			sourceUrl: isGenerated // if it's a generated API doc, then we don't have a link
+				? parsed.data.sourceUrl ?? null
+				: `${section.id}/${articleId}${extension}`,
+			content: parsed.content,
+			path:
+				section.id === 'getting-started'
+					? `/${articleId}`
+					: isUncategorized
+						? `/${section.id}/${articleId}`
+						: `/${section.id}/${categoryId}/${articleId}`,
+		}
+
+		if (isIndex) {
+			article.categoryIndex = -1
+			article.sectionIndex = -1
+			articles[section.id + '_index'] = article
+		} else {
+			if (category) {
+				if (article.id === category.id) {
+					article.categoryIndex = -1
+					article.sectionIndex = -1
+					articles[category.id + '_index'] = article
+				} else {
+					_categoryArticles[categoryId].push(article)
+				}
+			} else {
+				_ucg.push(article)
+			}
+		}
+	}
+
+	let sectionIndex = 0
+
+	// Sort ucg articles by date and add them to the articles table
+	_ucg.sort(sortArticles).forEach((article, i) => {
+		article.categoryIndex = i
+		article.sectionIndex = sectionIndex
+		articles[article.id] = article
+		sectionIndex++
+	})
+
+	const categories: Category[] = [
+		{
+			id: section.id + '_ucg',
+			type: 'category',
+			sectionId: section.id,
+			index: 0,
+			title: 'Uncategorized',
+			description: 'Articles that do not belong to a category.',
+			groups: [],
+			path: `/${section.id}/ucg`,
+			content: null,
+		},
+	]
+
+	// Sort categorized articles by date and add them to the articles table
+	section.categories.forEach((inputCategory, i) => {
+		const categoryArticles = _categoryArticles[inputCategory.id]
+
+		categoryArticles.sort(sortArticles).forEach((article, i) => {
+			article.categoryIndex = i
+			article.sectionIndex = sectionIndex
+			articles[article.id] = article
+			sectionIndex++
+		})
+
+		if (categoryArticles.length) {
+			categories.push({
+				...inputCategory,
+				type: 'category',
+				sectionId: section.id,
+				index: i + 1,
+				path: `/${section.id}/${inputCategory.id}`,
+				content: null,
+				groups: inputCategory.groups.map(({ id }, i) => ({
+					id,
+					title: id,
+					index: i,
+					type: 'group',
+					sectionId: section.id,
+					categoryId: inputCategory.id,
+					description: null,
+					content: null,
+					path: `/${section.id}/${inputCategory.id}/${id}`,
+				})),
+			})
+		}
+	})
+
+	return {
+		...section,
+		type: 'section',
+		sidebar_behavior: section.sidebar_behavior,
+		index,
+		categories,
+		content: '',
+		path: `/${section.id}`,
+	}
+}
+
+const sortArticles = (articleA: Article, articleB: Article) => {
+	const { categoryIndex: categoryIndexA, date: dateA = '01/01/1970' } = articleA
+	const { categoryIndex: categoryIndexB, date: dateB = '01/01/1970' } = articleB
+
+	return categoryIndexA === categoryIndexB
+		? new Date(dateB!).getTime() > new Date(dateA!).getTime()
+			? 1
+			: -1
+		: categoryIndexA < categoryIndexB
+			? -1
+			: 1
+}

apps/docs/scripts/functions/getApiMarkdown.ts

@@ -0,0 +1,427 @@
+import {
+	ApiClass,
+	ApiConstructSignature,
+	ApiConstructor,
+	ApiDeclaredItem,
+	ApiDocumentedItem,
+	ApiEnum,
+	ApiFunction,
+	ApiInterface,
+	ApiItem,
+	ApiItemKind,
+	ApiMethod,
+	ApiMethodSignature,
+	ApiNamespace,
+	ApiProperty,
+	ApiPropertySignature,
+	ApiReadonlyMixin,
+	ApiReleaseTagMixin,
+	ApiStaticMixin,
+	ApiTypeAlias,
+	ApiVariable,
+	Excerpt,
+	ReleaseTag,
+} from '@microsoft/api-extractor-model'
+import { MarkdownWriter, formatWithPrettier, getPath, getSlug } from '../utils'
+
+type Result = { markdown: string; keywords: string[] }
+
+const REPO_URL = 'https://github.com/tldraw/tldraw/blob/main/'
+
+const date = new Intl.DateTimeFormat('en-US', {
+	year: 'numeric',
+	month: '2-digit',
+	day: '2-digit',
+}).format(new Date())
+
+export async function getApiMarkdown(categoryName: string, item: ApiItem, j: number) {
+	const result: Result = { markdown: '', keywords: [] }
+	const toc: Result = { markdown: '', keywords: [] }
+	const membersResult: Result = { markdown: '', keywords: [] }
+
+	if (item.members) {
+		const constructors = []
+		const properties = []
+		const methods = []
+		for (const member of item.members) {
+			switch (member.kind) {
+				case ApiItemKind.Constructor:
+				case ApiItemKind.ConstructSignature:
+					constructors.push(member)
+					break
+				case ApiItemKind.Variable:
+				case ApiItemKind.Property:
+				case ApiItemKind.PropertySignature:
+					properties.push(member)
+					break
+				case ApiItemKind.Method:
+				case ApiItemKind.Function:
+				case ApiItemKind.MethodSignature:
+					methods.push(member)
+
+					break
+				case ApiItemKind.EnumMember:
+				case ApiItemKind.Class:
+				case ApiItemKind.TypeAlias:
+				case ApiItemKind.Interface:
+					// TODO: document these
+					break
+				default:
+					throw new Error(`Unknown member kind: ${member.kind} ${member.displayName}`)
+			}
+		}
+
+		const constructorResult: Result = { markdown: '', keywords: [] }
+		const propertiesResult: Result = { markdown: '', keywords: [] }
+		const methodsResult: Result = { markdown: '', keywords: [] }
+
+		if (constructors.length) {
+			for (const member of constructors) {
+				await addMarkdownForMember(constructorResult, member)
+				addHorizontalRule(constructorResult)
+			}
+			addMarkdown(membersResult, constructorResult.markdown)
+		}
+
+		if (properties.length) {
+			addMarkdown(toc, `- [Properties](#properties)\n`)
+			addMarkdown(propertiesResult, `## Properties\n\n`)
+			for (const member of properties) {
+				const slug = getSlug(member)
+				addMarkdown(toc, `  - [${member.displayName}](#${slug})\n`)
+				await addMarkdownForMember(propertiesResult, member)
+				addHorizontalRule(propertiesResult)
+			}
+			addMarkdown(membersResult, propertiesResult.markdown)
+		}
+
+		if (methods.length) {
+			addMarkdown(toc, `- [Methods](#methods)\n`)
+			addMarkdown(methodsResult, `## Methods\n\n`)
+			for (const member of methods) {
+				const slug = getSlug(member)
+				addMarkdown(toc, `  - [${member.displayName}](#${slug})\n`)
+				await addMarkdownForMember(methodsResult, member)
+				addHorizontalRule(methodsResult)
+			}
+			addMarkdown(membersResult, methodsResult.markdown)
+		}
+	}
+
+	await addFrontmatter(result, item, categoryName, j)
+
+	if (toc.markdown.length) {
+		result.markdown += `<details class="article__table-of-contents">\n\t<summary>Table of contents</summary>\n`
+		addMarkdown(result, toc.markdown)
+		result.markdown += `</details>\n\n`
+	}
+
+	addTags(result, item)
+
+	await addDocComment(result, item)
+
+	addReferences(result, item)
+	addLinkToSource(result, item)
+
+	if (membersResult.markdown.length) {
+		addHorizontalRule(result)
+		addMarkdown(result, membersResult.markdown)
+	}
+
+	return result
+}
+
+/* --------------------- Helpers -------------------- */
+
+function addMarkdown(result: Result, markdown: string) {
+	result.markdown += markdown
+}
+
+async function addMarkdownForMember(result: Result, member: ApiItem) {
+	if (member.displayName.startsWith('_')) return
+	addMemberName(result, member)
+	addTags(result, member)
+	await addDocComment(result, member)
+	addReferences(result, member)
+	addLinkToSource(result, member)
+}
+
+async function addFrontmatter(
+	result: Result,
+	member: ApiItem,
+	categoryName: string,
+	order: number
+) {
+	let description = ''
+	if (member instanceof ApiDocumentedItem && member.tsdocComment) {
+		const comment = await MarkdownWriter.docNodeToMarkdown(
+			member,
+			member.tsdocComment.summarySection
+		)
+		// only up to the first newline
+		description = comment.trim().split('\n')[0].replace(/:/g, '')
+	}
+
+	let kw = ''
+
+	if (result.keywords.length) {
+		kw += `\nkeywords:`
+		for (const k of result.keywords) {
+			if (k.startsWith('_')) continue
+			kw += `\n  - ${k.trim().split('\n')[0]}`
+		}
+	}
+
+	result.markdown += `---
+title: ${member.displayName}
+status: published
+description: ${description}
+category: ${categoryName}
+group: ${member.kind}
+author: api
+date: ${date}
+order: ${order}
+sourceUrl: ${'_fileUrlPath' in member ? member._fileUrlPath : ''}${kw}
+---
+`
+}
+
+function addHorizontalRule(result: Result) {
+	result.markdown += `---\n\n`
+}
+
+function addMemberName(result: Result, member: ApiItem) {
+	if (member.kind === 'Constructor') {
+		result.markdown += `### Constructor\n\n`
+		return
+	}
+
+	if (!member.displayName) return
+	result.markdown += `### \`${member.displayName}${member.kind === 'Method' ? '()' : ''}\`\n\n`
+}
+
+async function addDocComment(result: Result, member: ApiItem) {
+	if (!(member instanceof ApiDocumentedItem)) {
+		return
+	}
+
+	if (member.tsdocComment) {
+		result.markdown += await MarkdownWriter.docNodeToMarkdown(
+			member,
+			member.tsdocComment.summarySection
+		)
+
+		const exampleBlocks = member.tsdocComment.customBlocks.filter(
+			(block) => block.blockTag.tagNameWithUpperCase === '@EXAMPLE'
+		)
+
+		if (exampleBlocks.length) {
+			result.markdown += `\n\n`
+			result.markdown += `##### Example\n\n`
+			for (const example of exampleBlocks) {
+				result.markdown += await MarkdownWriter.docNodeToMarkdown(member, example.content)
+			}
+		}
+	}
+
+	if (
+		member instanceof ApiVariable ||
+		member instanceof ApiTypeAlias ||
+		member instanceof ApiProperty ||
+		member instanceof ApiPropertySignature ||
+		member instanceof ApiClass ||
+		member instanceof ApiFunction ||
+		member instanceof ApiInterface ||
+		member instanceof ApiEnum ||
+		member instanceof ApiNamespace ||
+		member instanceof ApiMethod
+	) {
+		result.markdown += `<ApiHeading>Signature</ApiHeading>\n\n`
+		result.markdown += await typeExcerptToMarkdown(member.excerpt, {
+			kind: member.kind,
+		})
+		result.markdown += `\n\n`
+	}
+
+	if (
+		member instanceof ApiMethod ||
+		member instanceof ApiMethodSignature ||
+		member instanceof ApiConstructor ||
+		member instanceof ApiConstructSignature ||
+		member instanceof ApiFunction
+	) {
+		if (!member.parameters.length) {
+			return
+		} else {
+			result.markdown += `<ApiHeading>Parameters</ApiHeading>\n\n`
+			result.markdown += '<ParametersTable>\n\n'
+			for (const param of member.parameters) {
+				result.markdown += '<ParametersTableRow>\n'
+				result.markdown += '<ParametersTableName>\n\n'
+				result.markdown += `\`${param.name}\`\n\n`
+				result.markdown += `</ParametersTableName>\n`
+				result.markdown += `<ParametersTableDescription>\n\n`
+				result.markdown += await typeExcerptToMarkdown(param.parameterTypeExcerpt, {
+					kind: 'ParameterType',
+					printWidth: 60,
+				})
+				result.markdown += `\n\n`
+				if (param.tsdocParamBlock) {
+					result.markdown += await MarkdownWriter.docNodeToMarkdown(
+						member,
+						param.tsdocParamBlock.content
+					)
+				}
+				result.markdown += `\n\n</ParametersTableDescription>\n`
+				result.markdown += `</ParametersTableRow>\n`
+			}
+			result.markdown += '</ParametersTable>\n\n'
+		}
+
+		if (!(member instanceof ApiConstructor)) {
+			result.markdown += `<ApiHeading>Returns</ApiHeading>\n\n`
+			result.markdown += await typeExcerptToMarkdown(member.returnTypeExcerpt, {
+				kind: 'ReturnType',
+			})
+			result.markdown += `\n\n`
+			if (member.tsdocComment && member.tsdocComment.returnsBlock) {
+				result.markdown += await MarkdownWriter.docNodeToMarkdown(
+					member,
+					member.tsdocComment.returnsBlock.content
+				)
+			}
+		}
+	} else if (
+		member instanceof ApiVariable ||
+		member instanceof ApiTypeAlias ||
+		member instanceof ApiProperty ||
+		member instanceof ApiPropertySignature ||
+		member instanceof ApiClass ||
+		member instanceof ApiInterface ||
+		member instanceof ApiEnum ||
+		member instanceof ApiNamespace
+	) {
+		const params = member.tsdocComment?.params
+		if (params && params.count > 0) {
+			result.markdown += `<ApiHeading>Parameters</ApiHeading>\n\n`
+			result.markdown += '<ParametersTable>\n\n'
+			for (const block of params.blocks) {
+				result.markdown += '<ParametersTableRow>\n'
+				result.markdown += '<ParametersTableName>\n\n'
+				result.markdown += `\`${block.parameterName}\`\n\n`
+				result.markdown += `</ParametersTableName>\n`
+				result.markdown += `<ParametersTableDescription>\n\n`
+				result.markdown += await MarkdownWriter.docNodeToMarkdown(member, block.content)
+				result.markdown += `\n\n</ParametersTableDescription>\n`
+				result.markdown += `</ParametersTableRow>\n`
+			}
+			result.markdown += '</ParametersTable>\n\n'
+		}
+	} else {
+		throw new Error('unknown member kind: ' + member.kind)
+	}
+}
+
+async function typeExcerptToMarkdown(
+	excerpt: Excerpt,
+	{ kind, printWidth }: { kind: ApiItemKind | 'ReturnType' | 'ParameterType'; printWidth?: number }
+) {
+	let code = ''
+	for (const token of excerpt.spannedTokens) {
+		code += token.text
+	}
+
+	code = code.replace(/^export /, '')
+	code = code.replace(/^declare /, '')
+
+	switch (kind) {
+		case ApiItemKind.CallSignature:
+		case ApiItemKind.EntryPoint:
+		case ApiItemKind.EnumMember:
+		case ApiItemKind.Function:
+		case ApiItemKind.Model:
+		case ApiItemKind.Namespace:
+		case ApiItemKind.None:
+		case ApiItemKind.Package:
+		case ApiItemKind.TypeAlias:
+			code = await formatWithPrettier(code, { printWidth })
+			break
+		case 'ReturnType':
+		case 'ParameterType':
+			code = await formatWithPrettier(`type X = () =>${code}`, { printWidth })
+			if (!code.startsWith('type X = () =>')) {
+				throw Error()
+			}
+			code = code = code.replace(/^type X = \(\) =>[ \n]/, '')
+			break
+		case ApiItemKind.Class:
+		case ApiItemKind.Enum:
+		case ApiItemKind.Interface:
+			code = await formatWithPrettier(`${code} {}`, { printWidth })
+			break
+		case ApiItemKind.Constructor:
+		case ApiItemKind.ConstructSignature:
+		case ApiItemKind.IndexSignature:
+		case ApiItemKind.Method:
+		case ApiItemKind.MethodSignature:
+		case ApiItemKind.Property:
+		case ApiItemKind.PropertySignature:
+		case ApiItemKind.Variable:
+			code = await formatWithPrettier(`class X { ${code} }`, { printWidth })
+			if (!(code.startsWith('class X {\n') && code.endsWith('\n}'))) {
+				throw Error()
+			}
+			code = code.slice('class X {\n'.length, -'\n}'.length)
+			code = code.replace(/^ {2}/gm, '')
+			break
+		default:
+			throw Error()
+	}
+
+	return ['```ts', code, '```'].join('\n')
+}
+
+function addTags(result: Result, member: ApiItem) {
+	const tags = []
+	if (ApiReleaseTagMixin.isBaseClassOf(member)) {
+		tags.push(ReleaseTag[member.releaseTag])
+	}
+	if (ApiStaticMixin.isBaseClassOf(member) && member.isStatic) {
+		tags.push('static')
+	}
+	if (ApiReadonlyMixin.isBaseClassOf(member) && member.isReadonly) {
+		tags.push('readonly')
+	}
+	tags.push(member.kind.toLowerCase())
+	result.markdown += `<Small>${tags.join(' ')}</Small>\n\n`
+}
+
+function addReferences(result: Result, member: ApiItem) {
+	if (!(member instanceof ApiDeclaredItem)) return
+	const references = new Set<string>()
+
+	member.excerptTokens.forEach((token) => {
+		if (token.kind !== 'Reference') return
+		const apiItemResult = member
+			.getAssociatedModel()!
+			.resolveDeclarationReference(token.canonicalReference!, member)
+		if (apiItemResult.errorMessage) {
+			return
+		}
+		const apiItem = apiItemResult.resolvedApiItem!
+		const url = `/gen/${getPath(apiItem)}`
+		references.add(`[${token.text}](${url})`)
+	})
+
+	if (references.size) {
+		result.markdown += `<ApiHeading>References</ApiHeading>\n\n`
+		result.markdown += Array.from(references).join(', ') + '\n\n'
+	}
+}
+
+function addLinkToSource(result: Result, member: ApiItem) {
+	if ('_fileUrlPath' in member && member._fileUrlPath) {
+		result.markdown += `<ApiHeading>Source</ApiHeading>\n\n`
+		result.markdown += `[${member._fileUrlPath}](${REPO_URL}${member._fileUrlPath})\n\n`
+	}
+}

apps/docs/scripts/functions/getVectorDbStats.ts

@@ -0,0 +1,7 @@
+import { getVectorDb } from '@/utils/ContentVectorDatabase'
+import { nicelog } from '@/utils/nicelog'
+
+export async function getVectorDbStats() {
+	const db = await getVectorDb()
+	nicelog(await db.index.getIndexStats())
+}

apps/docs/scripts/functions/package-list.ts

@@ -0,0 +1,7 @@
+export const TLDRAW_PACKAGES_TO_INCLUDE_IN_DOCS = [
+	'editor',
+	'store',
+	'tldraw',
+	'tlschema',
+	'validate',
+]

apps/docs/scripts/functions/refreshContent.ts

@@ -0,0 +1,26 @@
+import { addAuthors } from '@/utils/addAuthors'
+import { addContentToDb } from '@/utils/addContent'
+import { autoLinkDocs } from '@/utils/autoLinkDocs'
+import { nicelog } from '@/utils/nicelog'
+import { connect } from './connect'
+import { generateApiContent } from './generateApiContent'
+import { generateContent } from './generateContent'
+
+export async function refreshContent(opts = {} as { silent: boolean }) {
+	if (!opts.silent) nicelog('◦ Resetting database...')
+	const db = await connect({ reset: true })
+
+	if (!opts.silent) nicelog('◦ Adding authors to db...')
+	await addAuthors(db, await require('../../content/authors.json'))
+
+	if (!opts.silent) nicelog('◦ Generating / adding regular content to db...')
+	await addContentToDb(db, await generateContent())
+
+	if (!opts.silent) nicelog('◦ Generating / adding API content to db...')
+	await addContentToDb(db, await generateApiContent())
+
+	if (!opts.silent) nicelog('◦ Fixing links to API docs...')
+	await autoLinkDocs(db)
+
+	if (!opts.silent) nicelog('✔ Complete')
+}