Inline documentation links in type excerpts (#3931)

Before: 
<img width="667" alt="Screenshot 2024-06-12 at 15 54 38"
src="https://github.com/tldraw/tldraw/assets/1489520/3a5fc43c-fa2e-4b08-8e8b-c1c66decf7fa">

After: 
<img width="654" alt="Screenshot 2024-06-12 at 15 55 10"
src="https://github.com/tldraw/tldraw/assets/1489520/8c8abcaa-f156-4be4-a5e9-d1a4eff39ff4">

Previously, when items in our documentation referred to each other in
code snippets, we'd put the links to their documentation pages in a
separate "references" section at the bottom of the docs. Generally I
find that this makes links harder to find (they're not in-context) and
adds a fair bit of visual noise to our API documentation.

This diff moves those links inline by adding a post-processing step to
our highlighted code. This is slightly more involved than I wanted it to
be (see the comments in code.tsx for an explanation of why) but it gets
the job done. I've added small link icons next to linked code items - i
experimented with underlines and a 🔗 icon too, but this seemed to look
the best.

### Change Type

- [x] `docs` — Changes to the documentation, examples, or templates.
- [x] `improvement` — Improving existing features

apps/docs/components/mdx-components/code.tsx

@@ -2,10 +2,203 @@
 
 import { SandpackCodeViewer, SandpackFiles, SandpackProvider } from '@codesandbox/sandpack-react'
 import { useTheme } from 'next-themes'
-import { useEffect, useState } from 'react'
+import React, {
+	Fragment,
+	ReactElement,
+	ReactNode,
+	cloneElement,
+	createContext,
+	isValidElement,
+	useContext,
+	useEffect,
+	useMemo,
+	useState,
+} from 'react'
+import { A } from './generic'
 
-export const Code = (props: any) => {
+const CodeLinksContext = createContext<Record<string, string>>({})
-	return <code {...props} />
+
+export function CodeLinkProvider({
+	children,
+	links,
+}: {
+	children: React.ReactNode
+	links: Record<string, string>
+}) {
+	return <CodeLinksContext.Provider value={links}>{children}</CodeLinksContext.Provider>
+}
+
+export function Code({ children, ...props }: React.ComponentProps<'code'>) {
+	const codeLinks = useContext(CodeLinksContext)
+
+	const newChildren = useMemo(() => {
+		// to linkify code, we have to do quite a lot of work. we need to take the output of
+		// Highlight.js and transform it to add hyperlinks to certain tokens. There are a few things
+		// that make this difficult:
+		//
+		// 1, the structure is recursive. A function span will include a bunch of other spans making
+		// up the whole definition of the function, for example.
+		//
+		// 2, a given span doesn't necessarily correspond to a single identifier. For example, this
+		// code: `dispatch: (info: TLEventInfo) => this` will be split like this:
+		// - `dispatch`
+		// - `: (info: TLEventInfo) => `
+		// - `this`
+		//
+		// That means we need to take highlight.js's tokens and split them into our own tokens that
+		// contain single identifiers to linkify.
+		//
+		// 3, a single identifier can be split across multiple spans. For example,
+		// `Omit<Geometry2dOptions>` will be split like this:
+		// - `Omit`
+		// - `<`
+		// - `Geometry2`
+		// - `dOptions`
+		// - `>`
+		//
+		// I don't know why this happens, it feels like a bug. We handle this by keeping track of &
+		// merging consecutive tokens if they're identifiers with no non-identifier tokens in
+		// between.
+
+		// does this token look like a JS identifier?
+		function isIdentifier(token: string): boolean {
+			return /^[_$a-zA-Z\xA0-\uFFFF][_$a-zA-Z0-9\xA0-\uFFFF]*$/g.test(token)
+		}
+
+		// split the code into an array of identifiers, and the bits in between them
+		function tokenize(code: string): string[] {
+			const identifierRegex = /[_$a-zA-Z\xA0-\uFFFF][_$a-zA-Z0-9\xA0-\uFFFF]*/g
+
+			let currentIdx = 0
+			const tokens = []
+			for (const identifierMatch of code.matchAll(identifierRegex)) {
+				const [identifier] = identifierMatch
+				const idx = identifierMatch.index
+				if (idx > currentIdx) {
+					tokens.push(code.slice(currentIdx, idx))
+				}
+				tokens.push(identifier)
+				currentIdx = idx + identifier.length
+			}
+			if (currentIdx < code.length) {
+				tokens.push(code.slice(currentIdx))
+			}
+
+			return tokens
+		}
+
+		// recursively process the children array
+		function processChildrenArray(children: ReactNode): ReactNode {
+			if (!Array.isArray(children)) {
+				if (!children) return children
+				return processChildrenArray([children])
+			}
+
+			// these are the new linkified children, the result of this function
+			const newChildren = []
+
+			// in order to deal with token splitting/merging, we need to keep track of the last
+			// highlight span we saw. this has the right classes on it to colorize the current
+			// token.
+			let lastSeenHighlightSpan: ReactElement | null = null
+			// the current identifier that we're building up by merging consecutive tokens
+			let currentIdentifier: string | null = null
+			// whether the current span is closed, but we're still in the same identifier and might
+			// still need to append to it
+			let isCurrentSpanClosed = false
+
+			function startSpan(span: ReactElement) {
+				lastSeenHighlightSpan = span
+				isCurrentSpanClosed = false
+			}
+
+			function closeSpan() {
+				isCurrentSpanClosed = true
+				if (!currentIdentifier) {
+					lastSeenHighlightSpan = null
+				}
+			}
+
+			function pushInCurrentSpan(content: ReactNode) {
+				if (lastSeenHighlightSpan) {
+					newChildren.push(
+						cloneElement(lastSeenHighlightSpan, {
+							key: newChildren.length,
+							children: content,
+						})
+					)
+				} else {
+					newChildren.push(<Fragment key={newChildren.length}>{content}</Fragment>)
+				}
+			}
+
+			function finishCurrentIdentifier() {
+				if (currentIdentifier) {
+					const link = codeLinks[currentIdentifier]
+					if (link) {
+						pushInCurrentSpan(
+							<A href={link} className="code-link">
+								{currentIdentifier}
+							</A>
+						)
+					} else {
+						pushInCurrentSpan(currentIdentifier)
+					}
+					currentIdentifier = null
+				}
+				if (isCurrentSpanClosed) {
+					lastSeenHighlightSpan = null
+				}
+			}
+
+			function pushToken(token: string) {
+				if (isIdentifier(token)) {
+					if (currentIdentifier) {
+						currentIdentifier += token
+					} else {
+						currentIdentifier = token
+					}
+				} else {
+					finishCurrentIdentifier()
+					pushInCurrentSpan(token)
+				}
+			}
+
+			for (const child of children) {
+				if (typeof child === 'string') {
+					for (const token of tokenize(child)) {
+						pushToken(token)
+					}
+				} else if (isValidElement<{ children: ReactNode }>(child)) {
+					if (child.type === 'span' && typeof child.props.children === 'string') {
+						startSpan(child)
+						for (const token of tokenize(child.props.children)) {
+							pushToken(token)
+						}
+						closeSpan()
+					} else {
+						finishCurrentIdentifier()
+						newChildren.push(
+							cloneElement(child, {
+								key: newChildren.length,
+								children: processChildrenArray(child.props.children),
+							})
+						)
+					}
+				} else {
+					throw new Error(`Invalid code child: ${JSON.stringify(child)}`)
+				}
+			}
+
+			finishCurrentIdentifier()
+
+			return newChildren
+		}
+
+		return processChildrenArray(children)
+	}, [children, codeLinks])
+
+	return <code {...props}>{newChildren}</code>
 }
 
 export function CodeBlock({ code }: { code: SandpackFiles }) {

apps/docs/components/mdx-components/generic.tsx

@@ -1,3 +1,4 @@
+import Link from 'next/link'
 import React from 'react'
 
 /* ---------------------- Lists --------------------- */
@@ -76,7 +77,12 @@ export const A = (props: any) => {
 			? undefined
 			: '_blank'
 	const target = props.target ?? derivedTarget
-	return <a {...props} target={target} />
+
+	if (isLocalUrl) {
+		return <Link {...props} target={target} />
+	} else {
+		return <a {...props} target={target} />
+	}
 }
 
 export const Divider = (props: any) => {

apps/docs/components/mdx-components/index.ts

@@ -1,5 +1,6 @@
 import * as customComponents from '../article-components'
 import * as apiComponents from './api-docs'
+import { Code, CodeBlock, CodeLinkProvider } from './code'
 import {
 	A,
 	ApiHeading,
@@ -27,12 +28,11 @@ import {
 	Video,
 } from './generic'
 
-import { Code, CodeBlock } from './code'
-
 export const components = {
 	a: A,
 	blockquote: Blockquote,
 	code: Code,
+	CodeLinkProvider,
 	h1: Heading1,
 	h2: Heading2,
 	h3: Heading3,

apps/docs/scripts/functions/getApiMarkdown.ts

@@ -22,7 +22,6 @@ import {
 	ApiTypeAlias,
 	ApiVariable,
 	Excerpt,
-	ExcerptToken,
 	ReleaseTag,
 } from '@microsoft/api-extractor-model'
 import { MarkdownWriter, formatWithPrettier, getPath, getSlug } from '../utils'
@@ -118,9 +117,13 @@ export async function getApiMarkdown(
 				componentProps instanceof ApiDeclaredItem &&
 				componentProps?.kind !== 'Interface'
 			) {
-				propertiesResult.markdown += await typeExcerptToMarkdown(componentProps.excerpt, {
+				propertiesResult.markdown += await excerptToMarkdown(
-					kind: componentProps.kind,
+					componentProps,
-				})
+					componentProps.excerpt,
+					{
+						kind: componentProps.kind,
+					}
+				)
 			}
 			if (propertiesResult.markdown.trim()) {
 				addMarkdown(toc, `- [Properties](#properties)\n`)
@@ -153,7 +156,6 @@ export async function getApiMarkdown(
 
 	await addDocComment(model, result, item)
 
-	addReferences(model, result, item)
 	addLinkToSource(result, item)
 
 	if (membersResult.markdown.length) {
@@ -180,7 +182,6 @@ async function addMarkdownForMember(
 	addMemberName(result, member)
 	addTags(model, result, member, { isComponentProp })
 	await addDocComment(model, result, member, { isComponentProp })
-	addReferences(model, result, member)
 }
 
 async function addFrontmatter(
@@ -282,7 +283,7 @@ async function addDocComment(
 		member instanceof ApiMethod
 	) {
 		if (!isComponentProp) result.markdown += `<ApiHeading>Signature</ApiHeading>\n\n`
-		result.markdown += await typeExcerptToMarkdown(member.excerpt, {
+		result.markdown += await excerptToMarkdown(member, member.excerpt, {
 			kind: member.kind,
 		})
 		result.markdown += `\n\n`
@@ -308,7 +309,7 @@ async function addDocComment(
 				result.markdown += `\`${param.name}\`\n\n`
 				result.markdown += `</ParametersTableName>\n`
 				result.markdown += `<ParametersTableDescription>\n\n`
-				result.markdown += await typeExcerptToMarkdown(param.parameterTypeExcerpt, {
+				result.markdown += await excerptToMarkdown(member, param.parameterTypeExcerpt, {
 					kind: 'ParameterType',
 					printWidth: 60,
 				})
@@ -327,7 +328,7 @@ async function addDocComment(
 
 		if (!(member instanceof ApiConstructor)) {
 			result.markdown += `<ApiHeading>Returns</ApiHeading>\n\n`
-			result.markdown += await typeExcerptToMarkdown(member.returnTypeExcerpt, {
+			result.markdown += await excerptToMarkdown(member, member.returnTypeExcerpt, {
 				kind: 'ReturnType',
 			})
 			result.markdown += `\n\n`
@@ -369,13 +370,28 @@ async function addDocComment(
 	}
 }
 
-async function typeExcerptToMarkdown(
+async function excerptToMarkdown(
+	item: ApiItem,
 	excerpt: Excerpt,
 	{ kind, printWidth }: { kind: ApiItemKind | 'ReturnType' | 'ParameterType'; printWidth?: number }
 ) {
+	const links = {} as Record<string, string>
+
 	let code = ''
 	for (const token of excerpt.spannedTokens) {
 		code += token.text
+
+		if (!token.canonicalReference) continue
+
+		const apiItemResult = item
+			.getAssociatedModel()!
+			.resolveDeclarationReference(token.canonicalReference!, item)
+
+		if (apiItemResult.errorMessage) continue
+
+		const apiItem = apiItemResult.resolvedApiItem!
+		const url = `/reference/${getPath(apiItem)}`
+		links[token.text] = url
 	}
 
 	code = code.replace(/^export /, '')
@@ -425,7 +441,15 @@ async function typeExcerptToMarkdown(
 			throw Error()
 	}
 
-	return ['```ts', code, '```'].join('\n')
+	return [
+		`<CodeLinkProvider links={${JSON.stringify(links)}}>`,
+		'',
+		'```ts',
+		code,
+		'```',
+		'',
+		'</CodeLinkProvider>',
+	].join('\n')
 }
 
 function addTags(
@@ -456,40 +480,6 @@ function addTags(
 	result.markdown += `<Small>${tags.filter((t) => t.toLowerCase() !== 'none').join(' ')}</Small>\n\n`
 }
 
-function addReferences(model: TldrawApiModel, result: Result, member: ApiItem) {
-	if (!(member instanceof ApiDeclaredItem)) return
-	const references = new Set<string>()
-
-	function addToken(item: ApiDeclaredItem, token: ExcerptToken) {
-		if (token.kind !== 'Reference') return
-		const apiItemResult = item
-			.getAssociatedModel()!
-			.resolveDeclarationReference(token.canonicalReference!, item)
-		if (apiItemResult.errorMessage) {
-			return
-		}
-		const apiItem = apiItemResult.resolvedApiItem!
-		const url = `/reference/${getPath(apiItem)}`
-		references.add(`[${token.text}](${url})`)
-	}
-
-	member.excerptTokens.forEach((token) => {
-		addToken(member, token)
-	})
-
-	const componentProps = model.isComponent(member) ? model.getReactPropsItem(member) : null
-	if (componentProps && componentProps instanceof ApiDeclaredItem) {
-		componentProps.excerptTokens.forEach((token) => {
-			addToken(componentProps, token)
-		})
-	}
-
-	if (references.size) {
-		result.markdown += `<ApiHeading>References</ApiHeading>\n\n`
-		result.markdown += Array.from(references).join(', ') + '\n\n'
-	}
-}
-
 function addExtends(result: Result, item: ApiItem) {
 	const extendsTypes =
 		item instanceof ApiClass && item.extendsType
@@ -500,7 +490,30 @@ function addExtends(result: Result, item: ApiItem) {
 
 	if (!extendsTypes.length) return
 
-	result.markdown += `Extends \`${extendsTypes.map((type) => type.excerpt.text).join(', ')}\`.\n\n`
+	const links = {} as Record<string, string>
+	for (const type of extendsTypes) {
+		for (const token of type.excerpt.spannedTokens) {
+			if (!token.canonicalReference) continue
+
+			const apiItemResult = item
+				.getAssociatedModel()!
+				.resolveDeclarationReference(token.canonicalReference!, item)
+
+			if (apiItemResult.errorMessage) continue
+
+			const apiItem = apiItemResult.resolvedApiItem!
+			links[token.text] = `/reference/${getPath(apiItem)}`
+		}
+	}
+
+	result.markdown += [
+		`<CodeLinkProvider links={${JSON.stringify(links)}}>`,
+		'',
+		`Extends \`${extendsTypes.map((type) => type.excerpt.text).join(', ')}\`.`,
+		'',
+		'</CodeLinkProvider>',
+		'',
+	].join('\n')
 }
 
 function addLinkToSource(result: Result, member: ApiItem) {

apps/docs/styles/globals.css

@@ -526,6 +526,10 @@ body {
 	tab-size: 16px;
 }
 
+.article td pre code {
+	background-color: transparent;
+}
+
 .article ol {
 	margin: 20px 0px;
 	padding-left: 16px;

apps/docs/styles/hljs.css

@@ -77,3 +77,22 @@
 .hljs-link {
 	text-decoration: underline;
 }
+
+.hljs a.code-link {
+	color: inherit;
+}
+a.code-link::before {
+	content: '';
+	width: 15px;
+	height: 15px;
+	display: inline-block;
+	background: currentColor;
+	-webkit-mask: url('data:image/svg+xml;utf8,<svg width="15" height="15" viewBox="0 0 15 15" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M6.5 2.5H3.5C2.94772 2.5 2.5 2.94772 2.5 3.5V11.5C2.5 12.0523 2.94772 12.5 3.5 12.5H11.5C12.0523 12.5 12.5 12.0523 12.5 11.5V8.5M9.5 2.5H12.5M12.5 2.5V5.5M12.5 2.5L6.5 8.5" stroke="black" stroke-linecap="round" stroke-linejoin="round"/></svg>')
+		no-repeat 50% 50%;
+	mask: url('data:image/svg+xml;utf8,<svg width="15" height="15" viewBox="0 0 15 15" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M6.5 2.5H3.5C2.94772 2.5 2.5 2.94772 2.5 3.5V11.5C2.5 12.0523 2.94772 12.5 3.5 12.5H11.5C12.0523 12.5 12.5 12.0523 12.5 11.5V8.5M9.5 2.5H12.5M12.5 2.5V5.5M12.5 2.5L6.5 8.5" stroke="black" stroke-linecap="round" stroke-linejoin="round"/></svg>')
+		no-repeat 50% 50%;
+	-webkit-mask-size: cover;
+	mask-size: cover;
+	vertical-align: -2px;
+	margin-right: 2px;
+}