transfer-out: transfer out

This commit is contained in:
alex 2023-04-25 12:01:25 +01:00
parent ec84f64e63
commit 29ed921c67
1056 changed files with 154507 additions and 0 deletions

View file

@ -0,0 +1,317 @@
import { ApiModel } from '@microsoft/api-extractor-model'
import fs from 'fs'
import matter from 'gray-matter'
import path from 'path'
import authors from '../content/authors.json'
import sections from '../content/sections.json'
import {
Article,
Articles,
Category,
GeneratedContent,
Group,
MarkdownContent,
Section,
Status,
} from '../types/content-types'
import { getApiMarkdown } from './getApiMarkdown'
import { getSlug } from './utils'
const { log } = console
type InputCategory = {
id: string
title: string
description: string
groups: Group[]
}
type InputSection = {
id: string
title: string
description: string
categories: InputCategory[]
}
function generateSection(
section: InputSection,
content: MarkdownContent,
articles: Articles
): Section {
// A temporary table of categories
const _categories: Record<string, Category> = {}
// Uncategorized articles
const _ucg: Article[] = []
// A temporary table of articles mapped to categories
const _categoryArticles: Record<string, Article[]> = Object.fromEntries(
section.categories.map((category) => [category.id, []])
)
// The file directory for this section
const dir = path.join(process.cwd(), 'content', section.id)
fs.readdirSync(dir, { withFileTypes: false }).forEach((result: string | Buffer) => {
try {
const filename = result.toString()
const fileContent = fs.readFileSync(path.join(dir, filename)).toString()
const extension = path.extname(filename)
const articleId = filename.replace(extension, '')
const parsed = matter({ content: fileContent }, {})
if (process.env.NODE_ENV !== 'development' && parsed.data.status !== 'published') {
return
}
// If a category was provided but that category was not found in the section, throw an error
const category =
parsed.data.category && section.categories.find((c) => c.id === parsed.data.category)
if (parsed.data.category && !category) {
throw Error(
`Could not find a category for section ${section.id} with id ${parsed.data.category}.`
)
}
if (parsed.data.author && !authors[parsed.data.author as keyof typeof authors]) {
throw Error(`Could not find an author with id ${parsed.data.author}.`)
}
// By default, the category is ucg (uncategorized)
const { category: categoryId = 'ucg' } = parsed.data
const article: Article = {
id: articleId,
sectionIndex: 0,
groupIndex: -1,
groupId: parsed.data.group ?? null,
categoryIndex: parsed.data.order ?? -1,
sectionId: section.id,
categoryId: parsed.data.category ?? 'ucg',
status: parsed.data.status ?? Status.Draft,
title: parsed.data.title ?? 'Article',
description: parsed.data.description ?? 'An article for the docs site.',
hero: parsed.data.hero ?? null,
date: parsed.data.date ? new Date(parsed.data.date).toISOString() : null,
keywords: parsed.data.keywords ?? [],
next: null,
prev: null,
author: parsed.data.author
? authors[parsed.data.author as keyof typeof authors] ?? null
: null,
sourceUrl: `https://github.com/tldraw/tldraw-lite/tree/main/apps/docs/content/${section.id}/${articleId}${extension}`,
}
if (article.id === section.id) {
article.categoryIndex = -1
article.sectionIndex = -1
articles[section.id + '_index'] = article
content[section.id + '_index'] = parsed.content
} else {
if (category) {
if (article.id === category.id) {
article.categoryIndex = -1
article.sectionIndex = -1
articles[category.id + '_index'] = article
content[category.id + '_index'] = parsed.content
} else {
_categoryArticles[categoryId].push(article)
content[articleId] = parsed.content
}
} else {
_ucg.push(article)
content[articleId] = parsed.content
}
}
} catch (e) {
console.error(e)
}
})
const sortArticles = (articleA: Article, articleB: Article) => {
const { categoryIndex: categoryIndexA, date: dateA = '01/01/1970' } = articleA
const { categoryIndex: categoryIndexB, date: dateB = '01/01/1970' } = articleB
return categoryIndexA === categoryIndexB
? new Date(dateB!).getTime() > new Date(dateA!).getTime()
? 1
: -1
: categoryIndexA < categoryIndexB
? -1
: 1
}
let sectionIndex = 0
// Sort ucg articles by date and add them to the articles table
_ucg.sort(sortArticles).forEach((article, i) => {
article.categoryIndex = i
article.sectionIndex = sectionIndex++
article.prev = _ucg[i - 1]?.id ?? null
article.next = _ucg[i + 1]?.id ?? null
articles[article.id] = article
})
// Sort categorized articles by date and add them to the articles table
section.categories.forEach((category) => {
const categoryArticles = _categoryArticles[category.id]
categoryArticles.sort(sortArticles).forEach((article, i) => {
article.categoryIndex = i
article.sectionIndex = sectionIndex++
article.prev = categoryArticles[i - 1]?.id ?? null
article.next = categoryArticles[i + 1]?.id ?? null
articles[article.id] = article
})
_categories[category.id] = {
...category,
articleIds: categoryArticles.map((article) => article.id),
}
})
return {
...section,
categories: [
{
id: 'ucg',
title: 'Uncategorized',
description: 'Articles that do not belong to a category.',
groups: [],
articleIds: _ucg
.sort((a, b) => a.sectionIndex - b.sectionIndex)
.map((article) => article.id),
},
...section.categories.map(({ id }) => _categories[id]).filter((c) => c.articleIds.length > 0),
],
}
}
async function generateApiDocs() {
const apiInputSection: InputSection = {
id: 'gen' as string,
title: 'API',
description: "Reference for the tldraw package's APIs (generated).",
categories: [],
}
const addedCategories = new Set<string>()
const OUTPUT_DIR = path.join(process.cwd(), 'content', 'gen')
if (fs.existsSync(OUTPUT_DIR)) {
fs.rmdirSync(OUTPUT_DIR, { recursive: true })
}
fs.mkdirSync(OUTPUT_DIR)
// to include more packages in docs, add them to devDependencies in package.json
const packageJson = JSON.parse(fs.readFileSync(path.join(process.cwd(), 'package.json'), 'utf8'))
const tldrawPackagesToIncludeInDocs = Object.keys(packageJson.devDependencies).filter((dep) =>
dep.startsWith('@tldraw/')
)
const model = new ApiModel()
for (const packageName of tldrawPackagesToIncludeInDocs) {
// Get the file contents
const filePath = path.join(
process.cwd(),
'..',
'..',
'packages',
packageName.replace('@tldraw/', ''),
'api',
'api.json'
)
const packageModel = model.loadPackage(filePath)
const categoryName = packageModel.name.replace(`@tldraw/`, '')
if (!addedCategories.has(categoryName)) {
apiInputSection.categories!.push({
id: categoryName,
title: packageModel.name,
description: '',
groups: [
{
id: 'Namespace',
title: 'Namespaces',
},
{
id: 'Class',
title: 'Classes',
},
{
id: 'Function',
title: 'Functions',
},
{
id: 'Variable',
title: 'Variables',
},
{
id: 'Enum',
title: 'Enums',
},
{
id: 'Interface',
title: 'Interfaces',
},
{
id: 'TypeAlias',
title: 'TypeAliases',
},
],
})
addedCategories.add(categoryName)
}
const entrypoint = packageModel.entryPoints[0]
for (let j = 0; j < entrypoint.members.length; j++) {
const item = entrypoint.members[j]
const result = await getApiMarkdown(categoryName, item, j)
const outputFileName = `${getSlug(item)}.mdx`
fs.writeFileSync(path.join(OUTPUT_DIR, outputFileName), result.markdown)
}
}
return apiInputSection
}
export async function generateContent(): Promise<GeneratedContent> {
const content: MarkdownContent = {}
const articles: Articles = {}
const apiSection = await generateApiDocs()
log('• Generating site content (content.json)')
try {
const outputSections: Section[] = [...(sections as InputSection[]), apiSection]
.map((section) => generateSection(section, content, articles))
.filter((section) => section.categories.some((c) => c.articleIds.length > 0))
log('✔ Generated site content.')
// Write to disk
const contentComplete = { sections: outputSections, content, articles }
fs.writeFileSync(
path.join(process.cwd(), 'content.json'),
JSON.stringify(contentComplete, null, 2)
)
return contentComplete
} catch (error) {
log(`x Could not generate site content.`)
throw error
}
}

View file

@ -0,0 +1,341 @@
import {
ApiClass,
ApiConstructSignature,
ApiConstructor,
ApiDeclaredItem,
ApiDocumentedItem,
ApiEnum,
ApiFunction,
ApiInterface,
ApiItem,
ApiItemKind,
ApiMethod,
ApiMethodSignature,
ApiNamespace,
ApiProperty,
ApiPropertySignature,
ApiReadonlyMixin,
ApiReleaseTagMixin,
ApiStaticMixin,
ApiTypeAlias,
ApiVariable,
Excerpt,
ReleaseTag,
} from '@microsoft/api-extractor-model'
import { assert, exhaustiveSwitchError } from '@tldraw/utils'
import { MarkdownWriter, formatWithPrettier, getPath, getSlug } from './utils'
type Result = { markdown: string }
const date = new Intl.DateTimeFormat('en-US', {
year: 'numeric',
month: '2-digit',
day: '2-digit',
}).format(new Date())
export async function getApiMarkdown(categoryName: string, item: ApiItem, j: number) {
const result = { markdown: '' }
addFrontmatter(result, item, categoryName, j)
addTags(result, item)
const toc: Result = { markdown: '' }
const membersResult: Result = { markdown: '' }
if (item.members) {
const constructors = []
const properties = []
const methods = []
for (const member of item.members) {
switch (member.kind) {
case ApiItemKind.Constructor:
case ApiItemKind.ConstructSignature:
constructors.push(member)
break
case ApiItemKind.Variable:
case ApiItemKind.Property:
case ApiItemKind.PropertySignature:
properties.push(member)
break
case ApiItemKind.Method:
case ApiItemKind.Function:
case ApiItemKind.MethodSignature:
methods.push(member)
break
case ApiItemKind.EnumMember:
case ApiItemKind.Class:
case ApiItemKind.TypeAlias:
case ApiItemKind.Interface:
// TODO: document these
break
default:
throw new Error(`Unknown member kind: ${member.kind} ${member.displayName}`)
}
}
const constructorResult = { markdown: '' }
const propertiesResult = { markdown: '' }
const methodsResult = { markdown: '' }
if (constructors.length) {
for (const member of constructors) {
await addMarkdownForMember(constructorResult, member)
addHorizontalRule(constructorResult)
}
addMarkdown(membersResult, constructorResult.markdown)
}
if (properties.length) {
addMarkdown(toc, `- [Properties](#properties)\n`)
addMarkdown(propertiesResult, `## Properties\n\n`)
for (const member of properties) {
addMarkdown(toc, ` - [${member.displayName}](#${getSlug(member)})\n`)
await addMarkdownForMember(propertiesResult, member)
addHorizontalRule(propertiesResult)
}
addMarkdown(membersResult, propertiesResult.markdown)
}
if (methods.length) {
addMarkdown(toc, `- [Methods](#methods)\n`)
addMarkdown(methodsResult, `## Methods\n\n`)
for (const member of methods) {
addMarkdown(toc, ` - [${member.displayName}](#${getSlug(member)})\n`)
await addMarkdownForMember(methodsResult, member)
addHorizontalRule(methodsResult)
}
addMarkdown(membersResult, methodsResult.markdown)
}
}
if (toc.markdown.length) {
result.markdown += `<details>\n\t<summary>Table of Contents</summary>\n`
addMarkdown(result, toc.markdown)
result.markdown += `</details>\n\n`
}
await addDocComment(result, item)
addReferences(result, item)
if (membersResult.markdown.length) {
addHorizontalRule(result)
addMarkdown(result, membersResult.markdown)
}
return result
}
/* --------------------- Helpers -------------------- */
function addMarkdown(result: Result, markdown: string) {
result.markdown += markdown
}
async function addMarkdownForMember(result: Result, member: ApiItem) {
addMemberName(result, member)
addTags(result, member)
await addDocComment(result, member)
addReferences(result, member)
}
function addFrontmatter(result: Result, member: ApiItem, categoryName: string, order: number) {
result.markdown += `---
title: ${member.displayName}
status: published
category: ${categoryName}
group: ${member.kind}
author: api
date: ${date}
order: ${order}
---`
}
function addHorizontalRule(result: Result) {
result.markdown += `---\n\n`
}
function addMemberName(result: Result, member: ApiItem) {
if (member.kind === 'Constructor') {
result.markdown += `### \`Constructor\`\n\n`
return
}
if (!member.displayName) return
result.markdown += `### \`${member.displayName}${
member.kind === 'Method' ? '()' : ''
}\` \\{#${getSlug(member)}}\n\n`
}
async function addDocComment(result: Result, member: ApiItem) {
if (!(member instanceof ApiDocumentedItem)) {
return
}
if (member.tsdocComment) {
result.markdown += await MarkdownWriter.docNodeToMarkdown(member.tsdocComment.summarySection)
const exampleBlocks = member.tsdocComment.customBlocks.filter(
(block) => block.blockTag.tagNameWithUpperCase === '@EXAMPLE'
)
if (exampleBlocks.length) {
result.markdown += `\n\n`
result.markdown += `##### Example\n\n`
for (const example of exampleBlocks) {
result.markdown += await MarkdownWriter.docNodeToMarkdown(example.content)
}
}
}
if (
member instanceof ApiMethod ||
member instanceof ApiMethodSignature ||
member instanceof ApiConstructor ||
member instanceof ApiConstructSignature ||
member instanceof ApiFunction
) {
result.markdown += `##### Parameters\n\n\n`
if (!member.parameters.length) {
result.markdown += `None\n\n`
} else {
result.markdown += '<ParametersTable>\n\n'
for (const param of member.parameters) {
result.markdown += '<ParametersTableRow>\n'
result.markdown += '<ParametersTableName>\n\n'
result.markdown += `\`${param.name}\`\n\n`
if (param.isOptional) {
result.markdown += ` <Small>(optional)</Small>\n\n`
}
result.markdown += `</ParametersTableName>\n`
result.markdown += `<ParametersTableDescription>\n\n`
result.markdown += await typeExcerptToMarkdown(param.parameterTypeExcerpt, {
kind: 'ParameterType',
printWidth: 60,
})
result.markdown += `\n\n`
if (param.tsdocParamBlock) {
result.markdown += await MarkdownWriter.docNodeToMarkdown(param.tsdocParamBlock.content)
}
result.markdown += `\n\n</ParametersTableDescription>\n`
result.markdown += `</ParametersTableRow>\n`
}
result.markdown += '</ParametersTable>\n\n'
}
if (!(member instanceof ApiConstructor)) {
result.markdown += `##### Returns\n\n\n`
result.markdown += await typeExcerptToMarkdown(member.returnTypeExcerpt, {
kind: 'ReturnType',
})
result.markdown += `\n\n`
if (member.tsdocComment && member.tsdocComment.returnsBlock) {
result.markdown += await MarkdownWriter.docNodeToMarkdown(
member.tsdocComment.returnsBlock.content
)
}
}
} else if (
member instanceof ApiVariable ||
member instanceof ApiTypeAlias ||
member instanceof ApiProperty ||
member instanceof ApiPropertySignature ||
member instanceof ApiClass ||
member instanceof ApiInterface ||
member instanceof ApiEnum ||
member instanceof ApiNamespace
) {
// no specific docs for these types
result.markdown += `##### Signature\n\n\n`
result.markdown += await typeExcerptToMarkdown(member.excerpt, { kind: member.kind })
result.markdown += `\n\n`
} else {
throw new Error('unknown member kind: ' + member.kind)
}
}
async function typeExcerptToMarkdown(
excerpt: Excerpt,
{ kind, printWidth }: { kind: ApiItemKind | 'ReturnType' | 'ParameterType'; printWidth?: number }
) {
let code = ''
for (const token of excerpt.spannedTokens) {
code += token.text
}
code = code.replace(/^export /, '')
code = code.replace(/^declare /, '')
switch (kind) {
case ApiItemKind.CallSignature:
case ApiItemKind.EntryPoint:
case ApiItemKind.EnumMember:
case ApiItemKind.Function:
case ApiItemKind.Model:
case ApiItemKind.Namespace:
case ApiItemKind.None:
case ApiItemKind.Package:
case ApiItemKind.TypeAlias:
code = await formatWithPrettier(code, { printWidth })
break
case 'ReturnType':
case 'ParameterType':
code = await formatWithPrettier(`type X = () =>${code}`, { printWidth })
assert(code.startsWith('type X = () =>'))
code = code = code.replace(/^type X = \(\) =>[ \n]/, '')
break
case ApiItemKind.Class:
case ApiItemKind.Enum:
case ApiItemKind.Interface:
code = await formatWithPrettier(`${code} {}`, { printWidth })
break
case ApiItemKind.Constructor:
case ApiItemKind.ConstructSignature:
case ApiItemKind.IndexSignature:
case ApiItemKind.Method:
case ApiItemKind.MethodSignature:
case ApiItemKind.Property:
case ApiItemKind.PropertySignature:
case ApiItemKind.Variable:
code = await formatWithPrettier(`class X { ${code} }`, { printWidth })
assert(code.startsWith('class X {\n'))
assert(code.endsWith('\n}'))
code = code.slice('class X {\n'.length, -'\n}'.length)
code = code.replace(/^ {2}/gm, '')
break
default:
exhaustiveSwitchError(kind)
}
return ['```ts', code, '```'].join('\n')
}
function addTags(result: Result, member: ApiItem) {
const tags = []
if (ApiReleaseTagMixin.isBaseClassOf(member)) {
tags.push(ReleaseTag[member.releaseTag])
}
if (ApiStaticMixin.isBaseClassOf(member) && member.isStatic) {
tags.push('Static')
}
if (ApiReadonlyMixin.isBaseClassOf(member) && member.isReadonly) {
tags.push('Readonly')
}
tags.push(member.kind)
result.markdown += `<Small>${tags.join(' ')}</Small>\n\n`
}
function addReferences(result: Result, member: ApiItem) {
if (!(member instanceof ApiDeclaredItem)) return
const references = new Set<string>()
member.excerptTokens.forEach((token) => {
if (token.kind !== 'Reference') return
const url = `/gen/${getPath(token as { canonicalReference: ApiItem['canonicalReference'] })}`
references.add(`[${token.text}](${url})`)
})
if (references.size) {
result.markdown += `##### References\n\n`
result.markdown += Array.from(references).join(', ') + '\n\n'
}
}

View file

@ -0,0 +1,11 @@
// import { buildDocs } from './build-docs'
import { generateContent } from './generateContent'
async function main() {
const { log } = console
log('Creating content for www.')
// await buildDocs()
await generateContent()
}
main()

120
apps/docs/scripts/utils.ts Normal file
View file

@ -0,0 +1,120 @@
import { ApiItem } from '@microsoft/api-extractor-model'
import {
DocCodeSpan,
DocEscapedText,
DocFencedCode,
DocNode,
DocParagraph,
DocPlainText,
DocSection,
DocSoftBreak,
} from '@microsoft/tsdoc'
import prettier from 'prettier'
export function getPath(item: { canonicalReference: ApiItem['canonicalReference'] }): string {
return item.canonicalReference
.toString()
.replace(/^@tldraw\/([^!]+)/, '$1/')
.replace(/[!:()#.]/g, '-')
.replace(/-+/g, '-')
.replace(/^-/, '')
.replace(/\/-/, '/')
.replace(/-$/, '')
}
export function getSlug(item: { canonicalReference: ApiItem['canonicalReference'] }): string {
return getPath(item).replace(/^[^/]+\//, '')
}
const prettierConfigPromise = prettier.resolveConfig(__dirname)
const languages: { [tag: string]: string | undefined } = {
ts: 'typescript',
tsx: 'typescript',
}
export async function formatWithPrettier(
code: string,
{
languageTag,
// roughly the width of our code blocks on a desktop
printWidth = 80,
}: { languageTag?: string; printWidth?: number } = {}
) {
const language = languages[languageTag || 'ts']
if (!language) {
throw new Error(`Unknown language: ${languageTag}`)
}
const prettierConfig = await prettierConfigPromise
const formattedCode = prettier.format(code, {
...prettierConfig,
parser: language,
printWidth,
tabWidth: 2,
useTabs: false,
})
return formattedCode.trimEnd()
}
export class MarkdownWriter {
static async docNodeToMarkdown(docNode: DocNode) {
const writer = new MarkdownWriter()
await writer.writeDocNode(docNode)
return writer.toString()
}
private result = ''
write(...parts: string[]): this {
this.result += parts.join('')
return this
}
endsWith(str: string) {
return this.result.endsWith(str)
}
writeIfNeeded(str: string): this {
if (!this.endsWith(str)) {
this.write(str)
}
return this
}
async writeDocNode(docNode: DocNode) {
if (docNode instanceof DocPlainText) {
this.write(docNode.text)
} else if (docNode instanceof DocSection || docNode instanceof DocParagraph) {
await this.writeDocNodes(docNode.nodes)
this.writeIfNeeded('\n\n')
} else if (docNode instanceof DocSoftBreak) {
this.writeIfNeeded('\n')
} else if (docNode instanceof DocCodeSpan) {
this.write('`', docNode.code, '`')
} else if (docNode instanceof DocFencedCode) {
this.writeIfNeeded('\n').write(
'```',
docNode.language,
'\n',
await formatWithPrettier(docNode.code, { languageTag: docNode.language }),
'\n',
'```\n'
)
} else if (docNode instanceof DocEscapedText) {
this.write(docNode.encodedText)
} else {
throw new Error(`Unknown docNode kind: ${docNode.kind}`)
}
}
async writeDocNodes(docNodes: readonly DocNode[]) {
for (const docNode of docNodes) {
await this.writeDocNode(docNode)
}
return this
}
toString() {
return this.result
}
}