Demo server bookmark unfurl endpoint (#4062)

This adds the HTMLRewriter-based bookmark unfurler to the demo server.
It moves the unfurler into worker-shared, and adds some better shared
error handling across our workers.

I removed the fallback bookmark fetcher where we try and fetch websites
locally. This will almost never work, as it requires sites to set public
CORS.

### Change type
- [x] `other`

apps/bemo-worker/src/worker.ts

@@ -2,10 +2,13 @@
 /// <reference types="@cloudflare/workers-types" />
 
 import {
-	createSentry,
+	getUrlMetadata,
+	handleApiRequest,
 	handleUserAssetGet,
 	handleUserAssetUpload,
 	notFound,
+	parseRequestQuery,
+	urlMetadataQueryValidator,
 } from '@tldraw/worker-shared'
 import { WorkerEntrypoint } from 'cloudflare:workers'
 import { Router, createCors } from 'itty-router'
@@ -18,7 +21,7 @@ const cors = createCors({ origins: ['*'] })
 export default class Worker extends WorkerEntrypoint<Environment> {
 	private readonly router = Router()
 		.all('*', cors.preflight)
-		.get('/v1/uploads/:objectName', (request) => {
+		.get('/uploads/:objectName', (request) => {
 			return handleUserAssetGet({
 				request,
 				bucket: this.env.BEMO_BUCKET,
@@ -26,7 +29,7 @@ export default class Worker extends WorkerEntrypoint<Environment> {
 				context: this.ctx,
 			})
 		})
-		.post('/v1/uploads/:objectName', async (request) => {
+		.post('/uploads/:objectName', async (request) => {
 			return handleUserAssetUpload({
 				request,
 				bucket: this.env.BEMO_BUCKET,
@@ -34,6 +37,10 @@ export default class Worker extends WorkerEntrypoint<Environment> {
 				context: this.ctx,
 			})
 		})
+		.get('/bookmarks/unfurl', async (request) => {
+			const query = parseRequestQuery(request, urlMetadataQueryValidator)
+			return Response.json(await getUrlMetadata(query))
+		})
 		.get('/do', async (request) => {
 			const bemo = this.env.BEMO_DO.get(this.env.BEMO_DO.idFromName('bemo-do'))
 			const message = await (await bemo.fetch(request)).json()
@@ -42,17 +49,12 @@ export default class Worker extends WorkerEntrypoint<Environment> {
 		.all('*', notFound)
 
 	override async fetch(request: Request): Promise<Response> {
-		try {
+		return handleApiRequest({
-			return await this.router.handle(request).then(cors.corsify)
+			router: this.router,
-		} catch (error) {
+			request,
-			const sentry = createSentry(this.ctx, this.env, request)
+			env: this.env,
-			console.error(error)
+			ctx: this.ctx,
-			// eslint-disable-next-line deprecation/deprecation
+			after: cors.corsify,
-			sentry?.captureException(error)
-			return new Response('Something went wrong', {
-				status: 500,
-				statusText: 'Internal Server Error',
 		})
 	}
 }
-}

apps/dotcom-asset-upload/src/worker.ts

@@ -2,20 +2,20 @@
 /// <reference types="@cloudflare/workers-types" />
 
 import {
-	createSentry,
+	createRouter,
+	handleApiRequest,
 	handleUserAssetGet,
 	handleUserAssetUpload,
 	notFound,
 } from '@tldraw/worker-shared'
 import { WorkerEntrypoint } from 'cloudflare:workers'
 import { createCors } from 'itty-cors'
-import { Router } from 'itty-router'
 import { Environment } from './types'
 
 const { preflight, corsify } = createCors({ origins: ['*'] })
 
 export default class Worker extends WorkerEntrypoint<Environment> {
-	readonly router = Router()
+	readonly router = createRouter<Environment>()
 		.all('*', preflight)
 		.get('/uploads/:objectName', async (request) => {
 			return handleUserAssetGet({
@@ -36,17 +36,12 @@ export default class Worker extends WorkerEntrypoint<Environment> {
 		.all('*', notFound)
 
 	override async fetch(request: Request) {
-		try {
+		return handleApiRequest({
-			return await this.router.handle(request, this.env, this.ctx).then(corsify)
+			router: this.router,
-		} catch (error) {
+			request,
-			const sentry = createSentry(this.ctx, this.env, request)
+			env: this.env,
-			console.error(error)
+			ctx: this.ctx,
-			// eslint-disable-next-line deprecation/deprecation
+			after: corsify,
-			sentry?.captureException(error)
-			return new Response('Something went wrong', {
-				status: 500,
-				statusText: 'Internal Server Error',
 		})
 	}
 }
-}

apps/dotcom-worker/src/worker.ts

@@ -6,9 +6,16 @@ import {
 	ROOM_OPEN_MODE,
 	ROOM_PREFIX,
 } from '@tldraw/dotcom-shared'
-import { T } from '@tldraw/validate'
+import {
-import { createSentry, notFound } from '@tldraw/worker-shared'
+	createRouter,
-import { Router, createCors, json } from 'itty-router'
+	getUrlMetadata,
+	handleApiRequest,
+	notFound,
+	parseRequestQuery,
+	urlMetadataQueryValidator,
+} from '@tldraw/worker-shared'
+import { WorkerEntrypoint } from 'cloudflare:workers'
+import { createCors, json } from 'itty-router'
 import { createRoom } from './routes/createRoom'
 import { createRoomSnapshot } from './routes/createRoomSnapshot'
 import { forwardRoomRequest } from './routes/forwardRoomRequest'
@@ -18,14 +25,13 @@ import { getRoomHistorySnapshot } from './routes/getRoomHistorySnapshot'
 import { getRoomSnapshot } from './routes/getRoomSnapshot'
 import { joinExistingRoom } from './routes/joinExistingRoom'
 import { Environment } from './types'
-import { unfurl } from './utils/unfurl'
 export { TLDrawDurableObject } from './TLDrawDurableObject'
 
 const { preflight, corsify } = createCors({
 	origins: Object.assign([], { includes: (origin: string) => isAllowedOrigin(origin) }),
 })
 
-const router = Router()
+const router = createRouter<Environment>()
 	.all('*', preflight)
 	.all('*', blockUnknownOrigins)
 	.post('/new-room', createRoom)
@@ -44,31 +50,20 @@ const router = Router()
 	.get(`/${ROOM_PREFIX}/:roomId/history/:timestamp`, getRoomHistorySnapshot)
 	.get('/readonly-slug/:roomId', getReadonlySlug)
 	.get('/unfurl', async (req) => {
-		if (typeof req.query.url !== 'string' || !T.httpUrl.isValid(req.query.url)) {
+		const query = parseRequestQuery(req, urlMetadataQueryValidator)
-			return new Response('url query param is required', { status: 400 })
+		return json(await getUrlMetadata(query))
-		}
-		return json(await unfurl(req.query.url))
 	})
 	.post(`/${ROOM_PREFIX}/:roomId/restore`, forwardRoomRequest)
 	.all('*', notFound)
 
-const Worker = {
+export default class Worker extends WorkerEntrypoint<Environment> {
-	fetch(request: Request, env: Environment, context: ExecutionContext) {
+	override async fetch(request: Request): Promise<Response> {
-		const sentry = createSentry(context, env, request)
+		return await handleApiRequest({
-
+			router,
-		return router
+			request,
-			.handle(request, env, context)
+			env: this.env,
-			.catch((err) => {
+			ctx: this.ctx,
-				console.error(err)
+			after: (response) => {
-				// eslint-disable-next-line deprecation/deprecation
-				sentry?.captureException(err)
-
-				return new Response('Something went wrong', {
-					status: 500,
-					statusText: 'Internal Server Error',
-				})
-			})
-			.then((response) => {
 				const setCookies = response.headers.getAll('set-cookie')
 				// unfortunately corsify mishandles the set-cookie header, so
 				// we need to manually add it back in
@@ -83,8 +78,9 @@ const Worker = {
 					newResponse.headers.append('set-cookie', cookie)
 				}
 				return newResponse
-			})
 			},
+		})
+	}
 }
 
 export function isAllowedOrigin(origin: string) {
@@ -117,5 +113,3 @@ async function blockUnknownOrigins(request: Request, env: Environment) {
 	// origin doesn't match, so we can continue
 	return undefined
 }
-
-export default Worker

apps/dotcom/src/utils/createAssetFromUrl.ts

@@ -9,6 +9,7 @@ interface ResponseBody {
 }
 
 export async function createAssetFromUrl({ url }: { type: 'url'; url: string }): Promise<TLAsset> {
+	const urlHash = getHashForString(url)
 	try {
 		// First, try to get the meta data from our endpoint
 		const fetchUrl =
@@ -18,65 +19,34 @@ export async function createAssetFromUrl({ url }: { type: 'url'; url: string }):
 				url,
 			}).toString()
 
-		const meta = (await (await fetch(fetchUrl)).json()) as ResponseBody
+		const meta = (await (await fetch(fetchUrl)).json()) as ResponseBody | null
 
 		return {
-			id: AssetRecordType.createId(getHashForString(url)),
+			id: AssetRecordType.createId(urlHash),
 			typeName: 'asset',
 			type: 'bookmark',
 			props: {
 				src: url,
-				description: meta.description ?? '',
+				description: meta?.description ?? '',
-				image: meta.image ?? '',
+				image: meta?.image ?? '',
-				favicon: meta.favicon ?? '',
+				favicon: meta?.favicon ?? '',
-				title: meta.title ?? '',
+				title: meta?.title ?? '',
 			},
 			meta: {},
 		}
 	} catch (error) {
-		// Otherwise, fallback to fetching data from the url
+		// Otherwise, fallback to a blank bookmark
-
-		let meta: { image: string; favicon: string; title: string; description: string }
-
-		try {
-			const resp = await fetch(url, {
-				method: 'GET',
-				mode: 'no-cors',
-			})
-			const html = await resp.text()
-			const doc = new DOMParser().parseFromString(html, 'text/html')
-			meta = {
-				image: doc.head.querySelector('meta[property="og:image"]')?.getAttribute('content') ?? '',
-				favicon:
-					doc.head.querySelector('link[rel="apple-touch-icon"]')?.getAttribute('href') ??
-					doc.head.querySelector('link[rel="icon"]')?.getAttribute('href') ??
-					'',
-				title: doc.head.querySelector('meta[property="og:title"]')?.getAttribute('content') ?? '',
-				description:
-					doc.head.querySelector('meta[property="og:description"]')?.getAttribute('content') ?? '',
-			}
-			if (!meta.image.startsWith('http')) {
-				meta.image = new URL(meta.image, url).href
-			}
-			if (!meta.favicon.startsWith('http')) {
-				meta.favicon = new URL(meta.favicon, url).href
-			}
-		} catch (error) {
 		console.error(error)
-			meta = { image: '', favicon: '', title: '', description: '' }
-		}
-
-		// Create the bookmark asset from the meta
 		return {
-			id: AssetRecordType.createId(getHashForString(url)),
+			id: AssetRecordType.createId(urlHash),
 			typeName: 'asset',
 			type: 'bookmark',
 			props: {
 				src: url,
-				image: meta.image,
+				description: '',
-				favicon: meta.favicon,
+				image: '',
-				title: meta.title,
+				favicon: '',
-				description: meta.description,
+				title: '',
 			},
 			meta: {},
 		}

packages/tlschema/api-report.md

@@ -104,7 +104,7 @@ export const CameraRecordType: RecordType<TLCamera, never>;
 export const canvasUiColorTypeValidator: T.Validator<"accent" | "black" | "laser" | "muted-1" | "selection-fill" | "selection-stroke" | "white">;
 
 // @public
-export function createAssetValidator<Type extends string, Props extends JsonObject>(type: Type, props: T.Validator<Props>): T.ObjectValidator<{ [P in T.ExtractRequiredKeys<{
+export function createAssetValidator<Type extends string, Props extends JsonObject>(type: Type, props: T.Validator<Props>): T.ObjectValidator<Expand<    { [P in T.ExtractRequiredKeys<{
 id: TLAssetId;
 meta: JsonObject;
 props: Props;
@@ -128,7 +128,7 @@ export function createAssetValidator<Type extends string, Props extends JsonObje
 props: Props;
 type: Type;
 typeName: 'asset';
-    }[P_1] | undefined; }>;
+}[P_1] | undefined; }>>;
 
 // @public (undocumented)
 export function createBindingId(id?: string): TLBindingId;
@@ -146,7 +146,7 @@ export function createBindingValidator<Type extends string, Props extends JsonOb
     [K in keyof Props]: T.Validatable<Props[K]>;
 }, meta?: {
     [K in keyof Meta]: T.Validatable<Meta[K]>;
-}): T.ObjectValidator<{ [P in T.ExtractRequiredKeys<TLBaseBinding<Type, Props>>]: TLBaseBinding<Type, Props>[P]; } & { [P_1 in T.ExtractOptionalKeys<TLBaseBinding<Type, Props>>]?: TLBaseBinding<Type, Props>[P_1] | undefined; }>;
+}): T.ObjectValidator<Expand<    { [P in T.ExtractRequiredKeys<TLBaseBinding<Type, Props>>]: TLBaseBinding<Type, Props>[P]; } & { [P_1 in T.ExtractOptionalKeys<TLBaseBinding<Type, Props>>]?: TLBaseBinding<Type, Props>[P_1] | undefined; }>>;
 
 // @public
 export const createPresenceStateDerivation: ($user: Signal<{
@@ -171,7 +171,7 @@ export function createShapeValidator<Type extends string, Props extends JsonObje
     [K in keyof Props]: T.Validatable<Props[K]>;
 }, meta?: {
     [K in keyof Meta]: T.Validatable<Meta[K]>;
-}): T.ObjectValidator<{ [P in T.ExtractRequiredKeys<TLBaseShape<Type, Props>>]: TLBaseShape<Type, Props>[P]; } & { [P_1 in T.ExtractOptionalKeys<TLBaseShape<Type, Props>>]?: TLBaseShape<Type, Props>[P_1] | undefined; }>;
+}): T.ObjectValidator<Expand<    { [P in T.ExtractRequiredKeys<TLBaseShape<Type, Props>>]: TLBaseShape<Type, Props>[P]; } & { [P_1 in T.ExtractOptionalKeys<TLBaseShape<Type, Props>>]?: TLBaseShape<Type, Props>[P_1] | undefined; }>>;
 
 // @public
 export function createTLSchema({ shapes, bindings, migrations, }?: {
@@ -588,7 +588,7 @@ export function idValidator<Id extends RecordId<UnknownRecord>>(prefix: Id['__ty
 export const ImageShapeCrop: T.ObjectValidator<{
     bottomRight: VecModel;
     topLeft: VecModel;
-} & {}>;
+}>;
 
 // @public (undocumented)
 export const imageShapeMigrations: TLPropsMigrations;
@@ -596,10 +596,10 @@ export const imageShapeMigrations: TLPropsMigrations;
 // @public (undocumented)
 export const imageShapeProps: {
     assetId: T.Validator<TLAssetId | null>;
-    crop: T.Validator<({
+    crop: T.Validator<{
         bottomRight: VecModel;
         topLeft: VecModel;
-    } & {}) | null>;
+    } | null>;
     h: T.Validator<number>;
     playing: T.Validator<boolean>;
     url: T.Validator<string>;
@@ -753,7 +753,7 @@ export const lineShapeProps: {
         index: IndexKey;
         x: number;
         y: number;
-    } & {}>;
+    }>;
     scale: T.Validator<number>;
     size: EnumStyleProp<"l" | "m" | "s" | "xl">;
     spline: EnumStyleProp<"cubic" | "line">;

packages/validate/api-report.md

@@ -4,6 +4,7 @@
 
 ```ts
 
+import { Expand } from '@tldraw/utils';
 import { IndexKey } from '@tldraw/utils';
 import { JsonValue } from '@tldraw/utils';
 
@@ -102,11 +103,11 @@ function numberUnion<Key extends string, Config extends UnionValidatorConfig<Key
 // @public
 function object<Shape extends object>(config: {
     readonly [K in keyof Shape]: Validatable<Shape[K]>;
-}): ObjectValidator<{
+}): ObjectValidator<Expand<{
     [P in ExtractRequiredKeys<Shape>]: Shape[P];
 } & {
     [P in ExtractOptionalKeys<Shape>]?: Shape[P];
-}>;
+}>>;
 
 // @public (undocumented)
 export class ObjectValidator<Shape extends object> extends Validator<Shape> {

packages/validate/src/lib/validation.ts

@@ -1,4 +1,5 @@
 import {
+	Expand,
 	IndexKey,
 	JsonValue,
 	STRUCTURED_CLONE_OBJECT_PROTOTYPE,
@@ -696,7 +697,11 @@ export type ExtractOptionalKeys<T extends object> = {
 export function object<Shape extends object>(config: {
 	readonly [K in keyof Shape]: Validatable<Shape[K]>
 }): ObjectValidator<
-	{ [P in ExtractRequiredKeys<Shape>]: Shape[P] } & { [P in ExtractOptionalKeys<Shape>]?: Shape[P] }
+	Expand<
+		{ [P in ExtractRequiredKeys<Shape>]: Shape[P] } & {
+			[P in ExtractOptionalKeys<Shape>]?: Shape[P]
+		}
+	>
 > {
 	return new ObjectValidator(config) as any
 }

packages/worker-shared/package.json

@@ -10,6 +10,8 @@
 	"dependencies": {
 		"@cloudflare/workers-types": "^4.20240620.0",
 		"@tldraw/utils": "workspace:*",
+		"@tldraw/validate": "workspace:*",
+		"itty-router": "^4.0.13",
 		"lazyrepo": "0.0.0-alpha.27",
 		"toucan-js": "^3.4.0",
 		"typescript": "^5.3.3"

packages/worker-shared/src/getUrlMetadata.ts

@@ -1,3 +1,9 @@
+import { T } from '@tldraw/validate'
+
+export const urlMetadataQueryValidator = T.object({
+	url: T.httpUrl,
+})
+
 class TextExtractor {
 	string = ''
 	text({ text }: any) {
@@ -38,18 +44,23 @@ class IconExtractor {
 	}
 }
 
-export async function unfurl(url: string) {
+export async function getUrlMetadata({ url }: { url: string }) {
 	const meta$ = new MetaExtractor()
 	const title$ = new TextExtractor()
 	const icon$ = new IconExtractor()
-	// we use cloudflare's special html parser https://developers.cloudflare.com/workers/runtime-apis/html-rewriter/
+
+	try {
 		await new HTMLRewriter()
 			.on('meta', meta$)
 			.on('title', title$)
 			.on('link', icon$)
 			.transform((await fetch(url)) as any)
-		.blob?.()
+			.blob()
+	} catch {
+		return null
+	}
 
+	// we use cloudflare's special html parser https://developers.cloudflare.com/workers/runtime-apis/html-rewriter/
 	const { og, twitter } = meta$
 	const title = og['og:title'] ?? twitter['twitter:title'] ?? title$.string ?? undefined
 	const description =

packages/worker-shared/src/handleRequest.ts

@@ -0,0 +1,73 @@
+import { T } from '@tldraw/validate'
+import { IRequest, RouteHandler, Router, RouterType, StatusError } from 'itty-router'
+import { SentryEnvironment, createSentry } from './sentry'
+
+export type ApiRoute<Env extends SentryEnvironment, Ctx extends ExecutionContext> = (
+	path: string,
+	...handlers: RouteHandler<IRequest, [env: Env, ctx: Ctx]>[]
+) => RouterType<ApiRoute<Env, Ctx>, [env: Env, ctx: Ctx]>
+
+export type ApiRouter<Env extends SentryEnvironment, Ctx extends ExecutionContext> = RouterType<
+	ApiRoute<Env, Ctx>,
+	[env: Env, ctx: Ctx]
+>
+
+export function createRouter<
+	Env extends SentryEnvironment,
+	Ctx extends ExecutionContext = ExecutionContext,
+>() {
+	const router: ApiRouter<Env, Ctx> = Router()
+	return router
+}
+
+export async function handleApiRequest<
+	Env extends SentryEnvironment,
+	Ctx extends ExecutionContext,
+>({
+	router,
+	request,
+	env,
+	ctx,
+	after,
+}: {
+	router: ApiRouter<Env, Ctx>
+	request: Request
+	env: Env
+	ctx: Ctx
+	after: (response: Response) => Response | Promise<Response>
+}) {
+	let response
+	try {
+		response = await router.handle(request, env, ctx)
+	} catch (error: any) {
+		if (error instanceof StatusError) {
+			console.error(`${error.status}: ${error.stack}`)
+			response = Response.json({ error: error.message }, { status: error.status })
+		} else {
+			response = Response.json({ error: 'Internal server error' }, { status: 500 })
+			console.error(error.stack ?? error)
+			// eslint-disable-next-line deprecation/deprecation
+			createSentry(ctx, env, request)?.captureException(error)
+		}
+	}
+
+	try {
+		return await after(response)
+	} catch (error: any) {
+		console.error(error.stack ?? error)
+		// eslint-disable-next-line deprecation/deprecation
+		createSentry(ctx, env, request)?.captureException(error)
+		return Response.json({ error: 'Internal server error' }, { status: 500 })
+	}
+}
+
+export function parseRequestQuery<Params>(request: IRequest, validator: T.Validator<Params>) {
+	try {
+		return validator.validate(request.query)
+	} catch (err) {
+		if (err instanceof T.ValidationError) {
+			throw new StatusError(400, `Query parameters: ${err.message}`)
+		}
+		throw err
+	}
+}

packages/worker-shared/src/index.ts

@@ -2,5 +2,13 @@
 /// <reference types="@cloudflare/workers-types" />
 
 export { notFound } from './errors'
+export { getUrlMetadata, urlMetadataQueryValidator } from './getUrlMetadata'
+export {
+	createRouter,
+	handleApiRequest,
+	parseRequestQuery,
+	type ApiRoute,
+	type ApiRouter,
+} from './handleRequest'
 export { createSentry } from './sentry'
 export { handleUserAssetGet, handleUserAssetUpload } from './userAssetUploads'

packages/worker-shared/src/sentry.ts

@@ -7,7 +7,7 @@ interface Context {
 	request?: Request
 }
 
-interface SentryEnvironment {
+export interface SentryEnvironment {
 	readonly SENTRY_DSN: string | undefined
 	readonly TLDRAW_ENV?: string | undefined
 	readonly WORKER_NAME: string | undefined

packages/worker-shared/tsconfig.json

@@ -9,6 +9,9 @@
 	"references": [
 		{
 			"path": "../utils"
+		},
+		{
+			"path": "../validate"
 		}
 	]
 }

yarn.lock

@@ -6357,6 +6357,8 @@ __metadata:
   dependencies:
     "@cloudflare/workers-types": "npm:^4.20240620.0"
     "@tldraw/utils": "workspace:*"
+    "@tldraw/validate": "workspace:*"
+    itty-router: "npm:^4.0.13"
     lazyrepo: "npm:0.0.0-alpha.27"
     toucan-js: "npm:^3.4.0"
     typescript: "npm:^5.3.3"