Demo server bookmark unfurl endpoint (#4062)

This adds the HTMLRewriter-based bookmark unfurler to the demo server.
It moves the unfurler into worker-shared, and adds some better shared
error handling across our workers.

I removed the fallback bookmark fetcher where we try and fetch websites
locally. This will almost never work, as it requires sites to set public
CORS.

### Change type
- [x] `other`
This commit is contained in:
alex 2024-07-03 11:48:34 +01:00 committed by GitHub
parent 51e81d8357
commit 8906bd8ffa
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 213 additions and 147 deletions

View file

@ -2,10 +2,13 @@
/// <reference types="@cloudflare/workers-types" />
import {
createSentry,
getUrlMetadata,
handleApiRequest,
handleUserAssetGet,
handleUserAssetUpload,
notFound,
parseRequestQuery,
urlMetadataQueryValidator,
} from '@tldraw/worker-shared'
import { WorkerEntrypoint } from 'cloudflare:workers'
import { Router, createCors } from 'itty-router'
@ -18,7 +21,7 @@ const cors = createCors({ origins: ['*'] })
export default class Worker extends WorkerEntrypoint<Environment> {
private readonly router = Router()
.all('*', cors.preflight)
.get('/v1/uploads/:objectName', (request) => {
.get('/uploads/:objectName', (request) => {
return handleUserAssetGet({
request,
bucket: this.env.BEMO_BUCKET,
@ -26,7 +29,7 @@ export default class Worker extends WorkerEntrypoint<Environment> {
context: this.ctx,
})
})
.post('/v1/uploads/:objectName', async (request) => {
.post('/uploads/:objectName', async (request) => {
return handleUserAssetUpload({
request,
bucket: this.env.BEMO_BUCKET,
@ -34,6 +37,10 @@ export default class Worker extends WorkerEntrypoint<Environment> {
context: this.ctx,
})
})
.get('/bookmarks/unfurl', async (request) => {
const query = parseRequestQuery(request, urlMetadataQueryValidator)
return Response.json(await getUrlMetadata(query))
})
.get('/do', async (request) => {
const bemo = this.env.BEMO_DO.get(this.env.BEMO_DO.idFromName('bemo-do'))
const message = await (await bemo.fetch(request)).json()
@ -42,17 +49,12 @@ export default class Worker extends WorkerEntrypoint<Environment> {
.all('*', notFound)
override async fetch(request: Request): Promise<Response> {
try {
return await this.router.handle(request).then(cors.corsify)
} catch (error) {
const sentry = createSentry(this.ctx, this.env, request)
console.error(error)
// eslint-disable-next-line deprecation/deprecation
sentry?.captureException(error)
return new Response('Something went wrong', {
status: 500,
statusText: 'Internal Server Error',
})
}
return handleApiRequest({
router: this.router,
request,
env: this.env,
ctx: this.ctx,
after: cors.corsify,
})
}
}

View file

@ -2,20 +2,20 @@
/// <reference types="@cloudflare/workers-types" />
import {
createSentry,
createRouter,
handleApiRequest,
handleUserAssetGet,
handleUserAssetUpload,
notFound,
} from '@tldraw/worker-shared'
import { WorkerEntrypoint } from 'cloudflare:workers'
import { createCors } from 'itty-cors'
import { Router } from 'itty-router'
import { Environment } from './types'
const { preflight, corsify } = createCors({ origins: ['*'] })
export default class Worker extends WorkerEntrypoint<Environment> {
readonly router = Router()
readonly router = createRouter<Environment>()
.all('*', preflight)
.get('/uploads/:objectName', async (request) => {
return handleUserAssetGet({
@ -36,17 +36,12 @@ export default class Worker extends WorkerEntrypoint<Environment> {
.all('*', notFound)
override async fetch(request: Request) {
try {
return await this.router.handle(request, this.env, this.ctx).then(corsify)
} catch (error) {
const sentry = createSentry(this.ctx, this.env, request)
console.error(error)
// eslint-disable-next-line deprecation/deprecation
sentry?.captureException(error)
return new Response('Something went wrong', {
status: 500,
statusText: 'Internal Server Error',
})
}
return handleApiRequest({
router: this.router,
request,
env: this.env,
ctx: this.ctx,
after: corsify,
})
}
}

View file

@ -6,9 +6,16 @@ import {
ROOM_OPEN_MODE,
ROOM_PREFIX,
} from '@tldraw/dotcom-shared'
import { T } from '@tldraw/validate'
import { createSentry, notFound } from '@tldraw/worker-shared'
import { Router, createCors, json } from 'itty-router'
import {
createRouter,
getUrlMetadata,
handleApiRequest,
notFound,
parseRequestQuery,
urlMetadataQueryValidator,
} from '@tldraw/worker-shared'
import { WorkerEntrypoint } from 'cloudflare:workers'
import { createCors, json } from 'itty-router'
import { createRoom } from './routes/createRoom'
import { createRoomSnapshot } from './routes/createRoomSnapshot'
import { forwardRoomRequest } from './routes/forwardRoomRequest'
@ -18,14 +25,13 @@ import { getRoomHistorySnapshot } from './routes/getRoomHistorySnapshot'
import { getRoomSnapshot } from './routes/getRoomSnapshot'
import { joinExistingRoom } from './routes/joinExistingRoom'
import { Environment } from './types'
import { unfurl } from './utils/unfurl'
export { TLDrawDurableObject } from './TLDrawDurableObject'
const { preflight, corsify } = createCors({
origins: Object.assign([], { includes: (origin: string) => isAllowedOrigin(origin) }),
})
const router = Router()
const router = createRouter<Environment>()
.all('*', preflight)
.all('*', blockUnknownOrigins)
.post('/new-room', createRoom)
@ -44,31 +50,20 @@ const router = Router()
.get(`/${ROOM_PREFIX}/:roomId/history/:timestamp`, getRoomHistorySnapshot)
.get('/readonly-slug/:roomId', getReadonlySlug)
.get('/unfurl', async (req) => {
if (typeof req.query.url !== 'string' || !T.httpUrl.isValid(req.query.url)) {
return new Response('url query param is required', { status: 400 })
}
return json(await unfurl(req.query.url))
const query = parseRequestQuery(req, urlMetadataQueryValidator)
return json(await getUrlMetadata(query))
})
.post(`/${ROOM_PREFIX}/:roomId/restore`, forwardRoomRequest)
.all('*', notFound)
const Worker = {
fetch(request: Request, env: Environment, context: ExecutionContext) {
const sentry = createSentry(context, env, request)
return router
.handle(request, env, context)
.catch((err) => {
console.error(err)
// eslint-disable-next-line deprecation/deprecation
sentry?.captureException(err)
return new Response('Something went wrong', {
status: 500,
statusText: 'Internal Server Error',
})
})
.then((response) => {
export default class Worker extends WorkerEntrypoint<Environment> {
override async fetch(request: Request): Promise<Response> {
return await handleApiRequest({
router,
request,
env: this.env,
ctx: this.ctx,
after: (response) => {
const setCookies = response.headers.getAll('set-cookie')
// unfortunately corsify mishandles the set-cookie header, so
// we need to manually add it back in
@ -83,8 +78,9 @@ const Worker = {
newResponse.headers.append('set-cookie', cookie)
}
return newResponse
})
},
},
})
}
}
export function isAllowedOrigin(origin: string) {
@ -117,5 +113,3 @@ async function blockUnknownOrigins(request: Request, env: Environment) {
// origin doesn't match, so we can continue
return undefined
}
export default Worker

View file

@ -9,6 +9,7 @@ interface ResponseBody {
}
export async function createAssetFromUrl({ url }: { type: 'url'; url: string }): Promise<TLAsset> {
const urlHash = getHashForString(url)
try {
// First, try to get the meta data from our endpoint
const fetchUrl =
@ -18,65 +19,34 @@ export async function createAssetFromUrl({ url }: { type: 'url'; url: string }):
url,
}).toString()
const meta = (await (await fetch(fetchUrl)).json()) as ResponseBody
const meta = (await (await fetch(fetchUrl)).json()) as ResponseBody | null
return {
id: AssetRecordType.createId(getHashForString(url)),
id: AssetRecordType.createId(urlHash),
typeName: 'asset',
type: 'bookmark',
props: {
src: url,
description: meta.description ?? '',
image: meta.image ?? '',
favicon: meta.favicon ?? '',
title: meta.title ?? '',
description: meta?.description ?? '',
image: meta?.image ?? '',
favicon: meta?.favicon ?? '',
title: meta?.title ?? '',
},
meta: {},
}
} catch (error) {
// Otherwise, fallback to fetching data from the url
let meta: { image: string; favicon: string; title: string; description: string }
try {
const resp = await fetch(url, {
method: 'GET',
mode: 'no-cors',
})
const html = await resp.text()
const doc = new DOMParser().parseFromString(html, 'text/html')
meta = {
image: doc.head.querySelector('meta[property="og:image"]')?.getAttribute('content') ?? '',
favicon:
doc.head.querySelector('link[rel="apple-touch-icon"]')?.getAttribute('href') ??
doc.head.querySelector('link[rel="icon"]')?.getAttribute('href') ??
'',
title: doc.head.querySelector('meta[property="og:title"]')?.getAttribute('content') ?? '',
description:
doc.head.querySelector('meta[property="og:description"]')?.getAttribute('content') ?? '',
}
if (!meta.image.startsWith('http')) {
meta.image = new URL(meta.image, url).href
}
if (!meta.favicon.startsWith('http')) {
meta.favicon = new URL(meta.favicon, url).href
}
} catch (error) {
console.error(error)
meta = { image: '', favicon: '', title: '', description: '' }
}
// Create the bookmark asset from the meta
// Otherwise, fallback to a blank bookmark
console.error(error)
return {
id: AssetRecordType.createId(getHashForString(url)),
id: AssetRecordType.createId(urlHash),
typeName: 'asset',
type: 'bookmark',
props: {
src: url,
image: meta.image,
favicon: meta.favicon,
title: meta.title,
description: meta.description,
description: '',
image: '',
favicon: '',
title: '',
},
meta: {},
}

View file

@ -104,31 +104,31 @@ export const CameraRecordType: RecordType<TLCamera, never>;
export const canvasUiColorTypeValidator: T.Validator<"accent" | "black" | "laser" | "muted-1" | "selection-fill" | "selection-stroke" | "white">;
// @public
export function createAssetValidator<Type extends string, Props extends JsonObject>(type: Type, props: T.Validator<Props>): T.ObjectValidator<{ [P in T.ExtractRequiredKeys<{
id: TLAssetId;
meta: JsonObject;
props: Props;
type: Type;
typeName: 'asset';
}>]: {
id: TLAssetId;
meta: JsonObject;
props: Props;
type: Type;
typeName: 'asset';
}[P]; } & { [P_1 in T.ExtractOptionalKeys<{
id: TLAssetId;
meta: JsonObject;
props: Props;
type: Type;
typeName: 'asset';
}>]?: {
id: TLAssetId;
meta: JsonObject;
props: Props;
type: Type;
typeName: 'asset';
}[P_1] | undefined; }>;
export function createAssetValidator<Type extends string, Props extends JsonObject>(type: Type, props: T.Validator<Props>): T.ObjectValidator<Expand< { [P in T.ExtractRequiredKeys<{
id: TLAssetId;
meta: JsonObject;
props: Props;
type: Type;
typeName: 'asset';
}>]: {
id: TLAssetId;
meta: JsonObject;
props: Props;
type: Type;
typeName: 'asset';
}[P]; } & { [P_1 in T.ExtractOptionalKeys<{
id: TLAssetId;
meta: JsonObject;
props: Props;
type: Type;
typeName: 'asset';
}>]?: {
id: TLAssetId;
meta: JsonObject;
props: Props;
type: Type;
typeName: 'asset';
}[P_1] | undefined; }>>;
// @public (undocumented)
export function createBindingId(id?: string): TLBindingId;
@ -146,7 +146,7 @@ export function createBindingValidator<Type extends string, Props extends JsonOb
[K in keyof Props]: T.Validatable<Props[K]>;
}, meta?: {
[K in keyof Meta]: T.Validatable<Meta[K]>;
}): T.ObjectValidator<{ [P in T.ExtractRequiredKeys<TLBaseBinding<Type, Props>>]: TLBaseBinding<Type, Props>[P]; } & { [P_1 in T.ExtractOptionalKeys<TLBaseBinding<Type, Props>>]?: TLBaseBinding<Type, Props>[P_1] | undefined; }>;
}): T.ObjectValidator<Expand< { [P in T.ExtractRequiredKeys<TLBaseBinding<Type, Props>>]: TLBaseBinding<Type, Props>[P]; } & { [P_1 in T.ExtractOptionalKeys<TLBaseBinding<Type, Props>>]?: TLBaseBinding<Type, Props>[P_1] | undefined; }>>;
// @public
export const createPresenceStateDerivation: ($user: Signal<{
@ -171,7 +171,7 @@ export function createShapeValidator<Type extends string, Props extends JsonObje
[K in keyof Props]: T.Validatable<Props[K]>;
}, meta?: {
[K in keyof Meta]: T.Validatable<Meta[K]>;
}): T.ObjectValidator<{ [P in T.ExtractRequiredKeys<TLBaseShape<Type, Props>>]: TLBaseShape<Type, Props>[P]; } & { [P_1 in T.ExtractOptionalKeys<TLBaseShape<Type, Props>>]?: TLBaseShape<Type, Props>[P_1] | undefined; }>;
}): T.ObjectValidator<Expand< { [P in T.ExtractRequiredKeys<TLBaseShape<Type, Props>>]: TLBaseShape<Type, Props>[P]; } & { [P_1 in T.ExtractOptionalKeys<TLBaseShape<Type, Props>>]?: TLBaseShape<Type, Props>[P_1] | undefined; }>>;
// @public
export function createTLSchema({ shapes, bindings, migrations, }?: {
@ -588,7 +588,7 @@ export function idValidator<Id extends RecordId<UnknownRecord>>(prefix: Id['__ty
export const ImageShapeCrop: T.ObjectValidator<{
bottomRight: VecModel;
topLeft: VecModel;
} & {}>;
}>;
// @public (undocumented)
export const imageShapeMigrations: TLPropsMigrations;
@ -596,10 +596,10 @@ export const imageShapeMigrations: TLPropsMigrations;
// @public (undocumented)
export const imageShapeProps: {
assetId: T.Validator<TLAssetId | null>;
crop: T.Validator<({
crop: T.Validator<{
bottomRight: VecModel;
topLeft: VecModel;
} & {}) | null>;
} | null>;
h: T.Validator<number>;
playing: T.Validator<boolean>;
url: T.Validator<string>;
@ -753,7 +753,7 @@ export const lineShapeProps: {
index: IndexKey;
x: number;
y: number;
} & {}>;
}>;
scale: T.Validator<number>;
size: EnumStyleProp<"l" | "m" | "s" | "xl">;
spline: EnumStyleProp<"cubic" | "line">;

View file

@ -4,6 +4,7 @@
```ts
import { Expand } from '@tldraw/utils';
import { IndexKey } from '@tldraw/utils';
import { JsonValue } from '@tldraw/utils';
@ -102,11 +103,11 @@ function numberUnion<Key extends string, Config extends UnionValidatorConfig<Key
// @public
function object<Shape extends object>(config: {
readonly [K in keyof Shape]: Validatable<Shape[K]>;
}): ObjectValidator<{
}): ObjectValidator<Expand<{
[P in ExtractRequiredKeys<Shape>]: Shape[P];
} & {
[P in ExtractOptionalKeys<Shape>]?: Shape[P];
}>;
}>>;
// @public (undocumented)
export class ObjectValidator<Shape extends object> extends Validator<Shape> {

View file

@ -1,4 +1,5 @@
import {
Expand,
IndexKey,
JsonValue,
STRUCTURED_CLONE_OBJECT_PROTOTYPE,
@ -696,7 +697,11 @@ export type ExtractOptionalKeys<T extends object> = {
export function object<Shape extends object>(config: {
readonly [K in keyof Shape]: Validatable<Shape[K]>
}): ObjectValidator<
{ [P in ExtractRequiredKeys<Shape>]: Shape[P] } & { [P in ExtractOptionalKeys<Shape>]?: Shape[P] }
Expand<
{ [P in ExtractRequiredKeys<Shape>]: Shape[P] } & {
[P in ExtractOptionalKeys<Shape>]?: Shape[P]
}
>
> {
return new ObjectValidator(config) as any
}

View file

@ -10,6 +10,8 @@
"dependencies": {
"@cloudflare/workers-types": "^4.20240620.0",
"@tldraw/utils": "workspace:*",
"@tldraw/validate": "workspace:*",
"itty-router": "^4.0.13",
"lazyrepo": "0.0.0-alpha.27",
"toucan-js": "^3.4.0",
"typescript": "^5.3.3"

View file

@ -1,3 +1,9 @@
import { T } from '@tldraw/validate'
export const urlMetadataQueryValidator = T.object({
url: T.httpUrl,
})
class TextExtractor {
string = ''
text({ text }: any) {
@ -38,18 +44,23 @@ class IconExtractor {
}
}
export async function unfurl(url: string) {
export async function getUrlMetadata({ url }: { url: string }) {
const meta$ = new MetaExtractor()
const title$ = new TextExtractor()
const icon$ = new IconExtractor()
// we use cloudflare's special html parser https://developers.cloudflare.com/workers/runtime-apis/html-rewriter/
await new HTMLRewriter()
.on('meta', meta$)
.on('title', title$)
.on('link', icon$)
.transform((await fetch(url)) as any)
.blob?.()
try {
await new HTMLRewriter()
.on('meta', meta$)
.on('title', title$)
.on('link', icon$)
.transform((await fetch(url)) as any)
.blob()
} catch {
return null
}
// we use cloudflare's special html parser https://developers.cloudflare.com/workers/runtime-apis/html-rewriter/
const { og, twitter } = meta$
const title = og['og:title'] ?? twitter['twitter:title'] ?? title$.string ?? undefined
const description =

View file

@ -0,0 +1,73 @@
import { T } from '@tldraw/validate'
import { IRequest, RouteHandler, Router, RouterType, StatusError } from 'itty-router'
import { SentryEnvironment, createSentry } from './sentry'
export type ApiRoute<Env extends SentryEnvironment, Ctx extends ExecutionContext> = (
path: string,
...handlers: RouteHandler<IRequest, [env: Env, ctx: Ctx]>[]
) => RouterType<ApiRoute<Env, Ctx>, [env: Env, ctx: Ctx]>
export type ApiRouter<Env extends SentryEnvironment, Ctx extends ExecutionContext> = RouterType<
ApiRoute<Env, Ctx>,
[env: Env, ctx: Ctx]
>
export function createRouter<
Env extends SentryEnvironment,
Ctx extends ExecutionContext = ExecutionContext,
>() {
const router: ApiRouter<Env, Ctx> = Router()
return router
}
export async function handleApiRequest<
Env extends SentryEnvironment,
Ctx extends ExecutionContext,
>({
router,
request,
env,
ctx,
after,
}: {
router: ApiRouter<Env, Ctx>
request: Request
env: Env
ctx: Ctx
after: (response: Response) => Response | Promise<Response>
}) {
let response
try {
response = await router.handle(request, env, ctx)
} catch (error: any) {
if (error instanceof StatusError) {
console.error(`${error.status}: ${error.stack}`)
response = Response.json({ error: error.message }, { status: error.status })
} else {
response = Response.json({ error: 'Internal server error' }, { status: 500 })
console.error(error.stack ?? error)
// eslint-disable-next-line deprecation/deprecation
createSentry(ctx, env, request)?.captureException(error)
}
}
try {
return await after(response)
} catch (error: any) {
console.error(error.stack ?? error)
// eslint-disable-next-line deprecation/deprecation
createSentry(ctx, env, request)?.captureException(error)
return Response.json({ error: 'Internal server error' }, { status: 500 })
}
}
export function parseRequestQuery<Params>(request: IRequest, validator: T.Validator<Params>) {
try {
return validator.validate(request.query)
} catch (err) {
if (err instanceof T.ValidationError) {
throw new StatusError(400, `Query parameters: ${err.message}`)
}
throw err
}
}

View file

@ -2,5 +2,13 @@
/// <reference types="@cloudflare/workers-types" />
export { notFound } from './errors'
export { getUrlMetadata, urlMetadataQueryValidator } from './getUrlMetadata'
export {
createRouter,
handleApiRequest,
parseRequestQuery,
type ApiRoute,
type ApiRouter,
} from './handleRequest'
export { createSentry } from './sentry'
export { handleUserAssetGet, handleUserAssetUpload } from './userAssetUploads'

View file

@ -7,7 +7,7 @@ interface Context {
request?: Request
}
interface SentryEnvironment {
export interface SentryEnvironment {
readonly SENTRY_DSN: string | undefined
readonly TLDRAW_ENV?: string | undefined
readonly WORKER_NAME: string | undefined

View file

@ -9,6 +9,9 @@
"references": [
{
"path": "../utils"
},
{
"path": "../validate"
}
]
}

View file

@ -6357,6 +6357,8 @@ __metadata:
dependencies:
"@cloudflare/workers-types": "npm:^4.20240620.0"
"@tldraw/utils": "workspace:*"
"@tldraw/validate": "workspace:*"
itty-router: "npm:^4.0.13"
lazyrepo: "npm:0.0.0-alpha.27"
toucan-js: "npm:^3.4.0"
typescript: "npm:^5.3.3"