Demo assets server (#4055)

Adds an assets server to the demo worker, and reworks the existing asset
server to use the same code. There are a few simplifications to the code
due to some DX improvements working with R2 and caches. I also removed
the `HEAD` request from the assets server: i took a look at our logs and
it's not actually used at all.

This also fixes an issue where users could overwrite the contents of the
asset uploads bucket.

### Change type

- [x] `other`
This commit is contained in:
alex 2024-07-02 14:08:50 +01:00 committed by GitHub
parent adb84d97e3
commit dcfc6da604
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 240 additions and 209 deletions

View file

@ -54,6 +54,7 @@ jobs:
VERCEL_PROJECT_ID: ${{ vars.VERCEL_DOTCOM_PROJECT_ID }}
VERCEL_ORG_ID: ${{ vars.VERCEL_ORG_ID }}
ASSET_UPLOAD_SENTRY_DSN: ${{ secrets.ASSET_UPLOAD_SENTRY_DSN }}
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
DISCORD_DEPLOY_WEBHOOK_URL: ${{ secrets.DISCORD_DEPLOY_WEBHOOK_URL }}

View file

@ -3,10 +3,12 @@ import { BemoDO } from './BemoDO'
export interface Environment {
// bindings
BEMO_DO: DurableObjectNamespace<BemoDO>
BEMO_BUCKET: R2Bucket
CF_VERSION_METADATA: WorkerVersionMetadata
// environment variables
TLDRAW_ENV: string | undefined
SENTRY_DSN: string | undefined
IS_LOCAL: string | undefined
WORKER_NAME: string | undefined
CF_VERSION_METADATA: WorkerVersionMetadata
}

View file

@ -1,7 +1,12 @@
/// <reference no-default-lib="true"/>
/// <reference types="@cloudflare/workers-types" />
import { createSentry } from '@tldraw/worker-shared'
import {
createSentry,
handleUserAssetGet,
handleUserAssetUpload,
notFound,
} from '@tldraw/worker-shared'
import { WorkerEntrypoint } from 'cloudflare:workers'
import { Router, createCors } from 'itty-router'
import { Environment } from './types'
@ -13,12 +18,28 @@ const cors = createCors({ origins: ['*'] })
export default class Worker extends WorkerEntrypoint<Environment> {
private readonly router = Router()
.all('*', cors.preflight)
.get('/v1/uploads/:objectName', (request) => {
return handleUserAssetGet({
request,
bucket: this.env.BEMO_BUCKET,
objectName: `asset-uploads/${request.params.objectName}`,
context: this.ctx,
})
})
.post('/v1/uploads/:objectName', async (request) => {
return handleUserAssetUpload({
request,
bucket: this.env.BEMO_BUCKET,
objectName: `asset-uploads/${request.params.objectName}`,
context: this.ctx,
})
})
.get('/do', async (request) => {
const bemo = this.env.BEMO_DO.get(this.env.BEMO_DO.idFromName('bemo-do'))
const message = await (await bemo.fetch(request)).json()
return Response.json(message)
})
.all('*', async () => new Response('Not found', { status: 404 }))
.all('*', notFound)
override async fetch(request: Request): Promise<Response> {
try {

View file

@ -9,7 +9,7 @@ ip = "0.0.0.0"
# these migrations are append-only. you can't change them. if you do need to change something, do so
# by creating new migrations
[[migrations]]
tag = "v1" # Should be unique for each entry
tag = "v1" # Should be unique for each entry
new_classes = ["BemoDO"]
@ -23,44 +23,51 @@ name = "dev-bemo"
# staging is the same as a preview on main:
[env.staging]
name = "canary-bemo"
routes = [
{ pattern = "canary-demo.tldraw.xyz", custom_domain = true }
]
routes = [{ pattern = "canary-demo.tldraw.xyz", custom_domain = true }]
# production gets the proper name
[env.production]
name = "production-bemo"
routes = [
{ pattern = "demo.tldraw.xyz", custom_domain = true }
]
routes = [{ pattern = "demo.tldraw.xyz", custom_domain = true }]
#################### Durable objects ####################
# durable objects have the same configuration in all environments:
[durable_objects]
bindings = [
{ name = "BEMO_DO", class_name = "BemoDO" },
]
bindings = [{ name = "BEMO_DO", class_name = "BemoDO" }]
[env.dev.durable_objects]
bindings = [
{ name = "BEMO_DO", class_name = "BemoDO" },
]
bindings = [{ name = "BEMO_DO", class_name = "BemoDO" }]
[env.preview.durable_objects]
bindings = [
{ name = "BEMO_DO", class_name = "BemoDO" },
]
bindings = [{ name = "BEMO_DO", class_name = "BemoDO" }]
[env.staging.durable_objects]
bindings = [
{ name = "BEMO_DO", class_name = "BemoDO" },
]
bindings = [{ name = "BEMO_DO", class_name = "BemoDO" }]
[env.production.durable_objects]
bindings = [
{ name = "BEMO_DO", class_name = "BemoDO" },
]
bindings = [{ name = "BEMO_DO", class_name = "BemoDO" }]
#################### R2 bucket ####################
# in dev, we write to the preview bucket and need a `preview_bucket_name`
[[env.dev.r2_buckets]]
binding = 'BEMO_BUCKET'
bucket_name = 'sync-demo-preview'
preview_bucket_name = 'sync-demo-preview'
# in preview and staging we write to the preview bucket
[[env.preview.r2_buckets]]
binding = 'BEMO_BUCKET'
bucket_name = 'sync-demo-preview'
[[env.stating.r2_buckets]]
binding = 'BEMO_BUCKET'
bucket_name = 'sync-demo-preview'
# in production, we write to the main bucket
[[env.production.r2_buckets]]
binding = "BEMO_BUCKET"
bucket_name = "sync-demo"
#################### Version metadata ####################
[version_metadata]
@ -76,4 +83,4 @@ binding = "CF_VERSION_METADATA"
binding = "CF_VERSION_METADATA"
[env.production.version_metadata]
binding = "CF_VERSION_METADATA"
binding = "CF_VERSION_METADATA"

View file

@ -16,6 +16,7 @@
"lint": "yarn run -T tsx ../../scripts/lint.ts"
},
"dependencies": {
"@tldraw/worker-shared": "workspace:*",
"itty-cors": "^0.3.4",
"itty-router": "^4.0.13"
},

View file

@ -1,6 +1,13 @@
export interface Env {
UPLOADS: R2Bucket
import { R2Bucket, WorkerVersionMetadata } from '@cloudflare/workers-types'
KV: KVNamespace
ASSET_UPLOADER_AUTH_TOKEN: string | undefined
export interface Environment {
// bindings
UPLOADS: R2Bucket
CF_VERSION_METADATA: WorkerVersionMetadata
// environment variables
TLDRAW_ENV: string | undefined
SENTRY_DSN: string | undefined
IS_LOCAL: string | undefined
WORKER_NAME: string | undefined
}

View file

@ -1,166 +1,52 @@
/// <reference no-default-lib="true"/>
/// <reference types="@cloudflare/workers-types" />
import {
createSentry,
handleUserAssetGet,
handleUserAssetUpload,
notFound,
} from '@tldraw/worker-shared'
import { WorkerEntrypoint } from 'cloudflare:workers'
import { createCors } from 'itty-cors'
import { Router } from 'itty-router'
import { Environment } from './types'
const { preflight, corsify } = createCors({ origins: ['*'] })
interface Env {
UPLOADS: R2Bucket
}
function parseRange(
encoded: string | null
): undefined | { offset: number; end: number; length: number } {
if (encoded === null) {
return
}
const parts = (encoded.split('bytes=')[1]?.split('-') ?? []).filter(Boolean)
if (parts.length !== 2) {
console.error('Not supported to skip specifying the beginning/ending byte at this time')
return
}
return {
offset: Number(parts[0]),
end: Number(parts[1]),
length: Number(parts[1]) + 1 - Number(parts[0]),
}
}
function objectNotFound(objectName: string): Response {
return new Response(`<html><body>R2 object "<b>${objectName}</b>" not found</body></html>`, {
status: 404,
headers: {
'content-type': 'text/html; charset=UTF-8',
},
})
}
const CACHE_CONTROL_SETTING = 's-maxage=604800'
const router = Router()
router
.all('*', preflight)
.get('/uploads/:objectName', async (request: Request, env: Env, ctx: ExecutionContext) => {
const url = new URL(request.url)
const range = parseRange(request.headers.get('range'))
// NOTE: caching will only work when this is deployed to
// a custom domain, not a workers.dev domain. It's a no-op
// otherwise.
// Construct the cache key from the cache URL
const cacheKey = new Request(url.toString(), request)
const cache = caches.default as Cache
// Check whether the value is already available in the cache
// if not, you will need to fetch it from R2, and store it in the cache
// for future access
let cachedResponse
if (!range) {
cachedResponse = await cache.match(cacheKey)
if (cachedResponse) {
return cachedResponse
}
}
const ifNoneMatch = request.headers.get('if-none-match')
let hs = request.headers
if (ifNoneMatch?.startsWith('W/')) {
hs = new Headers(request.headers)
hs.set('if-none-match', ifNoneMatch.slice(2))
}
// TODO: infer types from path
// @ts-expect-error
const object = await env.UPLOADS.get(request.params.objectName, {
range,
onlyIf: hs,
})
if (object === null) {
// TODO: infer types from path
// @ts-expect-error
return objectNotFound(request.params.objectName)
}
const headers = new Headers()
object.writeHttpMetadata(headers)
headers.set('etag', object.httpEtag)
if (range) {
headers.set('content-range', `bytes ${range.offset}-${range.end}/${object.size}`)
}
// Cache API respects Cache-Control headers. Setting s-max-age to 7 days
// Any changes made to the response here will be reflected in the cached value
headers.append('Cache-Control', CACHE_CONTROL_SETTING)
const hasBody = 'body' in object && object.body
const status = hasBody ? (range ? 206 : 200) : 304
const response = new Response(hasBody ? object.body : undefined, {
headers,
status,
})
// Store the response in the cache for future access
if (!range) {
const clonedResponse = response.clone()
// If the request was made with no-cache, we should not cache that in the headers.
clonedResponse?.headers.set('Cache-Control', CACHE_CONTROL_SETTING)
ctx.waitUntil(cache.put(cacheKey, clonedResponse))
}
return response
})
.head('/uploads/:objectName', async (request: Request, env: Env) => {
// TODO: infer types from path
// @ts-expect-error
const object = await env.UPLOADS.head(request.params.objectName)
if (object === null) {
// TODO: infer types from path
// @ts-expect-error
return objectNotFound(request.params.objectName)
}
const headers = new Headers()
object.writeHttpMetadata(headers)
headers.set('etag', object.httpEtag)
return new Response(null, {
headers,
})
})
.post('/uploads/:objectName', async (request: Request, env: Env) => {
// TODO: infer types from path
// @ts-expect-error
const object = await env.UPLOADS.put(request.params.objectName, request.body, {
httpMetadata: request.headers,
})
return new Response(null, {
headers: {
etag: object.httpEtag,
},
})
})
.get('*', () => new Response('Not found', { status: 404 }))
const Worker = {
async fetch(request: Request, env: Env, ctx: ExecutionContext) {
return router
.handle(request, env, ctx)
.catch((err) => {
// eslint-disable-next-line no-console
console.log(err, err.stack)
return new Response((err as Error).message, { status: 500 })
export default class Worker extends WorkerEntrypoint<Environment> {
readonly router = Router()
.all('*', preflight)
.get('/uploads/:objectName', async (request) => {
return handleUserAssetGet({
request,
bucket: this.env.UPLOADS,
objectName: request.params.objectName,
context: this.ctx,
})
.then(corsify)
},
}
})
.post('/uploads/:objectName', async (request) => {
return handleUserAssetUpload({
request,
bucket: this.env.UPLOADS,
objectName: request.params.objectName,
context: this.ctx,
})
})
.all('*', notFound)
export default Worker
override async fetch(request: Request) {
try {
return await this.router.handle(request, this.env, this.ctx).then(corsify)
} catch (error) {
const sentry = createSentry(this.ctx, this.env, request)
console.error(error)
// eslint-disable-next-line deprecation/deprecation
sentry?.captureException(error)
return new Response('Something went wrong', {
status: 500,
statusText: 'Internal Server Error',
})
}
}
}

View file

@ -5,5 +5,10 @@
"compilerOptions": {
"noEmit": true,
"emitDeclarationOnly": false
}
},
"references": [
{
"path": "../../packages/worker-shared"
}
]
}

View file

@ -63,3 +63,19 @@ binding = "MEASURE"
pattern = 'assets.tldraw.xyz'
custom_domain = true
zone_name = 'tldraw.xyz'
#################### Version metadata ####################
[version_metadata]
binding = "CF_VERSION_METADATA"
[env.dev.version_metadata]
binding = "CF_VERSION_METADATA"
[env.preview.version_metadata]
binding = "CF_VERSION_METADATA"
[env.staging.version_metadata]
binding = "CF_VERSION_METADATA"
[env.production.version_metadata]
binding = "CF_VERSION_METADATA"

View file

@ -1,14 +1,14 @@
import { ROOM_PREFIX } from '@tldraw/dotcom-shared'
import { notFound } from '@tldraw/worker-shared'
import { IRequest } from 'itty-router'
import { Environment } from '../types'
import { fourOhFour } from '../utils/fourOhFour'
import { isRoomIdTooLong, roomIdIsTooLong } from '../utils/roomIdIsTooLong'
// Forwards a room request to the durable object associated with that room
export async function forwardRoomRequest(request: IRequest, env: Environment): Promise<Response> {
const roomId = request.params.roomId
if (!roomId) return fourOhFour()
if (!roomId) return notFound()
if (isRoomIdTooLong(roomId)) return roomIdIsTooLong()
// Set up the durable object for this room

View file

@ -1,14 +1,14 @@
import { notFound } from '@tldraw/worker-shared'
import { IRequest } from 'itty-router'
import { getR2KeyForRoom } from '../r2'
import { Environment } from '../types'
import { fourOhFour } from '../utils/fourOhFour'
import { isRoomIdTooLong, roomIdIsTooLong } from '../utils/roomIdIsTooLong'
// Returns the history of a room as a list of objects with timestamps
export async function getRoomHistory(request: IRequest, env: Environment): Promise<Response> {
const roomId = request.params.roomId
if (!roomId) return fourOhFour()
if (!roomId) return notFound()
if (isRoomIdTooLong(roomId)) return roomIdIsTooLong()
const versionCacheBucket = env.ROOMS_HISTORY_EPHEMERAL

View file

@ -1,7 +1,7 @@
import { notFound } from '@tldraw/worker-shared'
import { IRequest } from 'itty-router'
import { getR2KeyForRoom } from '../r2'
import { Environment } from '../types'
import { fourOhFour } from '../utils/fourOhFour'
import { isRoomIdTooLong, roomIdIsTooLong } from '../utils/roomIdIsTooLong'
// Get a snapshot of the room at a given point in time
@ -11,7 +11,7 @@ export async function getRoomHistorySnapshot(
): Promise<Response> {
const roomId = request.params.roomId
if (!roomId) return fourOhFour()
if (!roomId) return notFound()
if (isRoomIdTooLong(roomId)) return roomIdIsTooLong()
const timestamp = request.params.timestamp

View file

@ -1,9 +1,9 @@
import { RoomSnapshot } from '@tldraw/tlsync'
import { notFound } from '@tldraw/worker-shared'
import { IRequest } from 'itty-router'
import { getR2KeyForSnapshot } from '../r2'
import { Environment } from '../types'
import { createSupabaseClient, noSupabaseSorry } from '../utils/createSupabaseClient'
import { fourOhFour } from '../utils/fourOhFour'
import { getSnapshotsTable } from '../utils/getSnapshotsTable'
import { R2Snapshot } from './createRoomSnapshot'
@ -24,7 +24,7 @@ function generateReponse(roomId: string, data: RoomSnapshot) {
// Returns a snapshot of the room at a given point in time
export async function getRoomSnapshot(request: IRequest, env: Environment): Promise<Response> {
const roomId = request.params.roomId
if (!roomId) return fourOhFour()
if (!roomId) return notFound()
// Get the parent slug if it exists
const parentSlug = await env.SNAPSHOT_SLUG_TO_PARENT_SLUG.get(roomId)
@ -53,7 +53,7 @@ export async function getRoomSnapshot(request: IRequest, env: Environment): Prom
.maybeSingle()
const data = result.data?.drawing as RoomSnapshot
if (!data) return fourOhFour()
if (!data) return notFound()
// Send back the snapshot!
return generateReponse(roomId, data)

View file

@ -1,7 +1,7 @@
import { ROOM_PREFIX, RoomOpenMode } from '@tldraw/dotcom-shared'
import { notFound } from '@tldraw/worker-shared'
import { IRequest } from 'itty-router'
import { Environment } from '../types'
import { fourOhFour } from '../utils/fourOhFour'
import { isRoomIdTooLong, roomIdIsTooLong } from '../utils/roomIdIsTooLong'
import { getSlug } from '../utils/roomOpenMode'
@ -11,7 +11,7 @@ export async function joinExistingRoom(
roomOpenMode: RoomOpenMode
): Promise<Response> {
const roomId = await getSlug(env, request.params.roomId, roomOpenMode)
if (!roomId) return fourOhFour()
if (!roomId) return notFound()
if (isRoomIdTooLong(roomId)) return roomIdIsTooLong()
// This needs to be a websocket request!
@ -21,5 +21,5 @@ export async function joinExistingRoom(
return env.TLDR_DOC.get(id).fetch(request)
}
return fourOhFour()
return notFound()
}

View file

@ -1,5 +0,0 @@
export async function fourOhFour() {
return new Response('Not found', {
status: 404,
})
}

View file

@ -7,7 +7,7 @@ import {
ROOM_PREFIX,
} from '@tldraw/dotcom-shared'
import { T } from '@tldraw/validate'
import { createSentry } from '@tldraw/worker-shared'
import { createSentry, notFound } from '@tldraw/worker-shared'
import { Router, createCors, json } from 'itty-router'
import { createRoom } from './routes/createRoom'
import { createRoomSnapshot } from './routes/createRoomSnapshot'
@ -18,7 +18,6 @@ import { getRoomHistorySnapshot } from './routes/getRoomHistorySnapshot'
import { getRoomSnapshot } from './routes/getRoomSnapshot'
import { joinExistingRoom } from './routes/joinExistingRoom'
import { Environment } from './types'
import { fourOhFour } from './utils/fourOhFour'
import { unfurl } from './utils/unfurl'
export { TLDrawDurableObject } from './TLDrawDurableObject'
@ -51,7 +50,7 @@ const router = Router()
return json(await unfurl(req.query.url))
})
.post(`/${ROOM_PREFIX}/:roomId/restore`, forwardRoomRequest)
.all('*', fourOhFour)
.all('*', notFound)
const Worker = {
fetch(request: Request, env: Environment, context: ExecutionContext) {

View file

@ -0,0 +1,3 @@
export function notFound() {
return Response.json({ error: 'Not found' }, { status: 404 })
}

View file

@ -1,4 +1,6 @@
/// <reference no-default-lib="true"/>
/// <reference types="@cloudflare/workers-types" />
export { notFound } from './errors'
export { createSentry } from './sentry'
export { handleUserAssetGet, handleUserAssetUpload } from './userAssetUploads'

View file

@ -0,0 +1,75 @@
import { ExecutionContext, R2Bucket } from '@cloudflare/workers-types'
import { notFound } from './errors'
interface UserAssetOpts {
request: Request
bucket: R2Bucket
objectName: string
context: ExecutionContext
}
export async function handleUserAssetUpload({
request,
bucket,
objectName,
}: UserAssetOpts): Promise<Response> {
if (await bucket.head(objectName)) {
return Response.json({ error: 'Asset already exists' }, { status: 409 })
}
const object = await bucket.put(objectName, request.body, {
httpMetadata: request.headers,
})
return Response.json({ object: objectName }, { headers: { etag: object.httpEtag } })
}
export async function handleUserAssetGet({ request, bucket, objectName, context }: UserAssetOpts) {
const cacheUrl = new URL(request.url)
const cacheKey = new Request(cacheUrl.toString(), request)
// this cache automatically handles range responses etc.
const cachedResponse = await caches.default.match(cacheKey)
if (cachedResponse) {
return cachedResponse
}
const object = await bucket.get(objectName, {
range: request.headers,
onlyIf: request.headers,
})
if (!object) {
return notFound()
}
const headers = new Headers()
object.writeHttpMetadata(headers)
headers.set('etag', object.httpEtag)
if (object.range) {
let start
let end
if ('suffix' in object.range) {
start = object.size - object.range.suffix
end = object.size - 1
} else {
start = object.range.offset ?? 0
end = object.range.length ? start + object.range.length - 1 : object.size - 1
}
headers.set('content-range', `bytes ${start}-${end}/${object.size}`)
}
// assets are immutable, so we can cache them basically forever:
headers.set('cache-control', 'public, max-age=31536000, immutable')
const body = 'body' in object && object.body ? object.body : null
const status = body ? (request.headers.get('range') !== null ? 206 : 200) : 304
if (status === 200) {
const [cacheBody, responseBody] = body!.tee()
// cache the response
context.waitUntil(caches.default.put(cacheKey, new Response(cacheBody, { headers, status })))
return new Response(responseBody, { headers, status })
}
return new Response(body, { headers, status })
}

View file

@ -30,6 +30,7 @@ const dotcom = path.relative(process.cwd(), path.resolve(__dirname, '../apps/dot
const env = makeEnv([
'APP_ORIGIN',
'ASSET_UPLOAD',
'ASSET_UPLOAD_SENTRY_DSN',
'ASSET_BUCKET_ORIGIN',
'CLOUDFLARE_ACCOUNT_ID',
'CLOUDFLARE_API_TOKEN',
@ -164,8 +165,9 @@ async function prepareDotcomApp() {
let didUpdateAssetUploadWorker = false
async function deployAssetUploadWorker({ dryRun }: { dryRun: boolean }) {
const workerId = `${previewId ?? env.TLDRAW_ENV}-tldraw-assets`
if (previewId && !didUpdateAssetUploadWorker) {
await setWranglerPreviewConfig(assetUpload, { name: `${previewId}-tldraw-assets` })
await setWranglerPreviewConfig(assetUpload, { name: workerId })
didUpdateAssetUploadWorker = true
}
@ -173,7 +175,15 @@ async function deployAssetUploadWorker({ dryRun }: { dryRun: boolean }) {
location: assetUpload,
dryRun,
env: env.TLDRAW_ENV,
vars: {},
vars: {
SENTRY_DSN: env.ASSET_UPLOAD_SENTRY_DSN,
TLDRAW_ENV: env.TLDRAW_ENV,
WORKER_NAME: workerId,
},
sentry: {
project: 'asset-upload-worker',
authToken: env.SENTRY_AUTH_TOKEN,
},
})
}

View file

@ -10312,6 +10312,7 @@ __metadata:
resolution: "dotcom-asset-upload@workspace:apps/dotcom-asset-upload"
dependencies:
"@cloudflare/workers-types": "npm:^4.20240620.0"
"@tldraw/worker-shared": "workspace:*"
"@types/ws": "npm:^8.5.9"
itty-cors: "npm:^0.3.4"
itty-router: "npm:^4.0.13"