assets: store in indexedDB, not as base64 (#3836)

this is take #2 of this PR https://github.com/tldraw/tldraw/pull/3745

As I look at LOD holistically and whether we have multiple sources when
working locally, I learned that our system used base64 encoding of
assets directly. Issue https://github.com/tldraw/tldraw/issues/3728

<img width="1350" alt="assetstore"
src="https://github.com/tldraw/tldraw/assets/469604/e7b41e29-6656-4d9b-b462-72d43b98f3f7">


The motivations and benefits are:
- store size: not having a huge base64 blobs injected in room data
- perf on loading snapshot: this helps with loading the room data more
quickly
- multiple sources: furthermore, if we do decide to have multiple
sources locally (for each asset), then we won't get a multiplicative
effect of even larger JSON blobs that have lots of base64 data in them
- encoding/decoding perf: this also saves the (slow) step of having to
base64 encode/decode our assets, we can just strictly with work with
blobs.


Todo:
- [x] decodes video and images
- [x] make sure it syncs to other tabs
- [x] make sure it syncs to other multiplayer room
- [x] fix tests


### Change Type

<!-- ❗ Please select a 'Scope' label ❗️ -->

- [x] `sdk` — Changes the tldraw SDK
- [ ] `dotcom` — Changes the tldraw.com web app
- [ ] `docs` — Changes to the documentation, examples, or templates.
- [ ] `vs code` — Changes to the vscode plugin
- [ ] `internal` — Does not affect user-facing stuff

<!-- ❗ Please select a 'Type' label ❗️ -->

- [ ] `bugfix` — Bug fix
- [ ] `feature` — New feature
- [x] `improvement` — Improving existing features
- [ ] `chore` — Updating dependencies, other boring stuff
- [ ] `galaxy brain` — Architectural changes
- [ ] `tests` — Changes to any test code
- [ ] `tools` — Changes to infrastructure, CI, internal scripts,
debugging tools, etc.
- [ ] `dunno` — I don't know


### Test Plan

1. Test the shit out of uploading/downloading video/image assets,
locally+multiplayer.

- [ ] Need to fix current tests and write new ones

### Release Notes

- Assets: store as reference to blob in indexedDB instead of storing
directly as base64 in the snapshot.

apps/dotcom/src/components/LocalEditor.tsx

@@ -88,7 +88,7 @@ const components: TLComponents = {
 
 export function LocalEditor() {
 	const handleUiEvent = useHandleUiEvents()
-	const sharingUiOverrides = useSharing()
+	const sharingUiOverrides = useSharing(SCRATCH_PERSISTENCE_KEY)
 	const fileSystemUiOverrides = useFileSystem({ isMultiplayer: false })
 
 	const handleMount = useCallback((editor: Editor) => {
@@ -106,7 +106,7 @@ export function LocalEditor() {
 				overrides={[sharingUiOverrides, fileSystemUiOverrides]}
 				onUiEvent={handleUiEvent}
 				components={components}
-				assetOptions={{ onResolveAsset: resolveAsset }}
+				assetOptions={{ onResolveAsset: resolveAsset(SCRATCH_PERSISTENCE_KEY) }}
 				inferDarkMode
 			>
 				<LocalMigration />

apps/dotcom/src/components/MultiplayerEditor.tsx

@@ -159,7 +159,7 @@ export function MultiplayerEditor({
 				initialState={isReadonly ? 'hand' : 'select'}
 				onUiEvent={handleUiEvent}
 				components={components}
-				assetOptions={{ onResolveAsset: resolveAsset }}
+				assetOptions={{ onResolveAsset: resolveAsset() }}
 				inferDarkMode
 			>
 				<UrlStateSync />

apps/dotcom/src/utils/assetHandler.test.ts

@@ -1,12 +1,14 @@
 import { TLAsset } from 'tldraw'
 import { resolveAsset } from './assetHandler'
 
+const PERSISTENCE_KEY = 'tldraw'
+const resolver = resolveAsset(PERSISTENCE_KEY)
 const FILE_SIZE = 1024 * 1024 * 2
 
 describe('resolveAsset', () => {
 	it('should return null if the asset is null', async () => {
 		expect(
-			await resolveAsset(null, {
+			await resolver(null, {
 				screenScale: -1,
 				steppedScreenScale: 1,
 				dpr: 1,
@@ -17,7 +19,7 @@ describe('resolveAsset', () => {
 
 	it('should return null if the asset is undefined', async () => {
 		expect(
-			await resolveAsset(undefined, {
+			await resolver(undefined, {
 				screenScale: -1,
 				steppedScreenScale: 1,
 				dpr: 1,
@@ -29,7 +31,7 @@ describe('resolveAsset', () => {
 	it('should return null if the asset has no src', async () => {
 		const asset = { type: 'image', props: { w: 100, fileSize: FILE_SIZE } }
 		expect(
-			await resolveAsset(asset as TLAsset, {
+			await resolver(asset as TLAsset, {
 				screenScale: -1,
 				steppedScreenScale: 1,
 				dpr: 1,
@@ -44,7 +46,7 @@ describe('resolveAsset', () => {
 			props: { src: 'http://example.com/video.mp4', fileSize: FILE_SIZE },
 		}
 		expect(
-			await resolveAsset(asset as TLAsset, {
+			await resolver(asset as TLAsset, {
 				screenScale: -1,
 				steppedScreenScale: 1,
 				dpr: 1,
@@ -53,10 +55,23 @@ describe('resolveAsset', () => {
 		).toBe('http://example.com/video.mp4')
 	})
 
+	it('should return the original src for if original is asked for', async () => {
+		const asset = { type: 'image', props: { src: 'http://example.com/image.jpg', w: 100 } }
+		expect(
+			await resolver(asset as TLAsset, {
+				screenScale: -1,
+				steppedScreenScale: 1,
+				dpr: 1,
+				networkEffectiveType: '4g',
+				shouldResolveToOriginalImage: true,
+			})
+		).toBe('http://example.com/image.jpg')
+	})
+
 	it('should return the original src if it does not start with http or https', async () => {
 		const asset = { type: 'image', props: { src: 'data:somedata', w: 100, fileSize: FILE_SIZE } }
 		expect(
-			await resolveAsset(asset as TLAsset, {
+			await resolver(asset as TLAsset, {
 				screenScale: -1,
 				steppedScreenScale: 1,
 				dpr: 1,
@@ -76,7 +91,7 @@ describe('resolveAsset', () => {
 			},
 		}
 		expect(
-			await resolveAsset(asset as TLAsset, {
+			await resolver(asset as TLAsset, {
 				screenScale: -1,
 				steppedScreenScale: 1,
 				dpr: 1,
@@ -91,7 +106,7 @@ describe('resolveAsset', () => {
 			props: { src: 'http://example.com/small.png', w: 100, fileSize: 1024 * 1024 },
 		}
 		expect(
-			await resolveAsset(asset as TLAsset, {
+			await resolver(asset as TLAsset, {
 				screenScale: -1,
 				steppedScreenScale: 1,
 				dpr: 1,
@@ -106,7 +121,7 @@ describe('resolveAsset', () => {
 			props: { src: 'http://example.com/doc.pdf', w: 100, fileSize: FILE_SIZE },
 		}
 		expect(
-			await resolveAsset(asset as TLAsset, {
+			await resolver(asset as TLAsset, {
 				screenScale: -1,
 				steppedScreenScale: 1,
 				dpr: 1,
@@ -121,7 +136,7 @@ describe('resolveAsset', () => {
 			props: { src: 'http://example.com/image.jpg', w: 100, fileSize: FILE_SIZE },
 		}
 		expect(
-			await resolveAsset(asset as TLAsset, {
+			await resolver(asset as TLAsset, {
 				screenScale: -1,
 				steppedScreenScale: 0.5,
 				dpr: 2,
@@ -138,7 +153,7 @@ describe('resolveAsset', () => {
 			props: { src: 'http://example.com/image.jpg', w: 100, fileSize: FILE_SIZE },
 		}
 		expect(
-			await resolveAsset(asset as TLAsset, {
+			await resolver(asset as TLAsset, {
 				screenScale: -1,
 				steppedScreenScale: 0.5,
 				dpr: 2,
@@ -155,7 +170,7 @@ describe('resolveAsset', () => {
 			props: { src: 'https://example.com/image.jpg', w: 100, fileSize: FILE_SIZE },
 		}
 		expect(
-			await resolveAsset(asset as TLAsset, {
+			await resolver(asset as TLAsset, {
 				screenScale: -1,
 				steppedScreenScale: 4,
 				dpr: 1,
@@ -172,7 +187,7 @@ describe('resolveAsset', () => {
 			props: { src: 'https://example.com/image.jpg', w: 100, fileSize: FILE_SIZE },
 		}
 		expect(
-			await resolveAsset(asset as TLAsset, {
+			await resolver(asset as TLAsset, {
 				screenScale: -1,
 				steppedScreenScale: 0.5,
 				dpr: 1,
@@ -189,7 +204,7 @@ describe('resolveAsset', () => {
 			props: { src: 'https://example.com/image.jpg', w: 100, fileSize: FILE_SIZE },
 		}
 		expect(
-			await resolveAsset(asset as TLAsset, {
+			await resolver(asset as TLAsset, {
 				screenScale: -1,
 				steppedScreenScale: 0.25,
 				dpr: 1,

apps/dotcom/src/utils/assetHandler.ts

@@ -1,40 +1,73 @@
-import { AssetContextProps, MediaHelpers, TLAsset } from 'tldraw'
+import {
+	AssetContextProps,
+	MediaHelpers,
+	TLAsset,
+	TLAssetId,
+	WeakCache,
+	getAssetFromIndexedDb,
+} from 'tldraw'
 import { ASSET_BUCKET_ORIGIN, ASSET_UPLOADER_URL } from './config'
 
-export async function resolveAsset(asset: TLAsset | null | undefined, context: AssetContextProps) {
-	if (!asset || !asset.props.src) return null
+const objectURLCache = new WeakCache<TLAsset, ReturnType<typeof getLocalAssetObjectURL>>()
 
-	// We don't deal with videos at the moment.
-	if (asset.type === 'video') return asset.props.src
+export const resolveAsset =
+	(persistenceKey?: string) =>
+	async (asset: TLAsset | null | undefined, context: AssetContextProps) => {
+		if (!asset || !asset.props.src) return null
 
-	// Assert it's an image to make TS happy.
-	if (asset.type !== 'image') return null
+		// We don't deal with videos at the moment.
+		if (asset.type === 'video') return asset.props.src
 
-	// Don't try to transform data: URLs, yikes.
-	if (!asset.props.src.startsWith('http:') && !asset.props.src.startsWith('https:'))
-		return asset.props.src
+		// Assert it's an image to make TS happy.
+		if (asset.type !== 'image') return null
 
-	// Don't try to transform animated images.
-	if (MediaHelpers.isAnimatedImageType(asset?.props.mimeType) || asset.props.isAnimated)
-		return asset.props.src
+		// Retrieve a local image from the DB.
+		if (persistenceKey && asset.props.src.startsWith('asset:')) {
+			return await objectURLCache.get(
+				asset,
+				async () => await getLocalAssetObjectURL(persistenceKey, asset.id)
+			)
+		}
 
-	// Assets that are under a certain file size aren't worth transforming (and incurring cost).
-	if (asset.props.fileSize === -1 || asset.props.fileSize < 1024 * 1024 * 1.5 /* 1.5 MB */)
-		return asset.props.src
+		// Don't try to transform data: URLs, yikes.
+		if (!asset.props.src.startsWith('http:') && !asset.props.src.startsWith('https:'))
+			return asset.props.src
 
-	// N.B. navigator.connection is only available in certain browsers (mainly Blink-based browsers)
-	// 4g is as high the 'effectiveType' goes and we can pick a lower effective image quality for slower connections.
-	const networkCompensation =
-		!context.networkEffectiveType || context.networkEffectiveType === '4g' ? 1 : 0.5
+		if (context.shouldResolveToOriginalImage) {
+			return asset.props.src
+		}
 
-	const width = Math.ceil(asset.props.w * context.steppedScreenScale * networkCompensation)
+		// Don't try to transform animated images.
+		if (MediaHelpers.isAnimatedImageType(asset?.props.mimeType) || asset.props.isAnimated)
+			return asset.props.src
 
-	if (process.env.NODE_ENV === 'development') {
-		return asset.props.src
+		// Assets that are under a certain file size aren't worth transforming (and incurring cost).
+		if (asset.props.fileSize === -1 || asset.props.fileSize < 1024 * 1024 * 1.5 /* 1.5 MB */)
+			return asset.props.src
+
+		// N.B. navigator.connection is only available in certain browsers (mainly Blink-based browsers)
+		// 4g is as high the 'effectiveType' goes and we can pick a lower effective image quality for slower connections.
+		const networkCompensation =
+			!context.networkEffectiveType || context.networkEffectiveType === '4g' ? 1 : 0.5
+
+		const width = Math.ceil(asset.props.w * context.steppedScreenScale * networkCompensation)
+
+		if (process.env.NODE_ENV === 'development') {
+			return asset.props.src
+		}
+
+		// On preview, builds the origin for the asset won't be the right one for the Cloudflare transform.
+		const src = asset.props.src.replace(ASSET_UPLOADER_URL, ASSET_BUCKET_ORIGIN)
+		return `${ASSET_BUCKET_ORIGIN}/cdn-cgi/image/format=auto,width=${width},dpr=${context.dpr},fit=scale-down,quality=92/${src}`
 	}
 
-	// On preview, builds the origin for the asset won't be the right one for the Cloudflare transform.
-	const src = asset.props.src.replace(ASSET_UPLOADER_URL, ASSET_BUCKET_ORIGIN)
-
-	return `${ASSET_BUCKET_ORIGIN}/cdn-cgi/image/format=auto,width=${width},dpr=${context.dpr},fit=scale-down,quality=92/${src}`
+async function getLocalAssetObjectURL(persistenceKey: string, assetId: TLAssetId) {
+	const blob = await getAssetFromIndexedDb({
+		assetId: assetId,
+		persistenceKey,
+	})
+	if (blob) {
+		return URL.createObjectURL(blob)
+	}
+	return null
 }

apps/dotcom/src/utils/cloneAssetForShare.ts

@@ -1,18 +1,32 @@
-import { TLAsset, fetch } from 'tldraw'
+import { TLAsset, fetch, getAssetFromIndexedDb } from 'tldraw'
 
 export async function cloneAssetForShare(
 	asset: TLAsset,
-	uploadFileToAsset: (file: File) => Promise<TLAsset>
+	uploadFileToAsset: (file: File) => Promise<TLAsset>,
+	persistenceKey: string
 ): Promise<TLAsset> {
 	if (asset.type === 'bookmark') return asset
-	if (asset.props.src) {
-		const dataUrlMatch = asset.props.src.match(/data:(.*?)(;base64)?,/)
-		if (!dataUrlMatch) return asset
 
-		const response = await fetch(asset.props.src)
-		const file = new File([await response.blob()], asset.props.name, {
-			type: dataUrlMatch[1] ?? asset.props.mimeType,
-		})
+	if (asset.props.src) {
+		let file: File | undefined
+		if (asset.props.src.startsWith('asset:')) {
+			const blob = await getAssetFromIndexedDb({ assetId: asset.id, persistenceKey })
+			if (blob) {
+				file = new File([blob], asset.props.name, {
+					type: asset.props.mimeType || '',
+				})
+			} else {
+				return asset
+			}
+		} else {
+			const dataUrlMatch = asset.props.src.match(/data:(.*?)(;base64)?,/)
+			if (!dataUrlMatch) return asset
+
+			const response = await fetch(asset.props.src)
+			file = new File([await response.blob()], asset.props.name, {
+				type: dataUrlMatch[1] ?? asset.props.mimeType,
+			})
+		}
 
 		const uploadedAsset = await uploadFileToAsset(file)
 

apps/dotcom/src/utils/sharing.ts

@@ -47,10 +47,11 @@ async function getSnapshotLink(
 	addToast: TLUiToastsContextType['addToast'],
 	msg: (id: TLUiTranslationKey) => string,
 	uploadFileToAsset: (file: File) => Promise<TLAsset>,
-	parentSlug: string | undefined
+	parentSlug: string | undefined,
+	persistenceKey: string
 ) {
 	handleUiEvent('share-snapshot' as UI_OVERRIDE_TODO_EVENT, { source } as UI_OVERRIDE_TODO_EVENT)
-	const data = await getRoomData(editor, addToast, msg, uploadFileToAsset)
+	const data = await getRoomData(editor, addToast, msg, uploadFileToAsset, persistenceKey)
 	if (!data) return ''
 
 	const res = await fetch(CREATE_SNAPSHOT_ENDPOINT, {
@@ -95,7 +96,7 @@ export async function getNewRoomResponse(snapshot: Snapshot) {
 	})
 }
 
-export function useSharing(): TLUiOverrides {
+export function useSharing(persistenceKey?: string): TLUiOverrides {
 	const navigate = useNavigate()
 	const params = useParams()
 	const roomId = params.roomId
@@ -132,7 +133,13 @@ export function useSharing(): TLUiOverrides {
 							})
 
 							handleUiEvent('share-project', { source })
-							const data = await getRoomData(editor, addToast, msg, uploadFileToAsset)
+							const data = await getRoomData(
+								editor,
+								addToast,
+								msg,
+								uploadFileToAsset,
+								persistenceKey || ''
+							)
 							if (!data) return
 
 							const res = await getNewRoomResponse({
@@ -182,7 +189,8 @@ export function useSharing(): TLUiOverrides {
 							addToast,
 							msg,
 							uploadFileToAsset,
-							roomId
+							roomId,
+							persistenceKey || ''
 						)
 						if (navigator?.clipboard?.write) {
 							await navigator.clipboard.write([
@@ -209,7 +217,7 @@ export function useSharing(): TLUiOverrides {
 				return actions
 			},
 		}),
-		[handleUiEvent, navigate, uploadFileToAsset, roomId, runningInIFrame]
+		[handleUiEvent, navigate, uploadFileToAsset, roomId, runningInIFrame, persistenceKey]
 	)
 }
 
@@ -217,7 +225,8 @@ async function getRoomData(
 	editor: Editor,
 	addToast: TLUiToastsContextType['addToast'],
 	msg: (id: TLUiTranslationKey) => string,
-	uploadFileToAsset: (file: File) => Promise<TLAsset>
+	uploadFileToAsset: (file: File) => Promise<TLAsset>,
+	persistenceKey: string
 ) {
 	const rawData = editor.store.serialize()
 
@@ -253,7 +262,7 @@ async function getRoomData(
 			// processed it
 			if (!asset) continue
 
-			data[asset.id] = await cloneAssetForShare(asset, uploadFileToAsset)
+			data[asset.id] = await cloneAssetForShare(asset, uploadFileToAsset, persistenceKey)
 			// remove the asset after processing so we don't clone it multiple times
 			assets.delete(asset.id)
 		}

apps/dotcom/src/utils/useFileSystem.tsx

@@ -127,7 +127,7 @@ export function getSaveFileCopyAction(
 			const defaultName =
 				saveFileNames.get(editor.store) || `${documentName}${TLDRAW_FILE_EXTENSION}`
 
-			const blobToSave = serializeTldrawJsonBlob(editor.store)
+			const blobToSave = serializeTldrawJsonBlob(editor)
 			let handle
 			try {
 				handle = await fileSave(blobToSave, {

apps/vscode/editor/src/ChangeResponder.tsx

@@ -48,7 +48,7 @@ export const ChangeResponder = () => {
 			vscode.postMessage({
 				type: 'vscode:editor-updated',
 				data: {
-					fileContents: await serializeTldrawJson(editor.store),
+					fileContents: await serializeTldrawJson(editor),
 				},
 			})
 		}, 250)

packages/editor/api-report.md

@@ -153,6 +153,8 @@ export interface AssetContextProps {
     // (undocumented)
     screenScale: number;
     // (undocumented)
+    shouldResolveToOriginalImage?: boolean;
+    // (undocumented)
     steppedScreenScale: number;
 }
 
@@ -1062,6 +1064,8 @@ export class Editor extends EventEmitter<TLEventMap> {
         select: boolean;
     }>): this;
     hasAncestor(shape: TLShape | TLShapeId | undefined, ancestorId: TLShapeId): boolean;
+    // (undocumented)
+    hasExternalAssetHandler(type: TLExternalAssetContent['type']): boolean;
     readonly history: HistoryManager<TLRecord>;
     inputs: {
         buttons: Set<number>;
@@ -1127,8 +1131,11 @@ export class Editor extends EventEmitter<TLEventMap> {
     resetZoom(point?: Vec, opts?: TLCameraMoveOptions): this;
     resizeShape(shape: TLShape | TLShapeId, scale: VecLike, options?: TLResizeShapeOptions): this;
     // (undocumented)
+    resolveAssetsInContent(content: TLContent | undefined): Promise<TLContent | undefined>;
+    // (undocumented)
     resolveAssetUrl(assetId: null | TLAssetId, context: {
-        screenScale: number;
+        screenScale?: number;
+        shouldResolveToOriginalImage?: boolean;
     }): Promise<null | string>;
     readonly root: StateNode;
     rotateShapesBy(shapes: TLShape[] | TLShapeId[], delta: number): this;

packages/editor/src/lib/editor/Editor.ts

@@ -53,6 +53,7 @@ import {
 	isShapeId,
 } from '@tldraw/tlschema'
 import {
+	FileHelpers,
 	IndexKey,
 	JsonObject,
 	PerformanceTracker,
@@ -64,6 +65,7 @@ import {
 	compact,
 	dedupe,
 	exhaustiveSwitchError,
+	fetch,
 	getIndexAbove,
 	getIndexBetween,
 	getIndices,
@@ -3927,24 +3929,30 @@ export class Editor extends EventEmitter<TLEventMap> {
 
 	async resolveAssetUrl(
 		assetId: TLAssetId | null,
-		context: { screenScale: number }
+		context: {
+			screenScale?: number
+			shouldResolveToOriginalImage?: boolean
+		}
 	): Promise<string | null> {
 		if (!assetId) return ''
 		const asset = this.getAsset(assetId)
 		if (!asset) return ''
 
+		const { screenScale, shouldResolveToOriginalImage } = context
+
 		// We only look at the zoom level at powers of 2.
 		const zoomStepFunction = (zoom: number) => Math.pow(2, Math.ceil(Math.log2(zoom)))
-		const steppedScreenScale = Math.max(0.125, zoomStepFunction(context.screenScale))
+		const steppedScreenScale = Math.max(0.125, zoomStepFunction(screenScale || 1))
 		const networkEffectiveType: string | null =
 			'connection' in navigator ? (navigator as any).connection.effectiveType : null
 		const dpr = this.getInstanceState().devicePixelRatio
 
 		return await this._assetOptions.get().onResolveAsset(asset!, {
-			screenScale: context.screenScale,
+			screenScale: screenScale || 1,
 			steppedScreenScale,
 			dpr,
 			networkEffectiveType,
+			shouldResolveToOriginalImage,
 		})
 	}
 
@@ -7694,6 +7702,10 @@ export class Editor extends EventEmitter<TLEventMap> {
 		return await this.externalAssetContentHandlers[info.type]?.(info as any)
 	}
 
+	hasExternalAssetHandler(type: TLExternalAssetContent['type']): boolean {
+		return !!this.externalAssetContentHandlers[type]
+	}
+
 	/** @internal */
 	externalContentHandlers: {
 		[K in TLExternalContent['type']]: {
@@ -7823,6 +7835,39 @@ export class Editor extends EventEmitter<TLEventMap> {
 		})
 	}
 
+	async resolveAssetsInContent(content: TLContent | undefined): Promise<TLContent | undefined> {
+		if (!content) return undefined
+
+		const assets: TLAsset[] = []
+		await Promise.allSettled(
+			content.assets.map(async (asset) => {
+				if (
+					(asset.type === 'image' || asset.type === 'video') &&
+					!asset.props.src?.startsWith('data:image') &&
+					!asset.props.src?.startsWith('http')
+				) {
+					const assetWithDataUrl = structuredClone(asset as TLImageAsset | TLVideoAsset)
+					const objectUrl = await this._assetOptions.get().onResolveAsset(asset!, {
+						screenScale: 1,
+						steppedScreenScale: 1,
+						dpr: 1,
+						networkEffectiveType: null,
+						shouldResolveToOriginalImage: true,
+					})
+					assetWithDataUrl.props.src = await FileHelpers.blobToDataUrl(
+						await fetch(objectUrl!).then((r) => r.blob())
+					)
+					assets.push(assetWithDataUrl)
+				} else {
+					assets.push(asset)
+				}
+			})
+		)
+		content.assets = assets
+
+		return content
+	}
+
 	/**
 	 * Place content into the editor.
 	 *

packages/editor/src/lib/editor/types/misc-types.ts

@@ -61,6 +61,7 @@ export interface AssetContextProps {
 	steppedScreenScale: number
 	dpr: number
 	networkEffectiveType: string | null
+	shouldResolveToOriginalImage?: boolean
 }
 
 /** @public */

packages/tldraw/api-report.md

@@ -847,6 +847,12 @@ export function getArrowTerminalsInArrowSpace(editor: Editor, shape: TLArrowShap
     start: Vec;
 };
 
+// @public (undocumented)
+export function getAssetFromIndexedDb({ persistenceKey, assetId, }: {
+    assetId: string;
+    persistenceKey: string;
+}): Promise<Blob | undefined>;
+
 // @public
 export function getEmbedInfo(inputUrl: string): TLEmbedResult;
 
@@ -1321,10 +1327,10 @@ export class SelectTool extends StateNode {
 export function SelectToolbarItem(): JSX_2.Element;
 
 // @public (undocumented)
-export function serializeTldrawJson(store: TLStore): Promise<string>;
+export function serializeTldrawJson(editor: Editor): Promise<string>;
 
 // @public (undocumented)
-export function serializeTldrawJsonBlob(store: TLStore): Promise<Blob>;
+export function serializeTldrawJsonBlob(editor: Editor): Promise<Blob>;
 
 // @internal (undocumented)
 export function setDefaultEditorAssetUrls(assetUrls: TLEditorAssetUrls): void;
@@ -1344,6 +1350,13 @@ export function StackMenuItems(): JSX_2.Element;
 // @public (undocumented)
 export function StarToolbarItem(): JSX_2.Element;
 
+// @public (undocumented)
+export function storeAssetInIndexedDb({ persistenceKey, assetId, blob, }: {
+    assetId: string;
+    blob: Blob;
+    persistenceKey: string;
+}): Promise<void>;
+
 // @public (undocumented)
 export interface StylePickerSetProps {
     // (undocumented)
@@ -3281,8 +3294,8 @@ export function useLocalStorageState<T = any>(key: string, defaultValue: T): rea
 
 // @public (undocumented)
 export function useMenuClipboardEvents(): {
-    copy: (source: TLUiEventSource) => void;
-    cut: (source: TLUiEventSource) => void;
+    copy: (source: TLUiEventSource) => Promise<void>;
+    cut: (source: TLUiEventSource) => Promise<void>;
     paste: (data: ClipboardItem[] | DataTransfer, source: TLUiEventSource, point?: VecLike) => Promise<void>;
 };
 

packages/tldraw/package.json

@@ -58,6 +58,7 @@
 		"canvas-size": "^1.2.6",
 		"classnames": "^2.3.2",
 		"hotkeys-js": "^3.11.2",
+		"idb": "^7.1.1",
 		"lz-string": "^1.4.4"
 	},
 	"peerDependencies": {

packages/tldraw/src/index.ts

@@ -2,7 +2,7 @@
 
 // eslint-disable-next-line local/no-export-star
 export * from '@tldraw/editor'
-
+export { getAssetFromIndexedDb, storeAssetInIndexedDb } from './lib/AssetBlobStore'
 export { Tldraw, type TLComponents, type TldrawProps } from './lib/Tldraw'
 export { TldrawImage, type TldrawImageProps } from './lib/TldrawImage'
 export { TldrawHandles } from './lib/canvas/TldrawHandles'

packages/tldraw/src/lib/AssetBlobStore.ts

@@ -0,0 +1,62 @@
+import { IDBPDatabase, openDB } from 'idb'
+
+// DO NOT CHANGE THESE WITHOUT ADDING MIGRATION LOGIC. DOING SO WOULD WIPE ALL EXISTING DATA.
+const STORE_PREFIX = 'TLDRAW_ASSET_STORE_v1'
+
+const Table = {
+	Assets: 'assets',
+} as const
+
+type StoreName = (typeof Table)[keyof typeof Table]
+
+async function withDb<T>(storeId: string, cb: (db: IDBPDatabase<StoreName>) => Promise<T>) {
+	const db = await openDB<StoreName>(storeId, 1, {
+		upgrade(database) {
+			if (!database.objectStoreNames.contains(Table.Assets)) {
+				database.createObjectStore(Table.Assets)
+			}
+		},
+	})
+	try {
+		return await cb(db)
+	} finally {
+		db.close()
+	}
+}
+
+/** @public */
+export async function getAssetFromIndexedDb({
+	persistenceKey,
+	assetId,
+}: {
+	persistenceKey: string
+	assetId: string
+}): Promise<Blob | undefined> {
+	const storeId = STORE_PREFIX + persistenceKey
+
+	return await withDb(storeId, async (db) => {
+		const tx = db.transaction([Table.Assets], 'readwrite')
+		const assetsStore = tx.objectStore(Table.Assets)
+		return await assetsStore.get(assetId)
+	})
+}
+
+/** @public */
+export async function storeAssetInIndexedDb({
+	persistenceKey,
+	assetId,
+	blob,
+}: {
+	persistenceKey: string
+	assetId: string
+	blob: Blob
+}) {
+	const storeId = STORE_PREFIX + persistenceKey
+
+	await withDb(storeId, async (db) => {
+		const tx = db.transaction([Table.Assets], 'readwrite')
+		const assetsStore = tx.objectStore(Table.Assets)
+		await assetsStore.put(blob, assetId)
+		await tx.done
+	})
+}

packages/tldraw/src/lib/Tldraw.tsx

@@ -28,6 +28,7 @@ import { TldrawSelectionForeground } from './canvas/TldrawSelectionForeground'
 import { defaultBindingUtils } from './defaultBindingUtils'
 import {
 	TLExternalContentProps,
+	defaultResolveAsset,
 	registerDefaultExternalContentHandlers,
 } from './defaultExternalContentHandlers'
 import { defaultShapeTools } from './defaultShapeTools'
@@ -118,7 +119,12 @@ export function Tldraw(props: TldrawProps) {
 		[_tools]
 	)
 
+	const persistenceKey = 'persistenceKey' in rest ? rest.persistenceKey : undefined
 	const assets = useDefaultEditorAssetsWithOverrides(rest.assetUrls)
+	const assetOptions = useMemo(
+		() => ({ onResolveAsset: defaultResolveAsset(persistenceKey), ...rest.assetOptions }),
+		[persistenceKey, rest.assetOptions]
+	)
 	const { done: preloadingComplete, error: preloadingError } = usePreloadAssets(assets)
 	if (preloadingError) {
 		return <ErrorScreen>Could not load assets. Please refresh the page.</ErrorScreen>
@@ -135,6 +141,7 @@ export function Tldraw(props: TldrawProps) {
 			shapeUtils={shapeUtilsWithDefaults}
 			bindingUtils={bindingUtilsWithDefaults}
 			tools={toolsWithDefaults}
+			assetOptions={assetOptions}
 		>
 			<TldrawUi {...rest} components={componentsWithDefault}>
 				<InsideOfEditorAndUiContext
@@ -142,6 +149,7 @@ export function Tldraw(props: TldrawProps) {
 					maxAssetSize={maxAssetSize}
 					acceptedImageMimeTypes={acceptedImageMimeTypes}
 					acceptedVideoMimeTypes={acceptedVideoMimeTypes}
+					persistenceKey={persistenceKey}
 					onMount={onMount}
 				/>
 				{children}
@@ -157,7 +165,8 @@ function InsideOfEditorAndUiContext({
 	acceptedImageMimeTypes = DEFAULT_SUPPORTED_IMAGE_TYPES,
 	acceptedVideoMimeTypes = DEFAULT_SUPPORT_VIDEO_TYPES,
 	onMount,
-}: Partial<TLExternalContentProps & { onMount: TLOnMountHandler }>) {
+	persistenceKey,
+}: Partial<TLExternalContentProps & { onMount: TLOnMountHandler; persistenceKey?: string }>) {
 	const editor = useEditor()
 	const toasts = useToasts()
 	const msg = useTranslation()
@@ -179,7 +188,8 @@ function InsideOfEditorAndUiContext({
 			{
 				toasts,
 				msg,
-			}
+			},
+			persistenceKey
 		)
 
 		// ...then we run the onMount prop, which may override the above

packages/tldraw/src/lib/defaultExternalContentHandlers.ts

@@ -13,6 +13,7 @@ import {
 	TLTextShapeProps,
 	Vec,
 	VecLike,
+	WeakCache,
 	assert,
 	compact,
 	createShapeId,
@@ -20,6 +21,7 @@ import {
 	getHashForBuffer,
 	getHashForString,
 } from '@tldraw/editor'
+import { getAssetFromIndexedDb, storeAssetInIndexedDb } from './AssetBlobStore'
 import { FONT_FAMILIES, FONT_SIZES, TEXT_PROPS } from './shapes/shared/default-shape-constants'
 import { TLUiToastsContextType } from './ui/context/toasts'
 import { useTranslation } from './ui/hooks/useTranslation/useTranslation'
@@ -47,7 +49,8 @@ export function registerDefaultExternalContentHandlers(
 		acceptedImageMimeTypes,
 		acceptedVideoMimeTypes,
 	}: TLExternalContentProps,
-	{ toasts, msg }: { toasts: TLUiToastsContextType; msg: ReturnType<typeof useTranslation> }
+	{ toasts, msg }: { toasts: TLUiToastsContextType; msg: ReturnType<typeof useTranslation> },
+	persistenceKey?: string
 ) {
 	// files -> asset
 	editor.registerExternalAssetHandler('file', async ({ file: _file }) => {
@@ -83,23 +86,33 @@ export function registerDefaultExternalContentHandlers(
 		}
 
 		const assetId: TLAssetId = AssetRecordType.createId(hash)
-
-		const asset = AssetRecordType.create({
+		const assetInfo = {
 			id: assetId,
 			type: isImageType ? 'image' : 'video',
 			typeName: 'asset',
 			props: {
 				name,
-				src: await FileHelpers.blobToDataUrl(file),
+				src: '',
 				w: size.w,
 				h: size.h,
 				fileSize: file.size,
 				mimeType: file.type,
 				isAnimated,
 			},
-		})
+		} as TLAsset
 
-		return asset
+		if (persistenceKey) {
+			assetInfo.props.src = assetId
+			await storeAssetInIndexedDb({
+				persistenceKey,
+				assetId,
+				blob: file,
+			})
+		} else {
+			assetInfo.props.src = await FileHelpers.blobToDataUrl(file)
+		}
+
+		return AssetRecordType.create(assetInfo)
 	})
 
 	// urls -> bookmark asset
@@ -561,3 +574,36 @@ export function createEmptyBookmarkShape(
 
 	return editor.getShape(partial.id) as TLBookmarkShape
 }
+
+const objectURLCache = new WeakCache<TLAsset, ReturnType<typeof getLocalAssetObjectURL>>()
+export const defaultResolveAsset =
+	(persistenceKey?: string) => async (asset: TLAsset | null | undefined) => {
+		if (!asset || !asset.props.src) return null
+
+		// We don't deal with videos at the moment.
+		if (asset.type === 'video') return asset.props.src
+
+		// Assert it's an image to make TS happy.
+		if (asset.type !== 'image') return null
+
+		// Retrieve a local image from the DB.
+		if (persistenceKey && asset.props.src.startsWith('asset:')) {
+			return await objectURLCache.get(
+				asset,
+				async () => await getLocalAssetObjectURL(persistenceKey, asset.id)
+			)
+		}
+
+		return asset.props.src
+	}
+
+async function getLocalAssetObjectURL(persistenceKey: string, assetId: TLAssetId) {
+	const blob = await getAssetFromIndexedDb({
+		assetId: assetId,
+		persistenceKey,
+	})
+	if (blob) {
+		return URL.createObjectURL(blob)
+	}
+	return null
+}

packages/tldraw/src/lib/shapes/image/ImageShapeUtil.tsx

@@ -67,11 +67,11 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 		const { asset, url } = useAsset(shape.props.assetId, shape.props.w)
 
 		useEffect(() => {
-			// If an image is not animated (that's handled below), then we preload the image
-			// because we might have different source urls for different zoom levels.
+			// We preload the image because we might have different source urls for different
+			// zoom levels.
 			// Preloading the image ensures that the browser caches the image and doesn't
 			// cause visual flickering when the image is loaded.
-			if (url && !this.isAnimated(shape)) {
+			if (url) {
 				let cancelled = false
 
 				const image = Image()
@@ -204,12 +204,22 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 	}
 
 	override async toSvg(shape: TLImageShape) {
-		const asset = shape.props.assetId ? this.editor.getAsset(shape.props.assetId) : null
+		if (!shape.props.assetId) return null
+
+		const asset = this.editor.getAsset(shape.props.assetId)
 
 		if (!asset) return null
 
-		let src = asset?.props.src || ''
-		if (src.startsWith('http') || src.startsWith('/') || src.startsWith('./')) {
+		let src = await this.editor.resolveAssetUrl(shape.props.assetId, {
+			shouldResolveToOriginalImage: true,
+		})
+		if (!src) return null
+		if (
+			src.startsWith('blob:') ||
+			src.startsWith('http') ||
+			src.startsWith('/') ||
+			src.startsWith('./')
+		) {
 			// If it's a remote image, we need to fetch it and convert it to a data URI
 			src = (await getDataURIFromURL(src)) || ''
 		}

packages/tldraw/src/lib/ui/hooks/useClipboardEvents.ts

@@ -482,8 +482,10 @@ async function handleClipboardThings(editor: Editor, things: ClipboardThing[], p
  * @param editor - The editor instance.
  * @public
  */
-const handleNativeOrMenuCopy = (editor: Editor) => {
-	const content = editor.getContentFromCurrentPage(editor.getSelectedShapeIds())
+const handleNativeOrMenuCopy = async (editor: Editor) => {
+	const content = await editor.resolveAssetsInContent(
+		editor.getContentFromCurrentPage(editor.getSelectedShapeIds())
+	)
 	if (!content) {
 		if (navigator && navigator.clipboard) {
 			navigator.clipboard.writeText('')
@@ -555,20 +557,20 @@ export function useMenuClipboardEvents() {
 	const trackEvent = useUiEvents()
 
 	const copy = useCallback(
-		function onCopy(source: TLUiEventSource) {
+		async function onCopy(source: TLUiEventSource) {
 			if (editor.getSelectedShapeIds().length === 0) return
 
-			handleNativeOrMenuCopy(editor)
+			await handleNativeOrMenuCopy(editor)
 			trackEvent('copy', { source })
 		},
 		[editor, trackEvent]
 	)
 
 	const cut = useCallback(
-		function onCut(source: TLUiEventSource) {
+		async function onCut(source: TLUiEventSource) {
 			if (editor.getSelectedShapeIds().length === 0) return
 
-			handleNativeOrMenuCopy(editor)
+			await handleNativeOrMenuCopy(editor)
 			editor.deleteShapes(editor.getSelectedShapeIds())
 			trackEvent('cut', { source })
 		},
@@ -617,7 +619,7 @@ export function useNativeClipboardEvents() {
 
 	useEffect(() => {
 		if (!appIsFocused) return
-		const copy = (e: ClipboardEvent) => {
+		const copy = async (e: ClipboardEvent) => {
 			if (
 				editor.getSelectedShapeIds().length === 0 ||
 				editor.getEditingShapeId() !== null ||
@@ -627,11 +629,11 @@ export function useNativeClipboardEvents() {
 			}
 
 			preventDefault(e)
-			handleNativeOrMenuCopy(editor)
+			await handleNativeOrMenuCopy(editor)
 			trackEvent('copy', { source: 'kbd' })
 		}
 
-		function cut(e: ClipboardEvent) {
+		async function cut(e: ClipboardEvent) {
 			if (
 				editor.getSelectedShapeIds().length === 0 ||
 				editor.getEditingShapeId() !== null ||
@@ -640,7 +642,7 @@ export function useNativeClipboardEvents() {
 				return
 			}
 			preventDefault(e)
-			handleNativeOrMenuCopy(editor)
+			await handleNativeOrMenuCopy(editor)
 			editor.deleteShapes(editor.getSelectedShapeIds())
 			trackEvent('cut', { source: 'kbd' })
 		}

packages/tldraw/src/lib/utils/export/export.ts

@@ -119,7 +119,7 @@ export async function exportToString(
 			return (await getSvgString(editor, ids, opts))?.svg
 		}
 		case 'json': {
-			const data = editor.getContentFromCurrentPage(ids)
+			const data = await editor.resolveAssetsInContent(editor.getContentFromCurrentPage(ids))
 			return JSON.stringify(data)
 		}
 		default: {

packages/tldraw/src/lib/utils/tldr/file.ts

@@ -175,9 +175,9 @@ function pruneUnusedAssets(records: TLRecord[]) {
 }
 
 /** @public */
-export async function serializeTldrawJson(store: TLStore): Promise<string> {
+export async function serializeTldrawJson(editor: Editor): Promise<string> {
 	const records: TLRecord[] = []
-	for (const record of store.allRecords()) {
+	for (const record of editor.store.allRecords()) {
 		switch (record.typeName) {
 			case 'asset':
 				if (
@@ -187,10 +187,14 @@ export async function serializeTldrawJson(store: TLStore): Promise<string> {
 				) {
 					let assetSrcToSave
 					try {
+						let src = record.props.src
+						if (!src.startsWith('http')) {
+							src =
+								(await editor.resolveAssetUrl(record.id, { shouldResolveToOriginalImage: true })) ||
+								''
+						}
 						// try to save the asset as a base64 string
-						assetSrcToSave = await FileHelpers.blobToDataUrl(
-							await (await fetch(record.props.src)).blob()
-						)
+						assetSrcToSave = await FileHelpers.blobToDataUrl(await (await fetch(src)).blob())
 					} catch {
 						// if that fails, just save the original src
 						assetSrcToSave = record.props.src
@@ -215,14 +219,14 @@ export async function serializeTldrawJson(store: TLStore): Promise<string> {
 
 	return JSON.stringify({
 		tldrawFileFormatVersion: LATEST_TLDRAW_FILE_FORMAT_VERSION,
-		schema: store.schema.serialize(),
+		schema: editor.store.schema.serialize(),
 		records: pruneUnusedAssets(records),
 	})
 }
 
 /** @public */
-export async function serializeTldrawJsonBlob(store: TLStore): Promise<Blob> {
-	return new Blob([await serializeTldrawJson(store)], { type: TLDRAW_FILE_MIMETYPE })
+export async function serializeTldrawJsonBlob(editor: Editor): Promise<Blob> {
+	return new Blob([await serializeTldrawJson(editor)], { type: TLDRAW_FILE_MIMETYPE })
 }
 
 /** @internal */

packages/validate/src/lib/validation.ts

@@ -984,7 +984,8 @@ export const linkUrl = string.check((value) => {
 	}
 })
 
-const validSrcProtocols = new Set(['http:', 'https:', 'data:'])
+// N.B. asset: is a reference to the local indexedDB object store.
+const validSrcProtocols = new Set(['http:', 'https:', 'data:', 'asset:'])
 
 /**
  * Validates that a valid is a url safe to load as an asset.

yarn.lock

@@ -21439,6 +21439,7 @@ __metadata:
     chokidar-cli: "npm:^3.0.0"
     classnames: "npm:^2.3.2"
     hotkeys-js: "npm:^3.11.2"
+    idb: "npm:^7.1.1"
     jest-canvas-mock: "npm:^2.5.2"
     jest-environment-jsdom: "npm:^29.4.3"
     lazyrepo: "npm:0.0.0-alpha.27"