assets: make option to transform urls dynamically / LOD (#3827)

this is take #2 of this PR https://github.com/tldraw/tldraw/pull/3764

This continues the idea kicked off in
https://github.com/tldraw/tldraw/pull/3684 to explore LOD and takes it
in a different direction.

Several things here to call out:
- our dotcom version would start to use Cloudflare's image transforms
- we don't rewrite non-image assets 
- we debounce zooming so that we're not swapping out images while
zooming (it creates jank)
- we load different images based on steps of .25 (maybe we want to make
this more, like 0.33). Feels like 0.5 might be a bit too much but we can
play around with it.
- we take into account network connection speed. if you're on 3g, for
example, we have the size of the image.
- dpr is taken into account - in our case, Cloudflare handles it. But if
it wasn't Cloudflare, we could add it to our width equation.
- we use Cloudflare's `fit=scale-down` setting to never scale _up_ an
image.
- we don't swap the image in until we've finished loading it
programatically (to avoid a blank image while it loads)

TODO
- [x] We need to enable Cloudflare's pricing on image transforms btw
@steveruizok 😉 - this won't work quite yet until we do that.


### Change Type

<!--  Please select a 'Scope' label ️ -->

- [x] `sdk` — Changes the tldraw SDK
- [ ] `dotcom` — Changes the tldraw.com web app
- [ ] `docs` — Changes to the documentation, examples, or templates.
- [ ] `vs code` — Changes to the vscode plugin
- [ ] `internal` — Does not affect user-facing stuff

<!--  Please select a 'Type' label ️ -->

- [ ] `bugfix` — Bug fix
- [x] `feature` — New feature
- [ ] `improvement` — Improving existing features
- [ ] `chore` — Updating dependencies, other boring stuff
- [ ] `galaxy brain` — Architectural changes
- [ ] `tests` — Changes to any test code
- [ ] `tools` — Changes to infrastructure, CI, internal scripts,
debugging tools, etc.
- [ ] `dunno` — I don't know


### Test Plan

1. Test images on staging, small, medium, large, mega
2. Test videos on staging

- [x] Unit Tests
- [ ] End to end tests

### Release Notes

- Assets: make option to transform urls dynamically to provide different
sized images on demand.
This commit is contained in:
Mime Čuvalo 2024-06-11 15:17:09 +01:00 committed by GitHub
parent 3adae06d9c
commit 6c846716c3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
35 changed files with 570 additions and 26 deletions

View file

@ -55,6 +55,7 @@ jobs:
GH_TOKEN: ${{ github.token }}
ASSET_UPLOAD: ${{ vars.ASSET_UPLOAD }}
ASSET_BUCKET_ORIGIN: ${{ vars.ASSET_BUCKET_ORIGIN }}
MULTIPLAYER_SERVER: ${{ vars.MULTIPLAYER_SERVER }}
SUPABASE_LITE_URL: ${{ vars.SUPABASE_LITE_URL }}
VERCEL_PROJECT_ID: ${{ vars.VERCEL_DOTCOM_PROJECT_ID }}

View file

@ -2,6 +2,7 @@ global.crypto ??= new (require('@peculiar/webcrypto').Crypto)()
process.env.MULTIPLAYER_SERVER = 'https://localhost:8787'
process.env.ASSET_UPLOAD = 'https://localhost:8788'
process.env.ASSET_BUCKET_ORIGIN = 'https://localhost:8788'
global.TextEncoder = require('util').TextEncoder
global.TextDecoder = require('util').TextDecoder

View file

@ -19,6 +19,7 @@ import {
ViewSubmenu,
useActions,
} from 'tldraw'
import { resolveAsset } from '../utils/assetHandler'
import { assetUrls } from '../utils/assetUrls'
import { createAssetFromUrl } from '../utils/createAssetFromUrl'
import { DebugMenuItems } from '../utils/migration/DebugMenuItems'
@ -105,6 +106,7 @@ export function LocalEditor() {
overrides={[sharingUiOverrides, fileSystemUiOverrides]}
onUiEvent={handleUiEvent}
components={components}
assetOptions={{ onResolveAsset: resolveAsset }}
inferDarkMode
>
<LocalMigration />

View file

@ -24,6 +24,7 @@ import {
} from 'tldraw'
import { useRemoteSyncClient } from '../hooks/useRemoteSyncClient'
import { UrlStateParams, useUrlState } from '../hooks/useUrlState'
import { resolveAsset } from '../utils/assetHandler'
import { assetUrls } from '../utils/assetUrls'
import { MULTIPLAYER_SERVER } from '../utils/config'
import { CursorChatMenuItem } from '../utils/context-menu/CursorChatMenuItem'
@ -158,6 +159,7 @@ export function MultiplayerEditor({
initialState={isReadonly ? 'hand' : 'select'}
onUiEvent={handleUiEvent}
components={components}
assetOptions={{ onResolveAsset: resolveAsset }}
inferDarkMode
>
<UrlStateSync />

View file

@ -54,6 +54,7 @@ export function useMultiplayerAssets(assetUploaderUrl: string) {
src: url,
w: size.w,
h: size.h,
fileSize: file.size,
mimeType: file.type,
isAnimated,
},

View file

@ -0,0 +1,202 @@
import { TLAsset } from 'tldraw'
import { resolveAsset } from './assetHandler'
const FILE_SIZE = 1024 * 1024 * 2
describe('resolveAsset', () => {
it('should return null if the asset is null', async () => {
expect(
await resolveAsset(null, {
screenScale: -1,
steppedScreenScale: 1,
dpr: 1,
networkEffectiveType: '4g',
})
).toBe(null)
})
it('should return null if the asset is undefined', async () => {
expect(
await resolveAsset(undefined, {
screenScale: -1,
steppedScreenScale: 1,
dpr: 1,
networkEffectiveType: '4g',
})
).toBe(null)
})
it('should return null if the asset has no src', async () => {
const asset = { type: 'image', props: { w: 100, fileSize: FILE_SIZE } }
expect(
await resolveAsset(asset as TLAsset, {
screenScale: -1,
steppedScreenScale: 1,
dpr: 1,
networkEffectiveType: '4g',
})
).toBe(null)
})
it('should return the original src for video types', async () => {
const asset = {
type: 'video',
props: { src: 'http://example.com/video.mp4', fileSize: FILE_SIZE },
}
expect(
await resolveAsset(asset as TLAsset, {
screenScale: -1,
steppedScreenScale: 1,
dpr: 1,
networkEffectiveType: '4g',
})
).toBe('http://example.com/video.mp4')
})
it('should return the original src if it does not start with http or https', async () => {
const asset = { type: 'image', props: { src: 'data:somedata', w: 100, fileSize: FILE_SIZE } }
expect(
await resolveAsset(asset as TLAsset, {
screenScale: -1,
steppedScreenScale: 1,
dpr: 1,
networkEffectiveType: '4g',
})
).toBe('data:somedata')
})
it('should return the original src if it is animated', async () => {
const asset = {
type: 'image',
props: {
src: 'http://example.com/animated.gif',
mimeType: 'image/gif',
w: 100,
fileSize: FILE_SIZE,
},
}
expect(
await resolveAsset(asset as TLAsset, {
screenScale: -1,
steppedScreenScale: 1,
dpr: 1,
networkEffectiveType: '4g',
})
).toBe('http://example.com/animated.gif')
})
it('should return the original src if it is under a certain file size', async () => {
const asset = {
type: 'image',
props: { src: 'http://example.com/small.png', w: 100, fileSize: 1024 * 1024 },
}
expect(
await resolveAsset(asset as TLAsset, {
screenScale: -1,
steppedScreenScale: 1,
dpr: 1,
networkEffectiveType: '4g',
})
).toBe('http://example.com/small.png')
})
it("should return null if the asset type is not 'image'", async () => {
const asset = {
type: 'document',
props: { src: 'http://example.com/doc.pdf', w: 100, fileSize: FILE_SIZE },
}
expect(
await resolveAsset(asset as TLAsset, {
screenScale: -1,
steppedScreenScale: 1,
dpr: 1,
networkEffectiveType: '4g',
})
).toBe(null)
})
it('should handle if network compensation is not available and zoom correctly', async () => {
const asset = {
type: 'image',
props: { src: 'http://example.com/image.jpg', w: 100, fileSize: FILE_SIZE },
}
expect(
await resolveAsset(asset as TLAsset, {
screenScale: -1,
steppedScreenScale: 0.5,
dpr: 2,
networkEffectiveType: null,
})
).toBe(
'https://localhost:8788/cdn-cgi/image/width=50,dpr=2,fit=scale-down,quality=92/http://example.com/image.jpg'
)
})
it('should handle network compensation and zoom correctly', async () => {
const asset = {
type: 'image',
props: { src: 'http://example.com/image.jpg', w: 100, fileSize: FILE_SIZE },
}
expect(
await resolveAsset(asset as TLAsset, {
screenScale: -1,
steppedScreenScale: 0.5,
dpr: 2,
networkEffectiveType: '3g',
})
).toBe(
'https://localhost:8788/cdn-cgi/image/width=25,dpr=2,fit=scale-down,quality=92/http://example.com/image.jpg'
)
})
it('should round zoom to powers of 2', async () => {
const asset = {
type: 'image',
props: { src: 'https://example.com/image.jpg', w: 100, fileSize: FILE_SIZE },
}
expect(
await resolveAsset(asset as TLAsset, {
screenScale: -1,
steppedScreenScale: 4,
dpr: 1,
networkEffectiveType: '4g',
})
).toBe(
'https://localhost:8788/cdn-cgi/image/width=400,dpr=1,fit=scale-down,quality=92/https://example.com/image.jpg'
)
})
it('should round zoom to the nearest 0.25 and apply network compensation', async () => {
const asset = {
type: 'image',
props: { src: 'https://example.com/image.jpg', w: 100, fileSize: FILE_SIZE },
}
expect(
await resolveAsset(asset as TLAsset, {
screenScale: -1,
steppedScreenScale: 0.5,
dpr: 1,
networkEffectiveType: '2g',
})
).toBe(
'https://localhost:8788/cdn-cgi/image/width=25,dpr=1,fit=scale-down,quality=92/https://example.com/image.jpg'
)
})
it('should set zoom to a minimum of 0.25 if zoom is below 0.25', async () => {
const asset = {
type: 'image',
props: { src: 'https://example.com/image.jpg', w: 100, fileSize: FILE_SIZE },
}
expect(
await resolveAsset(asset as TLAsset, {
screenScale: -1,
steppedScreenScale: 0.25,
dpr: 1,
networkEffectiveType: '4g',
})
).toBe(
'https://localhost:8788/cdn-cgi/image/width=25,dpr=1,fit=scale-down,quality=92/https://example.com/image.jpg'
)
})
})

View file

@ -0,0 +1,40 @@
import { AssetContextProps, MediaHelpers, TLAsset } from 'tldraw'
import { ASSET_BUCKET_ORIGIN, ASSET_UPLOADER_URL } from './config'
export async function resolveAsset(asset: TLAsset | null | undefined, context: AssetContextProps) {
if (!asset || !asset.props.src) return null
// We don't deal with videos at the moment.
if (asset.type === 'video') return asset.props.src
// Assert it's an image to make TS happy.
if (asset.type !== 'image') return null
// Don't try to transform data: URLs, yikes.
if (!asset.props.src.startsWith('http:') && !asset.props.src.startsWith('https:'))
return asset.props.src
// Don't try to transform animated images.
if (MediaHelpers.isAnimatedImageType(asset?.props.mimeType) || asset.props.isAnimated)
return asset.props.src
// Assets that are under a certain file size aren't worth transforming (and incurring cost).
if (asset.props.fileSize === -1 || asset.props.fileSize < 1024 * 1024 * 1.5 /* 1.5 MB */)
return asset.props.src
// N.B. navigator.connection is only available in certain browsers (mainly Blink-based browsers)
// 4g is as high the 'effectiveType' goes and we can pick a lower effective image quality for slower connections.
const networkCompensation =
!context.networkEffectiveType || context.networkEffectiveType === '4g' ? 1 : 0.5
const width = Math.ceil(asset.props.w * context.steppedScreenScale * networkCompensation)
if (process.env.NODE_ENV === 'development') {
return asset.props.src
}
// On preview, builds the origin for the asset won't be the right one for the Cloudflare transform.
const src = asset.props.src.replace(ASSET_UPLOADER_URL, ASSET_BUCKET_ORIGIN)
return `${ASSET_BUCKET_ORIGIN}/cdn-cgi/image/width=${width},dpr=${context.dpr},fit=scale-down,quality=92/${src}`
}

View file

@ -6,8 +6,14 @@ if (!process.env.ASSET_UPLOAD) {
throw new Error('Missing ASSET_UPLOAD env var')
}
if (!process.env.ASSET_BUCKET_ORIGIN) {
throw new Error('Missing ASSET_BUCKET_ORIGIN env var')
}
export const ASSET_UPLOADER_URL: string = process.env.ASSET_UPLOAD
export const ASSET_BUCKET_ORIGIN: string = process.env.ASSET_BUCKET_ORIGIN
export const CONTROL_SERVER: string =
process.env.NEXT_PUBLIC_CONTROL_SERVER || 'http://localhost:3001'

View file

@ -53,6 +53,7 @@ export async function createAssetFromFile({ file }: { type: 'file'; file: File }
w: size.w,
h: size.h,
mimeType: file.type,
fileSize: file.size,
isAnimated,
},
meta: {},

View file

@ -46,6 +46,11 @@ export default defineConfig((env) => ({
),
'process.env.MULTIPLAYER_SERVER': urlOrLocalFallback(env.mode, getMultiplayerServerURL(), 8787),
'process.env.ASSET_UPLOAD': urlOrLocalFallback(env.mode, process.env.ASSET_UPLOAD, 8788),
'process.env.ASSET_BUCKET_ORIGIN': urlOrLocalFallback(
env.mode,
process.env.ASSET_BUCKET_ORIGIN,
8788
),
'process.env.TLDRAW_ENV': JSON.stringify(process.env.TLDRAW_ENV ?? 'development'),
// Fall back to staging DSN for local develeopment, although you still need to
// modify the env check in 'sentry.client.config.ts' to get it reporting errors

View file

@ -58,6 +58,7 @@ export default function HostedImagesExample() {
src: url,
w: size.w,
h: size.h,
fileSize: file.size,
mimeType: file.type,
isAnimated,
},

View file

@ -48,6 +48,7 @@ export function ImageAnnotationEditor({
props: {
w: image.width,
h: image.height,
fileSize: -1,
mimeType: image.type,
src: image.src,
name: 'image',

View file

@ -21,6 +21,7 @@ export default function LocalImagesExample() {
src: '/tldraw.png', // You could also use a base64 encoded string here
w: imageWidth,
h: imageHeight,
fileSize: -1,
mimeType: 'image/png',
isAnimated: false,
},

View file

@ -43,6 +43,7 @@ export function PdfEditor({ pdf }: { pdf: Pdf }) {
props: {
w: page.bounds.w,
h: page.bounds.h,
fileSize: -1,
mimeType: 'image/png',
src: page.src,
name: 'page',

View file

@ -85,6 +85,7 @@ import { useComputed } from '@tldraw/state';
import { useQuickReactor } from '@tldraw/state';
import { useReactor } from '@tldraw/state';
import { useValue } from '@tldraw/state';
import { useValueDebounced } from '@tldraw/state';
import { VecModel } from '@tldraw/tlschema';
import { whyAmIRunning } from '@tldraw/state';
@ -144,6 +145,18 @@ export class Arc2d extends Geometry2d {
// @public
export function areAnglesCompatible(a: number, b: number): boolean;
// @public (undocumented)
export interface AssetContextProps {
// (undocumented)
dpr: number;
// (undocumented)
networkEffectiveType: null | string;
// (undocumented)
screenScale: number;
// (undocumented)
steppedScreenScale: number;
}
export { Atom }
export { atom }
@ -765,7 +778,7 @@ export class Edge2d extends Geometry2d {
// @public (undocumented)
export class Editor extends EventEmitter<TLEventMap> {
constructor({ store, user, shapeUtils, bindingUtils, tools, getContainer, cameraOptions, initialState, autoFocus, inferDarkMode, options, }: TLEditorOptions);
constructor({ store, user, shapeUtils, bindingUtils, tools, getContainer, cameraOptions, assetOptions, initialState, autoFocus, inferDarkMode, options, }: TLEditorOptions);
addOpenMenu(id: string): this;
alignShapes(shapes: TLShape[] | TLShapeId[], operation: 'bottom' | 'center-horizontal' | 'center-vertical' | 'left' | 'right' | 'top'): this;
animateShape(partial: null | TLShapePartial | undefined, opts?: Partial<{
@ -1114,6 +1127,10 @@ export class Editor extends EventEmitter<TLEventMap> {
reparentShapes(shapes: TLShape[] | TLShapeId[], parentId: TLParentId, insertIndex?: IndexKey): this;
resetZoom(point?: Vec, opts?: TLCameraMoveOptions): this;
resizeShape(shape: TLShape | TLShapeId, scale: VecLike, options?: TLResizeShapeOptions): this;
// (undocumented)
resolveAssetUrl(assetId: null | TLAssetId, context: {
screenScale: number;
}): Promise<null | string>;
readonly root: StateNode;
rotateShapesBy(shapes: TLShape[] | TLShapeId[], delta: number): this;
screenToPage(point: VecLike): Vec;
@ -2306,6 +2323,12 @@ export type TLAnyBindingUtilConstructor = TLBindingUtilConstructor<any>;
// @public (undocumented)
export type TLAnyShapeUtilConstructor = TLShapeUtilConstructor<any>;
// @public (undocumented)
export interface TLAssetOptions {
// (undocumented)
onResolveAsset: (asset: null | TLAsset | undefined, ctx: AssetContextProps) => Promise<null | string>;
}
// @public (undocumented)
export type TLBaseBoxShape = TLBaseShape<string, {
h: number;
@ -2478,6 +2501,7 @@ export const TldrawEditor: React_2.NamedExoticComponent<TldrawEditorProps>;
// @public
export interface TldrawEditorBaseProps {
assetOptions?: Partial<TLAssetOptions>;
autoFocus?: boolean;
bindingUtils?: readonly TLAnyBindingUtilConstructor[];
cameraOptions?: Partial<TLCameraOptions>;
@ -2622,6 +2646,7 @@ export interface TLEditorComponents {
// @public (undocumented)
export interface TLEditorOptions {
assetOptions?: Partial<TLAssetOptions>;
autoFocus?: boolean;
bindingUtils: readonly TLBindingUtilConstructor<TLUnknownBinding>[];
cameraOptions?: Partial<TLCameraOptions>;
@ -3453,6 +3478,8 @@ export function useTransform(ref: React.RefObject<HTMLElement | SVGElement>, x?:
export { useValue }
export { useValueDebounced }
// @public (undocumented)
export class Vec {
constructor(x?: number, y?: number, z?: number);

View file

@ -28,6 +28,7 @@ export {
useQuickReactor,
useReactor,
useValue,
useValueDebounced,
whyAmIRunning,
type Atom,
type Signal,
@ -239,8 +240,10 @@ export {
type TLHistoryMark,
} from './lib/editor/types/history-types'
export {
type AssetContextProps,
type OptionalKeys,
type RequiredKeys,
type TLAssetOptions,
type TLCameraConstraints,
type TLCameraMoveOptions,
type TLCameraOptions,

View file

@ -20,7 +20,7 @@ import { TLAnyBindingUtilConstructor } from './config/defaultBindings'
import { TLAnyShapeUtilConstructor } from './config/defaultShapes'
import { Editor } from './editor/Editor'
import { TLStateNodeConstructor } from './editor/tools/StateNode'
import { TLCameraOptions } from './editor/types/misc-types'
import { TLAssetOptions, TLCameraOptions } from './editor/types/misc-types'
import { ContainerProvider, useContainer } from './hooks/useContainer'
import { useCursor } from './hooks/useCursor'
import { useDarkMode } from './hooks/useDarkMode'
@ -127,6 +127,11 @@ export interface TldrawEditorBaseProps {
*/
cameraOptions?: Partial<TLCameraOptions>
/**
* Asset options for the editor.
*/
assetOptions?: Partial<TLAssetOptions>
/**
* Options for the editor.
*/
@ -300,6 +305,7 @@ function TldrawEditorWithReadyStore({
autoFocus = true,
inferDarkMode,
cameraOptions,
assetOptions,
options,
}: Required<
TldrawEditorProps & {
@ -325,6 +331,7 @@ function TldrawEditorWithReadyStore({
autoFocus: initialAutoFocus,
inferDarkMode,
cameraOptions,
assetOptions,
options,
})
setEditor(editor)
@ -343,6 +350,7 @@ function TldrawEditorWithReadyStore({
initialAutoFocus,
inferDarkMode,
cameraOptions,
assetOptions,
options,
])

View file

@ -1,4 +1,4 @@
import { TLCameraOptions } from './editor/types/misc-types'
import { TLAssetOptions, TLCameraOptions } from './editor/types/misc-types'
import { EASINGS } from './primitives/easings'
/** @internal */
@ -10,6 +10,11 @@ export const DEFAULT_CAMERA_OPTIONS: TLCameraOptions = {
zoomSteps: [0.1, 0.25, 0.5, 1, 2, 4, 8],
}
/** @internal */
export const DEFAULT_ASSET_OPTIONS: TLAssetOptions = {
onResolveAsset: async (asset) => asset?.props.src || '',
}
/** @internal */
export const DEFAULT_ANIMATION_OPTIONS = {
duration: 0,

View file

@ -85,6 +85,7 @@ import { checkBindings } from '../config/defaultBindings'
import { checkShapesAndAddCore } from '../config/defaultShapes'
import {
DEFAULT_ANIMATION_OPTIONS,
DEFAULT_ASSET_OPTIONS,
DEFAULT_CAMERA_OPTIONS,
INTERNAL_POINTER_IDS,
LEFT_MOUSE_BUTTON,
@ -144,6 +145,7 @@ import { TLHistoryBatchOptions } from './types/history-types'
import {
OptionalKeys,
RequiredKeys,
TLAssetOptions,
TLCameraMoveOptions,
TLCameraOptions,
TLSvgOptions,
@ -207,7 +209,10 @@ export interface TLEditorOptions {
* Options for the editor's camera.
*/
cameraOptions?: Partial<TLCameraOptions>
/**
* Options for the editor's assets.
*/
assetOptions?: Partial<TLAssetOptions>
options?: Partial<TldrawOptions>
}
@ -221,6 +226,7 @@ export class Editor extends EventEmitter<TLEventMap> {
tools,
getContainer,
cameraOptions,
assetOptions,
initialState,
autoFocus,
inferDarkMode,
@ -246,6 +252,8 @@ export class Editor extends EventEmitter<TLEventMap> {
this._cameraOptions.set({ ...DEFAULT_CAMERA_OPTIONS, ...cameraOptions })
this._assetOptions.set({ ...DEFAULT_ASSET_OPTIONS, ...assetOptions })
this.user = new UserPreferencesManager(user ?? createTLUser(), inferDarkMode ?? false)
this.getContainer = getContainer ?? (() => document.body)
@ -3815,6 +3823,8 @@ export class Editor extends EventEmitter<TLEventMap> {
/* --------------------- Assets --------------------- */
private _assetOptions = atom('asset options', DEFAULT_ASSET_OPTIONS)
/** @internal */
@computed private _getAllAssetsQuery() {
return this.store.query.records('asset')
@ -3915,6 +3925,29 @@ export class Editor extends EventEmitter<TLEventMap> {
return this.store.get(typeof asset === 'string' ? asset : asset.id) as TLAsset | undefined
}
async resolveAssetUrl(
assetId: TLAssetId | null,
context: { screenScale: number }
): Promise<string | null> {
if (!assetId) return ''
const asset = this.getAsset(assetId)
if (!asset) return ''
// We only look at the zoom level at powers of 2.
const zoomStepFunction = (zoom: number) => Math.pow(2, Math.ceil(Math.log2(zoom)))
const steppedScreenScale = Math.max(0.125, zoomStepFunction(context.screenScale))
const networkEffectiveType: string | null =
'connection' in navigator ? (navigator as any).connection.effectiveType : null
const dpr = this.getInstanceState().devicePixelRatio
return await this._assetOptions.get().onResolveAsset(asset!, {
screenScale: context.screenScale,
steppedScreenScale,
dpr,
networkEffectiveType,
})
}
/* --------------------- Shapes --------------------- */
@computed

View file

@ -1,4 +1,4 @@
import { BoxModel } from '@tldraw/tlschema'
import { BoxModel, TLAsset } from '@tldraw/tlschema'
import { Box } from '../../primitives/Box'
import { VecLike } from '../../primitives/Vec'
@ -55,6 +55,22 @@ export interface TLCameraOptions {
constraints?: TLCameraConstraints
}
/** @public */
export interface AssetContextProps {
screenScale: number
steppedScreenScale: number
dpr: number
networkEffectiveType: string | null
}
/** @public */
export interface TLAssetOptions {
onResolveAsset: (
asset: TLAsset | null | undefined,
ctx: AssetContextProps
) => Promise<string | null>
}
/** @public */
export interface TLCameraConstraints {
/** The bounds (in page space) of the constrained space */

View file

@ -207,6 +207,9 @@ export function useValue<Value>(value: Signal<Value>): Value;
// @public (undocumented)
export function useValue<Value>(name: string, fn: () => Value, deps: unknown[]): Value;
// @public
export function useValueDebounced<Value>(name: string, fn: () => Value, deps: unknown[], ms: number): Value;
// @public
export function whyAmIRunning(): void;

View file

@ -5,3 +5,4 @@ export { useQuickReactor } from './useQuickReactor'
export { useReactor } from './useReactor'
export { useStateTracking } from './useStateTracking'
export { useValue } from './useValue'
export { useValueDebounced } from './useValueDebounced'

View file

@ -0,0 +1,33 @@
/* eslint-disable prefer-rest-params */
import { useEffect, useState } from 'react'
import { useValue } from './useValue'
/**
* Extracts the value from a signal and subscribes to it, debouncing the value by the given number of milliseconds.
*
* @see [[useValue]] for more information.
*
* @public
*/
export function useValueDebounced<Value>(
name: string,
fn: () => Value,
deps: unknown[],
ms: number
): Value
/** @public */
export function useValueDebounced<Value>(): Value {
const args = [...arguments].slice(0, -1) as Parameters<typeof useValue>
const ms = arguments[arguments.length - 1] as number
const value = useValue(...args) as Value
const [debouncedValue, setDebouncedValue] = useState<Value>(value)
useEffect(() => {
const timer = setTimeout(() => {
setDebouncedValue(value)
}, ms)
return () => clearTimeout(timer)
}, [value, ms])
return debouncedValue
}

View file

@ -23,7 +23,7 @@ import {
import { FONT_FAMILIES, FONT_SIZES, TEXT_PROPS } from './shapes/shared/default-shape-constants'
import { TLUiToastsContextType } from './ui/context/toasts'
import { useTranslation } from './ui/hooks/useTranslation/useTranslation'
import { containBoxSize, downsizeImage } from './utils/assets/assets'
import { containBoxSize } from './utils/assets/assets'
import { getEmbedInfo } from './utils/embeds/embeds'
import { cleanupText, isRightToLeftLanguage } from './utils/text/text'
@ -82,14 +82,6 @@ export function registerDefaultExternalContentHandlers(
}
}
// Always rescale the image
if (!isAnimated && MediaHelpers.isStaticImageType(file.type)) {
file = await downsizeImage(file, size.w, size.h, {
type: file.type,
quality: 0.92,
})
}
const assetId: TLAssetId = AssetRecordType.createId(hash)
const asset = AssetRecordType.create({
@ -101,6 +93,7 @@ export function registerDefaultExternalContentHandlers(
src: await FileHelpers.blobToDataUrl(file),
w: size.w,
h: size.h,
fileSize: file.size,
mimeType: file.type,
isAnimated,
},

View file

@ -18,6 +18,7 @@ import {
import { useEffect, useState } from 'react'
import { BrokenAssetIcon } from '../shared/BrokenAssetIcon'
import { HyperlinkButton } from '../shared/HyperlinkButton'
import { useAsset } from '../shared/useAsset'
import { usePrefersReducedMotion } from '../shared/usePrefersReducedMotion'
async function getDataURIFromURL(url: string): Promise<string> {
@ -61,15 +62,35 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
const isCropping = this.editor.getCroppingShapeId() === shape.id
const prefersReducedMotion = usePrefersReducedMotion()
const [staticFrameSrc, setStaticFrameSrc] = useState('')
const asset = shape.props.assetId ? this.editor.getAsset(shape.props.assetId) : undefined
const [loadedSrc, setLoadedSrc] = useState('')
const isSelected = shape.id === this.editor.getOnlySelectedShapeId()
const { asset, url } = useAsset(shape.props.assetId, shape.props.w)
useEffect(() => {
if (asset?.props.src && this.isAnimated(shape)) {
// If an image is not animated (that's handled below), then we preload the image
// because we might have different source urls for different zoom levels.
// Preloading the image ensures that the browser caches the image and doesn't
// cause visual flickering when the image is loaded.
if (url && !this.isAnimated(shape)) {
let cancelled = false
if (!url) return
const image = Image()
image.onload = () => {
if (cancelled) return
setLoadedSrc(url)
}
image.src = url
return () => {
cancelled = true
}
}
}, [url, shape])
useEffect(() => {
if (url && this.isAnimated(shape)) {
let cancelled = false
const url = asset.props.src
if (!url) return
const image = Image()
@ -85,6 +106,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
ctx.drawImage(image, 0, 0)
setStaticFrameSrc(canvas.toDataURL())
setLoadedSrc(url)
}
image.crossOrigin = 'anonymous'
image.referrerPolicy = 'strict-origin-when-cross-origin'
@ -94,7 +116,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
cancelled = true
}
}
}, [prefersReducedMotion, asset?.props, shape])
}, [prefersReducedMotion, url, shape])
if (asset?.type === 'bookmark') {
throw Error("Bookmark assets can't be rendered as images")
@ -108,7 +130,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
const containerStyle = getCroppedContainerStyle(shape)
if (!asset?.props.src) {
if (!url) {
return (
<HTMLContainer
id={shape.id}
@ -141,7 +163,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
style={{
opacity: 0.1,
backgroundImage: `url(${
!shape.props.playing || reduceMotion ? staticFrameSrc : asset.props.src
!shape.props.playing || reduceMotion ? staticFrameSrc : loadedSrc
})`,
}}
draggable={false}
@ -157,7 +179,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
className="tl-image"
style={{
backgroundImage: `url(${
!shape.props.playing || reduceMotion ? staticFrameSrc : asset.props.src
!shape.props.playing || reduceMotion ? staticFrameSrc : loadedSrc
})`,
}}
draggable={false}

View file

@ -0,0 +1,31 @@
import { TLAssetId, useEditor, useValueDebounced } from '@tldraw/editor'
import { useEffect, useState } from 'react'
/** @internal */
export function useAsset(assetId: TLAssetId | null, width: number) {
const editor = useEditor()
const [url, setUrl] = useState<string | null>(null)
const asset = assetId ? editor.getAsset(assetId) : null
const shapeScale = asset && 'w' in asset.props ? width / asset.props.w : 1
// We debounce the zoom level to reduce the number of times we fetch a new image and,
// more importantly, to not cause zooming in and out to feel janky.
const debouncedScreenScale = useValueDebounced(
'zoom level',
() => editor.getZoomLevel() * shapeScale,
[editor, shapeScale],
500
)
useEffect(() => {
async function resolve() {
const resolvedUrl = await editor.resolveAssetUrl(assetId, {
screenScale: debouncedScreenScale,
})
setUrl(resolvedUrl)
}
resolve()
}, [assetId, debouncedScreenScale, editor])
return { asset, url }
}

View file

@ -11,6 +11,7 @@ import {
import { ReactEventHandler, useCallback, useEffect, useRef, useState } from 'react'
import { BrokenAssetIcon } from '../shared/BrokenAssetIcon'
import { HyperlinkButton } from '../shared/HyperlinkButton'
import { useAsset } from '../shared/useAsset'
import { usePrefersReducedMotion } from '../shared/usePrefersReducedMotion'
/** @public */
@ -36,7 +37,7 @@ export class VideoShapeUtil extends BaseBoxShapeUtil<TLVideoShape> {
component(shape: TLVideoShape) {
const { editor } = this
const showControls = editor.getShapeGeometry(shape).bounds.w * editor.getZoomLevel() >= 110
const asset = shape.props.assetId ? editor.getAsset(shape.props.assetId) : null
const { asset, url } = useAsset(shape.props.assetId, shape.props.w)
const { time, playing } = shape.props
const isEditing = useIsEditing(shape.id)
const prefersReducedMotion = usePrefersReducedMotion()
@ -157,7 +158,7 @@ export class VideoShapeUtil extends BaseBoxShapeUtil<TLVideoShape> {
>
<div className="tl-counter-scaled">
<div className="tl-video-container">
{asset?.props.src ? (
{url ? (
<video
ref={rVideo}
style={isEditing ? { pointerEvents: 'all' } : undefined}
@ -178,7 +179,7 @@ export class VideoShapeUtil extends BaseBoxShapeUtil<TLVideoShape> {
onLoadedData={handleLoadedData}
hidden={!isLoaded}
>
<source src={asset.props.src} />
<source src={url} />
</video>
) : (
<BrokenAssetIcon />

View file

@ -290,6 +290,7 @@ export async function pasteExcalidrawContent(editor: Editor, clipboard: any, poi
props: {
w: element.width,
h: element.height,
fileSize: file.size,
name: element.id ?? 'Untitled',
isAnimated: false,
mimeType: file.mimeType,

View file

@ -68,6 +68,7 @@ export function buildFromV1Document(editor: Editor, _document: unknown) {
props: {
w: coerceDimension(v1Asset.size[0]),
h: coerceDimension(v1Asset.size[1]),
fileSize: -1,
name: v1Asset.fileName ?? 'Untitled',
isAnimated: false,
mimeType: null,
@ -91,6 +92,7 @@ export function buildFromV1Document(editor: Editor, _document: unknown) {
props: {
w: coerceDimension(v1Asset.size[0]),
h: coerceDimension(v1Asset.size[1]),
fileSize: -1,
name: v1Asset.fileName ?? 'Untitled',
isAnimated: true,
mimeType: null,

View file

@ -541,6 +541,7 @@ describe('snapshots', () => {
props: {
w: 1200,
h: 800,
fileSize: -1,
name: '',
isAnimated: false,
mimeType: 'png',

View file

@ -1187,6 +1187,7 @@ export type TLHighlightShapeProps = RecordPropsType<typeof highlightShapeProps>;
// @public
export type TLImageAsset = TLBaseAsset<'image', {
fileSize: number;
h: number;
isAnimated: boolean;
mimeType: null | string;
@ -1486,6 +1487,7 @@ export type TLUnknownShape = TLBaseShape<string, object>;
// @public
export type TLVideoAsset = TLBaseAsset<'video', {
fileSize: number;
h: number;
isAnimated: boolean;
mimeType: null | string;

View file

@ -12,6 +12,7 @@ export type TLImageAsset = TLBaseAsset<
{
w: number
h: number
fileSize: number
name: string
isAnimated: boolean
mimeType: string | null
@ -25,6 +26,7 @@ export const imageAssetValidator: T.Validator<TLImageAsset> = createAssetValidat
T.object({
w: T.number,
h: T.number,
fileSize: T.number,
name: T.string,
isAnimated: T.boolean,
mimeType: T.string.nullable(),
@ -36,6 +38,7 @@ const Versions = createMigrationIds('com.tldraw.asset.image', {
AddIsAnimated: 1,
RenameWidthHeight: 2,
MakeUrlsValid: 3,
AddFileSize: 4,
} as const)
export { Versions as imageAssetVersions }
@ -81,5 +84,14 @@ export const imageAssetMigrations = createRecordMigrationSequence({
// noop
},
},
{
id: Versions.AddFileSize,
up: (asset: any) => {
asset.props.fileSize = -1
},
down: (asset: any) => {
delete asset.props.fileSize
},
},
],
})

View file

@ -12,6 +12,7 @@ export type TLVideoAsset = TLBaseAsset<
{
w: number
h: number
fileSize: number
name: string
isAnimated: boolean
mimeType: string | null
@ -25,6 +26,7 @@ export const videoAssetValidator: T.Validator<TLVideoAsset> = createAssetValidat
T.object({
w: T.number,
h: T.number,
fileSize: T.number,
name: T.string,
isAnimated: T.boolean,
mimeType: T.string.nullable(),
@ -36,6 +38,7 @@ const Versions = createMigrationIds('com.tldraw.asset.video', {
AddIsAnimated: 1,
RenameWidthHeight: 2,
MakeUrlsValid: 3,
AddFileSize: 4,
} as const)
export { Versions as videoAssetVersions }
@ -81,5 +84,14 @@ export const videoAssetMigrations = createRecordMigrationSequence({
// noop
},
},
{
id: Versions.AddFileSize,
up: (asset: any) => {
asset.props.fileSize = -1
},
down: (asset: any) => {
delete asset.props.fileSize
},
},
],
})

View file

@ -98,6 +98,78 @@ describe('TLImageAsset AddIsAnimated', () => {
})
})
describe('TLVideoAsset AddFileSize', () => {
const oldAsset = {
id: '1',
type: 'video',
props: {
src: 'https://www.youtube.com/watch?v=1',
name: 'video',
width: 100,
height: 100,
mimeType: 'video/mp4',
},
}
const newAsset = {
id: '1',
type: 'video',
props: {
src: 'https://www.youtube.com/watch?v=1',
name: 'video',
width: 100,
height: 100,
mimeType: 'video/mp4',
fileSize: -1,
},
}
const { up, down } = getTestMigration(videoAssetVersions.AddFileSize)
test('up works as expected', () => {
expect(up(oldAsset)).toEqual(newAsset)
})
test('down works as expected', () => {
expect(down(newAsset)).toEqual(oldAsset)
})
})
describe('TLImageAsset AddFileSize', () => {
const oldAsset = {
id: '1',
type: 'image',
props: {
src: 'https://www.youtube.com/watch?v=1',
name: 'image',
width: 100,
height: 100,
mimeType: 'image/gif',
},
}
const newAsset = {
id: '1',
type: 'image',
props: {
src: 'https://www.youtube.com/watch?v=1',
name: 'image',
width: 100,
height: 100,
mimeType: 'image/gif',
fileSize: -1,
},
}
const { up, down } = getTestMigration(imageAssetVersions.AddFileSize)
test('up works as expected', () => {
expect(up(oldAsset)).toEqual(newAsset)
})
test('down works as expected', () => {
expect(down(newAsset)).toEqual(oldAsset)
})
})
const ShapeRecord = createRecordType('shape', {
validator: { validate: (record) => record as TLShape },
scope: 'document',

View file

@ -24,6 +24,7 @@ const dotcom = path.relative(process.cwd(), path.resolve(__dirname, '../apps/dot
const env = makeEnv([
'APP_ORIGIN',
'ASSET_UPLOAD',
'ASSET_BUCKET_ORIGIN',
'CLOUDFLARE_ACCOUNT_ID',
'CLOUDFLARE_API_TOKEN',
'DISCORD_DEPLOY_WEBHOOK_URL',