From 3c1aee492a4ba6acf10620b70445af291f7dc6b6 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 10 Jan 2024 14:41:18 +0000 Subject: [PATCH] faster image processing in default asset handler (#2441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ![Kapture 2024-01-10 at 13 42 06](https://github.com/tldraw/tldraw/assets/1489520/616bcda7-c05b-46f1-b985-3a36bb5c9476) (gif is with 6x CPU throttling to make the effect more visible) This is the first of a few diffs I'm working on to make dropping images onto the canvas feel a lot faster. There are three main changes here: 1. We operate on `Blob`s and `File`s rather than data urls. This saves a fair bit on converting to/from base64 all the time. I've updated our `MediaHelper` APIs to encourage the same in consumers. 2. We only check the max canvas size (slow) if images are above a certain dimension that we consider "safe" (8k x 8k) 3. Switching from the `downscale` npm library to canvas native downscaling. that library claims to give better results than the browser, but hasn't been updated in ~7 years. in modern browsers, we can opt-in to native high-quality image smoothing to achieve similar results much faster than with an algorithm implemented in pure JS. I want to follow this up with a system to show image placeholders whilst we're waiting for long-running operations like resizing etc but i'm going to split that out into its own diff as it'll involve some fairly complex changes to the history management API. ### Change Type - [x] `major` — Breaking change [^1]: publishes a `patch` release, for devDependencies use `internal` [^2]: will not publish a new version ### Test Plan 1. Tested manually, unit tests & end-to-end tests pass --- .../hosted-images/HostedImagesExample.tsx | 4 +- packages/tldraw/api-report.md | 14 +- packages/tldraw/api/api.json | 210 ++++++----- packages/tldraw/package.json | 2 - packages/tldraw/src/index.ts | 2 +- .../src/lib/defaultExternalContentHandlers.ts | 132 +++---- .../shapes/shared/getBrowserCanvasMaxSize.tsx | 2 + packages/tldraw/src/lib/utils/assets.ts | 103 ------ .../tldraw/src/lib/utils/assets/assets.ts | 99 +++-- packages/utils/api-report.md | 12 +- packages/utils/api/api.json | 342 +++++++++++++++++- packages/utils/src/index.ts | 2 +- packages/utils/src/lib/hash.ts | 15 + packages/utils/src/lib/media.ts | 122 +++++-- public-yarn.lock | 16 - 15 files changed, 674 insertions(+), 403 deletions(-) delete mode 100644 packages/tldraw/src/lib/utils/assets.ts diff --git a/apps/examples/src/examples/hosted-images/HostedImagesExample.tsx b/apps/examples/src/examples/hosted-images/HostedImagesExample.tsx index d7e3f9078..a06ab289b 100644 --- a/apps/examples/src/examples/hosted-images/HostedImagesExample.tsx +++ b/apps/examples/src/examples/hosted-images/HostedImagesExample.tsx @@ -39,12 +39,12 @@ export default function HostedImagesExample() { if (['image/jpeg', 'image/png', 'image/gif', 'image/svg+xml'].includes(file.type)) { shapeType = 'image' - size = await MediaHelpers.getImageSizeFromSrc(url) + size = await MediaHelpers.getImageSize(file) isAnimated = file.type === 'image/gif' && (await isGifAnimated(file)) } else { shapeType = 'video' isAnimated = true - size = await MediaHelpers.getVideoSizeFromSrc(url) + size = await MediaHelpers.getVideoSize(file) } const asset: TLAsset = AssetRecordType.create({ diff --git a/packages/tldraw/api-report.md b/packages/tldraw/api-report.md index fc7a61589..bfbefde80 100644 --- a/packages/tldraw/api-report.md +++ b/packages/tldraw/api-report.md @@ -320,6 +320,12 @@ declare namespace Dialog { } export { Dialog } +// @public +export function downsizeImage(blob: Blob, width: number, height: number, opts?: { + type?: string | undefined; + quality?: number | undefined; +}): Promise; + // @public (undocumented) export class DrawShapeTool extends StateNode { // (undocumented) @@ -692,12 +698,6 @@ export class GeoShapeUtil extends BaseBoxShapeUtil { // @public export function getEmbedInfo(inputUrl: string): TLEmbedResult; -// @public -export function getResizedImageDataUrl(dataURLForImage: string, width: number, height: number, opts?: { - type?: string | undefined; - quality?: number | undefined; -}): Promise; - // @public (undocumented) export function getSvgAsImage(svg: SVGElement, isSafari: boolean, options: { type: 'jpeg' | 'png' | 'webp'; @@ -834,7 +834,7 @@ function Indicator(): JSX.Element; export const Input: React_3.ForwardRefExoticComponent>; // @public (undocumented) -export function isGifAnimated(file: File): Promise; +export function isGifAnimated(file: Blob): Promise; // @public (undocumented) function Item({ noClose, ...props }: DropdownMenuItemProps): JSX.Element; diff --git a/packages/tldraw/api/api.json b/packages/tldraw/api/api.json index 8738eb989..762853bab 100644 --- a/packages/tldraw/api/api.json +++ b/packages/tldraw/api/api.json @@ -2827,6 +2827,114 @@ } ] }, + { + "kind": "Function", + "canonicalReference": "@tldraw/tldraw!downsizeImage:function(1)", + "docComment": "/**\n * Resize an image Blob to be smaller than it is currently.\n *\n * @param image - The image Blob.\n *\n * @param width - The desired width.\n *\n * @param height - The desired height.\n *\n * @param opts - Options for the image.\n *\n * @example\n * ```ts\n * const image = await (await fetch('/image.jpg')).blob()\n * const size = await getImageSize(image)\n * const resizedImage = await downsizeImage(image, size.w / 2, size.h / 2, { type: \"image/jpeg\", quality: 0.92 })\n * ```\n *\n * @public\n */\n", + "excerptTokens": [ + { + "kind": "Content", + "text": "export declare function downsizeImage(blob: " + }, + { + "kind": "Reference", + "text": "Blob", + "canonicalReference": "!Blob:interface" + }, + { + "kind": "Content", + "text": ", width: " + }, + { + "kind": "Content", + "text": "number" + }, + { + "kind": "Content", + "text": ", height: " + }, + { + "kind": "Content", + "text": "number" + }, + { + "kind": "Content", + "text": ", opts?: " + }, + { + "kind": "Content", + "text": "{\n type?: string | undefined;\n quality?: number | undefined;\n}" + }, + { + "kind": "Content", + "text": "): " + }, + { + "kind": "Reference", + "text": "Promise", + "canonicalReference": "!Promise:interface" + }, + { + "kind": "Content", + "text": "<" + }, + { + "kind": "Reference", + "text": "Blob", + "canonicalReference": "!Blob:interface" + }, + { + "kind": "Content", + "text": ">" + }, + { + "kind": "Content", + "text": ";" + } + ], + "fileUrlPath": "packages/tldraw/src/lib/utils/assets/assets.ts", + "returnTypeTokenRange": { + "startIndex": 9, + "endIndex": 13 + }, + "releaseTag": "Public", + "overloadIndex": 1, + "parameters": [ + { + "parameterName": "blob", + "parameterTypeTokenRange": { + "startIndex": 1, + "endIndex": 2 + }, + "isOptional": false + }, + { + "parameterName": "width", + "parameterTypeTokenRange": { + "startIndex": 3, + "endIndex": 4 + }, + "isOptional": false + }, + { + "parameterName": "height", + "parameterTypeTokenRange": { + "startIndex": 5, + "endIndex": 6 + }, + "isOptional": false + }, + { + "parameterName": "opts", + "parameterTypeTokenRange": { + "startIndex": 7, + "endIndex": 8 + }, + "isOptional": true + } + ], + "name": "downsizeImage" + }, { "kind": "Class", "canonicalReference": "@tldraw/tldraw!DrawShapeTool:class", @@ -7673,104 +7781,6 @@ ], "name": "getEmbedInfo" }, - { - "kind": "Function", - "canonicalReference": "@tldraw/tldraw!getResizedImageDataUrl:function(1)", - "docComment": "/**\n * Get the size of an image from its source.\n *\n * @param dataURLForImage - The image file as a string.\n *\n * @param width - The desired width.\n *\n * @param height - The desired height.\n *\n * @param opts - Options for the image.\n *\n * @example\n * ```ts\n * const size = await getImageSize('https://example.com/image.jpg')\n * const dataUrl = await getResizedImageDataUrl('https://example.com/image.jpg', size.w, size.h, { type: \"image/jpeg\", quality: 0.92 })\n * ```\n *\n * @public\n */\n", - "excerptTokens": [ - { - "kind": "Content", - "text": "export declare function getResizedImageDataUrl(dataURLForImage: " - }, - { - "kind": "Content", - "text": "string" - }, - { - "kind": "Content", - "text": ", width: " - }, - { - "kind": "Content", - "text": "number" - }, - { - "kind": "Content", - "text": ", height: " - }, - { - "kind": "Content", - "text": "number" - }, - { - "kind": "Content", - "text": ", opts?: " - }, - { - "kind": "Content", - "text": "{\n type?: string | undefined;\n quality?: number | undefined;\n}" - }, - { - "kind": "Content", - "text": "): " - }, - { - "kind": "Reference", - "text": "Promise", - "canonicalReference": "!Promise:interface" - }, - { - "kind": "Content", - "text": "" - }, - { - "kind": "Content", - "text": ";" - } - ], - "fileUrlPath": "packages/tldraw/src/lib/utils/assets/assets.ts", - "returnTypeTokenRange": { - "startIndex": 9, - "endIndex": 11 - }, - "releaseTag": "Public", - "overloadIndex": 1, - "parameters": [ - { - "parameterName": "dataURLForImage", - "parameterTypeTokenRange": { - "startIndex": 1, - "endIndex": 2 - }, - "isOptional": false - }, - { - "parameterName": "width", - "parameterTypeTokenRange": { - "startIndex": 3, - "endIndex": 4 - }, - "isOptional": false - }, - { - "parameterName": "height", - "parameterTypeTokenRange": { - "startIndex": 5, - "endIndex": 6 - }, - "isOptional": false - }, - { - "parameterName": "opts", - "parameterTypeTokenRange": { - "startIndex": 7, - "endIndex": 8 - }, - "isOptional": true - } - ], - "name": "getResizedImageDataUrl" - }, { "kind": "Function", "canonicalReference": "@tldraw/tldraw!getSvgAsImage:function(1)", @@ -9679,8 +9689,8 @@ }, { "kind": "Reference", - "text": "File", - "canonicalReference": "!File:interface" + "text": "Blob", + "canonicalReference": "!Blob:interface" }, { "kind": "Content", diff --git a/packages/tldraw/package.json b/packages/tldraw/package.json index 79497e9f9..3b98a1ca2 100644 --- a/packages/tldraw/package.json +++ b/packages/tldraw/package.json @@ -55,7 +55,6 @@ "@tldraw/editor": "workspace:*", "canvas-size": "^1.2.6", "classnames": "^2.3.2", - "downscale": "^1.0.6", "hotkeys-js": "^3.11.2", "lz-string": "^1.4.4" }, @@ -69,7 +68,6 @@ "@testing-library/react": "^14.0.0", "@types/canvas-size": "^1.2.0", "@types/classnames": "^2.3.1", - "@types/downscale": "^1.0.4", "@types/lz-string": "^1.3.34", "chokidar-cli": "^3.0.0", "jest-canvas-mock": "^2.5.2", diff --git a/packages/tldraw/src/index.ts b/packages/tldraw/src/index.ts index a81f60c7b..de52d3e45 100644 --- a/packages/tldraw/src/index.ts +++ b/packages/tldraw/src/index.ts @@ -147,7 +147,7 @@ export { DEFAULT_ACCEPTED_IMG_TYPE, DEFAULT_ACCEPTED_VID_TYPE, containBoxSize, - getResizedImageDataUrl, + downsizeImage, isGifAnimated, } from './lib/utils/assets/assets' export { getEmbedInfo } from './lib/utils/embeds/embeds' diff --git a/packages/tldraw/src/lib/defaultExternalContentHandlers.ts b/packages/tldraw/src/lib/defaultExternalContentHandlers.ts index 6947aba2f..ba6b4f427 100644 --- a/packages/tldraw/src/lib/defaultExternalContentHandlers.ts +++ b/packages/tldraw/src/lib/defaultExternalContentHandlers.ts @@ -12,12 +12,14 @@ import { TLTextShapeProps, Vec, VecLike, + assert, compact, createShapeId, + getHashForBuffer, getHashForString, } from '@tldraw/editor' import { FONT_FAMILIES, FONT_SIZES, TEXT_PROPS } from './shapes/shared/default-shape-constants' -import { containBoxSize, getResizedImageDataUrl, isGifAnimated } from './utils/assets/assets' +import { containBoxSize, downsizeImage, isGifAnimated } from './utils/assets/assets' import { getEmbedInfo } from './utils/embeds/embeds' import { cleanupText, isRightToLeftLanguage, truncateStringWithEllipsis } from './utils/text/text' @@ -43,87 +45,63 @@ export function registerDefaultExternalContentHandlers( }: TLExternalContentProps ) { // files -> asset - editor.registerExternalAssetHandler('file', async ({ file }) => { - return await new Promise((resolve, reject) => { - if ( - !acceptedImageMimeTypes.includes(file.type) && - !acceptedVideoMimeTypes.includes(file.type) - ) { - console.warn(`File type not allowed: ${file.type}`) - reject() + editor.registerExternalAssetHandler('file', async ({ file: _file }) => { + const name = _file.name + let file: Blob = _file + const isImageType = acceptedImageMimeTypes.includes(file.type) + const isVideoType = acceptedVideoMimeTypes.includes(file.type) + + assert(isImageType || isVideoType, `File type not allowed: ${file.type}`) + assert( + file.size <= maxAssetSize, + `File size too big: ${(file.size / 1024).toFixed()}kb > ${(maxAssetSize / 1024).toFixed()}kb` + ) + + if (file.type === 'video/quicktime') { + // hack to make .mov videos work + file = new Blob([file], { type: 'video/mp4' }) + } + + let size = isImageType + ? await MediaHelpers.getImageSize(file) + : await MediaHelpers.getVideoSize(file) + + const isAnimated = file.type === 'image/gif' ? await isGifAnimated(file) : isVideoType + + const hash = await getHashForBuffer(await file.arrayBuffer()) + + if (isFinite(maxImageDimension)) { + const resizedSize = containBoxSize(size, { w: maxImageDimension, h: maxImageDimension }) + if (size !== resizedSize && (file.type === 'image/jpeg' || file.type === 'image/png')) { + size = resizedSize } + } - if (file.size > maxAssetSize) { - console.warn( - `File size too big: ${(file.size / 1024).toFixed()}kb > ${( - maxAssetSize / 1024 - ).toFixed()}kb` - ) - reject() - } + // Always rescale the image + if (file.type === 'image/jpeg' || file.type === 'image/png') { + file = await downsizeImage(file, size.w, size.h, { + type: file.type, + quality: 0.92, + }) + } - const reader = new FileReader() - reader.onerror = () => reject(reader.error) - reader.onload = async () => { - let dataUrl = reader.result as string + const assetId: TLAssetId = AssetRecordType.createId(hash) - // Hack to make .mov videos work via dataURL. - if (file.type === 'video/quicktime' && dataUrl.includes('video/quicktime')) { - dataUrl = dataUrl.replace('video/quicktime', 'video/mp4') - } - - const isImageType = acceptedImageMimeTypes.includes(file.type) - - let size: { - w: number - h: number - } - let isAnimated: boolean - - if (isImageType) { - size = await MediaHelpers.getImageSizeFromSrc(dataUrl) - isAnimated = file.type === 'image/gif' && (await isGifAnimated(file)) - } else { - isAnimated = true - size = await MediaHelpers.getVideoSizeFromSrc(dataUrl) - } - - if (isFinite(maxImageDimension)) { - const resizedSize = containBoxSize(size, { w: maxImageDimension, h: maxImageDimension }) - if (size !== resizedSize && (file.type === 'image/jpeg' || file.type === 'image/png')) { - size = resizedSize - } - } - - // Always rescale the image - if (file.type === 'image/jpeg' || file.type === 'image/png') { - dataUrl = await getResizedImageDataUrl(dataUrl, size.w, size.h, { - type: file.type, - quality: 0.92, - }) - } - - const assetId: TLAssetId = AssetRecordType.createId(getHashForString(dataUrl)) - - const asset = AssetRecordType.create({ - id: assetId, - type: isImageType ? 'image' : 'video', - typeName: 'asset', - props: { - name: file.name, - src: dataUrl, - w: size.w, - h: size.h, - mimeType: file.type, - isAnimated, - }, - }) - - resolve(asset) - } - - reader.readAsDataURL(file) + const asset = AssetRecordType.create({ + id: assetId, + type: isImageType ? 'image' : 'video', + typeName: 'asset', + props: { + name, + src: await MediaHelpers.blobToDataUrl(file), + w: size.w, + h: size.h, + mimeType: file.type, + isAnimated, + }, }) + + return asset }) // urls -> bookmark asset diff --git a/packages/tldraw/src/lib/shapes/shared/getBrowserCanvasMaxSize.tsx b/packages/tldraw/src/lib/shapes/shared/getBrowserCanvasMaxSize.tsx index ef97218ad..cbfa94e2f 100644 --- a/packages/tldraw/src/lib/shapes/shared/getBrowserCanvasMaxSize.tsx +++ b/packages/tldraw/src/lib/shapes/shared/getBrowserCanvasMaxSize.tsx @@ -26,3 +26,5 @@ async function calculateBrowserCanvasMaxSize(): Promise { maxArea: maxArea.width * maxArea.height, } } + +export const MAX_SAFE_CANVAS_DIMENSION = 8192 diff --git a/packages/tldraw/src/lib/utils/assets.ts b/packages/tldraw/src/lib/utils/assets.ts deleted file mode 100644 index 7fccf30d3..000000000 --- a/packages/tldraw/src/lib/utils/assets.ts +++ /dev/null @@ -1,103 +0,0 @@ -import downscale from 'downscale' -import { getBrowserCanvasMaxSize } from '../shapes/shared/getBrowserCanvasMaxSize' -import { isAnimated } from './assets/is-gif-animated' - -type BoxWidthHeight = { - w: number - h: number -} - -/** - * Contains the size within the given box size - * - * @param originalSize - The size of the asset - * @param containBoxSize - The container size - * @returns Adjusted size - * @public - */ -export function containBoxSize( - originalSize: BoxWidthHeight, - containBoxSize: BoxWidthHeight -): BoxWidthHeight { - const overByXScale = originalSize.w / containBoxSize.w - const overByYScale = originalSize.h / containBoxSize.h - - if (overByXScale <= 1 && overByYScale <= 1) { - return originalSize - } else if (overByXScale > overByYScale) { - return { - w: originalSize.w / overByXScale, - h: originalSize.h / overByXScale, - } - } else { - return { - w: originalSize.w / overByYScale, - h: originalSize.h / overByYScale, - } - } -} - -/** - * Get the size of an image from its source. - * - * @example - * ```ts - * const size = await getImageSize('https://example.com/image.jpg') - * const dataUrl = await getResizedImageDataUrl('https://example.com/image.jpg', size.w, size.h, { type: "image/jpeg", quality: 0.92 }) - * ``` - * - * @param dataURLForImage - The image file as a string. - * @param width - The desired width. - * @param height - The desired height. - * @param opts - Options for the image. - * @public - */ -export async function getResizedImageDataUrl( - dataURLForImage: string, - width: number, - height: number, - opts = {} as { type?: string; quality?: number } -): Promise { - let desiredWidth = width * 2 - let desiredHeight = height * 2 - const { type = 'image/jpeg', quality = 0.92 } = opts - - const canvasSizes = await getBrowserCanvasMaxSize() - - const aspectRatio = width / height - - if (desiredWidth > canvasSizes.maxWidth) { - desiredWidth = canvasSizes.maxWidth - desiredHeight = desiredWidth / aspectRatio - } - - if (desiredHeight > canvasSizes.maxHeight) { - desiredHeight = canvasSizes.maxHeight - desiredWidth = desiredHeight * aspectRatio - } - - if (desiredWidth * desiredHeight > canvasSizes.maxArea) { - const ratio = Math.sqrt(canvasSizes.maxArea / (desiredWidth * desiredHeight)) - desiredWidth *= ratio - desiredHeight *= ratio - } - - return await downscale(dataURLForImage, desiredWidth, desiredHeight, { imageType: type, quality }) -} - -/** @public */ -export const DEFAULT_ACCEPTED_IMG_TYPE = ['image/jpeg', 'image/png', 'image/gif', 'image/svg+xml'] -/** @public */ -export const DEFAULT_ACCEPTED_VID_TYPE = ['video/mp4', 'video/quicktime'] - -/** @public */ -export async function isGifAnimated(file: File): Promise { - return await new Promise((resolve, reject) => { - const reader = new FileReader() - reader.onerror = () => reject(reader.error) - reader.onload = () => { - resolve(reader.result ? isAnimated(reader.result as ArrayBuffer) : false) - } - reader.readAsArrayBuffer(file) - }) -} diff --git a/packages/tldraw/src/lib/utils/assets/assets.ts b/packages/tldraw/src/lib/utils/assets/assets.ts index 9864f5752..1d14af089 100644 --- a/packages/tldraw/src/lib/utils/assets/assets.ts +++ b/packages/tldraw/src/lib/utils/assets/assets.ts @@ -1,5 +1,8 @@ -import downscale from 'downscale' -import { getBrowserCanvasMaxSize } from '../../shapes/shared/getBrowserCanvasMaxSize' +import { MediaHelpers, assertExists } from '@tldraw/editor' +import { + MAX_SAFE_CANVAS_DIMENSION, + getBrowserCanvasMaxSize, +} from '../../shapes/shared/getBrowserCanvasMaxSize' import { isAnimated } from './is-gif-animated' type BoxWidthHeight = { @@ -38,54 +41,77 @@ export function containBoxSize( } /** - * Get the size of an image from its source. + * Resize an image Blob to be smaller than it is currently. * * @example * ```ts - * const size = await getImageSize('https://example.com/image.jpg') - * const dataUrl = await getResizedImageDataUrl('https://example.com/image.jpg', size.w, size.h, { type: "image/jpeg", quality: 0.92 }) + * const image = await (await fetch('/image.jpg')).blob() + * const size = await getImageSize(image) + * const resizedImage = await downsizeImage(image, size.w / 2, size.h / 2, { type: "image/jpeg", quality: 0.92 }) * ``` * - * @param dataURLForImage - The image file as a string. + * @param image - The image Blob. * @param width - The desired width. * @param height - The desired height. * @param opts - Options for the image. * @public */ -export async function getResizedImageDataUrl( - dataURLForImage: string, +export async function downsizeImage( + blob: Blob, width: number, height: number, opts = {} as { type?: string; quality?: number } -): Promise { - let desiredWidth = width * 2 - let desiredHeight = height * 2 - const { type = 'image/jpeg', quality = 0.92 } = opts +): Promise { + const image = await MediaHelpers.usingObjectURL(blob, MediaHelpers.loadImage) + let desiredWidth = Math.min(width * 2, image.naturalWidth) + let desiredHeight = Math.min(height * 2, image.naturalHeight) + const { type = blob.type, quality = 0.92 } = opts - const canvasSizes = await getBrowserCanvasMaxSize() + if (desiredWidth > MAX_SAFE_CANVAS_DIMENSION || desiredHeight > MAX_SAFE_CANVAS_DIMENSION) { + const canvasSizes = await getBrowserCanvasMaxSize() - const aspectRatio = width / height + const aspectRatio = width / height - if (desiredWidth > canvasSizes.maxWidth) { - desiredWidth = canvasSizes.maxWidth - desiredHeight = desiredWidth / aspectRatio + if (desiredWidth > canvasSizes.maxWidth) { + desiredWidth = canvasSizes.maxWidth + desiredHeight = desiredWidth / aspectRatio + } + + if (desiredHeight > canvasSizes.maxHeight) { + desiredHeight = canvasSizes.maxHeight + desiredWidth = desiredHeight * aspectRatio + } + + if (desiredWidth * desiredHeight > canvasSizes.maxArea) { + const ratio = Math.sqrt(canvasSizes.maxArea / (desiredWidth * desiredHeight)) + desiredWidth *= ratio + desiredHeight *= ratio + } } - if (desiredHeight > canvasSizes.maxHeight) { - desiredHeight = canvasSizes.maxHeight - desiredWidth = desiredHeight * aspectRatio - } + const canvas = document.createElement('canvas') + canvas.width = desiredWidth + canvas.height = desiredHeight + const ctx = assertExists( + canvas.getContext('2d', { willReadFrequently: true }), + 'Could not get canvas context' + ) + ctx.imageSmoothingEnabled = true + ctx.imageSmoothingQuality = 'high' + ctx.drawImage(image, 0, 0, desiredWidth, desiredHeight) - if (desiredWidth * desiredHeight > canvasSizes.maxArea) { - const ratio = Math.sqrt(canvasSizes.maxArea / (desiredWidth * desiredHeight)) - desiredWidth *= ratio - desiredHeight *= ratio - } - - return await downscale(dataURLForImage, desiredWidth, desiredHeight, { - // downscale expects the type without the `image/` prefix - imageType: type.replace('image/', ''), - quality, + return new Promise((resolve, reject) => { + canvas.toBlob( + (blob) => { + if (blob) { + resolve(blob) + } else { + reject(new Error('Could not resize image')) + } + }, + type, + quality + ) }) } @@ -95,13 +121,6 @@ export const DEFAULT_ACCEPTED_IMG_TYPE = ['image/jpeg', 'image/png', 'image/gif' export const DEFAULT_ACCEPTED_VID_TYPE = ['video/mp4', 'video/quicktime'] /** @public */ -export async function isGifAnimated(file: File): Promise { - return await new Promise((resolve, reject) => { - const reader = new FileReader() - reader.onerror = () => reject(reader.error) - reader.onload = () => { - resolve(reader.result ? isAnimated(reader.result as ArrayBuffer) : false) - } - reader.readAsArrayBuffer(file) - }) +export async function isGifAnimated(file: Blob): Promise { + return isAnimated(await file.arrayBuffer()) } diff --git a/packages/utils/api-report.md b/packages/utils/api-report.md index 6b095a262..6184f3cec 100644 --- a/packages/utils/api-report.md +++ b/packages/utils/api-report.md @@ -68,6 +68,9 @@ export function getErrorAnnotations(error: Error): ErrorAnnotations; // @public export function getFirstFromIterable(set: Map | Set): T; +// @public +export function getHashForBuffer(buffer: ArrayBuffer): string; + // @public export function getHashForObject(obj: any): string; @@ -124,14 +127,19 @@ export function mapObjectMapValues( // @public export class MediaHelpers { - static getImageSizeFromSrc(dataURL: string): Promise<{ + static blobToDataUrl(blob: Blob): Promise; + static getImageSize(blob: Blob): Promise<{ w: number; h: number; }>; - static getVideoSizeFromSrc(src: string): Promise<{ + static getVideoSize(blob: Blob): Promise<{ w: number; h: number; }>; + static loadImage(src: string): Promise; + static loadVideo(src: string): Promise; + // (undocumented) + static usingObjectURL(blob: Blob, fn: (url: string) => Promise): Promise; } // @internal (undocumented) diff --git a/packages/utils/api/api.json b/packages/utils/api/api.json index f02515946..6ef8756aa 100644 --- a/packages/utils/api/api.json +++ b/packages/utils/api/api.json @@ -654,6 +654,52 @@ ], "name": "getFirstFromIterable" }, + { + "kind": "Function", + "canonicalReference": "@tldraw/utils!getHashForBuffer:function(1)", + "docComment": "/**\n * Hash an ArrayBuffer using the FNV-1a algorithm.\n *\n * @public\n */\n", + "excerptTokens": [ + { + "kind": "Content", + "text": "export declare function getHashForBuffer(buffer: " + }, + { + "kind": "Reference", + "text": "ArrayBuffer", + "canonicalReference": "!ArrayBuffer:interface" + }, + { + "kind": "Content", + "text": "): " + }, + { + "kind": "Content", + "text": "string" + }, + { + "kind": "Content", + "text": ";" + } + ], + "fileUrlPath": "packages/utils/src/lib/hash.ts", + "returnTypeTokenRange": { + "startIndex": 3, + "endIndex": 4 + }, + "releaseTag": "Public", + "overloadIndex": 1, + "parameters": [ + { + "parameterName": "buffer", + "parameterTypeTokenRange": { + "startIndex": 1, + "endIndex": 2 + }, + "isOptional": false + } + ], + "name": "getHashForBuffer" + }, { "kind": "Function", "canonicalReference": "@tldraw/utils!getHashForObject:function(1)", @@ -1246,16 +1292,71 @@ "members": [ { "kind": "Method", - "canonicalReference": "@tldraw/utils!MediaHelpers.getImageSizeFromSrc:member(1)", - "docComment": "/**\n * Get the size of an image from its source.\n *\n * @param dataURL - The file as a string.\n *\n * @public\n */\n", + "canonicalReference": "@tldraw/utils!MediaHelpers.blobToDataUrl:member(1)", + "docComment": "/**\n * Read a blob into a data url\n *\n * @public\n */\n", "excerptTokens": [ { "kind": "Content", - "text": "static getImageSizeFromSrc(dataURL: " + "text": "static blobToDataUrl(blob: " + }, + { + "kind": "Reference", + "text": "Blob", + "canonicalReference": "!Blob:interface" }, { "kind": "Content", - "text": "string" + "text": "): " + }, + { + "kind": "Reference", + "text": "Promise", + "canonicalReference": "!Promise:interface" + }, + { + "kind": "Content", + "text": "" + }, + { + "kind": "Content", + "text": ";" + } + ], + "isStatic": true, + "returnTypeTokenRange": { + "startIndex": 3, + "endIndex": 5 + }, + "releaseTag": "Public", + "isProtected": false, + "overloadIndex": 1, + "parameters": [ + { + "parameterName": "blob", + "parameterTypeTokenRange": { + "startIndex": 1, + "endIndex": 2 + }, + "isOptional": false + } + ], + "isOptional": false, + "isAbstract": false, + "name": "blobToDataUrl" + }, + { + "kind": "Method", + "canonicalReference": "@tldraw/utils!MediaHelpers.getImageSize:member(1)", + "docComment": "/**\n * Get the size of an image blob\n *\n * @param dataURL - A Blob containing the image.\n *\n * @public\n */\n", + "excerptTokens": [ + { + "kind": "Content", + "text": "static getImageSize(blob: " + }, + { + "kind": "Reference", + "text": "Blob", + "canonicalReference": "!Blob:interface" }, { "kind": "Content", @@ -1285,7 +1386,7 @@ "overloadIndex": 1, "parameters": [ { - "parameterName": "dataURL", + "parameterName": "blob", "parameterTypeTokenRange": { "startIndex": 1, "endIndex": 2 @@ -1295,20 +1396,21 @@ ], "isOptional": false, "isAbstract": false, - "name": "getImageSizeFromSrc" + "name": "getImageSize" }, { "kind": "Method", - "canonicalReference": "@tldraw/utils!MediaHelpers.getVideoSizeFromSrc:member(1)", - "docComment": "/**\n * Get the size of a video from its source.\n *\n * @param src - The source of the video.\n *\n * @public\n */\n", + "canonicalReference": "@tldraw/utils!MediaHelpers.getVideoSize:member(1)", + "docComment": "/**\n * Get the size of a video blob\n *\n * @param src - A SharedBlob containing the video\n *\n * @public\n */\n", "excerptTokens": [ { "kind": "Content", - "text": "static getVideoSizeFromSrc(src: " + "text": "static getVideoSize(blob: " }, { - "kind": "Content", - "text": "string" + "kind": "Reference", + "text": "Blob", + "canonicalReference": "!Blob:interface" }, { "kind": "Content", @@ -1336,6 +1438,68 @@ "releaseTag": "Public", "isProtected": false, "overloadIndex": 1, + "parameters": [ + { + "parameterName": "blob", + "parameterTypeTokenRange": { + "startIndex": 1, + "endIndex": 2 + }, + "isOptional": false + } + ], + "isOptional": false, + "isAbstract": false, + "name": "getVideoSize" + }, + { + "kind": "Method", + "canonicalReference": "@tldraw/utils!MediaHelpers.loadImage:member(1)", + "docComment": "/**\n * Load an image from a url.\n *\n * @public\n */\n", + "excerptTokens": [ + { + "kind": "Content", + "text": "static loadImage(src: " + }, + { + "kind": "Content", + "text": "string" + }, + { + "kind": "Content", + "text": "): " + }, + { + "kind": "Reference", + "text": "Promise", + "canonicalReference": "!Promise:interface" + }, + { + "kind": "Content", + "text": "<" + }, + { + "kind": "Reference", + "text": "HTMLImageElement", + "canonicalReference": "!HTMLImageElement:interface" + }, + { + "kind": "Content", + "text": ">" + }, + { + "kind": "Content", + "text": ";" + } + ], + "isStatic": true, + "returnTypeTokenRange": { + "startIndex": 3, + "endIndex": 7 + }, + "releaseTag": "Public", + "isProtected": false, + "overloadIndex": 1, "parameters": [ { "parameterName": "src", @@ -1348,7 +1512,161 @@ ], "isOptional": false, "isAbstract": false, - "name": "getVideoSizeFromSrc" + "name": "loadImage" + }, + { + "kind": "Method", + "canonicalReference": "@tldraw/utils!MediaHelpers.loadVideo:member(1)", + "docComment": "/**\n * Load a video from a url.\n *\n * @public\n */\n", + "excerptTokens": [ + { + "kind": "Content", + "text": "static loadVideo(src: " + }, + { + "kind": "Content", + "text": "string" + }, + { + "kind": "Content", + "text": "): " + }, + { + "kind": "Reference", + "text": "Promise", + "canonicalReference": "!Promise:interface" + }, + { + "kind": "Content", + "text": "<" + }, + { + "kind": "Reference", + "text": "HTMLVideoElement", + "canonicalReference": "!HTMLVideoElement:interface" + }, + { + "kind": "Content", + "text": ">" + }, + { + "kind": "Content", + "text": ";" + } + ], + "isStatic": true, + "returnTypeTokenRange": { + "startIndex": 3, + "endIndex": 7 + }, + "releaseTag": "Public", + "isProtected": false, + "overloadIndex": 1, + "parameters": [ + { + "parameterName": "src", + "parameterTypeTokenRange": { + "startIndex": 1, + "endIndex": 2 + }, + "isOptional": false + } + ], + "isOptional": false, + "isAbstract": false, + "name": "loadVideo" + }, + { + "kind": "Method", + "canonicalReference": "@tldraw/utils!MediaHelpers.usingObjectURL:member(1)", + "docComment": "", + "excerptTokens": [ + { + "kind": "Content", + "text": "static usingObjectURL(blob: " + }, + { + "kind": "Reference", + "text": "Blob", + "canonicalReference": "!Blob:interface" + }, + { + "kind": "Content", + "text": ", fn: " + }, + { + "kind": "Content", + "text": "(url: string) => " + }, + { + "kind": "Reference", + "text": "Promise", + "canonicalReference": "!Promise:interface" + }, + { + "kind": "Content", + "text": "" + }, + { + "kind": "Content", + "text": "): " + }, + { + "kind": "Reference", + "text": "Promise", + "canonicalReference": "!Promise:interface" + }, + { + "kind": "Content", + "text": "" + }, + { + "kind": "Content", + "text": ";" + } + ], + "typeParameters": [ + { + "typeParameterName": "T", + "constraintTokenRange": { + "startIndex": 0, + "endIndex": 0 + }, + "defaultTypeTokenRange": { + "startIndex": 0, + "endIndex": 0 + } + } + ], + "isStatic": true, + "returnTypeTokenRange": { + "startIndex": 7, + "endIndex": 9 + }, + "releaseTag": "Public", + "isProtected": false, + "overloadIndex": 1, + "parameters": [ + { + "parameterName": "blob", + "parameterTypeTokenRange": { + "startIndex": 1, + "endIndex": 2 + }, + "isOptional": false + }, + { + "parameterName": "fn", + "parameterTypeTokenRange": { + "startIndex": 3, + "endIndex": 6 + }, + "isOptional": false + } + ], + "isOptional": false, + "isAbstract": false, + "name": "usingObjectURL" } ], "implementsTokenRanges": [] diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index dcfd8ec12..e68407b18 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -20,7 +20,7 @@ export { debounce } from './lib/debounce' export { annotateError, getErrorAnnotations } from './lib/error' export { FileHelpers } from './lib/file' export { noop, omitFromStackTrace, throttle } from './lib/function' -export { getHashForObject, getHashForString, lns } from './lib/hash' +export { getHashForBuffer, getHashForObject, getHashForString, lns } from './lib/hash' export { getFirstFromIterable } from './lib/iterable' export type { JsonArray, JsonObject, JsonPrimitive, JsonValue } from './lib/json-value' export { MediaHelpers } from './lib/media' diff --git a/packages/utils/src/lib/hash.ts b/packages/utils/src/lib/hash.ts index 5e22c8bba..6937f2b5c 100644 --- a/packages/utils/src/lib/hash.ts +++ b/packages/utils/src/lib/hash.ts @@ -21,6 +21,21 @@ export function getHashForObject(obj: any) { return getHashForString(JSON.stringify(obj)) } +/** + * Hash an ArrayBuffer using the FNV-1a algorithm. + * + * @public + */ +export function getHashForBuffer(buffer: ArrayBuffer) { + const view = new DataView(buffer) + let hash = 0 + for (let i = 0; i < view.byteLength; i++) { + hash = (hash << 5) - hash + view.getUint8(i) + hash |= 0 // Convert to 32bit integer + } + return hash + '' +} + /** @public */ export function lns(str: string) { const result = str.split('') diff --git a/packages/utils/src/lib/media.ts b/packages/utils/src/lib/media.ts index 7204ce3ee..5deb32b37 100644 --- a/packages/utils/src/lib/media.ts +++ b/packages/utils/src/lib/media.ts @@ -1,4 +1,3 @@ -import { FileHelpers } from './file' import { PngHelpers } from './png' /** @@ -8,18 +7,16 @@ import { PngHelpers } from './png' */ export class MediaHelpers { /** - * Get the size of a video from its source. - * - * @param src - The source of the video. + * Load a video from a url. * @public */ - static async getVideoSizeFromSrc(src: string): Promise<{ w: number; h: number }> { - return await new Promise((resolve, reject) => { + static loadVideo(src: string): Promise { + return new Promise((resolve, reject) => { const video = document.createElement('video') - video.onloadeddata = () => resolve({ w: video.videoWidth, h: video.videoHeight }) + video.onloadeddata = () => resolve(video) video.onerror = (e) => { console.error(e) - reject(new Error('Could not get video size')) + reject(new Error('Could not load video')) } video.crossOrigin = 'anonymous' video.src = src @@ -27,45 +24,90 @@ export class MediaHelpers { } /** - * Get the size of an image from its source. - * - * @param dataURL - The file as a string. + * Load an image from a url. * @public */ - static async getImageSizeFromSrc(dataURL: string): Promise<{ w: number; h: number }> { - return await new Promise((resolve, reject) => { + static loadImage(src: string): Promise { + return new Promise((resolve, reject) => { const img = new Image() - img.onload = async () => { - try { - const blob = await FileHelpers.base64ToFile(dataURL) - const view = new DataView(blob) - if (PngHelpers.isPng(view, 0)) { - const physChunk = PngHelpers.findChunk(view, 'pHYs') - if (physChunk) { - const physData = PngHelpers.parsePhys(view, physChunk.dataOffset) - if (physData.unit === 0 && physData.ppux === physData.ppuy) { - const pixelRatio = Math.max(physData.ppux / 2834.5, 1) - resolve({ - w: Math.round(img.width / pixelRatio), - h: Math.round(img.height / pixelRatio), - }) - return + img.onload = () => resolve(img) + img.onerror = (e) => { + console.error(e) + reject(new Error('Could not load image')) + } + img.crossOrigin = 'anonymous' + img.src = src + }) + } + + /** + * Read a blob into a data url + * @public + */ + static blobToDataUrl(blob: Blob): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => resolve(reader.result as string) + reader.onerror = (e) => { + console.error(e) + reject(new Error('Could not read blob')) + } + reader.readAsDataURL(blob) + }) + } + + /** + * Get the size of a video blob + * + * @param src - A SharedBlob containing the video + * @public + */ + static async getVideoSize(blob: Blob): Promise<{ w: number; h: number }> { + return MediaHelpers.usingObjectURL(blob, async (url) => { + const video = await MediaHelpers.loadVideo(url) + return { w: video.videoWidth, h: video.videoHeight } + }) + } + + /** + * Get the size of an image blob + * + * @param dataURL - A Blob containing the image. + * @public + */ + static async getImageSize(blob: Blob): Promise<{ w: number; h: number }> { + const image = await MediaHelpers.usingObjectURL(blob, MediaHelpers.loadImage) + + try { + if (blob.type === 'image/png') { + const view = new DataView(await blob.arrayBuffer()) + if (PngHelpers.isPng(view, 0)) { + const physChunk = PngHelpers.findChunk(view, 'pHYs') + if (physChunk) { + const physData = PngHelpers.parsePhys(view, physChunk.dataOffset) + if (physData.unit === 0 && physData.ppux === physData.ppuy) { + const pixelRatio = Math.max(physData.ppux / 2834.5, 1) + return { + w: Math.round(image.naturalWidth / pixelRatio), + h: Math.round(image.naturalHeight / pixelRatio), } } } - - resolve({ w: img.width, h: img.height }) - } catch (err) { - console.error(err) - resolve({ w: img.width, h: img.height }) } } - img.onerror = (err) => { - console.error(err) - reject(new Error('Could not get image size')) - } - img.crossOrigin = 'anonymous' - img.src = dataURL - }) + } catch (err) { + console.error(err) + return { w: image.naturalWidth, h: image.naturalHeight } + } + return { w: image.naturalWidth, h: image.naturalHeight } + } + + static async usingObjectURL(blob: Blob, fn: (url: string) => Promise): Promise { + const url = URL.createObjectURL(blob) + try { + return await fn(url) + } finally { + URL.revokeObjectURL(url) + } } } diff --git a/public-yarn.lock b/public-yarn.lock index 654e83186..521856c12 100644 --- a/public-yarn.lock +++ b/public-yarn.lock @@ -4562,12 +4562,10 @@ __metadata: "@tldraw/editor": "workspace:*" "@types/canvas-size": ^1.2.0 "@types/classnames": ^2.3.1 - "@types/downscale": ^1.0.4 "@types/lz-string": ^1.3.34 canvas-size: ^1.2.6 chokidar-cli: ^3.0.0 classnames: ^2.3.2 - downscale: ^1.0.6 hotkeys-js: ^3.11.2 jest-canvas-mock: ^2.5.2 jest-environment-jsdom: ^28.1.2 @@ -4831,13 +4829,6 @@ __metadata: languageName: node linkType: hard -"@types/downscale@npm:^1.0.4": - version: 1.0.4 - resolution: "@types/downscale@npm:1.0.4" - checksum: b29ca279b616e06896c01795effa5c483e243e85a137462f98eb43dec3598acab3f72b0637225a4087080e0326cf4bde89ba21d0473a9254a7661c5fd29f98d9 - languageName: node - linkType: hard - "@types/estree-jsx@npm:^0.0.1": version: 0.0.1 resolution: "@types/estree-jsx@npm:0.0.1" @@ -8015,13 +8006,6 @@ __metadata: languageName: node linkType: hard -"downscale@npm:^1.0.6": - version: 1.0.6 - resolution: "downscale@npm:1.0.6" - checksum: f69beb8fe7711964b259ef3d3502fd9f5dd0c4472aa279537dc58d69dcff8a619c77b0209db65841aab014930b97e82f92d43558acc1e569407b8fbc63fb04ee - languageName: node - linkType: hard - "duplexify@npm:^3.5.0, duplexify@npm:^3.6.0": version: 3.7.1 resolution: "duplexify@npm:3.7.1"