faster image processing in default asset handler (#2441)
 (gif is with 6x CPU throttling to make the effect more visible) This is the first of a few diffs I'm working on to make dropping images onto the canvas feel a lot faster. There are three main changes here: 1. We operate on `Blob`s and `File`s rather than data urls. This saves a fair bit on converting to/from base64 all the time. I've updated our `MediaHelper` APIs to encourage the same in consumers. 2. We only check the max canvas size (slow) if images are above a certain dimension that we consider "safe" (8k x 8k) 3. Switching from the `downscale` npm library to canvas native downscaling. that library claims to give better results than the browser, but hasn't been updated in ~7 years. in modern browsers, we can opt-in to native high-quality image smoothing to achieve similar results much faster than with an algorithm implemented in pure JS. I want to follow this up with a system to show image placeholders whilst we're waiting for long-running operations like resizing etc but i'm going to split that out into its own diff as it'll involve some fairly complex changes to the history management API. ### Change Type - [x] `major` — Breaking change [^1]: publishes a `patch` release, for devDependencies use `internal` [^2]: will not publish a new version ### Test Plan 1. Tested manually, unit tests & end-to-end tests pass
This commit is contained in:
alex
GitHub
parent
7902dc65c3
commit
3c1aee492a
15 changed files with 674 additions and 403 deletions
|
|
@ -68,6 +68,9 @@ export function getErrorAnnotations(error: Error): ErrorAnnotations;
|
|||
// @public
|
||||
export function getFirstFromIterable<T = unknown>(set: Map<any, T> | Set<T>): T;
|
||||
|
||||
// @public
|
||||
export function getHashForBuffer(buffer: ArrayBuffer): string;
|
||||
|
||||
// @public
|
||||
export function getHashForObject(obj: any): string;
|
||||
|
||||
|
|
@ -124,14 +127,19 @@ export function mapObjectMapValues<Key extends string, ValueBefore, ValueAfter>(
|
|||
|
||||
// @public
|
||||
export class MediaHelpers {
|
||||
static getImageSizeFromSrc(dataURL: string): Promise<{
|
||||
static blobToDataUrl(blob: Blob): Promise<string>;
|
||||
static getImageSize(blob: Blob): Promise<{
|
||||
w: number;
|
||||
h: number;
|
||||
}>;
|
||||
static getVideoSizeFromSrc(src: string): Promise<{
|
||||
static getVideoSize(blob: Blob): Promise<{
|
||||
w: number;
|
||||
h: number;
|
||||
}>;
|
||||
static loadImage(src: string): Promise<HTMLImageElement>;
|
||||
static loadVideo(src: string): Promise<HTMLVideoElement>;
|
||||
// (undocumented)
|
||||
static usingObjectURL<T>(blob: Blob, fn: (url: string) => Promise<T>): Promise<T>;
|
||||
}
|
||||
|
||||
// @internal (undocumented)
|
||||
|
|
|
|||
|
|
@ -654,6 +654,52 @@
|
|||
],
|
||||
"name": "getFirstFromIterable"
|
||||
},
|
||||
{
|
||||
"kind": "Function",
|
||||
"canonicalReference": "@tldraw/utils!getHashForBuffer:function(1)",
|
||||
"docComment": "/**\n * Hash an ArrayBuffer using the FNV-1a algorithm.\n *\n * @public\n */\n",
|
||||
"excerptTokens": [
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "export declare function getHashForBuffer(buffer: "
|
||||
},
|
||||
{
|
||||
"kind": "Reference",
|
||||
"text": "ArrayBuffer",
|
||||
"canonicalReference": "!ArrayBuffer:interface"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "): "
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "string"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": ";"
|
||||
}
|
||||
],
|
||||
"fileUrlPath": "packages/utils/src/lib/hash.ts",
|
||||
"returnTypeTokenRange": {
|
||||
"startIndex": 3,
|
||||
"endIndex": 4
|
||||
},
|
||||
"releaseTag": "Public",
|
||||
"overloadIndex": 1,
|
||||
"parameters": [
|
||||
{
|
||||
"parameterName": "buffer",
|
||||
"parameterTypeTokenRange": {
|
||||
"startIndex": 1,
|
||||
"endIndex": 2
|
||||
},
|
||||
"isOptional": false
|
||||
}
|
||||
],
|
||||
"name": "getHashForBuffer"
|
||||
},
|
||||
{
|
||||
"kind": "Function",
|
||||
"canonicalReference": "@tldraw/utils!getHashForObject:function(1)",
|
||||
|
|
@ -1246,16 +1292,71 @@
|
|||
"members": [
|
||||
{
|
||||
"kind": "Method",
|
||||
"canonicalReference": "@tldraw/utils!MediaHelpers.getImageSizeFromSrc:member(1)",
|
||||
"docComment": "/**\n * Get the size of an image from its source.\n *\n * @param dataURL - The file as a string.\n *\n * @public\n */\n",
|
||||
"canonicalReference": "@tldraw/utils!MediaHelpers.blobToDataUrl:member(1)",
|
||||
"docComment": "/**\n * Read a blob into a data url\n *\n * @public\n */\n",
|
||||
"excerptTokens": [
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "static getImageSizeFromSrc(dataURL: "
|
||||
"text": "static blobToDataUrl(blob: "
|
||||
},
|
||||
{
|
||||
"kind": "Reference",
|
||||
"text": "Blob",
|
||||
"canonicalReference": "!Blob:interface"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "string"
|
||||
"text": "): "
|
||||
},
|
||||
{
|
||||
"kind": "Reference",
|
||||
"text": "Promise",
|
||||
"canonicalReference": "!Promise:interface"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "<string>"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": ";"
|
||||
}
|
||||
],
|
||||
"isStatic": true,
|
||||
"returnTypeTokenRange": {
|
||||
"startIndex": 3,
|
||||
"endIndex": 5
|
||||
},
|
||||
"releaseTag": "Public",
|
||||
"isProtected": false,
|
||||
"overloadIndex": 1,
|
||||
"parameters": [
|
||||
{
|
||||
"parameterName": "blob",
|
||||
"parameterTypeTokenRange": {
|
||||
"startIndex": 1,
|
||||
"endIndex": 2
|
||||
},
|
||||
"isOptional": false
|
||||
}
|
||||
],
|
||||
"isOptional": false,
|
||||
"isAbstract": false,
|
||||
"name": "blobToDataUrl"
|
||||
},
|
||||
{
|
||||
"kind": "Method",
|
||||
"canonicalReference": "@tldraw/utils!MediaHelpers.getImageSize:member(1)",
|
||||
"docComment": "/**\n * Get the size of an image blob\n *\n * @param dataURL - A Blob containing the image.\n *\n * @public\n */\n",
|
||||
"excerptTokens": [
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "static getImageSize(blob: "
|
||||
},
|
||||
{
|
||||
"kind": "Reference",
|
||||
"text": "Blob",
|
||||
"canonicalReference": "!Blob:interface"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
|
|
@ -1285,7 +1386,7 @@
|
|||
"overloadIndex": 1,
|
||||
"parameters": [
|
||||
{
|
||||
"parameterName": "dataURL",
|
||||
"parameterName": "blob",
|
||||
"parameterTypeTokenRange": {
|
||||
"startIndex": 1,
|
||||
"endIndex": 2
|
||||
|
|
@ -1295,20 +1396,21 @@
|
|||
],
|
||||
"isOptional": false,
|
||||
"isAbstract": false,
|
||||
"name": "getImageSizeFromSrc"
|
||||
"name": "getImageSize"
|
||||
},
|
||||
{
|
||||
"kind": "Method",
|
||||
"canonicalReference": "@tldraw/utils!MediaHelpers.getVideoSizeFromSrc:member(1)",
|
||||
"docComment": "/**\n * Get the size of a video from its source.\n *\n * @param src - The source of the video.\n *\n * @public\n */\n",
|
||||
"canonicalReference": "@tldraw/utils!MediaHelpers.getVideoSize:member(1)",
|
||||
"docComment": "/**\n * Get the size of a video blob\n *\n * @param src - A SharedBlob containing the video\n *\n * @public\n */\n",
|
||||
"excerptTokens": [
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "static getVideoSizeFromSrc(src: "
|
||||
"text": "static getVideoSize(blob: "
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "string"
|
||||
"kind": "Reference",
|
||||
"text": "Blob",
|
||||
"canonicalReference": "!Blob:interface"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
|
|
@ -1336,6 +1438,68 @@
|
|||
"releaseTag": "Public",
|
||||
"isProtected": false,
|
||||
"overloadIndex": 1,
|
||||
"parameters": [
|
||||
{
|
||||
"parameterName": "blob",
|
||||
"parameterTypeTokenRange": {
|
||||
"startIndex": 1,
|
||||
"endIndex": 2
|
||||
},
|
||||
"isOptional": false
|
||||
}
|
||||
],
|
||||
"isOptional": false,
|
||||
"isAbstract": false,
|
||||
"name": "getVideoSize"
|
||||
},
|
||||
{
|
||||
"kind": "Method",
|
||||
"canonicalReference": "@tldraw/utils!MediaHelpers.loadImage:member(1)",
|
||||
"docComment": "/**\n * Load an image from a url.\n *\n * @public\n */\n",
|
||||
"excerptTokens": [
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "static loadImage(src: "
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "string"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "): "
|
||||
},
|
||||
{
|
||||
"kind": "Reference",
|
||||
"text": "Promise",
|
||||
"canonicalReference": "!Promise:interface"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "<"
|
||||
},
|
||||
{
|
||||
"kind": "Reference",
|
||||
"text": "HTMLImageElement",
|
||||
"canonicalReference": "!HTMLImageElement:interface"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": ">"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": ";"
|
||||
}
|
||||
],
|
||||
"isStatic": true,
|
||||
"returnTypeTokenRange": {
|
||||
"startIndex": 3,
|
||||
"endIndex": 7
|
||||
},
|
||||
"releaseTag": "Public",
|
||||
"isProtected": false,
|
||||
"overloadIndex": 1,
|
||||
"parameters": [
|
||||
{
|
||||
"parameterName": "src",
|
||||
|
|
@ -1348,7 +1512,161 @@
|
|||
],
|
||||
"isOptional": false,
|
||||
"isAbstract": false,
|
||||
"name": "getVideoSizeFromSrc"
|
||||
"name": "loadImage"
|
||||
},
|
||||
{
|
||||
"kind": "Method",
|
||||
"canonicalReference": "@tldraw/utils!MediaHelpers.loadVideo:member(1)",
|
||||
"docComment": "/**\n * Load a video from a url.\n *\n * @public\n */\n",
|
||||
"excerptTokens": [
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "static loadVideo(src: "
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "string"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "): "
|
||||
},
|
||||
{
|
||||
"kind": "Reference",
|
||||
"text": "Promise",
|
||||
"canonicalReference": "!Promise:interface"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "<"
|
||||
},
|
||||
{
|
||||
"kind": "Reference",
|
||||
"text": "HTMLVideoElement",
|
||||
"canonicalReference": "!HTMLVideoElement:interface"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": ">"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": ";"
|
||||
}
|
||||
],
|
||||
"isStatic": true,
|
||||
"returnTypeTokenRange": {
|
||||
"startIndex": 3,
|
||||
"endIndex": 7
|
||||
},
|
||||
"releaseTag": "Public",
|
||||
"isProtected": false,
|
||||
"overloadIndex": 1,
|
||||
"parameters": [
|
||||
{
|
||||
"parameterName": "src",
|
||||
"parameterTypeTokenRange": {
|
||||
"startIndex": 1,
|
||||
"endIndex": 2
|
||||
},
|
||||
"isOptional": false
|
||||
}
|
||||
],
|
||||
"isOptional": false,
|
||||
"isAbstract": false,
|
||||
"name": "loadVideo"
|
||||
},
|
||||
{
|
||||
"kind": "Method",
|
||||
"canonicalReference": "@tldraw/utils!MediaHelpers.usingObjectURL:member(1)",
|
||||
"docComment": "",
|
||||
"excerptTokens": [
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "static usingObjectURL<T>(blob: "
|
||||
},
|
||||
{
|
||||
"kind": "Reference",
|
||||
"text": "Blob",
|
||||
"canonicalReference": "!Blob:interface"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": ", fn: "
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "(url: string) => "
|
||||
},
|
||||
{
|
||||
"kind": "Reference",
|
||||
"text": "Promise",
|
||||
"canonicalReference": "!Promise:interface"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "<T>"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "): "
|
||||
},
|
||||
{
|
||||
"kind": "Reference",
|
||||
"text": "Promise",
|
||||
"canonicalReference": "!Promise:interface"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": "<T>"
|
||||
},
|
||||
{
|
||||
"kind": "Content",
|
||||
"text": ";"
|
||||
}
|
||||
],
|
||||
"typeParameters": [
|
||||
{
|
||||
"typeParameterName": "T",
|
||||
"constraintTokenRange": {
|
||||
"startIndex": 0,
|
||||
"endIndex": 0
|
||||
},
|
||||
"defaultTypeTokenRange": {
|
||||
"startIndex": 0,
|
||||
"endIndex": 0
|
||||
}
|
||||
}
|
||||
],
|
||||
"isStatic": true,
|
||||
"returnTypeTokenRange": {
|
||||
"startIndex": 7,
|
||||
"endIndex": 9
|
||||
},
|
||||
"releaseTag": "Public",
|
||||
"isProtected": false,
|
||||
"overloadIndex": 1,
|
||||
"parameters": [
|
||||
{
|
||||
"parameterName": "blob",
|
||||
"parameterTypeTokenRange": {
|
||||
"startIndex": 1,
|
||||
"endIndex": 2
|
||||
},
|
||||
"isOptional": false
|
||||
},
|
||||
{
|
||||
"parameterName": "fn",
|
||||
"parameterTypeTokenRange": {
|
||||
"startIndex": 3,
|
||||
"endIndex": 6
|
||||
},
|
||||
"isOptional": false
|
||||
}
|
||||
],
|
||||
"isOptional": false,
|
||||
"isAbstract": false,
|
||||
"name": "usingObjectURL"
|
||||
}
|
||||
],
|
||||
"implementsTokenRanges": []
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ export { debounce } from './lib/debounce'
|
|||
export { annotateError, getErrorAnnotations } from './lib/error'
|
||||
export { FileHelpers } from './lib/file'
|
||||
export { noop, omitFromStackTrace, throttle } from './lib/function'
|
||||
export { getHashForObject, getHashForString, lns } from './lib/hash'
|
||||
export { getHashForBuffer, getHashForObject, getHashForString, lns } from './lib/hash'
|
||||
export { getFirstFromIterable } from './lib/iterable'
|
||||
export type { JsonArray, JsonObject, JsonPrimitive, JsonValue } from './lib/json-value'
|
||||
export { MediaHelpers } from './lib/media'
|
||||
|
|
|
|||
|
|
@ -21,6 +21,21 @@ export function getHashForObject(obj: any) {
|
|||
return getHashForString(JSON.stringify(obj))
|
||||
}
|
||||
|
||||
/**
|
||||
* Hash an ArrayBuffer using the FNV-1a algorithm.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export function getHashForBuffer(buffer: ArrayBuffer) {
|
||||
const view = new DataView(buffer)
|
||||
let hash = 0
|
||||
for (let i = 0; i < view.byteLength; i++) {
|
||||
hash = (hash << 5) - hash + view.getUint8(i)
|
||||
hash |= 0 // Convert to 32bit integer
|
||||
}
|
||||
return hash + ''
|
||||
}
|
||||
|
||||
/** @public */
|
||||
export function lns(str: string) {
|
||||
const result = str.split('')
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import { FileHelpers } from './file'
|
||||
import { PngHelpers } from './png'
|
||||
|
||||
/**
|
||||
|
|
@ -8,18 +7,16 @@ import { PngHelpers } from './png'
|
|||
*/
|
||||
export class MediaHelpers {
|
||||
/**
|
||||
* Get the size of a video from its source.
|
||||
*
|
||||
* @param src - The source of the video.
|
||||
* Load a video from a url.
|
||||
* @public
|
||||
*/
|
||||
static async getVideoSizeFromSrc(src: string): Promise<{ w: number; h: number }> {
|
||||
return await new Promise((resolve, reject) => {
|
||||
static loadVideo(src: string): Promise<HTMLVideoElement> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const video = document.createElement('video')
|
||||
video.onloadeddata = () => resolve({ w: video.videoWidth, h: video.videoHeight })
|
||||
video.onloadeddata = () => resolve(video)
|
||||
video.onerror = (e) => {
|
||||
console.error(e)
|
||||
reject(new Error('Could not get video size'))
|
||||
reject(new Error('Could not load video'))
|
||||
}
|
||||
video.crossOrigin = 'anonymous'
|
||||
video.src = src
|
||||
|
|
@ -27,45 +24,90 @@ export class MediaHelpers {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get the size of an image from its source.
|
||||
*
|
||||
* @param dataURL - The file as a string.
|
||||
* Load an image from a url.
|
||||
* @public
|
||||
*/
|
||||
static async getImageSizeFromSrc(dataURL: string): Promise<{ w: number; h: number }> {
|
||||
return await new Promise((resolve, reject) => {
|
||||
static loadImage(src: string): Promise<HTMLImageElement> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const img = new Image()
|
||||
img.onload = async () => {
|
||||
try {
|
||||
const blob = await FileHelpers.base64ToFile(dataURL)
|
||||
const view = new DataView(blob)
|
||||
if (PngHelpers.isPng(view, 0)) {
|
||||
const physChunk = PngHelpers.findChunk(view, 'pHYs')
|
||||
if (physChunk) {
|
||||
const physData = PngHelpers.parsePhys(view, physChunk.dataOffset)
|
||||
if (physData.unit === 0 && physData.ppux === physData.ppuy) {
|
||||
const pixelRatio = Math.max(physData.ppux / 2834.5, 1)
|
||||
resolve({
|
||||
w: Math.round(img.width / pixelRatio),
|
||||
h: Math.round(img.height / pixelRatio),
|
||||
})
|
||||
return
|
||||
img.onload = () => resolve(img)
|
||||
img.onerror = (e) => {
|
||||
console.error(e)
|
||||
reject(new Error('Could not load image'))
|
||||
}
|
||||
img.crossOrigin = 'anonymous'
|
||||
img.src = src
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a blob into a data url
|
||||
* @public
|
||||
*/
|
||||
static blobToDataUrl(blob: Blob): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader()
|
||||
reader.onload = () => resolve(reader.result as string)
|
||||
reader.onerror = (e) => {
|
||||
console.error(e)
|
||||
reject(new Error('Could not read blob'))
|
||||
}
|
||||
reader.readAsDataURL(blob)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the size of a video blob
|
||||
*
|
||||
* @param src - A SharedBlob containing the video
|
||||
* @public
|
||||
*/
|
||||
static async getVideoSize(blob: Blob): Promise<{ w: number; h: number }> {
|
||||
return MediaHelpers.usingObjectURL(blob, async (url) => {
|
||||
const video = await MediaHelpers.loadVideo(url)
|
||||
return { w: video.videoWidth, h: video.videoHeight }
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the size of an image blob
|
||||
*
|
||||
* @param dataURL - A Blob containing the image.
|
||||
* @public
|
||||
*/
|
||||
static async getImageSize(blob: Blob): Promise<{ w: number; h: number }> {
|
||||
const image = await MediaHelpers.usingObjectURL(blob, MediaHelpers.loadImage)
|
||||
|
||||
try {
|
||||
if (blob.type === 'image/png') {
|
||||
const view = new DataView(await blob.arrayBuffer())
|
||||
if (PngHelpers.isPng(view, 0)) {
|
||||
const physChunk = PngHelpers.findChunk(view, 'pHYs')
|
||||
if (physChunk) {
|
||||
const physData = PngHelpers.parsePhys(view, physChunk.dataOffset)
|
||||
if (physData.unit === 0 && physData.ppux === physData.ppuy) {
|
||||
const pixelRatio = Math.max(physData.ppux / 2834.5, 1)
|
||||
return {
|
||||
w: Math.round(image.naturalWidth / pixelRatio),
|
||||
h: Math.round(image.naturalHeight / pixelRatio),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resolve({ w: img.width, h: img.height })
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
resolve({ w: img.width, h: img.height })
|
||||
}
|
||||
}
|
||||
img.onerror = (err) => {
|
||||
console.error(err)
|
||||
reject(new Error('Could not get image size'))
|
||||
}
|
||||
img.crossOrigin = 'anonymous'
|
||||
img.src = dataURL
|
||||
})
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return { w: image.naturalWidth, h: image.naturalHeight }
|
||||
}
|
||||
return { w: image.naturalWidth, h: image.naturalHeight }
|
||||
}
|
||||
|
||||
static async usingObjectURL<T>(blob: Blob, fn: (url: string) => Promise<T>): Promise<T> {
|
||||
const url = URL.createObjectURL(blob)
|
||||
try {
|
||||
return await fn(url)
|
||||
} finally {
|
||||
URL.revokeObjectURL(url)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue