From d2d3e582e5c71bb15a710ed890270db728971ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mime=20=C4=8Cuvalo?= Date: Mon, 13 May 2024 09:29:43 +0100 Subject: [PATCH] assets: rework mime-type detection to be consistent/centralized; add support for webp/webm, apng, avif (#3730) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As I started working on image LOD stuff and wrapping my head around the codebase, this was bothering me. - there are missing popular types, especially WebP - there are places where we're copy/pasting the same list of types but they can get out-of-date with each other (also, one place described supporting webm but we didn't actually do that) This adds animated apng/avif detection as well (alongside our animated gif detection). Furthermore, it moves the gif logic to be alongside the png logic (they were in separate packages unnecessarily) ### Change Type - [x] `sdk` — Changes the tldraw SDK - [ ] `dotcom` — Changes the tldraw.com web app - [ ] `docs` — Changes to the documentation, examples, or templates. - [ ] `vs code` — Changes to the vscode plugin - [ ] `internal` — Does not affect user-facing stuff - [ ] `bugfix` — Bug fix - [ ] `feature` — New feature - [x] `improvement` — Improving existing features - [ ] `chore` — Updating dependencies, other boring stuff - [ ] `galaxy brain` — Architectural changes - [ ] `tests` — Changes to any test code - [ ] `tools` — Changes to infrastructure, CI, internal scripts, debugging tools, etc. - [ ] `dunno` — I don't know ### Release Notes - Images: unify list of acceptable types and expand to include webp, webm, apng, avif --- apps/dotcom/src/hooks/useMultiplayerAssets.ts | 5 +- apps/dotcom/src/utils/createAssetFromFile.ts | 5 +- .../hosted-images/HostedImagesExample.tsx | 5 +- .../examples/image-annotator/ImagePicker.tsx | 4 +- packages/tldraw/api-report.md | 13 +- packages/tldraw/src/index.ts | 8 +- packages/tldraw/src/lib/Tldraw.tsx | 15 +- .../src/lib/defaultExternalContentHandlers.ts | 12 +- .../src/lib/shapes/image/ImageShapeUtil.tsx | 24 ++- .../childStates/Crop/children/Idle.ts | 6 + .../tldraw/src/lib/ui/hooks/useInsertMedia.ts | 4 +- .../tldraw/src/lib/utils/assets/assets.ts | 11 -- .../tldraw/src/lib/utils/export/export.ts | 5 +- packages/utils/api-report.md | 17 ++ packages/utils/src/index.ts | 9 +- packages/utils/src/lib/media/apng.ts | 150 ++++++++++++++++++ packages/utils/src/lib/media/avif.ts | 4 + .../src/lib/media/gif.ts} | 2 +- packages/utils/src/lib/{ => media}/media.ts | 68 ++++++++ packages/utils/src/lib/{ => media}/png.ts | 6 +- packages/utils/src/lib/media/webp.ts | 25 +++ 21 files changed, 327 insertions(+), 71 deletions(-) create mode 100644 packages/utils/src/lib/media/apng.ts create mode 100644 packages/utils/src/lib/media/avif.ts rename packages/{tldraw/src/lib/utils/assets/is-gif-animated.ts => utils/src/lib/media/gif.ts} (97%) rename packages/utils/src/lib/{ => media}/media.ts (57%) rename packages/utils/src/lib/{ => media}/png.ts (96%) create mode 100644 packages/utils/src/lib/media/webp.ts diff --git a/apps/dotcom/src/hooks/useMultiplayerAssets.ts b/apps/dotcom/src/hooks/useMultiplayerAssets.ts index 796fad9ea..b6c184d27 100644 --- a/apps/dotcom/src/hooks/useMultiplayerAssets.ts +++ b/apps/dotcom/src/hooks/useMultiplayerAssets.ts @@ -1,7 +1,6 @@ import { useCallback } from 'react' import { AssetRecordType, - DEFAULT_ACCEPTED_IMG_TYPE, MediaHelpers, TLAsset, TLAssetId, @@ -25,7 +24,7 @@ export function useMultiplayerAssets(assetUploaderUrl: string) { const assetId: TLAssetId = AssetRecordType.createId(getHashForString(url)) - const isImageType = DEFAULT_ACCEPTED_IMG_TYPE.includes(file.type) + const isImageType = MediaHelpers.isImageType(file.type) let size: { w: number @@ -35,7 +34,7 @@ export function useMultiplayerAssets(assetUploaderUrl: string) { if (isImageType) { size = await MediaHelpers.getImageSize(file) - if (file.type === 'image/gif') { + if (MediaHelpers.isAnimatedImageType(file.type)) { isAnimated = true // await getIsGifAnimated(file) todo export me from editor } else { isAnimated = false diff --git a/apps/dotcom/src/utils/createAssetFromFile.ts b/apps/dotcom/src/utils/createAssetFromFile.ts index 47be943ba..b6004b117 100644 --- a/apps/dotcom/src/utils/createAssetFromFile.ts +++ b/apps/dotcom/src/utils/createAssetFromFile.ts @@ -1,6 +1,5 @@ import { AssetRecordType, - DEFAULT_ACCEPTED_IMG_TYPE, MediaHelpers, TLAsset, TLAssetId, @@ -23,7 +22,7 @@ export async function createAssetFromFile({ file }: { type: 'file'; file: File } const assetId: TLAssetId = AssetRecordType.createId(getHashForString(url)) - const isImageType = DEFAULT_ACCEPTED_IMG_TYPE.includes(file.type) + const isImageType = MediaHelpers.isImageType(file.type) let size: { w: number @@ -33,7 +32,7 @@ export async function createAssetFromFile({ file }: { type: 'file'; file: File } if (isImageType) { size = await MediaHelpers.getImageSize(file) - if (file.type === 'image/gif') { + if (MediaHelpers.isAnimatedImageType(file.type)) { isAnimated = true // await getIsGifAnimated(file) todo export me from editor } else { isAnimated = false diff --git a/apps/examples/src/examples/hosted-images/HostedImagesExample.tsx b/apps/examples/src/examples/hosted-images/HostedImagesExample.tsx index 5f86696de..12eee0901 100644 --- a/apps/examples/src/examples/hosted-images/HostedImagesExample.tsx +++ b/apps/examples/src/examples/hosted-images/HostedImagesExample.tsx @@ -7,7 +7,6 @@ import { TLAssetId, Tldraw, getHashForString, - isGifAnimated, uniqueId, } from 'tldraw' import 'tldraw/tldraw.css' @@ -40,10 +39,10 @@ export default function HostedImagesExample() { let shapeType: 'image' | 'video' //[c] - if (['image/jpeg', 'image/png', 'image/gif', 'image/svg+xml'].includes(file.type)) { + if (MediaHelpers.isImageType(file.type)) { shapeType = 'image' size = await MediaHelpers.getImageSize(file) - isAnimated = file.type === 'image/gif' && (await isGifAnimated(file)) + isAnimated = await MediaHelpers.isAnimated(file) } else { shapeType = 'video' isAnimated = true diff --git a/apps/examples/src/examples/image-annotator/ImagePicker.tsx b/apps/examples/src/examples/image-annotator/ImagePicker.tsx index 7769ca6b3..711cd6b3f 100644 --- a/apps/examples/src/examples/image-annotator/ImagePicker.tsx +++ b/apps/examples/src/examples/image-annotator/ImagePicker.tsx @@ -1,5 +1,5 @@ import { useState } from 'react' -import { FileHelpers, MediaHelpers } from 'tldraw' +import { DEFAULT_SUPPORTED_MEDIA_TYPE_LIST, FileHelpers, MediaHelpers } from 'tldraw' import anakin from './assets/anakin.jpeg' import distractedBf from './assets/distracted-bf.jpeg' import expandingBrain from './assets/expanding-brain.png' @@ -13,7 +13,7 @@ export function ImagePicker({ function onClickChooseImage() { const input = window.document.createElement('input') input.type = 'file' - input.accept = 'image/jpeg,image/png,image/gif,image/svg+xml,video/mp4,video/quicktime' + input.accept = DEFAULT_SUPPORTED_MEDIA_TYPE_LIST input.addEventListener('change', async (e) => { const fileList = (e.target as HTMLInputElement).files if (!fileList || fileList.length === 0) return diff --git a/packages/tldraw/api-report.md b/packages/tldraw/api-report.md index 9e046e89a..e5a364429 100644 --- a/packages/tldraw/api-report.md +++ b/packages/tldraw/api-report.md @@ -328,12 +328,6 @@ export function CutMenuItem(): JSX_2.Element; // @public (undocumented) export function DebugFlags(): JSX_2.Element | null; -// @public (undocumented) -export const DEFAULT_ACCEPTED_IMG_TYPE: string[]; - -// @public (undocumented) -export const DEFAULT_ACCEPTED_VID_TYPE: string[]; - // @public (undocumented) export const DefaultActionsMenu: NamedExoticComponent; @@ -586,7 +580,7 @@ export function ExportFileContentSubMenu(): JSX_2.Element; // @public export function exportToBlob({ editor, ids, format, opts, }: { editor: Editor; - format: 'jpeg' | 'json' | 'png' | 'svg' | 'webp'; + format: TLExportType; ids: TLShapeId[]; opts?: Partial; }): Promise; @@ -951,6 +945,8 @@ export class ImageShapeUtil extends BaseBoxShapeUtil { // (undocumented) indicator(shape: TLImageShape): JSX_2.Element | null; // (undocumented) + isAnimated(shape: TLImageShape): boolean; + // (undocumented) isAspectRatioLocked: () => boolean; // (undocumented) static migrations: TLPropsMigrations; @@ -976,9 +972,6 @@ export class ImageShapeUtil extends BaseBoxShapeUtil { static type: "image"; } -// @public (undocumented) -export function isGifAnimated(file: Blob): Promise; - // @public (undocumented) export function KeyboardShortcutsMenuItem(): JSX_2.Element | null; diff --git a/packages/tldraw/src/index.ts b/packages/tldraw/src/index.ts index c54422b70..72524d59f 100644 --- a/packages/tldraw/src/index.ts +++ b/packages/tldraw/src/index.ts @@ -115,13 +115,7 @@ export { } from './lib/ui/hooks/useTranslation/useTranslation' export { type TLUiIconType } from './lib/ui/icon-types' export { useDefaultHelpers, type TLUiOverrides } from './lib/ui/overrides' -export { - DEFAULT_ACCEPTED_IMG_TYPE, - DEFAULT_ACCEPTED_VID_TYPE, - containBoxSize, - downsizeImage, - isGifAnimated, -} from './lib/utils/assets/assets' +export { containBoxSize, downsizeImage } from './lib/utils/assets/assets' export { getEmbedInfo } from './lib/utils/embeds/embeds' export { copyAs } from './lib/utils/export/copyAs' export { exportToBlob, getSvgAsImage } from './lib/utils/export/export' diff --git a/packages/tldraw/src/lib/Tldraw.tsx b/packages/tldraw/src/lib/Tldraw.tsx index 86c2d3f0c..85b0a3ff9 100644 --- a/packages/tldraw/src/lib/Tldraw.tsx +++ b/packages/tldraw/src/lib/Tldraw.tsx @@ -1,4 +1,6 @@ import { + DEFAULT_SUPPORTED_IMAGE_TYPES, + DEFAULT_SUPPORT_VIDEO_TYPES, Editor, ErrorScreen, Expand, @@ -148,21 +150,12 @@ export function Tldraw(props: TldrawProps) { ) } -const defaultAcceptedImageMimeTypes = Object.freeze([ - 'image/jpeg', - 'image/png', - 'image/gif', - 'image/svg+xml', -]) - -const defaultAcceptedVideoMimeTypes = Object.freeze(['video/mp4', 'video/quicktime']) - // We put these hooks into a component here so that they can run inside of the context provided by TldrawEditor and TldrawUi. function InsideOfEditorAndUiContext({ maxImageDimension = 1000, maxAssetSize = 10 * 1024 * 1024, // 10mb - acceptedImageMimeTypes = defaultAcceptedImageMimeTypes, - acceptedVideoMimeTypes = defaultAcceptedVideoMimeTypes, + acceptedImageMimeTypes = DEFAULT_SUPPORTED_IMAGE_TYPES, + acceptedVideoMimeTypes = DEFAULT_SUPPORT_VIDEO_TYPES, onMount, }: Partial) { const editor = useEditor() diff --git a/packages/tldraw/src/lib/defaultExternalContentHandlers.ts b/packages/tldraw/src/lib/defaultExternalContentHandlers.ts index 647792061..18bcad4c5 100644 --- a/packages/tldraw/src/lib/defaultExternalContentHandlers.ts +++ b/packages/tldraw/src/lib/defaultExternalContentHandlers.ts @@ -22,7 +22,7 @@ import { import { FONT_FAMILIES, FONT_SIZES, TEXT_PROPS } from './shapes/shared/default-shape-constants' import { TLUiToastsContextType } from './ui/context/toasts' import { useTranslation } from './ui/hooks/useTranslation/useTranslation' -import { containBoxSize, downsizeImage, isGifAnimated } from './utils/assets/assets' +import { containBoxSize, downsizeImage } from './utils/assets/assets' import { getEmbedInfo } from './utils/embeds/embeds' import { cleanupText, isRightToLeftLanguage, truncateStringWithEllipsis } from './utils/text/text' @@ -32,9 +32,9 @@ export type TLExternalContentProps = { maxImageDimension: number // The maximum size (in bytes) of an asset. Assets larger than this will be rejected. Defaults to 10mb (10 * 1024 * 1024). maxAssetSize: number - // The mime types of images that are allowed to be handled. Defaults to ['image/jpeg', 'image/png', 'image/gif', 'image/svg+xml']. + // The mime types of images that are allowed to be handled. Defaults to DEFAULT_SUPPORTED_IMAGE_TYPES. acceptedImageMimeTypes: readonly string[] - // The mime types of videos that are allowed to be handled. Defaults to ['video/mp4', 'video/webm', 'video/quicktime']. + // The mime types of videos that are allowed to be handled. Defaults to DEFAULT_SUPPORT_VIDEO_TYPES. acceptedVideoMimeTypes: readonly string[] } @@ -70,19 +70,19 @@ export function registerDefaultExternalContentHandlers( ? await MediaHelpers.getImageSize(file) : await MediaHelpers.getVideoSize(file) - const isAnimated = file.type === 'image/gif' ? await isGifAnimated(file) : isVideoType + const isAnimated = (await MediaHelpers.isAnimated(file)) || isVideoType const hash = await getHashForBuffer(await file.arrayBuffer()) if (isFinite(maxImageDimension)) { const resizedSize = containBoxSize(size, { w: maxImageDimension, h: maxImageDimension }) - if (size !== resizedSize && (file.type === 'image/jpeg' || file.type === 'image/png')) { + if (size !== resizedSize && MediaHelpers.isStaticImageType(file.type)) { size = resizedSize } } // Always rescale the image - if (file.type === 'image/jpeg' || file.type === 'image/png') { + if (!isAnimated && MediaHelpers.isStaticImageType(file.type)) { file = await downsizeImage(file, size.w, size.h, { type: file.type, quality: 0.92, diff --git a/packages/tldraw/src/lib/shapes/image/ImageShapeUtil.tsx b/packages/tldraw/src/lib/shapes/image/ImageShapeUtil.tsx index 4d2ae51a6..a69c9a660 100644 --- a/packages/tldraw/src/lib/shapes/image/ImageShapeUtil.tsx +++ b/packages/tldraw/src/lib/shapes/image/ImageShapeUtil.tsx @@ -3,6 +3,7 @@ import { BaseBoxShapeUtil, FileHelpers, HTMLContainer, + MediaHelpers, TLImageShape, TLOnDoubleClickHandler, TLShapePartial, @@ -43,6 +44,17 @@ export class ImageShapeUtil extends BaseBoxShapeUtil { } } + isAnimated(shape: TLImageShape) { + const asset = shape.props.assetId ? this.editor.getAsset(shape.props.assetId) : undefined + + if (!asset) return false + + return ( + ('mimeType' in asset.props && MediaHelpers.isAnimatedImageType(asset?.props.mimeType)) || + ('isAnimated' in asset.props && asset.props.isAnimated) + ) + } + component(shape: TLImageShape) { const isCropping = this.editor.getCroppingShapeId() === shape.id const prefersReducedMotion = usePrefersReducedMotion() @@ -53,7 +65,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil { const isSelected = shape.id === this.editor.getOnlySelectedShapeId() useEffect(() => { - if (asset?.props.src && 'mimeType' in asset.props && asset?.props.mimeType === 'image/gif') { + if (asset?.props.src && this.isAnimated(shape)) { let cancelled = false const url = asset.props.src if (!url) return @@ -79,7 +91,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil { cancelled = true } } - }, [prefersReducedMotion, asset?.props]) + }, [prefersReducedMotion, asset?.props, shape]) if (asset?.type === 'bookmark') { throw Error("Bookmark assets can't be rendered as images") @@ -92,8 +104,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil { // We only want to reduce motion for mimeTypes that have motion const reduceMotion = - prefersReducedMotion && - (asset?.props.mimeType?.includes('video') || asset?.props.mimeType?.includes('gif')) + prefersReducedMotion && (asset?.props.mimeType?.includes('video') || this.isAnimated(shape)) const containerStyle = getCroppedContainerStyle(shape) @@ -151,7 +162,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil { }} draggable={false} /> - {asset.props.isAnimated && !shape.props.playing && ( + {this.isAnimated(shape) && !shape.props.playing && (
GIF
)} @@ -218,8 +229,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil { if (!asset) return - const canPlay = - asset.props.src && 'mimeType' in asset.props && asset.props.mimeType === 'image/gif' + const canPlay = asset.props.src && this.isAnimated(shape) if (!canPlay) return diff --git a/packages/tldraw/src/lib/tools/SelectTool/childStates/Crop/children/Idle.ts b/packages/tldraw/src/lib/tools/SelectTool/childStates/Crop/children/Idle.ts index 0c1a2493f..6b22a5abf 100644 --- a/packages/tldraw/src/lib/tools/SelectTool/childStates/Crop/children/Idle.ts +++ b/packages/tldraw/src/lib/tools/SelectTool/childStates/Crop/children/Idle.ts @@ -133,7 +133,13 @@ export class Idle extends StateNode { if (info.target === 'selection') { util.onDoubleClickEdge?.(shape) + return } + + // If the user double clicks the canvas, we want to cancel cropping, + // especially if it's an animated image, we want the image to continue playing. + this.cancel() + this.editor.root.handleEvent(info) } override onKeyDown: TLEventHandlers['onKeyDown'] = () => { diff --git a/packages/tldraw/src/lib/ui/hooks/useInsertMedia.ts b/packages/tldraw/src/lib/ui/hooks/useInsertMedia.ts index e8ded7bb1..05ef04fda 100644 --- a/packages/tldraw/src/lib/ui/hooks/useInsertMedia.ts +++ b/packages/tldraw/src/lib/ui/hooks/useInsertMedia.ts @@ -1,4 +1,4 @@ -import { useEditor } from '@tldraw/editor' +import { DEFAULT_SUPPORTED_MEDIA_TYPE_LIST, useEditor } from '@tldraw/editor' import { useCallback, useEffect, useRef } from 'react' export function useInsertMedia() { @@ -8,7 +8,7 @@ export function useInsertMedia() { useEffect(() => { const input = window.document.createElement('input') input.type = 'file' - input.accept = 'image/jpeg,image/png,image/gif,image/svg+xml,video/mp4,video/quicktime' + input.accept = DEFAULT_SUPPORTED_MEDIA_TYPE_LIST input.multiple = true inputRef.current = input async function onchange(e: Event) { diff --git a/packages/tldraw/src/lib/utils/assets/assets.ts b/packages/tldraw/src/lib/utils/assets/assets.ts index c20e7f00c..9c9b62160 100644 --- a/packages/tldraw/src/lib/utils/assets/assets.ts +++ b/packages/tldraw/src/lib/utils/assets/assets.ts @@ -1,6 +1,5 @@ import { MediaHelpers, assertExists } from '@tldraw/editor' import { clampToBrowserMaxCanvasSize } from '../../shapes/shared/getBrowserCanvasMaxSize' -import { isAnimated } from './is-gif-animated' type BoxWidthHeight = { w: number @@ -91,13 +90,3 @@ export async function downsizeImage( ) }) } - -/** @public */ -export const DEFAULT_ACCEPTED_IMG_TYPE = ['image/jpeg', 'image/png', 'image/gif', 'image/svg+xml'] -/** @public */ -export const DEFAULT_ACCEPTED_VID_TYPE = ['video/mp4', 'video/quicktime'] - -/** @public */ -export async function isGifAnimated(file: Blob): Promise { - return isAnimated(await file.arrayBuffer()) -} diff --git a/packages/tldraw/src/lib/utils/export/export.ts b/packages/tldraw/src/lib/utils/export/export.ts index d68c76767..45bd1034a 100644 --- a/packages/tldraw/src/lib/utils/export/export.ts +++ b/packages/tldraw/src/lib/utils/export/export.ts @@ -7,6 +7,7 @@ import { exhaustiveSwitchError, } from '@tldraw/editor' import { clampToBrowserMaxCanvasSize } from '../../shapes/shared/getBrowserCanvasMaxSize' +import { TLExportType } from './exportAs' /** @public */ export async function getSvgAsImage( @@ -143,7 +144,7 @@ export async function exportToBlob({ }: { editor: Editor ids: TLShapeId[] - format: 'svg' | 'png' | 'jpeg' | 'webp' | 'json' + format: TLExportType opts?: Partial }): Promise { switch (format) { @@ -185,7 +186,7 @@ const mimeTypeByFormat = { export function exportToBlobPromise( editor: Editor, ids: TLShapeId[], - format: 'svg' | 'png' | 'jpeg' | 'webp' | 'json', + format: TLExportType, opts = {} as Partial ): { blobPromise: Promise; mimeType: string } { return { diff --git a/packages/utils/api-report.md b/packages/utils/api-report.md index 66d93dfcb..d3b698a95 100644 --- a/packages/utils/api-report.md +++ b/packages/utils/api-report.md @@ -37,6 +37,15 @@ export function debounce(callback: (...args: T) => Promi // @public export function dedupe(input: T[], equals?: (a: any, b: any) => boolean): T[]; +// @public (undocumented) +export const DEFAULT_SUPPORT_VIDEO_TYPES: readonly string[]; + +// @public (undocumented) +export const DEFAULT_SUPPORTED_IMAGE_TYPES: readonly string[]; + +// @public (undocumented) +export const DEFAULT_SUPPORTED_MEDIA_TYPE_LIST: string; + // @internal export function deleteFromLocalStorage(key: string): void; @@ -195,6 +204,14 @@ export class MediaHelpers { h: number; w: number; }>; + // (undocumented) + static isAnimated(file: Blob): Promise; + // (undocumented) + static isAnimatedImageType(mimeType: null | string): boolean; + // (undocumented) + static isImageType(mimeType: string): boolean; + // (undocumented) + static isStaticImageType(mimeType: null | string): boolean; static loadImage(src: string): Promise; static loadVideo(src: string): Promise; // (undocumented) diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 0f25b217e..5c7affa53 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -25,7 +25,13 @@ export { noop, omitFromStackTrace, throttle } from './lib/function' export { getHashForBuffer, getHashForObject, getHashForString, lns } from './lib/hash' export { getFirstFromIterable } from './lib/iterable' export type { JsonArray, JsonObject, JsonPrimitive, JsonValue } from './lib/json-value' -export { MediaHelpers } from './lib/media' +export { + DEFAULT_SUPPORTED_IMAGE_TYPES, + DEFAULT_SUPPORTED_MEDIA_TYPE_LIST, + DEFAULT_SUPPORT_VIDEO_TYPES, + MediaHelpers, +} from './lib/media/media' +export { PngHelpers } from './lib/media/png' export { invLerp, lerp, modulate, rng } from './lib/number' export { areObjectsShallowEqual, @@ -39,7 +45,6 @@ export { objectMapValues, } from './lib/object' export { measureAverageDuration, measureCbDuration, measureDuration } from './lib/perf' -export { PngHelpers } from './lib/png' export { type IndexKey } from './lib/reordering/IndexKey' export { ZERO_INDEX_KEY, diff --git a/packages/utils/src/lib/media/apng.ts b/packages/utils/src/lib/media/apng.ts new file mode 100644 index 000000000..2a5ff1571 --- /dev/null +++ b/packages/utils/src/lib/media/apng.ts @@ -0,0 +1,150 @@ +/*! + * MIT License: https://github.com/vHeemstra/is-apng/blob/main/license + * Copyright (c) Philip van Heemstra + */ + +export function isApngAnimated(buffer: ArrayBuffer): boolean { + const view = new Uint8Array(buffer) + + if ( + !view || + !((typeof Buffer !== 'undefined' && Buffer.isBuffer(view)) || view instanceof Uint8Array) || + view.length < 16 + ) { + return false + } + + const isPNG = + view[0] === 0x89 && + view[1] === 0x50 && + view[2] === 0x4e && + view[3] === 0x47 && + view[4] === 0x0d && + view[5] === 0x0a && + view[6] === 0x1a && + view[7] === 0x0a + + if (!isPNG) { + return false + } + + /** + * Returns the index of the first occurrence of a sequence in an typed array, or -1 if it is not present. + * + * Works similar to `Array.prototype.indexOf()`, but it searches for a sequence of array values (bytes). + * The bytes in the `haystack` array are decoded (UTF-8) and then used to search for `needle`. + * + * @param haystack `Uint8Array` + * Array to search in. + * + * @param needle `string | RegExp` + * The value to locate in the array. + * + * @param fromIndex `number` + * The array index at which to begin the search. + * + * @param upToIndex `number` + * The array index up to which to search. + * If omitted, search until the end. + * + * @param chunksize `number` + * Size of the chunks used when searching (default 1024). + * + * @returns boolean + * Whether the array holds Animated PNG data. + */ + function indexOfSubstring( + haystack: Uint8Array, + needle: string | RegExp, + fromIndex: number, + upToIndex?: number, + chunksize = 1024 /* Bytes */ + ) { + /** + * Adopted from: https://stackoverflow.com/a/67771214/2142071 + */ + + if (!needle) { + return -1 + } + needle = new RegExp(needle, 'g') + + // The needle could get split over two chunks. + // So, at every chunk we prepend the last few characters + // of the last chunk. + const needle_length = needle.source.length + const decoder = new TextDecoder() + + // Handle search offset in line with + // `Array.prototype.indexOf()` and `TypedArray.prototype.subarray()`. + const full_haystack_length = haystack.length + if (typeof upToIndex === 'undefined') { + upToIndex = full_haystack_length + } + if (fromIndex >= full_haystack_length || upToIndex <= 0 || fromIndex >= upToIndex) { + return -1 + } + haystack = haystack.subarray(fromIndex, upToIndex) + + let position = -1 + let current_index = 0 + let full_length = 0 + let needle_buffer = '' + + outer: while (current_index < haystack.length) { + const next_index = current_index + chunksize + // subarray doesn't copy + const chunk = haystack.subarray(current_index, next_index) + const decoded = decoder.decode(chunk, { stream: true }) + + const text = needle_buffer + decoded + + let match: RegExpExecArray | null + let last_index = -1 + while ((match = needle.exec(text)) !== null) { + last_index = match.index - needle_buffer.length + position = full_length + last_index + break outer + } + + current_index = next_index + full_length += decoded.length + + // Check that the buffer doesn't itself include the needle + // this would cause duplicate finds (we could also use a Set to avoid that). + const needle_index = + last_index > -1 ? last_index + needle_length : decoded.length - needle_length + needle_buffer = decoded.slice(needle_index) + } + + // Correct for search offset. + if (position >= 0) { + position += fromIndex >= 0 ? fromIndex : full_haystack_length + fromIndex + } + + return position + } + + // APNGs have an animation control chunk ('acTL') preceding the IDATs. + // See: https://en.wikipedia.org/wiki/APNG#File_format + const idatIdx = indexOfSubstring(view, 'IDAT', 12) + if (idatIdx >= 12) { + const actlIdx = indexOfSubstring(view, 'acTL', 8, idatIdx) + return actlIdx >= 8 + } + + return false +} + +// globalThis.isApng = isApng + +// (new TextEncoder()).encode('IDAT') +// Decimal: [73, 68, 65, 84] +// Hex: [0x49, 0x44, 0x41, 0x54] + +// (new TextEncoder()).encode('acTL') +// Decimal: [97, 99, 84, 76] +// Hex: [0x61, 0x63, 0x54, 0x4C] + +// const idatIdx = buffer.indexOf('IDAT') +// const actlIdx = buffer.indexOf('acTL') diff --git a/packages/utils/src/lib/media/avif.ts b/packages/utils/src/lib/media/avif.ts new file mode 100644 index 000000000..1ec3bb484 --- /dev/null +++ b/packages/utils/src/lib/media/avif.ts @@ -0,0 +1,4 @@ +export const isAvifAnimated = (buffer: ArrayBuffer) => { + const view = new Uint8Array(buffer) + return view[3] === 44 +} diff --git a/packages/tldraw/src/lib/utils/assets/is-gif-animated.ts b/packages/utils/src/lib/media/gif.ts similarity index 97% rename from packages/tldraw/src/lib/utils/assets/is-gif-animated.ts rename to packages/utils/src/lib/media/gif.ts index 50658cb01..4bcdbfaa9 100644 --- a/packages/tldraw/src/lib/utils/assets/is-gif-animated.ts +++ b/packages/utils/src/lib/media/gif.ts @@ -31,7 +31,7 @@ export function isGIF(buffer: ArrayBuffer): boolean { * * @public */ -export function isAnimated(buffer: ArrayBuffer): boolean { +export function isGifAnimated(buffer: ArrayBuffer): boolean { const view = new Uint8Array(buffer) let hasColorTable, colorTableSize let offset = 0 diff --git a/packages/utils/src/lib/media.ts b/packages/utils/src/lib/media/media.ts similarity index 57% rename from packages/utils/src/lib/media.ts rename to packages/utils/src/lib/media/media.ts index 50cf01ad9..b89172dbc 100644 --- a/packages/utils/src/lib/media.ts +++ b/packages/utils/src/lib/media/media.ts @@ -1,4 +1,40 @@ +import { isApngAnimated } from './apng' +import { isAvifAnimated } from './avif' +import { isGifAnimated } from './gif' import { PngHelpers } from './png' +import { isWebpAnimated } from './webp' + +/** @public */ +export const DEFAULT_SUPPORTED_VECTOR_IMAGE_TYPES = Object.freeze(['image/svg+xml']) +/** @public */ +export const DEFAULT_SUPPORTED_STATIC_IMAGE_TYPES = Object.freeze([ + 'image/jpeg', + 'image/png', + 'image/webp', +]) +/** @public */ +export const DEFAULT_SUPPORTED_ANIMATED_IMAGE_TYPES = Object.freeze([ + 'image/gif', + 'image/apng', + 'image/avif', +]) +/** @public */ +export const DEFAULT_SUPPORTED_IMAGE_TYPES = Object.freeze([ + ...DEFAULT_SUPPORTED_STATIC_IMAGE_TYPES, + ...DEFAULT_SUPPORTED_VECTOR_IMAGE_TYPES, + ...DEFAULT_SUPPORTED_ANIMATED_IMAGE_TYPES, +]) +/** @public */ +export const DEFAULT_SUPPORT_VIDEO_TYPES = Object.freeze([ + 'video/mp4', + 'video/webm', + 'video/quicktime', +]) +/** @public */ +export const DEFAULT_SUPPORTED_MEDIA_TYPE_LIST = [ + ...DEFAULT_SUPPORTED_IMAGE_TYPES, + ...DEFAULT_SUPPORT_VIDEO_TYPES, +].join(',') /** * Helpers for media @@ -86,6 +122,38 @@ export class MediaHelpers { return { w: image.naturalWidth, h: image.naturalHeight } } + static async isAnimated(file: Blob): Promise { + if (file.type === 'image/gif') { + return isGifAnimated(await file.arrayBuffer()) + } + + if (file.type === 'image/avif') { + return isAvifAnimated(await file.arrayBuffer()) + } + + if (file.type === 'image/webp') { + return isWebpAnimated(await file.arrayBuffer()) + } + + if (file.type === 'image/apng') { + return isApngAnimated(await file.arrayBuffer()) + } + + return false + } + + static isAnimatedImageType(mimeType: string | null): boolean { + return DEFAULT_SUPPORTED_ANIMATED_IMAGE_TYPES.includes(mimeType || '') + } + + static isStaticImageType(mimeType: string | null): boolean { + return DEFAULT_SUPPORTED_STATIC_IMAGE_TYPES.includes(mimeType || '') + } + + static isImageType(mimeType: string): boolean { + return DEFAULT_SUPPORTED_IMAGE_TYPES.includes(mimeType) + } + static async usingObjectURL(blob: Blob, fn: (url: string) => Promise): Promise { const url = URL.createObjectURL(blob) try { diff --git a/packages/utils/src/lib/png.ts b/packages/utils/src/lib/media/png.ts similarity index 96% rename from packages/utils/src/lib/png.ts rename to packages/utils/src/lib/media/png.ts index b1bbd5703..3cbb3dfc1 100644 --- a/packages/utils/src/lib/png.ts +++ b/packages/utils/src/lib/media/png.ts @@ -43,7 +43,11 @@ if (typeof Int32Array !== 'undefined') { TABLE = new Int32Array(TABLE) } -// crc32, https://github.com/alexgorbatchev/crc/blob/master/src/calculators/crc32.ts +/*! + * MIT License: https://github.com/alexgorbatchev/crc/blob/master/LICENSE + * Copyright: 2014 Alex Gorbatchev + * Code: crc32, https://github.com/alexgorbatchev/crc/blob/master/src/calculators/crc32.ts + */ const crc: CRCCalculator = (current, previous) => { let crc = previous === 0 ? 0 : ~~previous! ^ -1 diff --git a/packages/utils/src/lib/media/webp.ts b/packages/utils/src/lib/media/webp.ts new file mode 100644 index 000000000..f74c5d576 --- /dev/null +++ b/packages/utils/src/lib/media/webp.ts @@ -0,0 +1,25 @@ +/*! + * MIT License: https://github.com/sindresorhus/is-webp/blob/main/license + * Copyright (c) Sindre Sorhus (https://sindresorhus.com) + */ +function isWebp(view: Uint8Array) { + if (!view || view.length < 12) { + return false + } + + return view[8] === 87 && view[9] === 69 && view[10] === 66 && view[11] === 80 +} + +export function isWebpAnimated(buffer: ArrayBuffer) { + const view = new Uint8Array(buffer) + + if (!isWebp(view)) { + return false + } + + if (!view || view.length < 21) { + return false + } + + return ((view[20] >> 1) & 1) === 1 +}