From fe032ed9423fe4182ddee258b53cb0d08dffda28 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 24 Mar 2022 15:13:11 -0600 Subject: [PATCH] Step 8.5: Move specific image utilities out of ContentMessages --- src/ContentMessages.ts | 106 +-------------- src/components/views/messages/MImageBody.tsx | 2 +- .../views/messages/MStickerBody.tsx | 2 +- src/components/views/messages/MVideoBody.tsx | 2 +- src/utils/image-media.ts | 121 ++++++++++++++++++ 5 files changed, 125 insertions(+), 108 deletions(-) create mode 100644 src/utils/image-media.ts diff --git a/src/ContentMessages.ts b/src/ContentMessages.ts index 7dc5b7f4a8..884943608f 100644 --- a/src/ContentMessages.ts +++ b/src/ContentMessages.ts @@ -40,7 +40,6 @@ import { UploadStartedPayload, } from "./dispatcher/payloads/UploadPayload"; import { IUpload } from "./models/IUpload"; -import { BlurhashEncoder } from "./BlurhashEncoder"; import SettingsStore from "./settings/SettingsStore"; import { decorateStartSendingTime, sendRoundTripMetric } from "./sendTimePerformanceMetrics"; import { TimelineRenderingType } from "./contexts/RoomContext"; @@ -49,20 +48,14 @@ import { addReplyToMessageContent } from "./utils/Reply"; import ErrorDialog from "./components/views/dialogs/ErrorDialog"; import UploadFailureDialog from "./components/views/dialogs/UploadFailureDialog"; import UploadConfirmDialog from "./components/views/dialogs/UploadConfirmDialog"; - -const MAX_WIDTH = 800; -const MAX_HEIGHT = 600; +import { createThumbnail } from "./utils/image-media"; // scraped out of a macOS hidpi (5660ppm) screenshot png // 5669 px (x-axis) , 5669 px (y-axis) , per metre const PHYS_HIDPI = [0x00, 0x00, 0x16, 0x25, 0x00, 0x00, 0x16, 0x25, 0x01]; -export const BLURHASH_FIELD = "xyz.amorgan.blurhash"; // MSC2448 - export class UploadCanceledError extends Error {} -type ThumbnailableElement = HTMLImageElement | HTMLVideoElement; - interface IMediaConfig { "m.upload.size"?: number; } @@ -78,103 +71,6 @@ interface IContent { url?: string; } -interface IThumbnail { - info: { - // eslint-disable-next-line camelcase - thumbnail_info: { - w: number; - h: number; - mimetype: string; - size: number; - }; - w: number; - h: number; - [BLURHASH_FIELD]: string; - }; - thumbnail: Blob; -} - -/** - * Create a thumbnail for a image DOM element. - * The image will be smaller than MAX_WIDTH and MAX_HEIGHT. - * The thumbnail will have the same aspect ratio as the original. - * Draws the element into a canvas using CanvasRenderingContext2D.drawImage - * Then calls Canvas.toBlob to get a blob object for the image data. - * - * Since it needs to calculate the dimensions of the source image and the - * thumbnailed image it returns an info object filled out with information - * about the original image and the thumbnail. - * - * @param {HTMLElement} element The element to thumbnail. - * @param {number} inputWidth The width of the image in the input element. - * @param {number} inputHeight the width of the image in the input element. - * @param {string} mimeType The mimeType to save the blob as. - * @param {boolean} calculateBlurhash Whether to calculate a blurhash of the given image too. - * @return {Promise} A promise that resolves with an object with an info key - * and a thumbnail key. - */ -export async function createThumbnail( - element: ThumbnailableElement, - inputWidth: number, - inputHeight: number, - mimeType: string, - calculateBlurhash = true, -): Promise { - let targetWidth = inputWidth; - let targetHeight = inputHeight; - if (targetHeight > MAX_HEIGHT) { - targetWidth = Math.floor(targetWidth * (MAX_HEIGHT / targetHeight)); - targetHeight = MAX_HEIGHT; - } - if (targetWidth > MAX_WIDTH) { - targetHeight = Math.floor(targetHeight * (MAX_WIDTH / targetWidth)); - targetWidth = MAX_WIDTH; - } - - let canvas: HTMLCanvasElement | OffscreenCanvas; - let context: CanvasRenderingContext2D; - try { - canvas = new window.OffscreenCanvas(targetWidth, targetHeight); - context = canvas.getContext("2d"); - } catch (e) { - // Fallback support for other browsers (Safari and Firefox for now) - canvas = document.createElement("canvas"); - (canvas as HTMLCanvasElement).width = targetWidth; - (canvas as HTMLCanvasElement).height = targetHeight; - context = canvas.getContext("2d"); - } - - context.drawImage(element, 0, 0, targetWidth, targetHeight); - - let thumbnailPromise: Promise; - - if (window.OffscreenCanvas) { - thumbnailPromise = (canvas as OffscreenCanvas).convertToBlob({ type: mimeType }); - } else { - thumbnailPromise = new Promise(resolve => (canvas as HTMLCanvasElement).toBlob(resolve, mimeType)); - } - - const imageData = context.getImageData(0, 0, targetWidth, targetHeight); - // thumbnailPromise and blurhash promise are being awaited concurrently - const blurhash = calculateBlurhash ? await BlurhashEncoder.instance.getBlurhash(imageData) : undefined; - const thumbnail = await thumbnailPromise; - - return { - info: { - thumbnail_info: { - w: targetWidth, - h: targetHeight, - mimetype: thumbnail.type, - size: thumbnail.size, - }, - w: inputWidth, - h: inputHeight, - [BLURHASH_FIELD]: blurhash, - }, - thumbnail, - }; -} - /** * Load a file into a newly created image element. * diff --git a/src/components/views/messages/MImageBody.tsx b/src/components/views/messages/MImageBody.tsx index 0f9fe45a29..4ef8e8d497 100644 --- a/src/components/views/messages/MImageBody.tsx +++ b/src/components/views/messages/MImageBody.tsx @@ -29,7 +29,7 @@ import { _t } from '../../../languageHandler'; import SettingsStore from "../../../settings/SettingsStore"; import Spinner from '../elements/Spinner'; import { Media, mediaFromContent } from "../../../customisations/Media"; -import { BLURHASH_FIELD, createThumbnail } from "../../../ContentMessages"; +import { BLURHASH_FIELD, createThumbnail } from "../../../utils/image-media"; import { IMediaEventContent } from '../../../customisations/models/IMediaEventContent'; import ImageView from '../elements/ImageView'; import { IBodyProps } from "./IBodyProps"; diff --git a/src/components/views/messages/MStickerBody.tsx b/src/components/views/messages/MStickerBody.tsx index c29d4c225b..eb56d8d2e5 100644 --- a/src/components/views/messages/MStickerBody.tsx +++ b/src/components/views/messages/MStickerBody.tsx @@ -17,7 +17,7 @@ limitations under the License. import React from 'react'; import MImageBody from './MImageBody'; -import { BLURHASH_FIELD } from "../../../ContentMessages"; +import { BLURHASH_FIELD } from "../../../utils/image-media"; import Tooltip from "../elements/Tooltip"; export default class MStickerBody extends MImageBody { diff --git a/src/components/views/messages/MVideoBody.tsx b/src/components/views/messages/MVideoBody.tsx index cacd19c5a8..f4733df19f 100644 --- a/src/components/views/messages/MVideoBody.tsx +++ b/src/components/views/messages/MVideoBody.tsx @@ -22,7 +22,7 @@ import { _t } from '../../../languageHandler'; import SettingsStore from "../../../settings/SettingsStore"; import InlineSpinner from '../elements/InlineSpinner'; import { mediaFromContent } from "../../../customisations/Media"; -import { BLURHASH_FIELD } from "../../../ContentMessages"; +import { BLURHASH_FIELD } from "../../../utils/image-media"; import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent"; import { IBodyProps } from "./IBodyProps"; import MFileBody from "./MFileBody"; diff --git a/src/utils/image-media.ts b/src/utils/image-media.ts new file mode 100644 index 0000000000..a57e4b841a --- /dev/null +++ b/src/utils/image-media.ts @@ -0,0 +1,121 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { BlurhashEncoder } from "../BlurhashEncoder"; + +type ThumbnailableElement = HTMLImageElement | HTMLVideoElement; + +interface IThumbnail { + info: { + // eslint-disable-next-line camelcase + thumbnail_info: { + w: number; + h: number; + mimetype: string; + size: number; + }; + w: number; + h: number; + [BLURHASH_FIELD]: string; + }; + thumbnail: Blob; +} + +export const BLURHASH_FIELD = "xyz.amorgan.blurhash"; // MSC2448 + +const MAX_WIDTH = 800; +const MAX_HEIGHT = 600; + +/** + * Create a thumbnail for a image DOM element. + * The image will be smaller than MAX_WIDTH and MAX_HEIGHT. + * The thumbnail will have the same aspect ratio as the original. + * Draws the element into a canvas using CanvasRenderingContext2D.drawImage + * Then calls Canvas.toBlob to get a blob object for the image data. + * + * Since it needs to calculate the dimensions of the source image and the + * thumbnailed image it returns an info object filled out with information + * about the original image and the thumbnail. + * + * @param {HTMLElement} element The element to thumbnail. + * @param {number} inputWidth The width of the image in the input element. + * @param {number} inputHeight the width of the image in the input element. + * @param {string} mimeType The mimeType to save the blob as. + * @param {boolean} calculateBlurhash Whether to calculate a blurhash of the given image too. + * @return {Promise} A promise that resolves with an object with an info key + * and a thumbnail key. + */ +export async function createThumbnail( + element: ThumbnailableElement, + inputWidth: number, + inputHeight: number, + mimeType: string, + calculateBlurhash = true, +): Promise { + let targetWidth = inputWidth; + let targetHeight = inputHeight; + if (targetHeight > MAX_HEIGHT) { + targetWidth = Math.floor(targetWidth * (MAX_HEIGHT / targetHeight)); + targetHeight = MAX_HEIGHT; + } + if (targetWidth > MAX_WIDTH) { + targetHeight = Math.floor(targetHeight * (MAX_WIDTH / targetWidth)); + targetWidth = MAX_WIDTH; + } + + let canvas: HTMLCanvasElement | OffscreenCanvas; + let context: CanvasRenderingContext2D; + try { + canvas = new window.OffscreenCanvas(targetWidth, targetHeight); + context = canvas.getContext("2d"); + } catch (e) { + // Fallback support for other browsers (Safari and Firefox for now) + canvas = document.createElement("canvas"); + (canvas as HTMLCanvasElement).width = targetWidth; + (canvas as HTMLCanvasElement).height = targetHeight; + context = canvas.getContext("2d"); + } + + context.drawImage(element, 0, 0, targetWidth, targetHeight); + + let thumbnailPromise: Promise; + + if (window.OffscreenCanvas) { + thumbnailPromise = (canvas as OffscreenCanvas).convertToBlob({ type: mimeType }); + } else { + thumbnailPromise = new Promise(resolve => (canvas as HTMLCanvasElement).toBlob(resolve, mimeType)); + } + + const imageData = context.getImageData(0, 0, targetWidth, targetHeight); + // thumbnailPromise and blurhash promise are being awaited concurrently + const blurhash = calculateBlurhash ? await BlurhashEncoder.instance.getBlurhash(imageData) : undefined; + const thumbnail = await thumbnailPromise; + + return { + info: { + thumbnail_info: { + w: targetWidth, + h: targetHeight, + mimetype: thumbnail.type, + size: thumbnail.size, + }, + w: inputWidth, + h: inputHeight, + [BLURHASH_FIELD]: blurhash, + }, + thumbnail, + }; +}