diff --git a/__mocks__/workerMock.js b/__mocks__/workerMock.js new file mode 100644 index 0000000000..6ee585673e --- /dev/null +++ b/__mocks__/workerMock.js @@ -0,0 +1 @@ +module.exports = jest.fn(); diff --git a/package.json b/package.json index 27c4f39a09..e80ed8dd5a 100644 --- a/package.json +++ b/package.json @@ -187,7 +187,8 @@ "\\$webapp/i18n/languages.json": "/__mocks__/languages.json", "decoderWorker\\.min\\.js": "/__mocks__/empty.js", "decoderWorker\\.min\\.wasm": "/__mocks__/empty.js", - "waveWorker\\.min\\.js": "/__mocks__/empty.js" + "waveWorker\\.min\\.js": "/__mocks__/empty.js", + "workers/(.+)\\.worker\\.ts": "/__mocks__/workerMock.js" }, "transformIgnorePatterns": [ "/node_modules/(?!matrix-js-sdk).+$" diff --git a/src/BlurhashEncoder.ts b/src/BlurhashEncoder.ts new file mode 100644 index 0000000000..2aee370fe9 --- /dev/null +++ b/src/BlurhashEncoder.ts @@ -0,0 +1,60 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { defer, IDeferred } from "matrix-js-sdk/src/utils"; + +// @ts-ignore - `.ts` is needed here to make TS happy +import BlurhashWorker from "./workers/blurhash.worker.ts"; + +interface IBlurhashWorkerResponse { + seq: number; + blurhash: string; +} + +export class BlurhashEncoder { + private static internalInstance = new BlurhashEncoder(); + + public static get instance(): BlurhashEncoder { + return BlurhashEncoder.internalInstance; + } + + private readonly worker: Worker; + private seq = 0; + private pendingDeferredMap = new Map>(); + + constructor() { + this.worker = new BlurhashWorker(); + this.worker.onmessage = this.onMessage; + } + + private onMessage = (ev: MessageEvent) => { + const { seq, blurhash } = ev.data; + const deferred = this.pendingDeferredMap.get(seq); + if (deferred) { + this.pendingDeferredMap.delete(seq); + deferred.resolve(blurhash); + } + }; + + public getBlurhash(imageData: ImageData): Promise { + const seq = this.seq++; + const deferred = defer(); + this.pendingDeferredMap.set(seq, deferred); + this.worker.postMessage({ seq, imageData }); + return deferred.promise; + } +} + diff --git a/src/ContentMessages.tsx b/src/ContentMessages.tsx index b752886b8a..0c65a7bd35 100644 --- a/src/ContentMessages.tsx +++ b/src/ContentMessages.tsx @@ -17,7 +17,6 @@ limitations under the License. */ import React from "react"; -import { encode } from "blurhash"; import { MatrixClient } from "matrix-js-sdk/src/client"; import dis from './dispatcher/dispatcher'; @@ -28,7 +27,6 @@ import RoomViewStore from './stores/RoomViewStore'; import encrypt from "browser-encrypt-attachment"; import extractPngChunks from "png-chunks-extract"; import Spinner from "./components/views/elements/Spinner"; - import { Action } from "./dispatcher/actions"; import CountlyAnalytics from "./CountlyAnalytics"; import { @@ -40,6 +38,7 @@ import { } from "./dispatcher/payloads/UploadPayload"; import { IUpload } from "./models/IUpload"; import { IAbortablePromise, IImageInfo } from "matrix-js-sdk/src/@types/partials"; +import { BlurhashEncoder } from "./BlurhashEncoder"; const MAX_WIDTH = 800; const MAX_HEIGHT = 600; @@ -103,55 +102,62 @@ interface IThumbnail { * @return {Promise} A promise that resolves with an object with an info key * and a thumbnail key. */ -function createThumbnail( +async function createThumbnail( element: ThumbnailableElement, inputWidth: number, inputHeight: number, mimeType: string, ): Promise { - return new Promise((resolve) => { - let targetWidth = inputWidth; - let targetHeight = inputHeight; - if (targetHeight > MAX_HEIGHT) { - targetWidth = Math.floor(targetWidth * (MAX_HEIGHT / targetHeight)); - targetHeight = MAX_HEIGHT; - } - if (targetWidth > MAX_WIDTH) { - targetHeight = Math.floor(targetHeight * (MAX_WIDTH / targetWidth)); - targetWidth = MAX_WIDTH; - } + let targetWidth = inputWidth; + let targetHeight = inputHeight; + if (targetHeight > MAX_HEIGHT) { + targetWidth = Math.floor(targetWidth * (MAX_HEIGHT / targetHeight)); + targetHeight = MAX_HEIGHT; + } + if (targetWidth > MAX_WIDTH) { + targetHeight = Math.floor(targetHeight * (MAX_WIDTH / targetWidth)); + targetWidth = MAX_WIDTH; + } - const canvas = document.createElement("canvas"); + let canvas: HTMLCanvasElement | OffscreenCanvas; + if (window.OffscreenCanvas) { + canvas = new window.OffscreenCanvas(targetWidth, targetHeight); + } else { + canvas = document.createElement("canvas"); canvas.width = targetWidth; canvas.height = targetHeight; - const context = canvas.getContext("2d"); - context.drawImage(element, 0, 0, targetWidth, targetHeight); - const imageData = context.getImageData(0, 0, targetWidth, targetHeight); - const blurhash = encode( - imageData.data, - imageData.width, - imageData.height, - // use 4 components on the longer dimension, if square then both - imageData.width >= imageData.height ? 4 : 3, - imageData.height >= imageData.width ? 4 : 3, - ); - canvas.toBlob(function(thumbnail) { - resolve({ - info: { - thumbnail_info: { - w: targetWidth, - h: targetHeight, - mimetype: thumbnail.type, - size: thumbnail.size, - }, - w: inputWidth, - h: inputHeight, - [BLURHASH_FIELD]: blurhash, - }, - thumbnail, - }); - }, mimeType); - }); + } + + const context = canvas.getContext("2d"); + context.drawImage(element, 0, 0, targetWidth, targetHeight); + + let thumbnailPromise: Promise; + + if (window.OffscreenCanvas) { + thumbnailPromise = (canvas as OffscreenCanvas).convertToBlob({ type: mimeType }); + } else { + thumbnailPromise = new Promise(resolve => (canvas as HTMLCanvasElement).toBlob(resolve, mimeType)); + } + + const imageData = context.getImageData(0, 0, targetWidth, targetHeight); + // thumbnailPromise and blurhash promise are being awaited concurrently + const blurhash = await BlurhashEncoder.instance.getBlurhash(imageData); + const thumbnail = await thumbnailPromise; + + return { + info: { + thumbnail_info: { + w: targetWidth, + h: targetHeight, + mimetype: thumbnail.type, + size: thumbnail.size, + }, + w: inputWidth, + h: inputHeight, + [BLURHASH_FIELD]: blurhash, + }, + thumbnail, + }; } /** diff --git a/src/workers/blurhash.worker.ts b/src/workers/blurhash.worker.ts new file mode 100644 index 0000000000..031cc67c90 --- /dev/null +++ b/src/workers/blurhash.worker.ts @@ -0,0 +1,38 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { encode } from "blurhash"; + +const ctx: Worker = self as any; + +interface IBlurhashWorkerRequest { + seq: number; + imageData: ImageData; +} + +ctx.addEventListener("message", (event: MessageEvent): void => { + const { seq, imageData } = event.data; + const blurhash = encode( + imageData.data, + imageData.width, + imageData.height, + // use 4 components on the longer dimension, if square then both + imageData.width >= imageData.height ? 4 : 3, + imageData.height >= imageData.width ? 4 : 3, + ); + + ctx.postMessage({ seq, blurhash }); +});