Offload some more waveform processing onto a worker (#9223)

cypress/e2e/audio-player/audio-player.spec.ts

@@ -204,8 +204,9 @@ describe("Audio player", () => {
             // Assert that the counter is zero before clicking the play button
             cy.contains(".mx_AudioPlayer_seek [role='timer']", "00:00").should("exist");
 
-            // Find and click "Play" button
+            // Find and click "Play" button, the wait is to make the test less flaky
-            cy.findByRole("button", { name: "Play" }).click();
+            cy.findByRole("button", { name: "Play" }).should("exist");
+            cy.wait(500).findByRole("button", { name: "Play" }).click();
 
             // Assert that "Pause" button can be found
             cy.findByRole("button", { name: "Pause" }).should("exist");
@@ -339,8 +340,9 @@ describe("Audio player", () => {
                         // Assert that the counter is zero before clicking the play button
                         cy.contains(".mx_AudioPlayer_seek [role='timer']", "00:00").should("exist");
 
-                        // Find and click "Play" button
+                        // Find and click "Play" button, the wait is to make the test less flaky
-                        cy.findByRole("button", { name: "Play" }).click();
+                        cy.findByRole("button", { name: "Play" }).should("exist");
+                        cy.wait(500).findByRole("button", { name: "Play" }).click();
 
                         // Assert that "Pause" button can be found
                         cy.findByRole("button", { name: "Pause" }).should("exist");
@@ -349,7 +351,7 @@ describe("Audio player", () => {
                         cy.contains(".mx_AudioPlayer_seek [role='timer']", "00:00").should("exist");
 
                         // Assert that "Play" button can be found
-                        cy.findByRole("button", { name: "Play" }).should("exist");
+                        cy.findByRole("button", { name: "Play" }).should("exist").should("not.have.attr", "disabled");
                     });
                 })
                 .realHover()

src/BlurhashEncoder.ts

@@ -14,15 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { defer, IDeferred } from "matrix-js-sdk/src/utils";
-
 // @ts-ignore - `.ts` is needed here to make TS happy
-import BlurhashWorker from "./workers/blurhash.worker.ts";
+import BlurhashWorker, { Request, Response } from "./workers/blurhash.worker.ts";
-
+import { WorkerManager } from "./WorkerManager";
-interface IBlurhashWorkerResponse {
-    seq: number;
-    blurhash: string;
-}
 
 export class BlurhashEncoder {
     private static internalInstance = new BlurhashEncoder();
@@ -31,29 +25,9 @@ export class BlurhashEncoder {
         return BlurhashEncoder.internalInstance;
     }
 
-    private readonly worker: Worker;
+    private readonly worker = new WorkerManager<Request, Response>(BlurhashWorker);
-    private seq = 0;
-    private pendingDeferredMap = new Map<number, IDeferred<string>>();
-
-    public constructor() {
-        this.worker = new BlurhashWorker();
-        this.worker.onmessage = this.onMessage;
-    }
-
-    private onMessage = (ev: MessageEvent<IBlurhashWorkerResponse>): void => {
-        const { seq, blurhash } = ev.data;
-        const deferred = this.pendingDeferredMap.get(seq);
-        if (deferred) {
-            this.pendingDeferredMap.delete(seq);
-            deferred.resolve(blurhash);
-        }
-    };
 
     public getBlurhash(imageData: ImageData): Promise<string> {
-        const seq = this.seq++;
+        return this.worker.call({ imageData }).then((resp) => resp.blurhash);
-        const deferred = defer<string>();
-        this.pendingDeferredMap.set(seq, deferred);
-        this.worker.postMessage({ seq, imageData });
-        return deferred.promise;
     }
 }

src/WorkerManager.ts

@@ -0,0 +1,46 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { defer, IDeferred } from "matrix-js-sdk/src/utils";
+
+import { WorkerPayload } from "./workers/worker";
+
+export class WorkerManager<Request extends {}, Response> {
+    private readonly worker: Worker;
+    private seq = 0;
+    private pendingDeferredMap = new Map<number, IDeferred<Response>>();
+
+    public constructor(WorkerConstructor: { new (): Worker }) {
+        this.worker = new WorkerConstructor();
+        this.worker.onmessage = this.onMessage;
+    }
+
+    private onMessage = (ev: MessageEvent<Response & WorkerPayload>): void => {
+        const deferred = this.pendingDeferredMap.get(ev.data.seq);
+        if (deferred) {
+            this.pendingDeferredMap.delete(ev.data.seq);
+            deferred.resolve(ev.data);
+        }
+    };
+
+    public call(request: Request): Promise<Response> {
+        const seq = this.seq++;
+        const deferred = defer<Response>();
+        this.pendingDeferredMap.set(seq, deferred);
+        this.worker.postMessage({ seq, ...request });
+        return deferred.promise;
+    }
+}

src/audio/ManagedPlayback.ts

@@ -14,8 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { DEFAULT_WAVEFORM, Playback } from "./Playback";
+import { Playback } from "./Playback";
 import { PlaybackManager } from "./PlaybackManager";
+import { DEFAULT_WAVEFORM } from "./consts";
 
 /**
  * A managed playback is a Playback instance that is guided by a PlaybackManager.

src/audio/Playback.ts

@@ -17,13 +17,18 @@ limitations under the License.
 import EventEmitter from "events";
 import { SimpleObservable } from "matrix-widget-api";
 import { logger } from "matrix-js-sdk/src/logger";
+import { defer } from "matrix-js-sdk/src/utils";
 
+// @ts-ignore - `.ts` is needed here to make TS happy
+import PlaybackWorker, { Request, Response } from "../workers/playback.worker.ts";
 import { UPDATE_EVENT } from "../stores/AsyncStore";
-import { arrayFastResample, arrayRescale, arraySeed, arraySmoothingResample } from "../utils/arrays";
+import { arrayFastResample } from "../utils/arrays";
 import { IDestroyable } from "../utils/IDestroyable";
 import { PlaybackClock } from "./PlaybackClock";
 import { createAudioContext, decodeOgg } from "./compat";
 import { clamp } from "../utils/numbers";
+import { WorkerManager } from "../WorkerManager";
+import { DEFAULT_WAVEFORM, PLAYBACK_WAVEFORM_SAMPLES } from "./consts";
 
 export enum PlaybackState {
     Decoding = "decoding",
@@ -32,25 +37,7 @@ export enum PlaybackState {
     Playing = "playing", // active progress through timeline
 }
 
-export interface PlaybackInterface {
-    readonly liveData: SimpleObservable<number[]>;
-    readonly timeSeconds: number;
-    readonly durationSeconds: number;
-    skipTo(timeSeconds: number): Promise<void>;
-}
-
-export const PLAYBACK_WAVEFORM_SAMPLES = 39;
 const THUMBNAIL_WAVEFORM_SAMPLES = 100; // arbitrary: [30,120]
-export const DEFAULT_WAVEFORM = arraySeed(0, PLAYBACK_WAVEFORM_SAMPLES);
-
-function makePlaybackWaveform(input: number[]): number[] {
-    // First, convert negative amplitudes to positive so we don't detect zero as "noisy".
-    const noiseWaveform = input.map((v) => Math.abs(v));
-
-    // Then, we'll resample the waveform using a smoothing approach so we can keep the same rough shape.
-    // We also rescale the waveform to be 0-1 so we end up with a clamped waveform to rely upon.
-    return arrayRescale(arraySmoothingResample(noiseWaveform, PLAYBACK_WAVEFORM_SAMPLES), 0, 1);
-}
 
 export interface PlaybackInterface {
     readonly currentState: PlaybackState;
@@ -68,14 +55,15 @@ export class Playback extends EventEmitter implements IDestroyable, PlaybackInte
     public readonly thumbnailWaveform: number[];
 
     private readonly context: AudioContext;
-    private source: AudioBufferSourceNode | MediaElementAudioSourceNode;
+    private source?: AudioBufferSourceNode | MediaElementAudioSourceNode;
     private state = PlaybackState.Decoding;
-    private audioBuf: AudioBuffer;
+    private audioBuf?: AudioBuffer;
-    private element: HTMLAudioElement;
+    private element?: HTMLAudioElement;
     private resampledWaveform: number[];
     private waveformObservable = new SimpleObservable<number[]>();
     private readonly clock: PlaybackClock;
     private readonly fileSize: number;
+    private readonly worker = new WorkerManager<Request, Response>(PlaybackWorker);
 
     /**
      * Creates a new playback instance from a buffer.
@@ -178,12 +166,11 @@ export class Playback extends EventEmitter implements IDestroyable, PlaybackInte
             // 5mb
             logger.log("Audio file too large: processing through <audio /> element");
             this.element = document.createElement("AUDIO") as HTMLAudioElement;
-            const prom = new Promise((resolve, reject) => {
+            const deferred = defer<unknown>();
-                this.element.onloadeddata = () => resolve(null);
+            this.element.onloadeddata = deferred.resolve;
-                this.element.onerror = (e) => reject(e);
+            this.element.onerror = deferred.reject;
-            });
             this.element.src = URL.createObjectURL(new Blob([this.buf]));
-            await prom; // make sure the audio element is ready for us
+            await deferred.promise; // make sure the audio element is ready for us
         } else {
             // Safari compat: promise API not supported on this function
             this.audioBuf = await new Promise((resolve, reject) => {
@@ -218,20 +205,23 @@ export class Playback extends EventEmitter implements IDestroyable, PlaybackInte
 
             // Update the waveform to the real waveform once we have channel data to use. We don't
             // exactly trust the user-provided waveform to be accurate...
-            const waveform = Array.from(this.audioBuf.getChannelData(0));
+            this.resampledWaveform = await this.makePlaybackWaveform(this.audioBuf.getChannelData(0));
-            this.resampledWaveform = makePlaybackWaveform(waveform);
         }
 
         this.waveformObservable.update(this.resampledWaveform);
 
         this.clock.flagLoadTime(); // must happen first because setting the duration fires a clock update
-        this.clock.durationSeconds = this.element ? this.element.duration : this.audioBuf.duration;
+        this.clock.durationSeconds = this.element?.duration ?? this.audioBuf!.duration;
 
         // Signal that we're not decoding anymore. This is done last to ensure the clock is updated for
         // when the downstream callers try to use it.
         this.emit(PlaybackState.Stopped); // signal that we're not decoding anymore
     }
 
+    private makePlaybackWaveform(input: Float32Array): Promise<number[]> {
+        return this.worker.call({ data: Array.from(input) }).then((resp) => resp.waveform);
+    }
+
     private onPlaybackEnd = async (): Promise<void> => {
         await this.context.suspend();
         this.emit(PlaybackState.Stopped);
@@ -269,7 +259,7 @@ export class Playback extends EventEmitter implements IDestroyable, PlaybackInte
             this.source = this.context.createMediaElementSource(this.element);
         } else {
             this.source = this.context.createBufferSource();
-            this.source.buffer = this.audioBuf;
+            this.source.buffer = this.audioBuf ?? null;
         }
 
         this.source.addEventListener("ended", this.onPlaybackEnd);

src/audio/PlaybackManager.ts

@@ -14,8 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { DEFAULT_WAVEFORM, Playback, PlaybackState } from "./Playback";
+import { Playback, PlaybackState } from "./Playback";
 import { ManagedPlayback } from "./ManagedPlayback";
+import { DEFAULT_WAVEFORM } from "./consts";
 
 /**
  * Handles management of playback instances to ensure certain functionality, like

src/audio/consts.ts

@@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import { arraySeed } from "../utils/arrays";
+
 export const WORKLET_NAME = "mx-voice-worklet";
 
 export enum PayloadEvent {
@@ -35,3 +37,6 @@ export interface IAmplitudePayload extends IPayload {
     forIndex: number;
     amplitude: number;
 }
+
+export const PLAYBACK_WAVEFORM_SAMPLES = 39;
+export const DEFAULT_WAVEFORM = arraySeed(0, PLAYBACK_WAVEFORM_SAMPLES);

src/components/views/audio_messages/PlaybackWaveform.tsx

@@ -18,8 +18,9 @@ import React from "react";
 
 import { arraySeed, arrayTrimFill } from "../../../utils/arrays";
 import Waveform from "./Waveform";
-import { Playback, PLAYBACK_WAVEFORM_SAMPLES } from "../../../audio/Playback";
+import { Playback } from "../../../audio/Playback";
 import { percentageOf } from "../../../utils/numbers";
+import { PLAYBACK_WAVEFORM_SAMPLES } from "../../../audio/consts";
 
 interface IProps {
     playback: Playback;

src/utils/arrays.ts

@@ -57,6 +57,7 @@ export function arrayFastResample(input: number[], points: number): number[] {
  * @param {number} points The number of samples to end up with.
  * @returns {number[]} The resampled array.
  */
+// ts-prune-ignore-next
 export function arraySmoothingResample(input: number[], points: number): number[] {
     if (input.length === points) return input; // short-circuit a complicated call
 
@@ -99,6 +100,7 @@ export function arraySmoothingResample(input: number[], points: number): number[
  * @param {number} newMax The maximum value to scale to.
  * @returns {number[]} The rescaled array.
  */
+// ts-prune-ignore-next
 export function arrayRescale(input: number[], newMin: number, newMax: number): number[] {
     const min: number = Math.min(...input);
     const max: number = Math.max(...input);

src/workers/blurhash.worker.ts

@@ -16,14 +16,19 @@ limitations under the License.
 
 import { encode } from "blurhash";
 
+import { WorkerPayload } from "./worker";
+
 const ctx: Worker = self as any;
 
-interface IBlurhashWorkerRequest {
+export interface Request {
-    seq: number;
     imageData: ImageData;
 }
 
-ctx.addEventListener("message", (event: MessageEvent<IBlurhashWorkerRequest>): void => {
+export interface Response {
+    blurhash: string;
+}
+
+ctx.addEventListener("message", (event: MessageEvent<Request & WorkerPayload>): void => {
     const { seq, imageData } = event.data;
     const blurhash = encode(
         imageData.data,

src/workers/playback.worker.ts

@@ -0,0 +1,42 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { WorkerPayload } from "./worker";
+import { arrayRescale, arraySmoothingResample } from "../utils/arrays";
+import { PLAYBACK_WAVEFORM_SAMPLES } from "../audio/consts";
+
+const ctx: Worker = self as any;
+
+export interface Request {
+    data: number[];
+}
+
+export interface Response {
+    waveform: number[];
+}
+
+ctx.addEventListener("message", async (event: MessageEvent<Request & WorkerPayload>): Promise<void> => {
+    const { seq, data } = event.data;
+
+    // First, convert negative amplitudes to positive so we don't detect zero as "noisy".
+    const noiseWaveform = data.map((v) => Math.abs(v));
+
+    // Then, we'll resample the waveform using a smoothing approach so we can keep the same rough shape.
+    // We also rescale the waveform to be 0-1 so we end up with a clamped waveform to rely upon.
+    const waveform = arrayRescale(arraySmoothingResample(noiseWaveform, PLAYBACK_WAVEFORM_SAMPLES), 0, 1);
+
+    ctx.postMessage({ seq, waveform });
+});

src/workers/worker.ts

@@ -0,0 +1,19 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export interface WorkerPayload {
+    seq: number;
+}

test/WorkerManager-test.ts

@@ -0,0 +1,59 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { WorkerManager } from "../src/WorkerManager";
+
+describe("WorkerManager", () => {
+    it("should generate consecutive sequence numbers for each call", () => {
+        const postMessage = jest.fn();
+        const manager = new WorkerManager(jest.fn(() => ({ postMessage } as unknown as Worker)));
+
+        manager.call({ data: "One" });
+        manager.call({ data: "Two" });
+        manager.call({ data: "Three" });
+
+        const one = postMessage.mock.calls.find((c) => c[0].data === "One")!;
+        const two = postMessage.mock.calls.find((c) => c[0].data === "Two")!;
+        const three = postMessage.mock.calls.find((c) => c[0].data === "Three")!;
+
+        expect(one[0].seq).toBe(0);
+        expect(two[0].seq).toBe(1);
+        expect(three[0].seq).toBe(2);
+    });
+
+    it("should support resolving out of order", async () => {
+        const postMessage = jest.fn();
+        const worker = { postMessage } as unknown as Worker;
+        const manager = new WorkerManager(jest.fn(() => worker));
+
+        const oneProm = manager.call({ data: "One" });
+        const twoProm = manager.call({ data: "Two" });
+        const threeProm = manager.call({ data: "Three" });
+
+        const one = postMessage.mock.calls.find((c) => c[0].data === "One")![0].seq;
+        const two = postMessage.mock.calls.find((c) => c[0].data === "Two")![0].seq;
+        const three = postMessage.mock.calls.find((c) => c[0].data === "Three")![0].seq;
+
+        worker.onmessage!({ data: { seq: one, data: 1 } } as MessageEvent);
+        await expect(oneProm).resolves.toEqual(expect.objectContaining({ data: 1 }));
+
+        worker.onmessage!({ data: { seq: three, data: 3 } } as MessageEvent);
+        await expect(threeProm).resolves.toEqual(expect.objectContaining({ data: 3 }));
+
+        worker.onmessage!({ data: { seq: two, data: 2 } } as MessageEvent);
+        await expect(twoProm).resolves.toEqual(expect.objectContaining({ data: 2 }));
+    });
+});

test/audio/Playback-test.ts

@@ -20,6 +20,12 @@ import { logger } from "matrix-js-sdk/src/logger";
 import { createAudioContext, decodeOgg } from "../../src/audio/compat";
 import { Playback, PlaybackState } from "../../src/audio/Playback";
 
+jest.mock("../../src/WorkerManager", () => ({
+    WorkerManager: jest.fn(() => ({
+        call: jest.fn().mockResolvedValue({ waveform: [0, 0, 1, 1] }),
+    })),
+}));
+
 jest.mock("../../src/audio/compat", () => ({
     createAudioContext: jest.fn(),
     decodeOgg: jest.fn(),

test/components/views/audio_messages/RecordingPlayback-test.tsx

@@ -26,6 +26,12 @@ import { createAudioContext } from "../../../../src/audio/compat";
 import { flushPromises } from "../../../test-utils";
 import { IRoomState } from "../../../../src/components/structures/RoomView";
 
+jest.mock("../../../../src/WorkerManager", () => ({
+    WorkerManager: jest.fn(() => ({
+        call: jest.fn().mockResolvedValue({ waveform: [0, 0, 1, 1] }),
+    })),
+}));
+
 jest.mock("../../../../src/audio/compat", () => ({
     createAudioContext: jest.fn(),
     decodeOgg: jest.fn().mockResolvedValue({}),