From 3cc3138cb5eb4f5b419b91b3f3a5e73e1453a9d9 Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Sat, 24 Aug 2024 16:03:06 +0000
Subject: [PATCH] web/libav: use fitting class for `Chunk`s to avoid
 unnecessary copies

---
 web/src/lib/libav/encode.ts    | 31 ++++++++---
 web/src/lib/libav/webcodecs.ts | 97 ++++++++++++++++++++++++++++------
 2 files changed, 104 insertions(+), 24 deletions(-)

diff --git a/web/src/lib/libav/encode.ts b/web/src/lib/libav/encode.ts
index 78e5b9f1..0b60db00 100644
--- a/web/src/lib/libav/encode.ts
+++ b/web/src/lib/libav/encode.ts
@@ -4,6 +4,10 @@ import * as LibAVWebCodecs from "@imput/libavjs-webcodecs-bridge";
 import { BufferStream } from "../buffer-stream";
 import WebCodecsWrapper from "./webcodecs";
 import LibAVWrapper from "./instance";
+import {
+    EncodedAudioChunk as PolyfilledEncodedAudioChunk,
+    EncodedVideoChunk as PolyfilledEncodedVideoChunk
+} from "@imput/libavjs-webcodecs-polyfill";
 
 const QUEUE_THRESHOLD_MIN = 16;
 const QUEUE_THRESHOLD_MAX = 128;
@@ -53,7 +57,7 @@ export default class EncodeLibAV extends LibAVWrapper {
                 const {
                     pipe,
                     stream: ostream
-                } = await this.#createEncoder(stream, 'mp3');
+                } = await this.#createEncoder(stream, 'flac');
 
                 pipes.push({
                     decoder: await this.#createDecoder(stream),
@@ -267,14 +271,25 @@ export default class EncodeLibAV extends LibAVWrapper {
     }
 
     #decodePacket(decoder: Decoder, packet: Packet, stream: Stream) {
-        let chunk;
-        if (WebCodecsWrapper.isVideo(decoder)) {
-            chunk = LibAVWebCodecs.packetToEncodedVideoChunk(packet, stream);
-        } else if (WebCodecsWrapper.isAudio(decoder)) {
-            chunk = LibAVWebCodecs.packetToEncodedAudioChunk(packet, stream);
-        }
+        let decoderType;
 
-        decoder.decode(chunk);
+        if (decoderType = WebCodecsWrapper.isVideo(decoder)) {
+            const EncodedVideoChunk = decoderType === 'polyfilled' ? PolyfilledEncodedVideoChunk : window.EncodedVideoChunk;
+            WebCodecsWrapper.decodeVideo(
+                LibAVWebCodecs.packetToEncodedVideoChunk(
+                    packet, stream, { EncodedVideoChunk }
+                ),
+                decoder as VideoDecoder
+            );
+        } else if (decoderType = WebCodecsWrapper.isAudio(decoder)) {
+            const EncodedAudioChunk = decoderType === 'polyfilled' ? PolyfilledEncodedAudioChunk : window.EncodedAudioChunk;
+            WebCodecsWrapper.decodeAudio(
+                LibAVWebCodecs.packetToEncodedAudioChunk(
+                    packet, stream, { EncodedAudioChunk }
+                ),
+                decoder as AudioDecoder,
+            );
+        }
     }
 
     async* #demux(fmt_ctx: number) {
diff --git a/web/src/lib/libav/webcodecs.ts b/web/src/lib/libav/webcodecs.ts
index ddfac5e7..b70752e9 100644
--- a/web/src/lib/libav/webcodecs.ts
+++ b/web/src/lib/libav/webcodecs.ts
@@ -139,25 +139,90 @@ export default class WebCodecsWrapper {
     }
 
     static isVideo(obj: unknown) {
-        return ('VideoEncoder' in window && obj instanceof VideoEncoder)
-                || ('VideoDecoder' in window && obj instanceof VideoDecoder)
-                || ('VideoFrame' in window && obj instanceof VideoFrame)
-                || ('EncodedVideoChunk' in window && obj instanceof EncodedVideoChunk)
-                || obj instanceof LibAVPolyfill.VideoEncoder
-                || obj instanceof LibAVPolyfill.VideoDecoder
-                || obj instanceof LibAVPolyfill.VideoFrame
-                || obj instanceof LibAVPolyfill.EncodedVideoChunk;
+        const isNative = ('VideoEncoder' in window && obj instanceof VideoEncoder)
+                            || ('VideoDecoder' in window && obj instanceof VideoDecoder)
+                            || ('VideoFrame' in window && obj instanceof VideoFrame)
+                            || ('EncodedVideoChunk' in window && obj instanceof EncodedVideoChunk);
+        if (isNative) {
+            return 'native';
+        }
+
+        const isPolyfilled = obj instanceof LibAVPolyfill.VideoEncoder
+                            || obj instanceof LibAVPolyfill.VideoDecoder
+                            || obj instanceof LibAVPolyfill.VideoFrame
+                            || obj instanceof LibAVPolyfill.EncodedVideoChunk;
+        if (isPolyfilled) {
+            return 'polyfilled';
+        }
     }
 
     static isAudio(obj: unknown) {
-        return ('AudioEncoder' in window && obj instanceof AudioEncoder)
-                || ('AudioDecoder' in window && obj instanceof AudioDecoder)
-                || ('AudioData' in window && obj instanceof AudioData)
-                || ('EncodedAudioChunk' in window && obj instanceof EncodedAudioChunk)
-                || obj instanceof LibAVPolyfill.AudioEncoder
-                || obj instanceof LibAVPolyfill.AudioDecoder
-                || obj instanceof LibAVPolyfill.AudioData
-                || obj instanceof LibAVPolyfill.EncodedAudioChunk;
+        const isNative = ('AudioEncoder' in window && obj instanceof AudioEncoder)
+                        || ('AudioDecoder' in window && obj instanceof AudioDecoder)
+                        || ('AudioData' in window && obj instanceof AudioData)
+                        || ('EncodedAudioChunk' in window && obj instanceof EncodedAudioChunk);
+
+        if (isNative) {
+            return 'native';
+        }
+
+        const isPolyfilled = obj instanceof LibAVPolyfill.AudioEncoder
+                            || obj instanceof LibAVPolyfill.AudioDecoder
+                            || obj instanceof LibAVPolyfill.AudioData
+                            || obj instanceof LibAVPolyfill.EncodedAudioChunk;
+        if (isPolyfilled) {
+            return 'polyfilled';
+        }
+    }
+
+    static decodeAudio(
+        data: EncodedAudioChunk | LibAVPolyfill.EncodedAudioChunk,
+        destination: AudioDecoder
+    ) {
+        const hasChunk = 'EncodedAudioChunk' in window
+        const isPolyfilled = hasChunk && window.EncodedAudioChunk === LibAVPolyfill.EncodedAudioChunk;
+        if (destination instanceof LibAVPolyfill.AudioDecoder) {
+            if (hasChunk && !isPolyfilled && data instanceof EncodedAudioChunk) {
+                data = LibAVPolyfill.EncodedAudioChunk.fromNative(data);
+                console.log('EncodedAudioChunk: native -> polyfill');
+            } else {
+                console.log('EncodedAudioChunk: passthrough (polyfill)');
+            }
+        } else {
+            if (data instanceof LibAVPolyfill.EncodedAudioChunk) {
+                data = data.toNative();
+                console.log('EncodedAudioChunk: polyfill -> native');
+            } else {
+                console.log('EncodedAudioChunk: passthrough (native)');
+            }
+        }
+
+        return destination.decode(data);
+    }
+
+    static decodeVideo(
+        data: EncodedVideoChunk | LibAVPolyfill.EncodedVideoChunk,
+        destination: VideoDecoder
+    ) {
+        const hasChunk = 'EncodedVideoChunk' in window
+        const isPolyfilled = hasChunk && window.EncodedVideoChunk === LibAVPolyfill.EncodedVideoChunk;
+        if (destination instanceof LibAVPolyfill.VideoDecoder) {
+            if (hasChunk && !isPolyfilled && data instanceof EncodedVideoChunk) {
+                data = LibAVPolyfill.EncodedVideoChunk.fromNative(data);
+                console.log('EncodedVideoChunk: native -> polyfill');
+            } else {
+                console.log('EncodedVideoChunk: passthrough (polyfill)');
+            }
+        } else {
+            if (data instanceof LibAVPolyfill.EncodedVideoChunk) {
+                data = data.toNative();
+                console.log('EncodedVideoChunk: polyfill -> native');
+            } else {
+                console.log('EncodedVideoChunk: passthrough (native)');
+            }
+        }
+
+        return destination.decode(data);
     }
 
     static encodeAudio(data: AudioData | LibAVPolyfill.AudioData, destination: AudioEncoder) {