From 9f21a6eb46e0f869b1a45fbd4f062d6888c19e46 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Fri, 23 Aug 2024 18:40:44 +0000 Subject: [PATCH] web/webcodecs: use polyfill as fallback --- pnpm-lock.yaml | 26 ++++++------ web/package.json | 4 +- web/src/lib/libav/encode.ts | 15 ++++--- web/src/lib/libav/webcodecs.ts | 73 ++++++++++++++++++++++++---------- 4 files changed, 73 insertions(+), 45 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 65d68041..ba5f67af 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -97,9 +97,12 @@ importers: '@imput/libav.js-remux-cli': specifier: ^5.7.6 version: 5.7.6 + '@imput/libavjs-webcodecs-bridge': + specifier: ^0.1.2 + version: 0.1.2 '@imput/libavjs-webcodecs-polyfill': - specifier: ^0.5.1 - version: 0.5.1 + specifier: ^0.5.2 + version: 0.5.2 '@imput/version-info': specifier: workspace:^ version: link:../packages/version-info @@ -109,9 +112,6 @@ importers: '@vitejs/plugin-basic-ssl': specifier: ^1.1.0 version: 1.1.0(vite@5.3.5(@types/node@20.14.14)) - libavjs-webcodecs-bridge: - specifier: ^0.1.0 - version: 0.1.0 mime: specifier: ^4.0.4 version: 4.0.4 @@ -544,8 +544,11 @@ packages: '@imput/libav.js-remux-cli@5.7.6': resolution: {integrity: sha512-ofSSLjRF9RfZ3QMBlb7fhxso8p8xlDqU4qX8eCJCukCB15g7iBShthCyGVnYz+3lLoFu9klbvVal9bEncBj/FQ==} - '@imput/libavjs-webcodecs-polyfill@0.5.1': - resolution: {integrity: sha512-uQ5vawG/4ppLKDumkg8BjnvqKWzoKXxt7cj0lvTpB9ph1hiuzjNx8wmwMcXbeTVAzRcts+GtCTPpSF9rFosYBg==} + '@imput/libavjs-webcodecs-bridge@0.1.2': + resolution: {integrity: sha512-8QVvCe9CRe3oVGpMrx/RtHmPunrK+I52nUImM/FDbOt13nEq0rd+zu5ADIMOw7rq/8OgLqqf/o1yQ+Sa4/mhSw==} + + '@imput/libavjs-webcodecs-polyfill@0.5.2': + resolution: {integrity: sha512-3wyRFUTV7yJyYNJvSKzoZWNnGbSN/5RVj+Xx9vYH8/xt73TLpXBBqotGc7UiHNeSy6eFO0EjUzAmieBa+YjLrQ==} '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} @@ -1541,9 +1544,6 @@ packages: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} - libavjs-webcodecs-bridge@0.1.0: - resolution: {integrity: sha512-K9pGvW+k2sGnbaCEuhbLG6ylzKoQyVAA1WxO+qIzaYEeVumosaFeKtMzVAThcIy6VHo11rGVF6IqLfSEEQL7sg==} - lilconfig@3.1.2: resolution: {integrity: sha512-eop+wDAvpItUys0FWkHIKeC9ybYrTGbU41U5K7+bttZZeohvnY7M9dZ5kB21GNWiFT2q1OoPTvncPCgSOVO5ow==} engines: {node: '>=14'} @@ -2522,7 +2522,9 @@ snapshots: '@imput/libav.js-remux-cli@5.7.6': {} - '@imput/libavjs-webcodecs-polyfill@0.5.1': + '@imput/libavjs-webcodecs-bridge@0.1.2': {} + + '@imput/libavjs-webcodecs-polyfill@0.5.2': dependencies: '@ungap/global-this': 0.4.4 @@ -3572,8 +3574,6 @@ snapshots: prelude-ls: 1.2.1 type-check: 0.4.0 - libavjs-webcodecs-bridge@0.1.0: {} - lilconfig@3.1.2: {} lines-and-columns@1.2.4: {} diff --git a/web/package.json b/web/package.json index 1fab9977..36e34ecf 100644 --- a/web/package.json +++ b/web/package.json @@ -51,11 +51,11 @@ "@fontsource/ibm-plex-mono": "^5.0.13", "@imput/libav.js-encode-cli": "^5.7.6", "@imput/libav.js-remux-cli": "^5.7.6", - "@imput/libavjs-webcodecs-polyfill": "^0.5.1", + "@imput/libavjs-webcodecs-bridge": "^0.1.2", + "@imput/libavjs-webcodecs-polyfill": "^0.5.2", "@imput/version-info": "workspace:^", "@tabler/icons-svelte": "3.6.0", "@vitejs/plugin-basic-ssl": "^1.1.0", - "libavjs-webcodecs-bridge": "^0.1.0", "mime": "^4.0.4", "sveltekit-i18n": "^2.4.2", "ts-deepmerge": "^7.0.0", diff --git a/web/src/lib/libav/encode.ts b/web/src/lib/libav/encode.ts index e996b14f..1731cb56 100644 --- a/web/src/lib/libav/encode.ts +++ b/web/src/lib/libav/encode.ts @@ -1,7 +1,6 @@ -import LibAV, { type LibAV as LibAVInstance, type Packet, type Stream } from "@imput/libav.js-encode-cli"; -import type { Chunk, ChunkMetadata, Decoder, FFmpegProgressCallback, OutputStream, Pipeline, RenderingPipeline } from "../types/libav"; -import * as LibAVWebCodecs from "libavjs-webcodecs-bridge"; -import { BufferStream } from "./buffer-stream"; +import LibAV, { type Packet, type Stream } from "@imput/libav.js-encode-cli"; +import type { Chunk, ChunkMetadata, Decoder, OutputStream, Pipeline, RenderingPipeline } from "../types/libav"; +import * as LibAVWebCodecs from "@imput/libavjs-webcodecs-bridge"; import { BufferStream } from "../buffer-stream"; import WebCodecsWrapper from "./webcodecs"; import LibAVWrapper from "./instance"; @@ -44,7 +43,8 @@ export default class EncodeLibAV extends LibAVWrapper { const pipes: RenderingPipeline[] = []; const output_streams: OutputStream[] = []; for (const stream of streams) { - if (stream.codec_id === 61) { + // FIXME: support images. + if (stream.codec_id === 61 || stream.codec_id === 62) { pipes.push(null); output_streams.push(null); continue; @@ -53,7 +53,7 @@ export default class EncodeLibAV extends LibAVWrapper { const { pipe, stream: ostream - } = await this.#createEncoder(stream, 'avc1.64083e'); + } = await this.#createEncoder(stream, 'mp3'); pipes.push({ decoder: await this.#createDecoder(stream), @@ -312,12 +312,11 @@ export default class EncodeLibAV extends LibAVWrapper { if (stream.codec_type === libav.AVMEDIA_TYPE_VIDEO) { streamToConfig = LibAVWebCodecs.videoStreamToConfig; configToStream = LibAVWebCodecs.configToVideoStream; - initEncoder = webcodecs!.initVideoEncoder.bind(webcodecs); + initEncoder = webcodecs.initVideoEncoder.bind(webcodecs); } else if (stream.codec_type === libav.AVMEDIA_TYPE_AUDIO) { streamToConfig = LibAVWebCodecs.audioStreamToConfig; configToStream = LibAVWebCodecs.configToAudioStream; initEncoder = webcodecs.initAudioEncoder.bind(webcodecs); - codec = 'mp4a.40.29'; } else throw "Unknown type: " + stream.codec_type; const config = await streamToConfig(libav, stream, true); diff --git a/web/src/lib/libav/webcodecs.ts b/web/src/lib/libav/webcodecs.ts index 4f58472b..b1c712dc 100644 --- a/web/src/lib/libav/webcodecs.ts +++ b/web/src/lib/libav/webcodecs.ts @@ -16,31 +16,46 @@ export default class WebCodecsWrapper { async #load() { if (typeof this.#ready === 'undefined') { this.#ready = LibAVPolyfill.load({ - polyfill: false, + polyfill: true, LibAV: { LibAV: () => this.#libav } }); } await this.#ready; } + + // FIXME: save me generics. generics save me async #getDecoder(config: VideoDecoderConfig | AudioDecoderConfig) { if (has(config, 'numberOfChannels') && has(config, 'sampleRate')) { const audioConfig = config as AudioDecoderConfig; + for (const source of [ window, LibAVPolyfill ]) { + if (source === LibAVPolyfill) { + await this.#load(); + } - if ('AudioDecoder' in window && await window.AudioDecoder.isConfigSupported(audioConfig)) - return window.AudioDecoder; - - await this.#load(); - if (await LibAVPolyfill.AudioDecoder.isConfigSupported(audioConfig)) - return LibAVPolyfill.AudioDecoder; + try { + const { supported } = await source.AudioDecoder.isConfigSupported(audioConfig); + if (supported) return source.AudioDecoder; + } catch(e) { + console.error('AudioDecoder missing or does not support', config); + console.error(e); + } + } } else { const videoConfig = config as VideoDecoderConfig; - if ('VideoDecoder' in window && await window.VideoDecoder.isConfigSupported(videoConfig)) - return window.VideoDecoder; + for (const source of [ window, LibAVPolyfill ]) { + if (source === LibAVPolyfill) { + await this.#load(); + } - await this.#load(); - if (await LibAVPolyfill.VideoDecoder.isConfigSupported(videoConfig)) - return LibAVPolyfill.VideoDecoder; + try { + const { supported } = await source.VideoDecoder.isConfigSupported(videoConfig); + if (supported) return source.VideoDecoder; + } catch(e) { + console.error('VideoDecoder missing or does not support', config); + console.error(e); + } + } } return null; @@ -49,20 +64,34 @@ export default class WebCodecsWrapper { async #getEncoder(config: VideoEncoderConfig | AudioEncoderConfig) { if (has(config, 'numberOfChannels') && has(config, 'sampleRate')) { const audioConfig = config as AudioEncoderConfig; - if ('AudioEncoder' in window && await window.AudioEncoder.isConfigSupported(audioConfig)) - return window.AudioEncoder; + for (const source of [ window, LibAVPolyfill ]) { + if (source === LibAVPolyfill) { + await this.#load(); + } - await this.#load(); - if (await LibAVPolyfill.AudioEncoder.isConfigSupported(audioConfig)) - return LibAVPolyfill.AudioEncoder; + try { + const { supported } = await source.AudioEncoder.isConfigSupported(audioConfig); + if (supported) return source.AudioEncoder; + } catch(e) { + console.error('AudioEncoder missing or does not support', config); + console.error(e); + } + } } else if (has(config, 'width') && has(config, 'height')) { const videoConfig = config as VideoEncoderConfig; - if ('VideoEncoder' in window && await window.VideoEncoder.isConfigSupported(videoConfig)) - return window.VideoEncoder; + for (const source of [ window, LibAVPolyfill ]) { + if (source === LibAVPolyfill) { + await this.#load(); + } - await this.#load(); - if (await LibAVPolyfill.VideoEncoder.isConfigSupported(videoConfig)) - return LibAVPolyfill.VideoEncoder; + try { + const { supported } = await source.VideoEncoder.isConfigSupported(videoConfig); + if (supported) return source.VideoEncoder; + } catch(e) { + console.error('VideoEncoder missing or does not support', config); + console.error(e); + } + } } else throw new Error("unreachable"); return null;