blitzwing/cobalt-7
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

web/libav: use fitting class for Chunks to avoid unnecessary copies

Browse source
This commit is contained in:
dumbmoron 2024-08-24 16:03:06 +00:00
parent d7b5b9e1e1
commit 3cc3138cb5
No known key found for this signature in database
2 changed files with 104 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
web/src/lib/libav/encode.ts
View file

@ -4,6 +4,10 @@ import * as LibAVWebCodecs from "@imput/libavjs-webcodecs-bridge";
import { BufferStream } from "../buffer-stream"; import { BufferStream } from "../buffer-stream";
import WebCodecsWrapper from "./webcodecs"; import WebCodecsWrapper from "./webcodecs";
import LibAVWrapper from "./instance"; import LibAVWrapper from "./instance";
import {
EncodedAudioChunk as PolyfilledEncodedAudioChunk,
EncodedVideoChunk as PolyfilledEncodedVideoChunk
} from "@imput/libavjs-webcodecs-polyfill";
const QUEUE_THRESHOLD_MIN = 16; const QUEUE_THRESHOLD_MIN = 16;
const QUEUE_THRESHOLD_MAX = 128; const QUEUE_THRESHOLD_MAX = 128;
@ -53,7 +57,7 @@ export default class EncodeLibAV extends LibAVWrapper {
const { const {
pipe, pipe,
stream: ostream stream: ostream
} = await this.#createEncoder(stream, 'mp3'); } = await this.#createEncoder(stream, 'flac');
pipes.push({ pipes.push({
decoder: await this.#createDecoder(stream), decoder: await this.#createDecoder(stream),
@ -267,14 +271,25 @@ export default class EncodeLibAV extends LibAVWrapper {
} }
#decodePacket(decoder: Decoder, packet: Packet, stream: Stream) { #decodePacket(decoder: Decoder, packet: Packet, stream: Stream) {
let chunk; let decoderType;
if (WebCodecsWrapper.isVideo(decoder)) {
chunk = LibAVWebCodecs.packetToEncodedVideoChunk(packet, stream);
} else if (WebCodecsWrapper.isAudio(decoder)) {
chunk = LibAVWebCodecs.packetToEncodedAudioChunk(packet, stream);
}
decoder.decode(chunk); if (decoderType = WebCodecsWrapper.isVideo(decoder)) {
const EncodedVideoChunk = decoderType === 'polyfilled' ? PolyfilledEncodedVideoChunk : window.EncodedVideoChunk;
WebCodecsWrapper.decodeVideo(
LibAVWebCodecs.packetToEncodedVideoChunk(
packet, stream, { EncodedVideoChunk }
),
decoder as VideoDecoder
);
} else if (decoderType = WebCodecsWrapper.isAudio(decoder)) {
const EncodedAudioChunk = decoderType === 'polyfilled' ? PolyfilledEncodedAudioChunk : window.EncodedAudioChunk;
WebCodecsWrapper.decodeAudio(
LibAVWebCodecs.packetToEncodedAudioChunk(
packet, stream, { EncodedAudioChunk }
),
decoder as AudioDecoder,
);
}
} }
async* #demux(fmt_ctx: number) { async* #demux(fmt_ctx: number) {

77
web/src/lib/libav/webcodecs.ts
View file

@ -139,25 +139,90 @@ export default class WebCodecsWrapper {
} }
static isVideo(obj: unknown) { static isVideo(obj: unknown) {
return ('VideoEncoder' in window && obj instanceof VideoEncoder) const isNative = ('VideoEncoder' in window && obj instanceof VideoEncoder)
|| ('VideoDecoder' in window && obj instanceof VideoDecoder) || ('VideoDecoder' in window && obj instanceof VideoDecoder)
|| ('VideoFrame' in window && obj instanceof VideoFrame) || ('VideoFrame' in window && obj instanceof VideoFrame)
|| ('EncodedVideoChunk' in window && obj instanceof EncodedVideoChunk) || ('EncodedVideoChunk' in window && obj instanceof EncodedVideoChunk);
|| obj instanceof LibAVPolyfill.VideoEncoder if (isNative) {
return 'native';
}
const isPolyfilled = obj instanceof LibAVPolyfill.VideoEncoder
|| obj instanceof LibAVPolyfill.VideoDecoder || obj instanceof LibAVPolyfill.VideoDecoder
|| obj instanceof LibAVPolyfill.VideoFrame || obj instanceof LibAVPolyfill.VideoFrame
|| obj instanceof LibAVPolyfill.EncodedVideoChunk; || obj instanceof LibAVPolyfill.EncodedVideoChunk;
if (isPolyfilled) {
return 'polyfilled';
}
} }
static isAudio(obj: unknown) { static isAudio(obj: unknown) {
return ('AudioEncoder' in window && obj instanceof AudioEncoder) const isNative = ('AudioEncoder' in window && obj instanceof AudioEncoder)
|| ('AudioDecoder' in window && obj instanceof AudioDecoder) || ('AudioDecoder' in window && obj instanceof AudioDecoder)
|| ('AudioData' in window && obj instanceof AudioData) || ('AudioData' in window && obj instanceof AudioData)
|| ('EncodedAudioChunk' in window && obj instanceof EncodedAudioChunk) || ('EncodedAudioChunk' in window && obj instanceof EncodedAudioChunk);
|| obj instanceof LibAVPolyfill.AudioEncoder
if (isNative) {
return 'native';
}
const isPolyfilled = obj instanceof LibAVPolyfill.AudioEncoder
|| obj instanceof LibAVPolyfill.AudioDecoder || obj instanceof LibAVPolyfill.AudioDecoder
|| obj instanceof LibAVPolyfill.AudioData || obj instanceof LibAVPolyfill.AudioData
|| obj instanceof LibAVPolyfill.EncodedAudioChunk; || obj instanceof LibAVPolyfill.EncodedAudioChunk;
if (isPolyfilled) {
return 'polyfilled';
}
}
static decodeAudio(
data: EncodedAudioChunk | LibAVPolyfill.EncodedAudioChunk,
destination: AudioDecoder
) {
const hasChunk = 'EncodedAudioChunk' in window
const isPolyfilled = hasChunk && window.EncodedAudioChunk === LibAVPolyfill.EncodedAudioChunk;
if (destination instanceof LibAVPolyfill.AudioDecoder) {
if (hasChunk && !isPolyfilled && data instanceof EncodedAudioChunk) {
data = LibAVPolyfill.EncodedAudioChunk.fromNative(data);
console.log('EncodedAudioChunk: native -> polyfill');
} else {
console.log('EncodedAudioChunk: passthrough (polyfill)');
}
} else {
if (data instanceof LibAVPolyfill.EncodedAudioChunk) {
data = data.toNative();
console.log('EncodedAudioChunk: polyfill -> native');
} else {
console.log('EncodedAudioChunk: passthrough (native)');
}
}
return destination.decode(data);
}
static decodeVideo(
data: EncodedVideoChunk | LibAVPolyfill.EncodedVideoChunk,
destination: VideoDecoder
) {
const hasChunk = 'EncodedVideoChunk' in window
const isPolyfilled = hasChunk && window.EncodedVideoChunk === LibAVPolyfill.EncodedVideoChunk;
if (destination instanceof LibAVPolyfill.VideoDecoder) {
if (hasChunk && !isPolyfilled && data instanceof EncodedVideoChunk) {
data = LibAVPolyfill.EncodedVideoChunk.fromNative(data);
console.log('EncodedVideoChunk: native -> polyfill');
} else {
console.log('EncodedVideoChunk: passthrough (polyfill)');
}
} else {
if (data instanceof LibAVPolyfill.EncodedVideoChunk) {
data = data.toNative();
console.log('EncodedVideoChunk: polyfill -> native');
} else {
console.log('EncodedVideoChunk: passthrough (native)');
}
}
return destination.decode(data);
} }
static encodeAudio(data: AudioData | LibAVPolyfill.AudioData, destination: AudioEncoder) { static encodeAudio(data: AudioData | LibAVPolyfill.AudioData, destination: AudioEncoder) {