blitzwing/cobalt-7
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

web/libav: use fitting class for Chunks to avoid unnecessary copies

Browse source
This commit is contained in:
dumbmoron 2024-08-24 16:03:06 +00:00
parent d7b5b9e1e1
commit 3cc3138cb5
No known key found for this signature in database
2 changed files with 104 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
web/src/lib/libav/encode.ts
View file

@ -4,6 +4,10 @@ import * as LibAVWebCodecs from "@imput/libavjs-webcodecs-bridge";
import { BufferStream } from "../buffer-stream";
import WebCodecsWrapper from "./webcodecs";
import LibAVWrapper from "./instance";
import {
EncodedAudioChunk as PolyfilledEncodedAudioChunk,
EncodedVideoChunk as PolyfilledEncodedVideoChunk
} from "@imput/libavjs-webcodecs-polyfill";
const QUEUE_THRESHOLD_MIN = 16;
const QUEUE_THRESHOLD_MAX = 128;
@ -53,7 +57,7 @@ export default class EncodeLibAV extends LibAVWrapper {
const {
pipe,
stream: ostream
} = await this.#createEncoder(stream, 'mp3');
} = await this.#createEncoder(stream, 'flac');
pipes.push({
decoder: await this.#createDecoder(stream),
@ -267,14 +271,25 @@ export default class EncodeLibAV extends LibAVWrapper {
}
#decodePacket(decoder: Decoder, packet: Packet, stream: Stream) {
let chunk;
if (WebCodecsWrapper.isVideo(decoder)) {
chunk = LibAVWebCodecs.packetToEncodedVideoChunk(packet, stream);
} else if (WebCodecsWrapper.isAudio(decoder)) {
chunk = LibAVWebCodecs.packetToEncodedAudioChunk(packet, stream);
}
let decoderType;
decoder.decode(chunk);
if (decoderType = WebCodecsWrapper.isVideo(decoder)) {
const EncodedVideoChunk = decoderType === 'polyfilled' ? PolyfilledEncodedVideoChunk : window.EncodedVideoChunk;
WebCodecsWrapper.decodeVideo(
LibAVWebCodecs.packetToEncodedVideoChunk(
packet, stream, { EncodedVideoChunk }
),
decoder as VideoDecoder
);
} else if (decoderType = WebCodecsWrapper.isAudio(decoder)) {
const EncodedAudioChunk = decoderType === 'polyfilled' ? PolyfilledEncodedAudioChunk : window.EncodedAudioChunk;
WebCodecsWrapper.decodeAudio(
LibAVWebCodecs.packetToEncodedAudioChunk(
packet, stream, { EncodedAudioChunk }
),
decoder as AudioDecoder,
);
}
}
async* #demux(fmt_ctx: number) {

97
web/src/lib/libav/webcodecs.ts
View file

@ -139,25 +139,90 @@ export default class WebCodecsWrapper {
}
static isVideo(obj: unknown) {
return ('VideoEncoder' in window && obj instanceof VideoEncoder)
|| ('VideoDecoder' in window && obj instanceof VideoDecoder)
|| ('VideoFrame' in window && obj instanceof VideoFrame)
|| ('EncodedVideoChunk' in window && obj instanceof EncodedVideoChunk)
|| obj instanceof LibAVPolyfill.VideoEncoder
|| obj instanceof LibAVPolyfill.VideoDecoder
|| obj instanceof LibAVPolyfill.VideoFrame
|| obj instanceof LibAVPolyfill.EncodedVideoChunk;
const isNative = ('VideoEncoder' in window && obj instanceof VideoEncoder)
|| ('VideoDecoder' in window && obj instanceof VideoDecoder)
|| ('VideoFrame' in window && obj instanceof VideoFrame)
|| ('EncodedVideoChunk' in window && obj instanceof EncodedVideoChunk);
if (isNative) {
return 'native';
}
const isPolyfilled = obj instanceof LibAVPolyfill.VideoEncoder
|| obj instanceof LibAVPolyfill.VideoDecoder
|| obj instanceof LibAVPolyfill.VideoFrame
|| obj instanceof LibAVPolyfill.EncodedVideoChunk;
if (isPolyfilled) {
return 'polyfilled';
}
}
static isAudio(obj: unknown) {
return ('AudioEncoder' in window && obj instanceof AudioEncoder)
|| ('AudioDecoder' in window && obj instanceof AudioDecoder)
|| ('AudioData' in window && obj instanceof AudioData)
|| ('EncodedAudioChunk' in window && obj instanceof EncodedAudioChunk)
|| obj instanceof LibAVPolyfill.AudioEncoder
|| obj instanceof LibAVPolyfill.AudioDecoder
|| obj instanceof LibAVPolyfill.AudioData
|| obj instanceof LibAVPolyfill.EncodedAudioChunk;
const isNative = ('AudioEncoder' in window && obj instanceof AudioEncoder)
|| ('AudioDecoder' in window && obj instanceof AudioDecoder)
|| ('AudioData' in window && obj instanceof AudioData)
|| ('EncodedAudioChunk' in window && obj instanceof EncodedAudioChunk);
if (isNative) {
return 'native';
}
const isPolyfilled = obj instanceof LibAVPolyfill.AudioEncoder
|| obj instanceof LibAVPolyfill.AudioDecoder
|| obj instanceof LibAVPolyfill.AudioData
|| obj instanceof LibAVPolyfill.EncodedAudioChunk;
if (isPolyfilled) {
return 'polyfilled';
}
}
static decodeAudio(
data: EncodedAudioChunk | LibAVPolyfill.EncodedAudioChunk,
destination: AudioDecoder
) {
const hasChunk = 'EncodedAudioChunk' in window
const isPolyfilled = hasChunk && window.EncodedAudioChunk === LibAVPolyfill.EncodedAudioChunk;
if (destination instanceof LibAVPolyfill.AudioDecoder) {
if (hasChunk && !isPolyfilled && data instanceof EncodedAudioChunk) {
data = LibAVPolyfill.EncodedAudioChunk.fromNative(data);
console.log('EncodedAudioChunk: native -> polyfill');
} else {
console.log('EncodedAudioChunk: passthrough (polyfill)');
}
} else {
if (data instanceof LibAVPolyfill.EncodedAudioChunk) {
data = data.toNative();
console.log('EncodedAudioChunk: polyfill -> native');
} else {
console.log('EncodedAudioChunk: passthrough (native)');
}
}
return destination.decode(data);
}
static decodeVideo(
data: EncodedVideoChunk | LibAVPolyfill.EncodedVideoChunk,
destination: VideoDecoder
) {
const hasChunk = 'EncodedVideoChunk' in window
const isPolyfilled = hasChunk && window.EncodedVideoChunk === LibAVPolyfill.EncodedVideoChunk;
if (destination instanceof LibAVPolyfill.VideoDecoder) {
if (hasChunk && !isPolyfilled && data instanceof EncodedVideoChunk) {
data = LibAVPolyfill.EncodedVideoChunk.fromNative(data);
console.log('EncodedVideoChunk: native -> polyfill');
} else {
console.log('EncodedVideoChunk: passthrough (polyfill)');
}
} else {
if (data instanceof LibAVPolyfill.EncodedVideoChunk) {
data = data.toNative();
console.log('EncodedVideoChunk: polyfill -> native');
} else {
console.log('EncodedVideoChunk: passthrough (native)');
}
}
return destination.decode(data);
}
static encodeAudio(data: AudioData | LibAVPolyfill.AudioData, destination: AudioEncoder) {