PrivateCoffee/element-web
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Merge branch 'develop' of github.com:matrix-org/matrix-react-sdk into t3chguy/ts/9

Browse source
This commit is contained in:
Michael Telatynski 2021-07-01 08:21:59 +01:00
commit 6693993d08
36 changed files with 855 additions and 260 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
.eslintrc.js
View file

@ -24,6 +24,18 @@ module.exports = {
// It's disabled here, but we should using it sparingly.
"react/jsx-no-bind": "off",
"react/jsx-key": ["error"],
"no-restricted-properties": [
"error",
...buildRestrictedPropertiesOptions(
["window.innerHeight", "window.innerWidth", "window.visualViewport"],
"Use UIStore to access window dimensions instead.",
),
...buildRestrictedPropertiesOptions(
["*.mxcUrlToHttp", "*.getHttpUriForMxc"],
"Use Media helper instead to centralise access for customisation.",
),
],
},
overrides: [{
files: [
@ -49,21 +61,16 @@ module.exports = {
"@typescript-eslint/no-explicit-any": "off",
// We'd rather not do this but we do
"@typescript-eslint/ban-ts-comment": "off",
"no-restricted-properties": [
"error",
...buildRestrictedPropertiesOptions(
["window.innerHeight", "window.innerWidth", "window.visualViewport"],
"Use UIStore to access window dimensions instead",
),
],
},
}],
};
function buildRestrictedPropertiesOptions(properties, message) {
return properties.map(prop => {
const [object, property] = prop.split(".");
let [object, property] = prop.split(".");
if (object === "*") {
object = undefined;
}
return {
object,
property,

9
res/css/_components.scss
View file

@ -37,6 +37,11 @@
@import "./structures/_ViewSource.scss";
@import "./structures/auth/_CompleteSecurity.scss";
@import "./structures/auth/_Login.scss";
@import "./views/audio_messages/_AudioPlayer.scss";
@import "./views/audio_messages/_PlayPauseButton.scss";
@import "./views/audio_messages/_PlaybackContainer.scss";
@import "./views/audio_messages/_SeekBar.scss";
@import "./views/audio_messages/_Waveform.scss";
@import "./views/auth/_AuthBody.scss";
@import "./views/auth/_AuthButtons.scss";
@import "./views/auth/_AuthFooter.scss";
@ -165,6 +170,7 @@
@import "./views/messages/_MTextBody.scss";
@import "./views/messages/_MVideoBody.scss";
@import "./views/messages/_MVoiceMessageBody.scss";
@import "./views/messages/_MediaBody.scss";
@import "./views/messages/_MessageActionBar.scss";
@import "./views/messages/_MessageTimestamp.scss";
@import "./views/messages/_MjolnirBody.scss";
@ -253,9 +259,6 @@
@import "./views/toasts/_AnalyticsToast.scss";
@import "./views/toasts/_NonUrgentEchoFailureToast.scss";
@import "./views/verification/_VerificationShowSas.scss";
@import "./views/voice_messages/_PlayPauseButton.scss";
@import "./views/voice_messages/_PlaybackContainer.scss";
@import "./views/voice_messages/_Waveform.scss";
@import "./views/voip/_CallContainer.scss";
@import "./views/voip/_CallView.scss";
@import "./views/voip/_CallViewForRoom.scss";

2
res/css/structures/_GroupView.scss
View file

@ -323,7 +323,7 @@ limitations under the License.
}
.mx_GroupView_featuredThing .mx_BaseAvatar {
/* To prevent misalignment with mx_TintableSvg (in addButton) */
/* To prevent misalignment with img (in addButton) */
vertical-align: initial;
}

68
res/css/views/audio_messages/_AudioPlayer.scss Normal file
View file

@ -0,0 +1,68 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
.mx_AudioPlayer_container {
padding: 16px 12px 12px 12px;
max-width: 267px; // use max to make the control fit in the files/pinned panels
.mx_AudioPlayer_primaryContainer {
display: flex;
.mx_PlayPauseButton {
margin-right: 8px;
}
.mx_AudioPlayer_mediaInfo {
flex: 1;
overflow: hidden; // makes the ellipsis on the file name work
& > * {
display: block;
}
.mx_AudioPlayer_mediaName {
color: $primary-fg-color;
font-size: $font-15px;
line-height: $font-15px;
text-overflow: ellipsis;
white-space: nowrap;
overflow: hidden;
padding-bottom: 4px; // mimics the line-height differences in the Figma
}
.mx_AudioPlayer_byline {
font-size: $font-12px;
line-height: $font-12px;
}
}
}
.mx_AudioPlayer_seek {
display: flex;
align-items: center;
.mx_SeekBar {
flex: 1;
}
.mx_Clock {
width: $font-42px; // we're not using a monospace font, so fake it
min-width: $font-42px; // for flexbox
padding-left: 4px; // isolate from seek bar
text-align: right;
}
}
}

2
res/css/views/voice_messages/_PlayPauseButton.scss → res/css/views/audio_messages/_PlayPauseButton.scss
View file

@ -18,6 +18,8 @@ limitations under the License.
position: relative;
width: 32px;
height: 32px;
min-width: 32px; // for when the button is used in a flexbox
min-height: 32px; // for when the button is used in a flexbox
border-radius: 32px;
background-color: $voice-playback-button-bg-color;

8
res/css/views/voice_messages/_PlaybackContainer.scss → res/css/views/audio_messages/_PlaybackContainer.scss
View file

@ -22,17 +22,11 @@ limitations under the License.
// 7px top and bottom for visual design. 12px left & right, but the waveform (right)
// has a 1px padding on it that we want to account for.
padding: 7px 12px 7px 11px;
background-color: $voice-record-waveform-bg-color;
border-radius: 12px;
// Cheat at alignment a bit
display: flex;
align-items: center;
color: $voice-record-waveform-fg-color;
font-size: $font-14px;
line-height: $font-24px;
contain: content;
.mx_Waveform {
@ -45,7 +39,7 @@ limitations under the License.
&.mx_Waveform_bar_100pct {
// Small animation to remove the mechanical feel of progress
transition: background-color 250ms ease;
background-color: $voice-record-waveform-fg-color;
background-color: $message-body-panel-fg-color;
}
}
}

103
res/css/views/audio_messages/_SeekBar.scss Normal file
View file

@ -0,0 +1,103 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// CSS inspiration from:
// * https://www.w3schools.com/howto/howto_js_rangeslider.asp
// * https://stackoverflow.com/a/28283806
// * https://css-tricks.com/styling-cross-browser-compatible-range-inputs-css/
.mx_SeekBar {
// Dev note: we deliberately do not have the -ms-track (and friends) selectors because we don't
// need to support IE.
appearance: none; // default style override
width: 100%;
height: 1px;
background: $quaternary-fg-color;
outline: none; // remove blue selection border
position: relative; // for before+after pseudo elements later on
cursor: pointer;
&::-webkit-slider-thumb {
appearance: none; // default style override
// Dev note: This needs to be duplicated with the -moz-range-thumb selector
// because otherwise Edge (webkit) will fail to see the styles and just refuse
// to apply them.
width: 8px;
height: 8px;
border-radius: 8px;
background-color: $tertiary-fg-color;
cursor: pointer;
}
&::-moz-range-thumb {
width: 8px;
height: 8px;
border-radius: 8px;
background-color: $tertiary-fg-color;
cursor: pointer;
// Firefox adds a border on the thumb
border: none;
}
// This is for webkit support, but we can't limit the functionality of it to just webkit
// browsers. Firefox responds to webkit-prefixed values now, which means we can't use media
// or support queries to selectively apply the rule. An upside is that this CSS doesn't work
// in firefox, so it's just wasted CPU/GPU time.
&::before { // ::before to ensure it ends up under the thumb
content: '';
background-color: $tertiary-fg-color;
// Absolute positioning to ensure it overlaps with the existing bar
position: absolute;
top: 0;
left: 0;
// Sizing to match the bar
width: 100%;
height: 1px;
// And finally dynamic width without overly hurting the rendering engine.
transform-origin: 0 100%;
transform: scaleX(var(--fillTo));
}
// This is firefox's built-in support for the above, with 100% less hacks.
&::-moz-range-progress {
background-color: $tertiary-fg-color;
height: 1px;
}
&:disabled {
opacity: 0.5;
}
// Increase clickable area for the slider (approximately same size as browser default)
// We do it this way to keep the same padding and margins of the element, avoiding margin math.
// Source: https://front-back.com/expand-clickable-areas-for-a-better-touch-experience/
&::after {
content: '';
position: absolute;
top: -6px;
bottom: -6px;
left: 0;
right: 0;
}
}

0
res/css/views/voice_messages/_Waveform.scss → res/css/views/audio_messages/_Waveform.scss
View file

2
res/css/views/messages/_MImageBody.scss
View file

@ -43,7 +43,7 @@ limitations under the License.
top: 50%;
}
// Inner img and TintableSvg should be centered around 0, 0
// Inner img should be centered around 0, 0
.mx_MImageBody_thumbnail_spinner > * {
transform: translate(-50%, -50%);
}

28
res/css/views/messages/_MediaBody.scss Normal file
View file

@ -0,0 +1,28 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// A "media body" is any file upload looking thing, apart from images and videos (they
// have unique styles).
.mx_MediaBody {
background-color: $message-body-panel-bg-color;
border-radius: 12px;
color: $message-body-panel-fg-color;
font-size: $font-14px;
line-height: $font-24px;
}

2
res/themes/dark/css/_dark.scss
View file

@ -215,8 +215,6 @@ $message-body-panel-icon-fg-color: #21262C; // "Separator"
$message-body-panel-icon-bg-color: $tertiary-fg-color;
$voice-record-stop-border-color: $quaternary-fg-color;
$voice-record-waveform-bg-color: $message-body-panel-bg-color;
$voice-record-waveform-fg-color: $message-body-panel-fg-color;
$voice-record-waveform-incomplete-fg-color: $quaternary-fg-color;
$voice-record-icon-color: $quaternary-fg-color;
$voice-playback-button-bg-color: $message-body-panel-icon-bg-color;

5
res/themes/legacy-dark/css/_legacy-dark.scss
View file

@ -20,6 +20,9 @@ $tertiary-fg-color: $primary-fg-color;
$primary-bg-color: $bg-color;
$muted-fg-color: $header-panel-text-primary-color;
// Legacy theme backports
$quaternary-fg-color: #6F7882;
// used for dialog box text
$light-fg-color: $header-panel-text-secondary-color;
@ -209,8 +212,6 @@ $message-body-panel-icon-bg-color: $secondary-fg-color;
// See non-legacy dark for variable information
$voice-record-stop-border-color: #6F7882;
$voice-record-waveform-bg-color: $message-body-panel-bg-color;
$voice-record-waveform-fg-color: $message-body-panel-fg-color;
$voice-record-waveform-incomplete-fg-color: #6F7882;
$voice-record-icon-color: #6F7882;
$voice-playback-button-bg-color: $tertiary-fg-color;

5
res/themes/legacy-light/css/_legacy-light.scss
View file

@ -28,6 +28,9 @@ $tertiary-fg-color: $primary-fg-color;
$primary-bg-color: #ffffff;
$muted-fg-color: #61708b; // Commonly used in headings and relevant alt text
// Legacy theme backports
$quaternary-fg-color: #C1C6CD;
// used for dialog box text
$light-fg-color: #747474;
@ -334,8 +337,6 @@ $message-body-panel-icon-bg-color: $primary-bg-color;
$voice-record-stop-symbol-color: #ff4b55;
$voice-record-live-circle-color: #ff4b55;
$voice-record-stop-border-color: #E3E8F0;
$voice-record-waveform-bg-color: $message-body-panel-bg-color;
$voice-record-waveform-fg-color: $message-body-panel-fg-color;
$voice-record-waveform-incomplete-fg-color: #C1C6CD;
$voice-record-icon-color: $tertiary-fg-color;
$voice-playback-button-bg-color: $message-body-panel-icon-bg-color;

2
res/themes/light/css/_light.scss
View file

@ -335,8 +335,6 @@ $voice-record-stop-symbol-color: #ff4b55;
$voice-record-live-circle-color: #ff4b55;
$voice-record-stop-border-color: #E3E8F0; // "Separator"
$voice-record-waveform-bg-color: $message-body-panel-bg-color;
$voice-record-waveform-fg-color: $message-body-panel-fg-color;
$voice-record-waveform-incomplete-fg-color: $quaternary-fg-color;
$voice-record-icon-color: $tertiary-fg-color;
$voice-playback-button-bg-color: $message-body-panel-icon-bg-color;

9
src/components/structures/RoomView.tsx
View file

@ -1054,11 +1054,6 @@ export default class RoomView extends React.Component<IProps, IState> {
});
}
private updateTint() {
const room = this.state.room;
if (!room) return;
}
private onAccountData = (event: MatrixEvent) => {
const type = event.getType();
if ((type === "org.matrix.preview_urls" || type === "im.vector.web.settings") && this.state.room) {
@ -1705,10 +1700,6 @@ export default class RoomView extends React.Component<IProps, IState> {
// otherwise react calls it with null on each update.
private gatherTimelinePanelRef = r => {
this.messagePanel = r;
if (r) {
console.log("updateTint from RoomView.gatherTimelinePanelRef");
this.updateTint();
}
};
private getOldRoom() {

124
src/components/views/audio_messages/AudioPlayer.tsx Normal file
View file

@ -0,0 +1,124 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { Playback, PlaybackState } from "../../../voice/Playback";
import React, { createRef, ReactNode, RefObject } from "react";
import { UPDATE_EVENT } from "../../../stores/AsyncStore";
import PlayPauseButton from "./PlayPauseButton";
import { replaceableComponent } from "../../../utils/replaceableComponent";
import { formatBytes } from "../../../utils/FormattingUtils";
import DurationClock from "./DurationClock";
import { Key } from "../../../Keyboard";
import { _t } from "../../../languageHandler";
import SeekBar from "./SeekBar";
import PlaybackClock from "./PlaybackClock";
interface IProps {
// Playback instance to render. Cannot change during component lifecycle: create
// an all-new component instead.
playback: Playback;
mediaName: string;
}
interface IState {
playbackPhase: PlaybackState;
}
@replaceableComponent("views.audio_messages.AudioPlayer")
export default class AudioPlayer extends React.PureComponent<IProps, IState> {
private playPauseRef: RefObject<PlayPauseButton> = createRef();
private seekRef: RefObject<SeekBar> = createRef();
constructor(props: IProps) {
super(props);
this.state = {
playbackPhase: PlaybackState.Decoding, // default assumption
};
// We don't need to de-register: the class handles this for us internally
this.props.playback.on(UPDATE_EVENT, this.onPlaybackUpdate);
// Don't wait for the promise to complete - it will emit a progress update when it
// is done, and it's not meant to take long anyhow.
// noinspection JSIgnoredPromiseFromCall
this.props.playback.prepare();
}
private onPlaybackUpdate = (ev: PlaybackState) => {
this.setState({ playbackPhase: ev });
};
private onKeyDown = (ev: React.KeyboardEvent) => {
// stopPropagation() prevents the FocusComposer catch-all from triggering,
// but we need to do it on key down instead of press (even though the user
// interaction is typically on press).
if (ev.key === Key.SPACE) {
ev.stopPropagation();
this.playPauseRef.current?.toggleState();
} else if (ev.key === Key.ARROW_LEFT) {
ev.stopPropagation();
this.seekRef.current?.left();
} else if (ev.key === Key.ARROW_RIGHT) {
ev.stopPropagation();
this.seekRef.current?.right();
}
};
protected renderFileSize(): string {
const bytes = this.props.playback.sizeBytes;
if (!bytes) return null;
// Not translated here - we're just presenting the data which should already
// be translated if needed.
return `(${formatBytes(bytes)})`;
}
public render(): ReactNode {
// tabIndex=0 to ensure that the whole component becomes a tab stop, where we handle keyboard
// events for accessibility
return <div className='mx_MediaBody mx_AudioPlayer_container' tabIndex={0} onKeyDown={this.onKeyDown}>
<div className='mx_AudioPlayer_primaryContainer'>
<PlayPauseButton
playback={this.props.playback}
playbackPhase={this.state.playbackPhase}
tabIndex={-1} // prevent tabbing into the button
ref={this.playPauseRef}
/>
<div className='mx_AudioPlayer_mediaInfo'>
<span className='mx_AudioPlayer_mediaName'>
{this.props.mediaName || _t("Unnamed audio")}
</span>
<div className='mx_AudioPlayer_byline'>
<DurationClock playback={this.props.playback} />
&nbsp; {/* easiest way to introduce a gap between the components */}
{ this.renderFileSize() }
</div>
</div>
</div>
<div className='mx_AudioPlayer_seek'>
<SeekBar
playback={this.props.playback}
tabIndex={-1} // prevent tabbing into the bar
playbackPhase={this.state.playbackPhase}
ref={this.seekRef}
/>
<PlaybackClock playback={this.props.playback} defaultDisplaySeconds={0} />
</div>
</div>;
}
}

2
src/components/views/voice_messages/Clock.tsx → src/components/views/audio_messages/Clock.tsx
View file

@ -28,7 +28,7 @@ interface IState {
* Simply converts seconds into minutes and seconds. Note that hours will not be
* displayed, making it possible to see "82:29".
*/
@replaceableComponent("views.voice_messages.Clock")
@replaceableComponent("views.audio_messages.Clock")
export default class Clock extends React.Component<IProps, IState> {
public constructor(props) {
super(props);

55
src/components/views/audio_messages/DurationClock.tsx Normal file
View file

@ -0,0 +1,55 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React from "react";
import { replaceableComponent } from "../../../utils/replaceableComponent";
import Clock from "./Clock";
import { Playback } from "../../../voice/Playback";
interface IProps {
playback: Playback;
}
interface IState {
durationSeconds: number;
}
/**
* A clock which shows a clip's maximum duration.
*/
@replaceableComponent("views.audio_messages.DurationClock")
export default class DurationClock extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);
this.state = {
// we track the duration on state because we won't really know what the clip duration
// is until the first time update, and as a PureComponent we are trying to dedupe state
// updates as much as possible. This is just the easiest way to avoid a forceUpdate() or
// member property to track "did we get a duration".
durationSeconds: this.props.playback.clockInfo.durationSeconds,
};
this.props.playback.clockInfo.liveData.onUpdate(this.onTimeUpdate);
}
private onTimeUpdate = (time: number[]) => {
this.setState({ durationSeconds: time[1] });
};
public render() {
return <Clock seconds={this.state.durationSeconds} />;
}
}

14
src/components/views/voice_messages/LiveRecordingClock.tsx → src/components/views/audio_messages/LiveRecordingClock.tsx
View file

@ -1,9 +1,12 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -12,16 +15,13 @@ limitations under the License.
*/
import React from "react";
import Clock from "./Clock";
import { IRecordingUpdate, VoiceRecording } from "../../../voice/VoiceRecording";
import { replaceableComponent } from "../../../utils/replaceableComponent";
import Clock from "./Clock";
import { MarkedExecution } from "../../../utils/MarkedExecution";
import {
IRecordingUpdate,
VoiceRecording,
} from "../../../voice/VoiceRecording";
interface IProps {
recorder?: VoiceRecording;
recorder: VoiceRecording;
}
interface IState {
@ -31,7 +31,7 @@ interface IState {
/**
* A clock for a live recording.
*/
@replaceableComponent("views.voice_messages.LiveRecordingClock")
@replaceableComponent("views.audio_messages.LiveRecordingClock")
export default class LiveRecordingClock extends React.PureComponent<IProps, IState> {
private seconds = 0;
private scheduledUpdate = new MarkedExecution(

27
src/components/views/voice_messages/LiveRecordingWaveform.tsx → src/components/views/audio_messages/LiveRecordingWaveform.tsx
View file

@ -1,9 +1,12 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -12,16 +15,15 @@ limitations under the License.
*/
import React from "react";
import Waveform from "./Waveform";
import { IRecordingUpdate, RECORDING_PLAYBACK_SAMPLES, VoiceRecording } from "../../../voice/VoiceRecording";
import { replaceableComponent } from "../../../utils/replaceableComponent";
import { arrayFastResample } from "../../../utils/arrays";
import { percentageOf } from "../../../utils/numbers";
import Waveform from "./Waveform";
import { MarkedExecution } from "../../../utils/MarkedExecution";
import {
IRecordingUpdate,
VoiceRecording,
} from "../../../voice/VoiceRecording";
interface IProps {
recorder?: VoiceRecording;
recorder: VoiceRecording;
}
interface IState {
@ -31,7 +33,7 @@ interface IState {
/**
* A waveform which shows the waveform of a live recording
*/
@replaceableComponent("views.voice_messages.LiveRecordingWaveform")
@replaceableComponent("views.audio_messages.LiveRecordingWaveform")
export default class LiveRecordingWaveform extends React.PureComponent<IProps, IState> {
public static defaultProps = {
progress: 1,
@ -52,15 +54,18 @@ export default class LiveRecordingWaveform extends React.PureComponent<IProps, I
componentDidMount() {
this.props.recorder.liveData.onUpdate((update: IRecordingUpdate) => {
this.waveform = update.waveform;
const bars = arrayFastResample(Array.from(update.waveform), RECORDING_PLAYBACK_SAMPLES);
// The incoming data is between zero and one, but typically even screaming into a
// microphone won't send you over 0.6, so we artificially adjust the gain for the
// waveform. This results in a slightly more cinematic/animated waveform for the
// user.
this.waveform = bars.map(b => percentageOf(b, 0, 0.50));
this.scheduledUpdate.mark();
});
}
private updateWaveform() {
this.setState({
waveform: this.waveform,
});
this.setState({ waveform: this.waveform });
}
public render() {

20
src/components/views/voice_messages/PlayPauseButton.tsx → src/components/views/audio_messages/PlayPauseButton.tsx
View file

@ -21,7 +21,8 @@ import { _t } from "../../../languageHandler";
import { Playback, PlaybackState } from "../../../voice/Playback";
import classNames from "classnames";
interface IProps {
// omitted props are handled by render function
interface IProps extends Omit<React.ComponentProps<typeof AccessibleTooltipButton>, "title" | "onClick" | "disabled"> {
// Playback instance to manipulate. Cannot change during the component lifecycle.
playback: Playback;
@ -33,19 +34,25 @@ interface IProps {
* Displays a play/pause button (activating the play/pause function of the recorder)
* to be displayed in reference to a recording.
*/
@replaceableComponent("views.voice_messages.PlayPauseButton")
@replaceableComponent("views.audio_messages.PlayPauseButton")
export default class PlayPauseButton extends React.PureComponent<IProps> {
public constructor(props) {
super(props);
}
private onClick = async () => {
await this.props.playback.toggle();
private onClick = () => {
// noinspection JSIgnoredPromiseFromCall
this.toggleState();
};
public async toggleState() {
await this.props.playback.toggle();
}
public render(): ReactNode {
const isPlaying = this.props.playback.isPlaying;
const isDisabled = this.props.playbackPhase === PlaybackState.Decoding;
const { playback, playbackPhase, ...restProps } = this.props;
const isPlaying = playback.isPlaying;
const isDisabled = playbackPhase === PlaybackState.Decoding;
const classes = classNames('mx_PlayPauseButton', {
'mx_PlayPauseButton_play': !isPlaying,
'mx_PlayPauseButton_pause': isPlaying,
@ -56,6 +63,7 @@ export default class PlayPauseButton extends React.PureComponent<IProps> {
title={isPlaying ? _t("Pause") : _t("Play")}
onClick={this.onClick}
disabled={isDisabled}
{...restProps}
/>;
}
}

13
src/components/views/voice_messages/PlaybackClock.tsx → src/components/views/audio_messages/PlaybackClock.tsx
View file

@ -22,6 +22,11 @@ import { UPDATE_EVENT } from "../../../stores/AsyncStore";
interface IProps {
playback: Playback;
// The default number of seconds to show when the playback has completed or
// has not started. Not used during playback, even when paused. Defaults to
// clip duration length.
defaultDisplaySeconds?: number;
}
interface IState {
@ -33,7 +38,7 @@ interface IState {
/**
* A clock for a playback of a recording.
*/
@replaceableComponent("views.voice_messages.PlaybackClock")
@replaceableComponent("views.audio_messages.PlaybackClock")
export default class PlaybackClock extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);
@ -64,7 +69,11 @@ export default class PlaybackClock extends React.PureComponent<IProps, IState> {
public render() {
let seconds = this.state.seconds;
if (this.state.playbackPhase === PlaybackState.Stopped) {
seconds = this.state.durationSeconds;
if (Number.isFinite(this.props.defaultDisplaySeconds)) {
seconds = this.props.defaultDisplaySeconds;
} else {
seconds = this.state.durationSeconds;
}
}
return <Clock seconds={seconds} />;
}

2
src/components/views/voice_messages/PlaybackWaveform.tsx → src/components/views/audio_messages/PlaybackWaveform.tsx
View file

@ -33,7 +33,7 @@ interface IState {
/**
* A waveform which shows the waveform of a previously recorded recording
*/
@replaceableComponent("views.voice_messages.PlaybackWaveform")
@replaceableComponent("views.audio_messages.PlaybackWaveform")
export default class PlaybackWaveform extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);

4
src/components/views/voice_messages/RecordingPlayback.tsx → src/components/views/audio_messages/RecordingPlayback.tsx
View file

@ -20,6 +20,7 @@ import { UPDATE_EVENT } from "../../../stores/AsyncStore";
import PlaybackWaveform from "./PlaybackWaveform";
import PlayPauseButton from "./PlayPauseButton";
import PlaybackClock from "./PlaybackClock";
import { replaceableComponent } from "../../../utils/replaceableComponent";
interface IProps {
// Playback instance to render. Cannot change during component lifecycle: create
@ -31,6 +32,7 @@ interface IState {
playbackPhase: PlaybackState;
}
@replaceableComponent("views.audio_messages.RecordingPlayback")
export default class RecordingPlayback extends React.PureComponent<IProps, IState> {
constructor(props: IProps) {
super(props);
@ -53,7 +55,7 @@ export default class RecordingPlayback extends React.PureComponent<IProps, IStat
};
public render(): ReactNode {
return <div className='mx_VoiceMessagePrimaryContainer'>
return <div className='mx_MediaBody mx_VoiceMessagePrimaryContainer'>
<PlayPauseButton playback={this.props.playback} playbackPhase={this.state.playbackPhase} />
<PlaybackClock playback={this.props.playback} />
<PlaybackWaveform playback={this.props.playback} />

112
src/components/views/audio_messages/SeekBar.tsx Normal file
View file

@ -0,0 +1,112 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { Playback, PlaybackState } from "../../../voice/Playback";
import React, { ChangeEvent, CSSProperties, ReactNode } from "react";
import { replaceableComponent } from "../../../utils/replaceableComponent";
import { MarkedExecution } from "../../../utils/MarkedExecution";
import { percentageOf } from "../../../utils/numbers";
interface IProps {
// Playback instance to render. Cannot change during component lifecycle: create
// an all-new component instead.
playback: Playback;
// Tab index for the underlying component. Useful if the seek bar is in a managed state.
// Defaults to zero.
tabIndex?: number;
playbackPhase: PlaybackState;
}
interface IState {
percentage: number;
}
interface ISeekCSS extends CSSProperties {
'--fillTo': number;
}
const ARROW_SKIP_SECONDS = 5; // arbitrary
@replaceableComponent("views.audio_messages.SeekBar")
export default class SeekBar extends React.PureComponent<IProps, IState> {
// We use an animation frame request to avoid overly spamming prop updates, even if we aren't
// really using anything demanding on the CSS front.
private animationFrameFn = new MarkedExecution(
() => this.doUpdate(),
() => requestAnimationFrame(() => this.animationFrameFn.trigger()));
public static defaultProps = {
tabIndex: 0,
};
constructor(props: IProps) {
super(props);
this.state = {
percentage: 0,
};
// We don't need to de-register: the class handles this for us internally
this.props.playback.clockInfo.liveData.onUpdate(() => this.animationFrameFn.mark());
}
private doUpdate() {
this.setState({
percentage: percentageOf(
this.props.playback.clockInfo.timeSeconds,
0,
this.props.playback.clockInfo.durationSeconds),
});
}
public left() {
// noinspection JSIgnoredPromiseFromCall
this.props.playback.skipTo(this.props.playback.clockInfo.timeSeconds - ARROW_SKIP_SECONDS);
}
public right() {
// noinspection JSIgnoredPromiseFromCall
this.props.playback.skipTo(this.props.playback.clockInfo.timeSeconds + ARROW_SKIP_SECONDS);
}
private onChange = (ev: ChangeEvent<HTMLInputElement>) => {
// Thankfully, onChange is only called when the user changes the value, not when we
// change the value on the component. We can use this as a reliable "skip to X" function.
//
// noinspection JSIgnoredPromiseFromCall
this.props.playback.skipTo(Number(ev.target.value) * this.props.playback.clockInfo.durationSeconds);
};
public render(): ReactNode {
// We use a range input to avoid having to re-invent accessibility handling on
// a custom set of divs.
return <input
type="range"
className='mx_SeekBar'
tabIndex={this.props.tabIndex}
onChange={this.onChange}
min={0}
max={1}
value={this.state.percentage}
step={0.001}
style={{ '--fillTo': this.state.percentage } as ISeekCSS}
disabled={this.props.playbackPhase === PlaybackState.Decoding}
/>;
}
}

16
src/components/views/voice_messages/Waveform.tsx → src/components/views/audio_messages/Waveform.tsx
View file

@ -17,8 +17,13 @@ limitations under the License.
import React from "react";
import { replaceableComponent } from "../../../utils/replaceableComponent";
import classNames from "classnames";
import { CSSProperties } from "react";
export interface IProps {
interface WaveformCSSProperties extends CSSProperties {
'--barHeight': number;
}
interface IProps {
relHeights: number[]; // relative heights (0-1)
progress: number; // percent complete, 0-1, default 100%
}
@ -34,14 +39,7 @@ interface IState {
* For CSS purposes, a mx_Waveform_bar_100pct class is added when the bar should be
* "filled", as a demonstration of the progress property.
*/
import { CSSProperties } from "react";
export interface WaveformCSSProperties extends CSSProperties {
'--barHeight': number;
}
@replaceableComponent("views.voice_messages.Waveform")
@replaceableComponent("views.audio_messages.Waveform")
export default class Waveform extends React.PureComponent<IProps, IState> {
public static defaultProps = {
progress: 1,

112
src/components/views/messages/MAudioBody.js
View file

@ -1,112 +0,0 @@
/*
Copyright 2016 OpenMarket Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React from 'react';
import MFileBody from './MFileBody';
import { decryptFile } from '../../../utils/DecryptFile';
import { _t } from '../../../languageHandler';
import InlineSpinner from '../elements/InlineSpinner';
import { replaceableComponent } from "../../../utils/replaceableComponent";
import { mediaFromContent } from "../../../customisations/Media";
@replaceableComponent("views.messages.MAudioBody")
export default class MAudioBody extends React.Component {
constructor(props) {
super(props);
this.state = {
playing: false,
decryptedUrl: null,
decryptedBlob: null,
error: null,
};
}
onPlayToggle() {
this.setState({
playing: !this.state.playing,
});
}
_getContentUrl() {
const media = mediaFromContent(this.props.mxEvent.getContent());
if (media.isEncrypted) {
return this.state.decryptedUrl;
} else {
return media.srcHttp;
}
}
componentDidMount() {
const content = this.props.mxEvent.getContent();
if (content.file !== undefined && this.state.decryptedUrl === null) {
let decryptedBlob;
decryptFile(content.file).then(function(blob) {
decryptedBlob = blob;
return URL.createObjectURL(decryptedBlob);
}).then((url) => {
this.setState({
decryptedUrl: url,
decryptedBlob: decryptedBlob,
});
}, (err) => {
console.warn("Unable to decrypt attachment: ", err);
this.setState({
error: err,
});
});
}
}
componentWillUnmount() {
if (this.state.decryptedUrl) {
URL.revokeObjectURL(this.state.decryptedUrl);
}
}
render() {
const content = this.props.mxEvent.getContent();
if (this.state.error !== null) {
return (
<span className="mx_MAudioBody">
<img src={require("../../../../res/img/warning.svg")} width="16" height="16" />
{ _t("Error decrypting audio") }
</span>
);
}
if (content.file !== undefined && this.state.decryptedUrl === null) {
// Need to decrypt the attachment
// The attachment is decrypted in componentDidMount.
// For now add an img tag with a 16x16 spinner.
// Not sure how tall the audio player is so not sure how tall it should actually be.
return (
<span className="mx_MAudioBody">
<InlineSpinner />
</span>
);
}
const contentUrl = this._getContentUrl();
return (
<span className="mx_MAudioBody">
<audio src={contentUrl} controls />
<MFileBody {...this.props} decryptedBlob={this.state.decryptedBlob} showGenericPlaceholder={false} />
</span>
);
}
}

110
src/components/views/messages/MAudioBody.tsx Normal file
View file

@ -0,0 +1,110 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React from "react";
import { MatrixEvent } from "matrix-js-sdk/src/models/event";
import { replaceableComponent } from "../../../utils/replaceableComponent";
import { Playback } from "../../../voice/Playback";
import MFileBody from "./MFileBody";
import InlineSpinner from '../elements/InlineSpinner';
import { _t } from "../../../languageHandler";
import { mediaFromContent } from "../../../customisations/Media";
import { decryptFile } from "../../../utils/DecryptFile";
import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent";
import AudioPlayer from "../audio_messages/AudioPlayer";
interface IProps {
mxEvent: MatrixEvent;
}
interface IState {
error?: Error;
playback?: Playback;
decryptedBlob?: Blob;
}
@replaceableComponent("views.messages.MAudioBody")
export default class MAudioBody extends React.PureComponent<IProps, IState> {
constructor(props: IProps) {
super(props);
this.state = {};
}
public async componentDidMount() {
let buffer: ArrayBuffer;
const content: IMediaEventContent = this.props.mxEvent.getContent();
const media = mediaFromContent(content);
if (media.isEncrypted) {
try {
const blob = await decryptFile(content.file);
buffer = await blob.arrayBuffer();
this.setState({ decryptedBlob: blob });
} catch (e) {
this.setState({ error: e });
console.warn("Unable to decrypt audio message", e);
return; // stop processing the audio file
}
} else {
try {
buffer = await media.downloadSource().then(r => r.blob()).then(r => r.arrayBuffer());
} catch (e) {
this.setState({ error: e });
console.warn("Unable to download audio message", e);
return; // stop processing the audio file
}
}
// We should have a buffer to work with now: let's set it up
const playback = new Playback(buffer);
playback.clockInfo.populatePlaceholdersFrom(this.props.mxEvent);
this.setState({ playback });
// Note: the RecordingPlayback component will handle preparing the Playback class for us.
}
public componentWillUnmount() {
this.state.playback?.destroy();
}
public render() {
if (this.state.error) {
// TODO: @@TR: Verify error state
return (
<span className="mx_MAudioBody">
<img src={require("../../../../res/img/warning.svg")} width="16" height="16" />
{ _t("Error processing audio message") }
</span>
);
}
if (!this.state.playback) {
// TODO: @@TR: Verify loading/decrypting state
return (
<span className="mx_MAudioBody">
<InlineSpinner />
</span>
);
}
// At this point we should have a playable state
return (
<span className="mx_MAudioBody">
<AudioPlayer playback={this.state.playback} mediaName={this.props.mxEvent.getContent().body} />
<MFileBody {...this.props} decryptedBlob={this.state.decryptedBlob} showGenericPlaceholder={false} />
</span>
);
}
}

3
src/components/views/messages/MStickerBody.js
View file

@ -42,8 +42,7 @@ export default class MStickerBody extends MImageBody {
// Placeholder to show in place of the sticker image if
// img onLoad hasn't fired yet.
getPlaceholder() {
const TintableSVG = sdk.getComponent('elements.TintableSvg');
return <TintableSVG src={require("../../../../res/img/icons-show-stickers.svg")} width="75" height="75" />;
return <img src={require("../../../../res/img/icons-show-stickers.svg")} width="75" height="75" />;
}
// Tooltip to show on mouse over

2
src/components/views/messages/MVoiceMessageBody.tsx
View file

@ -23,7 +23,7 @@ import InlineSpinner from '../elements/InlineSpinner';
import { _t } from "../../../languageHandler";
import { mediaFromContent } from "../../../customisations/Media";
import { decryptFile } from "../../../utils/DecryptFile";
import RecordingPlayback from "../voice_messages/RecordingPlayback";
import RecordingPlayback from "../audio_messages/RecordingPlayback";
import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent";
interface IProps {

2
src/components/views/rooms/SimpleRoomHeader.js
View file

@ -27,7 +27,7 @@ export default class SimpleRoomHeader extends React.Component {
static propTypes = {
title: PropTypes.string,
// `src` to a TintableSvg. Optional.
// `src` to an image. Optional.
icon: PropTypes.string,
};

54
src/components/views/rooms/VoiceRecordComposerTile.tsx
View file

@ -18,22 +18,18 @@ import AccessibleTooltipButton from "../elements/AccessibleTooltipButton";
import { _t } from "../../../languageHandler";
import React, { ReactNode } from "react";
import {
IRecordingUpdate,
RECORDING_PLAYBACK_SAMPLES,
RecordingState,
VoiceRecording,
} from "../../../voice/VoiceRecording";
import { Room } from "matrix-js-sdk/src/models/room";
import { MatrixClientPeg } from "../../../MatrixClientPeg";
import classNames from "classnames";
import LiveRecordingWaveform from "../voice_messages/LiveRecordingWaveform";
import LiveRecordingWaveform from "../audio_messages/LiveRecordingWaveform";
import { replaceableComponent } from "../../../utils/replaceableComponent";
import { arrayFastResample, arraySeed } from "../../../utils/arrays";
import { percentageOf } from "../../../utils/numbers";
import LiveRecordingClock from "../voice_messages/LiveRecordingClock";
import LiveRecordingClock from "../audio_messages/LiveRecordingClock";
import { VoiceRecordingStore } from "../../../stores/VoiceRecordingStore";
import { UPDATE_EVENT } from "../../../stores/AsyncStore";
import RecordingPlayback from "../voice_messages/RecordingPlayback";
import RecordingPlayback from "../audio_messages/RecordingPlayback";
import { MsgType } from "matrix-js-sdk/src/@types/event";
import Modal from "../../../Modal";
import ErrorDialog from "../dialogs/ErrorDialog";
@ -46,8 +42,6 @@ interface IProps {
interface IState {
recorder?: VoiceRecording;
recordingPhase?: RecordingState;
relHeights: number[];
seconds: number;
}
/**
@ -55,58 +49,18 @@ interface IState {
*/
@replaceableComponent("views.rooms.VoiceRecordComposerTile")
export default class VoiceRecordComposerTile extends React.PureComponent<IProps, IState> {
private waveform: number[] = [];
private seconds = 0;
private scheduledAnimationFrame = false;
public constructor(props) {
super(props);
this.state = {
recorder: null, // no recording started by default
seconds: 0,
relHeights: arraySeed(0, RECORDING_PLAYBACK_SAMPLES),
};
}
public componentDidUpdate(prevProps, prevState) {
if (!prevState.recorder && this.state.recorder) {
this.state.recorder.liveData.onUpdate(this.onRecordingUpdate);
}
}
public async componentWillUnmount() {
await VoiceRecordingStore.instance.disposeRecording();
}
private onRecordingUpdate = (update: IRecordingUpdate): void => {
this.waveform = update.waveform;
this.seconds = update.timeSeconds;
if (this.scheduledAnimationFrame) {
return;
}
this.scheduledAnimationFrame = true;
// The audio recorder flushes data faster than the screen refresh rate
// Using requestAnimationFrame makes sure that we only flush the data
// to react once per tick to avoid unneeded work.
requestAnimationFrame(() => {
// The waveform and the downsample target are pretty close, so we should be fine to
// do this, despite the docs on arrayFastResample.
const bars = arrayFastResample(Array.from(this.waveform), RECORDING_PLAYBACK_SAMPLES);
this.setState({
// The incoming data is between zero and one, but typically even screaming into a
// microphone won't send you over 0.6, so we artificially adjust the gain for the
// waveform. This results in a slightly more cinematic/animated waveform for the
// user.
relHeights: bars.map(b => percentageOf(b, 0, 0.50)),
seconds: this.seconds,
});
this.scheduledAnimationFrame = false;
});
};
// called by composer
public async send() {
if (!this.state.recorder) {
@ -228,7 +182,7 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
}
// only other UI is the recording-in-progress UI
return <div className="mx_VoiceMessagePrimaryContainer mx_VoiceRecordComposerTile_recording">
return <div className="mx_MediaBody mx_VoiceMessagePrimaryContainer mx_VoiceRecordComposerTile_recording">
<LiveRecordingClock recorder={this.state.recorder} />
<LiveRecordingWaveform recorder={this.state.recorder} />
</div>;

4
src/customisations/Media.ts
View file

@ -75,6 +75,7 @@ export class Media {
* The HTTP URL for the source media.
*/
public get srcHttp(): string {
// eslint-disable-next-line no-restricted-properties
return this.client.mxcUrlToHttp(this.srcMxc);
}
@ -84,6 +85,7 @@ export class Media {
*/
public get thumbnailHttp(): string | undefined | null {
if (!this.hasThumbnail) return null;
// eslint-disable-next-line no-restricted-properties
return this.client.mxcUrlToHttp(this.thumbnailMxc);
}
@ -100,6 +102,7 @@ export class Media {
// scale using the device pixel ratio to keep images clear
width = Math.floor(width * window.devicePixelRatio);
height = Math.floor(height * window.devicePixelRatio);
// eslint-disable-next-line no-restricted-properties
return this.client.mxcUrlToHttp(this.thumbnailMxc, width, height, mode);
}
@ -114,6 +117,7 @@ export class Media {
// scale using the device pixel ratio to keep images clear
width = Math.floor(width * window.devicePixelRatio);
height = Math.floor(height * window.devicePixelRatio);
// eslint-disable-next-line no-restricted-properties
return this.client.mxcUrlToHttp(this.srcMxc, width, height, mode);
}

7
src/i18n/strings/en_EN.json
View file

@ -918,8 +918,6 @@
"Silence call": "Silence call",
"Decline": "Decline",
"Accept": "Accept",
"Pause": "Pause",
"Play": "Play",
"The other party cancelled the verification.": "The other party cancelled the verification.",
"Verified!": "Verified!",
"You've successfully verified this user.": "You've successfully verified this user.",
@ -1868,7 +1866,7 @@
"Ignored attempt to disable encryption": "Ignored attempt to disable encryption",
"Encryption not enabled": "Encryption not enabled",
"The encryption used by this room isn't supported.": "The encryption used by this room isn't supported.",
"Error decrypting audio": "Error decrypting audio",
"Error processing audio message": "Error processing audio message",
"React": "React",
"Edit": "Edit",
"Retry": "Retry",
@ -2602,6 +2600,9 @@
"Use email or phone to optionally be discoverable by existing contacts.": "Use email or phone to optionally be discoverable by existing contacts.",
"Use email to optionally be discoverable by existing contacts.": "Use email to optionally be discoverable by existing contacts.",
"Sign in with SSO": "Sign in with SSO",
"Unnamed audio": "Unnamed audio",
"Pause": "Pause",
"Play": "Play",
"Couldn't load page": "Couldn't load page",
"You must <a>register</a> to use this functionality": "You must <a>register</a> to use this functionality",
"You must join the room to see its files": "You must join the room to see its files",

92
src/voice/Playback.ts
View file

@ -58,6 +58,7 @@ export class Playback extends EventEmitter implements IDestroyable {
private resampledWaveform: number[];
private waveformObservable = new SimpleObservable<number[]>();
private readonly clock: PlaybackClock;
private readonly fileSize: number;
/**
* Creates a new playback instance from a buffer.
@ -67,12 +68,22 @@ export class Playback extends EventEmitter implements IDestroyable {
*/
constructor(private buf: ArrayBuffer, seedWaveform = DEFAULT_WAVEFORM) {
super();
// Capture the file size early as reading the buffer will result in a 0-length buffer left behind
this.fileSize = this.buf.byteLength;
this.context = createAudioContext();
this.resampledWaveform = arrayFastResample(seedWaveform ?? DEFAULT_WAVEFORM, PLAYBACK_WAVEFORM_SAMPLES);
this.waveformObservable.update(this.resampledWaveform);
this.clock = new PlaybackClock(this.context);
}
/**
* Size of the audio clip in bytes. May be zero if unknown. This is updated
* when the playback goes through phase changes.
*/
public get sizeBytes(): number {
return this.fileSize;
}
/**
* Stable waveform for the playback. Values are guaranteed to be between
* zero and one, inclusive.
@ -150,16 +161,9 @@ export class Playback extends EventEmitter implements IDestroyable {
public async play() {
// We can't restart a buffer source, so we need to create a new one if we hit the end
if (this.state === PlaybackState.Stopped) {
if (this.source) {
this.source.disconnect();
this.source.removeEventListener("ended", this.onPlaybackEnd);
}
this.source = this.context.createBufferSource();
this.source.connect(this.context.destination);
this.source.buffer = this.audioBuf;
this.source.start(); // start immediately
this.source.addEventListener("ended", this.onPlaybackEnd);
this.disconnectSource();
this.makeNewSourceBuffer();
this.source.start();
}
// We use the context suspend/resume functions because it allows us to pause a source
@ -169,6 +173,18 @@ export class Playback extends EventEmitter implements IDestroyable {
this.emit(PlaybackState.Playing);
}
private disconnectSource() {
this.source?.disconnect();
this.source?.removeEventListener("ended", this.onPlaybackEnd);
}
private makeNewSourceBuffer() {
this.source = this.context.createBufferSource();
this.source.buffer = this.audioBuf;
this.source.addEventListener("ended", this.onPlaybackEnd);
this.source.connect(this.context.destination);
}
public async pause() {
await this.context.suspend();
this.emit(PlaybackState.Paused);
@ -183,4 +199,60 @@ export class Playback extends EventEmitter implements IDestroyable {
if (this.isPlaying) await this.pause();
else await this.play();
}
public async skipTo(timeSeconds: number) {
// Dev note: this function talks a lot about clock desyncs. There is a clock running
// independently to the audio context and buffer so that accurate human-perceptible
// time can be exposed. The PlaybackClock class has more information, but the short
// version is that we need to line up the useful time (clip position) with the context
// time, and avoid as many deviations as possible as otherwise the user could see the
// wrong time, and we stop playback at the wrong time, etc.
timeSeconds = clamp(timeSeconds, 0, this.clock.durationSeconds);
// Track playing state so we don't cause seeking to start playing the track.
const isPlaying = this.isPlaying;
if (isPlaying) {
// Pause first so we can get an accurate measurement of time
await this.context.suspend();
}
// We can't simply tell the context/buffer to jump to a time, so we have to
// start a whole new buffer and start it from the new time offset.
const now = this.context.currentTime;
this.disconnectSource();
this.makeNewSourceBuffer();
// We have to resync the clock because it can get confused about where we're
// at in the audio clip.
this.clock.syncTo(now, timeSeconds);
// Always start the source to queue it up. We have to do this now (and pause
// quickly if we're not supposed to be playing) as otherwise the clock can desync
// when it comes time to the user hitting play. After a couple jumps, the user
// will have desynced the clock enough to be about 10-15 seconds off, while this
// keeps it as close to perfect as humans can perceive.
this.source.start(now, timeSeconds);
// Dev note: it's critical that the code gap between `this.source.start()` and
// `this.pause()` is as small as possible: we do not want to delay *anything*
// as that could cause a clock desync, or a buggy feeling as a single note plays
// during seeking.
if (isPlaying) {
// If we were playing before, continue the context so the clock doesn't desync.
await this.context.resume();
} else {
// As mentioned above, we'll have to pause the clip if we weren't supposed to
// be playing it just yet. If we didn't have this, the audio clip plays but all
// the states will be wrong: clock won't advance, pause state doesn't match the
// blaring noise leaving the user's speakers, etc.
//
// Also as mentioned, if the code gap is small enough then this should be
// executed immediately after the start time, leaving no feasible time for the
// user's speakers to play any sound.
await this.pause();
}
}
}

70
src/voice/PlaybackClock.ts
View file

@ -16,8 +16,44 @@ limitations under the License.
import { SimpleObservable } from "matrix-widget-api";
import { IDestroyable } from "../utils/IDestroyable";
import { MatrixEvent } from "matrix-js-sdk/src/models/event";
// Because keeping track of time is sufficiently complicated...
/**
* Tracks accurate human-perceptible time for an audio clip, as informed
* by managed playback. This clock is tightly coupled with the operation
* of the Playback class, making assumptions about how the provided
* AudioContext will be used (suspended/resumed to preserve time, etc).
*
* But why do we need a clock? The AudioContext exposes time information,
* and so does the audio buffer, but not in a way that is useful for humans
* to perceive. The audio buffer time is often lagged behind the context
* time due to internal processing delays of the audio API. Additionally,
* the context's time is tracked from when it was first initialized/started,
* not related to positioning within the clip. However, the context time
* is the most accurate time we can use to determine position within the
* clip if we're fast enough to track the pauses and stops.
*
* As a result, we track every play, pause, stop, and seek event from the
* Playback class (kinda: it calls us, which is close enough to the same
* thing). These events are then tracked on the AudioContext time scale,
* with assumptions that code execution will result in negligible desync
* of the clock, or at least no perceptible difference in time. It's
* extremely important that the calling code, and the clock's own code,
* is extremely fast between the event happening and the clock time being
* tracked - anything more than a dozen milliseconds is likely to stack up
* poorly, leading to clock desync.
*
* Clock desync can be dangerous for the stability of the playback controls:
* if the clock thinks the user is somewhere else in the clip, it could
* inform the playback of the wrong place in time, leading to dead air in
* the output or, if severe enough, a clock that won't stop running while
* the audio is paused/stopped. Other examples include the clip stopping at
* 90% time due to playback ending, the clip playing from the wrong spot
* relative to the time, and negative clock time.
*
* Note that the clip duration is fed to the clock: this is to ensure that
* we have the most accurate time possible to present.
*/
export class PlaybackClock implements IDestroyable {
private clipStart = 0;
private stopped = true;
@ -25,12 +61,13 @@ export class PlaybackClock implements IDestroyable {
private observable = new SimpleObservable<number[]>();
private timerId: number;
private clipDuration = 0;
private placeholderDuration = 0;
public constructor(private context: AudioContext) {
}
public get durationSeconds(): number {
return this.clipDuration;
return this.clipDuration || this.placeholderDuration;
}
public set durationSeconds(val: number) {
@ -39,6 +76,12 @@ export class PlaybackClock implements IDestroyable {
}
public get timeSeconds(): number {
// The modulo is to ensure that we're only looking at the most recent clip
// time, as the context is long-running and multiple plays might not be
// informed to us (if the control is looping, for example). By taking the
// remainder of the division operation, we're assuming that playback is
// incomplete or stopped, thus giving an accurate position within the active
// clip segment.
return (this.context.currentTime - this.clipStart) % this.clipDuration;
}
@ -47,13 +90,23 @@ export class PlaybackClock implements IDestroyable {
}
private checkTime = () => {
const now = this.timeSeconds;
const now = this.timeSeconds; // calculated dynamically
if (this.lastCheck !== now) {
this.observable.update([now, this.durationSeconds]);
this.lastCheck = now;
}
};
/**
* Populates default information about the audio clip from the event body.
* The placeholders will be overridden once known.
* @param {MatrixEvent} event The event to use for placeholders.
*/
public populatePlaceholdersFrom(event: MatrixEvent) {
const durationSeconds = Number(event.getContent()['info']?.['duration']);
if (Number.isFinite(durationSeconds)) this.placeholderDuration = durationSeconds;
}
/**
* Mark the time in the audio context where the clip starts/has been loaded.
* This is to ensure the clock isn't skewed into thinking it is ~0.5s into
@ -70,8 +123,9 @@ export class PlaybackClock implements IDestroyable {
}
if (!this.timerId) {
// case to number because the types are wrong
// 100ms interval to make sure the time is as accurate as possible
// cast to number because the types are wrong
// 100ms interval to make sure the time is as accurate as possible without
// being overly insane
this.timerId = <number><any>setInterval(this.checkTime, 100);
}
}
@ -80,6 +134,12 @@ export class PlaybackClock implements IDestroyable {
this.stopped = true;
}
public syncTo(contextTime: number, clipTime: number) {
this.clipStart = contextTime - clipTime;
this.stopped = false; // count as a mid-stream pause (if we were stopped)
this.checkTime();
}
public destroy() {
this.observable.close();
if (this.timerId) clearInterval(this.timerId);