From 50127da227f09aaad64e7f435e8ffe31ad86e1cb Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 21 Jun 2021 15:11:41 -0600 Subject: [PATCH 01/23] Replace MAudioBody with a voice message body as a template --- src/components/views/messages/MAudioBody.js | 112 ------------------- src/components/views/messages/MAudioBody.tsx | 111 ++++++++++++++++++ 2 files changed, 111 insertions(+), 112 deletions(-) delete mode 100644 src/components/views/messages/MAudioBody.js create mode 100644 src/components/views/messages/MAudioBody.tsx diff --git a/src/components/views/messages/MAudioBody.js b/src/components/views/messages/MAudioBody.js deleted file mode 100644 index 0d5e449fc0..0000000000 --- a/src/components/views/messages/MAudioBody.js +++ /dev/null @@ -1,112 +0,0 @@ -/* - Copyright 2016 OpenMarket Ltd - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -import React from 'react'; -import MFileBody from './MFileBody'; - -import { decryptFile } from '../../../utils/DecryptFile'; -import { _t } from '../../../languageHandler'; -import InlineSpinner from '../elements/InlineSpinner'; -import {replaceableComponent} from "../../../utils/replaceableComponent"; -import {mediaFromContent} from "../../../customisations/Media"; - -@replaceableComponent("views.messages.MAudioBody") -export default class MAudioBody extends React.Component { - constructor(props) { - super(props); - this.state = { - playing: false, - decryptedUrl: null, - decryptedBlob: null, - error: null, - }; - } - onPlayToggle() { - this.setState({ - playing: !this.state.playing, - }); - } - - _getContentUrl() { - const media = mediaFromContent(this.props.mxEvent.getContent()); - if (media.isEncrypted) { - return this.state.decryptedUrl; - } else { - return media.srcHttp; - } - } - - componentDidMount() { - const content = this.props.mxEvent.getContent(); - if (content.file !== undefined && this.state.decryptedUrl === null) { - let decryptedBlob; - decryptFile(content.file).then(function(blob) { - decryptedBlob = blob; - return URL.createObjectURL(decryptedBlob); - }).then((url) => { - this.setState({ - decryptedUrl: url, - decryptedBlob: decryptedBlob, - }); - }, (err) => { - console.warn("Unable to decrypt attachment: ", err); - this.setState({ - error: err, - }); - }); - } - } - - componentWillUnmount() { - if (this.state.decryptedUrl) { - URL.revokeObjectURL(this.state.decryptedUrl); - } - } - - render() { - const content = this.props.mxEvent.getContent(); - - if (this.state.error !== null) { - return ( - - - { _t("Error decrypting audio") } - - ); - } - - if (content.file !== undefined && this.state.decryptedUrl === null) { - // Need to decrypt the attachment - // The attachment is decrypted in componentDidMount. - // For now add an img tag with a 16x16 spinner. - // Not sure how tall the audio player is so not sure how tall it should actually be. - return ( - - - - ); - } - - const contentUrl = this._getContentUrl(); - - return ( - - - ); - } -} diff --git a/src/components/views/messages/MAudioBody.tsx b/src/components/views/messages/MAudioBody.tsx new file mode 100644 index 0000000000..9e77bc0893 --- /dev/null +++ b/src/components/views/messages/MAudioBody.tsx @@ -0,0 +1,111 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import React from "react"; +import {MatrixEvent} from "matrix-js-sdk/src/models/event"; +import {replaceableComponent} from "../../../utils/replaceableComponent"; +import {Playback} from "../../../voice/Playback"; +import MFileBody from "./MFileBody"; +import InlineSpinner from '../elements/InlineSpinner'; +import {_t} from "../../../languageHandler"; +import {mediaFromContent} from "../../../customisations/Media"; +import {decryptFile} from "../../../utils/DecryptFile"; +import RecordingPlayback from "../voice_messages/RecordingPlayback"; +import {IMediaEventContent} from "../../../customisations/models/IMediaEventContent"; + +interface IProps { + mxEvent: MatrixEvent; +} + +interface IState { + error?: Error; + playback?: Playback; + decryptedBlob?: Blob; +} + +@replaceableComponent("views.messages.MAudioBody") +export default class MAudioBody extends React.PureComponent { + constructor(props: IProps) { + super(props); + + this.state = {}; + } + + public async componentDidMount() { + let buffer: ArrayBuffer; + const content: IMediaEventContent = this.props.mxEvent.getContent(); + const media = mediaFromContent(content); + if (media.isEncrypted) { + try { + const blob = await decryptFile(content.file); + buffer = await blob.arrayBuffer(); + this.setState({decryptedBlob: blob}); + } catch (e) { + this.setState({error: e}); + console.warn("Unable to decrypt voice message", e); + return; // stop processing the audio file + } + } else { + try { + buffer = await media.downloadSource().then(r => r.blob()).then(r => r.arrayBuffer()); + } catch (e) { + this.setState({error: e}); + console.warn("Unable to download voice message", e); + return; // stop processing the audio file + } + } + + const waveform = content?.["org.matrix.msc1767.audio"]?.waveform?.map(p => p / 1024); + + // We should have a buffer to work with now: let's set it up + const playback = new Playback(buffer, waveform); + this.setState({ playback }); + // Note: the RecordingPlayback component will handle preparing the Playback class for us. + } + + public componentWillUnmount() { + this.state.playback?.destroy(); + } + + public render() { + if (this.state.error) { + // TODO: @@TR: Verify error state + return ( + + + { _t("Error processing voice message") } + + ); + } + + if (!this.state.playback) { + // TODO: @@TR: Verify loading/decrypting state + return ( + + + + ); + } + + // At this point we should have a playable state + return ( + + + + + ) + } +} From 9f2eba4351a8ac4977e783c68811fb71a48a0586 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 21 Jun 2021 15:13:12 -0600 Subject: [PATCH 02/23] Fix class identifiers --- res/css/_components.scss | 1 + res/css/views/messages/_MAudioBody.scss | 19 +++++++++++++++++++ src/components/views/messages/MAudioBody.tsx | 16 +++++++--------- 3 files changed, 27 insertions(+), 9 deletions(-) create mode 100644 res/css/views/messages/_MAudioBody.scss diff --git a/res/css/_components.scss b/res/css/_components.scss index ec3af8655e..c997881138 100644 --- a/res/css/_components.scss +++ b/res/css/_components.scss @@ -156,6 +156,7 @@ @import "./views/messages/_CreateEvent.scss"; @import "./views/messages/_DateSeparator.scss"; @import "./views/messages/_EventTileBubble.scss"; +@import "./views/messages/_MAudioBody.scss"; @import "./views/messages/_MEmoteBody.scss"; @import "./views/messages/_MFileBody.scss"; @import "./views/messages/_MImageBody.scss"; diff --git a/res/css/views/messages/_MAudioBody.scss b/res/css/views/messages/_MAudioBody.scss new file mode 100644 index 0000000000..2c9b6f7030 --- /dev/null +++ b/res/css/views/messages/_MAudioBody.scss @@ -0,0 +1,19 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.mx_MAudioBody { + display: inline-block; // makes the playback controls magically line up +} diff --git a/src/components/views/messages/MAudioBody.tsx b/src/components/views/messages/MAudioBody.tsx index 9e77bc0893..8568899323 100644 --- a/src/components/views/messages/MAudioBody.tsx +++ b/src/components/views/messages/MAudioBody.tsx @@ -55,7 +55,7 @@ export default class MAudioBody extends React.PureComponent { this.setState({decryptedBlob: blob}); } catch (e) { this.setState({error: e}); - console.warn("Unable to decrypt voice message", e); + console.warn("Unable to decrypt audio message", e); return; // stop processing the audio file } } else { @@ -63,15 +63,13 @@ export default class MAudioBody extends React.PureComponent { buffer = await media.downloadSource().then(r => r.blob()).then(r => r.arrayBuffer()); } catch (e) { this.setState({error: e}); - console.warn("Unable to download voice message", e); + console.warn("Unable to download audio message", e); return; // stop processing the audio file } } - const waveform = content?.["org.matrix.msc1767.audio"]?.waveform?.map(p => p / 1024); - // We should have a buffer to work with now: let's set it up - const playback = new Playback(buffer, waveform); + const playback = new Playback(buffer); this.setState({ playback }); // Note: the RecordingPlayback component will handle preparing the Playback class for us. } @@ -84,9 +82,9 @@ export default class MAudioBody extends React.PureComponent { if (this.state.error) { // TODO: @@TR: Verify error state return ( - + - { _t("Error processing voice message") } + { _t("Error processing audio message") } ); } @@ -94,7 +92,7 @@ export default class MAudioBody extends React.PureComponent { if (!this.state.playback) { // TODO: @@TR: Verify loading/decrypting state return ( - + ); @@ -102,7 +100,7 @@ export default class MAudioBody extends React.PureComponent { // At this point we should have a playable state return ( - + From 470778cbb84f875314643fdbc3e484d2c66e21fc Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 21 Jun 2021 15:18:34 -0600 Subject: [PATCH 03/23] Move voice message components to audio-generic directory --- res/css/_components.scss | 6 +++--- .../_PlayPauseButton.scss | 0 .../_PlaybackContainer.scss | 0 .../{voice_messages => audio_messages}/_Waveform.scss | 0 .../views/{voice_messages => audio_messages}/Clock.tsx | 4 ++-- .../LiveRecordingClock.tsx | 6 +++--- .../LiveRecordingWaveform.tsx | 10 +++++----- .../PlayPauseButton.tsx | 10 +++++----- .../PlaybackClock.tsx | 8 ++++---- .../PlaybackWaveform.tsx | 10 +++++----- .../RecordingPlayback.tsx | 6 ++++-- .../{voice_messages => audio_messages}/Waveform.tsx | 4 ++-- src/components/views/messages/MAudioBody.tsx | 2 +- src/components/views/messages/MVoiceMessageBody.tsx | 2 +- src/components/views/rooms/VoiceRecordComposerTile.tsx | 6 +++--- 15 files changed, 38 insertions(+), 36 deletions(-) rename res/css/views/{voice_messages => audio_messages}/_PlayPauseButton.scss (100%) rename res/css/views/{voice_messages => audio_messages}/_PlaybackContainer.scss (100%) rename res/css/views/{voice_messages => audio_messages}/_Waveform.scss (100%) rename src/components/views/{voice_messages => audio_messages}/Clock.tsx (92%) rename src/components/views/{voice_messages => audio_messages}/LiveRecordingClock.tsx (84%) rename src/components/views/{voice_messages => audio_messages}/LiveRecordingWaveform.tsx (83%) rename src/components/views/{voice_messages => audio_messages}/PlayPauseButton.tsx (86%) rename src/components/views/{voice_messages => audio_messages}/PlaybackClock.tsx (90%) rename src/components/views/{voice_messages => audio_messages}/PlaybackWaveform.tsx (85%) rename src/components/views/{voice_messages => audio_messages}/RecordingPlayback.tsx (88%) rename src/components/views/{voice_messages => audio_messages}/Waveform.tsx (93%) diff --git a/res/css/_components.scss b/res/css/_components.scss index c997881138..cb921e251c 100644 --- a/res/css/_components.scss +++ b/res/css/_components.scss @@ -37,6 +37,9 @@ @import "./structures/_ViewSource.scss"; @import "./structures/auth/_CompleteSecurity.scss"; @import "./structures/auth/_Login.scss"; +@import "./views/audio_messages/_PlayPauseButton.scss"; +@import "./views/audio_messages/_PlaybackContainer.scss"; +@import "./views/audio_messages/_Waveform.scss"; @import "./views/auth/_AuthBody.scss"; @import "./views/auth/_AuthButtons.scss"; @import "./views/auth/_AuthFooter.scss"; @@ -254,9 +257,6 @@ @import "./views/toasts/_AnalyticsToast.scss"; @import "./views/toasts/_NonUrgentEchoFailureToast.scss"; @import "./views/verification/_VerificationShowSas.scss"; -@import "./views/voice_messages/_PlayPauseButton.scss"; -@import "./views/voice_messages/_PlaybackContainer.scss"; -@import "./views/voice_messages/_Waveform.scss"; @import "./views/voip/_CallContainer.scss"; @import "./views/voip/_CallView.scss"; @import "./views/voip/_CallViewForRoom.scss"; diff --git a/res/css/views/voice_messages/_PlayPauseButton.scss b/res/css/views/audio_messages/_PlayPauseButton.scss similarity index 100% rename from res/css/views/voice_messages/_PlayPauseButton.scss rename to res/css/views/audio_messages/_PlayPauseButton.scss diff --git a/res/css/views/voice_messages/_PlaybackContainer.scss b/res/css/views/audio_messages/_PlaybackContainer.scss similarity index 100% rename from res/css/views/voice_messages/_PlaybackContainer.scss rename to res/css/views/audio_messages/_PlaybackContainer.scss diff --git a/res/css/views/voice_messages/_Waveform.scss b/res/css/views/audio_messages/_Waveform.scss similarity index 100% rename from res/css/views/voice_messages/_Waveform.scss rename to res/css/views/audio_messages/_Waveform.scss diff --git a/src/components/views/voice_messages/Clock.tsx b/src/components/views/audio_messages/Clock.tsx similarity index 92% rename from src/components/views/voice_messages/Clock.tsx rename to src/components/views/audio_messages/Clock.tsx index 23e6762c52..19ca1cb348 100644 --- a/src/components/views/voice_messages/Clock.tsx +++ b/src/components/views/audio_messages/Clock.tsx @@ -15,7 +15,7 @@ limitations under the License. */ import React from "react"; -import {replaceableComponent} from "../../../utils/replaceableComponent"; +import { replaceableComponent } from "../../../utils/replaceableComponent"; interface IProps { seconds: number; @@ -28,7 +28,7 @@ interface IState { * Simply converts seconds into minutes and seconds. Note that hours will not be * displayed, making it possible to see "82:29". */ -@replaceableComponent("views.voice_messages.Clock") +@replaceableComponent("views.audio_messages.Clock") export default class Clock extends React.Component { public constructor(props) { super(props); diff --git a/src/components/views/voice_messages/LiveRecordingClock.tsx b/src/components/views/audio_messages/LiveRecordingClock.tsx similarity index 84% rename from src/components/views/voice_messages/LiveRecordingClock.tsx rename to src/components/views/audio_messages/LiveRecordingClock.tsx index b82539eb16..41909dc201 100644 --- a/src/components/views/voice_messages/LiveRecordingClock.tsx +++ b/src/components/views/audio_messages/LiveRecordingClock.tsx @@ -15,8 +15,8 @@ limitations under the License. */ import React from "react"; -import {IRecordingUpdate, VoiceRecording} from "../../../voice/VoiceRecording"; -import {replaceableComponent} from "../../../utils/replaceableComponent"; +import { IRecordingUpdate, VoiceRecording } from "../../../voice/VoiceRecording"; +import { replaceableComponent } from "../../../utils/replaceableComponent"; import Clock from "./Clock"; interface IProps { @@ -30,7 +30,7 @@ interface IState { /** * A clock for a live recording. */ -@replaceableComponent("views.voice_messages.LiveRecordingClock") +@replaceableComponent("views.audio_messages.LiveRecordingClock") export default class LiveRecordingClock extends React.PureComponent { public constructor(props) { super(props); diff --git a/src/components/views/voice_messages/LiveRecordingWaveform.tsx b/src/components/views/audio_messages/LiveRecordingWaveform.tsx similarity index 83% rename from src/components/views/voice_messages/LiveRecordingWaveform.tsx rename to src/components/views/audio_messages/LiveRecordingWaveform.tsx index aab89f6ab1..27d165e613 100644 --- a/src/components/views/voice_messages/LiveRecordingWaveform.tsx +++ b/src/components/views/audio_messages/LiveRecordingWaveform.tsx @@ -15,10 +15,10 @@ limitations under the License. */ import React from "react"; -import {IRecordingUpdate, RECORDING_PLAYBACK_SAMPLES, VoiceRecording} from "../../../voice/VoiceRecording"; -import {replaceableComponent} from "../../../utils/replaceableComponent"; -import {arrayFastResample, arraySeed} from "../../../utils/arrays"; -import {percentageOf} from "../../../utils/numbers"; +import { IRecordingUpdate, RECORDING_PLAYBACK_SAMPLES, VoiceRecording } from "../../../voice/VoiceRecording"; +import { replaceableComponent } from "../../../utils/replaceableComponent"; +import { arrayFastResample, arraySeed } from "../../../utils/arrays"; +import { percentageOf } from "../../../utils/numbers"; import Waveform from "./Waveform"; interface IProps { @@ -32,7 +32,7 @@ interface IState { /** * A waveform which shows the waveform of a live recording */ -@replaceableComponent("views.voice_messages.LiveRecordingWaveform") +@replaceableComponent("views.audio_messages.LiveRecordingWaveform") export default class LiveRecordingWaveform extends React.PureComponent { public constructor(props) { super(props); diff --git a/src/components/views/voice_messages/PlayPauseButton.tsx b/src/components/views/audio_messages/PlayPauseButton.tsx similarity index 86% rename from src/components/views/voice_messages/PlayPauseButton.tsx rename to src/components/views/audio_messages/PlayPauseButton.tsx index 1f87eb012d..399cb169bb 100644 --- a/src/components/views/voice_messages/PlayPauseButton.tsx +++ b/src/components/views/audio_messages/PlayPauseButton.tsx @@ -14,11 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -import React, {ReactNode} from "react"; -import {replaceableComponent} from "../../../utils/replaceableComponent"; +import React, { ReactNode } from "react"; +import { replaceableComponent } from "../../../utils/replaceableComponent"; import AccessibleTooltipButton from "../elements/AccessibleTooltipButton"; -import {_t} from "../../../languageHandler"; -import {Playback, PlaybackState} from "../../../voice/Playback"; +import { _t } from "../../../languageHandler"; +import { Playback, PlaybackState } from "../../../voice/Playback"; import classNames from "classnames"; interface IProps { @@ -33,7 +33,7 @@ interface IProps { * Displays a play/pause button (activating the play/pause function of the recorder) * to be displayed in reference to a recording. */ -@replaceableComponent("views.voice_messages.PlayPauseButton") +@replaceableComponent("views.audio_messages.PlayPauseButton") export default class PlayPauseButton extends React.PureComponent { public constructor(props) { super(props); diff --git a/src/components/views/voice_messages/PlaybackClock.tsx b/src/components/views/audio_messages/PlaybackClock.tsx similarity index 90% rename from src/components/views/voice_messages/PlaybackClock.tsx rename to src/components/views/audio_messages/PlaybackClock.tsx index 2e8ec9a3e7..82108ae63c 100644 --- a/src/components/views/voice_messages/PlaybackClock.tsx +++ b/src/components/views/audio_messages/PlaybackClock.tsx @@ -15,10 +15,10 @@ limitations under the License. */ import React from "react"; -import {replaceableComponent} from "../../../utils/replaceableComponent"; +import { replaceableComponent } from "../../../utils/replaceableComponent"; import Clock from "./Clock"; -import {Playback, PlaybackState} from "../../../voice/Playback"; -import {UPDATE_EVENT} from "../../../stores/AsyncStore"; +import { Playback, PlaybackState } from "../../../voice/Playback"; +import { UPDATE_EVENT } from "../../../stores/AsyncStore"; interface IProps { playback: Playback; @@ -33,7 +33,7 @@ interface IState { /** * A clock for a playback of a recording. */ -@replaceableComponent("views.voice_messages.PlaybackClock") +@replaceableComponent("views.audio_messages.PlaybackClock") export default class PlaybackClock extends React.PureComponent { public constructor(props) { super(props); diff --git a/src/components/views/voice_messages/PlaybackWaveform.tsx b/src/components/views/audio_messages/PlaybackWaveform.tsx similarity index 85% rename from src/components/views/voice_messages/PlaybackWaveform.tsx rename to src/components/views/audio_messages/PlaybackWaveform.tsx index 2e9f163f5e..bb205e4222 100644 --- a/src/components/views/voice_messages/PlaybackWaveform.tsx +++ b/src/components/views/audio_messages/PlaybackWaveform.tsx @@ -15,11 +15,11 @@ limitations under the License. */ import React from "react"; -import {replaceableComponent} from "../../../utils/replaceableComponent"; -import {arraySeed, arrayTrimFill} from "../../../utils/arrays"; +import { replaceableComponent } from "../../../utils/replaceableComponent"; +import { arraySeed, arrayTrimFill } from "../../../utils/arrays"; import Waveform from "./Waveform"; -import {Playback, PLAYBACK_WAVEFORM_SAMPLES} from "../../../voice/Playback"; -import {percentageOf} from "../../../utils/numbers"; +import { Playback, PLAYBACK_WAVEFORM_SAMPLES } from "../../../voice/Playback"; +import { percentageOf } from "../../../utils/numbers"; interface IProps { playback: Playback; @@ -33,7 +33,7 @@ interface IState { /** * A waveform which shows the waveform of a previously recorded recording */ -@replaceableComponent("views.voice_messages.PlaybackWaveform") +@replaceableComponent("views.audio_messages.PlaybackWaveform") export default class PlaybackWaveform extends React.PureComponent { public constructor(props) { super(props); diff --git a/src/components/views/voice_messages/RecordingPlayback.tsx b/src/components/views/audio_messages/RecordingPlayback.tsx similarity index 88% rename from src/components/views/voice_messages/RecordingPlayback.tsx rename to src/components/views/audio_messages/RecordingPlayback.tsx index 776997cec2..32ce32805d 100644 --- a/src/components/views/voice_messages/RecordingPlayback.tsx +++ b/src/components/views/audio_messages/RecordingPlayback.tsx @@ -14,12 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {Playback, PlaybackState} from "../../../voice/Playback"; +import { Playback, PlaybackState } from "../../../voice/Playback"; import React, {ReactNode} from "react"; -import {UPDATE_EVENT} from "../../../stores/AsyncStore"; +import { UPDATE_EVENT } from "../../../stores/AsyncStore"; import PlaybackWaveform from "./PlaybackWaveform"; import PlayPauseButton from "./PlayPauseButton"; import PlaybackClock from "./PlaybackClock"; +import { replaceableComponent } from "../../../utils/replaceableComponent"; interface IProps { // Playback instance to render. Cannot change during component lifecycle: create @@ -31,6 +32,7 @@ interface IState { playbackPhase: PlaybackState; } +@replaceableComponent("views.audio_messages.RecordingPlayback") export default class RecordingPlayback extends React.PureComponent { constructor(props: IProps) { super(props); diff --git a/src/components/views/voice_messages/Waveform.tsx b/src/components/views/audio_messages/Waveform.tsx similarity index 93% rename from src/components/views/voice_messages/Waveform.tsx rename to src/components/views/audio_messages/Waveform.tsx index 840a5a12b3..dc7551025e 100644 --- a/src/components/views/voice_messages/Waveform.tsx +++ b/src/components/views/audio_messages/Waveform.tsx @@ -15,7 +15,7 @@ limitations under the License. */ import React from "react"; -import {replaceableComponent} from "../../../utils/replaceableComponent"; +import { replaceableComponent } from "../../../utils/replaceableComponent"; import classNames from "classnames"; interface IProps { @@ -34,7 +34,7 @@ interface IState { * For CSS purposes, a mx_Waveform_bar_100pct class is added when the bar should be * "filled", as a demonstration of the progress property. */ -@replaceableComponent("views.voice_messages.Waveform") +@replaceableComponent("views.audio_messages.Waveform") export default class Waveform extends React.PureComponent { public static defaultProps = { progress: 1, diff --git a/src/components/views/messages/MAudioBody.tsx b/src/components/views/messages/MAudioBody.tsx index 8568899323..ddcebd7a60 100644 --- a/src/components/views/messages/MAudioBody.tsx +++ b/src/components/views/messages/MAudioBody.tsx @@ -23,7 +23,7 @@ import InlineSpinner from '../elements/InlineSpinner'; import {_t} from "../../../languageHandler"; import {mediaFromContent} from "../../../customisations/Media"; import {decryptFile} from "../../../utils/DecryptFile"; -import RecordingPlayback from "../voice_messages/RecordingPlayback"; +import RecordingPlayback from "../audio_messages/RecordingPlayback"; import {IMediaEventContent} from "../../../customisations/models/IMediaEventContent"; interface IProps { diff --git a/src/components/views/messages/MVoiceMessageBody.tsx b/src/components/views/messages/MVoiceMessageBody.tsx index a7e3b1cd86..84cd6ce012 100644 --- a/src/components/views/messages/MVoiceMessageBody.tsx +++ b/src/components/views/messages/MVoiceMessageBody.tsx @@ -23,7 +23,7 @@ import InlineSpinner from '../elements/InlineSpinner'; import {_t} from "../../../languageHandler"; import {mediaFromContent} from "../../../customisations/Media"; import {decryptFile} from "../../../utils/DecryptFile"; -import RecordingPlayback from "../voice_messages/RecordingPlayback"; +import RecordingPlayback from "../audio_messages/RecordingPlayback"; import {IMediaEventContent} from "../../../customisations/models/IMediaEventContent"; interface IProps { diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index 122ba0ca0b..d8c481e332 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -21,12 +21,12 @@ import {RecordingState, VoiceRecording} from "../../../voice/VoiceRecording"; import {Room} from "matrix-js-sdk/src/models/room"; import {MatrixClientPeg} from "../../../MatrixClientPeg"; import classNames from "classnames"; -import LiveRecordingWaveform from "../voice_messages/LiveRecordingWaveform"; +import LiveRecordingWaveform from "../audio_messages/LiveRecordingWaveform"; import {replaceableComponent} from "../../../utils/replaceableComponent"; -import LiveRecordingClock from "../voice_messages/LiveRecordingClock"; +import LiveRecordingClock from "../audio_messages/LiveRecordingClock"; import {VoiceRecordingStore} from "../../../stores/VoiceRecordingStore"; import {UPDATE_EVENT} from "../../../stores/AsyncStore"; -import RecordingPlayback from "../voice_messages/RecordingPlayback"; +import RecordingPlayback from "../audio_messages/RecordingPlayback"; import {MsgType} from "matrix-js-sdk/src/@types/event"; import Modal from "../../../Modal"; import ErrorDialog from "../dialogs/ErrorDialog"; From dda60949c33d6924e9f9053ae04c87df2a5759d0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 21 Jun 2021 16:18:39 -0600 Subject: [PATCH 04/23] Introduce basic audio playback control --- .../views/audio_messages/AudioPlayer.tsx | 96 +++++++++++++++++++ .../views/audio_messages/DurationClock.tsx | 55 +++++++++++ .../views/audio_messages/PlayPauseButton.tsx | 18 +++- src/components/views/messages/MAudioBody.tsx | 25 ++--- src/voice/Playback.ts | 11 +++ src/voice/PlaybackClock.ts | 18 +++- 6 files changed, 203 insertions(+), 20 deletions(-) create mode 100644 src/components/views/audio_messages/AudioPlayer.tsx create mode 100644 src/components/views/audio_messages/DurationClock.tsx diff --git a/src/components/views/audio_messages/AudioPlayer.tsx b/src/components/views/audio_messages/AudioPlayer.tsx new file mode 100644 index 0000000000..82372aca74 --- /dev/null +++ b/src/components/views/audio_messages/AudioPlayer.tsx @@ -0,0 +1,96 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { Playback, PlaybackState } from "../../../voice/Playback"; +import React, { createRef, ReactNode, RefObject } from "react"; +import { UPDATE_EVENT } from "../../../stores/AsyncStore"; +import PlaybackWaveform from "./PlaybackWaveform"; +import PlayPauseButton from "./PlayPauseButton"; +import { replaceableComponent } from "../../../utils/replaceableComponent"; +import { formatBytes } from "../../../utils/FormattingUtils"; +import DurationClock from "./DurationClock"; +import { Key } from "../../../Keyboard"; + +interface IProps { + // Playback instance to render. Cannot change during component lifecycle: create + // an all-new component instead. + playback: Playback; +} + +interface IState { + playbackPhase: PlaybackState; +} + +@replaceableComponent("views.audio_messages.AudioPlayer") +export default class AudioPlayer extends React.PureComponent { + private playPauseRef: RefObject = createRef(); + + constructor(props: IProps) { + super(props); + + this.state = { + playbackPhase: PlaybackState.Decoding, // default assumption + }; + + // We don't need to de-register: the class handles this for us internally + this.props.playback.on(UPDATE_EVENT, this.onPlaybackUpdate); + + // Don't wait for the promise to complete - it will emit a progress update when it + // is done, and it's not meant to take long anyhow. + // noinspection JSIgnoredPromiseFromCall + this.props.playback.prepare(); + } + + private onPlaybackUpdate = (ev: PlaybackState) => { + this.setState({ playbackPhase: ev }); + }; + + private onKeyPress = (ev: React.KeyboardEvent) => { + if (ev.key === Key.SPACE) { + ev.stopPropagation(); + this.playPauseRef.current?.toggle(); + } + }; + + protected renderFileSize(): string { + const bytes = this.props.playback.sizeBytes; + if (!bytes) return null; + + // Not translated as these are units, and therefore universal + return `(${formatBytes(bytes)})`; + } + + public render(): ReactNode { + // tabIndex=0 to ensure that the whole component becomes a tab stop, where we handle keyboard + // events for accessibility + return
+
+ +
+ +   {/* easiest way to introduce a gap between the components */} + { this.renderFileSize() } +
+
+ +
+ } +} diff --git a/src/components/views/audio_messages/DurationClock.tsx b/src/components/views/audio_messages/DurationClock.tsx new file mode 100644 index 0000000000..f6271d1cf4 --- /dev/null +++ b/src/components/views/audio_messages/DurationClock.tsx @@ -0,0 +1,55 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import React from "react"; +import { replaceableComponent } from "../../../utils/replaceableComponent"; +import Clock from "./Clock"; +import { Playback } from "../../../voice/Playback"; + +interface IProps { + playback: Playback; +} + +interface IState { + durationSeconds: number; +} + +/** + * A clock which shows a clip's maximum duration. + */ +@replaceableComponent("views.audio_messages.DurationClock") +export default class DurationClock extends React.PureComponent { + public constructor(props) { + super(props); + + this.state = { + // we track the duration on state because we won't really know what the clip duration + // is until the first time update, and as a PureComponent we are trying to dedupe state + // updates as much as possible. This is just the easiest way to avoid a forceUpdate() or + // member property to track "did we get a duration". + durationSeconds: this.props.playback.clockInfo.durationSeconds, + }; + this.props.playback.clockInfo.liveData.onUpdate(this.onTimeUpdate); + } + + private onTimeUpdate = (time: number[]) => { + this.setState({durationSeconds: time[1]}); + }; + + public render() { + return ; + } +} diff --git a/src/components/views/audio_messages/PlayPauseButton.tsx b/src/components/views/audio_messages/PlayPauseButton.tsx index 399cb169bb..7d881a10e5 100644 --- a/src/components/views/audio_messages/PlayPauseButton.tsx +++ b/src/components/views/audio_messages/PlayPauseButton.tsx @@ -21,7 +21,8 @@ import { _t } from "../../../languageHandler"; import { Playback, PlaybackState } from "../../../voice/Playback"; import classNames from "classnames"; -interface IProps { +// omitted props are handled by render function +interface IProps extends Omit, "title" | "onClick" | "disabled"> { // Playback instance to manipulate. Cannot change during the component lifecycle. playback: Playback; @@ -39,13 +40,19 @@ export default class PlayPauseButton extends React.PureComponent { super(props); } - private onClick = async () => { - await this.props.playback.toggle(); + private onClick = () => { + // noinspection JSIgnoredPromiseFromCall + this.toggle(); }; + public async toggle() { + await this.props.playback.toggle(); + } + public render(): ReactNode { - const isPlaying = this.props.playback.isPlaying; - const isDisabled = this.props.playbackPhase === PlaybackState.Decoding; + const { playback, playbackPhase, ...restProps } = this.props; + const isPlaying = playback.isPlaying; + const isDisabled = playbackPhase === PlaybackState.Decoding; const classes = classNames('mx_PlayPauseButton', { 'mx_PlayPauseButton_play': !isPlaying, 'mx_PlayPauseButton_pause': isPlaying, @@ -56,6 +63,7 @@ export default class PlayPauseButton extends React.PureComponent { title={isPlaying ? _t("Pause") : _t("Play")} onClick={this.onClick} disabled={isDisabled} + {...restProps} />; } } diff --git a/src/components/views/messages/MAudioBody.tsx b/src/components/views/messages/MAudioBody.tsx index ddcebd7a60..a8f2304fe9 100644 --- a/src/components/views/messages/MAudioBody.tsx +++ b/src/components/views/messages/MAudioBody.tsx @@ -15,16 +15,16 @@ limitations under the License. */ import React from "react"; -import {MatrixEvent} from "matrix-js-sdk/src/models/event"; -import {replaceableComponent} from "../../../utils/replaceableComponent"; -import {Playback} from "../../../voice/Playback"; +import { MatrixEvent } from "matrix-js-sdk/src/models/event"; +import { replaceableComponent } from "../../../utils/replaceableComponent"; +import { Playback } from "../../../voice/Playback"; import MFileBody from "./MFileBody"; import InlineSpinner from '../elements/InlineSpinner'; -import {_t} from "../../../languageHandler"; -import {mediaFromContent} from "../../../customisations/Media"; -import {decryptFile} from "../../../utils/DecryptFile"; -import RecordingPlayback from "../audio_messages/RecordingPlayback"; -import {IMediaEventContent} from "../../../customisations/models/IMediaEventContent"; +import { _t } from "../../../languageHandler"; +import { mediaFromContent } from "../../../customisations/Media"; +import { decryptFile } from "../../../utils/DecryptFile"; +import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent"; +import AudioPlayer from "../audio_messages/AudioPlayer"; interface IProps { mxEvent: MatrixEvent; @@ -52,9 +52,9 @@ export default class MAudioBody extends React.PureComponent { try { const blob = await decryptFile(content.file); buffer = await blob.arrayBuffer(); - this.setState({decryptedBlob: blob}); + this.setState({ decryptedBlob: blob }); } catch (e) { - this.setState({error: e}); + this.setState({ error: e }); console.warn("Unable to decrypt audio message", e); return; // stop processing the audio file } @@ -62,7 +62,7 @@ export default class MAudioBody extends React.PureComponent { try { buffer = await media.downloadSource().then(r => r.blob()).then(r => r.arrayBuffer()); } catch (e) { - this.setState({error: e}); + this.setState({ error: e }); console.warn("Unable to download audio message", e); return; // stop processing the audio file } @@ -70,6 +70,7 @@ export default class MAudioBody extends React.PureComponent { // We should have a buffer to work with now: let's set it up const playback = new Playback(buffer); + playback.clockInfo.populatePlaceholdersFrom(this.props.mxEvent); this.setState({ playback }); // Note: the RecordingPlayback component will handle preparing the Playback class for us. } @@ -101,7 +102,7 @@ export default class MAudioBody extends React.PureComponent { // At this point we should have a playable state return ( - + ) diff --git a/src/voice/Playback.ts b/src/voice/Playback.ts index 61da435151..f1f91310b2 100644 --- a/src/voice/Playback.ts +++ b/src/voice/Playback.ts @@ -58,6 +58,7 @@ export class Playback extends EventEmitter implements IDestroyable { private resampledWaveform: number[]; private waveformObservable = new SimpleObservable(); private readonly clock: PlaybackClock; + private readonly fileSize: number; /** * Creates a new playback instance from a buffer. @@ -67,12 +68,22 @@ export class Playback extends EventEmitter implements IDestroyable { */ constructor(private buf: ArrayBuffer, seedWaveform = DEFAULT_WAVEFORM) { super(); + // Capture the file size early as reading the buffer will result in a 0-length buffer left behind + this.fileSize = this.buf.byteLength; this.context = createAudioContext(); this.resampledWaveform = arrayFastResample(seedWaveform ?? DEFAULT_WAVEFORM, PLAYBACK_WAVEFORM_SAMPLES); this.waveformObservable.update(this.resampledWaveform); this.clock = new PlaybackClock(this.context); } + /** + * Size of the audio clip in bytes. May be zero if unknown. This is updated + * when the playback goes through phase changes. + */ + public get sizeBytes(): number { + return this.fileSize; + } + /** * Stable waveform for the playback. Values are guaranteed to be between * zero and one, inclusive. diff --git a/src/voice/PlaybackClock.ts b/src/voice/PlaybackClock.ts index d6d36e861f..9c2d36923f 100644 --- a/src/voice/PlaybackClock.ts +++ b/src/voice/PlaybackClock.ts @@ -14,8 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {SimpleObservable} from "matrix-widget-api"; -import {IDestroyable} from "../utils/IDestroyable"; +import { SimpleObservable } from "matrix-widget-api"; +import { IDestroyable } from "../utils/IDestroyable"; +import { MatrixEvent } from "matrix-js-sdk/src/models/event"; // Because keeping track of time is sufficiently complicated... export class PlaybackClock implements IDestroyable { @@ -25,12 +26,13 @@ export class PlaybackClock implements IDestroyable { private observable = new SimpleObservable(); private timerId: number; private clipDuration = 0; + private placeholderDuration = 0; public constructor(private context: AudioContext) { } public get durationSeconds(): number { - return this.clipDuration; + return this.clipDuration || this.placeholderDuration; } public set durationSeconds(val: number) { @@ -54,6 +56,16 @@ export class PlaybackClock implements IDestroyable { } }; + /** + * Populates default information about the audio clip from the event body. + * The placeholders will be overridden once known. + * @param {MatrixEvent} event The event to use for placeholders. + */ + public populatePlaceholdersFrom(event: MatrixEvent) { + const durationSeconds = Number(event.getContent()['info']?.['duration']); + if (Number.isFinite(durationSeconds)) this.placeholderDuration = durationSeconds; + } + /** * Mark the time in the audio context where the clip starts/has been loaded. * This is to ensure the clock isn't skewed into thinking it is ~0.5s into From 8ce77e618f4dbeaa007bc1e3c513717c5998275a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 23 Jun 2021 13:25:07 -0600 Subject: [PATCH 05/23] Reduce pointless CSS vars by 2 --- res/css/views/audio_messages/_PlaybackContainer.scss | 6 +++--- res/themes/dark/css/_dark.scss | 2 -- res/themes/legacy-dark/css/_legacy-dark.scss | 2 -- res/themes/legacy-light/css/_legacy-light.scss | 2 -- res/themes/light/css/_light.scss | 2 -- 5 files changed, 3 insertions(+), 11 deletions(-) diff --git a/res/css/views/audio_messages/_PlaybackContainer.scss b/res/css/views/audio_messages/_PlaybackContainer.scss index 20def16d6a..e992dd10cc 100644 --- a/res/css/views/audio_messages/_PlaybackContainer.scss +++ b/res/css/views/audio_messages/_PlaybackContainer.scss @@ -22,14 +22,14 @@ limitations under the License. // 7px top and bottom for visual design. 12px left & right, but the waveform (right) // has a 1px padding on it that we want to account for. padding: 7px 12px 7px 11px; - background-color: $voice-record-waveform-bg-color; + background-color: $message-body-panel-bg-color; border-radius: 12px; // Cheat at alignment a bit display: flex; align-items: center; - color: $voice-record-waveform-fg-color; + color: $message-body-panel-fg-color; font-size: $font-14px; line-height: $font-24px; @@ -40,7 +40,7 @@ limitations under the License. &.mx_Waveform_bar_100pct { // Small animation to remove the mechanical feel of progress transition: background-color 250ms ease; - background-color: $voice-record-waveform-fg-color; + background-color: $message-body-panel-fg-color; } } } diff --git a/res/themes/dark/css/_dark.scss b/res/themes/dark/css/_dark.scss index 8b5fde3bd1..81fd3c892a 100644 --- a/res/themes/dark/css/_dark.scss +++ b/res/themes/dark/css/_dark.scss @@ -215,8 +215,6 @@ $message-body-panel-icon-fg-color: #21262C; // "Separator" $message-body-panel-icon-bg-color: $tertiary-fg-color; $voice-record-stop-border-color: $quaternary-fg-color; -$voice-record-waveform-bg-color: $message-body-panel-bg-color; -$voice-record-waveform-fg-color: $message-body-panel-fg-color; $voice-record-waveform-incomplete-fg-color: $quaternary-fg-color; $voice-record-icon-color: $quaternary-fg-color; $voice-playback-button-bg-color: $message-body-panel-icon-bg-color; diff --git a/res/themes/legacy-dark/css/_legacy-dark.scss b/res/themes/legacy-dark/css/_legacy-dark.scss index eb6dc40599..a6a038f290 100644 --- a/res/themes/legacy-dark/css/_legacy-dark.scss +++ b/res/themes/legacy-dark/css/_legacy-dark.scss @@ -209,8 +209,6 @@ $message-body-panel-icon-bg-color: $secondary-fg-color; // See non-legacy dark for variable information $voice-record-stop-border-color: #6F7882; -$voice-record-waveform-bg-color: $message-body-panel-bg-color; -$voice-record-waveform-fg-color: $message-body-panel-fg-color; $voice-record-waveform-incomplete-fg-color: #6F7882; $voice-record-icon-color: #6F7882; $voice-playback-button-bg-color: $tertiary-fg-color; diff --git a/res/themes/legacy-light/css/_legacy-light.scss b/res/themes/legacy-light/css/_legacy-light.scss index a6b180bab4..7c1495d935 100644 --- a/res/themes/legacy-light/css/_legacy-light.scss +++ b/res/themes/legacy-light/css/_legacy-light.scss @@ -334,8 +334,6 @@ $message-body-panel-icon-bg-color: $primary-bg-color; $voice-record-stop-symbol-color: #ff4b55; $voice-record-live-circle-color: #ff4b55; $voice-record-stop-border-color: #E3E8F0; -$voice-record-waveform-bg-color: $message-body-panel-bg-color; -$voice-record-waveform-fg-color: $message-body-panel-fg-color; $voice-record-waveform-incomplete-fg-color: #C1C6CD; $voice-record-icon-color: $tertiary-fg-color; $voice-playback-button-bg-color: $message-body-panel-icon-bg-color; diff --git a/res/themes/light/css/_light.scss b/res/themes/light/css/_light.scss index d8dab9c9c4..7e958c2af6 100644 --- a/res/themes/light/css/_light.scss +++ b/res/themes/light/css/_light.scss @@ -335,8 +335,6 @@ $voice-record-stop-symbol-color: #ff4b55; $voice-record-live-circle-color: #ff4b55; $voice-record-stop-border-color: #E3E8F0; // "Separator" -$voice-record-waveform-bg-color: $message-body-panel-bg-color; -$voice-record-waveform-fg-color: $message-body-panel-fg-color; $voice-record-waveform-incomplete-fg-color: $quaternary-fg-color; $voice-record-icon-color: $tertiary-fg-color; $voice-playback-button-bg-color: $message-body-panel-icon-bg-color; From ffef2e58cb72fba6551c8f2611007920cb0f62ab Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 23 Jun 2021 13:34:57 -0600 Subject: [PATCH 06/23] Unify audio message types to new media body --- res/css/_components.scss | 1 + .../audio_messages/_PlaybackContainer.scss | 6 ---- res/css/views/messages/_MediaBody.scss | 28 +++++++++++++++++++ .../views/audio_messages/AudioPlayer.tsx | 5 ++-- .../audio_messages/RecordingPlayback.tsx | 2 +- .../views/rooms/VoiceRecordComposerTile.tsx | 2 +- 6 files changed, 33 insertions(+), 11 deletions(-) create mode 100644 res/css/views/messages/_MediaBody.scss diff --git a/res/css/_components.scss b/res/css/_components.scss index cb921e251c..d86ce8d017 100644 --- a/res/css/_components.scss +++ b/res/css/_components.scss @@ -169,6 +169,7 @@ @import "./views/messages/_MTextBody.scss"; @import "./views/messages/_MVideoBody.scss"; @import "./views/messages/_MVoiceMessageBody.scss"; +@import "./views/messages/_MediaBody.scss"; @import "./views/messages/_MessageActionBar.scss"; @import "./views/messages/_MessageTimestamp.scss"; @import "./views/messages/_MjolnirBody.scss"; diff --git a/res/css/views/audio_messages/_PlaybackContainer.scss b/res/css/views/audio_messages/_PlaybackContainer.scss index e992dd10cc..c1192f188b 100644 --- a/res/css/views/audio_messages/_PlaybackContainer.scss +++ b/res/css/views/audio_messages/_PlaybackContainer.scss @@ -22,17 +22,11 @@ limitations under the License. // 7px top and bottom for visual design. 12px left & right, but the waveform (right) // has a 1px padding on it that we want to account for. padding: 7px 12px 7px 11px; - background-color: $message-body-panel-bg-color; - border-radius: 12px; // Cheat at alignment a bit display: flex; align-items: center; - color: $message-body-panel-fg-color; - font-size: $font-14px; - line-height: $font-24px; - .mx_Waveform { .mx_Waveform_bar { background-color: $voice-record-waveform-incomplete-fg-color; diff --git a/res/css/views/messages/_MediaBody.scss b/res/css/views/messages/_MediaBody.scss new file mode 100644 index 0000000000..12e441750c --- /dev/null +++ b/res/css/views/messages/_MediaBody.scss @@ -0,0 +1,28 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// A "media body" is any file upload looking thing, apart from images and videos (they +// have unique styles). + +.mx_MediaBody { + background-color: $message-body-panel-bg-color; + border-radius: 12px; + + color: $message-body-panel-fg-color; + font-size: $font-14px; + line-height: $font-24px; +} + diff --git a/src/components/views/audio_messages/AudioPlayer.tsx b/src/components/views/audio_messages/AudioPlayer.tsx index 82372aca74..93bfab3dcd 100644 --- a/src/components/views/audio_messages/AudioPlayer.tsx +++ b/src/components/views/audio_messages/AudioPlayer.tsx @@ -17,7 +17,6 @@ limitations under the License. import { Playback, PlaybackState } from "../../../voice/Playback"; import React, { createRef, ReactNode, RefObject } from "react"; import { UPDATE_EVENT } from "../../../stores/AsyncStore"; -import PlaybackWaveform from "./PlaybackWaveform"; import PlayPauseButton from "./PlayPauseButton"; import { replaceableComponent } from "../../../utils/replaceableComponent"; import { formatBytes } from "../../../utils/FormattingUtils"; @@ -76,7 +75,7 @@ export default class AudioPlayer extends React.PureComponent { public render(): ReactNode { // tabIndex=0 to ensure that the whole component becomes a tab stop, where we handle keyboard // events for accessibility - return
+ return
{ { this.renderFileSize() }
- + TODO: Seek bar
} } diff --git a/src/components/views/audio_messages/RecordingPlayback.tsx b/src/components/views/audio_messages/RecordingPlayback.tsx index 32ce32805d..7c7a0a87c1 100644 --- a/src/components/views/audio_messages/RecordingPlayback.tsx +++ b/src/components/views/audio_messages/RecordingPlayback.tsx @@ -55,7 +55,7 @@ export default class RecordingPlayback extends React.PureComponent + return
diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index d8c481e332..d956a3febf 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -177,7 +177,7 @@ export default class VoiceRecordComposerTile extends React.PureComponent + return
; From aaec9857fd2762901bf0356fdef3f48f42cf1064 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 23 Jun 2021 19:34:25 -0600 Subject: [PATCH 07/23] Add optional mark function callback --- src/utils/MarkedExecution.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/utils/MarkedExecution.ts b/src/utils/MarkedExecution.ts index b0b8fdf63d..01cc91adce 100644 --- a/src/utils/MarkedExecution.ts +++ b/src/utils/MarkedExecution.ts @@ -26,9 +26,11 @@ export class MarkedExecution { /** * Creates a MarkedExecution for the provided function. - * @param fn The function to be called upon trigger if marked. + * @param {Function} fn The function to be called upon trigger if marked. + * @param {Function} onMarkCallback A function that is called when a new mark is made. Not + * called if a mark is already flagged. */ - constructor(private fn: () => void) { + constructor(private fn: () => void, private onMarkCallback?: () => void) { } /** @@ -42,6 +44,7 @@ export class MarkedExecution { * Marks the function to be called upon trigger(). */ public mark() { + if (!this.marked) this.onMarkCallback?.(); this.marked = true; } From 9c752680ba9689f35df94d8efbdfae688a382849 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 23 Jun 2021 19:34:34 -0600 Subject: [PATCH 08/23] Tile styling and early behaviour --- res/css/_components.scss | 3 +- .../views/audio_messages/_AudioPlayer.scss | 68 +++++++++++ .../audio_messages/_PlayPauseButton.scss | 2 + res/css/views/audio_messages/_SeekBar.scss | 89 ++++++++++++++ res/css/views/messages/_MAudioBody.scss | 19 --- res/themes/legacy-dark/css/_legacy-dark.scss | 3 + .../legacy-light/css/_legacy-light.scss | 3 + .../views/audio_messages/AudioPlayer.tsx | 23 +++- .../views/audio_messages/PlaybackClock.tsx | 11 +- .../views/audio_messages/SeekBar.tsx | 110 ++++++++++++++++++ src/components/views/messages/MAudioBody.tsx | 2 +- src/i18n/strings/en_EN.json | 7 +- 12 files changed, 310 insertions(+), 30 deletions(-) create mode 100644 res/css/views/audio_messages/_AudioPlayer.scss create mode 100644 res/css/views/audio_messages/_SeekBar.scss delete mode 100644 res/css/views/messages/_MAudioBody.scss create mode 100644 src/components/views/audio_messages/SeekBar.tsx diff --git a/res/css/_components.scss b/res/css/_components.scss index d86ce8d017..1517527034 100644 --- a/res/css/_components.scss +++ b/res/css/_components.scss @@ -37,8 +37,10 @@ @import "./structures/_ViewSource.scss"; @import "./structures/auth/_CompleteSecurity.scss"; @import "./structures/auth/_Login.scss"; +@import "./views/audio_messages/_AudioPlayer.scss"; @import "./views/audio_messages/_PlayPauseButton.scss"; @import "./views/audio_messages/_PlaybackContainer.scss"; +@import "./views/audio_messages/_SeekBar.scss"; @import "./views/audio_messages/_Waveform.scss"; @import "./views/auth/_AuthBody.scss"; @import "./views/auth/_AuthButtons.scss"; @@ -159,7 +161,6 @@ @import "./views/messages/_CreateEvent.scss"; @import "./views/messages/_DateSeparator.scss"; @import "./views/messages/_EventTileBubble.scss"; -@import "./views/messages/_MAudioBody.scss"; @import "./views/messages/_MEmoteBody.scss"; @import "./views/messages/_MFileBody.scss"; @import "./views/messages/_MImageBody.scss"; diff --git a/res/css/views/audio_messages/_AudioPlayer.scss b/res/css/views/audio_messages/_AudioPlayer.scss new file mode 100644 index 0000000000..2d3777aeca --- /dev/null +++ b/res/css/views/audio_messages/_AudioPlayer.scss @@ -0,0 +1,68 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.mx_AudioPlayer_container { + padding: 16px 12px 11px 12px; + width: 267px; + + .mx_AudioPlayer_primaryContainer { + display: flex; + + .mx_PlayPauseButton { + margin-right: 8px; + } + + .mx_AudioPlayer_mediaInfo { + flex: 1; + overflow: hidden; // makes the ellipsis on the file name work + + & > * { + display: block; + } + + .mx_AudioPlayer_mediaName { + color: $primary-fg-color; + font-size: $font-15px; + line-height: $font-15px; + text-overflow: ellipsis; + white-space: nowrap; + overflow: hidden; + padding-bottom: 4px; // mimics the line-height differences in the Figma + } + + .mx_AudioPlayer_byline { + font-size: $font-12px; + line-height: $font-12px; + } + } + } + + .mx_AudioPlayer_seek { + display: flex; + align-items: center; + + .mx_SeekBar { + flex: 1; + } + + .mx_Clock { + width: $font-42px; // we're not using a monospace font, so fake it + min-width: $font-42px; // for flexbox + padding-left: 4px; // isolate from seek bar + text-align: right; + } + } +} diff --git a/res/css/views/audio_messages/_PlayPauseButton.scss b/res/css/views/audio_messages/_PlayPauseButton.scss index 6caedafa29..714da3e605 100644 --- a/res/css/views/audio_messages/_PlayPauseButton.scss +++ b/res/css/views/audio_messages/_PlayPauseButton.scss @@ -18,6 +18,8 @@ limitations under the License. position: relative; width: 32px; height: 32px; + min-width: 32px; // for when the button is used in a flexbox + min-height: 32px; // for when the button is used in a flexbox border-radius: 32px; background-color: $voice-playback-button-bg-color; diff --git a/res/css/views/audio_messages/_SeekBar.scss b/res/css/views/audio_messages/_SeekBar.scss new file mode 100644 index 0000000000..23fc0bf5db --- /dev/null +++ b/res/css/views/audio_messages/_SeekBar.scss @@ -0,0 +1,89 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// CSS inspiration from: +// * https://www.w3schools.com/howto/howto_js_rangeslider.asp +// * https://stackoverflow.com/a/28283806 +// * https://css-tricks.com/styling-cross-browser-compatible-range-inputs-css/ + +.mx_SeekBar { + // Dev note: we deliberately do not have the -ms-track (and friends) selectors because we don't + // need to support IE. + + appearance: none; // default style override + + width: 100%; + height: 1px; + background: $quaternary-fg-color; + outline: none; // remove blue selection border + position: relative; // for progress bar support later on + + &::-webkit-slider-thumb { + appearance: none; // default style override + + // Dev note: This needs to be duplicated with the -moz-range-thumb selector + // because otherwise Edge (webkit) will fail to see the styles and just refuse + // to apply them. + width: 8px; + height: 8px; + border-radius: 8px; + background-color: $tertiary-fg-color; + cursor: pointer; + } + + &::-moz-range-thumb { + width: 8px; + height: 8px; + border-radius: 8px; + background-color: $tertiary-fg-color; + cursor: pointer; + + // Firefox adds a border on the thumb + border: none; + } + + // This is for webkit support, but we can't limit the functionality of it to just webkit + // browsers. Firefox responds to webkit-prefixed values now, which means we can't use media + // or support queries to selectively apply the rule. An upside is that this CSS doesn't work + // in firefox, so it's just wasted CPU/GPU time. + &::before { // ::before to ensure it ends up under the thumb + content: ''; + background-color: $tertiary-fg-color; + + // Absolute positioning to ensure it overlaps with the existing bar + position: absolute; + top: 0; + left: 0; + + // Sizing to match the bar + width: 100%; + height: 1px; + + // And finally dynamic width without overly hurting the rendering engine. + transform-origin: 0 100%; + transform: scaleX(var(--fillTo)); + } + + // This is firefox's built-in support for the above, with 100% less hacks. + &::-moz-range-progress { + background-color: $tertiary-fg-color; + height: 1px; + } + + &:disabled { + opacity: 0.5; + } +} diff --git a/res/css/views/messages/_MAudioBody.scss b/res/css/views/messages/_MAudioBody.scss deleted file mode 100644 index 2c9b6f7030..0000000000 --- a/res/css/views/messages/_MAudioBody.scss +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2021 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -.mx_MAudioBody { - display: inline-block; // makes the playback controls magically line up -} diff --git a/res/themes/legacy-dark/css/_legacy-dark.scss b/res/themes/legacy-dark/css/_legacy-dark.scss index a6a038f290..df01efbe1e 100644 --- a/res/themes/legacy-dark/css/_legacy-dark.scss +++ b/res/themes/legacy-dark/css/_legacy-dark.scss @@ -20,6 +20,9 @@ $tertiary-fg-color: $primary-fg-color; $primary-bg-color: $bg-color; $muted-fg-color: $header-panel-text-primary-color; +// Legacy theme backports +$quaternary-fg-color: #6F7882; + // used for dialog box text $light-fg-color: $header-panel-text-secondary-color; diff --git a/res/themes/legacy-light/css/_legacy-light.scss b/res/themes/legacy-light/css/_legacy-light.scss index 7c1495d935..c7debcdabe 100644 --- a/res/themes/legacy-light/css/_legacy-light.scss +++ b/res/themes/legacy-light/css/_legacy-light.scss @@ -28,6 +28,9 @@ $tertiary-fg-color: $primary-fg-color; $primary-bg-color: #ffffff; $muted-fg-color: #61708b; // Commonly used in headings and relevant alt text +// Legacy theme backports +$quaternary-fg-color: #C1C6CD; + // used for dialog box text $light-fg-color: #747474; diff --git a/src/components/views/audio_messages/AudioPlayer.tsx b/src/components/views/audio_messages/AudioPlayer.tsx index 93bfab3dcd..336139e467 100644 --- a/src/components/views/audio_messages/AudioPlayer.tsx +++ b/src/components/views/audio_messages/AudioPlayer.tsx @@ -22,11 +22,16 @@ import { replaceableComponent } from "../../../utils/replaceableComponent"; import { formatBytes } from "../../../utils/FormattingUtils"; import DurationClock from "./DurationClock"; import { Key } from "../../../Keyboard"; +import { _t } from "../../../languageHandler"; +import SeekBar from "./SeekBar"; +import PlaybackClock from "./PlaybackClock"; interface IProps { // Playback instance to render. Cannot change during component lifecycle: create // an all-new component instead. playback: Playback; + + mediaName: string; } interface IState { @@ -83,13 +88,21 @@ export default class AudioPlayer extends React.PureComponent { tabIndex={-1} // prevent tabbing into the button ref={this.playPauseRef} /> -
- -   {/* easiest way to introduce a gap between the components */} - { this.renderFileSize() } +
+ + {this.props.mediaName || _t("Unnamed audio")} + +
+ +   {/* easiest way to introduce a gap between the components */} + { this.renderFileSize() } +
- TODO: Seek bar +
+ + +
} } diff --git a/src/components/views/audio_messages/PlaybackClock.tsx b/src/components/views/audio_messages/PlaybackClock.tsx index 82108ae63c..d94e0bed82 100644 --- a/src/components/views/audio_messages/PlaybackClock.tsx +++ b/src/components/views/audio_messages/PlaybackClock.tsx @@ -22,6 +22,11 @@ import { UPDATE_EVENT } from "../../../stores/AsyncStore"; interface IProps { playback: Playback; + + // The default number of seconds to show when the playback has completed or + // has not started. Not used during playback, even when paused. Defaults to + // clip duration length. + defaultDisplaySeconds?: number; } interface IState { @@ -64,7 +69,11 @@ export default class PlaybackClock extends React.PureComponent { public render() { let seconds = this.state.seconds; if (this.state.playbackPhase === PlaybackState.Stopped) { - seconds = this.state.durationSeconds; + if (Number.isFinite(this.props.defaultDisplaySeconds)) { + seconds = this.props.defaultDisplaySeconds; + } else { + seconds = this.state.durationSeconds; + } } return ; } diff --git a/src/components/views/audio_messages/SeekBar.tsx b/src/components/views/audio_messages/SeekBar.tsx new file mode 100644 index 0000000000..2561321df3 --- /dev/null +++ b/src/components/views/audio_messages/SeekBar.tsx @@ -0,0 +1,110 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { Playback, PlaybackState } from "../../../voice/Playback"; +import React, { ChangeEvent, createRef, CSSProperties, ReactNode, RefObject } from "react"; +import { UPDATE_EVENT } from "../../../stores/AsyncStore"; +import PlayPauseButton from "./PlayPauseButton"; +import { replaceableComponent } from "../../../utils/replaceableComponent"; +import { formatBytes } from "../../../utils/FormattingUtils"; +import DurationClock from "./DurationClock"; +import { Key } from "../../../Keyboard"; +import { _t } from "../../../languageHandler"; +import { MarkedExecution } from "../../../utils/MarkedExecution"; +import { percentageOf } from "../../../utils/numbers"; + +interface IProps { + // Playback instance to render. Cannot change during component lifecycle: create + // an all-new component instead. + playback: Playback; + + // Tab index for the underlying component. Useful if the seek bar is in a managed state. + // Defaults to zero. + tabIndex?: number; + + playbackPhase: PlaybackState; +} + +interface IState { + percentage: number; +} + +interface ISeekCSS extends CSSProperties { + '--fillTo': number; +} + +@replaceableComponent("views.audio_messages.SeekBar") +export default class SeekBar extends React.PureComponent { + // We use an animation frame request to avoid overly spamming prop updates, even if we aren't + // really using anything demanding on the CSS front. + + private animationFrameFn = new MarkedExecution( + () => this.doUpdate(), + () => requestAnimationFrame(() => this.animationFrameFn.trigger())); + + public static defaultProps = { + tabIndex: 0, + }; + + constructor(props: IProps) { + super(props); + + this.state = { + percentage: 0, + }; + + // We don't need to de-register: the class handles this for us internally + this.props.playback.clockInfo.liveData.onUpdate(() => this.animationFrameFn.mark()); + } + + private doUpdate() { + this.setState({ + percentage: percentageOf( + this.props.playback.clockInfo.timeSeconds, + 0, + this.props.playback.clockInfo.durationSeconds), + }); + } + + public left() { + console.log("@@ LEFT"); + } + + public right() { + console.log("@@ RIGHT"); + } + + private onChange = (ev: ChangeEvent) => { + console.log('@@ CHANGE', ev.target.value); + }; + + public render(): ReactNode { + // We use a range input to avoid having to re-invent accessibility handling on + // a custom set of divs. + return ; + } +} diff --git a/src/components/views/messages/MAudioBody.tsx b/src/components/views/messages/MAudioBody.tsx index a8f2304fe9..8791828828 100644 --- a/src/components/views/messages/MAudioBody.tsx +++ b/src/components/views/messages/MAudioBody.tsx @@ -102,7 +102,7 @@ export default class MAudioBody extends React.PureComponent { // At this point we should have a playable state return ( - + ) diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index bc62868a0f..9da5a41327 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -918,8 +918,6 @@ "Silence call": "Silence call", "Decline": "Decline", "Accept": "Accept", - "Pause": "Pause", - "Play": "Play", "The other party cancelled the verification.": "The other party cancelled the verification.", "Verified!": "Verified!", "You've successfully verified this user.": "You've successfully verified this user.", @@ -1870,7 +1868,7 @@ "Ignored attempt to disable encryption": "Ignored attempt to disable encryption", "Encryption not enabled": "Encryption not enabled", "The encryption used by this room isn't supported.": "The encryption used by this room isn't supported.", - "Error decrypting audio": "Error decrypting audio", + "Error processing audio message": "Error processing audio message", "React": "React", "Edit": "Edit", "Retry": "Retry", @@ -2602,6 +2600,9 @@ "Use email or phone to optionally be discoverable by existing contacts.": "Use email or phone to optionally be discoverable by existing contacts.", "Use email to optionally be discoverable by existing contacts.": "Use email to optionally be discoverable by existing contacts.", "Sign in with SSO": "Sign in with SSO", + "Unnamed audio": "Unnamed audio", + "Pause": "Pause", + "Play": "Play", "Couldn't load page": "Couldn't load page", "You must register to use this functionality": "You must register to use this functionality", "You must join the room to see its files": "You must join the room to see its files", From ebb6f1b602a0f5fb0d2c799db551d4a213c11602 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 24 Jun 2021 20:18:50 -0600 Subject: [PATCH 09/23] Add seeking and notes about clock desync --- res/css/views/audio_messages/_SeekBar.scss | 2 + .../views/audio_messages/AudioPlayer.tsx | 7 ++ .../views/audio_messages/SeekBar.tsx | 14 +++- src/voice/Playback.ts | 81 ++++++++++++++++--- src/voice/PlaybackClock.ts | 56 ++++++++++++- 5 files changed, 143 insertions(+), 17 deletions(-) diff --git a/res/css/views/audio_messages/_SeekBar.scss b/res/css/views/audio_messages/_SeekBar.scss index 23fc0bf5db..ca3f7aa562 100644 --- a/res/css/views/audio_messages/_SeekBar.scss +++ b/res/css/views/audio_messages/_SeekBar.scss @@ -31,6 +31,8 @@ limitations under the License. outline: none; // remove blue selection border position: relative; // for progress bar support later on + cursor: pointer; + &::-webkit-slider-thumb { appearance: none; // default style override diff --git a/src/components/views/audio_messages/AudioPlayer.tsx b/src/components/views/audio_messages/AudioPlayer.tsx index 336139e467..0767c814d8 100644 --- a/src/components/views/audio_messages/AudioPlayer.tsx +++ b/src/components/views/audio_messages/AudioPlayer.tsx @@ -41,6 +41,7 @@ interface IState { @replaceableComponent("views.audio_messages.AudioPlayer") export default class AudioPlayer extends React.PureComponent { private playPauseRef: RefObject = createRef(); + private seekRef: RefObject = createRef(); constructor(props: IProps) { super(props); @@ -66,6 +67,12 @@ export default class AudioPlayer extends React.PureComponent { if (ev.key === Key.SPACE) { ev.stopPropagation(); this.playPauseRef.current?.toggle(); + } else if (ev.key === Key.ARROW_LEFT) { + ev.stopPropagation(); + this.seekRef.current?.left(); + } else if (ev.key === Key.ARROW_RIGHT) { + ev.stopPropagation(); + this.seekRef.current?.right(); } }; diff --git a/src/components/views/audio_messages/SeekBar.tsx b/src/components/views/audio_messages/SeekBar.tsx index 2561321df3..09beed70b6 100644 --- a/src/components/views/audio_messages/SeekBar.tsx +++ b/src/components/views/audio_messages/SeekBar.tsx @@ -46,6 +46,8 @@ interface ISeekCSS extends CSSProperties { '--fillTo': number; } +const ARROW_SKIP_SECONDS = 5; // arbitrary + @replaceableComponent("views.audio_messages.SeekBar") export default class SeekBar extends React.PureComponent { // We use an animation frame request to avoid overly spamming prop updates, even if we aren't @@ -80,15 +82,21 @@ export default class SeekBar extends React.PureComponent { } public left() { - console.log("@@ LEFT"); + // noinspection JSIgnoredPromiseFromCall + this.props.playback.skipTo(this.props.playback.clockInfo.timeSeconds - ARROW_SKIP_SECONDS); } public right() { - console.log("@@ RIGHT"); + // noinspection JSIgnoredPromiseFromCall + this.props.playback.skipTo(this.props.playback.clockInfo.timeSeconds + ARROW_SKIP_SECONDS); } private onChange = (ev: ChangeEvent) => { - console.log('@@ CHANGE', ev.target.value); + // Thankfully, onChange is only called when the user changes the value, not when we + // change the value on the component. We can use this as a reliable "skip to X" function. + // + // noinspection JSIgnoredPromiseFromCall + this.props.playback.skipTo(Number(ev.target.value) * this.props.playback.clockInfo.durationSeconds); }; public render(): ReactNode { diff --git a/src/voice/Playback.ts b/src/voice/Playback.ts index f1f91310b2..6a120bf924 100644 --- a/src/voice/Playback.ts +++ b/src/voice/Playback.ts @@ -161,16 +161,9 @@ export class Playback extends EventEmitter implements IDestroyable { public async play() { // We can't restart a buffer source, so we need to create a new one if we hit the end if (this.state === PlaybackState.Stopped) { - if (this.source) { - this.source.disconnect(); - this.source.removeEventListener("ended", this.onPlaybackEnd); - } - - this.source = this.context.createBufferSource(); - this.source.connect(this.context.destination); - this.source.buffer = this.audioBuf; - this.source.start(); // start immediately - this.source.addEventListener("ended", this.onPlaybackEnd); + this.disconnectSource(); + this.makeNewSourceBuffer(); + this.source.start(); } // We use the context suspend/resume functions because it allows us to pause a source @@ -180,6 +173,18 @@ export class Playback extends EventEmitter implements IDestroyable { this.emit(PlaybackState.Playing); } + private disconnectSource() { + this.source?.disconnect(); + this.source?.removeEventListener("ended", this.onPlaybackEnd); + } + + private makeNewSourceBuffer() { + this.source = this.context.createBufferSource(); + this.source.buffer = this.audioBuf; + this.source.addEventListener("ended", this.onPlaybackEnd); + this.source.connect(this.context.destination); + } + public async pause() { await this.context.suspend(); this.emit(PlaybackState.Paused); @@ -194,4 +199,60 @@ export class Playback extends EventEmitter implements IDestroyable { if (this.isPlaying) await this.pause(); else await this.play(); } + + public async skipTo(timeSeconds: number) { + // Dev note: this function talks a lot about clock desyncs. There is a clock running + // independently to the audio context and buffer so that accurate human-perceptible + // time can be exposed. The PlaybackClock class has more information, but the short + // version is that we need to line up the useful time (clip position) with the context + // time, and avoid as many deviations as possible as otherwise the user could see the + // wrong time, and we stop playback at the wrong time, etc. + + timeSeconds = clamp(timeSeconds, 0, this.clock.durationSeconds); + + // Track playing state so we don't cause seeking to start playing the track. + const isPlaying = this.isPlaying; + + if (isPlaying) { + // Pause first so we can get an accurate measurement of time + await this.context.suspend(); + } + + // We can't simply tell the context/buffer to jump to a time, so we have to + // start a whole new buffer and start it from the new time offset. + const now = this.context.currentTime; + this.disconnectSource(); + this.makeNewSourceBuffer(); + + // We have to resync the clock because it can get confused about where we're + // at in the audio clip. + this.clock.syncTo(now, timeSeconds); + + // Always start the source to queue it up. We have to do this now (and pause + // quickly if we're not supposed to be playing) as otherwise the clock can desync + // when it comes time to the user hitting play. After a couple jumps, the user + // will have desynced the clock enough to be about 10-15 seconds off, while this + // keeps it as close to perfect as humans can perceive. + this.source.start(now, timeSeconds); + + // Dev note: it's critical that the code gap between `this.source.start()` and + // `this.pause()` is as small as possible: we do not want to delay *anything* + // as that could cause a clock desync, or a buggy feeling as a single note plays + // during seeking. + + if (isPlaying) { + // If we were playing before, continue the context so the clock doesn't desync. + await this.context.resume(); + } else { + // As mentioned above, we'll have to pause the clip if we weren't supposed to + // be playing it just yet. If we didn't have this, the audio clip plays but all + // the states will be wrong: clock won't advance, pause state doesn't match the + // blaring noise leaving the user's speakers, etc. + // + // Also as mentioned, if the code gap is small enough then this should be + // executed immediately after the start time, leaving no feasible time for the + // user's speakers to play any sound. + await this.pause(); + } + } } diff --git a/src/voice/PlaybackClock.ts b/src/voice/PlaybackClock.ts index 9c2d36923f..e3f41930de 100644 --- a/src/voice/PlaybackClock.ts +++ b/src/voice/PlaybackClock.ts @@ -18,7 +18,42 @@ import { SimpleObservable } from "matrix-widget-api"; import { IDestroyable } from "../utils/IDestroyable"; import { MatrixEvent } from "matrix-js-sdk/src/models/event"; -// Because keeping track of time is sufficiently complicated... +/** + * Tracks accurate human-perceptible time for an audio clip, as informed + * by managed playback. This clock is tightly coupled with the operation + * of the Playback class, making assumptions about how the provided + * AudioContext will be used (suspended/resumed to preserve time, etc). + * + * But why do we need a clock? The AudioContext exposes time information, + * and so does the audio buffer, but not in a way that is useful for humans + * to perceive. The audio buffer time is often lagged behind the context + * time due to internal processing delays of the audio API. Additionally, + * the context's time is tracked from when it was first initialized/started, + * not related to positioning within the clip. However, the context time + * is the most accurate time we can use to determine position within the + * clip if we're fast enough to track the pauses and stops. + * + * As a result, we track every play, pause, stop, and seek event from the + * Playback class (kinda: it calls us, which is close enough to the same + * thing). These events are then tracked on the AudioContext time scale, + * with assumptions that code execution will result in negligible desync + * of the clock, or at least no perceptible difference in time. It's + * extremely important that the calling code, and the clock's own code, + * is extremely fast between the event happening and the clock time being + * tracked - anything more than a dozen milliseconds is likely to stack up + * poorly, leading to clock desync. + * + * Clock desync can be dangerous for the stability of the playback controls: + * if the clock thinks the user is somewhere else in the clip, it could + * inform the playback of the wrong place in time, leading to dead air in + * the output or, if severe enough, a clock that won't stop running while + * the audio is paused/stopped. Other examples include the clip stopping at + * 90% time due to playback ending, the clip playing from the wrong spot + * relative to the time, and negative clock time. + * + * Note that the clip duration is fed to the clock: this is to ensure that + * we have the most accurate time possible to present. + */ export class PlaybackClock implements IDestroyable { private clipStart = 0; private stopped = true; @@ -41,6 +76,12 @@ export class PlaybackClock implements IDestroyable { } public get timeSeconds(): number { + // The modulo is to ensure that we're only looking at the most recent clip + // time, as the context is long-running and multiple plays might not be + // informed to us (if the control is looping, for example). By taking the + // remainder of the division operation, we're assuming that playback is + // incomplete or stopped, thus giving an accurate position within the active + // clip segment. return (this.context.currentTime - this.clipStart) % this.clipDuration; } @@ -49,7 +90,7 @@ export class PlaybackClock implements IDestroyable { } private checkTime = () => { - const now = this.timeSeconds; + const now = this.timeSeconds; // calculated dynamically if (this.lastCheck !== now) { this.observable.update([now, this.durationSeconds]); this.lastCheck = now; @@ -82,8 +123,9 @@ export class PlaybackClock implements IDestroyable { } if (!this.timerId) { - // case to number because the types are wrong - // 100ms interval to make sure the time is as accurate as possible + // cast to number because the types are wrong + // 100ms interval to make sure the time is as accurate as possible without + // being overly insane this.timerId = setInterval(this.checkTime, 100); } } @@ -92,6 +134,12 @@ export class PlaybackClock implements IDestroyable { this.stopped = true; } + public syncTo(contextTime: number, clipTime: number) { + this.clipStart = contextTime - clipTime; + this.stopped = false; // count as a mid-stream pause (if we were stopped) + this.checkTime(); + } + public destroy() { this.observable.close(); if (this.timerId) clearInterval(this.timerId); From c5a72ee6ad1c75307132ffe063450c77f77d51f7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 24 Jun 2021 20:26:40 -0600 Subject: [PATCH 10/23] Fix arrow seeking --- .../views/audio_messages/AudioPlayer.tsx | 16 ++++++++++++---- .../views/audio_messages/PlayPauseButton.tsx | 4 ++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/components/views/audio_messages/AudioPlayer.tsx b/src/components/views/audio_messages/AudioPlayer.tsx index 0767c814d8..2f74c7b069 100644 --- a/src/components/views/audio_messages/AudioPlayer.tsx +++ b/src/components/views/audio_messages/AudioPlayer.tsx @@ -63,10 +63,13 @@ export default class AudioPlayer extends React.PureComponent { this.setState({ playbackPhase: ev }); }; - private onKeyPress = (ev: React.KeyboardEvent) => { + private onKeyDown = (ev: React.KeyboardEvent) => { + // stopPropagation() prevents the FocusComposer catch-all from triggering, + // but we need to do it on key down instead of press (even though the user + // interaction is typically on press). if (ev.key === Key.SPACE) { ev.stopPropagation(); - this.playPauseRef.current?.toggle(); + this.playPauseRef.current?.toggleState(); } else if (ev.key === Key.ARROW_LEFT) { ev.stopPropagation(); this.seekRef.current?.left(); @@ -87,7 +90,7 @@ export default class AudioPlayer extends React.PureComponent { public render(): ReactNode { // tabIndex=0 to ensure that the whole component becomes a tab stop, where we handle keyboard // events for accessibility - return
+ return
{
- +
diff --git a/src/components/views/audio_messages/PlayPauseButton.tsx b/src/components/views/audio_messages/PlayPauseButton.tsx index 7d881a10e5..a4f1e770f2 100644 --- a/src/components/views/audio_messages/PlayPauseButton.tsx +++ b/src/components/views/audio_messages/PlayPauseButton.tsx @@ -42,10 +42,10 @@ export default class PlayPauseButton extends React.PureComponent { private onClick = () => { // noinspection JSIgnoredPromiseFromCall - this.toggle(); + this.toggleState(); }; - public async toggle() { + public async toggleState() { await this.props.playback.toggle(); } From dd53c25706757e5d373bc7bcedf19f9a2470ddee Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 24 Jun 2021 20:37:34 -0600 Subject: [PATCH 11/23] Fix right panel sizing --- res/css/views/audio_messages/_AudioPlayer.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/res/css/views/audio_messages/_AudioPlayer.scss b/res/css/views/audio_messages/_AudioPlayer.scss index 2d3777aeca..3f02e54aab 100644 --- a/res/css/views/audio_messages/_AudioPlayer.scss +++ b/res/css/views/audio_messages/_AudioPlayer.scss @@ -16,7 +16,7 @@ limitations under the License. .mx_AudioPlayer_container { padding: 16px 12px 11px 12px; - width: 267px; + max-width: 267px; // use max to make the control fit in the files/pinned panels .mx_AudioPlayer_primaryContainer { display: flex; From d724af600ff47555b1e108056b9bb383b5aab9dc Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 24 Jun 2021 20:41:00 -0600 Subject: [PATCH 12/23] Remove copy/paste fails --- src/components/views/audio_messages/SeekBar.tsx | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/components/views/audio_messages/SeekBar.tsx b/src/components/views/audio_messages/SeekBar.tsx index 09beed70b6..ed07c44482 100644 --- a/src/components/views/audio_messages/SeekBar.tsx +++ b/src/components/views/audio_messages/SeekBar.tsx @@ -15,14 +15,8 @@ limitations under the License. */ import { Playback, PlaybackState } from "../../../voice/Playback"; -import React, { ChangeEvent, createRef, CSSProperties, ReactNode, RefObject } from "react"; -import { UPDATE_EVENT } from "../../../stores/AsyncStore"; -import PlayPauseButton from "./PlayPauseButton"; +import React, { ChangeEvent, CSSProperties, ReactNode } from "react"; import { replaceableComponent } from "../../../utils/replaceableComponent"; -import { formatBytes } from "../../../utils/FormattingUtils"; -import DurationClock from "./DurationClock"; -import { Key } from "../../../Keyboard"; -import { _t } from "../../../languageHandler"; import { MarkedExecution } from "../../../utils/MarkedExecution"; import { percentageOf } from "../../../utils/numbers"; From 76caba0385ee5b333552a7457fd66823e937b668 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 25 Jun 2021 00:10:32 -0600 Subject: [PATCH 13/23] Restore mx_VoiceMessagePrimaryContainer class --- src/components/views/rooms/VoiceRecordComposerTile.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index d956a3febf..036a345af0 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -177,7 +177,7 @@ export default class VoiceRecordComposerTile extends React.PureComponent + return
; From e0a73a583e5f11ed3b42382c7f90b8554873e5dd Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 25 Jun 2021 00:19:57 -0600 Subject: [PATCH 14/23] Increase clickable area of seek bar --- res/css/views/audio_messages/_SeekBar.scss | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/res/css/views/audio_messages/_SeekBar.scss b/res/css/views/audio_messages/_SeekBar.scss index ca3f7aa562..602ce94916 100644 --- a/res/css/views/audio_messages/_SeekBar.scss +++ b/res/css/views/audio_messages/_SeekBar.scss @@ -29,7 +29,7 @@ limitations under the License. height: 1px; background: $quaternary-fg-color; outline: none; // remove blue selection border - position: relative; // for progress bar support later on + position: relative; // for before+after pseudo elements later on cursor: pointer; @@ -88,4 +88,16 @@ limitations under the License. &:disabled { opacity: 0.5; } + + // Increase clickable area for the slider (approximately same size as browser default) + // We do it this way to keep the same padding and margins of the element, avoiding margin math. + // Source: https://front-back.com/expand-clickable-areas-for-a-better-touch-experience/ + &:after { + content: ''; + position: absolute; + top: -6px; + bottom: -6px; + left: 0; + right: 0; + } } From 7a6ee7d91896f234a54a4def20a3bdba57acb04f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 25 Jun 2021 00:22:05 -0600 Subject: [PATCH 15/23] I know how CSS works. --- res/css/views/audio_messages/_SeekBar.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/res/css/views/audio_messages/_SeekBar.scss b/res/css/views/audio_messages/_SeekBar.scss index 602ce94916..d13fe4ac6a 100644 --- a/res/css/views/audio_messages/_SeekBar.scss +++ b/res/css/views/audio_messages/_SeekBar.scss @@ -92,7 +92,7 @@ limitations under the License. // Increase clickable area for the slider (approximately same size as browser default) // We do it this way to keep the same padding and margins of the element, avoiding margin math. // Source: https://front-back.com/expand-clickable-areas-for-a-better-touch-experience/ - &:after { + &::after { content: ''; position: absolute; top: -6px; From d6cf2346fe3da5881eac8ab5a0b282f7da71848b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 28 Jun 2021 21:00:36 -0600 Subject: [PATCH 16/23] Manually conflict resolve https://github.com/matrix-org/matrix-react-sdk/pull/6240 --- .../audio_messages/_PlaybackContainer.scss | 5 ++ .../audio_messages/LiveRecordingClock.tsx | 30 ++++++++--- .../audio_messages/LiveRecordingWaveform.tsx | 49 ++++++++++------- .../views/audio_messages/Waveform.tsx | 13 +++-- .../views/rooms/VoiceRecordComposerTile.tsx | 52 ++----------------- 5 files changed, 68 insertions(+), 81 deletions(-) diff --git a/res/css/views/audio_messages/_PlaybackContainer.scss b/res/css/views/audio_messages/_PlaybackContainer.scss index c1192f188b..fd01864bba 100644 --- a/res/css/views/audio_messages/_PlaybackContainer.scss +++ b/res/css/views/audio_messages/_PlaybackContainer.scss @@ -27,9 +27,14 @@ limitations under the License. display: flex; align-items: center; + contain: content; + .mx_Waveform { .mx_Waveform_bar { background-color: $voice-record-waveform-incomplete-fg-color; + height: 100%; + /* Variable set by a JS component */ + transform: scaleY(max(0.05, var(--barHeight))); &.mx_Waveform_bar_100pct { // Small animation to remove the mechanical feel of progress diff --git a/src/components/views/audio_messages/LiveRecordingClock.tsx b/src/components/views/audio_messages/LiveRecordingClock.tsx index 41909dc201..a9dbd3c52f 100644 --- a/src/components/views/audio_messages/LiveRecordingClock.tsx +++ b/src/components/views/audio_messages/LiveRecordingClock.tsx @@ -18,6 +18,7 @@ import React from "react"; import { IRecordingUpdate, VoiceRecording } from "../../../voice/VoiceRecording"; import { replaceableComponent } from "../../../utils/replaceableComponent"; import Clock from "./Clock"; +import { MarkedExecution } from "../../../utils/MarkedExecution"; interface IProps { recorder: VoiceRecording; @@ -32,16 +33,31 @@ interface IState { */ @replaceableComponent("views.audio_messages.LiveRecordingClock") export default class LiveRecordingClock extends React.PureComponent { - public constructor(props) { - super(props); + private seconds = 0; + private scheduledUpdate = new MarkedExecution( + () => this.updateClock(), + () => requestAnimationFrame(() => this.scheduledUpdate.trigger()), + ); - this.state = {seconds: 0}; - this.props.recorder.liveData.onUpdate(this.onRecordingUpdate); + constructor(props) { + super(props); + this.state = { + seconds: 0, + }; } - private onRecordingUpdate = (update: IRecordingUpdate) => { - this.setState({seconds: update.timeSeconds}); - }; + componentDidMount() { + this.props.recorder.liveData.onUpdate((update: IRecordingUpdate) => { + this.seconds = update.timeSeconds; + this.scheduledUpdate.mark(); + }); + } + + private updateClock() { + this.setState({ + seconds: this.seconds, + }); + } public render() { return ; diff --git a/src/components/views/audio_messages/LiveRecordingWaveform.tsx b/src/components/views/audio_messages/LiveRecordingWaveform.tsx index 27d165e613..6e88346cae 100644 --- a/src/components/views/audio_messages/LiveRecordingWaveform.tsx +++ b/src/components/views/audio_messages/LiveRecordingWaveform.tsx @@ -20,13 +20,14 @@ import { replaceableComponent } from "../../../utils/replaceableComponent"; import { arrayFastResample, arraySeed } from "../../../utils/arrays"; import { percentageOf } from "../../../utils/numbers"; import Waveform from "./Waveform"; +import { MarkedExecution } from "../../../utils/MarkedExecution"; interface IProps { recorder: VoiceRecording; } interface IState { - heights: number[]; + waveform: number[]; } /** @@ -34,27 +35,35 @@ interface IState { */ @replaceableComponent("views.audio_messages.LiveRecordingWaveform") export default class LiveRecordingWaveform extends React.PureComponent { - public constructor(props) { - super(props); - - this.state = {heights: arraySeed(0, RECORDING_PLAYBACK_SAMPLES)}; - this.props.recorder.liveData.onUpdate(this.onRecordingUpdate); - } - - private onRecordingUpdate = (update: IRecordingUpdate) => { - // The waveform and the downsample target are pretty close, so we should be fine to - // do this, despite the docs on arrayFastResample. - const bars = arrayFastResample(Array.from(update.waveform), RECORDING_PLAYBACK_SAMPLES); - this.setState({ - // The incoming data is between zero and one, but typically even screaming into a - // microphone won't send you over 0.6, so we artificially adjust the gain for the - // waveform. This results in a slightly more cinematic/animated waveform for the - // user. - heights: bars.map(b => percentageOf(b, 0, 0.50)), - }); + public static defaultProps = { + progress: 1, }; + private waveform: number[] = []; + private scheduledUpdate = new MarkedExecution( + () => this.updateWaveform(), + () => requestAnimationFrame(() => this.scheduledUpdate.trigger()), + ); + + constructor(props) { + super(props); + this.state = { + waveform: [], + }; + } + + componentDidMount() { + this.props.recorder.liveData.onUpdate((update: IRecordingUpdate) => { + this.waveform = update.waveform; + this.scheduledUpdate.mark(); + }); + } + + private updateWaveform() { + this.setState({ waveform: this.waveform }); + } + public render() { - return ; + return ; } } diff --git a/src/components/views/audio_messages/Waveform.tsx b/src/components/views/audio_messages/Waveform.tsx index dc7551025e..3b7a881754 100644 --- a/src/components/views/audio_messages/Waveform.tsx +++ b/src/components/views/audio_messages/Waveform.tsx @@ -17,6 +17,11 @@ limitations under the License. import React from "react"; import { replaceableComponent } from "../../../utils/replaceableComponent"; import classNames from "classnames"; +import { CSSProperties } from "react"; + +interface WaveformCSSProperties extends CSSProperties { + '--barHeight': number; +} interface IProps { relHeights: number[]; // relative heights (0-1) @@ -40,10 +45,6 @@ export default class Waveform extends React.PureComponent { progress: 1, }; - public constructor(props) { - super(props); - } - public render() { return
{this.props.relHeights.map((h, i) => { @@ -53,7 +54,9 @@ export default class Waveform extends React.PureComponent { 'mx_Waveform_bar': true, 'mx_Waveform_bar_100pct': isCompleteBar, }); - return ; + return ; })}
; } diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index c5a196c377..4e8f258ee8 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -18,22 +18,18 @@ import AccessibleTooltipButton from "../elements/AccessibleTooltipButton"; import { _t } from "../../../languageHandler"; import React, {ReactNode} from "react"; import { - IRecordingUpdate, - RECORDING_PLAYBACK_SAMPLES, RecordingState, VoiceRecording, } from "../../../voice/VoiceRecording"; import {Room} from "matrix-js-sdk/src/models/room"; import { MatrixClientPeg } from "../../../MatrixClientPeg"; import classNames from "classnames"; -import LiveRecordingWaveform from "../voice_messages/LiveRecordingWaveform"; +import LiveRecordingWaveform from "../audio_messages/LiveRecordingWaveform"; import { replaceableComponent } from "../../../utils/replaceableComponent"; -import { arrayFastResample, arraySeed } from "../../../utils/arrays"; -import { percentageOf } from "../../../utils/numbers"; -import LiveRecordingClock from "../voice_messages/LiveRecordingClock"; +import LiveRecordingClock from "../audio_messages/LiveRecordingClock"; import { VoiceRecordingStore } from "../../../stores/VoiceRecordingStore"; import {UPDATE_EVENT} from "../../../stores/AsyncStore"; -import RecordingPlayback from "../voice_messages/RecordingPlayback"; +import RecordingPlayback from "../audio_messages/RecordingPlayback"; import { MsgType } from "matrix-js-sdk/src/@types/event"; import Modal from "../../../Modal"; import ErrorDialog from "../dialogs/ErrorDialog"; @@ -46,8 +42,6 @@ interface IProps { interface IState { recorder?: VoiceRecording; recordingPhase?: RecordingState; - relHeights: number[]; - seconds: number; } /** @@ -55,58 +49,18 @@ interface IState { */ @replaceableComponent("views.rooms.VoiceRecordComposerTile") export default class VoiceRecordComposerTile extends React.PureComponent { - private waveform: number[] = []; - private seconds = 0; - private scheduledAnimationFrame = false; - public constructor(props) { super(props); this.state = { recorder: null, // no recording started by default - seconds: 0, - relHeights: arraySeed(0, RECORDING_PLAYBACK_SAMPLES), }; } - public componentDidUpdate(prevProps, prevState) { - if (!prevState.recorder && this.state.recorder) { - this.state.recorder.liveData.onUpdate(this.onRecordingUpdate); - } - } - public async componentWillUnmount() { await VoiceRecordingStore.instance.disposeRecording(); } - private onRecordingUpdate = (update: IRecordingUpdate): void => { - this.waveform = update.waveform; - this.seconds = update.timeSeconds; - - if (this.scheduledAnimationFrame) { - return; - } - - this.scheduledAnimationFrame = true; - // The audio recorder flushes data faster than the screen refresh rate - // Using requestAnimationFrame makes sure that we only flush the data - // to react once per tick to avoid unneeded work. - requestAnimationFrame(() => { - // The waveform and the downsample target are pretty close, so we should be fine to - // do this, despite the docs on arrayFastResample. - const bars = arrayFastResample(Array.from(this.waveform), RECORDING_PLAYBACK_SAMPLES); - this.setState({ - // The incoming data is between zero and one, but typically even screaming into a - // microphone won't send you over 0.6, so we artificially adjust the gain for the - // waveform. This results in a slightly more cinematic/animated waveform for the - // user. - relHeights: bars.map(b => percentageOf(b, 0, 0.50)), - seconds: this.seconds, - }); - this.scheduledAnimationFrame = false; - }); - } - // called by composer public async send() { if (!this.state.recorder) { From 7a4ceec9858c38c3ff90912553df60ec5686f808 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 28 Jun 2021 21:02:42 -0600 Subject: [PATCH 17/23] Restore clamped dimensions on voice recorder waveform --- .../views/audio_messages/LiveRecordingWaveform.tsx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/components/views/audio_messages/LiveRecordingWaveform.tsx b/src/components/views/audio_messages/LiveRecordingWaveform.tsx index 6e88346cae..b9c5f80f05 100644 --- a/src/components/views/audio_messages/LiveRecordingWaveform.tsx +++ b/src/components/views/audio_messages/LiveRecordingWaveform.tsx @@ -17,7 +17,7 @@ limitations under the License. import React from "react"; import { IRecordingUpdate, RECORDING_PLAYBACK_SAMPLES, VoiceRecording } from "../../../voice/VoiceRecording"; import { replaceableComponent } from "../../../utils/replaceableComponent"; -import { arrayFastResample, arraySeed } from "../../../utils/arrays"; +import { arrayFastResample } from "../../../utils/arrays"; import { percentageOf } from "../../../utils/numbers"; import Waveform from "./Waveform"; import { MarkedExecution } from "../../../utils/MarkedExecution"; @@ -54,7 +54,12 @@ export default class LiveRecordingWaveform extends React.PureComponent { - this.waveform = update.waveform; + const bars = arrayFastResample(Array.from(update.waveform), RECORDING_PLAYBACK_SAMPLES); + // The incoming data is between zero and one, but typically even screaming into a + // microphone won't send you over 0.6, so we artificially adjust the gain for the + // waveform. This results in a slightly more cinematic/animated waveform for the + // user. + this.waveform = bars.map(b => percentageOf(b, 0, 0.50)); this.scheduledUpdate.mark(); }); } From fdced3da1b156fefef6011e1f1395052b9fbddbe Mon Sep 17 00:00:00 2001 From: Germain Souquet Date: Wed, 30 Jun 2021 08:09:55 +0100 Subject: [PATCH 18/23] Remove reminescent references to the tinter --- res/css/structures/_GroupView.scss | 2 +- res/css/views/messages/_MImageBody.scss | 2 +- src/components/structures/RoomView.tsx | 9 --------- src/components/views/rooms/SimpleRoomHeader.js | 2 +- 4 files changed, 3 insertions(+), 12 deletions(-) diff --git a/res/css/structures/_GroupView.scss b/res/css/structures/_GroupView.scss index 2350d9f28a..60f9ebdd08 100644 --- a/res/css/structures/_GroupView.scss +++ b/res/css/structures/_GroupView.scss @@ -323,7 +323,7 @@ limitations under the License. } .mx_GroupView_featuredThing .mx_BaseAvatar { - /* To prevent misalignment with mx_TintableSvg (in addButton) */ + /* To prevent misalignment with img (in addButton) */ vertical-align: initial; } diff --git a/res/css/views/messages/_MImageBody.scss b/res/css/views/messages/_MImageBody.scss index 1c773c2f06..515d867da5 100644 --- a/res/css/views/messages/_MImageBody.scss +++ b/res/css/views/messages/_MImageBody.scss @@ -43,7 +43,7 @@ limitations under the License. top: 50%; } -// Inner img and TintableSvg should be centered around 0, 0 +// Inner img should be centered around 0, 0 .mx_MImageBody_thumbnail_spinner > * { transform: translate(-50%, -50%); } diff --git a/src/components/structures/RoomView.tsx b/src/components/structures/RoomView.tsx index 61685e7ba8..81000a87a6 100644 --- a/src/components/structures/RoomView.tsx +++ b/src/components/structures/RoomView.tsx @@ -1054,11 +1054,6 @@ export default class RoomView extends React.Component { }); } - private updateTint() { - const room = this.state.room; - if (!room) return; - } - private onAccountData = (event: MatrixEvent) => { const type = event.getType(); if ((type === "org.matrix.preview_urls" || type === "im.vector.web.settings") && this.state.room) { @@ -1705,10 +1700,6 @@ export default class RoomView extends React.Component { // otherwise react calls it with null on each update. private gatherTimelinePanelRef = r => { this.messagePanel = r; - if (r) { - console.log("updateTint from RoomView.gatherTimelinePanelRef"); - this.updateTint(); - } }; private getOldRoom() { diff --git a/src/components/views/rooms/SimpleRoomHeader.js b/src/components/views/rooms/SimpleRoomHeader.js index 2133ccabcd..768a456b35 100644 --- a/src/components/views/rooms/SimpleRoomHeader.js +++ b/src/components/views/rooms/SimpleRoomHeader.js @@ -27,7 +27,7 @@ export default class SimpleRoomHeader extends React.Component { static propTypes = { title: PropTypes.string, - // `src` to a TintableSvg. Optional. + // `src` to an image. Optional. icon: PropTypes.string, }; From f3b4a2181572407cdb345123509572b972bc9494 Mon Sep 17 00:00:00 2001 From: Germain Souquet Date: Wed, 30 Jun 2021 09:02:00 +0100 Subject: [PATCH 19/23] Remove Tinter reference --- src/components/views/messages/MStickerBody.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/components/views/messages/MStickerBody.js b/src/components/views/messages/MStickerBody.js index aca3dba37c..eb3635b0c0 100644 --- a/src/components/views/messages/MStickerBody.js +++ b/src/components/views/messages/MStickerBody.js @@ -42,8 +42,7 @@ export default class MStickerBody extends MImageBody { // Placeholder to show in place of the sticker image if // img onLoad hasn't fired yet. getPlaceholder() { - const TintableSVG = sdk.getComponent('elements.TintableSvg'); - return ; + return ; } // Tooltip to show on mouse over From a9f35e8c69c203b5b8c79ef345b4330faad35d06 Mon Sep 17 00:00:00 2001 From: "J. Ryan Stinnett" Date: Wed, 30 Jun 2021 14:19:39 +0100 Subject: [PATCH 20/23] Lint MXC APIs to centralise access This adds linting rules to ensure that MXC APIs are only accessed via the `Media` helper so they can be customised easily when desired. Fixes https://github.com/vector-im/element-web/issues/16933 --- .eslintrc.js | 11 +++++++++-- src/customisations/Media.ts | 4 ++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index bf6e245b93..381cbd7417 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -54,7 +54,11 @@ module.exports = { "error", ...buildRestrictedPropertiesOptions( ["window.innerHeight", "window.innerWidth", "window.visualViewport"], - "Use UIStore to access window dimensions instead", + "Use UIStore to access window dimensions instead.", + ), + ...buildRestrictedPropertiesOptions( + ["*.mxcUrlToHttp", "*.getHttpUriForMxc"], + "Use Media helper instead to centralise access for customisation.", ), ], }, @@ -63,7 +67,10 @@ module.exports = { function buildRestrictedPropertiesOptions(properties, message) { return properties.map(prop => { - const [object, property] = prop.split("."); + let [object, property] = prop.split("."); + if (object === "*") { + object = undefined; + } return { object, property, diff --git a/src/customisations/Media.ts b/src/customisations/Media.ts index f8ee834102..ce8f88f6f1 100644 --- a/src/customisations/Media.ts +++ b/src/customisations/Media.ts @@ -75,6 +75,7 @@ export class Media { * The HTTP URL for the source media. */ public get srcHttp(): string { + // eslint-disable-next-line no-restricted-properties return this.client.mxcUrlToHttp(this.srcMxc); } @@ -84,6 +85,7 @@ export class Media { */ public get thumbnailHttp(): string | undefined | null { if (!this.hasThumbnail) return null; + // eslint-disable-next-line no-restricted-properties return this.client.mxcUrlToHttp(this.thumbnailMxc); } @@ -100,6 +102,7 @@ export class Media { // scale using the device pixel ratio to keep images clear width = Math.floor(width * window.devicePixelRatio); height = Math.floor(height * window.devicePixelRatio); + // eslint-disable-next-line no-restricted-properties return this.client.mxcUrlToHttp(this.thumbnailMxc, width, height, mode); } @@ -114,6 +117,7 @@ export class Media { // scale using the device pixel ratio to keep images clear width = Math.floor(width * window.devicePixelRatio); height = Math.floor(height * window.devicePixelRatio); + // eslint-disable-next-line no-restricted-properties return this.client.mxcUrlToHttp(this.srcMxc, width, height, mode); } From 2baace7658e0cdbfcb0b0bf48d2157d52d79098a Mon Sep 17 00:00:00 2001 From: "J. Ryan Stinnett" Date: Wed, 30 Jun 2021 14:27:03 +0100 Subject: [PATCH 21/23] Apply restricted property rules to all files --- .eslintrc.js | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 381cbd7417..827b373949 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -24,6 +24,18 @@ module.exports = { // It's disabled here, but we should using it sparingly. "react/jsx-no-bind": "off", "react/jsx-key": ["error"], + + "no-restricted-properties": [ + "error", + ...buildRestrictedPropertiesOptions( + ["window.innerHeight", "window.innerWidth", "window.visualViewport"], + "Use UIStore to access window dimensions instead.", + ), + ...buildRestrictedPropertiesOptions( + ["*.mxcUrlToHttp", "*.getHttpUriForMxc"], + "Use Media helper instead to centralise access for customisation.", + ), + ], }, overrides: [{ files: [ @@ -49,18 +61,6 @@ module.exports = { "@typescript-eslint/no-explicit-any": "off", // We'd rather not do this but we do "@typescript-eslint/ban-ts-comment": "off", - - "no-restricted-properties": [ - "error", - ...buildRestrictedPropertiesOptions( - ["window.innerHeight", "window.innerWidth", "window.visualViewport"], - "Use UIStore to access window dimensions instead.", - ), - ...buildRestrictedPropertiesOptions( - ["*.mxcUrlToHttp", "*.getHttpUriForMxc"], - "Use Media helper instead to centralise access for customisation.", - ), - ], }, }], }; From a5a4f2ed7d0eee484b527f8e19138d1b5c65984f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 30 Jun 2021 13:29:37 -0600 Subject: [PATCH 22/23] Fix linter + merge --- .../views/audio_messages/AudioPlayer.tsx | 2 +- .../views/audio_messages/DurationClock.tsx | 2 +- .../audio_messages/RecordingPlayback.tsx | 6 +- .../views/audio_messages/SeekBar.tsx | 2 +- src/components/views/messages/MAudioBody.js | 112 ------------------ src/components/views/messages/MAudioBody.tsx | 2 +- 6 files changed, 7 insertions(+), 119 deletions(-) delete mode 100644 src/components/views/messages/MAudioBody.js diff --git a/src/components/views/audio_messages/AudioPlayer.tsx b/src/components/views/audio_messages/AudioPlayer.tsx index 2f74c7b069..fa7c10b81c 100644 --- a/src/components/views/audio_messages/AudioPlayer.tsx +++ b/src/components/views/audio_messages/AudioPlayer.tsx @@ -118,6 +118,6 @@ export default class AudioPlayer extends React.PureComponent { /> - + ; } } diff --git a/src/components/views/audio_messages/DurationClock.tsx b/src/components/views/audio_messages/DurationClock.tsx index f6271d1cf4..81852b5944 100644 --- a/src/components/views/audio_messages/DurationClock.tsx +++ b/src/components/views/audio_messages/DurationClock.tsx @@ -46,7 +46,7 @@ export default class DurationClock extends React.PureComponent { } private onTimeUpdate = (time: number[]) => { - this.setState({durationSeconds: time[1]}); + this.setState({ durationSeconds: time[1] }); }; public render() { diff --git a/src/components/views/audio_messages/RecordingPlayback.tsx b/src/components/views/audio_messages/RecordingPlayback.tsx index 7c7a0a87c1..a0dea1c6db 100644 --- a/src/components/views/audio_messages/RecordingPlayback.tsx +++ b/src/components/views/audio_messages/RecordingPlayback.tsx @@ -15,7 +15,7 @@ limitations under the License. */ import { Playback, PlaybackState } from "../../../voice/Playback"; -import React, {ReactNode} from "react"; +import React, { ReactNode } from "react"; import { UPDATE_EVENT } from "../../../stores/AsyncStore"; import PlaybackWaveform from "./PlaybackWaveform"; import PlayPauseButton from "./PlayPauseButton"; @@ -51,7 +51,7 @@ export default class RecordingPlayback extends React.PureComponent { - this.setState({playbackPhase: ev}); + this.setState({ playbackPhase: ev }); }; public render(): ReactNode { @@ -59,6 +59,6 @@ export default class RecordingPlayback extends React.PureComponent - + ; } } diff --git a/src/components/views/audio_messages/SeekBar.tsx b/src/components/views/audio_messages/SeekBar.tsx index ed07c44482..5231a2fb79 100644 --- a/src/components/views/audio_messages/SeekBar.tsx +++ b/src/components/views/audio_messages/SeekBar.tsx @@ -105,7 +105,7 @@ export default class SeekBar extends React.PureComponent { max={1} value={this.state.percentage} step={0.001} - style={{'--fillTo': this.state.percentage} as ISeekCSS} + style={{ '--fillTo': this.state.percentage } as ISeekCSS} disabled={this.props.playbackPhase === PlaybackState.Decoding} />; } diff --git a/src/components/views/messages/MAudioBody.js b/src/components/views/messages/MAudioBody.js deleted file mode 100644 index 977c88448b..0000000000 --- a/src/components/views/messages/MAudioBody.js +++ /dev/null @@ -1,112 +0,0 @@ -/* - Copyright 2016 OpenMarket Ltd - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -import React from 'react'; -import MFileBody from './MFileBody'; - -import { decryptFile } from '../../../utils/DecryptFile'; -import { _t } from '../../../languageHandler'; -import InlineSpinner from '../elements/InlineSpinner'; -import { replaceableComponent } from "../../../utils/replaceableComponent"; -import { mediaFromContent } from "../../../customisations/Media"; - -@replaceableComponent("views.messages.MAudioBody") -export default class MAudioBody extends React.Component { - constructor(props) { - super(props); - this.state = { - playing: false, - decryptedUrl: null, - decryptedBlob: null, - error: null, - }; - } - onPlayToggle() { - this.setState({ - playing: !this.state.playing, - }); - } - - _getContentUrl() { - const media = mediaFromContent(this.props.mxEvent.getContent()); - if (media.isEncrypted) { - return this.state.decryptedUrl; - } else { - return media.srcHttp; - } - } - - componentDidMount() { - const content = this.props.mxEvent.getContent(); - if (content.file !== undefined && this.state.decryptedUrl === null) { - let decryptedBlob; - decryptFile(content.file).then(function(blob) { - decryptedBlob = blob; - return URL.createObjectURL(decryptedBlob); - }).then((url) => { - this.setState({ - decryptedUrl: url, - decryptedBlob: decryptedBlob, - }); - }, (err) => { - console.warn("Unable to decrypt attachment: ", err); - this.setState({ - error: err, - }); - }); - } - } - - componentWillUnmount() { - if (this.state.decryptedUrl) { - URL.revokeObjectURL(this.state.decryptedUrl); - } - } - - render() { - const content = this.props.mxEvent.getContent(); - - if (this.state.error !== null) { - return ( - - - { _t("Error decrypting audio") } - - ); - } - - if (content.file !== undefined && this.state.decryptedUrl === null) { - // Need to decrypt the attachment - // The attachment is decrypted in componentDidMount. - // For now add an img tag with a 16x16 spinner. - // Not sure how tall the audio player is so not sure how tall it should actually be. - return ( - - - - ); - } - - const contentUrl = this._getContentUrl(); - - return ( - - - ); - } -} diff --git a/src/components/views/messages/MAudioBody.tsx b/src/components/views/messages/MAudioBody.tsx index 8791828828..bc7216f42c 100644 --- a/src/components/views/messages/MAudioBody.tsx +++ b/src/components/views/messages/MAudioBody.tsx @@ -105,6 +105,6 @@ export default class MAudioBody extends React.PureComponent { - ) + ); } } From da8a783ca027cc71fd927898f4d680c632b119a3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 30 Jun 2021 14:51:18 -0600 Subject: [PATCH 23/23] lint --- res/css/views/audio_messages/_AudioPlayer.scss | 2 +- src/components/views/audio_messages/AudioPlayer.tsx | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/res/css/views/audio_messages/_AudioPlayer.scss b/res/css/views/audio_messages/_AudioPlayer.scss index 3f02e54aab..9a65ad008f 100644 --- a/res/css/views/audio_messages/_AudioPlayer.scss +++ b/res/css/views/audio_messages/_AudioPlayer.scss @@ -15,7 +15,7 @@ limitations under the License. */ .mx_AudioPlayer_container { - padding: 16px 12px 11px 12px; + padding: 16px 12px 12px 12px; max-width: 267px; // use max to make the control fit in the files/pinned panels .mx_AudioPlayer_primaryContainer { diff --git a/src/components/views/audio_messages/AudioPlayer.tsx b/src/components/views/audio_messages/AudioPlayer.tsx index fa7c10b81c..66efa64658 100644 --- a/src/components/views/audio_messages/AudioPlayer.tsx +++ b/src/components/views/audio_messages/AudioPlayer.tsx @@ -83,7 +83,8 @@ export default class AudioPlayer extends React.PureComponent { const bytes = this.props.playback.sizeBytes; if (!bytes) return null; - // Not translated as these are units, and therefore universal + // Not translated here - we're just presenting the data which should already + // be translated if needed. return `(${formatBytes(bytes)})`; }