diff --git a/res/css/structures/_SpacePanel.scss b/res/css/structures/_SpacePanel.scss index 202eaf0f4d..59f2ea947c 100644 --- a/res/css/structures/_SpacePanel.scss +++ b/res/css/structures/_SpacePanel.scss @@ -35,7 +35,7 @@ $activeBorderColor: $secondary-fg-color; .mx_SpacePanel_spaceTreeWrapper { flex: 1; - overflow-y: scroll; + padding: 8px 8px 16px 0; } .mx_SpacePanel_toggleCollapse { @@ -59,11 +59,10 @@ $activeBorderColor: $secondary-fg-color; margin: 0; list-style: none; padding: 0; - padding-left: 16px; - } - .mx_AutoHideScrollbar { - padding: 8px 0 16px; + > .mx_SpaceItem { + padding-left: 16px; + } } .mx_SpaceButton_toggleCollapse { diff --git a/res/css/structures/_SpaceRoomView.scss b/res/css/structures/_SpaceRoomView.scss index 2e7cfb55d9..2dbf0fe0fe 100644 --- a/res/css/structures/_SpaceRoomView.scss +++ b/res/css/structures/_SpaceRoomView.scss @@ -224,35 +224,6 @@ $SpaceRoomViewInnerWidth: 428px; .mx_FacePile_faces { cursor: pointer; - - > span:hover { - .mx_BaseAvatar { - filter: brightness(0.8); - } - } - - > span:first-child { - position: relative; - - .mx_BaseAvatar { - filter: brightness(0.8); - } - - &::before { - content: ""; - z-index: 1; - position: absolute; - top: 0; - left: 0; - height: 30px; - width: 30px; - background: #ffffff; // white icon fill - mask-position: center; - mask-size: 24px; - mask-repeat: no-repeat; - mask-image: url('$(res)/img/element-icons/room/ellipsis.svg'); - } - } } } diff --git a/res/css/views/elements/_FacePile.scss b/res/css/views/elements/_FacePile.scss index 9a992f59d1..c691baffb5 100644 --- a/res/css/views/elements/_FacePile.scss +++ b/res/css/views/elements/_FacePile.scss @@ -20,7 +20,7 @@ limitations under the License. flex-direction: row-reverse; vertical-align: middle; - > span + span { + > .mx_FacePile_face + .mx_FacePile_face { margin-right: -8px; } @@ -31,9 +31,32 @@ limitations under the License. .mx_BaseAvatar_initial { margin: 1px; // to offset the border on the image } + + .mx_FacePile_more { + position: relative; + border-radius: 100%; + width: 30px; + height: 30px; + background-color: $groupFilterPanel-bg-color; + + &::before { + content: ""; + z-index: 1; + position: absolute; + top: 0; + left: 0; + height: inherit; + width: inherit; + background: $tertiary-fg-color; + mask-position: center; + mask-size: 20px; + mask-repeat: no-repeat; + mask-image: url('$(res)/img/element-icons/room/ellipsis.svg'); + } + } } - > span { + .mx_FacePile_summary { margin-left: 12px; font-size: $font-14px; line-height: $font-24px; diff --git a/res/themes/dark/css/_dark.scss b/res/themes/dark/css/_dark.scss index bd7057c3e4..925d268eb0 100644 --- a/res/themes/dark/css/_dark.scss +++ b/res/themes/dark/css/_dark.scss @@ -85,7 +85,7 @@ $dialog-close-fg-color: #9fa9ba; $dialog-background-bg-color: $header-panel-bg-color; $lightbox-background-bg-color: #000; -$lightbox-background-bg-opacity: 85%; +$lightbox-background-bg-opacity: 0.85; $settings-grey-fg-color: #a2a2a2; $settings-profile-placeholder-bg-color: #21262c; diff --git a/res/themes/legacy-dark/css/_legacy-dark.scss b/res/themes/legacy-dark/css/_legacy-dark.scss index 9b2365a621..28e6e22326 100644 --- a/res/themes/legacy-dark/css/_legacy-dark.scss +++ b/res/themes/legacy-dark/css/_legacy-dark.scss @@ -83,7 +83,7 @@ $dialog-close-fg-color: #9fa9ba; $dialog-background-bg-color: $header-panel-bg-color; $lightbox-background-bg-color: #000; -$lightbox-background-bg-opacity: 85%; +$lightbox-background-bg-opacity: 0.85; $settings-grey-fg-color: #a2a2a2; $settings-profile-placeholder-bg-color: #e7e7e7; diff --git a/res/themes/legacy-light/css/_legacy-light.scss b/res/themes/legacy-light/css/_legacy-light.scss index 0956f433b2..7b6bdad4a4 100644 --- a/res/themes/legacy-light/css/_legacy-light.scss +++ b/res/themes/legacy-light/css/_legacy-light.scss @@ -127,7 +127,7 @@ $dialog-close-fg-color: #c1c1c1; $dialog-background-bg-color: #e9e9e9; $lightbox-background-bg-color: #000; -$lightbox-background-bg-opacity: 95%; +$lightbox-background-bg-opacity: 0.95; $imagebody-giflabel: rgba(0, 0, 0, 0.7); $imagebody-giflabel-border: rgba(0, 0, 0, 0.2); diff --git a/res/themes/light/css/_light.scss b/res/themes/light/css/_light.scss index b307dbaba3..5b46138dae 100644 --- a/res/themes/light/css/_light.scss +++ b/res/themes/light/css/_light.scss @@ -118,7 +118,7 @@ $dialog-close-fg-color: #c1c1c1; $dialog-background-bg-color: #e9e9e9; $lightbox-background-bg-color: #000; -$lightbox-background-bg-opacity: 95%; +$lightbox-background-bg-opacity: 0.95; $imagebody-giflabel: rgba(0, 0, 0, 0.7); $imagebody-giflabel-border: rgba(0, 0, 0, 0.2); diff --git a/src/@types/global.d.ts b/src/@types/global.d.ts index ee0963e537..41257c21f0 100644 --- a/src/@types/global.d.ts +++ b/src/@types/global.d.ts @@ -129,4 +129,30 @@ declare global { // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error/columnNumber columnNumber?: number; } + + // https://github.com/microsoft/TypeScript/issues/28308#issuecomment-650802278 + interface AudioWorkletProcessor { + readonly port: MessagePort; + process( + inputs: Float32Array[][], + outputs: Float32Array[][], + parameters: Record + ): boolean; + } + + // https://github.com/microsoft/TypeScript/issues/28308#issuecomment-650802278 + const AudioWorkletProcessor: { + prototype: AudioWorkletProcessor; + new (options?: AudioWorkletNodeOptions): AudioWorkletProcessor; + }; + + // https://github.com/microsoft/TypeScript/issues/28308#issuecomment-650802278 + function registerProcessor( + name: string, + processorCtor: (new ( + options?: AudioWorkletNodeOptions + ) => AudioWorkletProcessor) & { + parameterDescriptors?: AudioParamDescriptor[]; + } + ); } diff --git a/src/components/views/elements/FacePile.tsx b/src/components/views/elements/FacePile.tsx index e223744352..aeca2e844b 100644 --- a/src/components/views/elements/FacePile.tsx +++ b/src/components/views/elements/FacePile.tsx @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import React, { HTMLAttributes } from "react"; +import React, { HTMLAttributes, ReactNode, useContext } from "react"; import { Room } from "matrix-js-sdk/src/models/room"; import { RoomMember } from "matrix-js-sdk/src/models/room-member"; import { sortBy } from "lodash"; @@ -24,6 +24,7 @@ import { _t } from "../../../languageHandler"; import DMRoomMap from "../../../utils/DMRoomMap"; import TextWithTooltip from "../elements/TextWithTooltip"; import { useRoomMembers } from "../../../hooks/useRoomMembers"; +import MatrixClientContext from "../../../contexts/MatrixClientContext"; const DEFAULT_NUM_FACES = 5; @@ -36,6 +37,7 @@ interface IProps extends HTMLAttributes { const isKnownMember = (member: RoomMember) => !!DMRoomMap.shared().getDMRoomsForUserId(member.userId)?.length; const FacePile = ({ room, onlyKnownUsers = true, numShown = DEFAULT_NUM_FACES, ...props }: IProps) => { + const cli = useContext(MatrixClientContext); let members = useRoomMembers(room); // sort users with an explicit avatar first @@ -46,21 +48,42 @@ const FacePile = ({ room, onlyKnownUsers = true, numShown = DEFAULT_NUM_FACES, . // sort known users first iteratees.unshift(member => isKnownMember(member)); } - if (members.length < 1) return null; - const shownMembers = sortBy(members, iteratees).slice(0, numShown); + // exclude ourselves from the shown members list + const shownMembers = sortBy(members.filter(m => m.userId !== cli.getUserId()), iteratees).slice(0, numShown); + if (shownMembers.length < 1) return null; + + // We reverse the order of the shown faces in CSS to simplify their visual overlap, + // reverse members in tooltip order to make the order between the two match up. + const commaSeparatedMembers = shownMembers.map(m => m.rawDisplayName).reverse().join(", "); + + let tooltip: ReactNode; + if (props.onClick) { + tooltip =
+
+ { _t("View all %(count)s members", { count: members.length }) } +
+
+ { _t("Including %(commaSeparatedMembers)s", { commaSeparatedMembers }) } +
+
; + } else { + tooltip = _t("%(count)s members including %(commaSeparatedMembers)s", { + count: members.length, + commaSeparatedMembers, + }); + } + return
-
- { shownMembers.map(member => { - return - - ; - }) } -
- { onlyKnownUsers && + + { members.length > numShown ? : null } + { shownMembers.map(m => + )} + + { onlyKnownUsers && { _t("%(count)s people you know have already joined", { count: members.length }) } } -
+ ; }; export default FacePile; diff --git a/src/components/views/elements/TextWithTooltip.js b/src/components/views/elements/TextWithTooltip.js index 0bd491768c..a6fc00fc2e 100644 --- a/src/components/views/elements/TextWithTooltip.js +++ b/src/components/views/elements/TextWithTooltip.js @@ -25,6 +25,7 @@ export default class TextWithTooltip extends React.Component { class: PropTypes.string, tooltipClass: PropTypes.string, tooltip: PropTypes.node.isRequired, + tooltipProps: PropTypes.object, }; constructor() { @@ -46,15 +47,17 @@ export default class TextWithTooltip extends React.Component { render() { const Tooltip = sdk.getComponent("elements.Tooltip"); - const {class: className, children, tooltip, tooltipClass, ...props} = this.props; + const {class: className, children, tooltip, tooltipClass, tooltipProps, ...props} = this.props; return ( {children} {this.state.hover && } + className={"mx_TextWithTooltip_tooltip"} + /> } ); } diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index 1210a44958..9b7f0da472 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -53,9 +53,38 @@ export default class VoiceRecordComposerTile extends React.PureComponent Math.round(v * 1024)), + }, }); await VoiceRecordingStore.instance.disposeRecording(); this.setState({recorder: null}); diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index 133d24e3c8..f1b700540f 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -1916,7 +1916,13 @@ "Please create a new issue on GitHub so that we can investigate this bug.": "Please create a new issue on GitHub so that we can investigate this bug.", "collapse": "collapse", "expand": "expand", + "View all %(count)s members|other": "View all %(count)s members", + "View all %(count)s members|one": "View 1 member", + "Including %(commaSeparatedMembers)s": "Including %(commaSeparatedMembers)s", + "%(count)s members including %(commaSeparatedMembers)s|other": "%(count)s members including %(commaSeparatedMembers)s", + "%(count)s members including %(commaSeparatedMembers)s|one": "%(commaSeparatedMembers)s", "%(count)s people you know have already joined|other": "%(count)s people you know have already joined", + "%(count)s people you know have already joined|one": "%(count)s person you know has already joined", "Rotate Right": "Rotate Right", "Rotate Left": "Rotate Left", "Zoom out": "Zoom out", diff --git a/src/stores/SpaceStore.tsx b/src/stores/SpaceStore.tsx index a6ccf314d9..982c3d5d9f 100644 --- a/src/stores/SpaceStore.tsx +++ b/src/stores/SpaceStore.tsx @@ -405,8 +405,7 @@ export class SpaceStoreClass extends AsyncStoreWithClient { this.spaceFilteredRooms.forEach((roomIds, s) => { // Update NotificationStates - const rooms = this.matrixClient.getRooms().filter(room => roomIds.has(room.roomId)); - this.getNotificationState(s)?.setRooms(rooms); + this.getNotificationState(s)?.setRooms(visibleRooms.filter(room => roomIds.has(room.roomId))); }); }, 100, {trailing: true, leading: true}); diff --git a/src/utils/arrays.ts b/src/utils/arrays.ts index 52308937f7..8ab66dfb29 100644 --- a/src/utils/arrays.ts +++ b/src/utils/arrays.ts @@ -54,7 +54,7 @@ export function arraySeed(val: T, length: number): T[] { * @param a The array to clone. Must be defined. * @returns A copy of the array. */ -export function arrayFastClone(a: any[]): any[] { +export function arrayFastClone(a: T[]): T[] { return a.slice(0, a.length); } diff --git a/src/voice/RecorderWorklet.ts b/src/voice/RecorderWorklet.ts new file mode 100644 index 0000000000..7343d37066 --- /dev/null +++ b/src/voice/RecorderWorklet.ts @@ -0,0 +1,67 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {IAmplitudePayload, ITimingPayload, PayloadEvent, WORKLET_NAME} from "./consts"; +import {percentageOf} from "../utils/numbers"; + +// from AudioWorkletGlobalScope: https://developer.mozilla.org/en-US/docs/Web/API/AudioWorkletGlobalScope +declare const currentTime: number; +// declare const currentFrame: number; +// declare const sampleRate: number; + +class MxVoiceWorklet extends AudioWorkletProcessor { + private nextAmplitudeSecond = 0; + + process(inputs, outputs, parameters) { + // We only fire amplitude updates once a second to avoid flooding the recording instance + // with useless data. Much of the data would end up discarded, so we ratelimit ourselves + // here. + const currentSecond = Math.round(currentTime); + if (currentSecond === this.nextAmplitudeSecond) { + // We're expecting exactly one mono input source, so just grab the very first frame of + // samples for the analysis. + const monoChan = inputs[0][0]; + + // The amplitude of the frame's samples is effectively the loudness of the frame. This + // translates into a bar which can be rendered as part of the whole recording clip's + // waveform. + // + // We translate the amplitude down to 0-1 for sanity's sake. + const minVal = Math.min(...monoChan); + const maxVal = Math.max(...monoChan); + const amplitude = percentageOf(maxVal, -1, 1) - percentageOf(minVal, -1, 1); + + this.port.postMessage({ + ev: PayloadEvent.AmplitudeMark, + amplitude: amplitude, + forSecond: currentSecond, + }); + this.nextAmplitudeSecond++; + } + + // We mostly use this worklet to fire regular clock updates through to components + this.port.postMessage({ev: PayloadEvent.Timekeep, timeSeconds: currentTime}); + + // We're supposed to return false when we're "done" with the audio clip, but seeing as + // we are acting as a passive processor we are never truly "done". The browser will clean + // us up when it is done with us. + return true; + } +} + +registerProcessor(WORKLET_NAME, MxVoiceWorklet); + +export default null; // to appease module loaders (we never use the export) diff --git a/src/voice/VoiceRecording.ts b/src/voice/VoiceRecording.ts index 55775ff786..b0cc3cd407 100644 --- a/src/voice/VoiceRecording.ts +++ b/src/voice/VoiceRecording.ts @@ -23,6 +23,8 @@ import {clamp} from "../utils/numbers"; import EventEmitter from "events"; import {IDestroyable} from "../utils/IDestroyable"; import {Singleflight} from "../utils/Singleflight"; +import {PayloadEvent, WORKLET_NAME} from "./consts"; +import {arrayFastClone} from "../utils/arrays"; const CHANNELS = 1; // stereo isn't important const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality. @@ -49,16 +51,34 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { private recorderSource: MediaStreamAudioSourceNode; private recorderStream: MediaStream; private recorderFFT: AnalyserNode; - private recorderProcessor: ScriptProcessorNode; + private recorderWorklet: AudioWorkletNode; private buffer = new Uint8Array(0); private mxc: string; private recording = false; private observable: SimpleObservable; + private amplitudes: number[] = []; // at each second mark, generated public constructor(private client: MatrixClient) { super(); } + public get finalWaveform(): number[] { + return arrayFastClone(this.amplitudes); + } + + public get contentType(): string { + return "audio/ogg"; + } + + public get contentLength(): number { + return this.buffer.length; + } + + public get durationSeconds(): number { + if (!this.recorder) throw new Error("Duration not available without a recording"); + return this.recorderContext.currentTime; + } + private async makeRecorder() { this.recorderStream = await navigator.mediaDevices.getUserMedia({ audio: { @@ -80,18 +100,34 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { // it makes the time domain less than helpful. this.recorderFFT.fftSize = 64; - // We use an audio processor to get accurate timing information. - // The size of the audio buffer largely decides how quickly we push timing/waveform data - // out of this class. Smaller buffers mean we update more frequently as we can't hold as - // many bytes. Larger buffers mean slower updates. For scale, 1024 gives us about 30Hz of - // updates and 2048 gives us about 20Hz. We use 1024 to get as close to perceived realtime - // as possible. Must be a power of 2. - this.recorderProcessor = this.recorderContext.createScriptProcessor(1024, CHANNELS, CHANNELS); + // Set up our worklet. We use this for timing information and waveform analysis: the + // web audio API prefers this be done async to avoid holding the main thread with math. + const mxRecorderWorkletPath = document.body.dataset.vectorRecorderWorkletScript; + if (!mxRecorderWorkletPath) { + throw new Error("Unable to create recorder: no worklet script registered"); + } + await this.recorderContext.audioWorklet.addModule(mxRecorderWorkletPath); + this.recorderWorklet = new AudioWorkletNode(this.recorderContext, WORKLET_NAME); // Connect our inputs and outputs this.recorderSource.connect(this.recorderFFT); - this.recorderSource.connect(this.recorderProcessor); - this.recorderProcessor.connect(this.recorderContext.destination); + this.recorderSource.connect(this.recorderWorklet); + this.recorderWorklet.connect(this.recorderContext.destination); + + // Dev note: we can't use `addEventListener` for some reason. It just doesn't work. + this.recorderWorklet.port.onmessage = (ev) => { + switch (ev.data['ev']) { + case PayloadEvent.Timekeep: + this.processAudioUpdate(ev.data['timeSeconds']); + break; + case PayloadEvent.AmplitudeMark: + // Sanity check to make sure we're adding about one sample per second + if (ev.data['forSecond'] === this.amplitudes.length) { + this.amplitudes.push(ev.data['amplitude']); + } + break; + } + }; this.recorder = new Recorder({ encoderPath, // magic from webpack @@ -138,7 +174,7 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { return this.mxc; } - private processAudioUpdate = (ev: AudioProcessingEvent) => { + private processAudioUpdate = (timeSeconds: number) => { if (!this.recording) return; // The time domain is the input to the FFT, which means we use an array of the same @@ -162,12 +198,12 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { this.observable.update({ waveform: translatedData, - timeSeconds: ev.playbackTime, + timeSeconds: timeSeconds, }); // Now that we've updated the data/waveform, let's do a time check. We don't want to // go horribly over the limit. We also emit a warning state if needed. - const secondsLeft = TARGET_MAX_LENGTH - ev.playbackTime; + const secondsLeft = TARGET_MAX_LENGTH - timeSeconds; if (secondsLeft <= 0) { // noinspection JSIgnoredPromiseFromCall - we aren't concerned with it overlapping this.stop(); @@ -191,7 +227,6 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { } this.observable = new SimpleObservable(); await this.makeRecorder(); - this.recorderProcessor.addEventListener("audioprocess", this.processAudioUpdate); await this.recorder.start(); this.recording = true; this.emit(RecordingState.Started); @@ -205,6 +240,7 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { // Disconnect the source early to start shutting down resources this.recorderSource.disconnect(); + this.recorderWorklet.disconnect(); await this.recorder.stop(); // close the context after the recorder so the recorder doesn't try to @@ -216,7 +252,6 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { // Finally do our post-processing and clean up this.recording = false; - this.recorderProcessor.removeEventListener("audioprocess", this.processAudioUpdate); await this.recorder.close(); this.emit(RecordingState.Ended); @@ -240,7 +275,7 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { this.emit(RecordingState.Uploading); this.mxc = await this.client.uploadContent(new Blob([this.buffer], { - type: "audio/ogg", + type: this.contentType, }), { onlyContentUri: false, // to stop the warnings in the console }).then(r => r['content_uri']); diff --git a/src/voice/consts.ts b/src/voice/consts.ts new file mode 100644 index 0000000000..c530c60f0b --- /dev/null +++ b/src/voice/consts.ts @@ -0,0 +1,37 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export const WORKLET_NAME = "mx-voice-worklet"; + +export enum PayloadEvent { + Timekeep = "timekeep", + AmplitudeMark = "amplitude_mark", +} + +export interface IPayload { + ev: PayloadEvent; +} + +export interface ITimingPayload extends IPayload { + ev: PayloadEvent.Timekeep; + timeSeconds: number; +} + +export interface IAmplitudePayload extends IPayload { + ev: PayloadEvent.AmplitudeMark; + forSecond: number; + amplitude: number; +}