Merge branch 'dbkr/call_hold' into dbkr/new_look_callview

This commit is contained in:
David Baker 2020-11-05 14:25:35 +00:00
commit d145b1081c
14 changed files with 292 additions and 291 deletions

View file

@ -229,4 +229,4 @@
@import "./views/verification/_VerificationShowSas.scss";
@import "./views/voip/_CallContainer.scss";
@import "./views/voip/_CallView.scss";
@import "./views/voip/_VideoView.scss";
@import "./views/voip/_VideoFeed.scss";

View file

@ -33,11 +33,11 @@ limitations under the License.
pointer-events: initial; // restore pointer events so the user can leave/interact
cursor: pointer;
.mx_VideoView {
.mx_CallView_video {
width: 350px;
}
.mx_VideoView_localVideoFeed {
.mx_VideoFeed_local {
border-radius: 8px;
overflow: hidden;
}

View file

@ -92,3 +92,10 @@ limitations under the License.
background-color: $primary-fg-color;
}
}
.mx_CallView_video {
width: 100%;
position: relative;
z-index: 30;
}

View file

@ -1,5 +1,5 @@
/*
Copyright 2015, 2016 OpenMarket Ltd
Copyright 2015, 2016, 2020 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -14,23 +14,17 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
.mx_VideoView {
width: 100%;
position: relative;
z-index: 30;
}
.mx_VideoView video {
.mx_VideoFeed video {
width: 100%;
}
.mx_VideoView_remoteVideoFeed {
.mx_VideoFeed_remote {
width: 100%;
background-color: #000;
z-index: 50;
}
.mx_VideoView_localVideoFeed {
.mx_VideoFeed_local {
width: 25%;
height: 25%;
position: absolute;
@ -39,11 +33,11 @@ limitations under the License.
z-index: 100;
}
.mx_VideoView_localVideoFeed video {
.mx_VideoFeed_local video {
width: auto;
height: 100%;
}
.mx_VideoView_localVideoFeed.mx_VideoView_localVideoFeed_flipped video {
.mx_VideoFeed_mirror video {
transform: scale(-1, 1);
}

View file

@ -69,6 +69,13 @@ declare global {
interface Document {
// https://developer.mozilla.org/en-US/docs/Web/API/Document/hasStorageAccess
hasStorageAccess?: () => Promise<boolean>;
// Safari & IE11 only have this prefixed: we used prefixed versions
// previously so let's continue to support them for now
webkitExitFullscreen(): Promise<void>;
msExitFullscreen(): Promise<void>;
readonly webkitFullscreenElement: Element | null;
readonly msFullscreenElement: Element | null;
}
interface Navigator {
@ -99,6 +106,13 @@ declare global {
type?: string;
}
interface Element {
// Safari & IE11 only have this prefixed: we used prefixed versions
// previously so let's continue to support them for now
webkitRequestFullScreen(options?: FullscreenOptions): Promise<void>;
msRequestFullscreen(options?: FullscreenOptions): Promise<void>;
}
interface Error {
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error/fileName
fileName?: string;

View file

@ -59,8 +59,7 @@ import {MatrixClientPeg} from './MatrixClientPeg';
import PlatformPeg from './PlatformPeg';
import Modal from './Modal';
import { _t } from './languageHandler';
// @ts-ignore - XXX: tsc doesn't like this: our js-sdk imports are complex so this isn't surprising
import Matrix from 'matrix-js-sdk';
import Matrix from 'matrix-js-sdk/src/browser-index';
import dis from './dispatcher/dispatcher';
import WidgetUtils from './utils/WidgetUtils';
import WidgetEchoStore from './stores/WidgetEchoStore';
@ -77,7 +76,7 @@ import ErrorDialog from "./components/views/dialogs/ErrorDialog";
import WidgetStore from "./stores/WidgetStore";
import { WidgetMessagingStore } from "./stores/widgets/WidgetMessagingStore";
import { ElementWidgetActions } from "./stores/widgets/ElementWidgetActions";
import { MatrixCall, CallErrorCode, CallState, CallEvent, CallParty, CallType } from "matrix-js-sdk/lib/webrtc/call";
import { MatrixCall, CallErrorCode, CallState, CallEvent, CallParty, CallType } from "matrix-js-sdk/src/webrtc/call";
import Analytics from './Analytics';
import CountlyAnalytics from "./CountlyAnalytics";
@ -98,6 +97,21 @@ export enum PlaceCallType {
ScreenSharing = 'screensharing',
}
function getRemoteAudioElement(): HTMLAudioElement {
// this needs to be somewhere at the top of the DOM which
// always exists to avoid audio interruptions.
// Might as well just use DOM.
const remoteAudioElement = document.getElementById("remoteAudio") as HTMLAudioElement;
if (!remoteAudioElement) {
console.error(
"Failed to find remoteAudio element - cannot play audio!" +
"You need to add an <audio/> to the DOM.",
);
return null;
}
return remoteAudioElement;
}
export default class CallHandler {
private calls = new Map<string, MatrixCall>();
private audioPromises = new Map<AudioID, Promise<void>>();
@ -291,6 +305,11 @@ export default class CallHandler {
});
}
private setCallAudioElement(call: MatrixCall) {
const audioElement = getRemoteAudioElement();
if (audioElement) call.setRemoteAudioElement(audioElement);
}
private setCallState(call: MatrixCall, status: CallState) {
console.log(
`Call state in ${call.roomId} changed to ${status}`,
@ -346,6 +365,8 @@ export default class CallHandler {
const call = Matrix.createNewMatrixCall(MatrixClientPeg.get(), roomId);
this.calls.set(roomId, call);
this.setCallListeners(call);
this.setCallAudioElement(call);
if (type === PlaceCallType.Voice) {
call.placeVoiceCall();
} else if (type === 'video') {
@ -451,6 +472,7 @@ export default class CallHandler {
Analytics.trackEvent('voip', 'receiveCall', 'type', call.type);
this.calls.set(call.roomId, call)
this.setCallListeners(call);
this.setCallAudioElement(call);
}
break;
case 'hangup':

View file

@ -46,6 +46,7 @@ import { EffectiveMembership, getEffectiveMembership, leaveRoomBehaviour } from
import SdkConfig from "./SdkConfig";
import SettingsStore from "./settings/SettingsStore";
import {UIFeature} from "./settings/UIFeature";
import CallHandler from "./CallHandler";
// XXX: workaround for https://github.com/microsoft/TypeScript/issues/31816
interface HTMLInputEvent extends Event {
@ -1042,6 +1043,32 @@ export const Commands = [
},
category: CommandCategories.actions,
}),
new Command({
command: "holdcall",
description: _td("Places the call in the current room on hold"),
category: CommandCategories.other,
runFn: function(roomId, args) {
const call = CallHandler.sharedInstance().getCallForRoom(roomId);
if (!call) {
return reject("No active call in this room");
}
call.setRemoteOnHold(true);
return success();
},
}),
new Command({
command: "unholdcall",
description: _td("Takes the call in the current room off hold"),
category: CommandCategories.other,
runFn: function(roomId, args) {
const call = CallHandler.sharedInstance().getCallForRoom(roomId);
if (!call) {
return reject("No active call in this room");
}
call.setRemoteOnHold(false);
return success();
},
}),
// Command definitions for autocompletion ONLY:
// /me is special because its not handled by SlashCommands.js and is instead done inside the Composer classes

View file

@ -71,7 +71,7 @@ import RoomHeader from "../views/rooms/RoomHeader";
import TintableSvg from "../views/elements/TintableSvg";
import {XOR} from "../../@types/common";
import { IThreepidInvite } from "../../stores/ThreepidInviteStore";
import { CallState, CallType, MatrixCall } from "matrix-js-sdk/lib/webrtc/call";
import { CallState, CallType, MatrixCall } from "matrix-js-sdk/src/webrtc/call";
import WidgetStore from "../../stores/WidgetStore";
import {UPDATE_EVENT} from "../../stores/AsyncStore";
import Notifier from "../../Notifier";

View file

@ -24,7 +24,7 @@ import dis from '../../../dispatcher/dispatcher';
import { ActionPayload } from '../../../dispatcher/payloads';
import PersistentApp from "../elements/PersistentApp";
import SettingsStore from "../../../settings/SettingsStore";
import { CallState, MatrixCall } from 'matrix-js-sdk/lib/webrtc/call';
import { CallState, MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
interface IProps {
}

View file

@ -15,17 +15,18 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
import React, {createRef} from 'react';
import React, { createRef } from 'react';
import Room from 'matrix-js-sdk/src/models/room';
import dis from '../../../dispatcher/dispatcher';
import CallHandler from '../../../CallHandler';
import {MatrixClientPeg} from '../../../MatrixClientPeg';
import { _t } from '../../../languageHandler';
import AccessibleButton from '../elements/AccessibleButton';
import VideoView from "./VideoView";
import VideoFeed, { VideoFeedType } from "./VideoFeed";
import RoomAvatar from "../avatars/RoomAvatar";
import PulsedAvatar from '../avatars/PulsedAvatar';
import { CallState, MatrixCall } from 'matrix-js-sdk/lib/webrtc/call';
import { CallState, CallType, MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
import { CallEvent } from 'matrix-js-sdk/src/webrtc/call';
interface IProps {
// js-sdk room object. If set, we will only show calls for the given
@ -50,53 +51,104 @@ interface IProps {
}
interface IState {
call: any;
call: MatrixCall;
isLocalOnHold: boolean,
}
function getFullScreenElement() {
return (
document.fullscreenElement ||
// moz omitted because firefox supports this unprefixed now (webkit here for safari)
document.webkitFullscreenElement ||
document.msFullscreenElement
);
}
function requestFullscreen(element: Element) {
const method = (
element.requestFullscreen ||
// moz omitted since firefox supports unprefixed now
element.webkitRequestFullScreen ||
element.msRequestFullscreen
);
if (method) method.call(element);
}
function exitFullscreen() {
const exitMethod = (
document.exitFullscreen ||
document.webkitExitFullscreen ||
document.msExitFullscreen
);
if (exitMethod) exitMethod.call(document);
}
export default class CallView extends React.Component<IProps, IState> {
private videoref: React.RefObject<any>;
private dispatcherRef: string;
public call: any;
private container = createRef<HTMLDivElement>();
constructor(props: IProps) {
super(props);
const call = this.getCall();
this.state = {
// the call this view is displaying (if any)
call: null,
};
call,
isLocalOnHold: call ? call.isLocalOnHold() : null,
}
this.videoref = createRef();
this.updateCallListeners(null, call);
}
public componentDidMount() {
this.dispatcherRef = dis.register(this.onAction);
this.showCall();
}
public componentWillUnmount() {
this.updateCallListeners(this.state.call, null);
dis.unregister(this.dispatcherRef);
}
private onAction = (payload) => {
// don't filter out payloads for room IDs other than props.room because
// we may be interested in the conf 1:1 room
if (payload.action !== 'call_state') {
return;
switch (payload.action) {
case 'video_fullscreen': {
if (!this.container.current) {
return;
}
if (payload.fullscreen) {
requestFullscreen(this.container.current);
} else if (getFullScreenElement()) {
exitFullscreen();
}
break;
}
case 'call_state': {
const newCall = this.getCall();
if (newCall !== this.state.call) {
this.updateCallListeners(this.state.call, newCall);
this.setState({
call: newCall,
isLocalOnHold: newCall ? newCall.isLocalOnHold() : null,
});
}
if (!newCall && getFullScreenElement()) {
exitFullscreen();
}
break;
}
}
this.showCall();
};
private showCall() {
private getCall(): MatrixCall {
let call: MatrixCall;
if (this.props.room) {
const roomId = this.props.room.roomId;
call = CallHandler.sharedInstance().getCallForRoom(roomId);
if (this.call) {
this.setState({ call: call });
}
// We don't currently show voice calls in this view when in the room:
// they're represented in the room status bar at the bottom instead
// (but this will all change with the new designs)
if (call && call.type == CallType.Voice) call = null;
} else {
call = CallHandler.sharedInstance().getAnyActiveCall();
// Ignore calls if we can't get the room associated with them.
@ -106,65 +158,68 @@ export default class CallView extends React.Component<IProps, IState> {
if (MatrixClientPeg.get().getRoom(call.roomId) === null) {
call = null;
}
this.setState({ call: call });
}
if (call) {
if (this.getVideoView()) {
call.setLocalVideoElement(this.getVideoView().getLocalVideoElement());
call.setRemoteVideoElement(this.getVideoView().getRemoteVideoElement());
// always use a separate element for audio stream playback.
// this is to let us move CallView around the DOM without interrupting remote audio
// during playback, by having the audio rendered by a top-level <audio/> element.
// rather than being rendered by the main remoteVideo <video/> element.
call.setRemoteAudioElement(this.getVideoView().getRemoteAudioElement());
}
}
if (call && call.type === "video" && call.state !== CallState.Ended && call.state !== CallState.Ringing) {
this.getVideoView().getLocalVideoElement().style.display = "block";
this.getVideoView().getRemoteVideoElement().style.display = "block";
} else {
this.getVideoView().getLocalVideoElement().style.display = "none";
this.getVideoView().getRemoteVideoElement().style.display = "none";
dis.dispatch({action: 'video_fullscreen', fullscreen: false});
}
if (this.props.onResize) {
this.props.onResize();
}
if (call && call.state == CallState.Ended) return null;
return call;
}
private getVideoView() {
return this.videoref.current;
private updateCallListeners(oldCall: MatrixCall, newCall: MatrixCall) {
if (oldCall === newCall) return;
if (oldCall) oldCall.removeListener(CallEvent.HoldUnhold, this.onCallHoldUnhold);
if (newCall) newCall.on(CallEvent.HoldUnhold, this.onCallHoldUnhold);
}
private onCallHoldUnhold = () => {
this.setState({
isLocalOnHold: this.state.call ? this.state.call.isLocalOnHold() : null,
});
};
public render() {
let view: React.ReactNode;
if (this.state.call && this.state.call.type === "voice") {
const client = MatrixClientPeg.get();
const callRoom = client.getRoom(this.state.call.roomId);
view = <AccessibleButton className="mx_CallView_voice" onClick={this.props.onClick}>
<PulsedAvatar>
<RoomAvatar
room={callRoom}
height={35}
width={35}
if (this.state.call) {
if (this.state.call.type === "voice") {
const client = MatrixClientPeg.get();
const callRoom = client.getRoom(this.state.call.roomId);
let caption = _t("Active call");
if (this.state.isLocalOnHold) {
// we currently have no UI for holding / unholding a call (apart from slash
// commands) so we don't disintguish between when we've put the call on hold
// (ie. we'd show an unhold button) and when the other side has put us on hold
// (where obviously we would not show such a button).
caption = _t("Call Paused");
}
view = <AccessibleButton className="mx_CallView_voice" onClick={this.props.onClick}>
<PulsedAvatar>
<RoomAvatar
room={callRoom}
height={35}
width={35}
/>
</PulsedAvatar>
<div>
<h1>{callRoom.name}</h1>
<p>{ caption }</p>
</div>
</AccessibleButton>;
} else {
// For video calls, we currently ignore the call hold state altogether
// (the video will just go black)
// if we're fullscreen, we don't want to set a maxHeight on the video element.
const maxVideoHeight = getFullScreenElement() ? null : this.props.maxVideoHeight;
view = <div className="mx_CallView_video" onClick={this.props.onClick}>
<VideoFeed type={VideoFeedType.Remote} call={this.state.call} onResize={this.props.onResize}
maxHeight={maxVideoHeight}
/>
</PulsedAvatar>
<div>
<h1>{callRoom.name}</h1>
<p>{ _t("Active call") }</p>
</div>
</AccessibleButton>;
} else {
view = <VideoView
ref={this.videoref}
onClick={this.props.onClick}
onResize={this.props.onResize}
maxHeight={this.props.maxVideoHeight}
/>;
<VideoFeed type={VideoFeedType.Local} call={this.state.call} />
</div>;
}
}
let hangup: React.ReactNode;
@ -180,10 +235,9 @@ export default class CallView extends React.Component<IProps, IState> {
/>;
}
return <div className={this.props.className}>
return <div className={this.props.className} ref={this.container}>
{view}
{hangup}
</div>;
}
}

View file

@ -1,58 +0,0 @@
/*
Copyright 2015, 2016 OpenMarket Ltd
Copyright 2019 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React, {createRef} from 'react';
import PropTypes from 'prop-types';
export default class VideoFeed extends React.Component {
static propTypes = {
// maxHeight style attribute for the video element
maxHeight: PropTypes.number,
// a callback which is called when the video element is resized
// due to a change in video metadata
onResize: PropTypes.func,
};
constructor(props) {
super(props);
this._vid = createRef();
}
componentDidMount() {
this._vid.current.addEventListener('resize', this.onResize);
}
componentWillUnmount() {
this._vid.current.removeEventListener('resize', this.onResize);
}
onResize = (e) => {
if (this.props.onResize) {
this.props.onResize(e);
}
};
render() {
return (
<video ref={this._vid} style={{maxHeight: this.props.maxHeight}}>
</video>
);
}
}

View file

@ -0,0 +1,80 @@
/*
Copyright 2015, 2016, 2019 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import classnames from 'classnames';
import { MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
import React, {createRef} from 'react';
import SettingsStore from "../../../settings/SettingsStore";
export enum VideoFeedType {
Local,
Remote,
}
interface IProps {
call: MatrixCall,
type: VideoFeedType,
// maxHeight style attribute for the video element
maxHeight?: number,
// a callback which is called when the video element is resized
// due to a change in video metadata
onResize?: (e: Event) => void,
}
export default class VideoFeed extends React.Component<IProps> {
private vid = createRef<HTMLVideoElement>();
componentDidMount() {
this.vid.current.addEventListener('resize', this.onResize);
if (this.props.type === VideoFeedType.Local) {
this.props.call.setLocalVideoElement(this.vid.current);
} else {
this.props.call.setRemoteVideoElement(this.vid.current);
}
}
componentWillUnmount() {
this.vid.current.removeEventListener('resize', this.onResize);
}
onResize = (e) => {
if (this.props.onResize) {
this.props.onResize(e);
}
};
render() {
const videoClasses = {
mx_VideoFeed: true,
mx_VideoFeed_local: this.props.type === VideoFeedType.Local,
mx_VideoFeed_remote: this.props.type === VideoFeedType.Remote,
mx_VideoFeed_mirror: (
this.props.type === VideoFeedType.Local &&
SettingsStore.getValue('VideoView.flipVideoHorizontally')
),
};
let videoStyle = {};
if (this.props.maxHeight) videoStyle = { maxHeight: this.props.maxHeight };
return <div className={classnames(videoClasses)}>
<video ref={this.vid} style={videoStyle}></video>
</div>;
}
}

View file

@ -1,142 +0,0 @@
/*
Copyright 2015, 2016 OpenMarket Ltd
Copyright 2019 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React, {createRef} from 'react';
import ReactDOM from 'react-dom';
import PropTypes from 'prop-types';
import classNames from 'classnames';
import * as sdk from '../../../index';
import dis from '../../../dispatcher/dispatcher';
import SettingsStore from "../../../settings/SettingsStore";
function getFullScreenElement() {
return (
document.fullscreenElement ||
document.mozFullScreenElement ||
document.webkitFullscreenElement ||
document.msFullscreenElement
);
}
export default class VideoView extends React.Component {
static propTypes = {
// maxHeight style attribute for the video element
maxHeight: PropTypes.number,
// a callback which is called when the user clicks on the video div
onClick: PropTypes.func,
// a callback which is called when the video element is resized due to
// a change in video metadata
onResize: PropTypes.func,
};
constructor(props) {
super(props);
this._local = createRef();
this._remote = createRef();
}
componentDidMount() {
this.dispatcherRef = dis.register(this.onAction);
}
componentWillUnmount() {
dis.unregister(this.dispatcherRef);
}
getRemoteVideoElement = () => {
return ReactDOM.findDOMNode(this._remote.current);
};
getRemoteAudioElement = () => {
// this needs to be somewhere at the top of the DOM which
// always exists to avoid audio interruptions.
// Might as well just use DOM.
const remoteAudioElement = document.getElementById("remoteAudio");
if (!remoteAudioElement) {
console.error("Failed to find remoteAudio element - cannot play audio!"
+ "You need to add an <audio/> to the DOM.");
}
return remoteAudioElement;
};
getLocalVideoElement = () => {
return ReactDOM.findDOMNode(this._local.current);
};
setContainer = (c) => {
this.container = c;
};
onAction = (payload) => {
switch (payload.action) {
case 'video_fullscreen': {
if (!this.container) {
return;
}
const element = this.container;
if (payload.fullscreen) {
const requestMethod = (
element.requestFullScreen ||
element.webkitRequestFullScreen ||
element.mozRequestFullScreen ||
element.msRequestFullscreen
);
requestMethod.call(element);
} else if (getFullScreenElement()) {
const exitMethod = (
document.exitFullscreen ||
document.mozCancelFullScreen ||
document.webkitExitFullscreen ||
document.msExitFullscreen
);
if (exitMethod) {
exitMethod.call(document);
}
}
break;
}
}
};
render() {
const VideoFeed = sdk.getComponent('voip.VideoFeed');
// if we're fullscreen, we don't want to set a maxHeight on the video element.
const maxVideoHeight = getFullScreenElement() ? null : this.props.maxHeight;
const localVideoFeedClasses = classNames("mx_VideoView_localVideoFeed",
{ "mx_VideoView_localVideoFeed_flipped":
SettingsStore.getValue('VideoView.flipVideoHorizontally'),
},
);
return (
<div className="mx_VideoView" ref={this.setContainer} onClick={this.props.onClick}>
<div className="mx_VideoView_remoteVideoFeed">
<VideoFeed ref={this._remote} onResize={this.props.onResize}
maxHeight={maxVideoHeight} />
</div>
<div className={localVideoFeedClasses}>
<VideoFeed ref={this._local} />
</div>
</div>
);
}
}

View file

@ -213,6 +213,8 @@
"Send a bug report with logs": "Send a bug report with logs",
"Opens chat with the given user": "Opens chat with the given user",
"Sends a message to the given user": "Sends a message to the given user",
"Places the call in the current room on hold": "Places the call in the current room on hold",
"Takes the call in the current room off hold": "Takes the call in the current room off hold",
"Displays action": "Displays action",
"Reason": "Reason",
"%(targetName)s accepted the invitation for %(displayName)s.": "%(targetName)s accepted the invitation for %(displayName)s.",
@ -528,6 +530,7 @@
"My Ban List": "My Ban List",
"This is your list of users/servers you have blocked - don't leave the room!": "This is your list of users/servers you have blocked - don't leave the room!",
"Active call": "Active call",
"Call Paused": "Call Paused",
"Unknown caller": "Unknown caller",
"Incoming voice call": "Incoming voice call",
"Incoming video call": "Incoming video call",