Make CallView use CallFeed

Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com>

res/css/views/voip/_CallView.scss

@@ -112,6 +112,7 @@ limitations under the License.
     z-index: 30;
     border-radius: 8px;
     overflow: hidden;
+    display: flex;
 }
 
 .mx_CallView_video_hold {

res/css/views/voip/_VideoFeed.scss

@@ -14,11 +14,21 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-.mx_VideoFeed_remote {
-    width: 100%;
-    height: 100%;
+.mx_VideoFeed_voice {
+    // We don't want to collide with the call controls that have 52px of height
+    padding-bottom: 52px;
+    background-color: $inverted-bg-color;
+}
+
+.mx_VideoFeed_video {
     background-color: #000;
-    z-index: 50;
+}
+
+.mx_VideoFeed_remote {
+    flex: 1;
+    display: flex;
+    justify-content: center;
+    align-items: center;
 }
 
 .mx_VideoFeed_local {

src/CallHandler.tsx

@@ -621,7 +621,6 @@ export default class CallHandler {
 
     private async placeCall(
         roomId: string, type: PlaceCallType,
-        localElement: HTMLVideoElement, remoteElement: HTMLVideoElement,
     ) {
         Analytics.trackEvent('voip', 'placeCall', 'type', type);
         CountlyAnalytics.instance.trackStartCall(roomId, type === PlaceCallType.Video, false);
@@ -643,10 +642,7 @@ export default class CallHandler {
         if (type === PlaceCallType.Voice) {
             call.placeVoiceCall();
         } else if (type === 'video') {
-            call.placeVideoCall(
-                remoteElement,
-                localElement,
-            );
+            call.placeVideoCall();
         } else if (type === PlaceCallType.ScreenSharing) {
             const screenCapErrorString = PlatformPeg.get().screenCaptureErrorString();
             if (screenCapErrorString) {
@@ -660,8 +656,6 @@ export default class CallHandler {
             }
 
             call.placeScreenSharingCall(
-                remoteElement,
-                localElement,
                 async () : Promise<DesktopCapturerSource> => {
                     const {finished} = Modal.createDialog(DesktopCapturerSourcePicker);
                     const [source] = await finished;
@@ -715,14 +709,12 @@ export default class CallHandler {
                     } else if (members.length === 2) {
                         console.info(`Place ${payload.type} call in ${payload.room_id}`);
 
-                        this.placeCall(payload.room_id, payload.type, payload.local_element, payload.remote_element);
+                        this.placeCall(payload.room_id, payload.type);
                     } else { // > 2
                         dis.dispatch({
                             action: "place_conference_call",
                             room_id: payload.room_id,
                             type: payload.type,
-                            remote_element: payload.remote_element,
-                            local_element: payload.local_element,
                         });
                     }
                 }

src/components/views/voip/CallView.tsx

@@ -20,7 +20,7 @@ import dis from '../../../dispatcher/dispatcher';
 import CallHandler from '../../../CallHandler';
 import {MatrixClientPeg} from '../../../MatrixClientPeg';
 import { _t, _td } from '../../../languageHandler';
-import VideoFeed, { VideoFeedType } from "./VideoFeed";
+import VideoFeed from './VideoFeed';
 import RoomAvatar from "../avatars/RoomAvatar";
 import { CallState, CallType, MatrixCall, CallEvent } from 'matrix-js-sdk/src/webrtc/call';
 import classNames from 'classnames';
@@ -30,6 +30,7 @@ import {alwaysAboveLeftOf, alwaysAboveRightOf, ChevronFace, ContextMenuButton} f
 import CallContextMenu from '../context_menus/CallContextMenu';
 import { avatarUrlForMember } from '../../../Avatar';
 import DialpadContextMenu from '../context_menus/DialpadContextMenu';
+import { CallFeed } from 'matrix-js-sdk/src/webrtc/callFeed';
 
 interface IProps {
         // The call for us to display
@@ -58,6 +59,7 @@ interface IState {
     controlsVisible: boolean,
     showMoreMenu: boolean,
     showDialpad: boolean,
+    feeds: CallFeed[],
 }
 
 function getFullScreenElement() {
@@ -112,6 +114,7 @@ export default class CallView extends React.Component<IProps, IState> {
             controlsVisible: true,
             showMoreMenu: false,
             showDialpad: false,
+            feeds: this.props.call.getFeeds(),
         }
 
         this.updateCallListeners(null, this.props.call);
@@ -169,11 +172,13 @@ export default class CallView extends React.Component<IProps, IState> {
             oldCall.removeListener(CallEvent.State, this.onCallState);
             oldCall.removeListener(CallEvent.LocalHoldUnhold, this.onCallLocalHoldUnhold);
             oldCall.removeListener(CallEvent.RemoteHoldUnhold, this.onCallRemoteHoldUnhold);
+            oldCall.removeListener(CallEvent.FeedsChanged, this.onFeedsChanged);
         }
         if (newCall) {
             newCall.on(CallEvent.State, this.onCallState);
             newCall.on(CallEvent.LocalHoldUnhold, this.onCallLocalHoldUnhold);
             newCall.on(CallEvent.RemoteHoldUnhold, this.onCallRemoteHoldUnhold);
+            newCall.on(CallEvent.FeedsChanged, this.onFeedsChanged);
         }
     }
 
@@ -183,6 +188,10 @@ export default class CallView extends React.Component<IProps, IState> {
         });
     };
 
+    private onFeedsChanged = (newFeeds: Array<CallFeed>) => {
+        this.setState({feeds: newFeeds});
+    }
+
     private onCallLocalHoldUnhold = () => {
         this.setState({
             isLocalOnHold: this.props.call.isLocalOnHold(),
@@ -486,44 +495,71 @@ export default class CallView extends React.Component<IProps, IState> {
             });
         }
 
+        const avatarSize = this.props.pipMode ? 76 : 160;
+        if (isOnHold) {
             if (this.props.call.type === CallType.Video) {
-            let localVideoFeed = null;
-            let onHoldContent = null;
-            let onHoldBackground = null;
-            const backgroundStyle: CSSProperties = {};
                 const containerClasses = classNames({
                     mx_CallView_video: true,
                     mx_CallView_video_hold: isOnHold,
                 });
-            if (isOnHold) {
-                onHoldContent = <div className="mx_CallView_video_holdContent">
+                let onHoldContent = null;
+                let onHoldBackground = null;
+                const backgroundStyle: CSSProperties = {};
+                onHoldContent = (
+                    <div className="mx_CallView_video_holdContent">
                         {onHoldText}
-                </div>;
+                    </div>
+                );
                 const backgroundAvatarUrl = avatarUrlForMember(
                 // is it worth getting the size of the div to pass here?
                     this.props.call.getOpponentMember(), 1024, 1024, 'crop',
                 );
                 backgroundStyle.backgroundImage = 'url(' + backgroundAvatarUrl + ')';
                 onHoldBackground = <div className="mx_CallView_video_holdBackground" style={backgroundStyle} />;
-            }
-            if (!this.state.vidMuted) {
-                localVideoFeed = <VideoFeed type={VideoFeedType.Local} call={this.props.call} />;
-            }
 
-            contentView = <div className={containerClasses} ref={this.contentRef} onMouseMove={this.onMouseMove}>
+                contentView = (
+                    <div className={containerClasses} ref={this.contentRef} onMouseMove={this.onMouseMove}>
                         {onHoldBackground}
-                <VideoFeed type={VideoFeedType.Remote} call={this.props.call} onResize={this.props.onResize} />
-                {localVideoFeed}
                         {onHoldContent}
                         {callControls}
-            </div>;
+                    </div>
+                );
             } else {
-            const avatarSize = this.props.pipMode ? 76 : 160;
                 const classes = classNames({
                     mx_CallView_voice: true,
                     mx_CallView_voice_hold: isOnHold,
                 });
 
+                contentView =(
+                    <div className={classes} onMouseMove={this.onMouseMove}>
+                        <div className="mx_CallView_voice_avatarsContainer">
+                            <div
+                                className="mx_CallView_voice_avatarContainer"
+                                style={{width: avatarSize, height: avatarSize}}
+                            >
+                                <RoomAvatar
+                                    room={callRoom}
+                                    height={avatarSize}
+                                    width={avatarSize}
+                                />
+                            </div>
+                        </div>
+                        <div className="mx_CallView_voice_holdText">{onHoldText}</div>
+                        {callControls}
+                    </div>
+                );
+            }
+        } else if (this.props.call.noIncomingFeeds()) {
+            // Here we're reusing the css classes from voice on hold, because
+            // I am lazy. If this gets merged, the CallView might be subject
+            // to change anyway - I might take an axe to this file in order to
+            // try to get other things working
+            const classes = classNames({
+                mx_CallView_voice: true,
+            });
+
+            // Saying "Connecting" here isn't really true, but the best thing
+            // I can come up with, but this might be subject to change as well
             contentView = <div className={classes} onMouseMove={this.onMouseMove}>
                 <div className="mx_CallView_voice_avatarsContainer">
                     <div className="mx_CallView_voice_avatarContainer" style={{width: avatarSize, height: avatarSize}}>
@@ -534,7 +570,33 @@ export default class CallView extends React.Component<IProps, IState> {
                         />
                     </div>
                 </div>
-                <div className="mx_CallView_voice_holdText">{onHoldText}</div>
+                <div className="mx_CallView_voice_holdText">{_t("Connecting")}</div>
+                {callControls}
+            </div>;
+        } else {
+            const containerClasses = classNames({
+                mx_CallView_video: true,
+            });
+
+            // TODO: Later the CallView should probably be reworked to support any
+            // number of feeds but now we can always expect there to be two feeds
+            const feeds = this.state.feeds.map((feed, i) => {
+                // Here we check to hide local audio feeds to achieve the same UI/UX
+                // as before. But once again this might be subject to change
+                if (feed.isAudioOnly() && feed.isLocal()) return;
+                return (
+                    <VideoFeed
+                        key={i}
+                        feed={feed}
+                        call={this.props.call}
+                        pipMode={this.props.pipMode}
+                        onResize={this.props.onResize}
+                    />
+                );
+            });
+
+            contentView = <div className={containerClasses} ref={this.contentRef} onMouseMove={this.onMouseMove}>
+                {feeds}
                 {callControls}
             </div>;
         }

src/components/views/voip/VideoFeed.tsx

@@ -18,50 +18,81 @@ import classnames from 'classnames';
 import { MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
 import React, {createRef} from 'react';
 import SettingsStore from "../../../settings/SettingsStore";
-
-export enum VideoFeedType {
-    Local,
-    Remote,
-}
+import { CallFeed, CallFeedEvent } from 'matrix-js-sdk/src/webrtc/callFeed';
+import { MatrixClientPeg } from '../../../MatrixClientPeg';
+import { logger } from 'matrix-js-sdk/src/logger';
+import MemberAvatar from "../avatars/MemberAvatar"
+import CallHandler from '../../../CallHandler';
 
 interface IProps {
     call: MatrixCall,
 
-    type: VideoFeedType,
+    feed: CallFeed,
+
+    // Whether this call view is for picture-in-pictue mode
+    // otherwise, it's the larger call view when viewing the room the call is in.
+    // This is sort of a proxy for a number of things but we currently have no
+    // need to control those things separately, so this is simpler.
+    pipMode?: boolean;
 
     // a callback which is called when the video element is resized
     // due to a change in video metadata
     onResize?: (e: Event) => void,
 }
 
-export default class VideoFeed extends React.Component<IProps> {
-    private vid = createRef<HTMLVideoElement>();
-
-    componentDidMount() {
-        this.vid.current.addEventListener('resize', this.onResize);
-        this.setVideoElement();
+interface IState {
+    audioOnly: boolean;
 }
 
-    componentDidUpdate(prevProps) {
-        if (this.props.call !== prevProps.call) {
-            this.setVideoElement();
+export default class VideoFeed extends React.Component<IProps, IState> {
+    private vid = createRef<HTMLVideoElement>();
+
+    constructor(props: IProps) {
+        super(props);
+
+        this.state = {
+            audioOnly: this.props.feed.isAudioOnly(),
+        };
+    }
+
+    componentDidMount() {
+        this.props.feed.addListener(CallFeedEvent.NewStream, this.onNewStream);
+        if (!this.vid.current) return;
+        // A note on calling methods on media elements:
+        // We used to have queues per media element to serialise all calls on those elements.
+        // The reason given for this was that load() and play() were racing. However, we now
+        // never call load() explicitly so this seems unnecessary. However, serialising every
+        // operation was causing bugs where video would not resume because some play command
+        // had got stuck and all media operations were queued up behind it. If necessary, we
+        // should serialise the ones that need to be serialised but then be able to interrupt
+        // them with another load() which will cancel the pending one, but since we don't call
+        // load() explicitly, it shouldn't be a problem. - Dave
+        this.vid.current.srcObject = this.props.feed.stream;
+        this.vid.current.autoplay = true;
+        this.vid.current.muted = true;
+        try {
+            this.vid.current.play();
+        } catch (e) {
+            logger.info("Failed to play video element with feed", this.props.feed, e);
         }
     }
 
     componentWillUnmount() {
+        this.props.feed.removeListener(CallFeedEvent.NewStream, this.onNewStream);
+        if (!this.vid.current) return;
         this.vid.current.removeEventListener('resize', this.onResize);
+        this.vid.current.pause();
+        this.vid.current.srcObject = null;
     }
 
-    private setVideoElement() {
-        if (this.props.type === VideoFeedType.Local) {
-            this.props.call.setLocalVideoElement(this.vid.current);
-        } else {
-            this.props.call.setRemoteVideoElement(this.vid.current);
-        }
+    onNewStream = (newStream: MediaStream) => {
+        this.setState({ audioOnly: this.props.feed.isAudioOnly()});
+        if (!this.vid.current) return;
+        this.vid.current.srcObject = newStream;
     }
 
     onResize = (e) => {
-        if (this.props.onResize) {
+        if (this.props.onResize && !this.props.feed.isLocal()) {
             this.props.onResize(e);
         }
     };
@@ -69,14 +100,35 @@ export default class VideoFeed extends React.Component<IProps> {
     render() {
         const videoClasses = {
             mx_VideoFeed: true,
-            mx_VideoFeed_local: this.props.type === VideoFeedType.Local,
-            mx_VideoFeed_remote: this.props.type === VideoFeedType.Remote,
+            mx_VideoFeed_local: this.props.feed.isLocal(),
+            mx_VideoFeed_remote: !this.props.feed.isLocal(),
+            mx_VideoFeed_voice: this.state.audioOnly,
+            mx_VideoFeed_video: !this.state.audioOnly,
             mx_VideoFeed_mirror: (
-                this.props.type === VideoFeedType.Local &&
+                this.props.feed.isLocal() &&
                 SettingsStore.getValue('VideoView.flipVideoHorizontally')
             ),
         };
 
-        return <video className={classnames(videoClasses)} ref={this.vid} />;
+        if (this.state.audioOnly) {
+            const callRoomId = CallHandler.roomIdForCall(this.props.call);
+            const callRoom = MatrixClientPeg.get().getRoom(callRoomId);
+            const member = callRoom.getMember(this.props.feed.userId);
+            const avatarSize = this.props.pipMode ? 76 : 160;
+
+            return (
+                <div className={classnames(videoClasses)} >
+                    <MemberAvatar
+                        member={member}
+                        height={avatarSize}
+                        width={avatarSize}
+                    />
+                </div>
+            );
+        } else {
+            return (
+                <video className={classnames(videoClasses)} ref={this.vid} />
+            );
+        }
     }
 }