From c6a058fb6fa0af190c117c2f3b5bc01a6eab17e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 19 Dec 2020 19:32:58 +0100 Subject: [PATCH 001/572] Added surround with MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- .../views/rooms/BasicMessageComposer.tsx | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/components/views/rooms/BasicMessageComposer.tsx b/src/components/views/rooms/BasicMessageComposer.tsx index 2ececdeaed..cd34e25926 100644 --- a/src/components/views/rooms/BasicMessageComposer.tsx +++ b/src/components/views/rooms/BasicMessageComposer.tsx @@ -418,6 +418,10 @@ export default class BasicMessageEditor extends React.Component }; private onKeyDown = (event: React.KeyboardEvent) => { + const selectionRange = getRangeForSelection(this.editorRef.current, this.props.model, document.getSelection()); + // trim the range as we want it to exclude leading/trailing spaces + selectionRange.trim(); + const model = this.props.model; const modKey = IS_MAC ? event.metaKey : event.ctrlKey; let handled = false; @@ -471,6 +475,43 @@ export default class BasicMessageEditor extends React.Component }); handled = true; // autocomplete or enter to send below shouldn't have any modifier keys pressed. + } else if (document.getSelection().type != "Caret") { + if (event.key === '(') { + this.historyManager.ensureLastChangesPushed(this.props.model); + this.modifiedFlag = true; + toggleInlineFormat(selectionRange, "(", ")"); + handled = true; + } else if (event.key === '[') { + this.historyManager.ensureLastChangesPushed(this.props.model); + this.modifiedFlag = true; + toggleInlineFormat(selectionRange, "[", "]"); + handled = true; + } else if (event.key === '{') { + this.historyManager.ensureLastChangesPushed(this.props.model); + this.modifiedFlag = true; + toggleInlineFormat(selectionRange, "{", "}"); + handled = true; + } else if (event.key === '<') { + this.historyManager.ensureLastChangesPushed(this.props.model); + this.modifiedFlag = true; + toggleInlineFormat(selectionRange, "<", ">"); + handled = true; + } else if (event.key === '"') { + this.historyManager.ensureLastChangesPushed(this.props.model); + this.modifiedFlag = true; + toggleInlineFormat(selectionRange, "\""); + handled = true; + } else if (event.key === '`') { + this.historyManager.ensureLastChangesPushed(this.props.model); + this.modifiedFlag = true; + toggleInlineFormat(selectionRange, "`"); + handled = true; + } else if (event.key === '\'') { + this.historyManager.ensureLastChangesPushed(this.props.model); + this.modifiedFlag = true; + toggleInlineFormat(selectionRange, "'"); + handled = true; + } } else { const metaOrAltPressed = event.metaKey || event.altKey; const modifierPressed = metaOrAltPressed || event.shiftKey; From e90f5ddf5b6ac14648d7fc36ecf414a04d668c4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 19 Dec 2020 19:36:56 +0100 Subject: [PATCH 002/572] Added a comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/components/views/rooms/BasicMessageComposer.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/components/views/rooms/BasicMessageComposer.tsx b/src/components/views/rooms/BasicMessageComposer.tsx index cd34e25926..587f13e8c2 100644 --- a/src/components/views/rooms/BasicMessageComposer.tsx +++ b/src/components/views/rooms/BasicMessageComposer.tsx @@ -512,6 +512,7 @@ export default class BasicMessageEditor extends React.Component toggleInlineFormat(selectionRange, "'"); handled = true; } + // Surround selected text with a character } else { const metaOrAltPressed = event.metaKey || event.altKey; const modifierPressed = metaOrAltPressed || event.shiftKey; From b330dd55a0cd61fe9b014d47c6dcfb085e835b93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 12 Feb 2021 07:53:09 +0100 Subject: [PATCH 003/572] Hide surround with behind a setting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/components/views/rooms/BasicMessageComposer.tsx | 3 ++- .../views/settings/tabs/user/PreferencesUserSettingsTab.js | 1 + src/settings/Settings.ts | 5 +++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/components/views/rooms/BasicMessageComposer.tsx b/src/components/views/rooms/BasicMessageComposer.tsx index 587f13e8c2..a91e92123b 100644 --- a/src/components/views/rooms/BasicMessageComposer.tsx +++ b/src/components/views/rooms/BasicMessageComposer.tsx @@ -418,6 +418,7 @@ export default class BasicMessageEditor extends React.Component }; private onKeyDown = (event: React.KeyboardEvent) => { + const surroundWith = SettingsStore.getValue("MessageComposerInput.surroundWith"); const selectionRange = getRangeForSelection(this.editorRef.current, this.props.model, document.getSelection()); // trim the range as we want it to exclude leading/trailing spaces selectionRange.trim(); @@ -475,7 +476,7 @@ export default class BasicMessageEditor extends React.Component }); handled = true; // autocomplete or enter to send below shouldn't have any modifier keys pressed. - } else if (document.getSelection().type != "Caret") { + } else if (surroundWith && document.getSelection().type != "Caret") { if (event.key === '(') { this.historyManager.ensureLastChangesPushed(this.props.model); this.modifiedFlag = true; diff --git a/src/components/views/settings/tabs/user/PreferencesUserSettingsTab.js b/src/components/views/settings/tabs/user/PreferencesUserSettingsTab.js index 4d8493401e..2544c03a22 100644 --- a/src/components/views/settings/tabs/user/PreferencesUserSettingsTab.js +++ b/src/components/views/settings/tabs/user/PreferencesUserSettingsTab.js @@ -34,6 +34,7 @@ export default class PreferencesUserSettingsTab extends React.Component { 'MessageComposerInput.suggestEmoji', 'sendTypingNotifications', 'MessageComposerInput.ctrlEnterToSend', + 'MessageComposerInput.surroundWith', ]; static TIMELINE_SETTINGS = [ diff --git a/src/settings/Settings.ts b/src/settings/Settings.ts index b239b809fe..ed9b37d632 100644 --- a/src/settings/Settings.ts +++ b/src/settings/Settings.ts @@ -336,6 +336,11 @@ export const SETTINGS: {[setting: string]: ISetting} = { displayName: isMac ? _td("Use Command + Enter to send a message") : _td("Use Ctrl + Enter to send a message"), default: false, }, + "MessageComposerInput.surroundWith": { + supportedLevels: LEVELS_ACCOUNT_SETTINGS, + displayName: _td("Use surround with"), + default: false, + }, "MessageComposerInput.autoReplaceEmoji": { supportedLevels: LEVELS_ACCOUNT_SETTINGS, displayName: _td('Automatically replace plain text Emoji'), From 3f0d7673725f12b99e48b0f2e94c9c0f78f9c5d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 12 Feb 2021 07:57:15 +0100 Subject: [PATCH 004/572] i18n MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/i18n/strings/en_EN.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index a9d31bb9f2..3af2a62c94 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -816,6 +816,7 @@ "Use Ctrl + F to search": "Use Ctrl + F to search", "Use Command + Enter to send a message": "Use Command + Enter to send a message", "Use Ctrl + Enter to send a message": "Use Ctrl + Enter to send a message", + "Use surround with": "Use surround with", "Automatically replace plain text Emoji": "Automatically replace plain text Emoji", "Mirror local video feed": "Mirror local video feed", "Enable Community Filter Panel": "Enable Community Filter Panel", From 4a6d8ebdf0d7143c6e41a09aa62dfbcf2620b11b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 7 May 2021 20:29:26 +0200 Subject: [PATCH 005/572] Add screensharing icons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- res/img/voip/screensharing-off.svg | 17 +++++++++++++++++ res/img/voip/screensharing-on.svg | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 res/img/voip/screensharing-off.svg create mode 100644 res/img/voip/screensharing-on.svg diff --git a/res/img/voip/screensharing-off.svg b/res/img/voip/screensharing-off.svg new file mode 100644 index 0000000000..c05ccdf9aa --- /dev/null +++ b/res/img/voip/screensharing-off.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/res/img/voip/screensharing-on.svg b/res/img/voip/screensharing-on.svg new file mode 100644 index 0000000000..1436ca7e37 --- /dev/null +++ b/res/img/voip/screensharing-on.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + From 198722eb41ac39bde68c41a63567dcf0c30ef33a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 7 May 2021 20:29:45 +0200 Subject: [PATCH 006/572] Add classes for screensharing buttons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- res/css/views/voip/_CallView.scss | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/res/css/views/voip/_CallView.scss b/res/css/views/voip/_CallView.scss index 0be75be28c..8ebe7aa607 100644 --- a/res/css/views/voip/_CallView.scss +++ b/res/css/views/voip/_CallView.scss @@ -353,6 +353,18 @@ limitations under the License. } } +.mx_CallView_callControls_button_screensharingOn { + &::before { + background-image: url('$(res)/img/voip/screensharing-on.svg'); + } +} + +.mx_CallView_callControls_button_screensharingOff { + &::before { + background-image: url('$(res)/img/voip/screensharing-off.svg'); + } +} + .mx_CallView_callControls_button_hangup { &::before { background-image: url('$(res)/img/voip/hangup.svg'); From 1f27354439a1572040bcf7c863b1a7298edf4d59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 7 May 2021 21:34:56 +0200 Subject: [PATCH 007/572] Add support for up to 4 feeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- res/css/views/voip/_VideoFeed.scss | 32 +++++++++++- src/components/views/voip/CallView.tsx | 69 ++++++++++++++----------- src/components/views/voip/VideoFeed.tsx | 5 +- 3 files changed, 71 insertions(+), 35 deletions(-) diff --git a/res/css/views/voip/_VideoFeed.scss b/res/css/views/voip/_VideoFeed.scss index 7d85ac264e..170bd89652 100644 --- a/res/css/views/voip/_VideoFeed.scss +++ b/res/css/views/voip/_VideoFeed.scss @@ -21,7 +21,7 @@ limitations under the License. } -.mx_VideoFeed_remote { +.mx_VideoFeed_primary { width: 100%; height: 100%; display: flex; @@ -33,7 +33,7 @@ limitations under the License. } } -.mx_VideoFeed_local { +.mx_VideoFeed_secondary { max-width: 25%; max-height: 25%; position: absolute; @@ -47,6 +47,34 @@ limitations under the License. } } +.mx_VideoFeed_tertiary { + max-width: 25%; + max-height: 25%; + position: absolute; + right: 10px; + bottom: 10px; + z-index: 100; + border-radius: 4px; + + &.mx_VideoFeed_video { + background-color: transparent; + } +} + +.mx_VideoFeed_quaternary { + max-width: 25%; + max-height: 25%; + position: absolute; + left: 10px; + top: 10px; + z-index: 100; + border-radius: 4px; + + &.mx_VideoFeed_video { + background-color: transparent; + } +} + .mx_VideoFeed_mirror { transform: scale(-1, 1); } diff --git a/src/components/views/voip/CallView.tsx b/src/components/views/voip/CallView.tsx index c084dacaa8..2027e997e1 100644 --- a/src/components/views/voip/CallView.tsx +++ b/src/components/views/voip/CallView.tsx @@ -33,6 +33,13 @@ import DialpadContextMenu from '../context_menus/DialpadContextMenu'; import { CallFeed } from 'matrix-js-sdk/src/webrtc/callFeed'; import {replaceableComponent} from "../../../utils/replaceableComponent"; +const FEED_CLASS_NAMES = [ + "mx_VideoFeed_primary", + "mx_VideoFeed_secondary", + "mx_VideoFeed_tertiary", + "mx_VideoFeed_quaternary", +]; + interface IProps { // The call for us to display call: MatrixCall, @@ -371,6 +378,34 @@ export default class CallView extends React.Component { this.props.call.transferToCall(transfereeCall); } + private renderFeeds(feeds: Array, offset = 0) { + const sortedFeeds = [...feeds].sort((a, b) => { + if (b.purpose === SDPStreamMetadataPurpose.Screenshare && !b.isLocal()) return 1; + if (a.isLocal() && !b.isLocal()) return 1; + return 0; + }); + + return sortedFeeds.map((feed, i) => { + i += offset; + // TODO: Later the CallView should probably be reworked to support + // any number of feeds but now we can't render more than 4 feeds + if (i >= 4) return; + // Here we check to hide local audio feeds to achieve the same UI/UX + // as before. But once again this might be subject to change + if (feed.isVideoMuted() && feed.isLocal()) return; + return ( + + ); + }); + } + public render() { const client = MatrixClientPeg.get(); const callRoomId = CallHandler.sharedInstance().roomIdForCall(this.props.call); @@ -594,20 +629,8 @@ export default class CallView extends React.Component { mx_CallView_voice: true, }); - const feeds = this.props.call.getLocalFeeds().map((feed, i) => { - // Here we check to hide local audio feeds to achieve the same UI/UX - // as before. But once again this might be subject to change - if (feed.isVideoMuted()) return; - return ( - - ); - }); + // We pass offset of one to avoid a feed being rendered as primary + const feeds = this.renderFeeds(this.props.call.getLocalFeeds(), 1); // Saying "Connecting" here isn't really true, but the best thing // I can come up with, but this might be subject to change as well @@ -631,23 +654,7 @@ export default class CallView extends React.Component { mx_CallView_video: true, }); - // TODO: Later the CallView should probably be reworked to support - // any number of feeds but now we can always expect there to be two - // feeds. This is because the js-sdk ignores any new incoming streams - const feeds = this.state.feeds.map((feed, i) => { - // Here we check to hide local audio feeds to achieve the same UI/UX - // as before. But once again this might be subject to change - if (feed.isVideoMuted() && feed.isLocal()) return; - return ( - - ); - }); + const feeds = this.renderFeeds(this.state.feeds); contentView =
{feeds} diff --git a/src/components/views/voip/VideoFeed.tsx b/src/components/views/voip/VideoFeed.tsx index d22fa055ce..3a7d49cfd4 100644 --- a/src/components/views/voip/VideoFeed.tsx +++ b/src/components/views/voip/VideoFeed.tsx @@ -37,6 +37,8 @@ interface IProps { // a callback which is called when the video element is resized // due to a change in video metadata onResize?: (e: Event) => void, + + className: string, } interface IState { @@ -121,8 +123,7 @@ export default class VideoFeed extends React.Component { render() { const videoClasses = { mx_VideoFeed: true, - mx_VideoFeed_local: this.props.feed.isLocal(), - mx_VideoFeed_remote: !this.props.feed.isLocal(), + [this.props.className]: true, mx_VideoFeed_voice: this.state.videoMuted, mx_VideoFeed_video: !this.state.videoMuted, mx_VideoFeed_mirror: ( From 5b2f941ce2b24d8c0909f4a40c50065ff6451c58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 7 May 2021 21:35:32 +0200 Subject: [PATCH 008/572] Add button to screenshare MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/components/views/voip/CallView.tsx | 29 ++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/components/views/voip/CallView.tsx b/src/components/views/voip/CallView.tsx index 2027e997e1..30f5db8593 100644 --- a/src/components/views/voip/CallView.tsx +++ b/src/components/views/voip/CallView.tsx @@ -32,6 +32,9 @@ import { avatarUrlForMember } from '../../../Avatar'; import DialpadContextMenu from '../context_menus/DialpadContextMenu'; import { CallFeed } from 'matrix-js-sdk/src/webrtc/callFeed'; import {replaceableComponent} from "../../../utils/replaceableComponent"; +import DesktopCapturerSourcePicker from "../elements/DesktopCapturerSourcePicker"; +import Modal from '../../../Modal'; +import { SDPStreamMetadataPurpose } from 'matrix-js-sdk/src/webrtc/callEventTypes'; const FEED_CLASS_NAMES = [ "mx_VideoFeed_primary", @@ -63,6 +66,7 @@ interface IState { isRemoteOnHold: boolean, micMuted: boolean, vidMuted: boolean, + screensharing: boolean, callState: CallState, controlsVisible: boolean, showMoreMenu: boolean, @@ -119,6 +123,7 @@ export default class CallView extends React.Component { isRemoteOnHold: this.props.call.isRemoteOnHold(), micMuted: this.props.call.isMicrophoneMuted(), vidMuted: this.props.call.isLocalVideoMuted(), + screensharing: this.props.call.isScreensharing(), callState: this.props.call.state, controlsVisible: true, showMoreMenu: false, @@ -292,6 +297,18 @@ export default class CallView extends React.Component { this.setState({vidMuted: newVal}); } + private onScreenshareClick = async () => { + const isScreensharing = await this.props.call.setScreensharingEnabled( + !this.state.screensharing, + async (): Promise => { + const {finished} = Modal.createDialog(DesktopCapturerSourcePicker); + const [source] = await finished; + return source; + }, + ); + this.setState({screensharing: isScreensharing}) + } + private onMoreClick = () => { if (this.controlsHideTimer) { clearTimeout(this.controlsHideTimer); @@ -452,6 +469,12 @@ export default class CallView extends React.Component { mx_CallView_callControls_button_vidOff: this.state.vidMuted, }); + const screensharingClasses = classNames({ + mx_CallView_callControls_button: true, + mx_CallView_callControls_button_screensharingOn: this.state.screensharing, + mx_CallView_callControls_button_screensharingOff: !this.state.screensharing, + }); + // Put the other states of the mic/video icons in the document to make sure they're cached // (otherwise the icon disappears briefly when toggled) const micCacheClasses = classNames({ @@ -478,6 +501,11 @@ export default class CallView extends React.Component { onClick={this.onVidMuteClick} /> : null; + const screensharingButton = this.props.call.opponentSupportsSDPStreamMetadata() ? : null; + // The dial pad & 'more' button actions are only relevant in a connected call // When not connected, we have to put something there to make the flexbox alignment correct const dialpadButton = this.state.callState === CallState.Connected ? { }} /> {vidMuteButton} + {screensharingButton}
{contextMenuButton} From 4c9d9dd2140c8ff645874fcb036eeb3e06994773 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 10:04:26 +0200 Subject: [PATCH 009/572] Enable screenshare in all video calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/components/views/voip/CallView.tsx | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/components/views/voip/CallView.tsx b/src/components/views/voip/CallView.tsx index 30f5db8593..90e442c641 100644 --- a/src/components/views/voip/CallView.tsx +++ b/src/components/views/voip/CallView.tsx @@ -501,10 +501,15 @@ export default class CallView extends React.Component { onClick={this.onVidMuteClick} /> : null; - const screensharingButton = this.props.call.opponentSupportsSDPStreamMetadata() ? : null; + let screensharingButton; + if (this.props.call.opponentSupportsSDPStreamMetadata() || this.props.call.type === CallType.Video) { + screensharingButton = ( + + ); + } // The dial pad & 'more' button actions are only relevant in a connected call // When not connected, we have to put something there to make the flexbox alignment correct From 430808ae2efa6248e49301a8ed7dfe8454073a41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 16:49:47 +0200 Subject: [PATCH 010/572] Simplifie CSS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- res/css/views/voip/_VideoFeed.scss | 28 ++++++------------------- src/components/views/voip/VideoFeed.tsx | 1 + 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/res/css/views/voip/_VideoFeed.scss b/res/css/views/voip/_VideoFeed.scss index 170bd89652..3d39735519 100644 --- a/res/css/views/voip/_VideoFeed.scss +++ b/res/css/views/voip/_VideoFeed.scss @@ -20,7 +20,6 @@ limitations under the License. background-color: $inverted-bg-color; } - .mx_VideoFeed_primary { width: 100%; height: 100%; @@ -33,12 +32,10 @@ limitations under the License. } } -.mx_VideoFeed_secondary { +.mx_VideoFeed_nonPrimary { max-width: 25%; max-height: 25%; position: absolute; - right: 10px; - top: 10px; z-index: 100; border-radius: 4px; @@ -47,32 +44,19 @@ limitations under the License. } } +.mx_VideoFeed_secondary { + right: 10px; + top: 10px; +} + .mx_VideoFeed_tertiary { - max-width: 25%; - max-height: 25%; - position: absolute; right: 10px; bottom: 10px; - z-index: 100; - border-radius: 4px; - - &.mx_VideoFeed_video { - background-color: transparent; - } } .mx_VideoFeed_quaternary { - max-width: 25%; - max-height: 25%; - position: absolute; left: 10px; top: 10px; - z-index: 100; - border-radius: 4px; - - &.mx_VideoFeed_video { - background-color: transparent; - } } .mx_VideoFeed_mirror { diff --git a/src/components/views/voip/VideoFeed.tsx b/src/components/views/voip/VideoFeed.tsx index 3a7d49cfd4..a545f100c8 100644 --- a/src/components/views/voip/VideoFeed.tsx +++ b/src/components/views/voip/VideoFeed.tsx @@ -123,6 +123,7 @@ export default class VideoFeed extends React.Component { render() { const videoClasses = { mx_VideoFeed: true, + mx_VideoFeed_nonPrimary: this.props.className !== "mx_VideoFeed_primary", [this.props.className]: true, mx_VideoFeed_voice: this.state.videoMuted, mx_VideoFeed_video: !this.state.videoMuted, From 69b0425c1088704106b986775309c58ad89f1216 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 18:50:13 +0200 Subject: [PATCH 011/572] Improve and fix sorting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/components/views/voip/CallView.tsx | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/components/views/voip/CallView.tsx b/src/components/views/voip/CallView.tsx index 90e442c641..0b3e1d5f48 100644 --- a/src/components/views/voip/CallView.tsx +++ b/src/components/views/voip/CallView.tsx @@ -203,7 +203,14 @@ export default class CallView extends React.Component { }; private onFeedsChanged = (newFeeds: Array) => { - this.setState({feeds: newFeeds}); + // Sort the feeds so that screensharing and remote feeds have priority + const sortedFeeds = [...newFeeds].sort((a, b) => { + if (b.purpose === SDPStreamMetadataPurpose.Screenshare && !b.isLocal()) return 1; + if (a.isLocal() && !b.isLocal()) return 1; + return -1; + }); + + this.setState({feeds: sortedFeeds}); }; private onCallLocalHoldUnhold = () => { @@ -396,13 +403,7 @@ export default class CallView extends React.Component { } private renderFeeds(feeds: Array, offset = 0) { - const sortedFeeds = [...feeds].sort((a, b) => { - if (b.purpose === SDPStreamMetadataPurpose.Screenshare && !b.isLocal()) return 1; - if (a.isLocal() && !b.isLocal()) return 1; - return 0; - }); - - return sortedFeeds.map((feed, i) => { + return feeds.map((feed, i) => { i += offset; // TODO: Later the CallView should probably be reworked to support // any number of feeds but now we can't render more than 4 feeds From 2749715050d0e55d0c155c0b67cfdddc149dd5b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 10 May 2021 12:26:28 +0200 Subject: [PATCH 012/572] Remove screensharing call type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/CallHandler.tsx | 25 ------------------------ src/components/views/rooms/RoomHeader.js | 9 +++++++-- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/src/CallHandler.tsx b/src/CallHandler.tsx index 0268ebfe46..7184deb0e7 100644 --- a/src/CallHandler.tsx +++ b/src/CallHandler.tsx @@ -80,7 +80,6 @@ import CountlyAnalytics from "./CountlyAnalytics"; import {UIFeature} from "./settings/UIFeature"; import { CallError } from "matrix-js-sdk/src/webrtc/call"; import { logger } from 'matrix-js-sdk/src/logger'; -import DesktopCapturerSourcePicker from "./components/views/elements/DesktopCapturerSourcePicker" import { Action } from './dispatcher/actions'; import VoipUserMapper from './VoipUserMapper'; import { addManagedHybridWidget, isManagedHybridWidgetEnabled } from './widgets/ManagedHybrid'; @@ -129,14 +128,9 @@ interface ThirdpartyLookupResponse { fields: ThirdpartyLookupResponseFields, } -// Unlike 'CallType' in js-sdk, this one includes screen sharing -// (because a screen sharing call is only a screen sharing call to the caller, -// to the callee it's just a video call, at least as far as the current impl -// is concerned). export enum PlaceCallType { Voice = 'voice', Video = 'video', - ScreenSharing = 'screensharing', } export enum CallHandlerEvent { @@ -689,25 +683,6 @@ export default class CallHandler extends EventEmitter { call.placeVoiceCall(); } else if (type === 'video') { call.placeVideoCall(); - } else if (type === PlaceCallType.ScreenSharing) { - const screenCapErrorString = PlatformPeg.get().screenCaptureErrorString(); - if (screenCapErrorString) { - this.removeCallForRoom(roomId); - console.log("Can't capture screen: " + screenCapErrorString); - Modal.createTrackedDialog('Call Handler', 'Unable to capture screen', ErrorDialog, { - title: _t('Unable to capture screen'), - description: screenCapErrorString, - }); - return; - } - - call.placeScreenSharingCall( - async (): Promise => { - const {finished} = Modal.createDialog(DesktopCapturerSourcePicker); - const [source] = await finished; - return source; - }, - ); } else { console.error("Unknown conf call type: " + type); } diff --git a/src/components/views/rooms/RoomHeader.js b/src/components/views/rooms/RoomHeader.js index f856f7f6ef..476b0e1edb 100644 --- a/src/components/views/rooms/RoomHeader.js +++ b/src/components/views/rooms/RoomHeader.js @@ -33,6 +33,7 @@ import RoomTopic from "../elements/RoomTopic"; import RoomName from "../elements/RoomName"; import {PlaceCallType} from "../../../CallHandler"; import {replaceableComponent} from "../../../utils/replaceableComponent"; +import Modal from '../../../Modal'; @replaceableComponent("views.rooms.RoomHeader") export default class RoomHeader extends React.Component { @@ -118,6 +119,10 @@ export default class RoomHeader extends React.Component { return !(currentPinEvent.getContent().pinned && currentPinEvent.getContent().pinned.length <= 0); } + _displayInfoDialogAboutScreensharing() { + + } + render() { let searchStatus = null; let cancelButton = null; @@ -241,8 +246,8 @@ export default class RoomHeader extends React.Component { videoCallButton = this.props.onCallPlaced( - ev.shiftKey ? PlaceCallType.ScreenSharing : PlaceCallType.Video)} + onClick={(ev) => ev.shiftKey ? + this._displayInfoDialogAboutScreensharing() : this.props.onCallPlaced(PlaceCallType.Video)} title={_t("Video call")} />; } From 135cdb2255ad32aba23f8c3f0d4ba059914b4f19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 10 May 2021 12:37:40 +0200 Subject: [PATCH 013/572] Add dialog with info about the screensharing change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/components/views/rooms/RoomHeader.js | 7 ++++++- src/i18n/strings/en_EN.json | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/components/views/rooms/RoomHeader.js b/src/components/views/rooms/RoomHeader.js index 476b0e1edb..126f5e6c15 100644 --- a/src/components/views/rooms/RoomHeader.js +++ b/src/components/views/rooms/RoomHeader.js @@ -34,6 +34,7 @@ import RoomName from "../elements/RoomName"; import {PlaceCallType} from "../../../CallHandler"; import {replaceableComponent} from "../../../utils/replaceableComponent"; import Modal from '../../../Modal'; +import InfoDialog from "../dialogs/InfoDialog"; @replaceableComponent("views.rooms.RoomHeader") export default class RoomHeader extends React.Component { @@ -120,7 +121,11 @@ export default class RoomHeader extends React.Component { } _displayInfoDialogAboutScreensharing() { - + Modal.createDialog(InfoDialog, { + title: _t("Screensharing has changed"), + description: _t("You don't have to shift-click anymore! You can now share " + + "your screen in any video call and in voice calls if other side supports it."), + }); } render() { diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index deeee2e60e..7175dd6910 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -53,7 +53,6 @@ "A microphone and webcam are plugged in and set up correctly": "A microphone and webcam are plugged in and set up correctly", "Permission is granted to use the webcam": "Permission is granted to use the webcam", "No other application is using the webcam": "No other application is using the webcam", - "Unable to capture screen": "Unable to capture screen", "VoIP is unsupported": "VoIP is unsupported", "You cannot place VoIP calls in this browser.": "You cannot place VoIP calls in this browser.", "Too Many Calls": "Too Many Calls", @@ -1522,6 +1521,8 @@ "Unnamed room": "Unnamed room", "World readable": "World readable", "Guests can join": "Guests can join", + "Screensharing has changed": "Screensharing has changed", + "You don't have to shift-click anymore! You can now share your screen in any video call and in voice calls if other side supports it.": "You don't have to shift-click anymore! You can now share your screen in any video call and in voice calls if other side supports it.", "(~%(count)s results)|other": "(~%(count)s results)", "(~%(count)s results)|one": "(~%(count)s result)", "Join Room": "Join Room", From 0eeb21dfaca6554eb8df8251604eeadeba3bb635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 10 May 2021 12:41:29 +0200 Subject: [PATCH 014/572] Remove unnecessary import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/CallHandler.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/CallHandler.tsx b/src/CallHandler.tsx index 7184deb0e7..4e20238431 100644 --- a/src/CallHandler.tsx +++ b/src/CallHandler.tsx @@ -56,7 +56,6 @@ limitations under the License. import React from 'react'; import {MatrixClientPeg} from './MatrixClientPeg'; -import PlatformPeg from './PlatformPeg'; import Modal from './Modal'; import { _t } from './languageHandler'; import dis from './dispatcher/dispatcher'; From 228b2ccf2d3c749d229a2a83508701ab3309da20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 10 May 2021 13:06:25 +0200 Subject: [PATCH 015/572] Increase z-index of call controls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- res/css/views/voip/_CallView.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/res/css/views/voip/_CallView.scss b/res/css/views/voip/_CallView.scss index 8ebe7aa607..21c948511d 100644 --- a/res/css/views/voip/_CallView.scss +++ b/res/css/views/voip/_CallView.scss @@ -291,6 +291,7 @@ limitations under the License. width: 100%; opacity: 1; transition: opacity 0.5s; + z-index: 200; // To be above _all_ feeds } .mx_CallView_callControls_hidden { From 90f4ad7a830e2f7c4c4150d2d0f6b83fcc6c311c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 10 May 2021 13:21:02 +0200 Subject: [PATCH 016/572] Always sort feeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/components/views/voip/CallView.tsx | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/components/views/voip/CallView.tsx b/src/components/views/voip/CallView.tsx index 0b3e1d5f48..a7ad6ef8c6 100644 --- a/src/components/views/voip/CallView.tsx +++ b/src/components/views/voip/CallView.tsx @@ -128,7 +128,7 @@ export default class CallView extends React.Component { controlsVisible: true, showMoreMenu: false, showDialpad: false, - feeds: this.props.call.getFeeds(), + feeds: this.sortFeeds(this.props.call.getFeeds()), } this.updateCallListeners(null, this.props.call); @@ -203,14 +203,7 @@ export default class CallView extends React.Component { }; private onFeedsChanged = (newFeeds: Array) => { - // Sort the feeds so that screensharing and remote feeds have priority - const sortedFeeds = [...newFeeds].sort((a, b) => { - if (b.purpose === SDPStreamMetadataPurpose.Screenshare && !b.isLocal()) return 1; - if (a.isLocal() && !b.isLocal()) return 1; - return -1; - }); - - this.setState({feeds: sortedFeeds}); + this.setState({feeds: this.sortFeeds(newFeeds)}); }; private onCallLocalHoldUnhold = () => { @@ -253,6 +246,15 @@ export default class CallView extends React.Component { this.showControls(); } + private sortFeeds(feeds: Array) { + // Sort the feeds so that screensharing and remote feeds have priority + return [...feeds].sort((a, b) => { + if (b.purpose === SDPStreamMetadataPurpose.Screenshare && !b.isLocal()) return 1; + if (a.isLocal() && !b.isLocal()) return 1; + return -1; + }); + } + private showControls() { if (this.state.showMoreMenu || this.state.showDialpad) return; From acd0fa4c0e929fa4293ba430257d13f4749d0c11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 10 May 2021 14:42:06 +0200 Subject: [PATCH 017/572] Add a comment about when it is possible to screenshare MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/components/views/voip/CallView.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/components/views/voip/CallView.tsx b/src/components/views/voip/CallView.tsx index a7ad6ef8c6..b49e729244 100644 --- a/src/components/views/voip/CallView.tsx +++ b/src/components/views/voip/CallView.tsx @@ -505,6 +505,9 @@ export default class CallView extends React.Component { /> : null; let screensharingButton; + // Screensharing is possible, if we can send a second stream and identify + // it using SDPStreamMetadata or if we can replace the already existing + // usermedia track by a screensharing track if (this.props.call.opponentSupportsSDPStreamMetadata() || this.props.call.type === CallType.Video) { screensharingButton = ( Date: Tue, 11 May 2021 13:07:30 +0200 Subject: [PATCH 018/572] Don't render any audio non-primary feeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/components/views/voip/CallView.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/components/views/voip/CallView.tsx b/src/components/views/voip/CallView.tsx index b49e729244..dc84cbed60 100644 --- a/src/components/views/voip/CallView.tsx +++ b/src/components/views/voip/CallView.tsx @@ -410,9 +410,10 @@ export default class CallView extends React.Component { // TODO: Later the CallView should probably be reworked to support // any number of feeds but now we can't render more than 4 feeds if (i >= 4) return; - // Here we check to hide local audio feeds to achieve the same UI/UX - // as before. But once again this might be subject to change - if (feed.isVideoMuted() && feed.isLocal()) return; + // Here we check to hide any non-main audio feeds from the UI + // This is because we don't want them to obstruct the view + // But once again this might be subject to change + if (feed.isVideoMuted() && i > 0) return; return ( Date: Thu, 13 May 2021 18:11:47 +0200 Subject: [PATCH 019/572] Show screensharign button only if connected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/components/views/voip/CallView.tsx | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/components/views/voip/CallView.tsx b/src/components/views/voip/CallView.tsx index dc84cbed60..a9d605c9f1 100644 --- a/src/components/views/voip/CallView.tsx +++ b/src/components/views/voip/CallView.tsx @@ -506,10 +506,14 @@ export default class CallView extends React.Component { /> : null; let screensharingButton; - // Screensharing is possible, if we can send a second stream and identify - // it using SDPStreamMetadata or if we can replace the already existing - // usermedia track by a screensharing track - if (this.props.call.opponentSupportsSDPStreamMetadata() || this.props.call.type === CallType.Video) { + // Screensharing is possible, if we can send a second stream and + // identify it using SDPStreamMetadata or if we can replace the already + // existing usermedia track by a screensharing track. We also need to be + // connected to know the state of the other side + if ( + (this.props.call.opponentSupportsSDPStreamMetadata() || this.props.call.type === CallType.Video) && + this.props.call.state === CallState.Connected + ) { screensharingButton = ( Date: Wed, 19 May 2021 08:46:04 +0200 Subject: [PATCH 020/572] Do not render the audio element if there is no audio track MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/components/views/voip/AudioFeed.tsx | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/components/views/voip/AudioFeed.tsx b/src/components/views/voip/AudioFeed.tsx index c78f0c0fc8..7fb57abcd2 100644 --- a/src/components/views/voip/AudioFeed.tsx +++ b/src/components/views/voip/AudioFeed.tsx @@ -23,9 +23,21 @@ interface IProps { feed: CallFeed, } -export default class AudioFeed extends React.Component { +interface IState { + audioMuted: boolean; +} + +export default class AudioFeed extends React.Component { private element = createRef(); + constructor(props: IProps) { + super(props); + + this.state = { + audioMuted: this.props.feed.isAudioMuted(), + }; + } + componentDidMount() { this.props.feed.addListener(CallFeedEvent.NewStream, this.onNewStream); this.playMedia(); @@ -38,6 +50,7 @@ export default class AudioFeed extends React.Component { private playMedia() { const element = this.element.current; + if (!element) return; const audioOutput = CallMediaHandler.getAudioOutput(); if (audioOutput) { @@ -75,6 +88,7 @@ export default class AudioFeed extends React.Component { private stopMedia() { const element = this.element.current; + if (!element) return; element.pause(); element.src = null; @@ -86,10 +100,16 @@ export default class AudioFeed extends React.Component { } private onNewStream = () => { + this.setState({ + audioMuted: this.props.feed.isAudioMuted(), + }); this.playMedia(); }; render() { + // Do not render the audio element if there is no audio track + if (this.state.audioMuted) return null; + return (