Merge pull request #4436 from matrix-org/travis/e2e-e2e-tests

Fix end-to-end tests for end-to-end encryption verification

src/components/views/right_panel/UserInfo.js

@@ -1308,8 +1308,9 @@ const BasicUserInfo = ({room, member, groupId, devices, isRoomEncrypted}) => {
         useHasCrossSigningKeys(cli, member, canVerify, setUpdating );
 
     if (canVerify) {
+        // Note: mx_UserInfo_verifyButton is for the end-to-end tests
         verifyButton = (
-            <AccessibleButton className="mx_UserInfo_field" onClick={() => {
+            <AccessibleButton className="mx_UserInfo_field mx_UserInfo_verifyButton" onClick={() => {
                 if (hasCrossSigningKeys) {
                     verifyUser(member);
                 } else {

src/components/views/right_panel/VerificationPanel.js

@@ -123,10 +123,17 @@ export default class VerificationPanel extends React.PureComponent {
             const sasLabel = showQR ?
                 _t("If you can't scan the code above, verify by comparing unique emoji.") :
                 _t("Verify by comparing unique emoji.");
+
+            // Note: mx_VerificationPanel_verifyByEmojiButton is for the end-to-end tests
             sasBlock = <div className="mx_UserInfo_container">
                 <h3>{_t("Verify by emoji")}</h3>
                 <p>{sasLabel}</p>
-                <AccessibleButton disabled={disabled} kind="primary" className="mx_UserInfo_wideButton" onClick={this._startSAS}>
+                <AccessibleButton
+                    disabled={disabled}
+                    kind="primary"
+                    className="mx_UserInfo_wideButton mx_VerificationPanel_verifyByEmojiButton"
+                    onClick={this._startSAS}
+                >
                     {_t("Verify by emoji")}
                 </AccessibleButton>
             </div>;

test/end-to-end-tests/src/scenarios/directory.js

@@ -20,7 +20,7 @@ const join = require('../usecases/join');
 const sendMessage = require('../usecases/send-message');
 const {receiveMessage} = require('../usecases/timeline');
 const {createRoom} = require('../usecases/create-room');
-const changeRoomSettings = require('../usecases/room-settings');
+const {changeRoomSettings} = require('../usecases/room-settings');
 
 module.exports = async function roomDirectoryScenarios(alice, bob) {
     console.log(" creating a public room and join through directory:");

test/end-to-end-tests/src/scenarios/e2e-encryption.js

@@ -1,6 +1,6 @@
 /*
 Copyright 2018 New Vector Ltd
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,42 +15,32 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// TODO: Update test for cross signing
+const sendMessage = require('../usecases/send-message');
-// https://github.com/vector-im/riot-web/issues/13226
+const acceptInvite = require('../usecases/accept-invite');
+const {receiveMessage} = require('../usecases/timeline');
+const {createDm} = require('../usecases/create-room');
+const {checkRoomSettings} = require('../usecases/room-settings');
+const {startSasVerifcation, acceptSasVerification} = require('../usecases/verify');
+const assert = require('assert');
 
-module.exports = async function() {
+module.exports = async function e2eEncryptionScenarios(alice, bob) {
-    console.log(" this is supposed to be an e2e test, but it's broken");
+    console.log(" creating an e2e encrypted DM and join through invite:");
+    await createDm(bob, ['@alice:localhost']);
+    await checkRoomSettings(bob, {encryption: true}); // for sanity, should be e2e-by-default
+    await acceptInvite(alice, 'bob');
+    // do sas verifcation
+    bob.log.step(`starts SAS verification with ${alice.username}`);
+    const bobSasPromise = startSasVerifcation(bob, alice.username);
+    const aliceSasPromise = acceptSasVerification(alice, bob.username);
+    // wait in parallel, so they don't deadlock on each other
+    // the logs get a bit messy here, but that's fine enough for debugging (hopefully)
+    const [bobSas, aliceSas] = await Promise.all([bobSasPromise, aliceSasPromise]);
+    assert.deepEqual(bobSas, aliceSas);
+    bob.log.done(`done (match for ${bobSas.join(", ")})`);
+    const aliceMessage = "Guess what I just heard?!";
+    await sendMessage(alice, aliceMessage);
+    await receiveMessage(bob, {sender: "alice", body: aliceMessage, encrypted: true});
+    const bobMessage = "You've got to tell me!";
+    await sendMessage(bob, bobMessage);
+    await receiveMessage(alice, {sender: "bob", body: bobMessage, encrypted: true});
 };
-
-// const sendMessage = require('../usecases/send-message');
-// const acceptInvite = require('../usecases/accept-invite');
-// const invite = require('../usecases/invite');
-// const {receiveMessage} = require('../usecases/timeline');
-// const {createRoom} = require('../usecases/create-room');
-// const changeRoomSettings = require('../usecases/room-settings');
-// const {startSasVerifcation, acceptSasVerification} = require('../usecases/verify');
-// const assert = require('assert');
-//
-// module.exports = async function e2eEncryptionScenarios(alice, bob) {
-//     console.log(" creating an e2e encrypted room and join through invite:");
-//     const room = "secrets";
-//     await createRoom(bob, room);
-//     await changeRoomSettings(bob, {encryption: true});
-//     // await cancelKeyBackup(bob);
-//     await invite(bob, "@alice:localhost");
-//     await acceptInvite(alice, room);
-//     // do sas verifcation
-//     bob.log.step(`starts SAS verification with ${alice.username}`);
-//     const bobSasPromise = startSasVerifcation(bob, alice.username);
-//     const aliceSasPromise = acceptSasVerification(alice, bob.username);
-//     // wait in parallel, so they don't deadlock on each other
-//     const [bobSas, aliceSas] = await Promise.all([bobSasPromise, aliceSasPromise]);
-//     assert.deepEqual(bobSas, aliceSas);
-//     bob.log.done(`done (match for ${bobSas.join(", ")})`);
-//     const aliceMessage = "Guess what I just heard?!";
-//     await sendMessage(alice, aliceMessage);
-//     await receiveMessage(bob, {sender: "alice", body: aliceMessage, encrypted: true});
-//     const bobMessage = "You've got to tell me!";
-//     await sendMessage(bob, bobMessage);
-//     await receiveMessage(alice, {sender: "bob", body: bobMessage, encrypted: true});
-// };

test/end-to-end-tests/src/scenarios/lazy-loading.js

@@ -25,7 +25,7 @@ const {
 } = require('../usecases/timeline');
 const {createRoom} = require('../usecases/create-room');
 const {getMembersInMemberlist} = require('../usecases/memberlist');
-const changeRoomSettings = require('../usecases/room-settings');
+const {changeRoomSettings} = require('../usecases/room-settings');
 const assert = require('assert');
 
 module.exports = async function lazyLoadingScenarios(alice, bob, charlies) {

test/end-to-end-tests/src/session.js

@@ -75,6 +75,10 @@ module.exports = class RiotSession {
         return this.getElementProperty(field, 'outerHTML');
     }
 
+    isChecked(field) {
+        return this.getElementProperty(field, 'checked');
+    }
+
     consoleLogs() {
         return this.consoleLog.buffer;
     }

test/end-to-end-tests/src/usecases/create-room.js

@@ -27,7 +27,7 @@ async function createRoom(session, roomName, encrypted=false) {
     const roomListHeaderLabels = await Promise.all(roomListHeaders.map(h => session.innerText(h)));
     const roomsIndex = roomListHeaderLabels.findIndex(l => l.toLowerCase().includes("rooms"));
     if (roomsIndex === -1) {
-        throw new Error("could not find room list section that contains rooms in header");
+        throw new Error("could not find room list section that contains 'rooms' in header");
     }
     const roomsHeader = roomListHeaders[roomsIndex];
     const addRoomButton = await roomsHeader.$(".mx_RoomSubList_addRoom");
@@ -48,4 +48,39 @@ async function createRoom(session, roomName, encrypted=false) {
     session.log.done();
 }
 
-module.exports = {openRoomDirectory, createRoom};
+async function createDm(session, invitees) {
+    session.log.step(`creates DM with ${JSON.stringify(invitees)}`);
+
+    const roomListHeaders = await session.queryAll('.mx_RoomSubList_labelContainer');
+    const roomListHeaderLabels = await Promise.all(roomListHeaders.map(h => session.innerText(h)));
+    const dmsIndex = roomListHeaderLabels.findIndex(l => l.toLowerCase().includes('direct messages'));
+    if (dmsIndex === -1) {
+        throw new Error("could not find room list section that contains 'direct messages' in header");
+    }
+    const dmsHeader = roomListHeaders[dmsIndex];
+    const startChatButton = await dmsHeader.$(".mx_RoomSubList_addRoom");
+    await startChatButton.click();
+
+    const inviteesEditor = await session.query('.mx_InviteDialog_editor textarea');
+    for (const target of invitees) {
+        await session.replaceInputText(inviteesEditor, target);
+        await session.delay(1000); // give it a moment to figure out a suggestion
+        // find the suggestion and accept it
+        const suggestions = await session.queryAll('.mx_InviteDialog_roomTile_userId');
+        const suggestionTexts = await Promise.all(suggestions.map(s => session.innerText(s)));
+        const suggestionIndex = suggestionTexts.indexOf(target);
+        if (suggestionIndex === -1) {
+            throw new Error(`failed to find a suggestion in the DM dialog to invite ${target} with`);
+        }
+        await suggestions[suggestionIndex].click();
+    }
+
+    // press the go button and hope for the best
+    const goButton = await session.query('.mx_InviteDialog_goButton');
+    await goButton.click();
+
+    await session.query('.mx_MessageComposer');
+    session.log.done();
+}
+
+module.exports = {openRoomDirectory, createRoom, createDm};

test/end-to-end-tests/src/usecases/room-settings.js

@@ -30,18 +30,102 @@ async function setSettingsToggle(session, toggle, enabled) {
     }
 }
 
-module.exports = async function changeRoomSettings(session, settings) {
+async function checkSettingsToggle(session, toggle, shouldBeEnabled) {
-    session.log.startGroup(`changes the room settings`);
+    const className = await session.getElementProperty(toggle, "className");
+    const checked = className.includes("mx_ToggleSwitch_on");
+    if (checked === shouldBeEnabled) {
+        session.log.done('set as expected');
+    } else {
+        // other logs in the area should give more context as to what this means.
+        throw new Error("settings toggle value didn't match expectation");
+    }
+}
+
+async function findTabs(session) {
     /// XXX delay is needed here, possibly because the header is being rerendered
     /// click doesn't do anything otherwise
     await session.delay(1000);
     const settingsButton = await session.query(".mx_RoomHeader .mx_AccessibleButton[title=Settings]");
     await settingsButton.click();
+
     //find tabs
     const tabButtons = await session.queryAll(".mx_RoomSettingsDialog .mx_TabbedView_tabLabel");
     const tabLabels = await Promise.all(tabButtons.map(t => session.innerText(t)));
     const securityTabButton = tabButtons[tabLabels.findIndex(l => l.toLowerCase().includes("security"))];
 
+    return {securityTabButton};
+}
+
+async function checkRoomSettings(session, expectedSettings) {
+    session.log.startGroup(`checks the room settings`);
+
+    const {securityTabButton} = await findTabs(session);
+    const generalSwitches = await session.queryAll(".mx_RoomSettingsDialog .mx_ToggleSwitch");
+    const isDirectory = generalSwitches[0];
+
+    if (typeof expectedSettings.directory === 'boolean') {
+        session.log.step(`checks directory listing is ${expectedSettings.directory}`);
+        await checkSettingsToggle(session, isDirectory, expectedSettings.directory);
+    }
+
+    if (expectedSettings.alias) {
+        session.log.step(`checks for local alias of ${expectedSettings.alias}`);
+        const summary = await session.query(".mx_RoomSettingsDialog .mx_AliasSettings summary");
+        await summary.click();
+        const localAliases = await session.query('.mx_RoomSettingsDialog .mx_AliasSettings .mx_EditableItem_item');
+        const localAliasTexts = await Promise.all(localAliases.map(a => session.innerText(a)));
+        if (localAliasTexts.find(a => a.includes(expectedSettings.alias))) {
+            session.log.done("present");
+        } else {
+            throw new Error(`could not find local alias ${expectedSettings.alias}`);
+        }
+    }
+
+    securityTabButton.click();
+    await session.delay(500);
+    const securitySwitches = await session.queryAll(".mx_RoomSettingsDialog .mx_ToggleSwitch");
+    const e2eEncryptionToggle = securitySwitches[0];
+
+    if (typeof expectedSettings.encryption === "boolean") {
+        session.log.step(`checks room e2e encryption is ${expectedSettings.encryption}`);
+        await checkSettingsToggle(session, e2eEncryptionToggle, expectedSettings.encryption);
+    }
+
+    if (expectedSettings.visibility) {
+        session.log.step(`checks visibility is ${expectedSettings.visibility}`);
+        const radios = await session.queryAll(".mx_RoomSettingsDialog input[type=radio]");
+        assert.equal(radios.length, 7);
+        const inviteOnly = radios[0];
+        const publicNoGuests = radios[1];
+        const publicWithGuests = radios[2];
+
+        let expectedRadio = null;
+        if (expectedSettings.visibility === "invite_only") {
+            expectedRadio = inviteOnly;
+        } else if (expectedSettings.visibility === "public_no_guests") {
+            expectedRadio = publicNoGuests;
+        } else if (expectedSettings.visibility === "public_with_guests") {
+            expectedRadio = publicWithGuests;
+        } else {
+            throw new Error(`unrecognized room visibility setting: ${expectedSettings.visibility}`);
+        }
+        if (await session.isChecked(expectedRadio)) {
+            session.log.done();
+        } else {
+            throw new Error("room visibility is not as expected");
+        }
+    }
+
+    const closeButton = await session.query(".mx_RoomSettingsDialog .mx_Dialog_cancelButton");
+    await closeButton.click();
+
+    session.log.endGroup();
+}
+
+async function changeRoomSettings(session, settings) {
+    session.log.startGroup(`changes the room settings`);
+
+    const {securityTabButton} = await findTabs(session);
     const generalSwitches = await session.queryAll(".mx_RoomSettingsDialog .mx_ToggleSwitch");
     const isDirectory = generalSwitches[0];
 
@@ -100,4 +184,6 @@ module.exports = async function changeRoomSettings(session, settings) {
     await closeButton.click();
 
     session.log.endGroup();
-};
+}
+
+module.exports = {checkRoomSettings, changeRoomSettings};

test/end-to-end-tests/src/usecases/verify.js

@@ -1,6 +1,6 @@
 /*
 Copyright 2019 New Vector Ltd
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,18 +17,21 @@ limitations under the License.
 
 const assert = require('assert');
 const {openMemberInfo} = require("./memberlist");
-const {assertDialog, acceptDialog} = require("./dialog");
-
-async function assertVerified(session) {
-    const dialogSubTitle = await session.innerText(await session.query(".mx_Dialog h2"));
-    assert(dialogSubTitle, "Verified!");
-}
 
 async function startVerification(session, name) {
+    session.log.step("opens their opponent's profile and starts verification");
     await openMemberInfo(session, name);
     // click verify in member info
-    const firstVerifyButton = await session.query(".mx_MemberDeviceInfo_verify");
+    const firstVerifyButton = await session.query(".mx_UserInfo_verifyButton");
     await firstVerifyButton.click();
+
+    // wait for the animation to finish
+    await session.delay(1000);
+
+    // click 'start verification'
+    const startVerifyButton = await session.query('.mx_UserInfo_container .mx_AccessibleButton_kind_primary');
+    await startVerifyButton.click();
+    session.log.done();
 }
 
 async function getSasCodes(session) {
@@ -38,33 +41,73 @@ async function getSasCodes(session) {
     return sasLabels;
 }
 
-module.exports.startSasVerifcation = async function(session, name) {
+async function doSasVerification(session) {
-    await startVerification(session, name);
+    session.log.step("hunts for the emoji to yell at their opponent");
-    // expect "Verify device" dialog and click "Begin Verification"
-    await assertDialog(session, "Verify device");
-    // click "Begin Verification"
-    await acceptDialog(session);
     const sasCodes = await getSasCodes(session);
-    // click "Verify"
+    session.log.done(sasCodes);
-    await acceptDialog(session);
+
-    await assertVerified(session);
+    // Assume they match
-    // click "Got it" when verification is done
+    session.log.step("assumes the emoji match");
-    await acceptDialog(session);
+    const matchButton = await session.query(".mx_VerificationShowSas .mx_AccessibleButton_kind_primary");
+    await matchButton.click();
+    session.log.done();
+
+    // Wait for a big green shield (universal sign that it worked)
+    session.log.step("waits for a green shield");
+    await session.query(".mx_VerificationPanel_verified_section .mx_E2EIcon_verified");
+    session.log.done();
+
+    // Click 'Got It'
+    session.log.step("confirms the green shield");
+    const doneButton = await session.query(".mx_VerificationPanel_verified_section .mx_AccessibleButton_kind_primary");
+    await doneButton.click();
+    session.log.done();
+
+    // Wait a bit for the animation
+    session.log.step("confirms their opponent has a green shield");
+    await session.delay(1000);
+
+    // Verify that we now have a green shield in their name (proving it still works)
+    await session.query('.mx_UserInfo_profile .mx_E2EIcon_verified');
+    session.log.done();
+
+    return sasCodes;
+}
+
+module.exports.startSasVerifcation = async function(session, name) {
+    session.log.startGroup("starts verification");
+    await startVerification(session, name);
+
+    // expect to be waiting (the presence of a spinner is a good thing)
+    await session.query('.mx_UserInfo_container .mx_EncryptionInfo_spinner');
+
+    const sasCodes = await doSasVerification(session);
+    session.log.endGroup();
     return sasCodes;
 };
 
 module.exports.acceptSasVerification = async function(session, name) {
-    await assertDialog(session, "Incoming Verification Request");
+    session.log.startGroup("accepts verification");
-    const opponentLabelElement = await session.query(".mx_IncomingSasDialog_opponentProfile h2");
+    const requestToast = await session.query('.mx_Toast_icon_verification');
-    const opponentLabel = await session.innerText(opponentLabelElement);
+
-    assert(opponentLabel, name);
+    // verify the toast is for verification
-    // click "Continue" button
+    const toastHeader = await requestToast.$("h2");
-    await acceptDialog(session);
+    const toastHeaderText = await session.innerText(toastHeader);
-    const sasCodes = await getSasCodes(session);
+    assert.equal(toastHeaderText, 'Verification Request');
-    // click "Verify"
+    const toastDescription = await requestToast.$(".mx_Toast_description");
-    await acceptDialog(session);
+    const toastDescText = await session.innerText(toastDescription);
-    await assertVerified(session);
+    assert.equal(toastDescText.startsWith(name), true,
-    // click "Got it" when verification is done
+        `verification opponent mismatch: expected to start with '${name}', got '${toastDescText}'`);
-    await acceptDialog(session);
+
+    // accept the verification
+    const acceptButton = await requestToast.$(".mx_AccessibleButton_kind_primary");
+    await acceptButton.click();
+
+    // find the emoji button
+    const startEmojiButton = await session.query(".mx_VerificationPanel_verifyByEmojiButton");
+    await startEmojiButton.click();
+
+    const sasCodes = await doSasVerification(session);
+    session.log.endGroup();
     return sasCodes;
 };