Merge branch 'develop' into jaywink/hosting-provider-iframe

package.json

@@ -76,6 +76,8 @@
     "highlight.js": "^10.1.2",
     "html-entities": "^1.3.1",
     "is-ip": "^2.0.0",
+    "katex": "^0.12.0",
+    "cheerio": "^1.0.0-rc.3",
     "linkifyjs": "^2.1.9",
     "lodash": "^4.17.19",
     "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#develop",

src/HtmlUtils.tsx

@@ -27,9 +27,12 @@ import _linkifyString from 'linkifyjs/string';
 import classNames from 'classnames';
 import EMOJIBASE_REGEX from 'emojibase-regex';
 import url from 'url';
+import katex from 'katex';
+import { AllHtmlEntities } from 'html-entities';
+import SettingsStore from './settings/SettingsStore';
+import cheerio from 'cheerio';
 
 import {MatrixClientPeg} from './MatrixClientPeg';
-import SettingsStore from './settings/SettingsStore';
 import {tryTransformPermalinkToLocalHref} from "./utils/permalinks/Permalinks";
 import {SHORTCODE_TO_EMOJI, getEmojiFromUnicode} from "./emoji";
 import ReplyThread from "./components/views/elements/ReplyThread";
@@ -240,7 +243,8 @@ const sanitizeHtmlParams: IExtendedSanitizeOptions = {
     allowedAttributes: {
         // custom ones first:
         font: ['color', 'data-mx-bg-color', 'data-mx-color', 'style'], // custom to matrix
-        span: ['data-mx-bg-color', 'data-mx-color', 'data-mx-spoiler', 'style'], // custom to matrix
+        span: ['data-mx-maths', 'data-mx-bg-color', 'data-mx-color', 'data-mx-spoiler', 'style'], // custom to matrix
+        div: ['data-mx-maths'],
         a: ['href', 'name', 'target', 'rel'], // remote target: custom to matrix
         img: ['src', 'width', 'height', 'alt', 'title'],
         ol: ['start'],
@@ -414,6 +418,21 @@ export function bodyToHtml(content: IContent, highlights: string[], opts: IOpts
         if (isHtmlMessage) {
             isDisplayedWithHtml = true;
             safeBody = sanitizeHtml(formattedBody, sanitizeParams);
+
+            if (SettingsStore.getValue("feature_latex_maths")) {
+                const phtml = cheerio.load(safeBody,
+                    { _useHtmlParser2: true, decodeEntities: false })
+                phtml('div, span[data-mx-maths!=""]').replaceWith(function(i, e) {
+                    return katex.renderToString(
+                        AllHtmlEntities.decode(phtml(e).attr('data-mx-maths')),
+                        {
+                            throwOnError: false,
+                            displayMode: e.name == 'div',
+                            output: "htmlAndMathml",
+                        });
+                });
+                safeBody = phtml.html();
+            }
         }
     } finally {
         delete sanitizeParams.textFilter;
@@ -515,7 +534,6 @@ export function checkBlockNode(node: Node) {
         case "H6":
         case "PRE":
         case "BLOCKQUOTE":
-        case "DIV":
         case "P":
         case "UL":
         case "OL":
@@ -528,6 +546,9 @@ export function checkBlockNode(node: Node) {
         case "TH":
         case "TD":
             return true;
+        case "DIV":
+            // don't treat math nodes as block nodes for deserializing
+            return !(node as HTMLElement).hasAttribute("data-mx-maths");
         default:
             return false;
     }

src/Lifecycle.ts

@@ -49,6 +49,7 @@ import {SSO_HOMESERVER_URL_KEY, SSO_ID_SERVER_URL_KEY} from "./BasePlatform";
 import ThreepidInviteStore from "./stores/ThreepidInviteStore";
 import CountlyAnalytics from "./CountlyAnalytics";
 import CallHandler from './CallHandler';
+import LifecycleCustomisations from "./customisations/Lifecycle";
 
 const HOMESERVER_URL_KEY = "mx_hs_url";
 const ID_SERVER_URL_KEY = "mx_is_url";
@@ -716,6 +717,7 @@ export async function onLoggedOut(): Promise<void> {
     dis.dispatch({action: 'on_logged_out'}, true);
     stopMatrixClient();
     await clearStorage({deleteEverything: true});
+    LifecycleCustomisations.onLoggedOutAndStorageCleared?.();
 }
 
 /**

src/Markdown.js

@@ -23,6 +23,11 @@ const ALLOWED_HTML_TAGS = ['sub', 'sup', 'del', 'u'];
 const TEXT_NODES = ['text', 'softbreak', 'linebreak', 'paragraph', 'document'];
 
 function is_allowed_html_tag(node) {
+    if (node.literal != null &&
+        node.literal.match('^<((div|span) data-mx-maths="[^"]*"|\/(div|span))>$') != null) {
+        return true;
+    }
+
     // Regex won't work for tags with attrs, but we only
     // allow <del> anyway.
     const matches = /^<\/?(.*)>$/.exec(node.literal);
@@ -30,6 +35,7 @@ function is_allowed_html_tag(node) {
         const tag = matches[1];
         return ALLOWED_HTML_TAGS.indexOf(tag) > -1;
     }
+
     return false;
 }
 

src/components/views/settings/ChangePassword.js

@@ -21,9 +21,18 @@ import PropTypes from 'prop-types';
 import {MatrixClientPeg} from "../../../MatrixClientPeg";
 import AccessibleButton from '../elements/AccessibleButton';
 import Spinner from '../elements/Spinner';
+import withValidation from '../elements/Validation';
 import { _t } from '../../../languageHandler';
 import * as sdk from "../../../index";
 import Modal from "../../../Modal";
+import PassphraseField from "../auth/PassphraseField";
+import CountlyAnalytics from "../../../CountlyAnalytics";
+
+const FIELD_OLD_PASSWORD = 'field_old_password';
+const FIELD_NEW_PASSWORD = 'field_new_password';
+const FIELD_NEW_PASSWORD_CONFIRM = 'field_new_password_confirm';
+
+const PASSWORD_MIN_SCORE = 3; // safely unguessable: moderate protection from offline slow-hash scenario.
 
 export default class ChangePassword extends React.Component {
     static propTypes = {
@@ -63,6 +72,7 @@ export default class ChangePassword extends React.Component {
     }
 
     state = {
+        fieldValid: {},
         phase: ChangePassword.Phases.Edit,
         oldPassword: "",
         newPassword: "",
@@ -168,26 +178,84 @@ export default class ChangePassword extends React.Component {
         );
     };
 
+    markFieldValid(fieldID, valid) {
+        const { fieldValid } = this.state;
+        fieldValid[fieldID] = valid;
+        this.setState({
+            fieldValid,
+        });
+    }
+
     onChangeOldPassword = (ev) => {
         this.setState({
             oldPassword: ev.target.value,
         });
     };
 
+    onOldPasswordValidate = async fieldState => {
+        const result = await this.validateOldPasswordRules(fieldState);
+        this.markFieldValid(FIELD_OLD_PASSWORD, result.valid);
+        return result;
+    };
+
+    validateOldPasswordRules = withValidation({
+        rules: [
+            {
+                key: "required",
+                test: ({ value, allowEmpty }) => allowEmpty || !!value,
+                invalid: () => _t("Passwords can't be empty"),
+            },
+         ],
+    });
+
     onChangeNewPassword = (ev) => {
         this.setState({
             newPassword: ev.target.value,
         });
     };
 
+    onNewPasswordValidate = result => {
+        this.markFieldValid(FIELD_NEW_PASSWORD, result.valid);
+    };
+
     onChangeNewPasswordConfirm = (ev) => {
         this.setState({
             newPasswordConfirm: ev.target.value,
         });
     };
 
-    onClickChange = (ev) => {
+    onNewPasswordConfirmValidate = async fieldState => {
+        const result = await this.validatePasswordConfirmRules(fieldState);
+        this.markFieldValid(FIELD_NEW_PASSWORD_CONFIRM, result.valid);
+        return result;
+    };
+
+    validatePasswordConfirmRules = withValidation({
+        rules: [
+            {
+                key: "required",
+                test: ({ value, allowEmpty }) => allowEmpty || !!value,
+                invalid: () => _t("Confirm password"),
+            },
+            {
+                key: "match",
+                test({ value }) {
+                    return !value || value === this.state.newPassword;
+                },
+                invalid: () => _t("Passwords don't match"),
+            },
+         ],
+    });
+
+    onClickChange = async (ev) => {
         ev.preventDefault();
+
+        const allFieldsValid = await this.verifyFieldsBeforeSubmit();
+        if (!allFieldsValid) {
+            CountlyAnalytics.instance.track("onboarding_registration_submit_failed");
+            return;
+        }
+
         const oldPassword = this.state.oldPassword;
         const newPassword = this.state.newPassword;
         const confirmPassword = this.state.newPasswordConfirm;
@@ -201,9 +269,75 @@ export default class ChangePassword extends React.Component {
         }
     };
 
-    render() {
+    async verifyFieldsBeforeSubmit() {
-        // TODO: Live validation on `new pw == confirm pw`
+        // Blur the active element if any, so we first run its blur validation,
+        // which is less strict than the pass we're about to do below for all fields.
+        const activeElement = document.activeElement;
+        if (activeElement) {
+            activeElement.blur();
+        }
 
+        const fieldIDsInDisplayOrder = [
+            FIELD_OLD_PASSWORD,
+            FIELD_NEW_PASSWORD,
+            FIELD_NEW_PASSWORD_CONFIRM,
+        ];
+
+        // Run all fields with stricter validation that no longer allows empty
+        // values for required fields.
+        for (const fieldID of fieldIDsInDisplayOrder) {
+            const field = this[fieldID];
+            if (!field) {
+                continue;
+            }
+            // We must wait for these validations to finish before queueing
+            // up the setState below so our setState goes in the queue after
+            // all the setStates from these validate calls (that's how we
+            // know they've finished).
+            await field.validate({ allowEmpty: false });
+        }
+
+        // Validation and state updates are async, so we need to wait for them to complete
+        // first. Queue a `setState` callback and wait for it to resolve.
+        await new Promise(resolve => this.setState({}, resolve));
+
+        if (this.allFieldsValid()) {
+            return true;
+        }
+
+        const invalidField = this.findFirstInvalidField(fieldIDsInDisplayOrder);
+
+        if (!invalidField) {
+            return true;
+        }
+
+        // Focus the first invalid field and show feedback in the stricter mode
+        // that no longer allows empty values for required fields.
+        invalidField.focus();
+        invalidField.validate({ allowEmpty: false, focused: true });
+        return false;
+    }
+
+    allFieldsValid() {
+        const keys = Object.keys(this.state.fieldValid);
+        for (let i = 0; i < keys.length; ++i) {
+            if (!this.state.fieldValid[keys[i]]) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    findFirstInvalidField(fieldIDs) {
+        for (const fieldID of fieldIDs) {
+            if (!this.state.fieldValid[fieldID] && this[fieldID]) {
+                return this[fieldID];
+            }
+        }
+        return null;
+    }
+
+    render() {
         const rowClassName = this.props.rowClassName;
         const buttonClassName = this.props.buttonClassName;
 
@@ -213,28 +347,35 @@ export default class ChangePassword extends React.Component {
                     <form className={this.props.className} onSubmit={this.onClickChange}>
                         <div className={rowClassName}>
                             <Field
+                                ref={field => this[FIELD_OLD_PASSWORD] = field}
                                 type="password"
                                 label={_t('Current password')}
                                 value={this.state.oldPassword}
                                 onChange={this.onChangeOldPassword}
+                                onValidate={this.onOldPasswordValidate}
                             />
                         </div>
                         <div className={rowClassName}>
-                            <Field
+                            <PassphraseField
+                                fieldRef={field => this[FIELD_NEW_PASSWORD] = field}
                                 type="password"
-                                label={_t('New Password')}
+                                label='New Password'
+                                minScore={PASSWORD_MIN_SCORE}
                                 value={this.state.newPassword}
                                 autoFocus={this.props.autoFocusNewPasswordInput}
                                 onChange={this.onChangeNewPassword}
+                                onValidate={this.onNewPasswordValidate}
                                 autoComplete="new-password"
                             />
                         </div>
                         <div className={rowClassName}>
                             <Field
+                                ref={field => this[FIELD_NEW_PASSWORD_CONFIRM] = field}
                                 type="password"
                                 label={_t("Confirm password")}
                                 value={this.state.newPasswordConfirm}
                                 onChange={this.onChangeNewPasswordConfirm}
+                                onValidate={this.onNewPasswordConfirmValidate}
                                 autoComplete="new-password"
                             />
                         </div>

src/customisations/Lifecycle.ts

@@ -0,0 +1,30 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+function onLoggedOutAndStorageCleared(): void {
+    // E.g. redirect user or call other APIs after logout
+}
+
+// This interface summarises all available customisation points and also marks
+// them all as optional. This allows customisers to only define and export the
+// customisations they need while still maintaining type safety.
+export interface ILifecycleCustomisations {
+    onLoggedOutAndStorageCleared?: typeof onLoggedOutAndStorageCleared;
+}
+
+// A real customisation module will define and export one or more of the
+// customisation points that make up `ILifecycleCustomisations`.
+export default {} as ILifecycleCustomisations;

src/customisations/Security.ts

@@ -67,24 +67,13 @@ function setupEncryptionNeeded(kind: SetupEncryptionKind): boolean {
 // them all as optional. This allows customisers to only define and export the
 // customisations they need while still maintaining type safety.
 export interface ISecurityCustomisations {
-    examineLoginResponse?: (
+    examineLoginResponse?: typeof examineLoginResponse;
-        response: any,
+    persistCredentials?: typeof persistCredentials;
-        credentials: IMatrixClientCreds,
+    createSecretStorageKey?: typeof createSecretStorageKey,
-    ) => void;
+    getSecretStorageKey?: typeof getSecretStorageKey,
-    persistCredentials?: (
+    catchAccessSecretStorageError?: typeof catchAccessSecretStorageError,
-        credentials: IMatrixClientCreds,
+    setupEncryptionNeeded?: typeof setupEncryptionNeeded,
-    ) => void;
+    getDehydrationKey?: typeof getDehydrationKey,
-    createSecretStorageKey?: () => Uint8Array,
-    getSecretStorageKey?: () => Uint8Array,
-    catchAccessSecretStorageError?: (
-        e: Error,
-    ) => void,
-    setupEncryptionNeeded?: (
-        kind: SetupEncryptionKind,
-    ) => boolean,
-    getDehydrationKey?: (
-        keyInfo: ISecretStorageKeyInfo,
-    ) => Promise<Uint8Array>,
 }
 
 // A real customisation module will define and export one or more of the

src/editor/deserialize.ts

@@ -21,6 +21,7 @@ import { walkDOMDepthFirst } from "./dom";
 import { checkBlockNode } from "../HtmlUtils";
 import { getPrimaryPermalinkEntity } from "../utils/permalinks/Permalinks";
 import { PartCreator } from "./parts";
+import SdkConfig from "../SdkConfig";
 
 function parseAtRoomMentions(text: string, partCreator: PartCreator) {
     const ATROOM = "@room";
@@ -130,6 +131,23 @@ function parseElement(n: HTMLElement, partCreator: PartCreator, lastNode: HTMLEl
             }
             break;
         }
+        case "DIV":
+        case "SPAN": {
+            // math nodes are translated back into delimited latex strings
+            if (n.hasAttribute("data-mx-maths")) {
+                const delimLeft = (n.nodeName == "SPAN") ?
+                    (SdkConfig.get()['latex_maths_delims'] || {})['inline_left'] || "$" :
+                    (SdkConfig.get()['latex_maths_delims'] || {})['display_left'] || "$$";
+                const delimRight = (n.nodeName == "SPAN") ?
+                    (SdkConfig.get()['latex_maths_delims'] || {})['inline_right'] || "$" :
+                    (SdkConfig.get()['latex_maths_delims'] || {})['display_right'] || "$$";
+                const tex = n.getAttribute("data-mx-maths");
+                return partCreator.plain(delimLeft + tex + delimRight);
+            } else if (!checkDescendInto(n)) {
+                return partCreator.plain(n.textContent);
+            }
+            break;
+        }
         case "OL":
             state.listIndex.push((<HTMLOListElement>n).start || 1);
             /* falls through */

src/editor/serialize.ts

@@ -18,6 +18,10 @@ limitations under the License.
 import Markdown from '../Markdown';
 import {makeGenericPermalink} from "../utils/permalinks/Permalinks";
 import EditorModel from "./model";
+import { AllHtmlEntities } from 'html-entities';
+import SettingsStore from '../settings/SettingsStore';
+import SdkConfig from '../SdkConfig';
+import cheerio from 'cheerio';
 
 export function mdSerialize(model: EditorModel) {
     return model.parts.reduce((html, part) => {
@@ -38,10 +42,43 @@ export function mdSerialize(model: EditorModel) {
 }
 
 export function htmlSerializeIfNeeded(model: EditorModel, {forceHTML = false} = {}) {
-    const md = mdSerialize(model);
+    let md = mdSerialize(model);
+
+    if (SettingsStore.getValue("feature_latex_maths")) {
+        const displayPattern = (SdkConfig.get()['latex_maths_delims'] || {})['display_pattern'] ||
+            "\\$\\$(([^$]|\\\\\\$)*)\\$\\$";
+        const inlinePattern = (SdkConfig.get()['latex_maths_delims'] || {})['inline_pattern'] ||
+            "\\$(([^$]|\\\\\\$)*)\\$";
+
+        md = md.replace(RegExp(displayPattern, "gm"), function(m, p1) {
+            const p1e = AllHtmlEntities.encode(p1);
+            return `<div data-mx-maths="${p1e}">\n\n</div>\n\n`;
+        });
+
+        md = md.replace(RegExp(inlinePattern, "gm"), function(m, p1) {
+            const p1e = AllHtmlEntities.encode(p1);
+            return `<span data-mx-maths="${p1e}"></span>`;
+        });
+
+        // make sure div tags always start on a new line, otherwise it will confuse
+        // the markdown parser
+        md = md.replace(/(.)<div/g, function(m, p1) { return `${p1}\n<div`; });
+    }
+
     const parser = new Markdown(md);
     if (!parser.isPlainText() || forceHTML) {
-        return parser.toHTML();
+        // feed Markdown output to HTML parser
+        const phtml = cheerio.load(parser.toHTML(),
+            { _useHtmlParser2: true, decodeEntities: false })
+
+        // add fallback output for latex math, which should not be interpreted as markdown
+        phtml('div, span').each(function(i, e) {
+            const tex = phtml(e).attr('data-mx-maths')
+            if (tex) {
+                phtml(e).html(`<code>${tex}</code>`)
+            }
+        });
+        return phtml.html();
     }
     // ensure removal of escape backslashes in non-Markdown messages
     if (md.indexOf("\\") > -1) {

src/i18n/strings/en_EN.json

@@ -755,6 +755,7 @@
     "%(senderName)s: %(reaction)s": "%(senderName)s: %(reaction)s",
     "%(senderName)s: %(stickerName)s": "%(senderName)s: %(stickerName)s",
     "Change notification settings": "Change notification settings",
+    "Render LaTeX maths in messages": "Render LaTeX maths in messages",
     "Communities v2 prototypes. Requires compatible homeserver. Highly experimental - use with caution.": "Communities v2 prototypes. Requires compatible homeserver. Highly experimental - use with caution.",
     "New spinner design": "New spinner design",
     "Message Pinning": "Message Pinning",
@@ -954,9 +955,9 @@
     "Changing password will currently reset any end-to-end encryption keys on all sessions, making encrypted chat history unreadable, unless you first export your room keys and re-import them afterwards. In future this will be improved.": "Changing password will currently reset any end-to-end encryption keys on all sessions, making encrypted chat history unreadable, unless you first export your room keys and re-import them afterwards. In future this will be improved.",
     "Export E2E room keys": "Export E2E room keys",
     "Do you want to set an email address?": "Do you want to set an email address?",
-    "Current password": "Current password",
-    "New Password": "New Password",
     "Confirm password": "Confirm password",
+    "Passwords don't match": "Passwords don't match",
+    "Current password": "Current password",
     "Change Password": "Change Password",
     "Your homeserver does not support cross-signing.": "Your homeserver does not support cross-signing.",
     "Cross-signing is ready for use.": "Cross-signing is ready for use.",
@@ -2303,7 +2304,6 @@
     "If you don't specify an email address, you won't be able to reset your password. Are you sure?": "If you don't specify an email address, you won't be able to reset your password. Are you sure?",
     "Use an email address to recover your account": "Use an email address to recover your account",
     "Enter email address (required on this homeserver)": "Enter email address (required on this homeserver)",
-    "Passwords don't match": "Passwords don't match",
     "Other users can invite you to rooms using your contact details": "Other users can invite you to rooms using your contact details",
     "Enter phone number (required on this homeserver)": "Enter phone number (required on this homeserver)",
     "Use lowercase letters, numbers, dashes and underscores only": "Use lowercase letters, numbers, dashes and underscores only",
@@ -2489,6 +2489,7 @@
     "Your Matrix account on <underlinedServerName />": "Your Matrix account on <underlinedServerName />",
     "No identity server is configured: add one in server settings to reset your password.": "No identity server is configured: add one in server settings to reset your password.",
     "Sign in instead": "Sign in instead",
+    "New Password": "New Password",
     "A verification email will be sent to your inbox to confirm setting your new password.": "A verification email will be sent to your inbox to confirm setting your new password.",
     "Send Reset Email": "Send Reset Email",
     "An email has been sent to %(emailAddress)s. Once you've followed the link it contains, click below.": "An email has been sent to %(emailAddress)s. Once you've followed the link it contains, click below.",

src/settings/Settings.ts

@@ -117,6 +117,12 @@ export interface ISetting {
 }
 
 export const SETTINGS: {[setting: string]: ISetting} = {
+    "feature_latex_maths": {
+        isFeature: true,
+        displayName: _td("Render LaTeX maths in messages"),
+        supportedLevels: LEVELS_FEATURE,
+        default: false,
+    },
     "feature_communities_v2_prototypes": {
         isFeature: true,
         displayName: _td(

test/components/views/messages/TextualBody-test.js

@@ -36,6 +36,7 @@ describe("<TextualBody />", () => {
         MatrixClientPeg.matrixClient = {
             getRoom: () => mkStubRoom("room_id"),
             getAccountData: () => undefined,
+            isGuest: () => false,
         };
 
         const ev = mkEvent({
@@ -59,6 +60,7 @@ describe("<TextualBody />", () => {
         MatrixClientPeg.matrixClient = {
             getRoom: () => mkStubRoom("room_id"),
             getAccountData: () => undefined,
+            isGuest: () => false,
         };
 
         const ev = mkEvent({
@@ -83,6 +85,7 @@ describe("<TextualBody />", () => {
             MatrixClientPeg.matrixClient = {
                 getRoom: () => mkStubRoom("room_id"),
                 getAccountData: () => undefined,
+                isGuest: () => false,
             };
         });
 
@@ -135,6 +138,7 @@ describe("<TextualBody />", () => {
                 getHomeserverUrl: () => "https://my_server/",
                 on: () => undefined,
                 removeListener: () => undefined,
+                isGuest: () => false,
             };
         });
 

yarn.lock

@@ -6206,6 +6206,13 @@ jsx-ast-utils@^2.4.1:
     array-includes "^3.1.1"
     object.assign "^4.1.0"
 
+katex@^0.12.0:
+  version "0.12.0"
+  resolved "https://registry.yarnpkg.com/katex/-/katex-0.12.0.tgz#2fb1c665dbd2b043edcf8a1f5c555f46beaa0cb9"
+  integrity sha512-y+8btoc/CK70XqcHqjxiGWBOeIL8upbS0peTPXTvgrh21n1RiWWcIpSWM+4uXq+IAgNh9YYQWdc7LVDPDAEEAg==
+  dependencies:
+    commander "^2.19.0"
+
 kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0:
   version "3.2.2"
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64"