Switch from cheerio to DOMParser (#10929)

* Add tests around feature_latex_maths

* Switch from cheerio to DOMParser

* strict

* Iterate

package.json

@@ -68,7 +68,6 @@
         "@testing-library/react-hooks": "^8.0.1",
         "await-lock": "^2.1.0",
         "blurhash": "^1.1.3",
-        "cheerio": "^1.0.0-rc.9",
         "classnames": "^2.2.6",
         "commonmark": "^0.30.0",
         "counterpart": "^0.18.6",

src/HtmlUtils.tsx

@@ -19,7 +19,6 @@ limitations under the License.
 
 import React, { LegacyRef, ReactElement, ReactNode } from "react";
 import sanitizeHtml from "sanitize-html";
-import { load as cheerio } from "cheerio";
 import classNames from "classnames";
 import EMOJIBASE_REGEX from "emojibase-regex";
 import { merge, split } from "lodash";
@@ -549,27 +548,19 @@ export function bodyToHtml(content: IContent, highlights: Optional<string[]>, op
             }
 
             safeBody = sanitizeHtml(formattedBody!, sanitizeParams);
-            const phtml = cheerio(safeBody, {
+            const phtml = new DOMParser().parseFromString(safeBody, "text/html");
-                // @ts-ignore: The `_useHtmlParser2` internal option is the
+            const isPlainText = phtml.body.innerHTML === phtml.body.textContent;
-                // simplest way to both parse and render using `htmlparser2`.
-                _useHtmlParser2: true,
-                decodeEntities: false,
-            });
-            const isPlainText = phtml.html() === phtml.root().text();
             isHtmlMessage = !isPlainText;
 
             if (isHtmlMessage && SettingsStore.getValue("feature_latex_maths")) {
-                // @ts-ignore - The types for `replaceWith` wrongly expect
+                [...phtml.querySelectorAll<HTMLElement>("div, span[data-mx-maths]")].forEach((e) => {
-                // Cheerio instance to be returned.
+                    e.outerHTML = katex.renderToString(decode(e.getAttribute("data-mx-maths")), {
-                phtml('div, span[data-mx-maths!=""]').replaceWith(function (i, e) {
-                    return katex.renderToString(decode(phtml(e).attr("data-mx-maths")), {
                         throwOnError: false,
-                        // @ts-ignore - `e` can be an Element, not just a Node
+                        displayMode: e.tagName == "DIV",
-                        displayMode: e.name == "div",
                         output: "htmlAndMathml",
                     });
                 });
-                safeBody = phtml.html();
+                safeBody = phtml.body.innerHTML;
             }
         } else if (highlighter) {
             safeBody = highlighter.applyHighlights(escapeHtml(plainBody), safeHighlights!).join("");

src/editor/serialize.ts

@@ -16,7 +16,6 @@ limitations under the License.
 */
 
 import { encode } from "html-entities";
-import { load as cheerio } from "cheerio";
 import escapeHtml from "escape-html";
 
 import Markdown from "../Markdown";
@@ -133,8 +132,7 @@ export function htmlSerializeFromMdIfNeeded(md: string, { forceHTML = false } =
             });
         });
 
-        // make sure div tags always start on a new line, otherwise it will confuse
+        // make sure div tags always start on a new line, otherwise it will confuse the markdown parser
-        // the markdown parser
         md = md.replace(/(.)<div/g, function (m, p1) {
             return `${p1}\n<div`;
         });
@@ -143,39 +141,29 @@ export function htmlSerializeFromMdIfNeeded(md: string, { forceHTML = false } =
     const parser = new Markdown(md);
     if (!parser.isPlainText() || forceHTML) {
         // feed Markdown output to HTML parser
-        const phtml = cheerio(parser.toHTML(), {
+        const phtml = new DOMParser().parseFromString(parser.toHTML(), "text/html");
-            // @ts-ignore: The `_useHtmlParser2` internal option is the
-            // simplest way to both parse and render using `htmlparser2`.
-            _useHtmlParser2: true,
-            decodeEntities: false,
-        });
 
         if (SettingsStore.getValue("feature_latex_maths")) {
             // original Markdown without LaTeX replacements
             const parserOrig = new Markdown(orig);
-            const phtmlOrig = cheerio(parserOrig.toHTML(), {
+            const phtmlOrig = new DOMParser().parseFromString(parserOrig.toHTML(), "text/html");
-                // @ts-ignore: The `_useHtmlParser2` internal option is the
-                // simplest way to both parse and render using `htmlparser2`.
-                _useHtmlParser2: true,
-                decodeEntities: false,
-            });
 
             // since maths delimiters are handled before Markdown,
             // code blocks could contain mangled content.
             // replace code blocks with original content
-            phtmlOrig("code").each(function (i) {
+            [...phtmlOrig.getElementsByTagName("code")].forEach((e, i) => {
-                phtml("code").eq(i).text(phtmlOrig("code").eq(i).text());
+                phtml.getElementsByTagName("code").item(i)!.textContent = e.textContent;
             });
 
             // add fallback output for latex math, which should not be interpreted as markdown
-            phtml("div, span").each(function (i, e) {
+            [...phtml.querySelectorAll("div, span")].forEach((e, i) => {
-                const tex = phtml(e).attr("data-mx-maths");
+                const tex = e.getAttribute("data-mx-maths");
                 if (tex) {
-                    phtml(e).html(`<code>${tex}</code>`);
+                    e.innerHTML = `<code>${tex}</code>`;
                 }
             });
         }
-        return phtml.html();
+        return phtml.body.innerHTML;
     }
     // ensure removal of escape backslashes in non-Markdown messages
     if (md.indexOf("\\") > -1) {

test/HtmlUtils-test.tsx

@@ -131,4 +131,40 @@ describe("bodyToHtml", () => {
 
         expect(asFragment()).toMatchSnapshot();
     });
+
+    describe("feature_latex_maths", () => {
+        beforeEach(() => {
+            jest.spyOn(SettingsStore, "getValue").mockImplementation((feature) => feature === "feature_latex_maths");
+        });
+
+        it("should render inline katex", () => {
+            const html = getHtml({
+                body: "hello \\xi world",
+                msgtype: "m.text",
+                formatted_body: 'hello <span data-mx-maths="\\xi"><code>\\xi</code></span> world',
+                format: "org.matrix.custom.html",
+            });
+            expect(html).toMatchSnapshot();
+        });
+
+        it("should render block katex", () => {
+            const html = getHtml({
+                body: "hello \\xi world",
+                msgtype: "m.text",
+                formatted_body: '<p>hello</p><div data-mx-maths="\\xi"><code>\\xi</code></div><p>world</p>',
+                format: "org.matrix.custom.html",
+            });
+            expect(html).toMatchSnapshot();
+        });
+
+        it("should not mangle code blocks", () => {
+            const html = getHtml({
+                body: "hello \\xi world",
+                msgtype: "m.text",
+                formatted_body: "<p>hello</p><pre><code>$\\xi$</code></pre><p>world</p>",
+                format: "org.matrix.custom.html",
+            });
+            expect(html).toMatchSnapshot();
+        });
+    });
 });

test/__snapshots__/HtmlUtils-test.tsx.snap

@@ -15,3 +15,9 @@ exports[`bodyToHtml should generate big emoji for an emoji-only reply to a messa
   </span>
 </DocumentFragment>
 `;
+
+exports[`bodyToHtml feature_latex_maths should not mangle code blocks 1`] = `"<p>hello</p><pre><code>$\\xi$</code></pre><p>world</p>"`;
+
+exports[`bodyToHtml feature_latex_maths should render block katex 1`] = `"<p>hello</p><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>ξ</mi></mrow><annotation encoding="application/x-tex">\\xi</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8889em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.04601em;">ξ</span></span></span></span></span><p>world</p>"`;
+
+exports[`bodyToHtml feature_latex_maths should render inline katex 1`] = `"hello <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>ξ</mi></mrow><annotation encoding="application/x-tex">\\xi</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8889em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.04601em;">ξ</span></span></span></span> world"`;

test/editor/serialize-test.ts

@@ -17,6 +17,7 @@ limitations under the License.
 import EditorModel from "../../src/editor/model";
 import { htmlSerializeIfNeeded } from "../../src/editor/serialize";
 import { createPartCreator } from "./mock";
+import SettingsStore from "../../src/settings/SettingsStore";
 
 describe("editor/serialize", function () {
     describe("with markdown", function () {
@@ -75,6 +76,7 @@ describe("editor/serialize", function () {
             expect(html).toBe("*hello* world < hey world!");
         });
     });
+
     describe("with plaintext", function () {
         it("markdown remains plaintext", function () {
             const pc = createPartCreator();
@@ -102,4 +104,42 @@ describe("editor/serialize", function () {
             expect(html).toBe("hello world");
         });
     });
+
+    describe("feature_latex_maths", () => {
+        beforeEach(() => {
+            jest.spyOn(SettingsStore, "getValue").mockImplementation((feature) => feature === "feature_latex_maths");
+        });
+
+        it("should support inline katex", () => {
+            const pc = createPartCreator();
+            const model = new EditorModel([pc.plain("hello $\\xi$ world")], pc);
+            const html = htmlSerializeIfNeeded(model, {});
+            expect(html).toMatchInlineSnapshot(`"hello <span data-mx-maths="\\xi"><code>\\xi</code></span> world"`);
+        });
+
+        it("should support block katex", () => {
+            const pc = createPartCreator();
+            const model = new EditorModel([pc.plain("hello \n$$\\xi$$\n world")], pc);
+            const html = htmlSerializeIfNeeded(model, {});
+            expect(html).toMatchInlineSnapshot(`
+                "<p>hello</p>
+                <div data-mx-maths="\\xi"><code>\\xi</code></div>
+                <p>world</p>
+                "
+            `);
+        });
+
+        it("should not mangle code blocks", () => {
+            const pc = createPartCreator();
+            const model = new EditorModel([pc.plain("hello\n```\n$\\xi$\n```\nworld")], pc);
+            const html = htmlSerializeIfNeeded(model, {});
+            expect(html).toMatchInlineSnapshot(`
+                "<p>hello</p>
+                <pre><code>$\\xi$
+                </code></pre>
+                <p>world</p>
+                "
+            `);
+        });
+    });
 });

yarn.lock

@@ -3028,11 +3028,6 @@ blurhash@^1.1.3:
   resolved "https://registry.yarnpkg.com/blurhash/-/blurhash-1.1.5.tgz#3034104cd5dce5a3e5caa871ae2f0f1f2d0ab566"
   integrity sha512-a+LO3A2DfxTaTztsmkbLYmUzUeApi0LZuKalwbNmqAHR6HhJGMt1qSV/R3wc+w4DL28holjqO3Bg74aUGavGjg==
 
-boolbase@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e"
-  integrity sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==
-
 brace-expansion@^1.1.7:
   version "1.1.11"
   resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
@@ -3189,31 +3184,6 @@ check-more-types@^2.24.0:
   resolved "https://registry.yarnpkg.com/check-more-types/-/check-more-types-2.24.0.tgz#1420ffb10fd444dcfc79b43891bbfffd32a84600"
   integrity sha512-Pj779qHxV2tuapviy1bSZNEL1maXr13bPYpsvSDB68HlYcYuhlDrmGd63i0JHMCLKzc7rUSNIrpdJlhVlNwrxA==
 
-cheerio-select@^2.1.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/cheerio-select/-/cheerio-select-2.1.0.tgz#4d8673286b8126ca2a8e42740d5e3c4884ae21b4"
-  integrity sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==
-  dependencies:
-    boolbase "^1.0.0"
-    css-select "^5.1.0"
-    css-what "^6.1.0"
-    domelementtype "^2.3.0"
-    domhandler "^5.0.3"
-    domutils "^3.0.1"
-
-cheerio@^1.0.0-rc.9:
-  version "1.0.0-rc.12"
-  resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-1.0.0-rc.12.tgz#788bf7466506b1c6bf5fae51d24a2c4d62e47683"
-  integrity sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==
-  dependencies:
-    cheerio-select "^2.1.0"
-    dom-serializer "^2.0.0"
-    domhandler "^5.0.3"
-    domutils "^3.0.1"
-    htmlparser2 "^8.0.1"
-    parse5 "^7.0.0"
-    parse5-htmlparser2-tree-adapter "^7.0.0"
-
 chokidar@^3.4.0, chokidar@^3.5.1:
   version "3.5.3"
   resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd"
@@ -3519,17 +3489,6 @@ css-functions-list@^3.1.0:
   resolved "https://registry.yarnpkg.com/css-functions-list/-/css-functions-list-3.1.0.tgz#cf5b09f835ad91a00e5959bcfc627cd498e1321b"
   integrity sha512-/9lCvYZaUbBGvYUgYGFJ4dcYiyqdhSjG7IPVluoV8A1ILjkF7ilmhp1OGUz8n+nmBcu0RNrQAzgD8B6FJbrt2w==
 
-css-select@^5.1.0:
-  version "5.1.0"
-  resolved "https://registry.yarnpkg.com/css-select/-/css-select-5.1.0.tgz#b8ebd6554c3637ccc76688804ad3f6a6fdaea8a6"
-  integrity sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==
-  dependencies:
-    boolbase "^1.0.0"
-    css-what "^6.1.0"
-    domhandler "^5.0.2"
-    domutils "^3.0.1"
-    nth-check "^2.0.1"
-
 css-tree@^2.3.1:
   version "2.3.1"
   resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-2.3.1.tgz#10264ce1e5442e8572fc82fbe490644ff54b5c20"
@@ -3538,11 +3497,6 @@ css-tree@^2.3.1:
     mdn-data "2.0.30"
     source-map-js "^1.0.1"
 
-css-what@^6.1.0:
-  version "6.1.0"
-  resolved "https://registry.yarnpkg.com/css-what/-/css-what-6.1.0.tgz#fb5effcf76f1ddea2c81bdfaa4de44e79bac70f4"
-  integrity sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==
-
 css.escape@^1.5.1:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/css.escape/-/css.escape-1.5.1.tgz#42e27d4fa04ae32f931a4b4d4191fa9cddee97cb"
@@ -5080,7 +5034,7 @@ html-tags@^3.3.1:
   resolved "https://registry.yarnpkg.com/html-tags/-/html-tags-3.3.1.tgz#a04026a18c882e4bba8a01a3d39cfe465d40b5ce"
   integrity sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ==
 
-htmlparser2@^8.0.0, htmlparser2@^8.0.1:
+htmlparser2@^8.0.0:
   version "8.0.2"
   resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-8.0.2.tgz#f002151705b383e62433b5cf466f5b716edaec21"
   integrity sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==
@@ -6767,13 +6721,6 @@ npm-run-path@^4.0.0, npm-run-path@^4.0.1:
   dependencies:
     path-key "^3.0.0"
 
-nth-check@^2.0.1:
-  version "2.1.1"
-  resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.1.1.tgz#c9eab428effce36cd6b92c924bdb000ef1f1ed1d"
-  integrity sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==
-  dependencies:
-    boolbase "^1.0.0"
-
 nwsapi@^2.2.2:
   version "2.2.3"
   resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.3.tgz#00e04dfd5a4a751e5ec2fecdc75dfd2f0db820fa"
@@ -6982,14 +6929,6 @@ parse-srcset@^1.0.2:
   resolved "https://registry.yarnpkg.com/parse-srcset/-/parse-srcset-1.0.2.tgz#f2bd221f6cc970a938d88556abc589caaaa2bde1"
   integrity sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q==
 
-parse5-htmlparser2-tree-adapter@^7.0.0:
-  version "7.0.0"
-  resolved "https://registry.yarnpkg.com/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz#23c2cc233bcf09bb7beba8b8a69d46b08c62c2f1"
-  integrity sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==
-  dependencies:
-    domhandler "^5.0.2"
-    parse5 "^7.0.0"
-
 parse5@^7.0.0, parse5@^7.1.1:
   version "7.1.2"
   resolved "https://registry.yarnpkg.com/parse5/-/parse5-7.1.2.tgz#0736bebbfd77793823240a23b7fc5e010b7f8e32"