Replace i18n generation script with something matching our project

We've been relying on flow being close enough to TypeScript for so long that it is starting to run into issues. Here we switch to babel's parser given we already use babel in the project.

Babel's parser is also *slightly* faster, allowing us to generate strings 0.1s faster.

package.json

@@ -105,6 +105,7 @@
   "devDependencies": {
     "@babel/cli": "^7.10.5",
     "@babel/core": "^7.10.5",
+    "@babel/parser": "^7.11.0",
     "@babel/plugin-proposal-class-properties": "^7.10.4",
     "@babel/plugin-proposal-decorators": "^7.10.5",
     "@babel/plugin-proposal-export-default-from": "^7.10.4",
@@ -117,6 +118,7 @@
     "@babel/preset-react": "^7.10.4",
     "@babel/preset-typescript": "^7.10.4",
     "@babel/register": "^7.10.5",
+    "@babel/traverse": "^7.11.0",
     "@peculiar/webcrypto": "^1.1.2",
     "@types/classnames": "^2.2.10",
     "@types/counterpart": "^0.18.1",
@@ -145,9 +147,7 @@
     "eslint-plugin-flowtype": "^2.50.3",
     "eslint-plugin-react": "^7.20.3",
     "eslint-plugin-react-hooks": "^2.5.1",
-    "estree-walker": "^0.9.0",
     "file-loader": "^3.0.1",
-    "flow-parser": "0.57.3",
     "glob": "^5.0.15",
     "jest": "^24.9.0",
     "jest-canvas-mock": "^2.2.0",

scripts/gen-i18n.js

@@ -18,7 +18,7 @@ limitations under the License.
 
 /**
  * Regenerates the translations en_EN file by walking the source tree and
- * parsing each file with flow-parser. Emits a JSON file with the
+ * parsing each file with the appropriate parser. Emits a JSON file with the
  * translatable strings mapped to themselves in the order they appeared
  * in the files and grouped by the file they appeared in.
  *
@@ -29,8 +29,8 @@ const path = require('path');
 
 const walk = require('walk');
 
-const flowParser = require('flow-parser');
+const parser = require("@babel/parser");
-const estreeWalker = require('estree-walker');
+const traverse = require("@babel/traverse");
 
 const TRANSLATIONS_FUNCS = ['_t', '_td'];
 
@@ -44,17 +44,9 @@ const OUTPUT_FILE = 'src/i18n/strings/en_EN.json';
 // to a project that's actively maintained.
 const SEARCH_PATHS = ['src', 'res'];
 
-const FLOW_PARSER_OPTS = {
-  esproposal_class_instance_fields: true,
-  esproposal_class_static_fields: true,
-  esproposal_decorators: true,
-  esproposal_export_star_as: true,
-  types: true,
-};
-
 function getObjectValue(obj, key) {
     for (const prop of obj.properties) {
-        if (prop.key.type == 'Identifier' && prop.key.name == key) {
+        if (prop.key.type === 'Identifier' && prop.key.name === key) {
             return prop.value;
         }
     }
@@ -62,11 +54,11 @@ function getObjectValue(obj, key) {
 }
 
 function getTKey(arg) {
-    if (arg.type == 'Literal') {
+    if (arg.type === 'Literal' || arg.type === "StringLiteral") {
         return arg.value;
-    } else if (arg.type == 'BinaryExpression' && arg.operator == '+') {
+    } else if (arg.type === 'BinaryExpression' && arg.operator === '+') {
         return getTKey(arg.left) + getTKey(arg.right);
-    } else if (arg.type == 'TemplateLiteral') {
+    } else if (arg.type === 'TemplateLiteral') {
         return arg.quasis.map((q) => {
             return q.value.raw;
         }).join('');
@@ -110,81 +102,112 @@ function getFormatStrings(str) {
 }
 
 function getTranslationsJs(file) {
-    const tree = flowParser.parse(fs.readFileSync(file, { encoding: 'utf8' }), FLOW_PARSER_OPTS);
+    const contents = fs.readFileSync(file, { encoding: 'utf8' });
 
     const trs = new Set();
 
-    estreeWalker.walk(tree, {
+    try {
-        enter: function(node, parent) {
+        const plugins = [
-            if (
+            // https://babeljs.io/docs/en/babel-parser#plugins
-                node.type == 'CallExpression' &&
+            "classProperties",
-                TRANSLATIONS_FUNCS.includes(node.callee.name)
+            "objectRestSpread",
-            ) {
+            "throwExpressions",
-                const tKey = getTKey(node.arguments[0]);
+            "exportDefaultFrom",
-                // This happens whenever we call _t with non-literals (ie. whenever we've
+            "decorators-legacy",
-                // had to use a _td to compensate) so is expected.
+        ];
-                if (tKey === null) return;
 
-                // check the format string against the args
+        if (file.endsWith(".js") || file.endsWith(".jsx")) {
-                // We only check _t: _td has no args
+            // all JS is assumed to be flow or react
-                if (node.callee.name === '_t') {
+            plugins.push("flow", "jsx");
-                    try {
+        } else if (file.endsWith(".ts")) {
-                        const placeholders = getFormatStrings(tKey);
+            // TS can't use JSX unless it's a TSX file (otherwise angle casts fail)
-                        for (const placeholder of placeholders) {
+            plugins.push("typescript");
-                            if (node.arguments.length < 2) {
+        } else if (file.endsWith(".tsx")) {
-                                throw new Error(`Placeholder found ('${placeholder}') but no substitutions given`);
+            // When the file is a TSX file though, enable JSX parsing
-                            }
+            plugins.push("typescript", "jsx");
-                            const value = getObjectValue(node.arguments[1], placeholder);
+        }
-                            if (value === null) {
-                                throw new Error(`No value found for placeholder '${placeholder}'`);
-                            }
-                        }
 
-                        // Validate tag replacements
+        const babelParsed = parser.parse(contents, {
-                        if (node.arguments.length > 2) {
+            allowImportExportEverywhere: true,
-                            const tagMap = node.arguments[2];
+            errorRecovery: true,
-                            for (const prop of tagMap.properties || []) {
+            sourceFilename: file,
-                                if (prop.key.type === 'Literal') {
+            tokens: true,
-                                    const tag = prop.key.value;
+            plugins,
-                                    // RegExp same as in src/languageHandler.js
+        });
-                                    const regexp = new RegExp(`(<${tag}>(.*?)<\\/${tag}>|<${tag}>|<${tag}\\s*\\/>)`);
+        traverse.default(babelParsed, {
-                                    if (!tKey.match(regexp)) {
+            enter: (p) => {
-                                        throw new Error(`No match for ${regexp} in ${tKey}`);
+                const node = p.node;
+                if (p.isCallExpression() && node.callee && TRANSLATIONS_FUNCS.includes(node.callee.name)) {
+                    const tKey = getTKey(node.arguments[0]);
+
+                    // This happens whenever we call _t with non-literals (ie. whenever we've
+                    // had to use a _td to compensate) so is expected.
+                    if (tKey === null) return;
+
+                    // check the format string against the args
+                    // We only check _t: _td has no args
+                    if (node.callee.name === '_t') {
+                        try {
+                            const placeholders = getFormatStrings(tKey);
+                            for (const placeholder of placeholders) {
+                                if (node.arguments.length < 2) {
+                                    throw new Error(`Placeholder found ('${placeholder}') but no substitutions given`);
+                                }
+                                const value = getObjectValue(node.arguments[1], placeholder);
+                                if (value === null) {
+                                    throw new Error(`No value found for placeholder '${placeholder}'`);
+                                }
+                            }
+
+                            // Validate tag replacements
+                            if (node.arguments.length > 2) {
+                                const tagMap = node.arguments[2];
+                                for (const prop of tagMap.properties || []) {
+                                    if (prop.key.type === 'Literal') {
+                                        const tag = prop.key.value;
+                                        // RegExp same as in src/languageHandler.js
+                                        const regexp = new RegExp(`(<${tag}>(.*?)<\\/${tag}>|<${tag}>|<${tag}\\s*\\/>)`);
+                                        if (!tKey.match(regexp)) {
+                                            throw new Error(`No match for ${regexp} in ${tKey}`);
+                                        }
                                     }
                                 }
                             }
-                        }
 
-                    } catch (e) {
+                        } catch (e) {
-                        console.log();
+                            console.log();
-                        console.error(`ERROR: ${file}:${node.loc.start.line} ${tKey}`);
+                            console.error(`ERROR: ${file}:${node.loc.start.line} ${tKey}`);
-                        console.error(e);
+                            console.error(e);
-                        process.exit(1);
+                            process.exit(1);
-                    }
-                }
-
-                let isPlural = false;
-                if (node.arguments.length > 1 && node.arguments[1].type == 'ObjectExpression') {
-                    const countVal = getObjectValue(node.arguments[1], 'count');
-                    if (countVal) {
-                        isPlural = true;
-                    }
-                }
-
-                if (isPlural) {
-                    trs.add(tKey + "|other");
-                    const plurals = enPlurals[tKey];
-                    if (plurals) {
-                        for (const pluralType of Object.keys(plurals)) {
-                            trs.add(tKey + "|" + pluralType);
                         }
                     }
-                } else {
+
-                    trs.add(tKey);
+                    let isPlural = false;
+                    if (node.arguments.length > 1 && node.arguments[1].type === 'ObjectExpression') {
+                        const countVal = getObjectValue(node.arguments[1], 'count');
+                        if (countVal) {
+                            isPlural = true;
+                        }
+                    }
+
+                    if (isPlural) {
+                        trs.add(tKey + "|other");
+                        const plurals = enPlurals[tKey];
+                        if (plurals) {
+                            for (const pluralType of Object.keys(plurals)) {
+                                trs.add(tKey + "|" + pluralType);
+                            }
+                        }
+                    } else {
+                        trs.add(tKey);
+                    }
                 }
-            }
+            },
-        }
+        });
-    });
+    } catch (e) {
+        console.error(e);
+        process.exit(1);
+    }
 
     return trs;
 }

yarn.lock

@@ -111,6 +111,15 @@
     jsesc "^2.5.1"
     source-map "^0.5.0"
 
+"@babel/generator@^7.11.0":
+  version "7.11.0"
+  resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.11.0.tgz#4b90c78d8c12825024568cbe83ee6c9af193585c"
+  integrity sha512-fEm3Uzw7Mc9Xi//qU20cBKatTfs2aOtKqmvy/Vm7RkJEGFQ4xc9myCfbXxqK//ZS8MR/ciOHw6meGASJuKmDfQ==
+  dependencies:
+    "@babel/types" "^7.11.0"
+    jsesc "^2.5.1"
+    source-map "^0.5.0"
+
 "@babel/helper-annotate-as-pure@^7.10.1":
   version "7.10.1"
   resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.10.1.tgz#f6d08acc6f70bbd59b436262553fb2e259a1a268"
@@ -400,6 +409,13 @@
   dependencies:
     "@babel/types" "^7.10.4"
 
+"@babel/helper-split-export-declaration@^7.11.0":
+  version "7.11.0"
+  resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.11.0.tgz#f8a491244acf6a676158ac42072911ba83ad099f"
+  integrity sha512-74Vejvp6mHkGE+m+k5vHY93FX2cAtrw1zXrZXRlG4l410Nm9PxfEiVTn1PjDPV5SnmieiueY4AFg2xqhNFuuZg==
+  dependencies:
+    "@babel/types" "^7.11.0"
+
 "@babel/helper-validator-identifier@^7.10.1", "@babel/helper-validator-identifier@^7.10.3":
   version "7.10.3"
   resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.3.tgz#60d9847f98c4cea1b279e005fdb7c28be5412d15"
@@ -466,6 +482,11 @@
   resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.10.5.tgz#e7c6bf5a7deff957cec9f04b551e2762909d826b"
   integrity sha512-wfryxy4bE1UivvQKSQDU4/X6dr+i8bctjUjj8Zyt3DQy7NtPizJXT8M52nqpNKL+nq2PW8lxk4ZqLj0fD4B4hQ==
 
+"@babel/parser@^7.11.0":
+  version "7.11.0"
+  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.11.0.tgz#a9d7e11aead25d3b422d17b2c6502c8dddef6a5d"
+  integrity sha512-qvRvi4oI8xii8NllyEc4MDJjuZiNaRzyb7Y7lup1NqJV8TZHF4O27CcP+72WPn/k1zkgJ6WJfnIbk4jTsVAZHw==
+
 "@babel/plugin-proposal-async-generator-functions@^7.10.4":
   version "7.10.5"
   resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.10.5.tgz#3491cabf2f7c179ab820606cec27fed15e0e8558"
@@ -1213,6 +1234,21 @@
     globals "^11.1.0"
     lodash "^4.17.19"
 
+"@babel/traverse@^7.11.0":
+  version "7.11.0"
+  resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.11.0.tgz#9b996ce1b98f53f7c3e4175115605d56ed07dd24"
+  integrity sha512-ZB2V+LskoWKNpMq6E5UUCrjtDUh5IOTAyIl0dTjIEoXum/iKWkoIEKIRDnUucO6f+2FzNkE0oD4RLKoPIufDtg==
+  dependencies:
+    "@babel/code-frame" "^7.10.4"
+    "@babel/generator" "^7.11.0"
+    "@babel/helper-function-name" "^7.10.4"
+    "@babel/helper-split-export-declaration" "^7.11.0"
+    "@babel/parser" "^7.11.0"
+    "@babel/types" "^7.11.0"
+    debug "^4.1.0"
+    globals "^11.1.0"
+    lodash "^4.17.19"
+
 "@babel/types@^7.0.0", "@babel/types@^7.10.1", "@babel/types@^7.10.2", "@babel/types@^7.3.0", "@babel/types@^7.4.0", "@babel/types@^7.4.4", "@babel/types@^7.7.0":
   version "7.10.2"
   resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.10.2.tgz#30283be31cad0dbf6fb00bd40641ca0ea675172d"
@@ -1231,6 +1267,15 @@
     lodash "^4.17.19"
     to-fast-properties "^2.0.0"
 
+"@babel/types@^7.11.0":
+  version "7.11.0"
+  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.11.0.tgz#2ae6bf1ba9ae8c3c43824e5861269871b206e90d"
+  integrity sha512-O53yME4ZZI0jO1EVGtF1ePGl0LHirG4P1ibcD80XyzZcKhcMFeCXmh4Xb1ifGBIV233Qg12x4rBfQgA+tmOukA==
+  dependencies:
+    "@babel/helper-validator-identifier" "^7.10.4"
+    lodash "^4.17.19"
+    to-fast-properties "^2.0.0"
+
 "@cnakazawa/watch@^1.0.3":
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.4.tgz#f864ae85004d0fcab6f50be9141c4da368d1656a"
@@ -4178,11 +4223,6 @@ estree-walker@^0.6.1:
   resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-0.6.1.tgz#53049143f40c6eb918b23671d1fe3219f3a1b362"
   integrity sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==
 
-estree-walker@^0.9.0:
-  version "0.9.0"
-  resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-0.9.0.tgz#9116372f09c02fd88fcafb0c04343631012a0aa6"
-  integrity sha512-12U47o7XHUX329+x3FzNVjCx3SHEzMF0nkDv7r/HnBzX/xNTKxajBk6gyygaxrAFtLj39219oMfbtxv4KpaOiA==
-
 esutils@^2.0.2:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64"
@@ -4525,11 +4565,6 @@ flatted@^2.0.0:
   resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.2.tgz#4575b21e2bcee7434aa9be662f4b7b5f9c2b5138"
   integrity sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==
 
-flow-parser@0.57.3:
-  version "0.57.3"
-  resolved "https://registry.yarnpkg.com/flow-parser/-/flow-parser-0.57.3.tgz#b8d241a1b1cbae043afa7976e39f269988d8fe34"
-  integrity sha1-uNJBobHLrgQ6+nl2458mmYjY/jQ=
-
 flush-write-stream@^1.0.0:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8"