Replace i18n generation script with something matching our project

We've been relying on flow being close enough to TypeScript for so long that it is starting to run into issues. Here we switch to babel's parser given we already use babel in the project.

Babel's parser is also *slightly* faster, allowing us to generate strings 0.1s faster.

package.json

@@ -105,6 +105,7 @@
   "devDependencies": {
     "@babel/cli": "^7.10.5",
     "@babel/core": "^7.10.5",
+    "@babel/parser": "^7.11.0",
     "@babel/plugin-proposal-class-properties": "^7.10.4",
     "@babel/plugin-proposal-decorators": "^7.10.5",
     "@babel/plugin-proposal-export-default-from": "^7.10.4",
@@ -117,6 +118,7 @@
     "@babel/preset-react": "^7.10.4",
     "@babel/preset-typescript": "^7.10.4",
     "@babel/register": "^7.10.5",
+    "@babel/traverse": "^7.11.0",
     "@peculiar/webcrypto": "^1.1.2",
     "@types/classnames": "^2.2.10",
     "@types/counterpart": "^0.18.1",
@@ -145,9 +147,7 @@
     "eslint-plugin-flowtype": "^2.50.3",
     "eslint-plugin-react": "^7.20.3",
     "eslint-plugin-react-hooks": "^2.5.1",
-    "estree-walker": "^0.9.0",
     "file-loader": "^3.0.1",
-    "flow-parser": "0.57.3",
     "glob": "^5.0.15",
     "jest": "^24.9.0",
     "jest-canvas-mock": "^2.2.0",

scripts/gen-i18n.js

@@ -18,7 +18,7 @@ limitations under the License.
 
 /**
  * Regenerates the translations en_EN file by walking the source tree and
- * parsing each file with flow-parser. Emits a JSON file with the
+ * parsing each file with the appropriate parser. Emits a JSON file with the
  * translatable strings mapped to themselves in the order they appeared
  * in the files and grouped by the file they appeared in.
  *
@@ -29,8 +29,8 @@ const path = require('path');
 
 const walk = require('walk');
 
-const flowParser = require('flow-parser');
+const parser = require("@babel/parser");
-const estreeWalker = require('estree-walker');
+const traverse = require("@babel/traverse");
 
 const TRANSLATIONS_FUNCS = ['_t', '_td'];
 
@@ -44,17 +44,9 @@ const OUTPUT_FILE = 'src/i18n/strings/en_EN.json';
 // to a project that's actively maintained.
 const SEARCH_PATHS = ['src', 'res'];
 
-const FLOW_PARSER_OPTS = {
-  esproposal_class_instance_fields: true,
-  esproposal_class_static_fields: true,
-  esproposal_decorators: true,
-  esproposal_export_star_as: true,
-  types: true,
-};
-
 function getObjectValue(obj, key) {
     for (const prop of obj.properties) {
-        if (prop.key.type == 'Identifier' && prop.key.name == key) {
+        if (prop.key.type === 'Identifier' && prop.key.name === key) {
             return prop.value;
         }
     }
@@ -62,11 +54,11 @@ function getObjectValue(obj, key) {
 }
 
 function getTKey(arg) {
-    if (arg.type == 'Literal') {
+    if (arg.type === 'Literal' || arg.type === "StringLiteral") {
         return arg.value;
-    } else if (arg.type == 'BinaryExpression' && arg.operator == '+') {
+    } else if (arg.type === 'BinaryExpression' && arg.operator === '+') {
         return getTKey(arg.left) + getTKey(arg.right);
-    } else if (arg.type == 'TemplateLiteral') {
+    } else if (arg.type === 'TemplateLiteral') {
         return arg.quasis.map((q) => {
             return q.value.raw;
         }).join('');
@@ -110,17 +102,44 @@ function getFormatStrings(str) {
 }
 
 function getTranslationsJs(file) {
-    const tree = flowParser.parse(fs.readFileSync(file, { encoding: 'utf8' }), FLOW_PARSER_OPTS);
+    const contents = fs.readFileSync(file, { encoding: 'utf8' });
 
     const trs = new Set();
 
-    estreeWalker.walk(tree, {
+    try {
-        enter: function(node, parent) {
+        const plugins = [
-            if (
+            // https://babeljs.io/docs/en/babel-parser#plugins
-                node.type == 'CallExpression' &&
+            "classProperties",
-                TRANSLATIONS_FUNCS.includes(node.callee.name)
+            "objectRestSpread",
-            ) {
+            "throwExpressions",
+            "exportDefaultFrom",
+            "decorators-legacy",
+        ];
+
+        if (file.endsWith(".js") || file.endsWith(".jsx")) {
+            // all JS is assumed to be flow or react
+            plugins.push("flow", "jsx");
+        } else if (file.endsWith(".ts")) {
+            // TS can't use JSX unless it's a TSX file (otherwise angle casts fail)
+            plugins.push("typescript");
+        } else if (file.endsWith(".tsx")) {
+            // When the file is a TSX file though, enable JSX parsing
+            plugins.push("typescript", "jsx");
+        }
+
+        const babelParsed = parser.parse(contents, {
+            allowImportExportEverywhere: true,
+            errorRecovery: true,
+            sourceFilename: file,
+            tokens: true,
+            plugins,
+        });
+        traverse.default(babelParsed, {
+            enter: (p) => {
+                const node = p.node;
+                if (p.isCallExpression() && node.callee && TRANSLATIONS_FUNCS.includes(node.callee.name)) {
                     const tKey = getTKey(node.arguments[0]);
+
                     // This happens whenever we call _t with non-literals (ie. whenever we've
                     // had to use a _td to compensate) so is expected.
                     if (tKey === null) return;
@@ -164,7 +183,7 @@ function getTranslationsJs(file) {
                     }
 
                     let isPlural = false;
-                if (node.arguments.length > 1 && node.arguments[1].type == 'ObjectExpression') {
+                    if (node.arguments.length > 1 && node.arguments[1].type === 'ObjectExpression') {
                         const countVal = getObjectValue(node.arguments[1], 'count');
                         if (countVal) {
                             isPlural = true;
@@ -183,8 +202,12 @@ function getTranslationsJs(file) {
                         trs.add(tKey);
                     }
                 }
-        }
+            },
         });
+    } catch (e) {
+        console.error(e);
+        process.exit(1);
+    }
 
     return trs;
 }

yarn.lock

@@ -111,6 +111,15 @@
     jsesc "^2.5.1"
     source-map "^0.5.0"
 
+"@babel/generator@^7.11.0":
+  version "7.11.0"
+  resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.11.0.tgz#4b90c78d8c12825024568cbe83ee6c9af193585c"
+  integrity sha512-fEm3Uzw7Mc9Xi//qU20cBKatTfs2aOtKqmvy/Vm7RkJEGFQ4xc9myCfbXxqK//ZS8MR/ciOHw6meGASJuKmDfQ==
+  dependencies:
+    "@babel/types" "^7.11.0"
+    jsesc "^2.5.1"
+    source-map "^0.5.0"
+
 "@babel/helper-annotate-as-pure@^7.10.1":
   version "7.10.1"
   resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.10.1.tgz#f6d08acc6f70bbd59b436262553fb2e259a1a268"
@@ -400,6 +409,13 @@
   dependencies:
     "@babel/types" "^7.10.4"
 
+"@babel/helper-split-export-declaration@^7.11.0":
+  version "7.11.0"
+  resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.11.0.tgz#f8a491244acf6a676158ac42072911ba83ad099f"
+  integrity sha512-74Vejvp6mHkGE+m+k5vHY93FX2cAtrw1zXrZXRlG4l410Nm9PxfEiVTn1PjDPV5SnmieiueY4AFg2xqhNFuuZg==
+  dependencies:
+    "@babel/types" "^7.11.0"
+
 "@babel/helper-validator-identifier@^7.10.1", "@babel/helper-validator-identifier@^7.10.3":
   version "7.10.3"
   resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.3.tgz#60d9847f98c4cea1b279e005fdb7c28be5412d15"
@@ -466,6 +482,11 @@
   resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.10.5.tgz#e7c6bf5a7deff957cec9f04b551e2762909d826b"
   integrity sha512-wfryxy4bE1UivvQKSQDU4/X6dr+i8bctjUjj8Zyt3DQy7NtPizJXT8M52nqpNKL+nq2PW8lxk4ZqLj0fD4B4hQ==
 
+"@babel/parser@^7.11.0":
+  version "7.11.0"
+  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.11.0.tgz#a9d7e11aead25d3b422d17b2c6502c8dddef6a5d"
+  integrity sha512-qvRvi4oI8xii8NllyEc4MDJjuZiNaRzyb7Y7lup1NqJV8TZHF4O27CcP+72WPn/k1zkgJ6WJfnIbk4jTsVAZHw==
+
 "@babel/plugin-proposal-async-generator-functions@^7.10.4":
   version "7.10.5"
   resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.10.5.tgz#3491cabf2f7c179ab820606cec27fed15e0e8558"
@@ -1213,6 +1234,21 @@
     globals "^11.1.0"
     lodash "^4.17.19"
 
+"@babel/traverse@^7.11.0":
+  version "7.11.0"
+  resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.11.0.tgz#9b996ce1b98f53f7c3e4175115605d56ed07dd24"
+  integrity sha512-ZB2V+LskoWKNpMq6E5UUCrjtDUh5IOTAyIl0dTjIEoXum/iKWkoIEKIRDnUucO6f+2FzNkE0oD4RLKoPIufDtg==
+  dependencies:
+    "@babel/code-frame" "^7.10.4"
+    "@babel/generator" "^7.11.0"
+    "@babel/helper-function-name" "^7.10.4"
+    "@babel/helper-split-export-declaration" "^7.11.0"
+    "@babel/parser" "^7.11.0"
+    "@babel/types" "^7.11.0"
+    debug "^4.1.0"
+    globals "^11.1.0"
+    lodash "^4.17.19"
+
 "@babel/types@^7.0.0", "@babel/types@^7.10.1", "@babel/types@^7.10.2", "@babel/types@^7.3.0", "@babel/types@^7.4.0", "@babel/types@^7.4.4", "@babel/types@^7.7.0":
   version "7.10.2"
   resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.10.2.tgz#30283be31cad0dbf6fb00bd40641ca0ea675172d"
@@ -1231,6 +1267,15 @@
     lodash "^4.17.19"
     to-fast-properties "^2.0.0"
 
+"@babel/types@^7.11.0":
+  version "7.11.0"
+  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.11.0.tgz#2ae6bf1ba9ae8c3c43824e5861269871b206e90d"
+  integrity sha512-O53yME4ZZI0jO1EVGtF1ePGl0LHirG4P1ibcD80XyzZcKhcMFeCXmh4Xb1ifGBIV233Qg12x4rBfQgA+tmOukA==
+  dependencies:
+    "@babel/helper-validator-identifier" "^7.10.4"
+    lodash "^4.17.19"
+    to-fast-properties "^2.0.0"
+
 "@cnakazawa/watch@^1.0.3":
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.4.tgz#f864ae85004d0fcab6f50be9141c4da368d1656a"
@@ -4178,11 +4223,6 @@ estree-walker@^0.6.1:
   resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-0.6.1.tgz#53049143f40c6eb918b23671d1fe3219f3a1b362"
   integrity sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==
 
-estree-walker@^0.9.0:
-  version "0.9.0"
-  resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-0.9.0.tgz#9116372f09c02fd88fcafb0c04343631012a0aa6"
-  integrity sha512-12U47o7XHUX329+x3FzNVjCx3SHEzMF0nkDv7r/HnBzX/xNTKxajBk6gyygaxrAFtLj39219oMfbtxv4KpaOiA==
-
 esutils@^2.0.2:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64"
@@ -4525,11 +4565,6 @@ flatted@^2.0.0:
   resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.2.tgz#4575b21e2bcee7434aa9be662f4b7b5f9c2b5138"
   integrity sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==
 
-flow-parser@0.57.3:
-  version "0.57.3"
-  resolved "https://registry.yarnpkg.com/flow-parser/-/flow-parser-0.57.3.tgz#b8d241a1b1cbae043afa7976e39f269988d8fe34"
-  integrity sha1-uNJBobHLrgQ6+nl2458mmYjY/jQ=
-
 flush-write-stream@^1.0.0:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8"