Merge pull request #26229 from vector-im/johannes/webpack-5

jest.config.ts

@@ -40,9 +40,9 @@ const config: Config = {
         "waveWorker\\.min\\.js": "<rootDir>/node_modules/matrix-react-sdk/__mocks__/empty.js",
         "context-filter-polyfill": "<rootDir>/node_modules/matrix-react-sdk/__mocks__/empty.js",
         "FontManager.ts": "<rootDir>/node_modules/matrix-react-sdk/__mocks__/FontManager.js",
-        "workers/(.+)\\.worker\\.ts": "<rootDir>/node_modules/matrix-react-sdk/__mocks__/workerMock.js",
+        "workers/(.+)Factory": "<rootDir>/node_modules/matrix-react-sdk/__mocks__/workerFactoryMock.js",
         "^!!raw-loader!.*": "jest-raw-loader",
-        "RecorderWorklet": "<rootDir>/node_modules/matrix-react-sdk/__mocks__/empty.js",
+        "recorderWorkletFactory": "<rootDir>/node_modules/matrix-react-sdk/__mocks__/empty.js",
         "^fetch-mock$": "<rootDir>/node_modules/fetch-mock",
     },
     transformIgnorePatterns: ["/node_modules/(?!matrix-js-sdk).+$", "/node_modules/(?!matrix-react-sdk).+$"],

package.json

@@ -43,12 +43,12 @@
         "build:modernizr": "modernizr -c .modernizr.json -d src/vector/modernizr.js",
         "build:bundle": "webpack --progress --mode production",
         "build:bundle-stats": "webpack --progress --mode production --json > webpack-stats.json",
-        "build:module_system": "tsc --project ./tsconfig.module_system.json && node ./lib/module_system/scripts/install.js",
+        "build:module_system": "ts-node --project ./tsconfig.module_system.json module_system/scripts/install.ts",
         "dist": "scripts/package.sh",
         "start": "concurrently --kill-others-on-fail --prefix \"{time} [{name}]\" -n modules,res,jitsi \"yarn build:module_system\" \"yarn build:res\" \"yarn build:jitsi\" && concurrently --kill-others-on-fail --prefix \"{time} [{name}]\" -n res,element-js \"yarn start:res\" \"yarn start:js\"",
         "start:https": "concurrently --kill-others-on-fail --prefix \"{time} [{name}]\" -n res,element-js \"yarn start:res\" \"yarn start:js --https\"",
         "start:res": "ts-node scripts/copy-res.ts -w",
-        "start:js": "webpack serve --output-path webapp --mode development",
+        "start:js": "webpack serve --output-path webapp --output-filename=bundles/_dev_/[name].js --output-chunk-filename=bundles/_dev_/[name].js --mode development",
         "lint": "yarn lint:types && yarn lint:js && yarn lint:style",
         "lint:js": "yarn lint:js:src && yarn lint:js:module_system",
         "lint:js:src": "eslint --max-warnings 0 src test && prettier --check .",
@@ -103,13 +103,12 @@
         "@babel/runtime": "^7.12.5",
         "@casualbot/jest-sonar-reporter": "2.2.7",
         "@principalstudio/html-webpack-inject-preload": "^1.2.7",
-        "@sentry/webpack-plugin": "^2.0.0",
+        "@sentry/webpack-plugin": "^2.7.1",
         "@svgr/webpack": "^5.5.0",
         "@testing-library/react": "^12.1.5",
         "@types/jest": "^29.0.0",
         "@types/jitsi-meet": "^2.0.2",
         "@types/jsrsasign": "^10.5.4",
-        "@types/loader-utils": "^2.0.4",
         "@types/lodash": "^4.14.197",
         "@types/node": "^16",
         "@types/node-fetch": "^2.6.4",
@@ -121,11 +120,13 @@
         "allchange": "^1.0.6",
         "babel-jest": "^29.0.0",
         "babel-loader": "^8.2.2",
+        "buffer": "^6.0.3",
         "chokidar": "^3.5.1",
         "concurrently": "^8.0.0",
-        "copy-webpack-plugin": "^6.0.0",
+        "copy-webpack-plugin": "^11.0.0",
         "cronstrue": "^2.41.0",
         "css-loader": "^4",
+        "css-minimizer-webpack-plugin": "^5.0.1",
         "dotenv": "^16.0.2",
         "eslint": "8.53.0",
         "eslint-config-google": "^0.14.0",
@@ -140,21 +141,19 @@
         "fetch-mock": "9.11.0",
         "fetch-mock-jest": "^1.5.1",
         "file-loader": "^6.0.0",
-        "html-webpack-plugin": "^4.5.2",
+        "html-webpack-plugin": "^5.5.3",
         "jest": "^29.0.0",
         "jest-canvas-mock": "2.5.2",
         "jest-environment-jsdom": "^29.0.0",
         "jest-mock": "^29.0.0",
         "jest-raw-loader": "^1.0.1",
-        "loader-utils": "^3.0.0",
         "matrix-mock-request": "^2.5.0",
         "matrix-web-i18n": "^3.1.3",
-        "mini-css-extract-plugin": "^1",
+        "mini-css-extract-plugin": "^2.7.6",
         "minimist": "^1.2.6",
         "mkdirp": "^3.0.0",
         "modernizr": "^3.12.0",
         "node-fetch": "^2.6.7",
-        "optimize-css-assets-webpack-plugin": "^6.0.0",
         "postcss": "^8.4.31",
         "postcss-easings": "^2.0.0",
         "postcss-hexrgba": "2.0.1",
@@ -166,25 +165,26 @@
         "postcss-scss": "^4.0.4",
         "postcss-simple-vars": "^5.0.2",
         "prettier": "2.8.8",
+        "process": "^0.11.10",
         "proxy-agent": "^6.3.0",
         "raw-loader": "^4.0.2",
         "rimraf": "^5.0.0",
         "semver": "^7.5.2",
+        "setimmediate": "^1.0.5",
         "string-replace-loader": "3",
         "style-loader": "2",
         "stylelint": "^15.10.1",
         "stylelint-config-standard": "^34.0.0",
         "stylelint-scss": "^5.0.0",
-        "terser-webpack-plugin": "^4.0.0",
+        "terser-webpack-plugin": "^5.3.9",
         "ts-node": "^10.9.1",
         "ts-prune": "^0.10.3",
         "typescript": "5.2.2",
-        "webpack": "^4.47.0",
+        "util": "^0.12.5",
+        "webpack": "^5.89.0",
         "webpack-bundle-analyzer": "^4.8.0",
         "webpack-cli": "^4.10.0",
         "webpack-dev-server": "^4.15.1",
-        "worker-loader": "^3.0.0",
-        "worklet-loader": "^2.0.0",
         "yaml": "^2.3.3"
     },
     "@casualbot/jest-sonar-reporter": {

recorder-worklet-loader.js

@@ -0,0 +1,45 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Inspired by https://github.com/reklawnos/worklet-loader which doesn't
+// formally support Webpack 5
+
+const SingleEntryPlugin = require("webpack/lib/SingleEntryPlugin");
+
+module.exports = function () {};
+
+module.exports.pitch = function pitch(request) {
+    const cb = this.async();
+    const filename = "recorder.worklet.js";
+
+    const compiler = this._compilation.createChildCompiler("worker", {
+        filename,
+        chunkFilename: `[id].${filename}`,
+        namedChunkFilename: null,
+    });
+
+    new SingleEntryPlugin(this.context, `!!${request}`, "main").apply(compiler);
+
+    compiler.runAsChild((err, entries, compilation) => {
+        if (err) {
+            return cb(err);
+        }
+        if (entries[0]) {
+            return cb(null, `module.exports = __webpack_public_path__ + ${JSON.stringify(entries[0].files[0])};`);
+        }
+        return cb(null, null);
+    });
+};

scripts/copy-res.ts

@@ -6,7 +6,7 @@ import parseArgs from "minimist";
 import * as chokidar from "chokidar";
 import * as fs from "node:fs";
 import _ from "lodash";
-import * as loaderUtils from "loader-utils";
+import { util } from "webpack";
 import { Translations } from "matrix-web-i18n";
 
 const REACT_I18N_BASE_PATH = "node_modules/matrix-react-sdk/src/i18n/strings/";
@@ -60,7 +60,7 @@ function prepareLangFile(lang: string, dest: string): [filename: string, json: s
 
     const json = JSON.stringify(translations, null, 4);
     const jsonBuffer = Buffer.from(json);
-    const digest = loaderUtils.getHashDigest(jsonBuffer, null, "hex", 7);
+    const digest = util.createHash("xxhash64").update(jsonBuffer).digest("hex").slice(0, 7);
     const filename = `${lang}.${digest}.json`;
 
     return [filename, json];

src/@types/loader-utils.d.ts

@@ -1,26 +0,0 @@
-/*
-Copyright 2023 The Matrix.org Foundation C.I.C.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-import * as LoaderUtils from "loader-utils";
-
-declare module "loader-utils" {
-    export function getHashDigest(
-        buffer: Buffer,
-        hashType: null,
-        digestType: LoaderUtils.DigestType,
-        maxLength: number,
-    ): string;
-}

src/vector/index.html

@@ -49,11 +49,10 @@
         <% }
     } %>
 
-    <% for (var i=0; i < htmlWebpackPlugin.tags.headTags.length; i++) {
+    <% for (const tag of htmlWebpackPlugin.tags.headTags) {
-        var tag = htmlWebpackPlugin.tags.headTags[i];
+        let path = tag.attributes && tag.attributes.href;
-        var path = tag.attributes && tag.attributes.href;
+        if (path && path.includes("/Inter/")) { %>
-        if (path.includes("/Inter/")) { %>
+            <link rel="preload" as="font" href="<%= path %>" crossorigin="anonymous"/>
-            <link rel="preload" as="font" href=".<%= path %>" crossorigin="anonymous"/>
         <% }
     } %>
 

src/vector/index.ts

@@ -25,6 +25,9 @@ import { extractErrorMessageFromError } from "matrix-react-sdk/src/components/vi
 import { parseQsFromFragment } from "./url_utils";
 import "./modernizr";
 
+// Make setImmediate available in bundle
+import "setimmediate";
+
 // Require common CSS here; this will make webpack process it into bundle.css.
 // Our own CSS (which is themed) is imported via separate webpack entry points
 // in webpack.config.js

tsconfig.json

@@ -4,7 +4,7 @@
         "emitDecoratorMetadata": false,
         "resolveJsonModule": true,
         "esModuleInterop": true,
-        "module": "commonjs",
+        "module": "es2022",
         "moduleResolution": "node",
         "target": "es2016",
         "noUnusedLocals": true,
@@ -28,6 +28,9 @@
         "./scripts/*.ts"
     ],
     "ts-node": {
-        "files": true
+        "files": true,
+        "moduleTypes": {
+            "*": "cjs"
+        }
     }
 }

webpack.config.js

@@ -6,7 +6,7 @@ const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 const TerserPlugin = require("terser-webpack-plugin");
-const OptimizeCSSAssetsPlugin = require("optimize-css-assets-webpack-plugin");
+const CssMinimizerPlugin = require("css-minimizer-webpack-plugin");
 const HtmlWebpackInjectPreload = require("@principalstudio/html-webpack-inject-preload");
 const { sentryWebpackPlugin } = require("@sentry/webpack-plugin");
 const crypto = require("crypto");
@@ -81,7 +81,15 @@ function parseOverridesToReplacements(overrides) {
             // because the input is effectively defined by the person running the build, we don't
             // need to do anything special to protect against regex overrunning, etc.
             new RegExp(oldPath.replace(/\//g, "[\\/\\\\]").replace(/\./g, "\\.")),
-            path.resolve(__dirname, newPath),
+            function (resource) {
+                resource.request = path.resolve(__dirname, newPath);
+                resource.createData.resource = path.resolve(__dirname, newPath);
+                // Starting with Webpack 5 we also need to set the context as otherwise replacing
+                // files in e.g. matrix-react-sdk with files from element-web will try to resolve
+                // them within matrix-react-sdk (https://github.com/webpack/webpack/issues/17716)
+                resource.context = path.dirname(resource.request);
+                resource.createData.context = path.dirname(resource.createData.resource);
+            },
         );
     });
 }
@@ -147,14 +155,6 @@ module.exports = (env, argv) => {
 
         bail: true,
 
-        node: {
-            // Mock out the NodeFS module: The opus decoder imports this wrongly.
-            fs: "empty",
-            net: "empty",
-            tls: "empty",
-            crypto: "empty",
-        },
-
         entry: {
             bundle: "./src/vector/index.ts",
             mobileguide: "./src/vector/mobile_guide/index.ts",
@@ -192,14 +192,13 @@ module.exports = (env, argv) => {
                 },
             },
 
-            // This fixes duplicate files showing up in chrome with sourcemaps enabled.
+            // Readable IDs for better debugging
-            // See https://github.com/webpack/webpack/issues/7128 for more info.
+            moduleIds: "named",
-            namedModules: false,
 
             // Minification is normally enabled by default for webpack in production mode, but
             // we use a CSS optimizer too and need to manage it ourselves.
             minimize: enableMinification,
-            minimizer: enableMinification ? [new TerserPlugin({}), new OptimizeCSSAssetsPlugin({})] : [],
+            minimizer: enableMinification ? [new TerserPlugin({}), new CssMinimizerPlugin()] : [],
 
             // Set the value of `process.env.NODE_ENV` for libraries like React
             // See also https://v4.webpack.js.org/configuration/optimization/#optimizationnodeenv
@@ -242,6 +241,20 @@ module.exports = (env, argv) => {
                 // Define a variable so the i18n stuff can load
                 "$webapp": path.resolve(__dirname, "webapp"),
             },
+            fallback: {
+                // Mock out the NodeFS module: The opus decoder imports this wrongly.
+                "fs": false,
+                "net": false,
+                "tls": false,
+                "crypto": false,
+
+                // Polyfill needed by counterpart
+                "util": require.resolve("util/"),
+                // Polyfill needed by matrix-js-sdk/src/crypto
+                "buffer": require.resolve("buffer/"),
+                // Polyfill needed by sentry
+                "process/browser": require.resolve("process/browser"),
+            },
         },
 
         module: {
@@ -268,15 +281,6 @@ module.exports = (env, argv) => {
                         replace: getThemesImports(),
                     },
                 },
-                {
-                    test: /\.worker\.ts$/,
-                    loader: "worker-loader",
-                    options: {
-                        // Prevent bundling workers since CSP forbids loading them
-                        // from another origin.
-                        filename: "[hash].worker.js",
-                    },
-                },
                 {
                     test: /\.(ts|js)x?$/,
                     include: (f) => {
@@ -452,22 +456,17 @@ module.exports = (env, argv) => {
                     },
                 },
                 {
-                    // Special case the recorder worklet as it can't end up HMR'd, but the worker-loader
+                    // Ideally we should use the built-in worklet support in Webpack 5 with the syntax
-                    // isn't good enough for us. Note that the worklet-loader is listed as "do not use",
+                    // described in https://github.com/webpack/webpack.js.org/issues/6869. However, this
-                    // however it seems to work fine for our purposes.
+                    // doesn't currently appear to work with our public path setup. So we handle this
+                    // with a custom loader instead.
                     test: /RecorderWorklet\.ts$/,
                     type: "javascript/auto",
                     use: [
-                        // executed last -> first, for some reason.
                         {
-                            loader: "worklet-loader",
+                            loader: path.resolve("./recorder-worklet-loader.js"),
-                            options: {
-                                // Override name so we know what it is in the output.
-                                name: "recorder-worklet.[hash:7].js",
-                            },
                         },
                         {
-                            // TS -> JS because the worklet-loader won't do this for us.
                             loader: "babel-loader",
                         },
                     ],
@@ -709,6 +708,7 @@ module.exports = (env, argv) => {
                         console.log(`::warning title=Sentry error::${err.message}`);
                     },
                 }),
+
             new webpack.EnvironmentPlugin(["VERSION"]),
 
             new CopyWebpackPlugin({
@@ -727,6 +727,13 @@ module.exports = (env, argv) => {
                     "contribute.json",
                 ],
             }),
+
+            // Automatically load buffer & process modules as we use them without explicitly
+            // importing them
+            new webpack.ProvidePlugin({
+                Buffer: ["buffer", "Buffer"],
+                process: "process/browser",
+            }),
         ].filter(Boolean),
 
         output: {
@@ -746,6 +753,15 @@ module.exports = (env, argv) => {
 
         // configuration for the webpack-dev-server
         devServer: {
+            client: {
+                overlay: {
+                    // Only show overlay on build errors as anything more can get annoying quickly
+                    errors: true,
+                    warnings: false,
+                    runtimeErrors: false,
+                },
+            },
+
             static: {
                 // Where to serve static assets from
                 directory: "./webapp",