Remove url npm dependency and use Web URL constructor (#10930)

package.json

@@ -118,7 +118,6 @@
         "sanitize-html": "2.10.0",
         "tar-js": "^0.3.0",
         "ua-parser-js": "^1.0.2",
-        "url": "^0.11.0",
         "what-input": "^5.2.10",
         "zxcvbn": "^4.4.2"
     },

src/ScalarAuthClient.ts

@@ -14,7 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import url from "url";
 import { SERVICE_TYPES } from "matrix-js-sdk/src/service-types";
 import { Room } from "matrix-js-sdk/src/models/room";
 import { logger } from "matrix-js-sdk/src/logger";
@@ -25,6 +24,7 @@ import { Service, startTermsFlow, TermsInteractionCallback, TermsNotSignedError
 import { MatrixClientPeg } from "./MatrixClientPeg";
 import SdkConfig from "./SdkConfig";
 import { WidgetType } from "./widgets/WidgetType";
+import { parseUrl } from "./utils/UrlUtils";
 
 // The version of the integration manager API we're intending to work with
 const imApiVersion = "1.1";
@@ -154,11 +154,10 @@ export default class ScalarAuthClient {
                     // Once we've fully transitioned to _matrix URLs, we can give people
                     // a grace period to update their configs, then use the rest url as
                     // a regular base url.
-                    const parsedImRestUrl = url.parse(this.apiUrl);
+                    const parsedImRestUrl = parseUrl(this.apiUrl);
-                    parsedImRestUrl.path = "";
                     parsedImRestUrl.pathname = "";
                     return startTermsFlow(
-                        [new Service(SERVICE_TYPES.IM, url.format(parsedImRestUrl), token)],
+                        [new Service(SERVICE_TYPES.IM, parsedImRestUrl.toString(), token)],
                         this.termsInteractionCallback,
                     ).then(() => {
                         return token;

src/components/views/dialogs/TermsDialog.tsx

@@ -14,7 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import url from "url";
 import React from "react";
 import { SERVICE_TYPES } from "matrix-js-sdk/src/service-types";
 
@@ -23,6 +22,7 @@ import DialogButtons from "../elements/DialogButtons";
 import BaseDialog from "./BaseDialog";
 import { ServicePolicyPair } from "../../../Terms";
 import ExternalLink from "../elements/ExternalLink";
+import { parseUrl } from "../../../utils/UrlUtils";
 
 interface ITermsCheckboxProps {
     onChange: (url: string, checked: boolean) => void;
@@ -130,7 +130,7 @@ export default class TermsDialog extends React.PureComponent<ITermsDialogProps,
     public render(): React.ReactNode {
         const rows: JSX.Element[] = [];
         for (const policiesAndService of this.props.policiesAndServicePairs) {
-            const parsedBaseUrl = url.parse(policiesAndService.service.baseUrl);
+            const parsedBaseUrl = parseUrl(policiesAndService.service.baseUrl);
 
             const policyValues = Object.values(policiesAndService.policies);
             for (let i = 0; i < policyValues.length; ++i) {

src/components/views/elements/AppPermission.tsx

@@ -17,7 +17,6 @@ limitations under the License.
 */
 
 import React from "react";
-import url from "url";
 import { RoomMember } from "matrix-js-sdk/src/models/room-member";
 
 import { _t } from "../../../languageHandler";
@@ -28,6 +27,7 @@ import MemberAvatar from "../avatars/MemberAvatar";
 import BaseAvatar from "../avatars/BaseAvatar";
 import AccessibleButton from "./AccessibleButton";
 import TextWithTooltip from "./TextWithTooltip";
+import { parseUrl } from "../../../utils/UrlUtils";
 
 interface IProps {
     url: string;
@@ -67,13 +67,12 @@ export default class AppPermission extends React.Component<IProps, IState> {
     }
 
     private parseWidgetUrl(): { isWrapped: boolean; widgetDomain: string | null } {
-        const widgetUrl = url.parse(this.props.url);
+        const widgetUrl = parseUrl(this.props.url);
-        const params = new URLSearchParams(widgetUrl.search ?? undefined);
 
         // HACK: We're relying on the query params when we should be relying on the widget's `data`.
         // This is a workaround for Scalar.
-        if (WidgetUtils.isScalarUrl(this.props.url) && params?.get("url")) {
+        if (WidgetUtils.isScalarUrl(this.props.url) && widgetUrl.searchParams.has("url")) {
-            const unwrappedUrl = url.parse(params.get("url")!);
+            const unwrappedUrl = parseUrl(widgetUrl.searchParams.get("url")!);
             return {
                 widgetDomain: unwrappedUrl.host || unwrappedUrl.hostname,
                 isWrapped: true,

src/components/views/elements/AppTile.tsx

@@ -17,7 +17,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import url from "url";
 import React, { ContextType, createRef, CSSProperties, MutableRefObject, ReactNode } from "react";
 import classNames from "classnames";
 import { IWidget, MatrixCapabilities } from "matrix-widget-api";
@@ -52,6 +51,7 @@ import { ElementWidgetCapabilities } from "../../../stores/widgets/ElementWidget
 import { WidgetMessagingStore } from "../../../stores/widgets/WidgetMessagingStore";
 import { SdkContextClass } from "../../../contexts/SDKContext";
 import { ModuleRunner } from "../../../modules/ModuleRunner";
+import { parseUrl } from "../../../utils/UrlUtils";
 
 interface IProps {
     app: IWidget | IApp;
@@ -265,7 +265,7 @@ export default class AppTile extends React.Component<IProps, IState> {
 
     private isMixedContent(): boolean {
         const parentContentProtocol = window.location.protocol;
-        const u = url.parse(this.props.app.url);
+        const u = parseUrl(this.props.app.url);
         const childContentProtocol = u.protocol;
         if (parentContentProtocol === "https:" && childContentProtocol !== "https:") {
             logger.warn(

src/components/views/settings/SetIdServer.tsx

@@ -14,7 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import url from "url";
 import React, { ReactNode } from "react";
 import { logger } from "matrix-js-sdk/src/logger";
 import { IThreepid } from "matrix-js-sdk/src/@types/threepids";
@@ -25,7 +24,7 @@ import Modal from "../../../Modal";
 import dis from "../../../dispatcher/dispatcher";
 import { getThreepidsWithBindStatus } from "../../../boundThreepids";
 import IdentityAuthClient from "../../../IdentityAuthClient";
-import { abbreviateUrl, unabbreviateUrl } from "../../../utils/UrlUtils";
+import { abbreviateUrl, parseUrl, unabbreviateUrl } from "../../../utils/UrlUtils";
 import { getDefaultIdentityServerUrl, doesIdentityServerHaveTerms } from "../../../utils/IdentityServerUtils";
 import { timeout } from "../../../utils/promise";
 import { ActionPayload } from "../../../dispatcher/payloads";
@@ -44,7 +43,7 @@ const REACHABILITY_TIMEOUT = 10000; // ms
  * @returns {string} null if url passes all checks, otherwise i18ned error string
  */
 async function checkIdentityServerUrl(u: string): Promise<string | null> {
-    const parsedUrl = url.parse(u);
+    const parsedUrl = parseUrl(u);
 
     if (parsedUrl.protocol !== "https:") return _t("Identity server URL must be HTTPS");
 

src/integrations/IntegrationManagerInstance.ts

@@ -14,7 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import url from "url";
 import { ComponentProps } from "react";
 import { logger } from "matrix-js-sdk/src/logger";
 
@@ -25,6 +24,7 @@ import Modal from "../Modal";
 import SettingsStore from "../settings/SettingsStore";
 import IntegrationManager from "../components/views/settings/IntegrationManager";
 import { IntegrationManagers } from "./IntegrationManagers";
+import { parseUrl } from "../utils/UrlUtils";
 
 export enum Kind {
     Account = "account",
@@ -42,15 +42,14 @@ export class IntegrationManagerInstance {
     ) {}
 
     public get name(): string {
-        const parsed = url.parse(this.uiUrl);
+        const parsed = parseUrl(this.uiUrl);
         return parsed.host ?? "";
     }
 
     public get trimmedApiUrl(): string {
-        const parsed = url.parse(this.apiUrl);
+        const parsed = parseUrl(this.apiUrl);
         parsed.pathname = "";
-        parsed.path = "";
+        return parsed.toString();
-        return url.format(parsed);
     }
 
     public getScalarClient(): ScalarAuthClient {

src/integrations/IntegrationManagers.ts

@@ -14,7 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import url from "url";
 import { logger } from "matrix-js-sdk/src/logger";
 import { ClientEvent, IClientWellKnown, MatrixClient } from "matrix-js-sdk/src/client";
 import { compare } from "matrix-js-sdk/src/utils";
@@ -27,6 +26,7 @@ import IntegrationsImpossibleDialog from "../components/views/dialogs/Integratio
 import IntegrationsDisabledDialog from "../components/views/dialogs/IntegrationsDisabledDialog";
 import WidgetUtils from "../utils/WidgetUtils";
 import { MatrixClientPeg } from "../MatrixClientPeg";
+import { parseUrl } from "../utils/UrlUtils";
 
 const KIND_PREFERENCE = [
     // Ordered: first is most preferred, last is least preferred.
@@ -199,7 +199,7 @@ export class IntegrationManagers {
         logger.log("Looking up integration manager via .well-known");
         if (domainName.startsWith("http:") || domainName.startsWith("https:")) {
             // trim off the scheme and just use the domain
-            domainName = url.parse(domainName).host!;
+            domainName = parseUrl(domainName).host;
         }
 
         let wkConfig: IClientWellKnown;

src/utils/UrlUtils.ts

@@ -14,8 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import url from "url";
-
 /**
  * If a url has no path component, etc. abbreviate it to just the hostname
  *
@@ -25,11 +23,16 @@ import url from "url";
 export function abbreviateUrl(u?: string): string {
     if (!u) return "";
 
-    const parsedUrl = url.parse(u);
+    let parsedUrl: URL;
+    try {
+        parsedUrl = parseUrl(u);
+    } catch (e) {
+        console.error(e);
         // if it's something we can't parse as a url then just return it
-    if (!parsedUrl) return u;
+        return u;
+    }
 
-    if (parsedUrl.path === "/") {
+    if (parsedUrl.pathname === "/") {
         // we ignore query / hash parts: these aren't relevant for IS server URLs
         return parsedUrl.host || "";
     }
@@ -42,8 +45,15 @@ export function unabbreviateUrl(u?: string): string {
 
     let longUrl = u;
     if (!u.startsWith("https://")) longUrl = "https://" + u;
-    const parsed = url.parse(longUrl);
+    const parsed = parseUrl(longUrl);
-    if (parsed.hostname === null) return u;
+    if (!parsed.hostname) return u;
 
     return longUrl;
 }
+
+export function parseUrl(u: string): URL {
+    if (!u.includes(":")) {
+        u = window.location.protocol + u;
+    }
+    return new URL(u);
+}

src/utils/WidgetUtils.ts

@@ -15,7 +15,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import * as url from "url";
 import { base32 } from "rfc4648";
 import { IWidget, IWidgetData } from "matrix-widget-api";
 import { Room } from "matrix-js-sdk/src/models/room";
@@ -36,6 +35,7 @@ import { Jitsi } from "../widgets/Jitsi";
 import { objectClone } from "./objects";
 import { _t } from "../languageHandler";
 import { IApp, isAppWidget } from "../stores/WidgetStore";
+import { parseUrl } from "./UrlUtils";
 
 // How long we wait for the state event echo to come back from the server
 // before waitFor[Room/User]Widget rejects its promise
@@ -106,7 +106,7 @@ export default class WidgetUtils {
             return false;
         }
 
-        const testUrl = url.parse(testUrlString);
+        const testUrl = parseUrl(testUrlString);
         let scalarUrls = SdkConfig.get().integrations_widgets_urls;
         if (!scalarUrls || scalarUrls.length === 0) {
             const defaultManager = IntegrationManagers.sharedInstance().getPrimaryManager();
@@ -118,7 +118,7 @@ export default class WidgetUtils {
         }
 
         for (let i = 0; i < scalarUrls.length; i++) {
-            const scalarUrl = url.parse(scalarUrls[i]);
+            const scalarUrl = parseUrl(scalarUrls[i]);
             if (testUrl && scalarUrl) {
                 if (
                     testUrl.protocol === scalarUrl.protocol &&

test/components/views/elements/AppTile-test.tsx

@@ -52,6 +52,16 @@ import { WidgetMessagingStore } from "../../../../src/stores/widgets/WidgetMessa
 import { ModuleRunner } from "../../../../src/modules/ModuleRunner";
 import { RoomPermalinkCreator } from "../../../../src/utils/permalinks/Permalinks";
 
+jest.mock("../../../../src/stores/OwnProfileStore", () => ({
+    OwnProfileStore: {
+        instance: {
+            isProfileInfoFetched: true,
+            removeListener: jest.fn(),
+            getHttpAvatarUrl: jest.fn().mockReturnValue("http://avatar_url"),
+        },
+    },
+}));
+
 describe("AppTile", () => {
     let cli: MatrixClient;
     let r1: Room;

test/utils/UrlUtils-test.ts

@@ -0,0 +1,51 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { abbreviateUrl, parseUrl, unabbreviateUrl } from "../../src/utils/UrlUtils";
+
+describe("abbreviateUrl", () => {
+    it("should return empty string if passed falsey", () => {
+        expect(abbreviateUrl(undefined)).toEqual("");
+    });
+
+    it("should abbreviate to host if empty pathname", () => {
+        expect(abbreviateUrl("https://foo/")).toEqual("foo");
+    });
+
+    it("should not abbreviate if has path parts", () => {
+        expect(abbreviateUrl("https://foo/path/parts")).toEqual("https://foo/path/parts");
+    });
+});
+
+describe("unabbreviateUrl", () => {
+    it("should return empty string if passed falsey", () => {
+        expect(unabbreviateUrl(undefined)).toEqual("");
+    });
+
+    it("should prepend https to input if it lacks it", () => {
+        expect(unabbreviateUrl("element.io")).toEqual("https://element.io");
+    });
+
+    it("should not prepend https to input if it has it", () => {
+        expect(unabbreviateUrl("https://element.io")).toEqual("https://element.io");
+    });
+});
+
+describe("parseUrl", () => {
+    it("should not throw on no proto", () => {
+        expect(() => parseUrl("test")).not.toThrow();
+    });
+});

yarn.lock

@@ -7313,11 +7313,6 @@ pump@^3.0.0:
     end-of-stream "^1.1.0"
     once "^1.3.1"
 
-punycode@1.3.2:
-  version "1.3.2"
-  resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d"
-  integrity sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==
-
 punycode@^2.1.0, punycode@^2.1.1:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.0.tgz#f67fa67c94da8f4d0cfff981aee4118064199b8f"
@@ -7352,11 +7347,6 @@ qs@~6.10.3:
   dependencies:
     side-channel "^1.0.4"
 
-querystring@0.2.0:
-  version "0.2.0"
-  resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
-  integrity sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==
-
 querystring@^0.2.0:
   version "0.2.1"
   resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.1.tgz#40d77615bb09d16902a85c3e38aa8b5ed761c2dd"
@@ -8611,14 +8601,6 @@ url-parse@^1.5.3:
     querystringify "^2.1.1"
     requires-port "^1.0.0"
 
-url@^0.11.0:
-  version "0.11.0"
-  resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1"
-  integrity sha512-kbailJa29QrtXnxgq+DdCEGlbTeYM2eJUxsz6vjZavrCYPMIFHMKQmSKYAIuUK2i7hgPm28a8piX5NTUtM/LKQ==
-  dependencies:
-    punycode "1.3.2"
-    querystring "0.2.0"
-
 use-callback-ref@^1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/use-callback-ref/-/use-callback-ref-1.3.0.tgz#772199899b9c9a50526fedc4993fc7fa1f7e32d5"