api/security: implement api keys as method of authentication

.gitignore

@@ -13,6 +13,7 @@ build
 .env.*
 !.env.example
 cookies.json
+keys.json
 
 # docker
 docker-compose.yml

api/package.json

@@ -33,7 +33,7 @@
         "express-rate-limit": "^6.3.0",
         "ffmpeg-static": "^5.1.0",
         "hls-parser": "^0.10.7",
-        "ipaddr.js": "2.1.0",
+        "ipaddr.js": "2.2.0",
         "nanoid": "^4.0.2",
         "node-cache": "^5.1.2",
         "psl": "1.9.0",

api/src/config.js

@@ -43,6 +43,11 @@ const env = {
                         && process.env.TURNSTILE_SECRET
                         && process.env.JWT_SECRET,
 
+    apiKeyURL: process.env.API_KEY_URL && new URL(process.env.API_KEY_URL),
+    authRequired: process.env.API_AUTH_REQUIRED === '1',
+
+    keyReloadInterval: 900,
+
     enabledServices,
 }
 

api/src/misc/console-text.js

@@ -5,12 +5,19 @@ function t(color, tt) {
 export function Bright(tt) {
     return t("\x1b[1m", tt)
 }
+
 export function Red(tt) {
     return t("\x1b[31m", tt)
 }
+
 export function Green(tt) {
     return t("\x1b[32m", tt)
 }
+
 export function Cyan(tt) {
     return t("\x1b[36m", tt)
 }
+
+export function Yellow(tt) {
+    return t("\x1b[93m", tt)
+}

api/src/security/api-keys.js

@@ -0,0 +1,201 @@
+import { env } from "../config.js";
+import { readFile } from "node:fs/promises";
+import { Yellow } from "../misc/console-text.js";
+import ip from "ipaddr.js";
+
+// this function is a modified variation of code
+// from https://stackoverflow.com/a/32402438/14855621
+const generateWildcardRegex = rule => {
+    var escapeRegex = (str) => str.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1");
+    return new RegExp("^" + rule.split("*").map(escapeRegex).join(".*") + "$");
+}
+
+const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+let keys = {};
+
+const ALLOWED_KEYS = new Set(['name', 'ips', 'userAgents', 'limit']);
+
+/* Expected format pseudotype:
+** type KeyFileContents = Record<
+**    UUIDv4String,
+**    {
+**        name?: string,
+**        limit?: number | "unlimited",
+**        ips?: CIDRString[],
+**        userAgents?: string[]
+**    }
+** >;
+*/
+
+const validateKeys = (input) => {
+    if (typeof input !== 'object' || input === null) {
+        throw "input is not an object";
+    }
+
+    if (Object.keys(input).some(x => !UUID_REGEX.test(x))) {
+        throw "key file contains invalid key(s)";
+    }
+
+    Object.values(input).forEach(details => {
+        if (typeof details !== 'object' || details === null) {
+            throw "some key(s) are incorrectly configured";
+        }
+
+        const unexpected_key = Object.keys(details).find(k => !ALLOWED_KEYS.has(k));
+        if (unexpected_key) {
+            throw "detail object contains unexpected key: " + unexpected_key;
+        }
+
+        if (details.limit && details.limit !== 'unlimited') {
+            if (typeof details.limit !== 'number')
+                throw "detail object contains invalid limit (not a number)";
+            else if (details.limit < 1)
+                throw "detail object contains invalid limit (not a number)";
+        }
+
+        if (details.ips) {
+            if (!Array.isArray(details.ips))
+                throw "details object contains value for `ips` which is not an array";
+
+            const invalid_ip = details.ips.find(
+                addr => typeof addr !== 'string' || (!ip.isValidCIDR(addr) && !ip.isValid(addr))
+            );
+
+            if (invalid_ip) {
+                throw "`ips` in details contains an invalid IP or CIDR range: " + invalid_ip;
+            }
+        }
+
+        if (details.userAgents) {
+            if (!Array.isArray(details.userAgents))
+                throw "details object contains value for `userAgents` which is not an array";
+
+            const invalid_ua = details.userAgents.find(ua => typeof ua !== 'string');
+            if (invalid_ua) {
+                throw "`userAgents` in details contains an invalid user agent: " + invalid_ua;
+            }
+        }
+    });
+}
+
+const formatKeys = (keyData) => {
+    const formatted = {};
+
+    for (let key in keyData) {
+        const data = keyData[key];
+        key = key.toLowerCase();
+
+        formatted[key] = {};
+
+        if (data.limit) {
+            formatted[key].limit = data.limit;
+        }
+
+        if (data.ips) {
+            formatted[key].ips = data.ips.map(addr => {
+                if (ip.isValid(addr)) {
+                    return [ ip.parse(addr), 32 ];
+                }
+
+                return ip.parseCIDR(addr);
+            });
+        }
+
+        if (data.userAgents) {
+            formatted[key].userAgents = data.userAgents.map(generateWildcardRegex);
+        }
+    }
+
+    return formatted;
+}
+
+const loadKeys = async (source) => {
+    let updated;
+    if (source.protocol === 'file:') {
+        const pathname = source.pathname === '/' ? '' : source.pathname;
+        updated = JSON.parse(
+            await readFile(
+                decodeURIComponent(source.host + pathname),
+                'utf8'
+            )
+        );
+    } else {
+        updated = await fetch(source).then(a => a.json());
+    }
+
+    validateKeys(updated);
+    keys = formatKeys(updated);
+}
+
+const wrapLoad = (url) => {
+    loadKeys(url)
+    .then(() => {})
+    .catch((e) => {
+        console.error(`${Yellow('[!]')} Failed loading API keys at ${new Date().toISOString()}.`);
+        console.error('Error:', e);
+    })
+}
+
+const err = (reason) => ({ success: false, error: reason });
+
+export const validateAuthorization = (req) => {
+    const authHeader = req.get('Authorization');
+
+    if (typeof authHeader !== 'string') {
+        return err("missing");
+    }
+
+    const [ authType, keyString ] = authHeader.split(' ', 2);
+    if (authType.toLowerCase() !== 'api-key') {
+        return err("not_api_key");
+    }
+
+    if (!UUID_REGEX.test(keyString) || `${authType} ${keyString}` !== authHeader) {
+        return err("invalid");
+    }
+
+    const matchingKey = keys[keyString.toLowerCase()];
+    if (!matchingKey) {
+        return err("not_found");
+    }
+
+    if (matchingKey.ips) {
+        let addr;
+        try {
+            addr = ip.parse(req.ip);
+        } catch {
+            return err("invalid_ip");
+        }
+
+        const ip_allowed = matchingKey.ips.some(
+            ([ allowed, size ]) => {
+                return addr.kind() === allowed.kind()
+                        && addr.match(allowed, size);
+            }
+        );
+
+        if (!ip_allowed) {
+            return err("ip_not_allowed");
+        }
+    }
+
+    if (matchingKey.userAgents) {
+        const userAgent = req.get('User-Agent');
+        if (!matchingKey.userAgents.some(regex => regex.test(userAgent))) {
+            return err("ua_not_allowed");
+        }
+    }
+
+    req.rateLimitKey = keyString.toLowerCase();
+    req.rateLimitMax = matchingKey.limit;
+
+    return { success: true };
+}
+
+export const setup = (url) => {
+    wrapLoad(url);
+    if (env.keyReloadInterval > 0) {
+        setInterval(() => wrapLoad(url), env.keyReloadInterval * 1000);
+    }
+}

docs/run-an-instance.md

@@ -72,6 +72,8 @@ sudo service nscd start
 | `RATELIMIT_MAX`       | `20`      | `30`                    | max requests per time window. requests above this amount will be blocked for the rate limit window duration. |
 | `DURATION_LIMIT`      | `10800`   | `18000`                 | max allowed video duration in **seconds**. |
 | `TUNNEL_LIFESPAN`     | `90`      | `120`                   | the duration for which tunnel info is stored in ram, **in seconds**. |
+| `API_KEY_URL`         | ➖        | `file://keys.json`      | the location of the api key database. for loading API keys, cobalt supports HTTP(S) urls, or local files by specifying a local path using the `file://` protocol. see the "api key file format" below for more details.  |
+| `API_AUTH_REQUIRED`   | ➖        | `1`                     | when set to `1`, the user always needs to be authenticated in some way before they can access the API (either via an api key or via turnstile, if enabled). |
 
 \* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)).
 
@@ -80,3 +82,49 @@ setting a `FREEBIND_CIDR` allows cobalt to pick a random IP for every download a
 requests it makes for that particular download. to use freebind in cobalt, you need to follow its [setup instructions](https://github.com/imputnet/freebind.js?tab=readme-ov-file#setup) first. if you configure this option while running cobalt
 in a docker container, you also need to set the `API_LISTEN_ADDRESS` env to `127.0.0.1`, and set
 `network_mode` for the container to `host`.
+
+#### api key file format
+the file is a JSON-serialized object with the following structure:
+```typescript
+
+type KeyFileContents = Record<
+    UUIDv4String,
+    {
+        name?: string,
+        limit?: number | "unlimited",
+        ips?: (CIDRString | IPString)[],
+        userAgents?: string[]
+    }
+>;
+```
+
+where *`UUIDv4String`* is a stringified version of a UUIDv4 identifier.
+- **name** is a field for your own reference, it is not used by cobalt anywhere.
+
+- **`limit`** specifies how many requests the API key can make during the window specified in the `RATELIMIT_WINDOW` env.
+    - when omitted, the limit specified in `RATELIMIT_MAX` will be used.
+
+- **`ips`** contains an array of allowlisted IP ranges, which can be specified both as individual ips or CIDR ranges (e.g. *`["192.168.42.69", "2001:db8::48", "10.0.0.0/8", "fe80::/10"]`*).
+    - when specified, only requests from these ip ranges can use the specified api key.
+    - when omitted, any IP can be used to make requests with that API key.
+
+- **`userAgents`** contains an array of allowed user agents, with support for wildcards (e.g. *`["cobaltbot/1.0", "Mozilla/5.0 * Chrome/*"]`*).
+    - when specified, requests with a `user-agent` that does not appear in this array will be rejected.
+    - when omitted, any user agent can be specified to make requests with that API key.
+
+- if both `ips` and `userAgents` are set, the tokens will be limited by both parameters.
+- if cobalt detects any problem with your key file, it will be ignored and a warning will be printed to the console.
+
+an example key file could look like this:
+```json
+{
+    "b5c7160a-b655-4c7a-b500-de839f094550": {
+        "limit": 10,
+        "ips": ["10.0.0.0/8", "192.168.42.42"],
+        "userAgents": ["*Chrome*"]
+    }
+}
+```
+
+if you are configuring a key file, **do not use the UUID from the example** but instead generate your own. you can do this by running the following command if you have node.js installed:
+`node -e "console.log(crypto.randomUUID())"`

pnpm-lock.yaml

@@ -38,8 +38,8 @@ importers:
         specifier: ^0.10.7
         version: 0.10.9
       ipaddr.js:
-        specifier: 2.1.0
+        specifier: 2.2.0
-        version: 2.1.0
+        version: 2.2.0
       nanoid:
         specifier: ^4.0.2
         version: 4.0.2
@@ -1448,6 +1448,10 @@ packages:
     resolution: {integrity: sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==}
     engines: {node: '>= 10'}
 
+  ipaddr.js@2.2.0:
+    resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==}
+    engines: {node: '>= 10'}
+
   is-binary-path@2.1.0:
     resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==}
     engines: {node: '>=8'}
@@ -3471,7 +3475,10 @@ snapshots:
 
   ipaddr.js@1.9.1: {}
 
-  ipaddr.js@2.1.0: {}
+  ipaddr.js@2.1.0:
+    optional: true
+
+  ipaddr.js@2.2.0: {}
 
   is-binary-path@2.1.0:
     dependencies: