merge: add freebind support (#490)

Dockerfile

@@ -1,14 +1,13 @@
 FROM node:18-bullseye-slim
 WORKDIR /app
 
-RUN apt-get update
-RUN apt-get install -y git
-RUN rm -rf /var/lib/apt/lists/*
-
 COPY package*.json ./
-RUN npm install
 
-RUN git clone -n https://github.com/imputnet/cobalt.git --depth 1 && mv cobalt/.git ./ && rm -rf cobalt
+RUN apt-get update && \
+    apt-get install -y git python3 build-essential && \
+    npm install && \
+    apt purge --autoremove -y python3 build-essential && \
+    rm -rf ~/.cache/ /var/lib/apt/lists/*
 
 COPY . .
 EXPOSE 9000

docs/run-an-instance.md

@@ -53,6 +53,7 @@ sudo service nscd start
 | variable name         | default   | example                 | description |
 |:----------------------|:----------|:------------------------|:------------|
 | `API_PORT`            | `9000`    |  `9000`                 | changes port from which api server is accessible. |
+| `API_LISTEN_ADDRESS`  | `0.0.0.0` |  `127.0.0.1`            | changes address from which api server is accessible. **if you are using docker, you usually don't need to configure this.** |
 | `API_URL`             | ➖        | `https://co.wuk.sh/`    | changes url from which api server is accessible. <br> ***REQUIRED TO RUN API***. |
 | `API_NAME`            | `unknown` | `ams-1`                 | api server name that is shown in `/api/serverInfo`. |
 | `CORS_WILDCARD`       | `1`       | `0`                     | toggles cross-origin resource sharing. <br> `0`: disabled. `1`: enabled. |
@@ -60,6 +61,7 @@ sudo service nscd start
 | `COOKIE_PATH`         | not used  | `/cookies.json`         | path for cookie file relative to main folder. |
 | `PROCESSING_PRIORITY` | not used  | `10`                    | changes `nice` value* for ffmpeg subprocess. available only on unix systems. |
 | `TIKTOK_DEVICE_INFO`  | ➖        | *see below*             | device info (including `iid` and `device_id`) for tiktok functionality. required for tiktok to work. |
+| `FREEBIND_CIDR`       | ➖        | `2001:db8::/32`         | IPv6 prefix used for randomly assigning addresses to cobalt requests. only supported on linux systems. for more info, see below. |
 
 \* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)).
 
@@ -85,6 +87,12 @@ you can compress the json to save space. if you're using a `.env` file then the
 TIKTOK_DEVICE_INFO='{"iid":"<install_id here>","device_id":"<device_id here>","channel":"googleplay","app_name":"musical_ly","version_code":"310503","device_platform":"android","device_type":"Redmi+7","os_version":"13"}'
 ```
 
+#### FREEBIND_CIDR
+setting a `FREEBIND_CIDR` allows cobalt to pick a random IP for every download and use it for all
+requests it makes for that particular download. to use freebind in cobalt, you need to follow its [setup instructions](https://github.com/imputnet/freebind.js?tab=readme-ov-file#setup) first. if you configure this option while running cobalt
+in a docker container, you also need to set the `API_LISTEN_ADDRESS` env to `127.0.0.1`, and set
+`network_mode` for the container to `host`.
+
 ### variables for web
 | variable name        | default              | example                 | description                                                                           |
 |:---------------------|:---------------------|:------------------------|:--------------------------------------------------------------------------------------|

package.json

@@ -38,8 +38,11 @@
         "node-cache": "^5.1.2",
         "psl": "1.9.0",
         "set-cookie-parser": "2.6.0",
-        "undici": "^6.7.0",
+        "undici": "^5.19.1",
         "url-pattern": "1.0.3",
         "youtubei.js": "^9.3.0"
+    },
+    "optionalDependencies": {
+        "freebind": "^0.2.2"
     }
 }

src/core/api.js

@@ -194,7 +194,7 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
         res.redirect('/api/json')
     });
 
-    app.listen(env.apiPort, () => {
+    app.listen(env.apiPort, env.listenAddress, () => {
         console.log(`\n` +
             `${Cyan("cobalt")} API ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\n` +
             `Start time: ${Bright(`${startTime.toUTCString()} (${startTimestamp})`)}\n\n` +

src/modules/config.js

@@ -1,5 +1,6 @@
 import UrlPattern from "url-pattern";
 import { loadJSON } from "./sub/loadFromFs.js";
+
 const config = loadJSON("./src/config.json");
 const packageJson = loadJSON("./package.json");
 const servicesConfigJson = loadJSON("./src/modules/processing/servicesConfig.json");
@@ -29,13 +30,15 @@ const
     apiEnvs = {
         apiPort: process.env.API_PORT || 9000,
         apiName: process.env.API_NAME || 'unknown',
+        listenAddress: process.env.API_LISTEN_ADDRESS,
         corsWildcard: process.env.CORS_WILDCARD !== '0',
         corsURL: process.env.CORS_URL,
         cookiePath: process.env.COOKIE_PATH,
-        processingPriority: process.platform !== "win32"
+        processingPriority: process.platform !== 'win32'
                                 && process.env.PROCESSING_PRIORITY
                                 && parseInt(process.env.PROCESSING_PRIORITY),
         tiktokDeviceInfo: process.env.TIKTOK_DEVICE_INFO && JSON.parse(process.env.TIKTOK_DEVICE_INFO),
+        freebindCIDR: process.platform === 'linux' && process.env.FREEBIND_CIDR,
         apiURL
     }
 
@@ -46,7 +49,7 @@ export const
     streamLifespan = config.streamLifespan,
     maxVideoDuration = config.maxVideoDuration,
     genericUserAgent = config.genericUserAgent,
-    repo = packageJson["bugs"]["url"].replace('/issues', ''),
+    repo = packageJson.bugs.url.replace('/issues', ''),
     authorInfo = config.authorInfo,
     donations = config.donations,
     ffmpegArgs = config.ffmpegArgs,

src/modules/processing/match.js

@@ -25,9 +25,21 @@ import streamable from "./services/streamable.js";
 import twitch from "./services/twitch.js";
 import rutube from "./services/rutube.js";
 import dailymotion from "./services/dailymotion.js";
+import { env } from '../config.js';
 
+let freebind;
 export default async function(host, patternMatch, url, lang, obj) {
     assert(url instanceof URL);
+    let dispatcher, requestIP;
+
+    if (env.freebindCIDR) {
+        if (!freebind) {
+            freebind = await import('freebind');
+        }
+
+        requestIP = freebind.ip.random(env.freebindCIDR);
+        dispatcher = freebind.dispatcherFromIP(requestIP, { strict: false });
+    }
 
     try {
         let r, isAudioOnly = !!obj.isAudioOnly, disableMetadata = !!obj.disableMetadata;
@@ -66,7 +78,8 @@ export default async function(host, patternMatch, url, lang, obj) {
                     format: obj.vCodec,
                     isAudioOnly: isAudioOnly,
                     isAudioMuted: obj.isAudioMuted,
-                    dubLang: obj.dubLang
+                    dubLang: obj.dubLang,
+                    dispatcher
                 }
 
                 if (url.hostname === 'music.youtube.com' || isAudioOnly === true) {
@@ -122,7 +135,8 @@ export default async function(host, patternMatch, url, lang, obj) {
             case "instagram":
                 r = await instagram({
                     ...patternMatch,
-                    quality: obj.vQuality
+                    quality: obj.vQuality,
+                    dispatcher
                 })
                 break;
             case "vine":
@@ -181,7 +195,8 @@ export default async function(host, patternMatch, url, lang, obj) {
         return matchActionDecider(
             r, host, obj.aFormat, isAudioOnly,
             lang, isAudioMuted, disableMetadata,
-            obj.filenamePattern, obj.twitterGif
+            obj.filenamePattern, obj.twitterGif,
+            requestIP
         )
     } catch (e) {
         return apiJSON(0, { t: genericError(lang, host) })

src/modules/processing/matchActionDecider.js

@@ -3,7 +3,7 @@ import { apiJSON } from "../sub/utils.js";
 import loc from "../../localization/manager.js";
 import createFilename from "./createFilename.js";
 
-export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern, toGif) {
+export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern, toGif, requestIP) {
     let action,
         responseType = 2,
         defaultParams = {
@@ -11,7 +11,8 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
             service: host,
             filename: r.filenameAttributes ?
                     createFilename(r.filenameAttributes, filenamePattern, isAudioOnly, isAudioMuted) : r.filename,
-            fileMetadata: !disableMetadata ? r.fileMetadata : false
+            fileMetadata: !disableMetadata ? r.fileMetadata : false,
+            requestIP
         },
         params = {},
         audioFormat = String(userFormat);

src/modules/processing/services/instagram.js

@@ -41,6 +41,9 @@ const cachedDtsg = {
     expiry: 0
 }
 
+export default function(obj) {
+    const dispatcher = obj.dispatcher;
+
     async function findDtsgId(cookie) {
         try {
             if (cachedDtsg.expiry > Date.now()) return cachedDtsg.value;
@@ -49,7 +52,8 @@ async function findDtsgId(cookie) {
                 headers: {
                     ...commonHeaders,
                     cookie
-            }
+                },
+                dispatcher
             }).then(r => r.text());
 
             const token = data.match(/"dtsg":{"token":"(.*?)"/)[1];
@@ -78,6 +82,7 @@ async function request(url, cookie, method = 'GET', requestData) {
             method,
             headers,
             body: requestData && new URLSearchParams(requestData),
+            dispatcher
         });
 
         if (data.headers.get('X-Ig-Set-Www-Claim') && cookie)
@@ -95,7 +100,8 @@ async function getMediaId(id, { cookie, token } = {}) {
                 ...mobileHeaders,
                 ...( token && { authorization: `Bearer ${token}` } ),
                 cookie
-        }
+            },
+            dispatcher
         }).then(r => r.json()).catch(() => {});
 
         return oembed?.media_id;
@@ -107,7 +113,8 @@ async function requestMobileApi(mediaId, { cookie, token } = {}) {
                 ...mobileHeaders,
                 ...( token && { authorization: `Bearer ${token}` } ),
                 cookie
-        }
+            },
+            dispatcher
         }).then(r => r.json()).catch(() => {});
 
         return mediaInfo?.items?.[0];
@@ -117,7 +124,8 @@ async function requestHTML(id, cookie) {
             headers: {
                 ...embedHeaders,
                 cookie
-        }
+            },
+            dispatcher
         }).then(r => r.text()).catch(() => {});
 
         let embedData = JSON.parse(data?.match(/"init",\[\],\[(.*?)\]\],/)[1]);
@@ -333,7 +341,6 @@ async function getStory(username, id) {
         return { error: 'ErrorCouldntFetch' };
     }
 
-export default function(obj) {
     const { postId, storyId, username } = obj;
     if (postId) return getPost(postId);
     if (username && storyId) return getStory(username, storyId);

src/modules/processing/services/youtube.js

@@ -1,8 +1,9 @@
-import { Innertube } from 'youtubei.js';
+import { Innertube, Session } from 'youtubei.js';
 import { maxVideoDuration } from '../../config.js';
 import { cleanString } from '../../sub/utils.js';
+import { fetch } from 'undici'
 
-const yt = await Innertube.create();
+const ytBase = await Innertube.create();
 
 const codecMatch = {
     h264: {
@@ -22,7 +23,27 @@ const codecMatch = {
     }
 }
 
+const cloneInnertube = (customFetch) => {
+    const session = new Session(
+        ytBase.session.context,
+        ytBase.session.key,
+        ytBase.session.api_version,
+        ytBase.session.account_index,
+        ytBase.session.player,
+        undefined,
+        customFetch ?? ytBase.session.http.fetch,
+        ytBase.session.cache
+    );
+
+    const yt = new Innertube(session);
+    return yt;
+}
+
 export default async function(o) {
+    const yt = cloneInnertube(
+        (input, init) => fetch(input, { ...init, dispatcher: o.dispatcher })
+    );
+
     let info, isDubbed, format = o.format || "h264";
     let quality = o.quality === "max" ? "9000" : o.quality; // 9000(p) - max quality
 

src/modules/stream/internal.js

@@ -18,6 +18,7 @@ async function* readChunks(streamInfo, size) {
                 ...getHeaders('youtube'),
                 Range: `bytes=${read}-${read + CHUNK_SIZE}`
             },
+            dispatcher: streamInfo.dispatcher,
             signal: streamInfo.controller.signal
         });
 
@@ -47,6 +48,7 @@ async function handleYoutubeStream(streamInfo, res) {
         const req = await fetch(streamInfo.url, {
             headers: getHeaders('youtube'),
             method: 'HEAD',
+            dispatcher: streamInfo.dispatcher,
             signal: streamInfo.controller.signal
         });
 
@@ -81,6 +83,7 @@ export async function internalStream(streamInfo, res) {
                 ...streamInfo.headers,
                 host: undefined
             },
+            dispatcher: streamInfo.dispatcher,
             signal: streamInfo.controller.signal,
             maxRedirections: 16
         });

src/modules/stream/manage.js

@@ -6,6 +6,9 @@ import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js";
 import { streamLifespan, env } from "../config.js";
 import { strict as assert } from "assert";
 
+// optional dependency
+const freebind = env.freebindCIDR && await import('freebind').catch(() => {});
+
 const M3U_SERVICES = ['dailymotion', 'vimeo', 'rutube'];
 
 const streamNoAccess = {
@@ -46,7 +49,8 @@ export function createStream(obj) {
             isAudioOnly: !!obj.isAudioOnly,
             copy: !!obj.copy,
             mute: !!obj.mute,
-            metadata: obj.fileMetadata || false
+            metadata: obj.fileMetadata || false,
+            requestIP: obj.requestIP
         };
 
     streamCache.set(
@@ -78,11 +82,17 @@ export function getInternalStream(id) {
 export function createInternalStream(url, obj = {}) {
     assert(typeof url === 'string');
 
+    let dispatcher;
+    if (obj.requestIP) {
+        dispatcher = freebind?.dispatcherFromIP(obj.requestIP, { strict: false })
+    }
+
     const streamID = nanoid();
     internalStreamCache[streamID] = {
         url,
         service: obj.service,
-        controller: new AbortController()
+        controller: new AbortController(),
+        dispatcher
     };
 
     let streamLink = new URL('/api/istream', `http://127.0.0.1:${env.apiPort}`);