blitzwing/cobalt-7
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

merge: add freebind support (#490)

Browse source
This commit is contained in:
wukko 2024-05-15 08:47:34 +06:00 committed by GitHub
commit bd16aec699
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 382 additions and 312 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
Dockerfile
View file

@ -1,14 +1,13 @@
FROM node:18-bullseye-slim
WORKDIR /app
RUN apt-get update
RUN apt-get install -y git
RUN rm -rf /var/lib/apt/lists/*
COPY package*.json ./
RUN npm install
RUN git clone -n https://github.com/imputnet/cobalt.git --depth 1 && mv cobalt/.git ./ && rm -rf cobalt
RUN apt-get update && \
apt-get install -y git python3 build-essential && \
npm install && \
apt purge --autoremove -y python3 build-essential && \
rm -rf ~/.cache/ /var/lib/apt/lists/*
COPY . .
EXPOSE 9000

8
docs/run-an-instance.md
View file

@ -53,6 +53,7 @@ sudo service nscd start
| variable name | default | example | description |
|:----------------------|:----------|:------------------------|:------------|
| `API_PORT` | `9000` | `9000` | changes port from which api server is accessible. |
| `API_LISTEN_ADDRESS` | `0.0.0.0` | `127.0.0.1` | changes address from which api server is accessible. **if you are using docker, you usually don't need to configure this.** |
| `API_URL` | ➖ | `https://co.wuk.sh/` | changes url from which api server is accessible. <br> ***REQUIRED TO RUN API***. |
| `API_NAME` | `unknown` | `ams-1` | api server name that is shown in `/api/serverInfo`. |
| `CORS_WILDCARD` | `1` | `0` | toggles cross-origin resource sharing. <br> `0`: disabled. `1`: enabled. |
@ -60,6 +61,7 @@ sudo service nscd start
| `COOKIE_PATH` | not used | `/cookies.json` | path for cookie file relative to main folder. |
| `PROCESSING_PRIORITY` | not used | `10` | changes `nice` value* for ffmpeg subprocess. available only on unix systems. |
| `TIKTOK_DEVICE_INFO` | ➖ | *see below* | device info (including `iid` and `device_id`) for tiktok functionality. required for tiktok to work. |
| `FREEBIND_CIDR` | ➖ | `2001:db8::/32` | IPv6 prefix used for randomly assigning addresses to cobalt requests. only supported on linux systems. for more info, see below. |
\* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)).
@ -85,6 +87,12 @@ you can compress the json to save space. if you're using a `.env` file then the
TIKTOK_DEVICE_INFO='{"iid":"<install_id here>","device_id":"<device_id here>","channel":"googleplay","app_name":"musical_ly","version_code":"310503","device_platform":"android","device_type":"Redmi+7","os_version":"13"}'
```
#### FREEBIND_CIDR
setting a `FREEBIND_CIDR` allows cobalt to pick a random IP for every download and use it for all
requests it makes for that particular download. to use freebind in cobalt, you need to follow its [setup instructions](https://github.com/imputnet/freebind.js?tab=readme-ov-file#setup) first. if you configure this option while running cobalt
in a docker container, you also need to set the `API_LISTEN_ADDRESS` env to `127.0.0.1`, and set
`network_mode` for the container to `host`.
### variables for web
| variable name | default | example | description |
|:---------------------|:---------------------|:------------------------|:--------------------------------------------------------------------------------------|

5
package.json
View file

@ -38,8 +38,11 @@
"node-cache": "^5.1.2",
"psl": "1.9.0",
"set-cookie-parser": "2.6.0",
"undici": "^6.7.0",
"undici": "^5.19.1",
"url-pattern": "1.0.3",
"youtubei.js": "^9.3.0"
},
"optionalDependencies": {
"freebind": "^0.2.2"
}
}

2
src/core/api.js
View file

@ -194,7 +194,7 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
res.redirect('/api/json')
});
app.listen(env.apiPort, () => {
app.listen(env.apiPort, env.listenAddress, () => {
console.log(`\n` +
`${Cyan("cobalt")} API ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\n` +
`Start time: ${Bright(`${startTime.toUTCString()} (${startTimestamp})`)}\n\n` +

7
src/modules/config.js
View file

@ -1,5 +1,6 @@
import UrlPattern from "url-pattern";
import { loadJSON } from "./sub/loadFromFs.js";
const config = loadJSON("./src/config.json");
const packageJson = loadJSON("./package.json");
const servicesConfigJson = loadJSON("./src/modules/processing/servicesConfig.json");
@ -29,13 +30,15 @@ const
apiEnvs = {
apiPort: process.env.API_PORT || 9000,
apiName: process.env.API_NAME || 'unknown',
listenAddress: process.env.API_LISTEN_ADDRESS,
corsWildcard: process.env.CORS_WILDCARD !== '0',
corsURL: process.env.CORS_URL,
cookiePath: process.env.COOKIE_PATH,
processingPriority: process.platform !== "win32"
processingPriority: process.platform !== 'win32'
&& process.env.PROCESSING_PRIORITY
&& parseInt(process.env.PROCESSING_PRIORITY),
tiktokDeviceInfo: process.env.TIKTOK_DEVICE_INFO && JSON.parse(process.env.TIKTOK_DEVICE_INFO),
freebindCIDR: process.platform === 'linux' && process.env.FREEBIND_CIDR,
apiURL
}
@ -46,7 +49,7 @@ export const
streamLifespan = config.streamLifespan,
maxVideoDuration = config.maxVideoDuration,
genericUserAgent = config.genericUserAgent,
repo = packageJson["bugs"]["url"].replace('/issues', ''),
repo = packageJson.bugs.url.replace('/issues', ''),
authorInfo = config.authorInfo,
donations = config.donations,
ffmpegArgs = config.ffmpegArgs,

21
src/modules/processing/match.js
View file

@ -25,9 +25,21 @@ import streamable from "./services/streamable.js";
import twitch from "./services/twitch.js";
import rutube from "./services/rutube.js";
import dailymotion from "./services/dailymotion.js";
import { env } from '../config.js';
let freebind;
export default async function(host, patternMatch, url, lang, obj) {
assert(url instanceof URL);
let dispatcher, requestIP;
if (env.freebindCIDR) {
if (!freebind) {
freebind = await import('freebind');
}
requestIP = freebind.ip.random(env.freebindCIDR);
dispatcher = freebind.dispatcherFromIP(requestIP, { strict: false });
}
try {
let r, isAudioOnly = !!obj.isAudioOnly, disableMetadata = !!obj.disableMetadata;
@ -66,7 +78,8 @@ export default async function(host, patternMatch, url, lang, obj) {
format: obj.vCodec,
isAudioOnly: isAudioOnly,
isAudioMuted: obj.isAudioMuted,
dubLang: obj.dubLang
dubLang: obj.dubLang,
dispatcher
}
if (url.hostname === 'music.youtube.com' || isAudioOnly === true) {
@ -122,7 +135,8 @@ export default async function(host, patternMatch, url, lang, obj) {
case "instagram":
r = await instagram({
...patternMatch,
quality: obj.vQuality
quality: obj.vQuality,
dispatcher
})
break;
case "vine":
@ -181,7 +195,8 @@ export default async function(host, patternMatch, url, lang, obj) {
return matchActionDecider(
r, host, obj.aFormat, isAudioOnly,
lang, isAudioMuted, disableMetadata,
obj.filenamePattern, obj.twitterGif
obj.filenamePattern, obj.twitterGif,
requestIP
)
} catch (e) {
return apiJSON(0, { t: genericError(lang, host) })

5
src/modules/processing/matchActionDecider.js
View file

@ -3,7 +3,7 @@ import { apiJSON } from "../sub/utils.js";
import loc from "../../localization/manager.js";
import createFilename from "./createFilename.js";
export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern, toGif) {
export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern, toGif, requestIP) {
let action,
responseType = 2,
defaultParams = {
@ -11,7 +11,8 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
service: host,
filename: r.filenameAttributes ?
createFilename(r.filenameAttributes, filenamePattern, isAudioOnly, isAudioMuted) : r.filename,
fileMetadata: !disableMetadata ? r.fileMetadata : false
fileMetadata: !disableMetadata ? r.fileMetadata : false,
requestIP
},
params = {},
audioFormat = String(userFormat);

17
src/modules/processing/services/instagram.js
View file

@ -41,6 +41,9 @@ const cachedDtsg = {
expiry: 0
}
export default function(obj) {
const dispatcher = obj.dispatcher;
async function findDtsgId(cookie) {
try {
if (cachedDtsg.expiry > Date.now()) return cachedDtsg.value;
@ -49,7 +52,8 @@ async function findDtsgId(cookie) {
headers: {
...commonHeaders,
cookie
}
},
dispatcher
}).then(r => r.text());
const token = data.match(/"dtsg":{"token":"(.*?)"/)[1];
@ -78,6 +82,7 @@ async function request(url, cookie, method = 'GET', requestData) {
method,
headers,
body: requestData && new URLSearchParams(requestData),
dispatcher
});
if (data.headers.get('X-Ig-Set-Www-Claim') && cookie)
@ -95,7 +100,8 @@ async function getMediaId(id, { cookie, token } = {}) {
...mobileHeaders,
...( token && { authorization: `Bearer ${token}` } ),
cookie
}
},
dispatcher
}).then(r => r.json()).catch(() => {});
return oembed?.media_id;
@ -107,7 +113,8 @@ async function requestMobileApi(mediaId, { cookie, token } = {}) {
...mobileHeaders,
...( token && { authorization: `Bearer ${token}` } ),
cookie
}
},
dispatcher
}).then(r => r.json()).catch(() => {});
return mediaInfo?.items?.[0];
@ -117,7 +124,8 @@ async function requestHTML(id, cookie) {
headers: {
...embedHeaders,
cookie
}
},
dispatcher
}).then(r => r.text()).catch(() => {});
let embedData = JSON.parse(data?.match(/"init",\[\],\[(.*?)\]\],/)[1]);
@ -333,7 +341,6 @@ async function getStory(username, id) {
return { error: 'ErrorCouldntFetch' };
}
export default function(obj) {
const { postId, storyId, username } = obj;
if (postId) return getPost(postId);
if (username && storyId) return getStory(username, storyId);

25
src/modules/processing/services/youtube.js
View file

@ -1,8 +1,9 @@
import { Innertube } from 'youtubei.js';
import { Innertube, Session } from 'youtubei.js';
import { maxVideoDuration } from '../../config.js';
import { cleanString } from '../../sub/utils.js';
import { fetch } from 'undici'
const yt = await Innertube.create();
const ytBase = await Innertube.create();
const codecMatch = {
h264: {
@ -22,7 +23,27 @@ const codecMatch = {
}
}
const cloneInnertube = (customFetch) => {
const session = new Session(
ytBase.session.context,
ytBase.session.key,
ytBase.session.api_version,
ytBase.session.account_index,
ytBase.session.player,
undefined,
customFetch ?? ytBase.session.http.fetch,
ytBase.session.cache
);
const yt = new Innertube(session);
return yt;
}
export default async function(o) {
const yt = cloneInnertube(
(input, init) => fetch(input, { ...init, dispatcher: o.dispatcher })
);
let info, isDubbed, format = o.format || "h264";
let quality = o.quality === "max" ? "9000" : o.quality; // 9000(p) - max quality

3
src/modules/stream/internal.js
View file

@ -18,6 +18,7 @@ async function* readChunks(streamInfo, size) {
...getHeaders('youtube'),
Range: `bytes=${read}-${read + CHUNK_SIZE}`
},
dispatcher: streamInfo.dispatcher,
signal: streamInfo.controller.signal
});
@ -47,6 +48,7 @@ async function handleYoutubeStream(streamInfo, res) {
const req = await fetch(streamInfo.url, {
headers: getHeaders('youtube'),
method: 'HEAD',
dispatcher: streamInfo.dispatcher,
signal: streamInfo.controller.signal
});
@ -81,6 +83,7 @@ export async function internalStream(streamInfo, res) {
...streamInfo.headers,
host: undefined
},
dispatcher: streamInfo.dispatcher,
signal: streamInfo.controller.signal,
maxRedirections: 16
});

14
src/modules/stream/manage.js
View file

@ -6,6 +6,9 @@ import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js";
import { streamLifespan, env } from "../config.js";
import { strict as assert } from "assert";
// optional dependency
const freebind = env.freebindCIDR && await import('freebind').catch(() => {});
const M3U_SERVICES = ['dailymotion', 'vimeo', 'rutube'];
const streamNoAccess = {
@ -46,7 +49,8 @@ export function createStream(obj) {
isAudioOnly: !!obj.isAudioOnly,
copy: !!obj.copy,
mute: !!obj.mute,
metadata: obj.fileMetadata || false
metadata: obj.fileMetadata || false,
requestIP: obj.requestIP
};
streamCache.set(
@ -78,11 +82,17 @@ export function getInternalStream(id) {
export function createInternalStream(url, obj = {}) {
assert(typeof url === 'string');
let dispatcher;
if (obj.requestIP) {
dispatcher = freebind?.dispatcherFromIP(obj.requestIP, { strict: false })
}
const streamID = nanoid();
internalStreamCache[streamID] = {
url,
service: obj.service,
controller: new AbortController()
controller: new AbortController(),
dispatcher
};
let streamLink = new URL('/api/istream', `http://127.0.0.1:${env.apiPort}`);