api: flatten code directories, better filenames, remove old files

api/package.json

@@ -10,9 +10,8 @@
     },
     "scripts": {
         "start": "node src/cobalt",
-        "setup": "node src/modules/setup",
+        "setup": "node src/util/setup",
         "test": "node src/util/test",
-        "build": "node src/modules/buildStatic",
         "token:youtube": "node src/util/generate-youtube-tokens"
     },
     "repository": {

api/src/cobalt.js

@@ -1,11 +1,11 @@
 import "dotenv/config";
-import "./modules/sub/alias-envs.js";
+import "./misc/alias-envs.js";
 
 import express from "express";
 
-import { Bright, Green, Red } from "./modules/sub/consoleText.js";
-import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js";
-import { env } from "./modules/config.js"
+import { Bright, Green, Red } from "./misc/console-text.js";
+import { getCurrentBranch, shortCommit } from "./misc/current-commit.js";
+import { env } from "./config.js"
 
 import path from 'path';
 import { fileURLToPath } from 'url';

api/src/config.js

@@ -1,9 +1,8 @@
 import UrlPattern from "url-pattern";
-import { loadJSON } from "./sub/loadFromFs.js";
+import { loadJSON } from "./misc/load-from-fs.js";
 
-const config = loadJSON("./src/config.json");
 const packageJson = loadJSON("./package.json");
-const servicesConfigJson = loadJSON("./src/modules/processing/servicesConfig.json");
+const servicesConfigJson = loadJSON("./src/processing/service-config.json");
 
 Object.values(servicesConfigJson.config).forEach(service => {
     service.patterns = service.patterns.map(
@@ -14,13 +13,23 @@ Object.values(servicesConfigJson.config).forEach(service => {
 })
 
 export const
+    version = packageJson.version,
+
+    genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
+    supportedAudio = ["mp3", "ogg", "wav", "opus"],
+    ffmpegArgs = {
+        webm: ["-c:v", "copy", "-c:a", "copy"],
+        mp4: ["-c:v", "copy", "-c:a", "copy", "-movflags", "faststart+frag_keyframe+empty_moov"],
+        copy: ["-c:a", "copy"],
+        audio: ["-ar", "48000", "-ac", "2", "-b:a", "320k"],
+        m4a: ["-movflags", "frag_keyframe+empty_moov"],
+        gif: ["-vf", "scale=-1:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse", "-loop", "0"]
+    },
+
     services = servicesConfigJson.config,
     hlsExceptions = servicesConfigJson.hlsExceptions,
     audioIgnore = servicesConfigJson.audioIgnore,
-    version = packageJson.version,
-    genericUserAgent = config.genericUserAgent,
-    ffmpegArgs = config.ffmpegArgs,
-    supportedAudio = config.supportedAudio,
+
     env = {
         apiURL: process.env.API_URL || '',
         apiPort: process.env.API_PORT || 9000,
@@ -33,15 +42,15 @@ export const
 
         cookiePath: process.env.COOKIE_PATH,
 
-        rateLimitWindow: (process.env.RATELIMIT_WINDOW  && parseInt(process.env.RATELIMIT_WINDOW)) || 60,
+        rateLimitWindow: (process.env.RATELIMIT_WINDOW && parseInt(process.env.RATELIMIT_WINDOW)) || 60,
         rateLimitMax: (process.env.RATELIMIT_MAX && parseInt(process.env.RATELIMIT_MAX)) || 20,
 
         durationLimit: (process.env.DURATION_LIMIT && parseInt(process.env.DURATION_LIMIT)) || 10800,
         streamLifespan: 90,
 
         processingPriority: process.platform !== 'win32'
-                                && process.env.PROCESSING_PRIORITY
-                                && parseInt(process.env.PROCESSING_PRIORITY),
+            && process.env.PROCESSING_PRIORITY
+            && parseInt(process.env.PROCESSING_PRIORITY),
 
         externalProxy: process.env.API_EXTERNAL_PROXY,
     }

api/src/config.json

@@ -1,103 +0,0 @@
-{
-    "genericUserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
-    "authorInfo": {
-        "support": {
-            "default": {
-                "email": {
-                    "emoji": "📧",
-                    "url": "mailto:support@cobalt.tools",
-                    "name": "support@cobalt.tools"
-                },
-                "twitter": {
-                    "emoji": "🐦",
-                    "url": "https://twitter.com/justusecobalt",
-                    "name": "@justusecobalt"
-                },
-                "discord": {
-                    "emoji": "👾",
-                    "url": "https://discord.gg/pQPt8HBUPu",
-                    "name": "cobalt discord server"
-                }
-            },
-            "ru": {
-                "telegram": {
-                    "emoji": "📬",
-                    "url": "https://t.me/justusecobalt_ru",
-                    "name": "канал в telegram"
-                },
-                "email": {
-                    "emoji": "📧",
-                    "url": "mailto:support@cobalt.tools",
-                    "name": "support@cobalt.tools"
-                }
-            }
-        }
-    },
-    "donations": {
-        "crypto": {
-            "monero": "4B1SNB6s8Pq1hxjNeKPEe8Qa8EP3zdL16Sqsa7QDoJcUecKQzEj9BMxWnEnTGu12doKLJBKRDUqnn6V9qfSdXpXi3Nw5Uod",
-            "litecoin": "ltc1qvp0xhrk2m7pa6p6z844qcslfyxv4p3vf95rhna",
-            "ethereum": "0x4B4cF23051c78c7A7E0eA09d39099621c46bc302",
-            "usdt-erc20": "0x4B4cF23051c78c7A7E0eA09d39099621c46bc302",
-            "usdt-trc20": "TVbx7YT3rBfu931Gxko6pRfXtedYqbgnBB",
-            "bitcoin": "bc1qlvcnlnyzfsgnuxyxsv3k0p0q0yln0azjpadyx4",
-            "bitcoin-alt": "18PKf6N2cHrmSzz9ZzTSvDd2jAkqGC7SxA",
-            "ton": "UQA3SO-hHZq1oCCT--u6or6ollB8fd2o52aD8mXiLk9iDZd3"
-        },
-        "links": {
-            "boosty": "https://boosty.to/wukko/donate"
-        }
-    },
-    "links": {
-        "saveToGalleryShortcut": "https://www.icloud.com/shortcuts/14e9aebf04b24156acc34ceccf7e6fcd",
-        "saveToFilesShortcut": "https://www.icloud.com/shortcuts/2134cd9d4d6b41448b2201f933542b2e",
-        "statusPage": "https://status.cobalt.tools/",
-        "troubleshootingGuide": "https://github.com/imputnet/cobalt/blob/current/docs/troubleshooting.md"
-    },
-    "celebrations": {
-        "01-01": "🎄",
-        "02-17": "😺",
-        "02-22": "😺",
-        "03-01": "😺",
-        "03-08": "💪",
-        "05-26": "🎂",
-        "08-08": "😺",
-        "08-26": "🐶",
-        "10-29": "😺",
-        "10-30": "🎃",
-        "10-31": "🎃",
-        "11-01": "🕯️",
-        "11-02": "🕯️",
-        "12-20": "🎄",
-        "12-21": "🎄",
-        "12-22": "🎄",
-        "12-23": "🎄",
-        "12-24": "🎄",
-        "12-25": "🎄",
-        "12-26": "🎄",
-        "12-27": "🎄",
-        "12-28": "🎄",
-        "12-29": "🎄",
-        "12-30": "🎄",
-        "12-31": "🎄"
-    },
-    "supportedAudio": ["mp3", "ogg", "wav", "opus"],
-    "ffmpegArgs": {
-        "webm": ["-c:v", "copy", "-c:a", "copy"],
-        "mp4": ["-c:v", "copy", "-c:a", "copy", "-movflags", "faststart+frag_keyframe+empty_moov"],
-        "copy": ["-c:a", "copy"],
-        "audio": ["-ar", "48000", "-ac", "2", "-b:a", "320k"],
-        "m4a": ["-movflags", "frag_keyframe+empty_moov"],
-        "gif": ["-vf", "scale=-1:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse", "-loop", "0"]
-    },
-    "sponsors": [{
-        "name": "royale",
-        "fullName": "RoyaleHosting",
-        "url": "https://royalehosting.net/?partner=cobalt",
-        "logo": {
-            "width": 605,
-            "height": 136,
-            "scale": 5
-        }
-    }]
-}

api/src/core/api.js

@@ -2,18 +2,18 @@ import cors from "cors";
 import rateLimit from "express-rate-limit";
 import { setGlobalDispatcher, ProxyAgent } from "undici";
 
-import { env, version } from "../modules/config.js";
+import { env, version } from "../config.js";
 
-import { generateHmac, generateSalt } from "../modules/sub/crypto.js";
-import { Bright, Cyan } from "../modules/sub/consoleText.js";
-import { languageCode } from "../modules/sub/utils.js";
+import { generateHmac, generateSalt } from "../misc/crypto.js";
+import { Bright, Cyan } from "../misc/console-text.js";
+import { languageCode } from "../misc/utils.js";
 
-import { createResponse, normalizeRequest, getIP } from "../modules/processing/request.js";
-import { verifyStream, getInternalStream } from "../modules/stream/manage.js";
-import { randomizeCiphers } from '../modules/sub/randomize-ciphers.js';
-import { extract } from "../modules/processing/url.js";
-import match from "../modules/processing/match.js";
-import stream from "../modules/stream/stream.js";
+import { createResponse, normalizeRequest, getIP } from "../processing/request.js";
+import { verifyStream, getInternalStream } from "../stream/manage.js";
+import { randomizeCiphers } from '../misc/randomize-ciphers.js';
+import { extract } from "../processing/url.js";
+import match from "../processing/match.js";
+import stream from "../stream/stream.js";
 
 const acceptRegex = /^application\/json(; charset=utf-8)?$/;
 

api/src/misc/alias-envs.js

@@ -1,4 +1,4 @@
-import { Red } from "./consoleText.js";
+import { Red } from "./console-text.js";
 
 const mapping = {
     apiPort: 'API_PORT',

api/src/misc/run-test.js

@@ -1,6 +1,6 @@
-import { normalizeRequest } from "../modules/processing/request.js";
-import match from "./processing/match.js";
-import { extract } from "./processing/url.js";
+import { normalizeRequest } from "../processing/request.js";
+import match from "../processing/match.js";
+import { extract } from "../processing/url.js";
 
 export async function runTest(url, params, expect) {
     const normalized = normalizeRequest({ url, ...params });

api/src/processing/cookie/manager.js

@@ -1,7 +1,7 @@
 import Cookie from './cookie.js';
 import { readFile, writeFile } from 'fs/promises';
 import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser';
-import { env } from '../../../modules/config.js'
+import { env } from '../../config.js';
 
 const WRITE_INTERVAL = 60000,
       cookiePath = env.cookiePath,

api/src/processing/match-action.js

@@ -1,6 +1,6 @@
 import { audioIgnore, services, supportedAudio } from "../config.js";
-import { createResponse } from "../processing/request.js";
-import createFilename from "./createFilename.js";
+import { createResponse } from "./request.js";
+import createFilename from "./create-filename.js";
 import { createStream } from "../stream/manage.js";
 
 export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern, toGif, requestIP) {

api/src/processing/match.js

@@ -1,10 +1,10 @@
 import { strict as assert } from "node:assert";
 
-import { env } from '../config.js';
+import { env } from "../config.js";
 import { createResponse } from "../processing/request.js";
 
-import { testers } from "./servicesPatternTesters.js";
-import matchActionDecider from "./matchActionDecider.js";
+import { testers } from "./service-patterns.js";
+import matchActionDecider from "./match-action.js";
 
 import bilibili from "./services/bilibili.js";
 import reddit from "./services/reddit.js";

api/src/processing/request.js

@@ -1,8 +1,8 @@
 import ipaddr from "ipaddr.js";
 
-import { normalizeURL } from "../processing/url.js";
+import { normalizeURL } from "./url.js";
 import { createStream } from "../stream/manage.js";
-import { verifyLanguageCode } from "../sub/utils.js";
+import { verifyLanguageCode } from "../misc/utils.js";
 
 const apiVar = {
     allowed: {

api/src/processing/service-patterns.js

@@ -31,8 +31,8 @@ export const testers = {
         || patternMatch.shortLink?.length <= 32,
 
     "snapchat": (patternMatch) =>
-        (patternMatch.username?.length <= 32 && (!patternMatch.storyId || patternMatch.storyId?.length <= 255)) 
-        || patternMatch.spotlightId?.length <= 255 
+        (patternMatch.username?.length <= 32 && (!patternMatch.storyId || patternMatch.storyId?.length <= 255))
+        || patternMatch.spotlightId?.length <= 255
         || patternMatch.shortLink?.length <= 16,
 
     "streamable": (patternMatch) =>

api/src/processing/services/ok.js

@@ -1,5 +1,5 @@
 import { genericUserAgent, env } from "../../config.js";
-import { cleanString } from "../../sub/utils.js";
+import { cleanString } from "../../misc/utils.js";
 
 const resolutions = {
     "ultra": "2160",

api/src/processing/services/pinterest.js

@@ -33,7 +33,7 @@ export default async function(o) {
     let imageLink = [...html.matchAll(imageRegex)]
                     .map(([, link]) => link)
                     .find(a => a.endsWith('.jpg') || a.endsWith('.gif'));
-                    
+
     if (imageLink) return {
         urls: imageLink,
         isPhoto: true

api/src/processing/services/rutube.js

@@ -1,7 +1,7 @@
 import HLS from 'hls-parser';
 
 import { env } from "../../config.js";
-import { cleanString } from '../../sub/utils.js';
+import { cleanString } from '../../misc/utils.js';
 
 async function requestJSON(url) {
     try {

api/src/processing/services/snapchat.js

@@ -1,5 +1,5 @@
 import { genericUserAgent } from "../../config.js";
-import { getRedirectingURL } from "../../sub/utils.js";
+import { getRedirectingURL } from "../../misc/utils.js";
 import { extract, normalizeURL } from "../url.js";
 
 const SPOTLIGHT_VIDEO_REGEX = /<link data-react-helmet="true" rel="preload" href="(https:\/\/cf-st\.sc-cdn\.net\/d\/[\w.?=]+&amp;uc=\d+)" as="video"\/>/;

api/src/processing/services/soundcloud.js

@@ -1,5 +1,5 @@
 import { env } from "../../config.js";
-import { cleanString } from "../../sub/utils.js";
+import { cleanString } from "../../misc/utils.js";
 
 const cachedID = {
     version: '',

api/src/processing/services/twitch.js

@@ -1,5 +1,5 @@
 import { env } from "../../config.js";
-import { cleanString } from '../../sub/utils.js';
+import { cleanString } from '../../misc/utils.js';
 
 const gqlURL = "https://gql.twitch.tv/gql";
 const clientIdHead = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" };

api/src/processing/services/vimeo.js

@@ -1,5 +1,5 @@
 import { env } from "../../config.js";
-import { cleanString, merge } from '../../sub/utils.js';
+import { cleanString, merge } from '../../misc/utils.js';
 
 import HLS from "hls-parser";
 

api/src/processing/services/vk.js

@@ -1,5 +1,5 @@
 import { genericUserAgent, env } from "../../config.js";
-import { cleanString } from "../../sub/utils.js";
+import { cleanString } from "../../misc/utils.js";
 
 const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240"];
 

api/src/processing/services/youtube.js

@@ -3,7 +3,7 @@ import { fetch } from "undici";
 import { Innertube, Session } from "youtubei.js";
 
 import { env } from "../../config.js";
-import { cleanString } from "../../sub/utils.js";
+import { cleanString } from "../../misc/utils.js";
 import { getCookie, updateCookieValues } from "../cookie/manager.js";
 
 const PLAYER_REFRESH_PERIOD = 1000 * 60 * 15; // ms

api/src/processing/url.js

@@ -54,6 +54,7 @@ function aliasURL(url) {
                 url = new URL(`https://bilibili.com/_tv${url.pathname}`);
             }
             break;
+
         case "b23":
             if (url.hostname === 'b23.tv' && parts.length === 2) {
                 url = new URL(`https://bilibili.com/_shortLink/${parts[1]}`)
@@ -179,4 +180,4 @@ export function extract(url) {
     }
 
     return { host, patternMatch };
-}
+}

api/src/stream/internal.js

@@ -28,7 +28,7 @@ async function* readChunks(streamInfo, size) {
         if (received < expected / 2n) {
             closeRequest(streamInfo.controller);
         }
-        
+
         for await (const data of chunk.body) {
             yield data;
         }
@@ -64,7 +64,7 @@ async function handleYoutubeStream(streamInfo, res) {
         }
 
         signal.addEventListener('abort', abortGenerator);
-    
+
         const stream = Readable.from(generator);
 
         for (const headerName of ['content-type', 'content-length']) {
@@ -119,4 +119,4 @@ export function internalStream(streamInfo, res) {
     }
 
     return handleGenericStream(streamInfo, res);
-}
+}

api/src/stream/manage.js

@@ -3,7 +3,7 @@ import { randomBytes } from "crypto";
 import { nanoid } from "nanoid";
 import { setMaxListeners } from "node:events";
 
-import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js";
+import { decryptStream, encryptStream, generateHmac } from "../misc/crypto.js";
 import { env } from "../config.js";
 import { strict as assert } from "assert";
 import { closeRequest } from "./shared.js";
@@ -83,7 +83,7 @@ export function createInternalStream(url, obj = {}) {
     let controller = obj.controller;
 
     if (!controller) {
-        controller = new AbortController(); 
+        controller = new AbortController();
         setMaxListeners(Infinity, controller.signal);
     }
 

api/src/stream/shared.js

@@ -42,4 +42,4 @@ export function pipe(from, to, done) {
       .on('close', done);
 
     from.pipe(to);
-}
+}

api/src/stream/types.js

@@ -3,7 +3,7 @@ import ffmpeg from "ffmpeg-static";
 import { spawn } from "child_process";
 import { create as contentDisposition } from "content-disposition-header";
 
-import { metadataManager } from "../sub/utils.js";
+import { metadataManager } from "../misc/utils.js";
 import { destroyInternalStream } from "./manage.js";
 import { env, ffmpegArgs, hlsExceptions } from "../config.js";
 import { getHeaders, closeRequest, closeResponse, pipe } from "./shared.js";

api/src/util/generate-youtube-tokens.js

@@ -1,5 +1,5 @@
 import { Innertube } from 'youtubei.js';
-import { Red } from '../modules/sub/consoleText.js'
+import { Red } from '../misc/console-text.js'
 
 const bail = (...msg) => {
     console.error(...msg);

api/src/util/setup.js

@@ -1,7 +1,7 @@
 import { existsSync, unlinkSync, appendFileSync } from "fs";
 import { createInterface } from "readline";
-import { Cyan, Bright } from "./sub/consoleText.js";
-import { loadJSON } from "./sub/loadFromFs.js";
+import { Cyan, Bright } from "./misc/console-text.js";
+import { loadJSON } from "./misc/load-from-fs.js";
 import { execSync } from "child_process";
 
 const { version } = loadJSON("./package.json");

api/src/util/test-ci.js

@@ -1,11 +1,10 @@
-import { env } from "../modules/config.js";
-import { runTest } from "../modules/test.js";
-import { loadLoc } from "../localization/manager.js";
-import { loadJSON } from "../modules/sub/loadFromFs.js";
-import { Red, Bright } from "../modules/sub/consoleText.js";
+import { env } from "../config.js";
+import { runTest } from "../misc/run-test.js";
+import { loadJSON } from "../misc/load-from-fs.js";
+import { Red, Bright } from "../misc/console-text.js";
 
 const tests = loadJSON('./src/util/tests.json');
-const services = loadJSON('./src/modules/processing/servicesConfig.json');
+const services = loadJSON('./src/processing/service-config.json');
 
 // services that are known to frequently fail due to external
 // factors (e.g. rate limiting)
@@ -38,7 +37,6 @@ switch (action) {
             console.error('no such service:', service);
         }
 
-        await loadLoc();
         env.streamLifespan = 10000;
         env.apiURL = 'http://x';
 
@@ -78,4 +76,4 @@ switch (action) {
     default:
         console.error('invalid action:', action);
         process.exitCode = 1;
-}
+}

api/src/util/test.js

@@ -1,12 +1,12 @@
 import "dotenv/config";
-import "../modules/sub/alias-envs.js";
+import "../misc/alias-envs.js";
 
-import { services } from "../modules/config.js";
-import { extract } from "../modules/processing/url.js";
-import match from "../modules/processing/match.js";
-import { loadJSON } from "../modules/sub/loadFromFs.js";
-import { normalizeRequest } from "../modules/processing/request.js";
-import { env } from "../modules/config.js";
+import { services } from "../config.js";
+import { extract } from "../processing/url.js";
+import match from "../processing/match.js";
+import { loadJSON } from "../misc/load-from-fs.js";
+import { normalizeRequest } from "../processing/request.js";
+import { env } from "../config.js";
 
 env.apiURL = 'http://localhost:9000'
 let tests = loadJSON('./src/util/tests.json');
@@ -29,7 +29,7 @@ for (let i in services) {
         console.log(`\nRunning tests for ${i}...\n`)
         for (let k = 0; k < tests[i].length; k++) {
             let test = tests[i][k];
-            
+
             console.log(`Running test ${k+1}: ${test.name}`);
             console.log('params:');
             let params = {...{url: test.url}, ...test.params};

api/src/util/testFilenamePresets.js

@@ -1,70 +0,0 @@
-import createFilename from "../modules/processing/createFilename.js";
-
-let tests = [
-    {
-        f: {
-            service: 'youtube',
-            id: 'MMK3L4W70g4',
-            title: "Loossemble (루셈블) - 'Sensitive' MV",
-            author: 'Loossemble',
-            youtubeDubName: false,
-            qualityLabel: '2160p',
-            resolution: '3840x2160',
-            extension: 'webm',
-            youtubeFormat: 'vp9'
-        },
-        isAudioOnly: false,
-        isAudioMuted: false
-    },
-    {
-        f: {
-            service: 'youtube',
-            id: 'MMK3L4W70g4',
-            title: "Loossemble (루셈블) - 'Sensitive' MV",
-            author: 'Loossemble',
-            youtubeDubName: false,
-            qualityLabel: '2160p',
-            resolution: '3840x2160',
-            extension: 'webm',
-            youtubeFormat: 'vp9'
-        },
-        isAudioOnly: true,
-        isAudioMuted: false
-    },
-    {
-        f: {
-            service: 'youtube',
-            id: 'MMK3L4W70g4',
-            title: "Loossemble (루셈블) - 'Sensitive' MV",
-            author: 'Loossemble',
-            youtubeDubName: false,
-            qualityLabel: '2160p',
-            resolution: '3840x2160',
-            extension: 'webm',
-            youtubeFormat: 'vp9'
-        },
-        isAudioOnly: false,
-        isAudioMuted: true
-    },
-    {
-        f: {
-            service: 'vimeo',
-            id: 'MMK3L4W70g4',
-            title: "Loossemble (루셈블) - 'Sensitive' MV",
-            author: 'Loossemble',
-            qualityLabel: '2160p',
-            resolution: '3840x2160',
-            extension: 'mp4'
-        },
-        isAudioOnly: false,
-        isAudioMuted: true
-    }
-]
-
-for (let i = 0; i < tests.length; i++) {
-    console.log(`---${i}---`)
-    console.log(createFilename(tests[i].f, "classic", tests[i].isAudioOnly, tests[i].isAudioMuted))
-    console.log(createFilename(tests[i].f, "basic", tests[i].isAudioOnly, tests[i].isAudioMuted))
-    console.log(createFilename(tests[i].f, "pretty", tests[i].isAudioOnly, tests[i].isAudioMuted))
-    console.log(createFilename(tests[i].f, "nerdy", tests[i].isAudioOnly, tests[i].isAudioMuted))
-}