From dd831e13e89bfc5a37246280e41468a3d1d8c4bd Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 3 Aug 2024 14:47:13 +0600 Subject: [PATCH] api: flatten code directories, better filenames, remove old files --- api/package.json | 3 +- api/src/cobalt.js | 8 +- api/src/{modules => }/config.js | 29 +++-- api/src/config.json | 103 ------------------ api/src/core/api.js | 20 ++-- api/src/{modules/sub => misc}/alias-envs.js | 2 +- .../consoleText.js => misc/console-text.js} | 0 api/src/{modules/sub => misc}/crypto.js | 0 .../current-commit.js} | 0 .../loadFromFs.js => misc/load-from-fs.js} | 0 .../sub => misc}/randomize-ciphers.js | 0 api/src/{modules/test.js => misc/run-test.js} | 6 +- api/src/{modules/sub => misc}/utils.js | 0 .../{modules => }/processing/cookie/cookie.js | 0 .../processing/cookie/manager.js | 2 +- .../create-filename.js} | 0 .../match-action.js} | 4 +- api/src/{modules => }/processing/match.js | 6 +- api/src/{modules => }/processing/request.js | 4 +- .../service-config.json} | 0 .../service-patterns.js} | 4 +- .../processing/services/bilibili.js | 0 .../processing/services/dailymotion.js | 0 .../processing/services/facebook.js | 0 .../processing/services/instagram.js | 0 .../{modules => }/processing/services/loom.js | 0 .../{modules => }/processing/services/ok.js | 2 +- .../processing/services/pinterest.js | 2 +- .../processing/services/reddit.js | 0 .../processing/services/rutube.js | 2 +- .../processing/services/snapchat.js | 2 +- .../processing/services/soundcloud.js | 2 +- .../processing/services/streamable.js | 0 .../processing/services/tiktok.js | 0 .../processing/services/tumblr.js | 0 .../processing/services/twitch.js | 2 +- .../processing/services/twitter.js | 0 .../processing/services/vimeo.js | 2 +- .../{modules => }/processing/services/vine.js | 0 .../{modules => }/processing/services/vk.js | 2 +- .../processing/services/youtube.js | 2 +- api/src/{modules => }/processing/url.js | 3 +- api/src/{modules => }/stream/internal-hls.js | 0 api/src/{modules => }/stream/internal.js | 6 +- api/src/{modules => }/stream/manage.js | 4 +- api/src/{modules => }/stream/shared.js | 2 +- api/src/{modules => }/stream/stream.js | 0 api/src/{modules => }/stream/types.js | 2 +- api/src/util/generate-youtube-tokens.js | 2 +- api/src/{modules => util}/setup.js | 4 +- api/src/util/test-ci.js | 14 +-- api/src/util/test.js | 16 +-- api/src/util/testFilenamePresets.js | 70 ------------ 53 files changed, 83 insertions(+), 249 deletions(-) rename api/src/{modules => }/config.js (56%) delete mode 100644 api/src/config.json rename api/src/{modules/sub => misc}/alias-envs.js (94%) rename api/src/{modules/sub/consoleText.js => misc/console-text.js} (100%) rename api/src/{modules/sub => misc}/crypto.js (100%) rename api/src/{modules/sub/currentCommit.js => misc/current-commit.js} (100%) rename api/src/{modules/sub/loadFromFs.js => misc/load-from-fs.js} (100%) rename api/src/{modules/sub => misc}/randomize-ciphers.js (100%) rename api/src/{modules/test.js => misc/run-test.js} (86%) rename api/src/{modules/sub => misc}/utils.js (100%) rename api/src/{modules => }/processing/cookie/cookie.js (100%) rename api/src/{modules => }/processing/cookie/manager.js (97%) rename api/src/{modules/processing/createFilename.js => processing/create-filename.js} (100%) rename api/src/{modules/processing/matchActionDecider.js => processing/match-action.js} (98%) rename api/src/{modules => }/processing/match.js (98%) rename api/src/{modules => }/processing/request.js (97%) rename api/src/{modules/processing/servicesConfig.json => processing/service-config.json} (100%) rename api/src/{modules/processing/servicesPatternTesters.js => processing/service-patterns.js} (96%) rename api/src/{modules => }/processing/services/bilibili.js (100%) rename api/src/{modules => }/processing/services/dailymotion.js (100%) rename api/src/{modules => }/processing/services/facebook.js (100%) rename api/src/{modules => }/processing/services/instagram.js (100%) rename api/src/{modules => }/processing/services/loom.js (100%) rename api/src/{modules => }/processing/services/ok.js (97%) rename api/src/{modules => }/processing/services/pinterest.js (98%) rename api/src/{modules => }/processing/services/reddit.js (100%) rename api/src/{modules => }/processing/services/rutube.js (98%) rename api/src/{modules => }/processing/services/snapchat.js (98%) rename api/src/{modules => }/processing/services/soundcloud.js (98%) rename api/src/{modules => }/processing/services/streamable.js (100%) rename api/src/{modules => }/processing/services/tiktok.js (100%) rename api/src/{modules => }/processing/services/tumblr.js (100%) rename api/src/{modules => }/processing/services/twitch.js (98%) rename api/src/{modules => }/processing/services/twitter.js (100%) rename api/src/{modules => }/processing/services/vimeo.js (98%) rename api/src/{modules => }/processing/services/vine.js (100%) rename api/src/{modules => }/processing/services/vk.js (97%) rename api/src/{modules => }/processing/services/youtube.js (99%) rename api/src/{modules => }/processing/url.js (99%) rename api/src/{modules => }/stream/internal-hls.js (100%) rename api/src/{modules => }/stream/internal.js (99%) rename api/src/{modules => }/stream/manage.js (97%) rename api/src/{modules => }/stream/shared.js (99%) rename api/src/{modules => }/stream/stream.js (100%) rename api/src/{modules => }/stream/types.js (99%) rename api/src/{modules => util}/setup.js (97%) delete mode 100644 api/src/util/testFilenamePresets.js diff --git a/api/package.json b/api/package.json index 7cafdf4f..f55ceb4a 100644 --- a/api/package.json +++ b/api/package.json @@ -10,9 +10,8 @@ }, "scripts": { "start": "node src/cobalt", - "setup": "node src/modules/setup", + "setup": "node src/util/setup", "test": "node src/util/test", - "build": "node src/modules/buildStatic", "token:youtube": "node src/util/generate-youtube-tokens" }, "repository": { diff --git a/api/src/cobalt.js b/api/src/cobalt.js index 33ed953d..53a5d71d 100644 --- a/api/src/cobalt.js +++ b/api/src/cobalt.js @@ -1,11 +1,11 @@ import "dotenv/config"; -import "./modules/sub/alias-envs.js"; +import "./misc/alias-envs.js"; import express from "express"; -import { Bright, Green, Red } from "./modules/sub/consoleText.js"; -import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js"; -import { env } from "./modules/config.js" +import { Bright, Green, Red } from "./misc/console-text.js"; +import { getCurrentBranch, shortCommit } from "./misc/current-commit.js"; +import { env } from "./config.js" import path from 'path'; import { fileURLToPath } from 'url'; diff --git a/api/src/modules/config.js b/api/src/config.js similarity index 56% rename from api/src/modules/config.js rename to api/src/config.js index 4a3a63ed..0f9bf07d 100644 --- a/api/src/modules/config.js +++ b/api/src/config.js @@ -1,9 +1,8 @@ import UrlPattern from "url-pattern"; -import { loadJSON } from "./sub/loadFromFs.js"; +import { loadJSON } from "./misc/load-from-fs.js"; -const config = loadJSON("./src/config.json"); const packageJson = loadJSON("./package.json"); -const servicesConfigJson = loadJSON("./src/modules/processing/servicesConfig.json"); +const servicesConfigJson = loadJSON("./src/processing/service-config.json"); Object.values(servicesConfigJson.config).forEach(service => { service.patterns = service.patterns.map( @@ -14,13 +13,23 @@ Object.values(servicesConfigJson.config).forEach(service => { }) export const + version = packageJson.version, + + genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36", + supportedAudio = ["mp3", "ogg", "wav", "opus"], + ffmpegArgs = { + webm: ["-c:v", "copy", "-c:a", "copy"], + mp4: ["-c:v", "copy", "-c:a", "copy", "-movflags", "faststart+frag_keyframe+empty_moov"], + copy: ["-c:a", "copy"], + audio: ["-ar", "48000", "-ac", "2", "-b:a", "320k"], + m4a: ["-movflags", "frag_keyframe+empty_moov"], + gif: ["-vf", "scale=-1:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse", "-loop", "0"] + }, + services = servicesConfigJson.config, hlsExceptions = servicesConfigJson.hlsExceptions, audioIgnore = servicesConfigJson.audioIgnore, - version = packageJson.version, - genericUserAgent = config.genericUserAgent, - ffmpegArgs = config.ffmpegArgs, - supportedAudio = config.supportedAudio, + env = { apiURL: process.env.API_URL || '', apiPort: process.env.API_PORT || 9000, @@ -33,15 +42,15 @@ export const cookiePath: process.env.COOKIE_PATH, - rateLimitWindow: (process.env.RATELIMIT_WINDOW && parseInt(process.env.RATELIMIT_WINDOW)) || 60, + rateLimitWindow: (process.env.RATELIMIT_WINDOW && parseInt(process.env.RATELIMIT_WINDOW)) || 60, rateLimitMax: (process.env.RATELIMIT_MAX && parseInt(process.env.RATELIMIT_MAX)) || 20, durationLimit: (process.env.DURATION_LIMIT && parseInt(process.env.DURATION_LIMIT)) || 10800, streamLifespan: 90, processingPriority: process.platform !== 'win32' - && process.env.PROCESSING_PRIORITY - && parseInt(process.env.PROCESSING_PRIORITY), + && process.env.PROCESSING_PRIORITY + && parseInt(process.env.PROCESSING_PRIORITY), externalProxy: process.env.API_EXTERNAL_PROXY, } diff --git a/api/src/config.json b/api/src/config.json deleted file mode 100644 index 640fb2f5..00000000 --- a/api/src/config.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "genericUserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36", - "authorInfo": { - "support": { - "default": { - "email": { - "emoji": "πŸ“§", - "url": "mailto:support@cobalt.tools", - "name": "support@cobalt.tools" - }, - "twitter": { - "emoji": "🐦", - "url": "https://twitter.com/justusecobalt", - "name": "@justusecobalt" - }, - "discord": { - "emoji": "πŸ‘Ύ", - "url": "https://discord.gg/pQPt8HBUPu", - "name": "cobalt discord server" - } - }, - "ru": { - "telegram": { - "emoji": "πŸ“¬", - "url": "https://t.me/justusecobalt_ru", - "name": "ΠΊΠ°Π½Π°Π» Π² telegram" - }, - "email": { - "emoji": "πŸ“§", - "url": "mailto:support@cobalt.tools", - "name": "support@cobalt.tools" - } - } - } - }, - "donations": { - "crypto": { - "monero": "4B1SNB6s8Pq1hxjNeKPEe8Qa8EP3zdL16Sqsa7QDoJcUecKQzEj9BMxWnEnTGu12doKLJBKRDUqnn6V9qfSdXpXi3Nw5Uod", - "litecoin": "ltc1qvp0xhrk2m7pa6p6z844qcslfyxv4p3vf95rhna", - "ethereum": "0x4B4cF23051c78c7A7E0eA09d39099621c46bc302", - "usdt-erc20": "0x4B4cF23051c78c7A7E0eA09d39099621c46bc302", - "usdt-trc20": "TVbx7YT3rBfu931Gxko6pRfXtedYqbgnBB", - "bitcoin": "bc1qlvcnlnyzfsgnuxyxsv3k0p0q0yln0azjpadyx4", - "bitcoin-alt": "18PKf6N2cHrmSzz9ZzTSvDd2jAkqGC7SxA", - "ton": "UQA3SO-hHZq1oCCT--u6or6ollB8fd2o52aD8mXiLk9iDZd3" - }, - "links": { - "boosty": "https://boosty.to/wukko/donate" - } - }, - "links": { - "saveToGalleryShortcut": "https://www.icloud.com/shortcuts/14e9aebf04b24156acc34ceccf7e6fcd", - "saveToFilesShortcut": "https://www.icloud.com/shortcuts/2134cd9d4d6b41448b2201f933542b2e", - "statusPage": "https://status.cobalt.tools/", - "troubleshootingGuide": "https://github.com/imputnet/cobalt/blob/current/docs/troubleshooting.md" - }, - "celebrations": { - "01-01": "πŸŽ„", - "02-17": "😺", - "02-22": "😺", - "03-01": "😺", - "03-08": "πŸ’ͺ", - "05-26": "πŸŽ‚", - "08-08": "😺", - "08-26": "🐢", - "10-29": "😺", - "10-30": "πŸŽƒ", - "10-31": "πŸŽƒ", - "11-01": "πŸ•―οΈ", - "11-02": "πŸ•―οΈ", - "12-20": "πŸŽ„", - "12-21": "πŸŽ„", - "12-22": "πŸŽ„", - "12-23": "πŸŽ„", - "12-24": "πŸŽ„", - "12-25": "πŸŽ„", - "12-26": "πŸŽ„", - "12-27": "πŸŽ„", - "12-28": "πŸŽ„", - "12-29": "πŸŽ„", - "12-30": "πŸŽ„", - "12-31": "πŸŽ„" - }, - "supportedAudio": ["mp3", "ogg", "wav", "opus"], - "ffmpegArgs": { - "webm": ["-c:v", "copy", "-c:a", "copy"], - "mp4": ["-c:v", "copy", "-c:a", "copy", "-movflags", "faststart+frag_keyframe+empty_moov"], - "copy": ["-c:a", "copy"], - "audio": ["-ar", "48000", "-ac", "2", "-b:a", "320k"], - "m4a": ["-movflags", "frag_keyframe+empty_moov"], - "gif": ["-vf", "scale=-1:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse", "-loop", "0"] - }, - "sponsors": [{ - "name": "royale", - "fullName": "RoyaleHosting", - "url": "https://royalehosting.net/?partner=cobalt", - "logo": { - "width": 605, - "height": 136, - "scale": 5 - } - }] -} diff --git a/api/src/core/api.js b/api/src/core/api.js index 74ea0307..ebdedb55 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -2,18 +2,18 @@ import cors from "cors"; import rateLimit from "express-rate-limit"; import { setGlobalDispatcher, ProxyAgent } from "undici"; -import { env, version } from "../modules/config.js"; +import { env, version } from "../config.js"; -import { generateHmac, generateSalt } from "../modules/sub/crypto.js"; -import { Bright, Cyan } from "../modules/sub/consoleText.js"; -import { languageCode } from "../modules/sub/utils.js"; +import { generateHmac, generateSalt } from "../misc/crypto.js"; +import { Bright, Cyan } from "../misc/console-text.js"; +import { languageCode } from "../misc/utils.js"; -import { createResponse, normalizeRequest, getIP } from "../modules/processing/request.js"; -import { verifyStream, getInternalStream } from "../modules/stream/manage.js"; -import { randomizeCiphers } from '../modules/sub/randomize-ciphers.js'; -import { extract } from "../modules/processing/url.js"; -import match from "../modules/processing/match.js"; -import stream from "../modules/stream/stream.js"; +import { createResponse, normalizeRequest, getIP } from "../processing/request.js"; +import { verifyStream, getInternalStream } from "../stream/manage.js"; +import { randomizeCiphers } from '../misc/randomize-ciphers.js'; +import { extract } from "../processing/url.js"; +import match from "../processing/match.js"; +import stream from "../stream/stream.js"; const acceptRegex = /^application\/json(; charset=utf-8)?$/; diff --git a/api/src/modules/sub/alias-envs.js b/api/src/misc/alias-envs.js similarity index 94% rename from api/src/modules/sub/alias-envs.js rename to api/src/misc/alias-envs.js index 24f6a856..5facb14b 100644 --- a/api/src/modules/sub/alias-envs.js +++ b/api/src/misc/alias-envs.js @@ -1,4 +1,4 @@ -import { Red } from "./consoleText.js"; +import { Red } from "./console-text.js"; const mapping = { apiPort: 'API_PORT', diff --git a/api/src/modules/sub/consoleText.js b/api/src/misc/console-text.js similarity index 100% rename from api/src/modules/sub/consoleText.js rename to api/src/misc/console-text.js diff --git a/api/src/modules/sub/crypto.js b/api/src/misc/crypto.js similarity index 100% rename from api/src/modules/sub/crypto.js rename to api/src/misc/crypto.js diff --git a/api/src/modules/sub/currentCommit.js b/api/src/misc/current-commit.js similarity index 100% rename from api/src/modules/sub/currentCommit.js rename to api/src/misc/current-commit.js diff --git a/api/src/modules/sub/loadFromFs.js b/api/src/misc/load-from-fs.js similarity index 100% rename from api/src/modules/sub/loadFromFs.js rename to api/src/misc/load-from-fs.js diff --git a/api/src/modules/sub/randomize-ciphers.js b/api/src/misc/randomize-ciphers.js similarity index 100% rename from api/src/modules/sub/randomize-ciphers.js rename to api/src/misc/randomize-ciphers.js diff --git a/api/src/modules/test.js b/api/src/misc/run-test.js similarity index 86% rename from api/src/modules/test.js rename to api/src/misc/run-test.js index 75bdab26..455e3dfd 100644 --- a/api/src/modules/test.js +++ b/api/src/misc/run-test.js @@ -1,6 +1,6 @@ -import { normalizeRequest } from "../modules/processing/request.js"; -import match from "./processing/match.js"; -import { extract } from "./processing/url.js"; +import { normalizeRequest } from "../processing/request.js"; +import match from "../processing/match.js"; +import { extract } from "../processing/url.js"; export async function runTest(url, params, expect) { const normalized = normalizeRequest({ url, ...params }); diff --git a/api/src/modules/sub/utils.js b/api/src/misc/utils.js similarity index 100% rename from api/src/modules/sub/utils.js rename to api/src/misc/utils.js diff --git a/api/src/modules/processing/cookie/cookie.js b/api/src/processing/cookie/cookie.js similarity index 100% rename from api/src/modules/processing/cookie/cookie.js rename to api/src/processing/cookie/cookie.js diff --git a/api/src/modules/processing/cookie/manager.js b/api/src/processing/cookie/manager.js similarity index 97% rename from api/src/modules/processing/cookie/manager.js rename to api/src/processing/cookie/manager.js index 2b997d96..25bf9c90 100644 --- a/api/src/modules/processing/cookie/manager.js +++ b/api/src/processing/cookie/manager.js @@ -1,7 +1,7 @@ import Cookie from './cookie.js'; import { readFile, writeFile } from 'fs/promises'; import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser'; -import { env } from '../../../modules/config.js' +import { env } from '../../config.js'; const WRITE_INTERVAL = 60000, cookiePath = env.cookiePath, diff --git a/api/src/modules/processing/createFilename.js b/api/src/processing/create-filename.js similarity index 100% rename from api/src/modules/processing/createFilename.js rename to api/src/processing/create-filename.js diff --git a/api/src/modules/processing/matchActionDecider.js b/api/src/processing/match-action.js similarity index 98% rename from api/src/modules/processing/matchActionDecider.js rename to api/src/processing/match-action.js index 6284f665..d5c58013 100644 --- a/api/src/modules/processing/matchActionDecider.js +++ b/api/src/processing/match-action.js @@ -1,6 +1,6 @@ import { audioIgnore, services, supportedAudio } from "../config.js"; -import { createResponse } from "../processing/request.js"; -import createFilename from "./createFilename.js"; +import { createResponse } from "./request.js"; +import createFilename from "./create-filename.js"; import { createStream } from "../stream/manage.js"; export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern, toGif, requestIP) { diff --git a/api/src/modules/processing/match.js b/api/src/processing/match.js similarity index 98% rename from api/src/modules/processing/match.js rename to api/src/processing/match.js index ce69f30b..381c9b98 100644 --- a/api/src/modules/processing/match.js +++ b/api/src/processing/match.js @@ -1,10 +1,10 @@ import { strict as assert } from "node:assert"; -import { env } from '../config.js'; +import { env } from "../config.js"; import { createResponse } from "../processing/request.js"; -import { testers } from "./servicesPatternTesters.js"; -import matchActionDecider from "./matchActionDecider.js"; +import { testers } from "./service-patterns.js"; +import matchActionDecider from "./match-action.js"; import bilibili from "./services/bilibili.js"; import reddit from "./services/reddit.js"; diff --git a/api/src/modules/processing/request.js b/api/src/processing/request.js similarity index 97% rename from api/src/modules/processing/request.js rename to api/src/processing/request.js index 0bee1691..899fcc44 100644 --- a/api/src/modules/processing/request.js +++ b/api/src/processing/request.js @@ -1,8 +1,8 @@ import ipaddr from "ipaddr.js"; -import { normalizeURL } from "../processing/url.js"; +import { normalizeURL } from "./url.js"; import { createStream } from "../stream/manage.js"; -import { verifyLanguageCode } from "../sub/utils.js"; +import { verifyLanguageCode } from "../misc/utils.js"; const apiVar = { allowed: { diff --git a/api/src/modules/processing/servicesConfig.json b/api/src/processing/service-config.json similarity index 100% rename from api/src/modules/processing/servicesConfig.json rename to api/src/processing/service-config.json diff --git a/api/src/modules/processing/servicesPatternTesters.js b/api/src/processing/service-patterns.js similarity index 96% rename from api/src/modules/processing/servicesPatternTesters.js rename to api/src/processing/service-patterns.js index 48f953c1..e8d169d5 100644 --- a/api/src/modules/processing/servicesPatternTesters.js +++ b/api/src/processing/service-patterns.js @@ -31,8 +31,8 @@ export const testers = { || patternMatch.shortLink?.length <= 32, "snapchat": (patternMatch) => - (patternMatch.username?.length <= 32 && (!patternMatch.storyId || patternMatch.storyId?.length <= 255)) - || patternMatch.spotlightId?.length <= 255 + (patternMatch.username?.length <= 32 && (!patternMatch.storyId || patternMatch.storyId?.length <= 255)) + || patternMatch.spotlightId?.length <= 255 || patternMatch.shortLink?.length <= 16, "streamable": (patternMatch) => diff --git a/api/src/modules/processing/services/bilibili.js b/api/src/processing/services/bilibili.js similarity index 100% rename from api/src/modules/processing/services/bilibili.js rename to api/src/processing/services/bilibili.js diff --git a/api/src/modules/processing/services/dailymotion.js b/api/src/processing/services/dailymotion.js similarity index 100% rename from api/src/modules/processing/services/dailymotion.js rename to api/src/processing/services/dailymotion.js diff --git a/api/src/modules/processing/services/facebook.js b/api/src/processing/services/facebook.js similarity index 100% rename from api/src/modules/processing/services/facebook.js rename to api/src/processing/services/facebook.js diff --git a/api/src/modules/processing/services/instagram.js b/api/src/processing/services/instagram.js similarity index 100% rename from api/src/modules/processing/services/instagram.js rename to api/src/processing/services/instagram.js diff --git a/api/src/modules/processing/services/loom.js b/api/src/processing/services/loom.js similarity index 100% rename from api/src/modules/processing/services/loom.js rename to api/src/processing/services/loom.js diff --git a/api/src/modules/processing/services/ok.js b/api/src/processing/services/ok.js similarity index 97% rename from api/src/modules/processing/services/ok.js rename to api/src/processing/services/ok.js index 33847cd8..8f177043 100644 --- a/api/src/modules/processing/services/ok.js +++ b/api/src/processing/services/ok.js @@ -1,5 +1,5 @@ import { genericUserAgent, env } from "../../config.js"; -import { cleanString } from "../../sub/utils.js"; +import { cleanString } from "../../misc/utils.js"; const resolutions = { "ultra": "2160", diff --git a/api/src/modules/processing/services/pinterest.js b/api/src/processing/services/pinterest.js similarity index 98% rename from api/src/modules/processing/services/pinterest.js rename to api/src/processing/services/pinterest.js index 3e9e1d32..5a5f9438 100644 --- a/api/src/modules/processing/services/pinterest.js +++ b/api/src/processing/services/pinterest.js @@ -33,7 +33,7 @@ export default async function(o) { let imageLink = [...html.matchAll(imageRegex)] .map(([, link]) => link) .find(a => a.endsWith('.jpg') || a.endsWith('.gif')); - + if (imageLink) return { urls: imageLink, isPhoto: true diff --git a/api/src/modules/processing/services/reddit.js b/api/src/processing/services/reddit.js similarity index 100% rename from api/src/modules/processing/services/reddit.js rename to api/src/processing/services/reddit.js diff --git a/api/src/modules/processing/services/rutube.js b/api/src/processing/services/rutube.js similarity index 98% rename from api/src/modules/processing/services/rutube.js rename to api/src/processing/services/rutube.js index 325e9ccc..c162eaf5 100644 --- a/api/src/modules/processing/services/rutube.js +++ b/api/src/processing/services/rutube.js @@ -1,7 +1,7 @@ import HLS from 'hls-parser'; import { env } from "../../config.js"; -import { cleanString } from '../../sub/utils.js'; +import { cleanString } from '../../misc/utils.js'; async function requestJSON(url) { try { diff --git a/api/src/modules/processing/services/snapchat.js b/api/src/processing/services/snapchat.js similarity index 98% rename from api/src/modules/processing/services/snapchat.js rename to api/src/processing/services/snapchat.js index a93f0933..1c1eaf6d 100644 --- a/api/src/modules/processing/services/snapchat.js +++ b/api/src/processing/services/snapchat.js @@ -1,5 +1,5 @@ import { genericUserAgent } from "../../config.js"; -import { getRedirectingURL } from "../../sub/utils.js"; +import { getRedirectingURL } from "../../misc/utils.js"; import { extract, normalizeURL } from "../url.js"; const SPOTLIGHT_VIDEO_REGEX = //; diff --git a/api/src/modules/processing/services/soundcloud.js b/api/src/processing/services/soundcloud.js similarity index 98% rename from api/src/modules/processing/services/soundcloud.js rename to api/src/processing/services/soundcloud.js index c36ec61d..82852b8f 100644 --- a/api/src/modules/processing/services/soundcloud.js +++ b/api/src/processing/services/soundcloud.js @@ -1,5 +1,5 @@ import { env } from "../../config.js"; -import { cleanString } from "../../sub/utils.js"; +import { cleanString } from "../../misc/utils.js"; const cachedID = { version: '', diff --git a/api/src/modules/processing/services/streamable.js b/api/src/processing/services/streamable.js similarity index 100% rename from api/src/modules/processing/services/streamable.js rename to api/src/processing/services/streamable.js diff --git a/api/src/modules/processing/services/tiktok.js b/api/src/processing/services/tiktok.js similarity index 100% rename from api/src/modules/processing/services/tiktok.js rename to api/src/processing/services/tiktok.js diff --git a/api/src/modules/processing/services/tumblr.js b/api/src/processing/services/tumblr.js similarity index 100% rename from api/src/modules/processing/services/tumblr.js rename to api/src/processing/services/tumblr.js diff --git a/api/src/modules/processing/services/twitch.js b/api/src/processing/services/twitch.js similarity index 98% rename from api/src/modules/processing/services/twitch.js rename to api/src/processing/services/twitch.js index 13cee19f..5712178c 100644 --- a/api/src/modules/processing/services/twitch.js +++ b/api/src/processing/services/twitch.js @@ -1,5 +1,5 @@ import { env } from "../../config.js"; -import { cleanString } from '../../sub/utils.js'; +import { cleanString } from '../../misc/utils.js'; const gqlURL = "https://gql.twitch.tv/gql"; const clientIdHead = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" }; diff --git a/api/src/modules/processing/services/twitter.js b/api/src/processing/services/twitter.js similarity index 100% rename from api/src/modules/processing/services/twitter.js rename to api/src/processing/services/twitter.js diff --git a/api/src/modules/processing/services/vimeo.js b/api/src/processing/services/vimeo.js similarity index 98% rename from api/src/modules/processing/services/vimeo.js rename to api/src/processing/services/vimeo.js index 30ee3368..a9f02040 100644 --- a/api/src/modules/processing/services/vimeo.js +++ b/api/src/processing/services/vimeo.js @@ -1,5 +1,5 @@ import { env } from "../../config.js"; -import { cleanString, merge } from '../../sub/utils.js'; +import { cleanString, merge } from '../../misc/utils.js'; import HLS from "hls-parser"; diff --git a/api/src/modules/processing/services/vine.js b/api/src/processing/services/vine.js similarity index 100% rename from api/src/modules/processing/services/vine.js rename to api/src/processing/services/vine.js diff --git a/api/src/modules/processing/services/vk.js b/api/src/processing/services/vk.js similarity index 97% rename from api/src/modules/processing/services/vk.js rename to api/src/processing/services/vk.js index e95f12be..a3ddd615 100644 --- a/api/src/modules/processing/services/vk.js +++ b/api/src/processing/services/vk.js @@ -1,5 +1,5 @@ import { genericUserAgent, env } from "../../config.js"; -import { cleanString } from "../../sub/utils.js"; +import { cleanString } from "../../misc/utils.js"; const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240"]; diff --git a/api/src/modules/processing/services/youtube.js b/api/src/processing/services/youtube.js similarity index 99% rename from api/src/modules/processing/services/youtube.js rename to api/src/processing/services/youtube.js index 9bfc3919..5dd40ee1 100644 --- a/api/src/modules/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -3,7 +3,7 @@ import { fetch } from "undici"; import { Innertube, Session } from "youtubei.js"; import { env } from "../../config.js"; -import { cleanString } from "../../sub/utils.js"; +import { cleanString } from "../../misc/utils.js"; import { getCookie, updateCookieValues } from "../cookie/manager.js"; const PLAYER_REFRESH_PERIOD = 1000 * 60 * 15; // ms diff --git a/api/src/modules/processing/url.js b/api/src/processing/url.js similarity index 99% rename from api/src/modules/processing/url.js rename to api/src/processing/url.js index 180ec6ee..a2b9cd7d 100644 --- a/api/src/modules/processing/url.js +++ b/api/src/processing/url.js @@ -54,6 +54,7 @@ function aliasURL(url) { url = new URL(`https://bilibili.com/_tv${url.pathname}`); } break; + case "b23": if (url.hostname === 'b23.tv' && parts.length === 2) { url = new URL(`https://bilibili.com/_shortLink/${parts[1]}`) @@ -179,4 +180,4 @@ export function extract(url) { } return { host, patternMatch }; -} \ No newline at end of file +} diff --git a/api/src/modules/stream/internal-hls.js b/api/src/stream/internal-hls.js similarity index 100% rename from api/src/modules/stream/internal-hls.js rename to api/src/stream/internal-hls.js diff --git a/api/src/modules/stream/internal.js b/api/src/stream/internal.js similarity index 99% rename from api/src/modules/stream/internal.js rename to api/src/stream/internal.js index 7a155a9a..253f98bb 100644 --- a/api/src/modules/stream/internal.js +++ b/api/src/stream/internal.js @@ -28,7 +28,7 @@ async function* readChunks(streamInfo, size) { if (received < expected / 2n) { closeRequest(streamInfo.controller); } - + for await (const data of chunk.body) { yield data; } @@ -64,7 +64,7 @@ async function handleYoutubeStream(streamInfo, res) { } signal.addEventListener('abort', abortGenerator); - + const stream = Readable.from(generator); for (const headerName of ['content-type', 'content-length']) { @@ -119,4 +119,4 @@ export function internalStream(streamInfo, res) { } return handleGenericStream(streamInfo, res); -} \ No newline at end of file +} diff --git a/api/src/modules/stream/manage.js b/api/src/stream/manage.js similarity index 97% rename from api/src/modules/stream/manage.js rename to api/src/stream/manage.js index af041a67..24270144 100644 --- a/api/src/modules/stream/manage.js +++ b/api/src/stream/manage.js @@ -3,7 +3,7 @@ import { randomBytes } from "crypto"; import { nanoid } from "nanoid"; import { setMaxListeners } from "node:events"; -import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js"; +import { decryptStream, encryptStream, generateHmac } from "../misc/crypto.js"; import { env } from "../config.js"; import { strict as assert } from "assert"; import { closeRequest } from "./shared.js"; @@ -83,7 +83,7 @@ export function createInternalStream(url, obj = {}) { let controller = obj.controller; if (!controller) { - controller = new AbortController(); + controller = new AbortController(); setMaxListeners(Infinity, controller.signal); } diff --git a/api/src/modules/stream/shared.js b/api/src/stream/shared.js similarity index 99% rename from api/src/modules/stream/shared.js rename to api/src/stream/shared.js index fd7d1569..91e1ac2f 100644 --- a/api/src/modules/stream/shared.js +++ b/api/src/stream/shared.js @@ -42,4 +42,4 @@ export function pipe(from, to, done) { .on('close', done); from.pipe(to); -} \ No newline at end of file +} diff --git a/api/src/modules/stream/stream.js b/api/src/stream/stream.js similarity index 100% rename from api/src/modules/stream/stream.js rename to api/src/stream/stream.js diff --git a/api/src/modules/stream/types.js b/api/src/stream/types.js similarity index 99% rename from api/src/modules/stream/types.js rename to api/src/stream/types.js index 071cb6d0..f566a196 100644 --- a/api/src/modules/stream/types.js +++ b/api/src/stream/types.js @@ -3,7 +3,7 @@ import ffmpeg from "ffmpeg-static"; import { spawn } from "child_process"; import { create as contentDisposition } from "content-disposition-header"; -import { metadataManager } from "../sub/utils.js"; +import { metadataManager } from "../misc/utils.js"; import { destroyInternalStream } from "./manage.js"; import { env, ffmpegArgs, hlsExceptions } from "../config.js"; import { getHeaders, closeRequest, closeResponse, pipe } from "./shared.js"; diff --git a/api/src/util/generate-youtube-tokens.js b/api/src/util/generate-youtube-tokens.js index ab877f96..5585705a 100644 --- a/api/src/util/generate-youtube-tokens.js +++ b/api/src/util/generate-youtube-tokens.js @@ -1,5 +1,5 @@ import { Innertube } from 'youtubei.js'; -import { Red } from '../modules/sub/consoleText.js' +import { Red } from '../misc/console-text.js' const bail = (...msg) => { console.error(...msg); diff --git a/api/src/modules/setup.js b/api/src/util/setup.js similarity index 97% rename from api/src/modules/setup.js rename to api/src/util/setup.js index 137f6271..7796ea22 100644 --- a/api/src/modules/setup.js +++ b/api/src/util/setup.js @@ -1,7 +1,7 @@ import { existsSync, unlinkSync, appendFileSync } from "fs"; import { createInterface } from "readline"; -import { Cyan, Bright } from "./sub/consoleText.js"; -import { loadJSON } from "./sub/loadFromFs.js"; +import { Cyan, Bright } from "./misc/console-text.js"; +import { loadJSON } from "./misc/load-from-fs.js"; import { execSync } from "child_process"; const { version } = loadJSON("./package.json"); diff --git a/api/src/util/test-ci.js b/api/src/util/test-ci.js index ed6962c7..6559aadc 100644 --- a/api/src/util/test-ci.js +++ b/api/src/util/test-ci.js @@ -1,11 +1,10 @@ -import { env } from "../modules/config.js"; -import { runTest } from "../modules/test.js"; -import { loadLoc } from "../localization/manager.js"; -import { loadJSON } from "../modules/sub/loadFromFs.js"; -import { Red, Bright } from "../modules/sub/consoleText.js"; +import { env } from "../config.js"; +import { runTest } from "../misc/run-test.js"; +import { loadJSON } from "../misc/load-from-fs.js"; +import { Red, Bright } from "../misc/console-text.js"; const tests = loadJSON('./src/util/tests.json'); -const services = loadJSON('./src/modules/processing/servicesConfig.json'); +const services = loadJSON('./src/processing/service-config.json'); // services that are known to frequently fail due to external // factors (e.g. rate limiting) @@ -38,7 +37,6 @@ switch (action) { console.error('no such service:', service); } - await loadLoc(); env.streamLifespan = 10000; env.apiURL = 'http://x'; @@ -78,4 +76,4 @@ switch (action) { default: console.error('invalid action:', action); process.exitCode = 1; -} \ No newline at end of file +} diff --git a/api/src/util/test.js b/api/src/util/test.js index 294ff7e7..2a2c084c 100644 --- a/api/src/util/test.js +++ b/api/src/util/test.js @@ -1,12 +1,12 @@ import "dotenv/config"; -import "../modules/sub/alias-envs.js"; +import "../misc/alias-envs.js"; -import { services } from "../modules/config.js"; -import { extract } from "../modules/processing/url.js"; -import match from "../modules/processing/match.js"; -import { loadJSON } from "../modules/sub/loadFromFs.js"; -import { normalizeRequest } from "../modules/processing/request.js"; -import { env } from "../modules/config.js"; +import { services } from "../config.js"; +import { extract } from "../processing/url.js"; +import match from "../processing/match.js"; +import { loadJSON } from "../misc/load-from-fs.js"; +import { normalizeRequest } from "../processing/request.js"; +import { env } from "../config.js"; env.apiURL = 'http://localhost:9000' let tests = loadJSON('./src/util/tests.json'); @@ -29,7 +29,7 @@ for (let i in services) { console.log(`\nRunning tests for ${i}...\n`) for (let k = 0; k < tests[i].length; k++) { let test = tests[i][k]; - + console.log(`Running test ${k+1}: ${test.name}`); console.log('params:'); let params = {...{url: test.url}, ...test.params}; diff --git a/api/src/util/testFilenamePresets.js b/api/src/util/testFilenamePresets.js deleted file mode 100644 index ba088cb0..00000000 --- a/api/src/util/testFilenamePresets.js +++ /dev/null @@ -1,70 +0,0 @@ -import createFilename from "../modules/processing/createFilename.js"; - -let tests = [ - { - f: { - service: 'youtube', - id: 'MMK3L4W70g4', - title: "Loossemble (λ£¨μ…ˆλΈ”) - 'Sensitive' MV", - author: 'Loossemble', - youtubeDubName: false, - qualityLabel: '2160p', - resolution: '3840x2160', - extension: 'webm', - youtubeFormat: 'vp9' - }, - isAudioOnly: false, - isAudioMuted: false - }, - { - f: { - service: 'youtube', - id: 'MMK3L4W70g4', - title: "Loossemble (λ£¨μ…ˆλΈ”) - 'Sensitive' MV", - author: 'Loossemble', - youtubeDubName: false, - qualityLabel: '2160p', - resolution: '3840x2160', - extension: 'webm', - youtubeFormat: 'vp9' - }, - isAudioOnly: true, - isAudioMuted: false - }, - { - f: { - service: 'youtube', - id: 'MMK3L4W70g4', - title: "Loossemble (λ£¨μ…ˆλΈ”) - 'Sensitive' MV", - author: 'Loossemble', - youtubeDubName: false, - qualityLabel: '2160p', - resolution: '3840x2160', - extension: 'webm', - youtubeFormat: 'vp9' - }, - isAudioOnly: false, - isAudioMuted: true - }, - { - f: { - service: 'vimeo', - id: 'MMK3L4W70g4', - title: "Loossemble (λ£¨μ…ˆλΈ”) - 'Sensitive' MV", - author: 'Loossemble', - qualityLabel: '2160p', - resolution: '3840x2160', - extension: 'mp4' - }, - isAudioOnly: false, - isAudioMuted: true - } -] - -for (let i = 0; i < tests.length; i++) { - console.log(`---${i}---`) - console.log(createFilename(tests[i].f, "classic", tests[i].isAudioOnly, tests[i].isAudioMuted)) - console.log(createFilename(tests[i].f, "basic", tests[i].isAudioOnly, tests[i].isAudioMuted)) - console.log(createFilename(tests[i].f, "pretty", tests[i].isAudioOnly, tests[i].isAudioMuted)) - console.log(createFilename(tests[i].f, "nerdy", tests[i].isAudioOnly, tests[i].isAudioMuted)) -}