From cc6345ff633b233f48f6e2f024a0e0057cb707eb Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 15 May 2024 21:39:44 +0600 Subject: [PATCH] api: move request functions to separate file - request status id is no longer a cryptic number - descriptive function names --- src/core/api.js | 8 +- src/modules/processing/match.js | 16 +- src/modules/processing/matchActionDecider.js | 26 ++-- src/modules/processing/request.js | 154 +++++++++++++++++++ src/modules/processing/services/reddit.js | 6 +- src/modules/sub/utils.js | 104 ------------- 6 files changed, 183 insertions(+), 131 deletions(-) create mode 100644 src/modules/processing/request.js diff --git a/src/core/api.js b/src/core/api.js index ab9c1787..228ab6db 100644 --- a/src/core/api.js +++ b/src/core/api.js @@ -6,14 +6,14 @@ const ipSalt = randomBytes(64).toString('hex'); import { env, version } from "../modules/config.js"; import match from "../modules/processing/match.js"; -import { apiJSON, checkJSONPost, getIP, languageCode } from "../modules/sub/utils.js"; +import { languageCode } from "../modules/sub/utils.js"; +import { createResponse, verifyRequest, getIP } from "../modules/processing/request.js"; import { Bright, Cyan } from "../modules/sub/consoleText.js"; import stream from "../modules/stream/stream.js"; import loc from "../localization/manager.js"; import { generateHmac } from "../modules/sub/crypto.js"; import { verifyStream, getInternalStream } from "../modules/stream/manage.js"; import { extract } from "../modules/processing/url.js"; -import { errorUnsupported } from "../modules/sub/errors.js"; export function runAPI(express, app, gitCommit, gitBranch, __dirname) { const corsConfig = !env.corsWildcard ? { @@ -100,7 +100,7 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { const request = req.body; const lang = languageCode(req); const fail = (t) => { - const { status, body } = apiJSON(0, { t: loc(lang, t) }); + const { status, body } = createResponse("error", { t: loc(lang, t) }); res.status(status).json(body); } @@ -113,7 +113,7 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { } request.dubLang = request.dubLang ? lang : false; - const normalizedRequest = checkJSONPost(request); + const normalizedRequest = verifyRequest(request); if (!normalizedRequest) { return fail('ErrorCantProcess'); } diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 4de3e9b8..73102dfb 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -1,6 +1,6 @@ import { strict as assert } from "node:assert"; -import { apiJSON } from "../sub/utils.js"; +import { createResponse } from "../processing/request.js"; import { errorUnsupported, genericError, brokenLink } from "../sub/errors.js"; import loc from "../../localization/manager.js"; @@ -45,8 +45,8 @@ export default async function(host, patternMatch, lang, obj) { try { let r, isAudioOnly = !!obj.isAudioOnly, disableMetadata = !!obj.disableMetadata; - if (!testers[host]) return apiJSON(0, { t: errorUnsupported(lang) }); - if (!(testers[host](patternMatch))) return apiJSON(0, { t: brokenLink(lang, host) }); + if (!testers[host]) return createResponse("error", { t: errorUnsupported(lang) }); + if (!(testers[host](patternMatch))) return createResponse("error", { t: brokenLink(lang, host) }); switch (host) { case "twitter": @@ -177,17 +177,17 @@ export default async function(host, patternMatch, lang, obj) { r = await dailymotion(patternMatch); break; default: - return apiJSON(0, { t: errorUnsupported(lang) }); + return createResponse("error", { t: errorUnsupported(lang) }); } if (r.isAudioOnly) isAudioOnly = true; let isAudioMuted = isAudioOnly ? false : obj.isAudioMuted; if (r.error && r.critical) - return apiJSON(6, { t: loc(lang, r.error) }) + return createResponse("critical", { t: loc(lang, r.error) }) if (r.error) - return apiJSON(0, { + return createResponse("error", { t: Array.isArray(r.error) ? loc(lang, r.error[0], r.error[1]) : loc(lang, r.error) @@ -199,7 +199,7 @@ export default async function(host, patternMatch, lang, obj) { obj.filenamePattern, obj.twitterGif, requestIP ) - } catch (e) { - return apiJSON(0, { t: genericError(lang, host) }) + } catch { + return createResponse("error", { t: genericError(lang, host) }) } } diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index d440cff6..cc2698e3 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -1,11 +1,11 @@ import { audioIgnore, services, supportedAudio } from "../config.js"; -import { apiJSON } from "../sub/utils.js"; +import { createResponse } from "../processing/request.js"; import loc from "../../localization/manager.js"; import createFilename from "./createFilename.js"; export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern, toGif, requestIP) { let action, - responseType = 2, + responseType = "stream", defaultParams = { u: r.urls, service: host, @@ -36,10 +36,10 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di switch (action) { default: - return apiJSON(0, { t: loc(lang, 'ErrorEmptyDownload') }); + return createResponse("error", { t: loc(lang, 'ErrorEmptyDownload') }); case "photo": - responseType = 1; + responseType = "redirect"; break; case "gif": @@ -56,11 +56,12 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di u: Array.isArray(r.urls) ? r.urls[0] : r.urls, mute: true } - if (host === "reddit" && r.typeId === 1) responseType = 1; + if (host === "reddit" && r.typeId === "redirect") + responseType = "redirect"; break; case "picker": - responseType = 5; + responseType = "picker"; switch (host) { case "instagram": case "twitter": @@ -98,7 +99,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di if (Array.isArray(r.urls)) { params = { type: "render" } } else { - responseType = 1; + responseType = "redirect"; } break; @@ -106,7 +107,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di if (r.type === "remux") { params = { type: r.type }; } else { - responseType = 1; + responseType = "redirect"; } break; @@ -121,14 +122,15 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di case "tumblr": case "pinterest": case "streamable": - responseType = 1; + responseType = "redirect"; break; } break; case "audio": - if ((host === "reddit" && r.typeId === 1) || audioIgnore.includes(host)) { - return apiJSON(0, { t: loc(lang, 'ErrorEmptyDownload') }) + if (audioIgnore.includes(host) + || (host === "reddit" && r.typeId === "redirect")) { + return createResponse("error", { t: loc(lang, 'ErrorEmptyDownload') }) } let processType = "render", @@ -178,5 +180,5 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di break; } - return apiJSON(responseType, {...defaultParams, ...params}) + return createResponse(responseType, {...defaultParams, ...params}) } diff --git a/src/modules/processing/request.js b/src/modules/processing/request.js new file mode 100644 index 00000000..9015e742 --- /dev/null +++ b/src/modules/processing/request.js @@ -0,0 +1,154 @@ +import ipaddr from "ipaddr.js"; + +import { normalizeURL } from "../processing/url.js"; +import { createStream } from "../stream/manage.js"; + +const apiVar = { + allowed: { + vCodec: ["h264", "av1", "vp9"], + vQuality: ["max", "4320", "2160", "1440", "1080", "720", "480", "360", "240", "144"], + aFormat: ["best", "mp3", "ogg", "wav", "opus"], + filenamePattern: ["classic", "pretty", "basic", "nerdy"] + }, + booleanOnly: [ + "isAudioOnly", + "isTTFullAudio", + "isAudioMuted", + "dubLang", + "disableMetadata", + "twitterGif", + "tiktokH265" + ] +} + +export function createResponse(responseType, responseData) { + try { + let status = 200, + response = {}; + + switch(responseType) { + case "error": + status = 400; + break; + + case "rate-limit": + status = 429; + break; + } + + switch (responseType) { + case "error": + case "success": + case "rate-limit": + response = { + text: responseData.t + } + break; + + case "redirect": + response = { + url: responseData.u + } + break; + + case "stream": + response = { + url: createStream(responseData) + } + break; + + case "picker": + let pickerType = "various", + audio = false; + + if (responseData.service === "tiktok") { + audio = responseData.u + pickerType = "images" + } + + response = { + pickerType: pickerType, + picker: responseData.picker, + audio: audio + } + break; + default: + throw "unreachable" + } + return { + status, + body: { + status: responseType, + ...response + } + } + } catch { + return { + status: 500, + body: { + status: "error", + text: "Internal Server Error" + } + } + } +} + +export function verifyRequest(request) { + try { + let template = { + url: normalizeURL(decodeURIComponent(request.url)), + vCodec: "h264", + vQuality: "720", + aFormat: "mp3", + filenamePattern: "classic", + isAudioOnly: false, + isTTFullAudio: false, + isAudioMuted: false, + disableMetadata: false, + dubLang: false, + twitterGif: false, + tiktokH265: false + } + + const requestKeys = Object.keys(request); + const templateKeys = Object.keys(template); + + if (requestKeys.length > templateKeys.length + 1 || !request.url) { + return false; + } + + for (const i in requestKeys) { + const key = requestKeys[i]; + const item = request[key]; + + if (String(key) !== "url" && templateKeys.includes(key)) { + if (apiVar.booleanOnly.includes(key)) { + template[key] = !!item; + } else if (apiVar.allowed[key] && apiVar.allowed[key].includes(item)) { + template[key] = String(item) + } + } + } + + if (template.dubLang) + template.dubLang = verifyLanguageCode(request.dubLang); + + return template + } catch { + return false + } +} + +export function getIP(req) { + const strippedIP = req.ip.replace(/^::ffff:/, ''); + const ip = ipaddr.parse(strippedIP); + if (ip.kind() === 'ipv4') { + return strippedIP; + } + + const prefix = 56; + const v6Bytes = ip.toByteArray(); + v6Bytes.fill(0, prefix / 8); + + return ipaddr.fromByteArray(v6Bytes).toString(); +} diff --git a/src/modules/processing/services/reddit.js b/src/modules/processing/services/reddit.js index e022f62c..75669538 100644 --- a/src/modules/processing/services/reddit.js +++ b/src/modules/processing/services/reddit.js @@ -68,7 +68,7 @@ export default async function(obj) { data = data[0]?.data?.children[0]?.data; if (data?.url?.endsWith('.gif')) return { - typeId: 1, + typeId: "redirect", urls: data.url } @@ -106,12 +106,12 @@ export default async function(obj) { let id = video.split('/')[3]; if (!audio) return { - typeId: 1, + typeId: "redirect", urls: video } return { - typeId: 2, + typeId: "stream", type: "render", urls: [video, audioFileLink], audioFilename: `reddit_${id}_audio`, diff --git a/src/modules/sub/utils.js b/src/modules/sub/utils.js index 59954445..e965bd9a 100644 --- a/src/modules/sub/utils.js +++ b/src/modules/sub/utils.js @@ -1,58 +1,5 @@ -import { normalizeURL } from "../processing/url.js"; -import { createStream } from "../stream/manage.js"; -import ipaddr from "ipaddr.js"; - -const apiVar = { - allowed: { - vCodec: ["h264", "av1", "vp9"], - vQuality: ["max", "4320", "2160", "1440", "1080", "720", "480", "360", "240", "144"], - aFormat: ["best", "mp3", "ogg", "wav", "opus"], - filenamePattern: ["classic", "pretty", "basic", "nerdy"] - }, - booleanOnly: [ - "isAudioOnly", - "isTTFullAudio", - "isAudioMuted", - "dubLang", - "disableMetadata", - "twitterGif", - "tiktokH265" - ] -} const forbiddenCharsString = ['}', '{', '%', '>', '<', '^', ';', '`', '$', '"', "@", '=']; -export function apiJSON(type, obj) { - try { - switch (type) { - case 0: - return { status: 400, body: { status: "error", text: obj.t } }; - case 1: - return { status: 200, body: { status: "redirect", url: obj.u } }; - case 2: - return { status: 200, body: { status: "stream", url: createStream(obj) } }; - case 3: - return { status: 200, body: { status: "success", text: obj.t } }; - case 4: - return { status: 429, body: { status: "rate-limit", text: obj.t } }; - case 5: - let pickerType = "various", audio = false - switch (obj.service) { - case "douyin": - case "tiktok": - audio = obj.u - pickerType = "images" - break; - } - return { status: 200, body: { status: "picker", pickerType: pickerType, picker: obj.picker, audio: audio } }; - case 6: // critical error, action should be taken by balancer/other server software - return { status: 500, body: { status: "error", text: obj.t, critical: true } }; - default: - return { status: 400, body: { status: "error", text: "Bad Request" } }; - } - } catch (e) { - return { status: 500, body: { status: "error", text: "Internal Server Error", critical: true } }; - } -} export function metadataManager(obj) { let keys = Object.keys(obj); let tags = ["album", "composer", "genre", "copyright", "encoded_by", "title", "language", "artist", "album_artist", "performer", "disc", "publisher", "track", "encoder", "compilation", "date", "creation_time", "comment"] @@ -79,57 +26,6 @@ export function unicodeDecode(str) { return String.fromCharCode(parseInt(unicode.replace(/\\u/g, ""), 16)); }); } -export function checkJSONPost(obj) { - let def = { - url: normalizeURL(decodeURIComponent(obj.url)), - vCodec: "h264", - vQuality: "720", - aFormat: "mp3", - filenamePattern: "classic", - isAudioOnly: false, - isTTFullAudio: false, - isAudioMuted: false, - disableMetadata: false, - dubLang: false, - twitterGif: false, - tiktokH265: false - } - try { - let objKeys = Object.keys(obj); - let defKeys = Object.keys(def); - if (objKeys.length > defKeys.length + 1 || !obj.url) return false; - - for (let i in objKeys) { - if (String(objKeys[i]) !== "url" && defKeys.includes(objKeys[i])) { - if (apiVar.booleanOnly.includes(objKeys[i])) { - def[objKeys[i]] = obj[objKeys[i]] ? true : false; - } else { - if (apiVar.allowed[objKeys[i]] && apiVar.allowed[objKeys[i]].includes(obj[objKeys[i]])) def[objKeys[i]] = String(obj[objKeys[i]]) - } - } - } - - if (def.dubLang) - def.dubLang = verifyLanguageCode(obj.dubLang); - - return def - } catch (e) { - return false - } -} -export function getIP(req) { - const strippedIP = req.ip.replace(/^::ffff:/, ''); - const ip = ipaddr.parse(strippedIP); - if (ip.kind() === 'ipv4') { - return strippedIP; - } - - const prefix = 56; - const v6Bytes = ip.toByteArray(); - v6Bytes.fill(0, prefix / 8); - - return ipaddr.fromByteArray(v6Bytes).toString(); -} export function cleanHTML(html) { let clean = html.replace(/ {4}/g, ''); clean = clean.replace(/\n/g, '');