From 4d369170ff80be1e4af991611097eaf6909fcb1d Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 19 May 2023 16:13:38 +0600 Subject: [PATCH] separated web and api, build improvements wip: - separate web and api servers. - script for building static pages. - building improvements. - async localisation preloading. --- .gitignore | 3 + package.json | 7 +- src/api.js | 196 +++++++++++++++++++++++++++++ src/cobalt.js | 28 +++-- src/front/cobalt.css | 5 +- src/localization/languages/en.json | 2 +- src/localization/languages/ru.json | 2 +- src/localization/manager.js | 15 +-- src/modules/build.js | 12 +- src/modules/buildStatic.js | 7 ++ src/modules/pageRender/elements.js | 3 +- src/modules/stream/manage.js | 2 +- src/web.js | 59 +++++++++ 13 files changed, 315 insertions(+), 26 deletions(-) create mode 100644 src/api.js create mode 100644 src/modules/buildStatic.js create mode 100644 src/web.js diff --git a/.gitignore b/.gitignore index 305746b2..8af52058 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ build # stuff i already made but delayed future + +# docker +docker-compose.yml diff --git a/package.json b/package.json index 1382b1f5..2c0f7258 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "5.7", + "version": "6.0-dev", "author": "wukko", "exports": "./src/cobalt.js", "type": "module", @@ -11,7 +11,10 @@ "scripts": { "start": "node src/cobalt", "setup": "node src/modules/setup", - "test": "node src/test/test" + "test": "node src/test/test", + "build": "node src/modules/buildStatic", + "api": "node src/api", + "web": "node src/web" }, "repository": { "type": "git", diff --git a/src/api.js b/src/api.js new file mode 100644 index 00000000..74c33e1f --- /dev/null +++ b/src/api.js @@ -0,0 +1,196 @@ +import "dotenv/config"; + +import express from "express"; +import cors from "cors"; +import rateLimit from "express-rate-limit"; +import { randomBytes } from "crypto"; + +const ipSalt = randomBytes(64).toString('hex'); + +import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js"; +import { appName, version } from "./modules/config.js"; +import { getJSON } from "./modules/api.js"; +import { apiJSON, checkJSONPost, getIP, languageCode } from "./modules/sub/utils.js"; +import { Bright, Cyan, Green, Red } from "./modules/sub/consoleText.js"; +import stream from "./modules/stream/stream.js"; +import loc, { loadLoc } from "./localization/manager.js"; +import { changelogHistory } from "./modules/pageRender/onDemand.js"; +import { sha256 } from "./modules/sub/crypto.js"; +import { celebrationsEmoji } from "./modules/pageRender/elements.js"; + +if (process.env.apiURL && process.env.apiPort) { + const commitHash = shortCommit(); + const branch = getCurrentBranch(); + const app = express(); + + app.disable('x-powered-by'); + + const corsConfig = process.env.cors === '0' ? { origin: process.env.webURL, optionsSuccessStatus: 200 } : {}; + + const apiLimiter = rateLimit({ + windowMs: 60000, + max: 25, + standardHeaders: false, + legacyHeaders: false, + keyGenerator: (req, res) => sha256(getIP(req), ipSalt), + handler: (req, res, next, opt) => { + res.status(429).json({ "status": "error", "text": loc(languageCode(req), 'ErrorRateLimit') }); + return; + } + }); + const apiLimiterStream = rateLimit({ + windowMs: 60000, + max: 28, + standardHeaders: false, + legacyHeaders: false, + keyGenerator: (req, res) => sha256(getIP(req), ipSalt), + handler: (req, res, next, opt) => { + res.status(429).json({ "status": "error", "text": loc(languageCode(req), 'ErrorRateLimit') }); + return; + } + }); + + const startTime = new Date(); + const startTimestamp = Math.floor(startTime.getTime()); + + // preload localization files + await loadLoc(); + + app.use('/api/:type', cors(corsConfig)); + app.use('/api/json', apiLimiter); + app.use('/api/stream', apiLimiterStream); + app.use('/api/onDemand', apiLimiter); + + app.use((req, res, next) => { + try { decodeURIComponent(req.path) } catch (e) { return res.redirect('/') } + next(); + }); + app.use((req, res, next) => { + if (req.header("user-agent") && req.header("user-agent").includes("Trident")) res.destroy(); + next(); + }); + app.use('/api/json', express.json({ + verify: (req, res, buf) => { + try { + JSON.parse(buf); + if (buf.length > 720) throw new Error(); + if (String(req.header('Content-Type')) !== "application/json") { + res.status(400).json({ 'status': 'error', 'text': 'invalid content type header' }); + return; + } + if (String(req.header('Accept')) !== "application/json") { + res.status(400).json({ 'status': 'error', 'text': 'invalid accept header' }); + return; + } + } catch(e) { + res.status(400).json({ 'status': 'error', 'text': 'invalid json body.' }); + return; + } + } + })); + + app.post('/api/json', async (req, res) => { + try { + let ip = sha256(getIP(req), ipSalt); + let lang = languageCode(req); + let j = apiJSON(0, { t: "Bad request" }); + try { + let request = req.body; + if (request.url) { + request.dubLang = request.dubLang ? lang : false; + let chck = checkJSONPost(request); + if (chck) chck["ip"] = ip; + j = chck ? await getJSON(chck["url"], lang, chck) : apiJSON(0, { t: loc(lang, 'ErrorCouldntFetch') }); + } else { + j = apiJSON(0, { t: loc(lang, 'ErrorNoLink') }); + } + } catch (e) { + j = apiJSON(0, { t: loc(lang, 'ErrorCantProcess') }); + } + res.status(j.status).json(j.body); + return; + } catch (e) { + res.destroy(); + return + } + }); + + app.get('/api/:type', (req, res) => { + try { + let ip = sha256(getIP(req), ipSalt); + switch (req.params.type) { + case 'stream': + if (req.query.p) { + res.status(200).json({ "status": "continue" }); + return; + } else if (req.query.t && req.query.h && req.query.e) { + stream(res, ip, req.query.t, req.query.h, req.query.e); + } else { + let j = apiJSON(0, { t: "no stream id" }) + res.status(j.status).json(j.body); + return; + } + break; + case 'onDemand': + if (req.query.blockId) { + let blockId = req.query.blockId.slice(0, 3); + let r, j; + switch(blockId) { + case "0": // changelog history + r = changelogHistory(); + j = r ? apiJSON(3, { t: r }) : apiJSON(0, { t: "couldn't render this block" }) + break; + case "1": // celebrations emoji + r = celebrationsEmoji(); + j = r ? apiJSON(3, { t: r }) : false + break; + default: + j = apiJSON(0, { t: "couldn't find a block with this id" }) + break; + } + if (j.body) { + res.status(j.status).json(j.body) + } else { + res.status(204).end() + } + } else { + let j = apiJSON(0, { t: "no block id" }); + res.status(j.status).json(j.body) + } + break; + case 'serverInfo': + res.status(200).json({ + version: version, + commit: commitHash, + branch: branch, + url: process.env.apiURL, + cors: process.env.cors, + startTime: `${startTimestamp}` + }); + break; + default: + let j = apiJSON(0, { t: "unknown response type" }) + res.status(j.status).json(j.body); + break; + } + } catch (e) { + res.status(500).json({ 'status': 'error', 'text': loc(languageCode(req), 'ErrorCantProcess') }); + return; + } + }); + app.get("/api/status", (req, res) => { + res.status(200).end() + }); + app.get("/favicon.ico", (req, res) => { + res.redirect('/icons/favicon.ico'); + }); + app.get("/*", (req, res) => { + res.redirect('/api/json') + }); + + app.listen(process.env.apiPort, () => { + console.log(`\n${Cyan(appName)} API ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${startTimestamp})`)}\n\nURL: ${Cyan(`${process.env.apiURL}`)}\nPort: ${process.env.apiPort}\n`) + }) +} else { + console.log(Red(`cobalt api hasn't been configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`)); +} diff --git a/src/cobalt.js b/src/cobalt.js index 990f5d31..066385a7 100644 --- a/src/cobalt.js +++ b/src/cobalt.js @@ -18,7 +18,7 @@ import { getJSON } from "./modules/api.js"; import { apiJSON, checkJSONPost, getIP, languageCode } from "./modules/sub/utils.js"; import { Bright, Cyan, Green, Red } from "./modules/sub/consoleText.js"; import stream from "./modules/stream/stream.js"; -import loc from "./localization/manager.js"; +import loc, { loadLoc } from "./localization/manager.js"; import { buildFront } from "./modules/build.js"; import { changelogHistory } from "./modules/pageRender/onDemand.js"; import { sha256 } from "./modules/sub/crypto.js"; @@ -56,7 +56,12 @@ if (process.env.selfURL && process.env.port) { return; } }); + + const startTime = new Date(); + const startTimestamp = Math.floor(startTime.getTime()); + // preload localization files and build static pages + await loadLoc(); await buildFront(commitHash, branch); app.use('/api/:type', cors(corsConfig)); @@ -64,7 +69,7 @@ if (process.env.selfURL && process.env.port) { app.use('/api/stream', apiLimiterStream); app.use('/api/onDemand', apiLimiter); - app.use('/', express.static('./min')); + app.use('/', express.static('./build/min')); app.use('/', express.static('./src/front')); app.use((req, res, next) => { @@ -164,6 +169,16 @@ if (process.env.selfURL && process.env.port) { res.status(j.status).json(j.body) } break; + case 'serverInfo': + res.status(200).json({ + version: version, + commit: commitHash, + branch: branch, + url: process.env.apiURL, + cors: process.env.cors, + startTime: `${startTimestamp}` + }); + break; default: let j = apiJSON(0, { t: "unknown response type" }) res.status(j.status).json(j.body); @@ -174,12 +189,12 @@ if (process.env.selfURL && process.env.port) { return; } }); + app.get("/api/status", (req, res) => { + res.status(200).end() + }); app.get("/api", (req, res) => { res.redirect('/api/json') }); - app.get("/status", (req, res) => { - res.status(200).end() - }); app.get("/", (req, res) => { res.sendFile(`${__dirname}/${findRendered(languageCode(req), req.header('user-agent') ? req.header('user-agent') : genericUserAgent)}`); }); @@ -191,8 +206,7 @@ if (process.env.selfURL && process.env.port) { }); app.listen(process.env.port, () => { - let startTime = new Date(); - console.log(`\n${Cyan(appName)} ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(new Date().getTime())})`)}\n\nURL: ${Cyan(`${process.env.selfURL}`)}\nPort: ${process.env.port}\n`) + console.log(`\n${Cyan(appName)} ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(startTimestamp)})`)}\n\nURL: ${Cyan(`${process.env.selfURL}`)}\nPort: ${process.env.port}\n`) }) } else { console.log(Red(`cobalt hasn't been configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`)); diff --git a/src/front/cobalt.css b/src/front/cobalt.css index 9f9c8a9c..338f0dc1 100644 --- a/src/front/cobalt.css +++ b/src/front/cobalt.css @@ -156,7 +156,7 @@ button:active, .text-to-copy:active { background: var(--accent-press); cursor: pointer; - transform: scale(0.95) + transform: scale(0.95); } .collapse-header:active { background: var(--accent-press); @@ -681,6 +681,9 @@ button:active, #about-donate-footer:active::before { opacity: 0; } +.popup-tabs-child { + width: 100%; +} /* adapt the page according to screen size */ @media screen and (min-width: 2300px) { html { diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json index 4f052578..4fe42dbd 100644 --- a/src/localization/languages/en.json +++ b/src/localization/languages/en.json @@ -47,7 +47,7 @@ "SettingsQualityDescription": "if selected quality isn't available, closest one is used instead.", "LinkGitHubChanges": ">> see previous commits and contribute on github", "NoScriptMessage": "{appName} uses javascript for api requests and interactive interface. you have to allow javascript to use this site. there are no pesty scripts, pinky promise.", - "DownloadPopupDescriptionIOS": "easiest way to save videos on ios:\n1. add this siri shortcut.\n2. press \"share\" above and select \"save to photos\" in appeared share sheet.\nif asked, review the permission request popup on top, and press \"always allow\".\n\nalternative method: press and hold the download button, hide the video preview, and select \"download linked file\" to download.\nthen, open safari downloads, select the file you downloaded, open share menu, and finally press \"save video\".", + "DownloadPopupDescriptionIOS": "easiest way to save videos on ios:\n1. add this siri shortcut.\n2. press \"share\" above and select \"save to photos\" in appeared share sheet.\nif asked, review the permission request, and press \"always allow\".\n\nalternative method:\npress and hold the download button, hide the video preview, and select \"download linked file\" to download.\nthen, open safari downloads, select the file you downloaded, open share menu, and finally press \"save video\".", "DownloadPopupDescription": "download button opens a new tab with requested file. you can disable this popup in settings.", "DownloadPopupWayToSave": "pick a way to save", "ClickToCopy": "press to copy", diff --git a/src/localization/languages/ru.json b/src/localization/languages/ru.json index 7f59f356..8a0845f8 100644 --- a/src/localization/languages/ru.json +++ b/src/localization/languages/ru.json @@ -47,7 +47,7 @@ "SettingsQualityDescription": "если выбранное качество недоступно, то выбирается ближайшее к нему.", "LinkGitHubChanges": ">> смотри предыдущие изменения на github", "NoScriptMessage": "{appName} использует javascript для обработки ссылок и интерактивного интерфейса. ты должен разрешить использование javascript, чтобы пользоваться сайтом. тут нет никаких зловредных скриптов, обещаю.", - "DownloadPopupDescriptionIOS": "наиболее простой метод скачивания видео на ios:\n1. добавь этот сценарий siri.\n2. нажми \"поделиться\" выше и выбери \"save to photos\" в открывшемся окне.\nесли появляется окно с запросом разрешения, то прочитай его, потом нажми \"всегда разрешать\".\n\nальтернативный метод: зажми кнопку \"скачать\", затем скрой превью и выбери \"загрузить файл по ссылке\" в появившемся окне.\nпотом открой загрузки в safari, выбери скачанный файл, нажми иконку \"поделиться\", и, наконец, нажми \"сохранить видео\".", + "DownloadPopupDescriptionIOS": "наиболее простой метод скачивания видео на ios:\n1. добавь этот сценарий siri.\n2. нажми \"поделиться\" выше и выбери \"save to photos\" в открывшемся окне.\nесли появляется окно с запросом разрешения, то прочитай его, потом нажми \"всегда разрешать\".\n\nальтернативный метод:\nзажми кнопку \"скачать\", затем скрой превью и выбери \"загрузить файл по ссылке\" в появившемся окне.\nпотом открой загрузки в safari, выбери скачанный файл, нажми иконку \"поделиться\", и, наконец, нажми \"сохранить видео\".", "DownloadPopupDescription": "кнопка скачивания открывает новое окно с файлом. ты можешь отключить выбор метода скачивания файла в настройках.", "DownloadPopupWayToSave": "выбери, как сохранить", "ClickToCopy": "нажми, чтобы скопировать", diff --git a/src/localization/manager.js b/src/localization/manager.js index 2f5a334a..e5eec9bb 100644 --- a/src/localization/manager.js +++ b/src/localization/manager.js @@ -7,16 +7,13 @@ const locPath = './src/localization/languages'; let loc = {} let languages = []; -export function loadLoc() { - fs.readdir(locPath, (err, files) => { - if (err) return false; - files.forEach(file => { - loc[file.split('.')[0]] = loadJson(`${locPath}/${file}`); - languages.push(file.split('.')[0]) - }); - }) +export async function loadLoc() { + const files = await fs.promises.readdir(locPath).catch((e) => { return [] }); + files.forEach(file => { + loc[file.split('.')[0]] = loadJson(`${locPath}/${file}`); + languages.push(file.split('.')[0]) + }); } -loadLoc(); export function replaceBase(s) { return s.replace(/\n/g, '
').replace(/{saveToGalleryShortcut}/g, links.saveToGalleryShortcut).replace(/{appName}/g, appName).replace(/{repo}/g, repo).replace(/\*;/g, "•"); diff --git a/src/modules/build.js b/src/modules/build.js index 6c4bee85..a2515b0e 100644 --- a/src/modules/build.js +++ b/src/modules/build.js @@ -1,6 +1,7 @@ import * as esbuild from "esbuild"; import * as fs from "fs"; -import { languageList } from "../localization/manager.js"; +import { loadLoc, languageList } from "../localization/manager.js"; + import page from "./pageRender/page.js"; function cleanHTML(html) { @@ -10,6 +11,9 @@ function cleanHTML(html) { } export async function buildFront(commitHash, branch) { try { + // preload localization files + await loadLoc(); + // build html if (!fs.existsSync('./build/')){ fs.mkdirSync('./build/'); @@ -17,6 +21,10 @@ export async function buildFront(commitHash, branch) { fs.mkdirSync('./build/pc/'); fs.mkdirSync('./build/mob/'); } + // get rid of old build path + if (fs.existsSync('./min')) { + fs.rmSync('./min', { recursive: true, force: true }); + } for (let i in languageList) { i = languageList[i]; let params = { @@ -36,7 +44,7 @@ export async function buildFront(commitHash, branch) { // build js & css await esbuild.build({ entryPoints: ['src/front/cobalt.js', 'src/front/cobalt.css'], - outdir: 'min/', + outdir: 'build/min/', minify: true, loader: { '.js': 'js', '.css': 'css', }, charset: 'utf8' diff --git a/src/modules/buildStatic.js b/src/modules/buildStatic.js new file mode 100644 index 00000000..d3ed909f --- /dev/null +++ b/src/modules/buildStatic.js @@ -0,0 +1,7 @@ +import { buildFront } from "./build.js"; +import { getCurrentBranch, shortCommit } from "./sub/currentCommit.js"; + +const commitHash = shortCommit(); +const branch = getCurrentBranch(); + +await buildFront(commitHash, branch); diff --git a/src/modules/pageRender/elements.js b/src/modules/pageRender/elements.js index cdf155ed..dd9861a0 100644 --- a/src/modules/pageRender/elements.js +++ b/src/modules/pageRender/elements.js @@ -89,14 +89,13 @@ export function multiPagePopup(obj) { tabs += `` tabContent += `
${obj.tabs[i]["content"]}
` } - tabs += `` return ` ` } export function collapsibleList(arr) { diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js index e01c2cb0..26dfc08c 100644 --- a/src/modules/stream/manage.js +++ b/src/modules/stream/manage.js @@ -39,7 +39,7 @@ export function createStream(obj) { exp = streamInfo.exp; ghmac = streamInfo.hmac; } - return `${process.env.selfURL}api/stream?t=${streamID}&e=${exp}&h=${ghmac}`; + return `${process.env.apiURL || process.env.selfURL}api/stream?t=${streamID}&e=${exp}&h=${ghmac}`; } export function verifyStream(ip, id, hmac, exp) { diff --git a/src/web.js b/src/web.js new file mode 100644 index 00000000..4f298975 --- /dev/null +++ b/src/web.js @@ -0,0 +1,59 @@ +import "dotenv/config"; + +import express from "express"; + +import path from 'path'; +import { fileURLToPath } from 'url'; +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename).slice(0, -4); // go up another level (get rid of src/) + +import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js"; +import { appName, genericUserAgent, version } from "./modules/config.js"; +import { languageCode } from "./modules/sub/utils.js"; +import { Bright, Cyan, Green, Red } from "./modules/sub/consoleText.js"; +import { loadLoc } from "./localization/manager.js"; +import { buildFront } from "./modules/build.js"; +import findRendered from "./modules/pageRender/findRendered.js"; + +if (process.env.webURL && process.env.webPort) { + const commitHash = shortCommit(); + const branch = getCurrentBranch(); + const app = express(); + + app.disable('x-powered-by'); + + // preload localization files and build static pages + await loadLoc(); + await buildFront(commitHash, branch); + + app.use('/', express.static('./build/min')); + app.use('/', express.static('./src/front')); + + app.use((req, res, next) => { + try { decodeURIComponent(req.path) } catch (e) { return res.redirect('/') } + next(); + }); + app.use((req, res, next) => { + if (req.header("user-agent") && req.header("user-agent").includes("Trident")) res.destroy(); + next(); + }); + app.get("/status", (req, res) => { + res.status(200).end() + }); + app.get("/", (req, res) => { + res.sendFile(`${__dirname}/${findRendered(languageCode(req), req.header('user-agent') ? req.header('user-agent') : genericUserAgent)}`); + }); + app.get("/favicon.ico", (req, res) => { + res.redirect('/icons/favicon.ico'); + }); + app.get("/*", (req, res) => { + res.redirect('/') + }); + + app.listen(process.env.webPort, () => { + let startTime = new Date(); + console.log(`\n${Cyan(appName)} WEB ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(new Date().getTime())})`)}\n\nURL: ${Cyan(`${process.env.webURL}`)}\nPort: ${process.env.webPort}\n`) + }) +} else { + console.log(Red(`cobalt web hasn't been configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`)); +}