separated web and api, build improvements

wip:
- separate web and api servers.
- script for building static pages.
- building improvements.
- async localisation preloading.

.gitignore

@@ -13,3 +13,6 @@ build
 
 # stuff i already made but delayed
 future
+
+# docker
+docker-compose.yml

package.json

@@ -1,7 +1,7 @@
 {
     "name": "cobalt",
     "description": "save what you love",
-    "version": "5.7",
+    "version": "6.0-dev",
     "author": "wukko",
     "exports": "./src/cobalt.js",
     "type": "module",
@@ -11,7 +11,10 @@
     "scripts": {
         "start": "node src/cobalt",
         "setup": "node src/modules/setup",
-        "test": "node src/test/test"
+        "test": "node src/test/test",
+        "build": "node src/modules/buildStatic",
+        "api": "node src/api",
+        "web": "node src/web"
     },
     "repository": {
         "type": "git",

src/api.js

@@ -0,0 +1,196 @@
+import "dotenv/config";
+
+import express from "express";
+import cors from "cors";
+import rateLimit from "express-rate-limit";
+import { randomBytes } from "crypto";
+
+const ipSalt = randomBytes(64).toString('hex');
+
+import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js";
+import { appName, version } from "./modules/config.js";
+import { getJSON } from "./modules/api.js";
+import { apiJSON, checkJSONPost, getIP, languageCode } from "./modules/sub/utils.js";
+import { Bright, Cyan, Green, Red } from "./modules/sub/consoleText.js";
+import stream from "./modules/stream/stream.js";
+import loc, { loadLoc } from "./localization/manager.js";
+import { changelogHistory } from "./modules/pageRender/onDemand.js";
+import { sha256 } from "./modules/sub/crypto.js";
+import { celebrationsEmoji } from "./modules/pageRender/elements.js";
+
+if (process.env.apiURL && process.env.apiPort) {
+    const commitHash = shortCommit();
+    const branch = getCurrentBranch();
+    const app = express();
+
+    app.disable('x-powered-by');
+
+    const corsConfig = process.env.cors === '0' ? { origin: process.env.webURL, optionsSuccessStatus: 200 } : {};
+
+    const apiLimiter = rateLimit({
+        windowMs: 60000,
+        max: 25,
+        standardHeaders: false,
+        legacyHeaders: false,
+        keyGenerator: (req, res) => sha256(getIP(req), ipSalt),
+        handler: (req, res, next, opt) => {
+            res.status(429).json({ "status": "error", "text": loc(languageCode(req), 'ErrorRateLimit') });
+            return;
+        }
+    });
+    const apiLimiterStream = rateLimit({
+        windowMs: 60000,
+        max: 28,
+        standardHeaders: false,
+        legacyHeaders: false,
+        keyGenerator: (req, res) => sha256(getIP(req), ipSalt),
+        handler: (req, res, next, opt) => {
+            res.status(429).json({ "status": "error", "text": loc(languageCode(req), 'ErrorRateLimit') });
+            return;
+        }
+    });
+    
+    const startTime = new Date();
+    const startTimestamp = Math.floor(startTime.getTime());
+
+    // preload localization files
+    await loadLoc();
+
+    app.use('/api/:type', cors(corsConfig));
+    app.use('/api/json', apiLimiter);
+    app.use('/api/stream', apiLimiterStream);
+    app.use('/api/onDemand', apiLimiter);
+
+    app.use((req, res, next) => {
+        try { decodeURIComponent(req.path) } catch (e) { return res.redirect('/') }
+        next();
+    });
+    app.use((req, res, next) => {
+        if (req.header("user-agent") && req.header("user-agent").includes("Trident")) res.destroy();
+        next();
+    });
+    app.use('/api/json', express.json({
+        verify: (req, res, buf) => {
+            try {
+                JSON.parse(buf);
+                if (buf.length > 720) throw new Error();
+                if (String(req.header('Content-Type')) !== "application/json") {
+                    res.status(400).json({ 'status': 'error', 'text': 'invalid content type header' });
+                    return;
+                }
+                if (String(req.header('Accept')) !== "application/json") {
+                    res.status(400).json({ 'status': 'error', 'text': 'invalid accept header' });
+                    return;
+                }
+            } catch(e) {
+                res.status(400).json({ 'status': 'error', 'text': 'invalid json body.' });
+                return;
+            }
+        }
+    }));
+
+    app.post('/api/json', async (req, res) => {
+        try {
+            let ip = sha256(getIP(req), ipSalt);
+            let lang = languageCode(req);
+            let j = apiJSON(0, { t: "Bad request" });
+            try {
+                let request = req.body;
+                if (request.url) {
+                    request.dubLang = request.dubLang ? lang : false;
+                    let chck = checkJSONPost(request);
+                    if (chck) chck["ip"] = ip;
+                    j = chck ? await getJSON(chck["url"], lang, chck) : apiJSON(0, { t: loc(lang, 'ErrorCouldntFetch') });
+                } else {
+                    j = apiJSON(0, { t: loc(lang, 'ErrorNoLink') });
+                }
+            } catch (e) {
+                j = apiJSON(0, { t: loc(lang, 'ErrorCantProcess') });
+            }
+            res.status(j.status).json(j.body);
+            return;
+        } catch (e) {
+            res.destroy();
+            return
+        }
+    });
+
+    app.get('/api/:type', (req, res) => {
+        try {
+            let ip = sha256(getIP(req), ipSalt);
+            switch (req.params.type) {
+                case 'stream':
+                    if (req.query.p) {
+                        res.status(200).json({ "status": "continue" });
+                        return;
+                    } else if (req.query.t && req.query.h && req.query.e) {
+                        stream(res, ip, req.query.t, req.query.h, req.query.e);
+                    } else {
+                        let j = apiJSON(0, { t: "no stream id" })
+                        res.status(j.status).json(j.body);
+                        return;
+                    }
+                    break;
+                case 'onDemand':
+                    if (req.query.blockId) {
+                        let blockId = req.query.blockId.slice(0, 3);
+                        let r, j;
+                        switch(blockId) {
+                            case "0": // changelog history
+                                r = changelogHistory();
+                                j = r ? apiJSON(3, { t: r }) : apiJSON(0, { t: "couldn't render this block" })
+                                break;
+                            case "1": // celebrations emoji
+                                r = celebrationsEmoji();
+                                j = r ? apiJSON(3, { t: r }) : false
+                                break;
+                            default:
+                                j = apiJSON(0, { t: "couldn't find a block with this id" })
+                                break;
+                        }
+                        if (j.body) {
+                            res.status(j.status).json(j.body)
+                        } else {
+                            res.status(204).end()
+                        }
+                    } else {
+                        let j = apiJSON(0, { t: "no block id" });
+                        res.status(j.status).json(j.body)
+                    }
+                    break;
+                case 'serverInfo':
+                    res.status(200).json({
+                        version: version,
+                        commit: commitHash,
+                        branch: branch,
+                        url: process.env.apiURL,
+                        cors: process.env.cors,
+                        startTime: `${startTimestamp}`
+                    });
+                    break;
+                default:
+                    let j = apiJSON(0, { t: "unknown response type" })
+                    res.status(j.status).json(j.body);
+                    break;
+            }
+        } catch (e) {
+            res.status(500).json({ 'status': 'error', 'text': loc(languageCode(req), 'ErrorCantProcess') });
+            return;
+        }
+    });
+    app.get("/api/status", (req, res) => {
+        res.status(200).end()
+    });
+    app.get("/favicon.ico", (req, res) => {
+        res.redirect('/icons/favicon.ico');
+    });
+    app.get("/*", (req, res) => {
+        res.redirect('/api/json')
+    });
+
+    app.listen(process.env.apiPort, () => {
+        console.log(`\n${Cyan(appName)} API ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${startTimestamp})`)}\n\nURL: ${Cyan(`${process.env.apiURL}`)}\nPort: ${process.env.apiPort}\n`)
+    })
+} else {
+    console.log(Red(`cobalt api hasn't been configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`));
+}

src/cobalt.js

@@ -18,7 +18,7 @@ import { getJSON } from "./modules/api.js";
 import { apiJSON, checkJSONPost, getIP, languageCode } from "./modules/sub/utils.js";
 import { Bright, Cyan, Green, Red } from "./modules/sub/consoleText.js";
 import stream from "./modules/stream/stream.js";
-import loc from "./localization/manager.js";
+import loc, { loadLoc } from "./localization/manager.js";
 import { buildFront } from "./modules/build.js";
 import { changelogHistory } from "./modules/pageRender/onDemand.js";
 import { sha256 } from "./modules/sub/crypto.js";
@@ -57,6 +57,11 @@ if (process.env.selfURL && process.env.port) {
         }
     });
     
+    const startTime = new Date();
+    const startTimestamp = Math.floor(startTime.getTime());
+
+    // preload localization files and build static pages
+    await loadLoc();
     await buildFront(commitHash, branch);
 
     app.use('/api/:type', cors(corsConfig));
@@ -64,7 +69,7 @@ if (process.env.selfURL && process.env.port) {
     app.use('/api/stream', apiLimiterStream);
     app.use('/api/onDemand', apiLimiter);
 
-    app.use('/', express.static('./min'));
+    app.use('/', express.static('./build/min'));
     app.use('/', express.static('./src/front'));
 
     app.use((req, res, next) => {
@@ -164,6 +169,16 @@ if (process.env.selfURL && process.env.port) {
                         res.status(j.status).json(j.body)
                     }
                     break;
+                case 'serverInfo':
+                    res.status(200).json({
+                        version: version,
+                        commit: commitHash,
+                        branch: branch,
+                        url: process.env.apiURL,
+                        cors: process.env.cors,
+                        startTime: `${startTimestamp}`
+                    });
+                    break;
                 default:
                     let j = apiJSON(0, { t: "unknown response type" })
                     res.status(j.status).json(j.body);
@@ -174,12 +189,12 @@ if (process.env.selfURL && process.env.port) {
             return;
         }
     });
+    app.get("/api/status", (req, res) => {
+        res.status(200).end()
+    });
     app.get("/api", (req, res) => {
         res.redirect('/api/json')
     });
-    app.get("/status", (req, res) => {
-        res.status(200).end()
-    });
     app.get("/", (req, res) => {
         res.sendFile(`${__dirname}/${findRendered(languageCode(req), req.header('user-agent') ? req.header('user-agent') : genericUserAgent)}`);
     });
@@ -191,8 +206,7 @@ if (process.env.selfURL && process.env.port) {
     });
 
     app.listen(process.env.port, () => {
-        let startTime = new Date();
+        console.log(`\n${Cyan(appName)} ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(startTimestamp)})`)}\n\nURL: ${Cyan(`${process.env.selfURL}`)}\nPort: ${process.env.port}\n`)
-        console.log(`\n${Cyan(appName)} ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(new Date().getTime())})`)}\n\nURL: ${Cyan(`${process.env.selfURL}`)}\nPort: ${process.env.port}\n`)
     })
 } else {
     console.log(Red(`cobalt hasn't been configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`));

src/front/cobalt.css

@@ -156,7 +156,7 @@ button:active,
 .text-to-copy:active {
     background: var(--accent-press);
     cursor: pointer;
-    transform: scale(0.95)
+    transform: scale(0.95);
 }
 .collapse-header:active {
     background: var(--accent-press);
@@ -681,6 +681,9 @@ button:active,
 #about-donate-footer:active::before {
     opacity: 0;
 }
+.popup-tabs-child {
+    width: 100%;
+}
 /* adapt the page according to screen size */
 @media screen and (min-width: 2300px) {
     html {

src/localization/languages/en.json

@@ -47,7 +47,7 @@
         "SettingsQualityDescription": "if selected quality isn't available, closest one is used instead.",
         "LinkGitHubChanges": ">> see previous commits and contribute on github",
         "NoScriptMessage": "{appName} uses javascript for api requests and interactive interface. you have to allow javascript to use this site. there are no pesty scripts, pinky promise.",
-        "DownloadPopupDescriptionIOS": "easiest way to save videos on ios:\n1. add <a class=\"text-backdrop italic\" href=\"{saveToGalleryShortcut}\" target=\"_blank\">this siri shortcut</a>.\n2. press \"share\" above and select \"save to photos\" in appeared share sheet.\nif asked, review the permission request popup on top, and press \"always allow\".\n\nalternative method: press and hold the download button, hide the video preview, and select \"download linked file\" to download.\nthen, open safari downloads, select the file you downloaded, open share menu, and finally press \"save video\".",
+        "DownloadPopupDescriptionIOS": "easiest way to save videos on ios:\n1. add <a class=\"text-backdrop italic\" href=\"{saveToGalleryShortcut}\" target=\"_blank\">this siri shortcut</a>.\n2. press \"share\" above and select \"save to photos\" in appeared share sheet.\nif asked, review the permission request, and press \"always allow\".\n\nalternative method:\npress and hold the download button, hide the video preview, and select \"download linked file\" to download.\nthen, open safari downloads, select the file you downloaded, open share menu, and finally press \"save video\".",
         "DownloadPopupDescription": "download button opens a new tab with requested file. you can disable this popup in settings.",
         "DownloadPopupWayToSave": "pick a way to save",
         "ClickToCopy": "press to copy",

src/localization/languages/ru.json

@@ -47,7 +47,7 @@
         "SettingsQualityDescription": "если выбранное качество недоступно, то выбирается ближайшее к нему.",
         "LinkGitHubChanges": ">> смотри предыдущие изменения на github",
         "NoScriptMessage": "{appName} использует javascript для обработки ссылок и интерактивного интерфейса. ты должен разрешить использование javascript, чтобы пользоваться сайтом. тут нет никаких зловредных скриптов, обещаю.",
-        "DownloadPopupDescriptionIOS": "наиболее простой метод скачивания видео на ios:\n1. добавь <a class=\"text-backdrop italic\" href=\"{saveToGalleryShortcut}\" target=\"_blank\">этот сценарий siri</a>.\n2. нажми \"поделиться\" выше и выбери \"save to photos\" в открывшемся окне.\nесли появляется окно с запросом разрешения, то прочитай его, потом нажми \"всегда разрешать\".\n\nальтернативный метод: зажми кнопку \"скачать\", затем скрой превью и выбери \"загрузить файл по ссылке\" в появившемся окне.\nпотом открой загрузки в safari, выбери скачанный файл, нажми иконку \"поделиться\", и, наконец, нажми \"сохранить видео\".",
+        "DownloadPopupDescriptionIOS": "наиболее простой метод скачивания видео на ios:\n1. добавь <a class=\"text-backdrop italic\" href=\"{saveToGalleryShortcut}\" target=\"_blank\">этот сценарий siri</a>.\n2. нажми \"поделиться\" выше и выбери \"save to photos\" в открывшемся окне.\nесли появляется окно с запросом разрешения, то прочитай его, потом нажми \"всегда разрешать\".\n\nальтернативный метод:\nзажми кнопку \"скачать\", затем скрой превью и выбери \"загрузить файл по ссылке\" в появившемся окне.\nпотом открой загрузки в safari, выбери скачанный файл, нажми иконку \"поделиться\", и, наконец, нажми \"сохранить видео\".",
         "DownloadPopupDescription": "кнопка скачивания открывает новое окно с файлом. ты можешь отключить выбор метода скачивания файла в настройках.",
         "DownloadPopupWayToSave": "выбери, как сохранить",
         "ClickToCopy": "нажми, чтобы скопировать",

src/localization/manager.js

@@ -7,16 +7,13 @@ const locPath = './src/localization/languages';
 let loc = {}
 let languages = [];
 
-export function loadLoc() {
+export async function loadLoc() {
-    fs.readdir(locPath, (err, files) => {
+    const files = await fs.promises.readdir(locPath).catch((e) => { return [] });
-        if (err) return false;
     files.forEach(file => {
         loc[file.split('.')[0]] = loadJson(`${locPath}/${file}`);
         languages.push(file.split('.')[0])
     });
-    })
 }
-loadLoc();
 
 export function replaceBase(s) {
     return s.replace(/\n/g, '<br/>').replace(/{saveToGalleryShortcut}/g, links.saveToGalleryShortcut).replace(/{appName}/g, appName).replace(/{repo}/g, repo).replace(/\*;/g, "&bull;");

src/modules/build.js

@@ -1,6 +1,7 @@
 import * as esbuild from "esbuild";
 import * as fs from "fs";
-import { languageList } from "../localization/manager.js";
+import { loadLoc, languageList } from "../localization/manager.js";
+
 import page from "./pageRender/page.js";
 
 function cleanHTML(html) {
@@ -10,6 +11,9 @@ function cleanHTML(html) {
 }
 export async function buildFront(commitHash, branch) {
     try {
+        // preload localization files
+        await loadLoc();
+
         // build html
         if (!fs.existsSync('./build/')){
             fs.mkdirSync('./build/');
@@ -17,6 +21,10 @@ export async function buildFront(commitHash, branch) {
             fs.mkdirSync('./build/pc/');
             fs.mkdirSync('./build/mob/');
         }
+        // get rid of old build path
+        if (fs.existsSync('./min')) {
+            fs.rmSync('./min', { recursive: true, force: true });
+        }
         for (let i in languageList) {
             i = languageList[i];
             let params = {
@@ -36,7 +44,7 @@ export async function buildFront(commitHash, branch) {
         // build js & css
         await esbuild.build({
             entryPoints: ['src/front/cobalt.js', 'src/front/cobalt.css'],
-            outdir: 'min/',
+            outdir: 'build/min/',
             minify: true,
             loader: { '.js': 'js', '.css': 'css', },
             charset: 'utf8'

src/modules/buildStatic.js

@@ -0,0 +1,7 @@
+import { buildFront } from "./build.js";
+import { getCurrentBranch, shortCommit } from "./sub/currentCommit.js";
+
+const commitHash = shortCommit();
+const branch = getCurrentBranch();
+
+await buildFront(commitHash, branch);

src/modules/pageRender/elements.js

@@ -89,14 +89,13 @@ export function multiPagePopup(obj) {
         tabs += `<button id="tab-button-${obj.name}-${obj.tabs[i]["name"]}" class="switch tab tab-${obj.name}" onclick="changeTab(event, 'tab-${obj.name}-${obj.tabs[i]["name"]}', '${obj.name}')">${obj.tabs[i]["title"]}</button>`
         tabContent += `<div id="tab-${obj.name}-${obj.tabs[i]["name"]}" class="popup-tab-content tab-content-${obj.name}">${obj.tabs[i]["content"]}</div>`
     }
-    tabs += `<button id="close-button" class="switch tab-${obj.name}" onclick="popup('${obj.name}', 0)" ${obj.closeAria ? `aria-label="${obj.closeAria}"` : ''}>x</button>`
     return `
     <div id="popup-${obj.name}" class="popup center box scrollable" style="visibility: hidden;">
         <div id="popup-content">${obj.header ? `<div id="popup-header" class="popup-header">
         ${obj.header.aboveTitle ? `<a id="popup-above-title" target="_blank" href="${obj.header.aboveTitle.url}">${obj.header.aboveTitle.text}</a>` : ''}
         ${obj.header.title ? `<div id="popup-title">${obj.header.title}</div>` : ''}
         ${obj.header.subtitle ? `<div id="popup-subtitle">${obj.header.subtitle}</div>` : ''}</div>` : ''}${tabContent}</div>
-        <div id="popup-tabs" class="switches popup-tabs">${tabs}</div>
+        <div id="popup-tabs" class="switches popup-tabs"><div class="switches popup-tabs-child">${tabs}</div><button id="close-button" class="switch tab-${obj.name}" onclick="popup('${obj.name}', 0)" ${obj.closeAria ? `aria-label="${obj.closeAria}"` : ''}>x</button></div>
     </div>`
 }
 export function collapsibleList(arr) {

src/modules/stream/manage.js

@@ -39,7 +39,7 @@ export function createStream(obj) {
         exp = streamInfo.exp;
         ghmac = streamInfo.hmac;
     }
-    return `${process.env.selfURL}api/stream?t=${streamID}&e=${exp}&h=${ghmac}`;
+    return `${process.env.apiURL || process.env.selfURL}api/stream?t=${streamID}&e=${exp}&h=${ghmac}`;
 }
 
 export function verifyStream(ip, id, hmac, exp) {

src/web.js

@@ -0,0 +1,59 @@
+import "dotenv/config";
+
+import express from "express";
+
+import path from 'path';
+import { fileURLToPath } from 'url';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename).slice(0, -4); // go up another level (get rid of src/)
+
+import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js";
+import { appName, genericUserAgent, version } from "./modules/config.js";
+import { languageCode } from "./modules/sub/utils.js";
+import { Bright, Cyan, Green, Red } from "./modules/sub/consoleText.js";
+import { loadLoc } from "./localization/manager.js";
+import { buildFront } from "./modules/build.js";
+import findRendered from "./modules/pageRender/findRendered.js";
+
+if (process.env.webURL && process.env.webPort) {
+    const commitHash = shortCommit();
+    const branch = getCurrentBranch();
+    const app = express();
+
+    app.disable('x-powered-by');
+
+    // preload localization files and build static pages
+    await loadLoc();
+    await buildFront(commitHash, branch);
+
+    app.use('/', express.static('./build/min'));
+    app.use('/', express.static('./src/front'));
+
+    app.use((req, res, next) => {
+        try { decodeURIComponent(req.path) } catch (e) { return res.redirect('/') }
+        next();
+    });
+    app.use((req, res, next) => {
+        if (req.header("user-agent") && req.header("user-agent").includes("Trident")) res.destroy();
+        next();
+    });
+    app.get("/status", (req, res) => {
+        res.status(200).end()
+    });
+    app.get("/", (req, res) => {
+        res.sendFile(`${__dirname}/${findRendered(languageCode(req), req.header('user-agent') ? req.header('user-agent') : genericUserAgent)}`);
+    });
+    app.get("/favicon.ico", (req, res) => {
+        res.redirect('/icons/favicon.ico');
+    });
+    app.get("/*", (req, res) => {
+        res.redirect('/')
+    });
+
+    app.listen(process.env.webPort, () => {
+        let startTime = new Date();
+        console.log(`\n${Cyan(appName)} WEB ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(new Date().getTime())})`)}\n\nURL: ${Cyan(`${process.env.webURL}`)}\nPort: ${process.env.webPort}\n`)
+    })
+} else {
+    console.log(Red(`cobalt web hasn't been configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`));
+}