From a3daa65148abf7287b7f134a2b265ff04d7d8503 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 10 Mar 2023 00:41:17 +0600 Subject: [PATCH] 5.2 - page render caching - onDemand block caching - page html minify - better rate limiting - minor cobalt.js clean up - page render platform indication in settings popup all these changes are aimed to improve performance and responsiveness !! not final version of 5.2 !! --- .gitignore | 3 + package.json | 4 +- src/cobalt.js | 81 ++++++++++---------------- src/localization/manager.js | 6 +- src/modules/build.js | 37 +++++++++++- src/modules/emoji.js | 4 +- src/modules/pageRender/findRendered.js | 11 ++++ src/modules/pageRender/onDemand.js | 4 ++ src/modules/pageRender/page.js | 6 +- src/modules/sub/utils.js | 2 +- 10 files changed, 100 insertions(+), 58 deletions(-) create mode 100644 src/modules/pageRender/findRendered.js diff --git a/.gitignore b/.gitignore index 97df23a8..9cf0b290 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ package-lock.json # esbuild min + +#page build +build diff --git a/package.json b/package.json index 66881e87..d91e86f6 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "5.1.2", + "version": "5.2-dev", "author": "wukko", "exports": "./src/cobalt.js", "type": "module", @@ -33,6 +33,6 @@ "node-cache": "^5.1.2", "url-pattern": "1.0.3", "xml-js": "^1.6.11", - "youtubei.js": "^3.1.1" + "youtubei.js": "^3.3.0" } } diff --git a/src/cobalt.js b/src/cobalt.js index b5fde3a1..c3ae9402 100644 --- a/src/cobalt.js +++ b/src/cobalt.js @@ -5,10 +5,14 @@ import cors from "cors"; import * as fs from "fs"; import rateLimit from "express-rate-limit"; +import path from 'path'; +import { fileURLToPath } from 'url'; +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename).slice(0, -4); // go up another level (get rid of src/) + import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js"; import { appName, genericUserAgent, version } from "./modules/config.js"; import { getJSON } from "./modules/api.js"; -import renderPage from "./modules/pageRender/page.js"; import { apiJSON, checkJSONPost, languageCode } from "./modules/sub/utils.js"; import { Bright, Cyan, Green, Red } from "./modules/sub/consoleText.js"; import stream from "./modules/stream/stream.js"; @@ -16,6 +20,7 @@ import loc from "./localization/manager.js"; import { buildFront } from "./modules/build.js"; import { changelogHistory } from "./modules/pageRender/onDemand.js"; import { sha256 } from "./modules/sub/crypto.js"; +import findRendered from "./modules/pageRender/findRendered.js"; const commitHash = shortCommit(); const branch = getCurrentBranch(); @@ -26,42 +31,38 @@ app.disable('x-powered-by'); if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt && process.env.port) { const apiLimiter = rateLimit({ windowMs: 60000, - max: 12, - standardHeaders: true, + max: 18, + standardHeaders: false, legacyHeaders: false, + keyGenerator: (req, res) => sha256(req.ip.replace('::ffff:', ''), process.env.streamSalt), handler: (req, res, next, opt) => { res.status(429).json({ "status": "error", "text": loc(languageCode(req), 'ErrorRateLimit') }); } }); const apiLimiterStream = rateLimit({ windowMs: 60000, - max: 12, - standardHeaders: true, + max: 18, + standardHeaders: false, legacyHeaders: false, + keyGenerator: (req, res) => sha256(req.ip.replace('::ffff:', ''), process.env.streamSalt), handler: (req, res, next, opt) => { res.status(429).json({ "status": "error", "text": loc(languageCode(req), 'ErrorRateLimit') }); } }); - await buildFront(); + await buildFront(commitHash, branch); + app.use('/api/', apiLimiter); app.use('/api/stream', apiLimiterStream); app.use('/', express.static('./min')); app.use('/', express.static('./src/front')); app.use((req, res, next) => { - try { - decodeURIComponent(req.path) - } - catch (e) { - return res.redirect(process.env.selfURL); - } + try { decodeURIComponent(req.path) } catch (e) { return res.redirect(process.env.selfURL) } next(); }); app.use((req, res, next) => { - if (req.header("user-agent") && req.header("user-agent").includes("Trident")) { - res.destroy() - } + if (req.header("user-agent") && req.header("user-agent").includes("Trident")) res.destroy(); next(); }); app.use('/api/json', express.json({ @@ -77,36 +78,25 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt && } })); - app.post('/api/:type', cors({ origin: process.env.selfURL, optionsSuccessStatus: 200 }), async (req, res) => { + app.post('/api/json', cors({ origin: process.env.selfURL, optionsSuccessStatus: 200 }), async (req, res) => { try { let ip = sha256(req.header('x-forwarded-for') ? req.header('x-forwarded-for') : req.ip.replace('::ffff:', ''), process.env.streamSalt); let lang = languageCode(req); - switch (req.params.type) { - case 'json': - try { - let request = req.body; - request.dubLang = request.dubLang ? lang : false; - let chck = checkJSONPost(request); - if (request.url && chck) { - chck["ip"] = ip; - let j = await getJSON(chck["url"], lang, chck) - res.status(j.status).json(j.body); - } else if (request.url && !chck) { - let j = apiJSON(0, { t: loc(lang, 'ErrorCouldntFetch') }); - res.status(j.status).json(j.body); - } else { - let j = apiJSON(0, { t: loc(lang, 'ErrorNoLink') }) - res.status(j.status).json(j.body); - } - } catch (e) { - res.status(500).json({ 'status': 'error', 'text': loc(lang, 'ErrorCantProcess') }) - } - break; - default: - let j = apiJSON(0, { t: "unknown response type" }) - res.status(j.status).json(j.body); - break; + let j = apiJSON(0, { t: "Bad request" }); + try { + let request = req.body; + if (request.url) { + request.dubLang = request.dubLang ? lang : false; + let chck = checkJSONPost(request); + if (chck) chck["ip"] = ip; + j = chck ? await getJSON(chck["url"], lang, chck) : apiJSON(0, { t: loc(lang, 'ErrorCouldntFetch') }); + } else { + j = apiJSON(0, { t: loc(lang, 'ErrorNoLink') }); + } + } catch (e) { + j = apiJSON(0, { t: loc(lang, 'ErrorCantProcess') }); } + res.status(j.status).json(j.body); } catch (e) { res.status(500).json({ 'status': 'error', 'text': loc(languageCode(req), 'ErrorCantProcess') }) } @@ -154,18 +144,11 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt && res.status(500).json({ 'status': 'error', 'text': loc(languageCode(req), 'ErrorCantProcess') }) } }); - app.get("/api", (req, res) => { res.redirect('/api/json') }); app.get("/", (req, res) => { - res.send(renderPage({ - "hash": commitHash, - "type": "default", - "lang": languageCode(req), - "useragent": req.header('user-agent') ? req.header('user-agent') : genericUserAgent, - "branch": branch - })) + res.sendFile(`${__dirname}/${findRendered(languageCode(req), req.header('user-agent') ? req.header('user-agent') : genericUserAgent)}`); }); app.get("/favicon.ico", (req, res) => { res.redirect('/icons/favicon.ico'); diff --git a/src/localization/manager.js b/src/localization/manager.js index b2650870..273300b8 100644 --- a/src/localization/manager.js +++ b/src/localization/manager.js @@ -5,16 +5,19 @@ import loadJson from "../modules/sub/loadJSON.js"; const locPath = './src/localization/languages' let loc = {} +let languages = []; export function loadLoc() { fs.readdir(locPath, (err, files) => { if (err) return false; files.forEach(file => { - loc[file.split('.')[0]] = loadJson(`${locPath}/${file}`) + loc[file.split('.')[0]] = loadJson(`${locPath}/${file}`); + languages.push(file.split('.')[0]) }); }) } loadLoc(); + export function replaceBase(s) { return s.replace(/\n/g, '
').replace(/{appName}/g, appName).replace(/{repo}/g, repo).replace(/\*;/g, "•"); } @@ -42,3 +45,4 @@ export default function(lang, string, replacement) { return `!!${string}!!` } } +export let languageList = languages; diff --git a/src/modules/build.js b/src/modules/build.js index 0dd7279e..515e7123 100644 --- a/src/modules/build.js +++ b/src/modules/build.js @@ -1,12 +1,45 @@ import * as esbuild from "esbuild"; +import * as fs from "fs"; +import { languageList } from "../localization/manager.js"; +import page from "./pageRender/page.js"; -export async function buildFront() { +function cleanHTML(html) { + let clean = html.replace(/ /g, ''); + clean = clean.replace(/\n/g, ''); + return clean +} +export async function buildFront(commitHash, branch) { try { + // build html + if (!fs.existsSync('./build/')){ + fs.mkdirSync('./build/'); + fs.mkdirSync('./build/ios/'); + fs.mkdirSync('./build/pc/'); + fs.mkdirSync('./build/mob/'); + } + for (let i in languageList) { + i = languageList[i]; + let params = { + "hash": commitHash, + "lang": i, + "useragent": "pc", + "branch": branch + } + fs.writeFileSync(`./build/pc/${i}.html`, cleanHTML(page(params))); + + params["useragent"] = "iphone os"; + fs.writeFileSync(`./build/ios/${i}.html`, cleanHTML(page(params))); + + params["useragent"] = "android"; + fs.writeFileSync(`./build/mob/${i}.html`, cleanHTML(page(params))); + } + // build js & css await esbuild.build({ entryPoints: ['src/front/cobalt.js', 'src/front/cobalt.css'], outdir: 'min/', minify: true, - loader: { '.js': 'js', '.css': 'css' } + loader: { '.js': 'js', '.css': 'css', }, + charset: 'utf8' }) } catch (e) { return; diff --git a/src/modules/emoji.js b/src/modules/emoji.js index a2d9a0e8..59214288 100644 --- a/src/modules/emoji.js +++ b/src/modules/emoji.js @@ -33,8 +33,8 @@ let sizing = { } export default function(emoji, size, disablePadding) { if (!size) size = 22; - let padding = size !== 22 ? `margin-right:${sizing[size] ? sizing[size] : "0.4"}rem;` : ``; + let padding = size !== 22 ? `margin-right:${sizing[size] ? sizing[size] : "0.4"}rem;` : false; if (disablePadding) padding = 'margin-right:0!important;'; if (!names[emoji]) emoji = "❓"; - return `${emoji}` + return `` } diff --git a/src/modules/pageRender/findRendered.js b/src/modules/pageRender/findRendered.js new file mode 100644 index 00000000..92986480 --- /dev/null +++ b/src/modules/pageRender/findRendered.js @@ -0,0 +1,11 @@ +import { languageList } from "../../localization/manager.js"; + +export default function(lang, userAgent) { + let language = languageList.includes(lang) ? lang : "en"; + + let ua = userAgent.toLowerCase(); + let platform = (ua.match("android") || ua.match("iphone os")) ? "mob" : "pc"; + if (platform === "mob" && ua.match("iphone os")) platform = "ios"; + + return `/build/${platform}/${language}.html`; +} diff --git a/src/modules/pageRender/onDemand.js b/src/modules/pageRender/onDemand.js index 6fed7c0b..ce29d642 100644 --- a/src/modules/pageRender/onDemand.js +++ b/src/modules/pageRender/onDemand.js @@ -1,6 +1,9 @@ import changelogManager from "../changelog/changelogManager.js" +let cache = {} + export function changelogHistory() { // blockId 0 + if (cache['0']) return cache['0']; let history = changelogManager("history"); let render = ``; @@ -9,5 +12,6 @@ export function changelogHistory() { // blockId 0 let separator = (i !== 0 && i !== historyLen) ? '
' : '' render += `${separator}${history[i]["banner"] ? `
` : ''}` } + cache['0'] = render; return render; } diff --git a/src/modules/pageRender/page.js b/src/modules/pageRender/page.js index 4c405937..bd8c7864 100644 --- a/src/modules/pageRender/page.js +++ b/src/modules/pageRender/page.js @@ -33,6 +33,10 @@ export default function(obj) { let ua = obj.useragent.toLowerCase(); let isIOS = ua.match("iphone os"); let isMobile = ua.match("android") || ua.match("iphone os"); + + let platform = isMobile ? "m" : "p"; + if (isMobile && isIOS) platform = "i"; + audioFormats[0]["text"] = t('SettingsAudioFormatBest'); try { @@ -186,7 +190,7 @@ export default function(obj) { closeAria: t('AccessibilityClosePopup'), header: { aboveTitle: { - text: `v.${version}-${obj.hash} (${obj.branch})`, + text: `v.${version}-${obj.hash}${platform} (${obj.branch})`, url: `${repo}/commit/${obj.hash}` }, title: `${emoji("⚙️", 30)} ${t('TitlePopupSettings')}` diff --git a/src/modules/sub/utils.js b/src/modules/sub/utils.js index 0581863d..f1377df4 100644 --- a/src/modules/sub/utils.js +++ b/src/modules/sub/utils.js @@ -108,7 +108,7 @@ export function checkJSONPost(obj) { } try { let objKeys = Object.keys(obj); - if (!(objKeys.length <= 8 && obj.url)) return false; + if (!(objKeys.length <= 9 && obj.url)) return false; let defKeys = Object.keys(def); for (let i in objKeys) {