From fd0357e52da4033d440057fc5eebcf895589adea Mon Sep 17 00:00:00 2001 From: wukko Date: Sun, 20 Aug 2023 16:51:39 +0600 Subject: [PATCH] move cookies to processing slight clean up and better example --- src/modules/cookie/cookie.js | 43 ---------- src/modules/cookie/cookies.json.example | 9 --- src/modules/cookie/manager.js | 78 ------------------- src/modules/processing/cookie/cookie.js | 37 +++++++++ .../processing/cookie/cookies_example.json | 5 ++ src/modules/processing/cookie/manager.js | 58 ++++++++++++++ src/modules/processing/services/instagram.js | 8 +- src/test/tests.json | 8 -- 8 files changed, 106 insertions(+), 140 deletions(-) delete mode 100644 src/modules/cookie/cookie.js delete mode 100644 src/modules/cookie/cookies.json.example delete mode 100644 src/modules/cookie/manager.js create mode 100644 src/modules/processing/cookie/cookie.js create mode 100644 src/modules/processing/cookie/cookies_example.json create mode 100644 src/modules/processing/cookie/manager.js diff --git a/src/modules/cookie/cookie.js b/src/modules/cookie/cookie.js deleted file mode 100644 index 01a07de5..00000000 --- a/src/modules/cookie/cookie.js +++ /dev/null @@ -1,43 +0,0 @@ -import { strict as assert } from 'node:assert' - -export default class Cookie { - constructor(input) { - assert(typeof input === 'object'); - this._values = {} - this.set(input); - } - - set(values) { - Object.entries(values).forEach( - ([ key, value ]) => this._values[key] = value - ) - } - - unset(keys) { - for (const key of keys) - delete this._values[key] - } - - static fromString(str) { - const obj = {} - - str - .split('; ') - .forEach(cookie => { - const key = cookie.split('=')[0] - const value = cookie.split('=').splice(1).join('=') - obj[key] = decodeURIComponent(value) - }) - - return new Cookie(obj); - } - - toString() { - return Object.entries(this._values) - .map(([ name, value ]) => `${name}=${encodeURIComponent(value)}`) - .join('; '); - } - - toJSON() { return this.toString() } - values() { return Object.freeze({ ...this._values }) } -} \ No newline at end of file diff --git a/src/modules/cookie/cookies.json.example b/src/modules/cookie/cookies.json.example deleted file mode 100644 index 60cab650..00000000 --- a/src/modules/cookie/cookies.json.example +++ /dev/null @@ -1,9 +0,0 @@ -{ - "instagram": [ - "cookie=asd; bla=bla; fake=cookie" - ], - "youtube": [ - "epic=google_cookie", - "epic=another_epic; youtube=cookie" - ] -} \ No newline at end of file diff --git a/src/modules/cookie/manager.js b/src/modules/cookie/manager.js deleted file mode 100644 index 4e29151a..00000000 --- a/src/modules/cookie/manager.js +++ /dev/null @@ -1,78 +0,0 @@ -import path from 'path' -import Cookie from './cookie.js' -import { readFile, writeFile } from 'fs/promises' -import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser' - -const WRITE_INTERVAL = 60000, - COOKIE_PATH = process.env.COOKIE_PATH, - COUNTER = Symbol('counter'); - -let cookies = {}, dirty = false, intervalId; - -const setup = async () => { - try { - if (!COOKIE_PATH) - return - - cookies = await readFile(COOKIE_PATH, 'utf8') - cookies = JSON.parse(cookies) - intervalId = setInterval(writeChanges, WRITE_INTERVAL) - } catch { /* no cookies for you */ } -} - -setup() - -function writeChanges() { - if (!dirty) return - dirty = false - - writeFile( - COOKIE_PATH, - JSON.stringify(cookies, null, 4) - ).catch(e => { - console.error('warn: cookies failed to save, stopping interval') - console.error('exception:', e) - clearInterval(intervalId) - }) -} - -export function getCookie(service) { - if (!cookies[service] || !cookies[service].length) - return - - let n - if (cookies[service][COUNTER] === undefined) { - n = cookies[service][COUNTER] = 0 - } else { - ++cookies[service][COUNTER] - n = (cookies[service][COUNTER] %= cookies[service].length) - } - - const cookie = cookies[service][n] - if (typeof cookie === 'string') - cookies[service][n] = Cookie.fromString(cookie) - - return cookies[service][n] -} - -// todo: expiry checking? domain checking? -// might be pointless for the purposes of cobalt -export function updateCookie(cookie, headers) { - const parsed = parseSetCookie( - splitCookiesString(headers.get('set-cookie')) - ), values = {} - - cookie.unset( - parsed - .filter(c => c.expires < new Date()) - .map(c => c.name) - ) - - parsed - .filter(c => c.expires > new Date()) - .forEach(c => values[c.name] = c.value); - - cookie.set(values) - if (Object.keys(values).length) - dirty = true -} diff --git a/src/modules/processing/cookie/cookie.js b/src/modules/processing/cookie/cookie.js new file mode 100644 index 00000000..996ab7c7 --- /dev/null +++ b/src/modules/processing/cookie/cookie.js @@ -0,0 +1,37 @@ +import { strict as assert } from 'node:assert'; + +export default class Cookie { + constructor(input) { + assert(typeof input === 'object'); + this._values = {}; + this.set(input) + } + set(values) { + Object.entries(values).forEach( + ([ key, value ]) => this._values[key] = value + ) + } + unset(keys) { + for (const key of keys) delete this._values[key] + } + static fromString(str) { + const obj = {}; + + str.split('; ').forEach(cookie => { + const key = cookie.split('=')[0]; + const value = cookie.split('=').splice(1).join('='); + obj[key] = decodeURIComponent(value) + }) + + return new Cookie(obj) + } + toString() { + return Object.entries(this._values).map(([ name, value ]) => `${name}=${encodeURIComponent(value)}`).join('; ') + } + toJSON() { + return this.toString() + } + values() { + return Object.freeze({ ...this._values }) + } +} diff --git a/src/modules/processing/cookie/cookies_example.json b/src/modules/processing/cookie/cookies_example.json new file mode 100644 index 00000000..faaeb569 --- /dev/null +++ b/src/modules/processing/cookie/cookies_example.json @@ -0,0 +1,5 @@ +{ + "instagram": [ + "mid=replace; ig_did=this; csrftoken=cookie" + ] +} diff --git a/src/modules/processing/cookie/manager.js b/src/modules/processing/cookie/manager.js new file mode 100644 index 00000000..2e585b12 --- /dev/null +++ b/src/modules/processing/cookie/manager.js @@ -0,0 +1,58 @@ +import Cookie from './cookie.js'; +import { readFile, writeFile } from 'fs/promises'; +import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser'; + +const WRITE_INTERVAL = 60000, + COOKIE_PATH = process.env.COOKIE_PATH, + COUNTER = Symbol('counter'); + +let cookies = {}, dirty = false, intervalId; + +const setup = async () => { + try { + if (!COOKIE_PATH) return; + + cookies = await readFile(COOKIE_PATH, 'utf8'); + cookies = JSON.parse(cookies); + intervalId = setInterval(writeChanges, WRITE_INTERVAL) + } catch { /* no cookies for you */ } +} + +setup(); + +function writeChanges() { + if (!dirty) return; + dirty = false; + + writeFile(COOKIE_PATH, JSON.stringify(cookies, null, 4)).catch(() => { + clearInterval(intervalId) + }) +} + +export function getCookie(service) { + if (!cookies[service] || !cookies[service].length) return; + + let n; + if (cookies[service][COUNTER] === undefined) { + n = cookies[service][COUNTER] = 0 + } else { + ++cookies[service][COUNTER] + n = (cookies[service][COUNTER] %= cookies[service].length) + } + + const cookie = cookies[service][n]; + if (typeof cookie === 'string') cookies[service][n] = Cookie.fromString(cookie); + + return cookies[service][n] +} + +export function updateCookie(cookie, headers) { + const parsed = parseSetCookie(splitCookiesString(headers.get('set-cookie'))), + values = {} + + cookie.unset(parsed.filter(c => c.expires < new Date()).map(c => c.name)); + parsed.filter(c => c.expires > new Date()).forEach(c => values[c.name] = c.value); + + cookie.set(values); + if (Object.keys(values).length) dirty = true +} diff --git a/src/modules/processing/services/instagram.js b/src/modules/processing/services/instagram.js index b45af479..eac0722e 100644 --- a/src/modules/processing/services/instagram.js +++ b/src/modules/processing/services/instagram.js @@ -1,6 +1,6 @@ import { createStream } from "../../stream/manage.js"; import { genericUserAgent } from "../../config.js"; -import { getCookie, updateCookie } from '../../cookie/manager.js'; +import { getCookie, updateCookie } from '../cookie/manager.js'; export default async function(obj) { let data; @@ -67,7 +67,11 @@ export default async function(obj) { } if (single) { - return { urls: single, filename: `instagram_${obj.id}.mp4`, audioFilename: `instagram_${obj.id}_audio` } + return { + urls: single, + filename: `instagram_${obj.id}.mp4`, + audioFilename: `instagram_${obj.id}_audio` + } } else if (multiple.length) { return { picker: multiple } } else { diff --git a/src/test/tests.json b/src/test/tests.json index bfac3bec..695431de 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -834,14 +834,6 @@ } }], "instagram": [{ - "name": "several videos in a post (picker)", - "url": "https://www.instagram.com/p/CqifaD0qiDt/", - "params": {}, - "expected": { - "code": 200, - "status": "picker" - } - }, { "name": "reel", "url": "https://www.instagram.com/reel/CoEBV3eM4QR/", "params": {},