From a2216510b772c4145d1142a0372b0072a9b2291e Mon Sep 17 00:00:00 2001 From: dumbmoron <136796770+dumbmoron@users.noreply.github.com> Date: Wed, 16 Aug 2023 19:49:55 +0000 Subject: [PATCH 1/3] add cookie support usage: - create cookies.json file somewhere, preferrably outside cobalt directory - in docker, you can bind mount it (`volumes` in composefile) - if you don't want cobalt to update the cookies, set it to `:ro` (cobalt will print a warning about this, ignore it) - set COOKIE_PATH to the absolute path of this file - enjoy? usage in services: probably the simplest api ever - import { getCookie, updateCookie } from '../../cookie/manager.js'; - const cookie = getCookie(''); - add this to headers - `headers: { cookie }` - after fetch is done, save potential cookie updates: updateCookie(cookie, fetch.headers) - see instagram.js for example usage --- .gitignore | 3 + package.json | 1 + src/modules/cookie/cookie.js | 43 +++++++++++ src/modules/cookie/cookies.json.example | 9 +++ src/modules/cookie/manager.js | 78 ++++++++++++++++++++ src/modules/processing/services/instagram.js | 7 +- 6 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 src/modules/cookie/cookie.js create mode 100644 src/modules/cookie/cookies.json.example create mode 100644 src/modules/cookie/manager.js diff --git a/.gitignore b/.gitignore index 7d12aa29..887344cc 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ docker-compose.yml # vscode .vscode + +# cookie file +cookies.json diff --git a/package.json b/package.json index e43894a3..9b8ad056 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "got": "^12.1.0", "nanoid": "^4.0.2", "node-cache": "^5.1.2", + "set-cookie-parser": "2.6.0", "url-pattern": "1.0.3", "xml-js": "^1.6.11", "youtubei.js": "^5.4.0" diff --git a/src/modules/cookie/cookie.js b/src/modules/cookie/cookie.js new file mode 100644 index 00000000..01a07de5 --- /dev/null +++ b/src/modules/cookie/cookie.js @@ -0,0 +1,43 @@ +import { strict as assert } from 'node:assert' + +export default class Cookie { + constructor(input) { + assert(typeof input === 'object'); + this._values = {} + this.set(input); + } + + set(values) { + Object.entries(values).forEach( + ([ key, value ]) => this._values[key] = value + ) + } + + unset(keys) { + for (const key of keys) + delete this._values[key] + } + + static fromString(str) { + const obj = {} + + str + .split('; ') + .forEach(cookie => { + const key = cookie.split('=')[0] + const value = cookie.split('=').splice(1).join('=') + obj[key] = decodeURIComponent(value) + }) + + return new Cookie(obj); + } + + toString() { + return Object.entries(this._values) + .map(([ name, value ]) => `${name}=${encodeURIComponent(value)}`) + .join('; '); + } + + toJSON() { return this.toString() } + values() { return Object.freeze({ ...this._values }) } +} \ No newline at end of file diff --git a/src/modules/cookie/cookies.json.example b/src/modules/cookie/cookies.json.example new file mode 100644 index 00000000..60cab650 --- /dev/null +++ b/src/modules/cookie/cookies.json.example @@ -0,0 +1,9 @@ +{ + "instagram": [ + "cookie=asd; bla=bla; fake=cookie" + ], + "youtube": [ + "epic=google_cookie", + "epic=another_epic; youtube=cookie" + ] +} \ No newline at end of file diff --git a/src/modules/cookie/manager.js b/src/modules/cookie/manager.js new file mode 100644 index 00000000..4e29151a --- /dev/null +++ b/src/modules/cookie/manager.js @@ -0,0 +1,78 @@ +import path from 'path' +import Cookie from './cookie.js' +import { readFile, writeFile } from 'fs/promises' +import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser' + +const WRITE_INTERVAL = 60000, + COOKIE_PATH = process.env.COOKIE_PATH, + COUNTER = Symbol('counter'); + +let cookies = {}, dirty = false, intervalId; + +const setup = async () => { + try { + if (!COOKIE_PATH) + return + + cookies = await readFile(COOKIE_PATH, 'utf8') + cookies = JSON.parse(cookies) + intervalId = setInterval(writeChanges, WRITE_INTERVAL) + } catch { /* no cookies for you */ } +} + +setup() + +function writeChanges() { + if (!dirty) return + dirty = false + + writeFile( + COOKIE_PATH, + JSON.stringify(cookies, null, 4) + ).catch(e => { + console.error('warn: cookies failed to save, stopping interval') + console.error('exception:', e) + clearInterval(intervalId) + }) +} + +export function getCookie(service) { + if (!cookies[service] || !cookies[service].length) + return + + let n + if (cookies[service][COUNTER] === undefined) { + n = cookies[service][COUNTER] = 0 + } else { + ++cookies[service][COUNTER] + n = (cookies[service][COUNTER] %= cookies[service].length) + } + + const cookie = cookies[service][n] + if (typeof cookie === 'string') + cookies[service][n] = Cookie.fromString(cookie) + + return cookies[service][n] +} + +// todo: expiry checking? domain checking? +// might be pointless for the purposes of cobalt +export function updateCookie(cookie, headers) { + const parsed = parseSetCookie( + splitCookiesString(headers.get('set-cookie')) + ), values = {} + + cookie.unset( + parsed + .filter(c => c.expires < new Date()) + .map(c => c.name) + ) + + parsed + .filter(c => c.expires > new Date()) + .forEach(c => values[c.name] = c.value); + + cookie.set(values) + if (Object.keys(values).length) + dirty = true +} diff --git a/src/modules/processing/services/instagram.js b/src/modules/processing/services/instagram.js index d1714393..b45af479 100644 --- a/src/modules/processing/services/instagram.js +++ b/src/modules/processing/services/instagram.js @@ -1,5 +1,6 @@ import { createStream } from "../../stream/manage.js"; import { genericUserAgent } from "../../config.js"; +import { getCookie, updateCookie } from '../../cookie/manager.js'; export default async function(obj) { let data; @@ -14,6 +15,8 @@ export default async function(obj) { shortcode: obj.id })) + const cookie = getCookie('instagram'); + data = await fetch(url, { headers: { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', @@ -25,9 +28,11 @@ export default async function(obj) { 'Sec-Fetch-Site': 'same-origin', 'upgrade-insecure-requests': '1', 'accept-encoding': 'gzip, deflate, br', - 'accept-language': 'en-US,en;q=0.9,en;q=0.8' + 'accept-language': 'en-US,en;q=0.9,en;q=0.8', + cookie } }) + updateCookie(cookie, data.headers); data = (await data.json()).data; } catch (e) { data = false; From fd0357e52da4033d440057fc5eebcf895589adea Mon Sep 17 00:00:00 2001 From: wukko Date: Sun, 20 Aug 2023 16:51:39 +0600 Subject: [PATCH 2/3] move cookies to processing slight clean up and better example --- src/modules/cookie/cookie.js | 43 ---------- src/modules/cookie/cookies.json.example | 9 --- src/modules/cookie/manager.js | 78 ------------------- src/modules/processing/cookie/cookie.js | 37 +++++++++ .../processing/cookie/cookies_example.json | 5 ++ src/modules/processing/cookie/manager.js | 58 ++++++++++++++ src/modules/processing/services/instagram.js | 8 +- src/test/tests.json | 8 -- 8 files changed, 106 insertions(+), 140 deletions(-) delete mode 100644 src/modules/cookie/cookie.js delete mode 100644 src/modules/cookie/cookies.json.example delete mode 100644 src/modules/cookie/manager.js create mode 100644 src/modules/processing/cookie/cookie.js create mode 100644 src/modules/processing/cookie/cookies_example.json create mode 100644 src/modules/processing/cookie/manager.js diff --git a/src/modules/cookie/cookie.js b/src/modules/cookie/cookie.js deleted file mode 100644 index 01a07de5..00000000 --- a/src/modules/cookie/cookie.js +++ /dev/null @@ -1,43 +0,0 @@ -import { strict as assert } from 'node:assert' - -export default class Cookie { - constructor(input) { - assert(typeof input === 'object'); - this._values = {} - this.set(input); - } - - set(values) { - Object.entries(values).forEach( - ([ key, value ]) => this._values[key] = value - ) - } - - unset(keys) { - for (const key of keys) - delete this._values[key] - } - - static fromString(str) { - const obj = {} - - str - .split('; ') - .forEach(cookie => { - const key = cookie.split('=')[0] - const value = cookie.split('=').splice(1).join('=') - obj[key] = decodeURIComponent(value) - }) - - return new Cookie(obj); - } - - toString() { - return Object.entries(this._values) - .map(([ name, value ]) => `${name}=${encodeURIComponent(value)}`) - .join('; '); - } - - toJSON() { return this.toString() } - values() { return Object.freeze({ ...this._values }) } -} \ No newline at end of file diff --git a/src/modules/cookie/cookies.json.example b/src/modules/cookie/cookies.json.example deleted file mode 100644 index 60cab650..00000000 --- a/src/modules/cookie/cookies.json.example +++ /dev/null @@ -1,9 +0,0 @@ -{ - "instagram": [ - "cookie=asd; bla=bla; fake=cookie" - ], - "youtube": [ - "epic=google_cookie", - "epic=another_epic; youtube=cookie" - ] -} \ No newline at end of file diff --git a/src/modules/cookie/manager.js b/src/modules/cookie/manager.js deleted file mode 100644 index 4e29151a..00000000 --- a/src/modules/cookie/manager.js +++ /dev/null @@ -1,78 +0,0 @@ -import path from 'path' -import Cookie from './cookie.js' -import { readFile, writeFile } from 'fs/promises' -import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser' - -const WRITE_INTERVAL = 60000, - COOKIE_PATH = process.env.COOKIE_PATH, - COUNTER = Symbol('counter'); - -let cookies = {}, dirty = false, intervalId; - -const setup = async () => { - try { - if (!COOKIE_PATH) - return - - cookies = await readFile(COOKIE_PATH, 'utf8') - cookies = JSON.parse(cookies) - intervalId = setInterval(writeChanges, WRITE_INTERVAL) - } catch { /* no cookies for you */ } -} - -setup() - -function writeChanges() { - if (!dirty) return - dirty = false - - writeFile( - COOKIE_PATH, - JSON.stringify(cookies, null, 4) - ).catch(e => { - console.error('warn: cookies failed to save, stopping interval') - console.error('exception:', e) - clearInterval(intervalId) - }) -} - -export function getCookie(service) { - if (!cookies[service] || !cookies[service].length) - return - - let n - if (cookies[service][COUNTER] === undefined) { - n = cookies[service][COUNTER] = 0 - } else { - ++cookies[service][COUNTER] - n = (cookies[service][COUNTER] %= cookies[service].length) - } - - const cookie = cookies[service][n] - if (typeof cookie === 'string') - cookies[service][n] = Cookie.fromString(cookie) - - return cookies[service][n] -} - -// todo: expiry checking? domain checking? -// might be pointless for the purposes of cobalt -export function updateCookie(cookie, headers) { - const parsed = parseSetCookie( - splitCookiesString(headers.get('set-cookie')) - ), values = {} - - cookie.unset( - parsed - .filter(c => c.expires < new Date()) - .map(c => c.name) - ) - - parsed - .filter(c => c.expires > new Date()) - .forEach(c => values[c.name] = c.value); - - cookie.set(values) - if (Object.keys(values).length) - dirty = true -} diff --git a/src/modules/processing/cookie/cookie.js b/src/modules/processing/cookie/cookie.js new file mode 100644 index 00000000..996ab7c7 --- /dev/null +++ b/src/modules/processing/cookie/cookie.js @@ -0,0 +1,37 @@ +import { strict as assert } from 'node:assert'; + +export default class Cookie { + constructor(input) { + assert(typeof input === 'object'); + this._values = {}; + this.set(input) + } + set(values) { + Object.entries(values).forEach( + ([ key, value ]) => this._values[key] = value + ) + } + unset(keys) { + for (const key of keys) delete this._values[key] + } + static fromString(str) { + const obj = {}; + + str.split('; ').forEach(cookie => { + const key = cookie.split('=')[0]; + const value = cookie.split('=').splice(1).join('='); + obj[key] = decodeURIComponent(value) + }) + + return new Cookie(obj) + } + toString() { + return Object.entries(this._values).map(([ name, value ]) => `${name}=${encodeURIComponent(value)}`).join('; ') + } + toJSON() { + return this.toString() + } + values() { + return Object.freeze({ ...this._values }) + } +} diff --git a/src/modules/processing/cookie/cookies_example.json b/src/modules/processing/cookie/cookies_example.json new file mode 100644 index 00000000..faaeb569 --- /dev/null +++ b/src/modules/processing/cookie/cookies_example.json @@ -0,0 +1,5 @@ +{ + "instagram": [ + "mid=replace; ig_did=this; csrftoken=cookie" + ] +} diff --git a/src/modules/processing/cookie/manager.js b/src/modules/processing/cookie/manager.js new file mode 100644 index 00000000..2e585b12 --- /dev/null +++ b/src/modules/processing/cookie/manager.js @@ -0,0 +1,58 @@ +import Cookie from './cookie.js'; +import { readFile, writeFile } from 'fs/promises'; +import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser'; + +const WRITE_INTERVAL = 60000, + COOKIE_PATH = process.env.COOKIE_PATH, + COUNTER = Symbol('counter'); + +let cookies = {}, dirty = false, intervalId; + +const setup = async () => { + try { + if (!COOKIE_PATH) return; + + cookies = await readFile(COOKIE_PATH, 'utf8'); + cookies = JSON.parse(cookies); + intervalId = setInterval(writeChanges, WRITE_INTERVAL) + } catch { /* no cookies for you */ } +} + +setup(); + +function writeChanges() { + if (!dirty) return; + dirty = false; + + writeFile(COOKIE_PATH, JSON.stringify(cookies, null, 4)).catch(() => { + clearInterval(intervalId) + }) +} + +export function getCookie(service) { + if (!cookies[service] || !cookies[service].length) return; + + let n; + if (cookies[service][COUNTER] === undefined) { + n = cookies[service][COUNTER] = 0 + } else { + ++cookies[service][COUNTER] + n = (cookies[service][COUNTER] %= cookies[service].length) + } + + const cookie = cookies[service][n]; + if (typeof cookie === 'string') cookies[service][n] = Cookie.fromString(cookie); + + return cookies[service][n] +} + +export function updateCookie(cookie, headers) { + const parsed = parseSetCookie(splitCookiesString(headers.get('set-cookie'))), + values = {} + + cookie.unset(parsed.filter(c => c.expires < new Date()).map(c => c.name)); + parsed.filter(c => c.expires > new Date()).forEach(c => values[c.name] = c.value); + + cookie.set(values); + if (Object.keys(values).length) dirty = true +} diff --git a/src/modules/processing/services/instagram.js b/src/modules/processing/services/instagram.js index b45af479..eac0722e 100644 --- a/src/modules/processing/services/instagram.js +++ b/src/modules/processing/services/instagram.js @@ -1,6 +1,6 @@ import { createStream } from "../../stream/manage.js"; import { genericUserAgent } from "../../config.js"; -import { getCookie, updateCookie } from '../../cookie/manager.js'; +import { getCookie, updateCookie } from '../cookie/manager.js'; export default async function(obj) { let data; @@ -67,7 +67,11 @@ export default async function(obj) { } if (single) { - return { urls: single, filename: `instagram_${obj.id}.mp4`, audioFilename: `instagram_${obj.id}_audio` } + return { + urls: single, + filename: `instagram_${obj.id}.mp4`, + audioFilename: `instagram_${obj.id}_audio` + } } else if (multiple.length) { return { picker: multiple } } else { diff --git a/src/test/tests.json b/src/test/tests.json index bfac3bec..695431de 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -834,14 +834,6 @@ } }], "instagram": [{ - "name": "several videos in a post (picker)", - "url": "https://www.instagram.com/p/CqifaD0qiDt/", - "params": {}, - "expected": { - "code": 200, - "status": "picker" - } - }, { "name": "reel", "url": "https://www.instagram.com/reel/CoEBV3eM4QR/", "params": {}, From c72d9b0ed15d2951c6aada4836a462fd2aca96c4 Mon Sep 17 00:00:00 2001 From: wukko Date: Sun, 20 Aug 2023 16:58:04 +0600 Subject: [PATCH 3/3] change cookie path capitalization for consistency with other env stuff --- docker-compose.example.yml | 4 +++- src/modules/processing/cookie/manager.js | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docker-compose.example.yml b/docker-compose.example.yml index 20cf43c6..bd7d9157 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -20,7 +20,9 @@ services: - apiURL=https://co.wuk.sh/ # replace apiName with your instance's distinctive name - apiName=eu-nl - + # if you want to use cookies when fetching data from services, uncomment the next line + #- cookiePath=/cookies.json + # see src/modules/processing/cookie/cookies_example.json for example file. cobalt-web: build: . diff --git a/src/modules/processing/cookie/manager.js b/src/modules/processing/cookie/manager.js index 2e585b12..437100a0 100644 --- a/src/modules/processing/cookie/manager.js +++ b/src/modules/processing/cookie/manager.js @@ -3,16 +3,16 @@ import { readFile, writeFile } from 'fs/promises'; import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser'; const WRITE_INTERVAL = 60000, - COOKIE_PATH = process.env.COOKIE_PATH, + cookiePath = process.env.cookiePath, COUNTER = Symbol('counter'); let cookies = {}, dirty = false, intervalId; const setup = async () => { try { - if (!COOKIE_PATH) return; + if (!cookiePath) return; - cookies = await readFile(COOKIE_PATH, 'utf8'); + cookies = await readFile(cookiePath, 'utf8'); cookies = JSON.parse(cookies); intervalId = setInterval(writeChanges, WRITE_INTERVAL) } catch { /* no cookies for you */ } @@ -24,7 +24,7 @@ function writeChanges() { if (!dirty) return; dirty = false; - writeFile(COOKIE_PATH, JSON.stringify(cookies, null, 4)).catch(() => { + writeFile(cookiePath, JSON.stringify(cookies, null, 4)).catch(() => { clearInterval(intervalId) }) }