diff --git a/.gitignore b/.gitignore index 7d12aa29..887344cc 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ docker-compose.yml # vscode .vscode + +# cookie file +cookies.json diff --git a/docker-compose.example.yml b/docker-compose.example.yml index 20cf43c6..bd7d9157 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -20,7 +20,9 @@ services: - apiURL=https://co.wuk.sh/ # replace apiName with your instance's distinctive name - apiName=eu-nl - + # if you want to use cookies when fetching data from services, uncomment the next line + #- cookiePath=/cookies.json + # see src/modules/processing/cookie/cookies_example.json for example file. cobalt-web: build: . diff --git a/package.json b/package.json index e43894a3..9b8ad056 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "got": "^12.1.0", "nanoid": "^4.0.2", "node-cache": "^5.1.2", + "set-cookie-parser": "2.6.0", "url-pattern": "1.0.3", "xml-js": "^1.6.11", "youtubei.js": "^5.4.0" diff --git a/src/modules/processing/cookie/cookie.js b/src/modules/processing/cookie/cookie.js new file mode 100644 index 00000000..996ab7c7 --- /dev/null +++ b/src/modules/processing/cookie/cookie.js @@ -0,0 +1,37 @@ +import { strict as assert } from 'node:assert'; + +export default class Cookie { + constructor(input) { + assert(typeof input === 'object'); + this._values = {}; + this.set(input) + } + set(values) { + Object.entries(values).forEach( + ([ key, value ]) => this._values[key] = value + ) + } + unset(keys) { + for (const key of keys) delete this._values[key] + } + static fromString(str) { + const obj = {}; + + str.split('; ').forEach(cookie => { + const key = cookie.split('=')[0]; + const value = cookie.split('=').splice(1).join('='); + obj[key] = decodeURIComponent(value) + }) + + return new Cookie(obj) + } + toString() { + return Object.entries(this._values).map(([ name, value ]) => `${name}=${encodeURIComponent(value)}`).join('; ') + } + toJSON() { + return this.toString() + } + values() { + return Object.freeze({ ...this._values }) + } +} diff --git a/src/modules/processing/cookie/cookies_example.json b/src/modules/processing/cookie/cookies_example.json new file mode 100644 index 00000000..faaeb569 --- /dev/null +++ b/src/modules/processing/cookie/cookies_example.json @@ -0,0 +1,5 @@ +{ + "instagram": [ + "mid=replace; ig_did=this; csrftoken=cookie" + ] +} diff --git a/src/modules/processing/cookie/manager.js b/src/modules/processing/cookie/manager.js new file mode 100644 index 00000000..437100a0 --- /dev/null +++ b/src/modules/processing/cookie/manager.js @@ -0,0 +1,58 @@ +import Cookie from './cookie.js'; +import { readFile, writeFile } from 'fs/promises'; +import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser'; + +const WRITE_INTERVAL = 60000, + cookiePath = process.env.cookiePath, + COUNTER = Symbol('counter'); + +let cookies = {}, dirty = false, intervalId; + +const setup = async () => { + try { + if (!cookiePath) return; + + cookies = await readFile(cookiePath, 'utf8'); + cookies = JSON.parse(cookies); + intervalId = setInterval(writeChanges, WRITE_INTERVAL) + } catch { /* no cookies for you */ } +} + +setup(); + +function writeChanges() { + if (!dirty) return; + dirty = false; + + writeFile(cookiePath, JSON.stringify(cookies, null, 4)).catch(() => { + clearInterval(intervalId) + }) +} + +export function getCookie(service) { + if (!cookies[service] || !cookies[service].length) return; + + let n; + if (cookies[service][COUNTER] === undefined) { + n = cookies[service][COUNTER] = 0 + } else { + ++cookies[service][COUNTER] + n = (cookies[service][COUNTER] %= cookies[service].length) + } + + const cookie = cookies[service][n]; + if (typeof cookie === 'string') cookies[service][n] = Cookie.fromString(cookie); + + return cookies[service][n] +} + +export function updateCookie(cookie, headers) { + const parsed = parseSetCookie(splitCookiesString(headers.get('set-cookie'))), + values = {} + + cookie.unset(parsed.filter(c => c.expires < new Date()).map(c => c.name)); + parsed.filter(c => c.expires > new Date()).forEach(c => values[c.name] = c.value); + + cookie.set(values); + if (Object.keys(values).length) dirty = true +} diff --git a/src/modules/processing/services/instagram.js b/src/modules/processing/services/instagram.js index d1714393..eac0722e 100644 --- a/src/modules/processing/services/instagram.js +++ b/src/modules/processing/services/instagram.js @@ -1,5 +1,6 @@ import { createStream } from "../../stream/manage.js"; import { genericUserAgent } from "../../config.js"; +import { getCookie, updateCookie } from '../cookie/manager.js'; export default async function(obj) { let data; @@ -14,6 +15,8 @@ export default async function(obj) { shortcode: obj.id })) + const cookie = getCookie('instagram'); + data = await fetch(url, { headers: { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', @@ -25,9 +28,11 @@ export default async function(obj) { 'Sec-Fetch-Site': 'same-origin', 'upgrade-insecure-requests': '1', 'accept-encoding': 'gzip, deflate, br', - 'accept-language': 'en-US,en;q=0.9,en;q=0.8' + 'accept-language': 'en-US,en;q=0.9,en;q=0.8', + cookie } }) + updateCookie(cookie, data.headers); data = (await data.json()).data; } catch (e) { data = false; @@ -62,7 +67,11 @@ export default async function(obj) { } if (single) { - return { urls: single, filename: `instagram_${obj.id}.mp4`, audioFilename: `instagram_${obj.id}_audio` } + return { + urls: single, + filename: `instagram_${obj.id}.mp4`, + audioFilename: `instagram_${obj.id}_audio` + } } else if (multiple.length) { return { picker: multiple } } else { diff --git a/src/test/tests.json b/src/test/tests.json index 97286611..5ba36b1b 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -850,14 +850,6 @@ } }], "instagram": [{ - "name": "several videos in a post (picker)", - "url": "https://www.instagram.com/p/CqifaD0qiDt/", - "params": {}, - "expected": { - "code": 200, - "status": "picker" - } - }, { "name": "reel", "url": "https://www.instagram.com/reel/CoEBV3eM4QR/", "params": {},