#177: add cookie support

merge pull request #177 from dumbmoron/cookie

.gitignore

@@ -17,3 +17,6 @@ docker-compose.yml
 
 # vscode
 .vscode
+
+# cookie file
+cookies.json

docker-compose.example.yml

@@ -20,7 +20,9 @@ services:
             - apiURL=https://co.wuk.sh/
             # replace apiName with your instance's distinctive name
             - apiName=eu-nl
-
+            # if you want to use cookies when fetching data from services, uncomment the next line
+            #- cookiePath=/cookies.json
+            # see src/modules/processing/cookie/cookies_example.json for example file.
 
     cobalt-web:
         build: .

package.json

@@ -33,6 +33,7 @@
         "got": "^12.1.0",
         "nanoid": "^4.0.2",
         "node-cache": "^5.1.2",
+        "set-cookie-parser": "2.6.0",
         "url-pattern": "1.0.3",
         "xml-js": "^1.6.11",
         "youtubei.js": "^5.4.0"

src/modules/processing/cookie/cookie.js

@@ -0,0 +1,37 @@
+import { strict as assert } from 'node:assert';
+
+export default class Cookie {
+    constructor(input) {
+        assert(typeof input === 'object');
+        this._values = {};
+        this.set(input)
+    }
+    set(values) {
+        Object.entries(values).forEach(
+            ([ key, value ]) => this._values[key] = value
+        )
+    }
+    unset(keys) {
+        for (const key of keys) delete this._values[key]
+    }
+    static fromString(str) {
+        const obj = {};
+
+        str.split('; ').forEach(cookie => {
+            const key = cookie.split('=')[0];
+            const value = cookie.split('=').splice(1).join('=');
+            obj[key] = decodeURIComponent(value)
+        })
+
+        return new Cookie(obj)
+    }
+    toString() {
+        return Object.entries(this._values).map(([ name, value ]) => `${name}=${encodeURIComponent(value)}`).join('; ')
+    }
+    toJSON() {
+        return this.toString()
+    }
+    values() {
+        return Object.freeze({ ...this._values })
+    }
+}

src/modules/processing/cookie/cookies_example.json

@@ -0,0 +1,5 @@
+{
+    "instagram": [
+        "mid=replace; ig_did=this; csrftoken=cookie"
+    ]
+}

src/modules/processing/cookie/manager.js

@@ -0,0 +1,58 @@
+import Cookie from './cookie.js';
+import { readFile, writeFile } from 'fs/promises';
+import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser';
+
+const WRITE_INTERVAL = 60000,
+      cookiePath = process.env.cookiePath,
+      COUNTER = Symbol('counter');
+
+let cookies = {}, dirty = false, intervalId;
+
+const setup = async () => {
+    try {
+        if (!cookiePath) return;
+
+        cookies = await readFile(cookiePath, 'utf8');
+        cookies = JSON.parse(cookies);
+        intervalId = setInterval(writeChanges, WRITE_INTERVAL)
+    } catch { /* no cookies for you */ }
+}
+
+setup();
+
+function writeChanges() {
+    if (!dirty) return;
+    dirty = false;
+
+    writeFile(cookiePath, JSON.stringify(cookies, null, 4)).catch(() => {
+        clearInterval(intervalId)
+    })
+}
+
+export function getCookie(service) {
+    if (!cookies[service] || !cookies[service].length) return;
+
+    let n;
+    if (cookies[service][COUNTER] === undefined) {
+        n = cookies[service][COUNTER] = 0
+    } else {
+        ++cookies[service][COUNTER]
+        n = (cookies[service][COUNTER] %= cookies[service].length)
+    }
+
+    const cookie = cookies[service][n];
+    if (typeof cookie === 'string') cookies[service][n] = Cookie.fromString(cookie);
+
+    return cookies[service][n]
+}
+
+export function updateCookie(cookie, headers) {
+    const parsed = parseSetCookie(splitCookiesString(headers.get('set-cookie'))),
+          values = {}
+
+    cookie.unset(parsed.filter(c => c.expires < new Date()).map(c => c.name));
+    parsed.filter(c => c.expires > new Date()).forEach(c => values[c.name] = c.value);
+
+    cookie.set(values);
+    if (Object.keys(values).length) dirty = true
+}

src/modules/processing/services/instagram.js

@@ -1,5 +1,6 @@
 import { createStream } from "../../stream/manage.js";
 import { genericUserAgent } from "../../config.js";
+import { getCookie, updateCookie } from '../cookie/manager.js';
 
 export default async function(obj) {
     let data;
@@ -14,6 +15,8 @@ export default async function(obj) {
             shortcode: obj.id
         }))
 
+        const cookie = getCookie('instagram');
+
         data = await fetch(url, {
             headers: {
                 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
@@ -25,9 +28,11 @@ export default async function(obj) {
                 'Sec-Fetch-Site': 'same-origin',
                 'upgrade-insecure-requests': '1',
                 'accept-encoding': 'gzip, deflate, br',
-                'accept-language': 'en-US,en;q=0.9,en;q=0.8'
+                'accept-language': 'en-US,en;q=0.9,en;q=0.8',
+                cookie
             }
         })
+        updateCookie(cookie, data.headers);
         data = (await data.json()).data;
     } catch (e) {
         data = false;
@@ -62,7 +67,11 @@ export default async function(obj) {
     }
 
     if (single) {
-        return { urls: single, filename: `instagram_${obj.id}.mp4`, audioFilename: `instagram_${obj.id}_audio` }
+        return {
+            urls: single,
+            filename: `instagram_${obj.id}.mp4`,
+            audioFilename: `instagram_${obj.id}_audio`
+        }
     } else if (multiple.length) {
         return { picker: multiple }
     } else {

src/test/tests.json

@@ -850,14 +850,6 @@
         }
     }],
     "instagram": [{
-        "name": "several videos in a post (picker)",
-        "url": "https://www.instagram.com/p/CqifaD0qiDt/",
-        "params": {},
-        "expected": {
-            "code": 200,
-            "status": "picker"
-        }
-    }, {
         "name": "reel",
         "url": "https://www.instagram.com/reel/CoEBV3eM4QR/",
         "params": {},