From 21d5b4b8d457c0bfe3dfe17029c235e115957200 Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Sun, 16 Jun 2024 10:52:54 +0000
Subject: [PATCH 01/67] instagram: use correct id when requesting from mobile
 API

---
 src/modules/processing/services/instagram.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/modules/processing/services/instagram.js b/src/modules/processing/services/instagram.js
index 2fc533d8..6ea05178 100644
--- a/src/modules/processing/services/instagram.js
+++ b/src/modules/processing/services/instagram.js
@@ -256,11 +256,11 @@ export default function(obj) {
             if (!media_id && cookie) media_id = await getMediaId(id, { cookie });
 
             // mobile api (bearer)
-            if (media_id && token) data = await requestMobileApi(id, { token });
+            if (media_id && token) data = await requestMobileApi(media_id, { token });
 
             // mobile api (no cookie, cookie)
-            if (!data && media_id) data = await requestMobileApi(id);
-            if (!data && media_id && cookie) data = await requestMobileApi(id, { cookie });
+            if (media_id && !data) data = await requestMobileApi(media_id);
+            if (media_id && cookie && !data) data = await requestMobileApi(media_id, { cookie });
 
             // html embed (no cookie, cookie)
             if (!data) data = await requestHTML(id);

From ef97ff06af7a7d77de943712cfd7176dfe8e5f90 Mon Sep 17 00:00:00 2001
From: jj <log@riseup.net>
Date: Sat, 22 Jun 2024 12:57:30 +0200
Subject: [PATCH 02/67] stream: fix some memory leaks in internal stream
 handling (#581)

---
 src/modules/stream/internal.js | 23 ++++++++++++++---------
 src/modules/stream/manage.js   | 11 ++++++++++-
 2 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/src/modules/stream/internal.js b/src/modules/stream/internal.js
index 535bba2d..ef8ec6f7 100644
--- a/src/modules/stream/internal.js
+++ b/src/modules/stream/internal.js
@@ -1,6 +1,5 @@
 import { request } from 'undici';
 import { Readable } from 'node:stream';
-import { assert } from 'console';
 import { getHeaders, pipe } from './shared.js';
 import { handleHlsPlaylist, isHlsRequest } from './internal-hls.js';
 
@@ -36,21 +35,17 @@ async function* readChunks(streamInfo, size) {
 
         read += received;
     }
-} 
-
-function chunkedStream(streamInfo, size) {
-    assert(streamInfo.controller instanceof AbortController);
-    const stream = Readable.from(readChunks(streamInfo, size));
-    return stream;
 }
 
 async function handleYoutubeStream(streamInfo, res) {
+    const { signal } = streamInfo.controller;
+
     try {
         const req = await fetch(streamInfo.url, {
             headers: getHeaders('youtube'),
             method: 'HEAD',
             dispatcher: streamInfo.dispatcher,
-            signal: streamInfo.controller.signal
+            signal
         });
 
         streamInfo.url = req.url;
@@ -60,7 +55,16 @@ async function handleYoutubeStream(streamInfo, res) {
             return res.end();
         }
 
-        const stream = chunkedStream(streamInfo, size);
+        const generator = readChunks(streamInfo, size);
+
+        const abortGenerator = () => {
+            generator.return();
+            signal.removeEventListener('abort', abortGenerator);
+        }
+
+        signal.addEventListener('abort', abortGenerator);
+    
+        const stream = Readable.from(generator);
 
         for (const headerName of ['content-type', 'content-length']) {
             const headerValue = req.headers.get(headerName);
@@ -69,6 +73,7 @@ async function handleYoutubeStream(streamInfo, res) {
 
         pipe(stream, res, () => res.end());
     } catch {
+        signal.abort();
         res.end();
     }
 }
diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js
index 0dec1972..b02c6084 100644
--- a/src/modules/stream/manage.js
+++ b/src/modules/stream/manage.js
@@ -78,16 +78,25 @@ export function createInternalStream(url, obj = {}) {
     }
 
     const streamID = nanoid();
+    const controller = new AbortController();
     internalStreamCache[streamID] = {
         url,
         service: obj.service,
         headers: obj.headers,
-        controller: new AbortController(),
+        controller,
         dispatcher
     };
 
     let streamLink = new URL('/api/istream', `http://127.0.0.1:${env.apiPort}`);
     streamLink.searchParams.set('id', streamID);
+
+    const cleanup = () => {
+        destroyInternalStream(streamLink);
+        controller.signal.removeEventListener('abort', cleanup);
+    }
+
+    controller.signal.addEventListener('abort', cleanup);
+
     return streamLink.toString();
 }
 

From a5e00be37655866dbe1f741a48e3c48f0cef3dbd Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Sat, 22 Jun 2024 17:02:50 +0600
Subject: [PATCH 03/67] services: add support for m.vk.com links

closes #576
---
 src/modules/processing/servicesConfig.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json
index ae4cd9f0..d727b9a5 100644
--- a/src/modules/processing/servicesConfig.json
+++ b/src/modules/processing/servicesConfig.json
@@ -33,6 +33,7 @@
         "vk": {
             "alias": "vk video & clips",
             "patterns": ["video:userId_:videoId", "clip:userId_:videoId", "clips:duplicate?z=clip:userId_:videoId"],
+            "subdomains": ["m"],
             "enabled": true
         },
         "ok": {

From 7815838751799071bdc5b1c38637344a8ef28b86 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Sat, 22 Jun 2024 17:03:43 +0600
Subject: [PATCH 04/67] package: bump version to 7.14.5

---
 package-lock.json | 4 ++--
 package.json      | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index a61508b0..9c5434d9 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
     "name": "cobalt",
-    "version": "7.14.4",
+    "version": "7.14.5",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "cobalt",
-            "version": "7.14.4",
+            "version": "7.14.5",
             "license": "AGPL-3.0",
             "dependencies": {
                 "content-disposition-header": "0.6.0",
diff --git a/package.json b/package.json
index 1d44f380..b2224a50 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
 {
     "name": "cobalt",
     "description": "save what you love",
-    "version": "7.14.4",
+    "version": "7.14.5",
     "author": "imput",
     "exports": "./src/cobalt.js",
     "type": "module",

From a6733ef0cc64545b99b97995f781f8bc8691a13b Mon Sep 17 00:00:00 2001
From: jj <log@riseup.net>
Date: Sun, 23 Jun 2024 14:51:36 +0200
Subject: [PATCH 05/67] stream/internal: refactor, abort controller in more
 places (#583)

---
 src/modules/stream/internal.js | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/src/modules/stream/internal.js b/src/modules/stream/internal.js
index ef8ec6f7..5a28af93 100644
--- a/src/modules/stream/internal.js
+++ b/src/modules/stream/internal.js
@@ -39,6 +39,7 @@ async function* readChunks(streamInfo, size) {
 
 async function handleYoutubeStream(streamInfo, res) {
     const { signal } = streamInfo.controller;
+    const cleanup = () => (res.end(), streamInfo.controller.abort());
 
     try {
         const req = await fetch(streamInfo.url, {
@@ -52,7 +53,7 @@ async function handleYoutubeStream(streamInfo, res) {
         const size = BigInt(req.headers.get('content-length'));
 
         if (req.status !== 200 || !size) {
-            return res.end();
+            return cleanup();
         }
 
         const generator = readChunks(streamInfo, size);
@@ -71,17 +72,15 @@ async function handleYoutubeStream(streamInfo, res) {
             if (headerValue) res.setHeader(headerName, headerValue);
         }
 
-        pipe(stream, res, () => res.end());
+        pipe(stream, res, cleanup);
     } catch {
-        signal.abort();
-        res.end();
+        cleanup();
     }
 }
 
-export async function internalStream(streamInfo, res) {
-    if (streamInfo.service === 'youtube') {
-        return handleYoutubeStream(streamInfo, res);
-    }
+async function handleGenericStream(streamInfo, res) {
+    const { signal } = streamInfo.controller;
+    const cleanup = () => (res.end(), streamInfo.controller.abort());
 
     try {
         const req = await request(streamInfo.url, {
@@ -90,7 +89,7 @@ export async function internalStream(streamInfo, res) {
                 host: undefined
             },
             dispatcher: streamInfo.dispatcher,
-            signal: streamInfo.controller.signal,
+            signal,
             maxRedirections: 16
         });
 
@@ -105,9 +104,17 @@ export async function internalStream(streamInfo, res) {
         if (isHlsRequest(req)) {
             await handleHlsPlaylist(streamInfo, req, res);
         } else {
-            pipe(req.body, res, () => res.end());
+            pipe(req.body, res, cleanup);
         }
     } catch {
-        streamInfo.controller.abort();
+        cleanup();
     }
+}
+
+export function internalStream(streamInfo, res) {
+    if (streamInfo.service === 'youtube') {
+        return handleYoutubeStream(streamInfo, res);
+    }
+
+    return handleGenericStream(streamInfo, res);
 }
\ No newline at end of file

From 33c3c398fc6361ddc4f684d8334e7a41482de7bf Mon Sep 17 00:00:00 2001
From: jj <log@riseup.net>
Date: Sun, 23 Jun 2024 17:37:02 +0200
Subject: [PATCH 06/67] stream/internal: don't abort immediately after close
 for generic streams (#584)

* stream: move closeRequest to shared functions

* stream: use closeRequest instead of abort() directly

* stream/internal: don't abort immediately after close for generic streams
---
 src/modules/stream/internal.js | 12 +++++++-----
 src/modules/stream/manage.js   |  3 ++-
 src/modules/stream/shared.js   |  4 ++++
 src/modules/stream/types.js    |  6 +-----
 4 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/src/modules/stream/internal.js b/src/modules/stream/internal.js
index 5a28af93..8ab2ec76 100644
--- a/src/modules/stream/internal.js
+++ b/src/modules/stream/internal.js
@@ -1,6 +1,6 @@
 import { request } from 'undici';
 import { Readable } from 'node:stream';
-import { getHeaders, pipe } from './shared.js';
+import { closeRequest, getHeaders, pipe } from './shared.js';
 import { handleHlsPlaylist, isHlsRequest } from './internal-hls.js';
 
 const CHUNK_SIZE = BigInt(8e6); // 8 MB
@@ -26,7 +26,7 @@ async function* readChunks(streamInfo, size) {
         const received = BigInt(chunk.headers['content-length']);
 
         if (received < expected / 2n) {
-            streamInfo.controller.abort();
+            closeRequest(streamInfo.controller);
         }
         
         for await (const data of chunk.body) {
@@ -39,7 +39,7 @@ async function* readChunks(streamInfo, size) {
 
 async function handleYoutubeStream(streamInfo, res) {
     const { signal } = streamInfo.controller;
-    const cleanup = () => (res.end(), streamInfo.controller.abort());
+    const cleanup = () => (res.end(), closeRequest(streamInfo.controller));
 
     try {
         const req = await fetch(streamInfo.url, {
@@ -80,7 +80,7 @@ async function handleYoutubeStream(streamInfo, res) {
 
 async function handleGenericStream(streamInfo, res) {
     const { signal } = streamInfo.controller;
-    const cleanup = () => (res.end(), streamInfo.controller.abort());
+    const cleanup = () => res.end();
 
     try {
         const req = await request(streamInfo.url, {
@@ -94,12 +94,13 @@ async function handleGenericStream(streamInfo, res) {
         });
 
         res.status(req.statusCode);
+        req.body.on('error', () => {});
 
         for (const [ name, value ] of Object.entries(req.headers))
             res.setHeader(name, value)
 
         if (req.statusCode < 200 || req.statusCode > 299)
-            return res.end();
+            return cleanup();
 
         if (isHlsRequest(req)) {
             await handleHlsPlaylist(streamInfo, req, res);
@@ -107,6 +108,7 @@ async function handleGenericStream(streamInfo, res) {
             pipe(req.body, res, cleanup);
         }
     } catch {
+        closeRequest(streamInfo.controller);
         cleanup();
     }
 }
diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js
index b02c6084..031d6711 100644
--- a/src/modules/stream/manage.js
+++ b/src/modules/stream/manage.js
@@ -5,6 +5,7 @@ import { nanoid } from "nanoid";
 import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js";
 import { env } from "../config.js";
 import { strict as assert } from "assert";
+import { closeRequest } from "./shared.js";
 
 // optional dependency
 const freebind = env.freebindCIDR && await import('freebind').catch(() => {});
@@ -109,7 +110,7 @@ export function destroyInternalStream(url) {
     const id = url.searchParams.get('id');
 
     if (internalStreamCache[id]) {
-        internalStreamCache[id].controller.abort();
+        closeRequest(internalStreamCache[id].controller);
         delete internalStreamCache[id];
     }
 }
diff --git a/src/modules/stream/shared.js b/src/modules/stream/shared.js
index 1e2f2e25..fd7d1569 100644
--- a/src/modules/stream/shared.js
+++ b/src/modules/stream/shared.js
@@ -16,6 +16,10 @@ const serviceHeaders = {
     }
 }
 
+export function closeRequest(controller) {
+    try { controller.abort() } catch {}
+}
+
 export function closeResponse(res) {
     if (!res.headersSent) {
         res.sendStatus(500);
diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js
index 2372d6c1..000b7f7f 100644
--- a/src/modules/stream/types.js
+++ b/src/modules/stream/types.js
@@ -6,7 +6,7 @@ import { create as contentDisposition } from "content-disposition-header";
 import { metadataManager } from "../sub/utils.js";
 import { destroyInternalStream } from "./manage.js";
 import { env, ffmpegArgs, hlsExceptions } from "../config.js";
-import { getHeaders, closeResponse, pipe } from "./shared.js";
+import { getHeaders, closeRequest, closeResponse, pipe } from "./shared.js";
 
 function toRawHeaders(headers) {
     return Object.entries(headers)
@@ -14,10 +14,6 @@ function toRawHeaders(headers) {
                  .join('');
 }
 
-function closeRequest(controller) {
-    try { controller.abort() } catch {}
-}
-
 function killProcess(p) {
     // ask the process to terminate itself gracefully
     p?.kill('SIGTERM');

From 08c7aa1ce11b5092e545ae26851554e08543fdd3 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Sun, 23 Jun 2024 22:13:36 +0600
Subject: [PATCH 07/67] stream: add support for remuxing multiple m3u8 files

---
 src/modules/processing/matchActionDecider.js | 14 ++++++++++----
 src/modules/stream/types.js                  |  4 ++++
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js
index f7ed3da9..74f0f8c7 100644
--- a/src/modules/processing/matchActionDecider.js
+++ b/src/modules/processing/matchActionDecider.js
@@ -24,7 +24,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
     else if (r.isGif && toGif) action = "gif";
     else if (isAudioMuted) action = "muteVideo";
     else if (isAudioOnly) action = "audio";
-    else if (r.isM3U8) action = "singleM3U8";
+    else if (r.isM3U8) action = "m3u8";
     else action = "video";
 
     if (action === "picker" || action === "audio") {
@@ -48,13 +48,19 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
             params = { type: "gif" }
             break;
 
-        case "singleM3U8":
-            params = { type: "remux" }
+        case "m3u8":
+            params = {
+                type: Array.isArray(r.urls) ? "render" : "remux"
+            }
             break;
 
         case "muteVideo":
+            let muteType = "mute";
+            if (Array.isArray(r.urls) && !r.isM3U8) {
+                muteType = "bridge";
+            }
             params = {
-                type: Array.isArray(r.urls) ? "bridge" : "mute",
+                type: muteType,
                 u: Array.isArray(r.urls) ? r.urls[0] : r.urls,
                 mute: true
             }
diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js
index 000b7f7f..af4aa2e5 100644
--- a/src/modules/stream/types.js
+++ b/src/modules/stream/types.js
@@ -92,6 +92,10 @@ export function streamLiveRender(streamInfo, res) {
 
         args = args.concat(ffmpegArgs[format]);
 
+        if (hlsExceptions.includes(streamInfo.service)) {
+            args.push('-bsf:a', 'aac_adtstoasc')
+        }
+
         if (streamInfo.metadata) {
             args = args.concat(metadataManager(streamInfo.metadata))
         }

From eb05c4b938cebc86a0702527f388e7e6395f4405 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Sun, 23 Jun 2024 22:17:11 +0600
Subject: [PATCH 08/67] stream/internal-hls: transform HLS map when defined in
 playlist header

---
 src/modules/stream/internal-hls.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/modules/stream/internal-hls.js b/src/modules/stream/internal-hls.js
index 9fa37e6e..3ed7e17b 100644
--- a/src/modules/stream/internal-hls.js
+++ b/src/modules/stream/internal-hls.js
@@ -23,6 +23,10 @@ function transformObject(streamInfo, hlsObject) {
 
     hlsObject.uri = createInternalStream(fullUrl.toString(), streamInfo);
 
+    if (hlsObject.map) {
+        hlsObject.map = transformObject(streamInfo, hlsObject.map);
+    }
+
     return hlsObject;
 }
 

From 0432232ea4211c9abc405159e722b5cddd51bf72 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Sun, 23 Jun 2024 23:02:57 +0600
Subject: [PATCH 09/67] vimeo: use HLS playlists instead of dash manifest

---
 src/modules/processing/services/vimeo.js | 60 ++++++++++++++++--------
 1 file changed, 40 insertions(+), 20 deletions(-)

diff --git a/src/modules/processing/services/vimeo.js b/src/modules/processing/services/vimeo.js
index 69a36eca..96364dac 100644
--- a/src/modules/processing/services/vimeo.js
+++ b/src/modules/processing/services/vimeo.js
@@ -1,6 +1,8 @@
 import { env } from "../../config.js";
 import { cleanString } from '../../sub/utils.js';
 
+import HLS from "hls-parser";
+
 const resolutionMatch = {
     "3840": "2160",
     "2732": "1440",
@@ -33,7 +35,7 @@ export default async function(obj) {
         url.searchParams.set('h', obj.password);
     }
 
-    let api = await fetch(url)
+    const api = await fetch(url)
                     .then(r => r.json())
                     .catch(() => {});
     if (!api) return { error: 'ErrorCouldntFetch' };
@@ -43,14 +45,15 @@ export default async function(obj) {
     if (!obj.isAudioOnly && JSON.stringify(api).includes('"progressive":[{'))
         downloadType = "progressive";
 
-    let fileMetadata = {
+    const fileMetadata = {
         title: cleanString(api.video.title.trim()),
         artist: cleanString(api.video.owner.name.trim()),
     }
 
     if (downloadType !== "dash") {
         if (qualityMatch[quality]) quality = qualityMatch[quality];
-        let all = api.request.files.progressive.sort((a, b) => Number(b.width) - Number(a.width));
+
+        const all = api.request.files.progressive.sort((a, b) => Number(b.width) - Number(a.width));
         let best = all[0];
 
         let bestQuality = all[0].quality.split('p')[0];
@@ -59,7 +62,7 @@ export default async function(obj) {
         }
 
         if (Number(quality) < Number(bestQuality)) {
-            best = all.find(i => i.quality.split('p')[0] === quality);
+            best = all.find(v => v.quality.split('p')[0] === quality);
         }
 
         if (!best) return { error: 'ErrorEmptyDownload' };
@@ -74,26 +77,43 @@ export default async function(obj) {
     if (api.video.duration > env.durationLimit)
         return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
 
-    let masterJSONURL = api.request.files.dash.cdns.akfire_interconnect_quic.url;
-    let masterJSON = await fetch(masterJSONURL).then(r => r.json()).catch(() => {});
+    const urlMasterHLS = api.request.files.hls.cdns.akfire_interconnect_quic.url;
 
-    if (!masterJSON) return { error: 'ErrorCouldntFetch' };
-    if (!masterJSON.video) return { error: 'ErrorEmptyDownload' };
+    const masterHLS = await fetch(urlMasterHLS)
+                            .then(r => r.text())
+                            .catch(() => {});
 
-    let masterJSON_Video = masterJSON.video
-                            .sort((a, b) => Number(b.width) - Number(a.width))
-                            .filter(a => ["dash", "mp42"].includes(a.format));
+    if (!masterHLS) return { error: 'ErrorCouldntFetch' };
 
-    let bestVideo = masterJSON_Video[0];
-    if (Number(quality) < Number(resolutionMatch[bestVideo.width])) {
-        bestVideo = masterJSON_Video.find(i => resolutionMatch[i.width] === quality)
+    const variants = HLS.parse(masterHLS).variants.sort(
+        (a, b) => Number(b.bandwidth) - Number(a.bandwidth)
+    );
+    if (!variants) return { error: 'ErrorEmptyDownload' };
+
+    let bestQuality;
+    if (Number(quality) < Number(resolutionMatch[variants[0].resolution.width])) {
+        bestQuality = variants.find(v =>
+            (Number(quality) === Number(resolutionMatch[v.resolution.width]))
+        );
+    }
+    if (!bestQuality) bestQuality = variants[0];
+
+    const expandLink = (url) => {
+        return new URL(url, urlMasterHLS).toString();
+    };
+
+    let urls = expandLink(bestQuality.uri);
+
+    const audioPath = bestQuality?.audio[0]?.uri;
+    if (audioPath) {
+        urls = [
+            urls,
+            expandLink(audioPath)
+        ]
     }
 
-    let masterM3U8 = `${masterJSONURL.split("/sep/")[0]}/sep/video/${bestVideo.id}/master.m3u8`;
-    const fallbackResolution = bestVideo.height > bestVideo.width ? bestVideo.width : bestVideo.height;
-
     return {
-        urls: masterM3U8,
+        urls,
         isM3U8: true,
         fileMetadata: fileMetadata,
         filenameAttributes: {
@@ -101,8 +121,8 @@ export default async function(obj) {
             id: obj.id,
             title: fileMetadata.title,
             author: fileMetadata.artist,
-            resolution: `${bestVideo.width}x${bestVideo.height}`,
-            qualityLabel: `${resolutionMatch[bestVideo.width] || fallbackResolution}p`,
+            resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`,
+            qualityLabel: `${resolutionMatch[bestQuality.resolution.width]}p`,
             extension: "mp4"
         }
     }

From cc4abbb3e26491daf211f47b935e7566a095ecb1 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Sun, 23 Jun 2024 23:15:05 +0600
Subject: [PATCH 10/67] vimeo: remove progressive parsing

it's no longer returned by the api
---
 src/modules/processing/services/vimeo.js | 39 ------------------------
 1 file changed, 39 deletions(-)

diff --git a/src/modules/processing/services/vimeo.js b/src/modules/processing/services/vimeo.js
index 96364dac..e48317d6 100644
--- a/src/modules/processing/services/vimeo.js
+++ b/src/modules/processing/services/vimeo.js
@@ -16,16 +16,6 @@ const resolutionMatch = {
     "426": "240"
 }
 
-const qualityMatch = {
-    "2160": "4K",
-    "1440": "2K",
-    "480": "540",
-
-    "4K": "2160",
-    "2K": "1440",
-    "540": "480"
-}
-
 export default async function(obj) {
     let quality = obj.quality === "max" ? "9000" : obj.quality;
     if (!quality || obj.isAudioOnly) quality = "9000";
@@ -40,40 +30,11 @@ export default async function(obj) {
                     .catch(() => {});
     if (!api) return { error: 'ErrorCouldntFetch' };
 
-    let downloadType = "dash";
-
-    if (!obj.isAudioOnly && JSON.stringify(api).includes('"progressive":[{'))
-        downloadType = "progressive";
-
     const fileMetadata = {
         title: cleanString(api.video.title.trim()),
         artist: cleanString(api.video.owner.name.trim()),
     }
 
-    if (downloadType !== "dash") {
-        if (qualityMatch[quality]) quality = qualityMatch[quality];
-
-        const all = api.request.files.progressive.sort((a, b) => Number(b.width) - Number(a.width));
-        let best = all[0];
-
-        let bestQuality = all[0].quality.split('p')[0];
-        if (qualityMatch[bestQuality]) {
-            bestQuality = qualityMatch[bestQuality]
-        }
-
-        if (Number(quality) < Number(bestQuality)) {
-            best = all.find(v => v.quality.split('p')[0] === quality);
-        }
-
-        if (!best) return { error: 'ErrorEmptyDownload' };
-
-        return {
-            urls: best.url,
-            audioFilename: `vimeo_${obj.id}_audio`,
-            filename: `vimeo_${obj.id}_${best.width}x${best.height}.mp4`
-        }
-    }
-
     if (api.video.duration > env.durationLimit)
         return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
 

From 850877369c05b1f6fb999bc2cdffc4a76e68e543 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Sun, 23 Jun 2024 23:22:58 +0600
Subject: [PATCH 11/67] vimeo: clean up & fix 144p quality

---
 src/modules/processing/services/vimeo.js | 31 +++++++++++++-----------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/src/modules/processing/services/vimeo.js b/src/modules/processing/services/vimeo.js
index e48317d6..2198bae7 100644
--- a/src/modules/processing/services/vimeo.js
+++ b/src/modules/processing/services/vimeo.js
@@ -4,21 +4,22 @@ import { cleanString } from '../../sub/utils.js';
 import HLS from "hls-parser";
 
 const resolutionMatch = {
-    "3840": "2160",
-    "2732": "1440",
-    "2560": "1440",
-    "2048": "1080",
-    "1920": "1080",
-    "1366": "720",
-    "1280": "720",
-    "960": "480",
-    "640": "360",
-    "426": "240"
+    "3840": 2160,
+    "2732": 1440,
+    "2560": 1440,
+    "2048": 1080,
+    "1920": 1080,
+    "1366": 720,
+    "1280": 720,
+    "960": 480,
+    "640": 360,
+    "426": 240
 }
 
 export default async function(obj) {
-    let quality = obj.quality === "max" ? "9000" : obj.quality;
-    if (!quality || obj.isAudioOnly) quality = "9000";
+    let quality = obj.quality === "max" ? 9000 : Number(obj.quality);
+    if (quality < 240) quality = 240;
+    if (!quality || obj.isAudioOnly) quality = 9000;
 
     const url = new URL(`https://player.vimeo.com/video/${obj.id}/config`);
     if (obj.password) {
@@ -52,9 +53,9 @@ export default async function(obj) {
     if (!variants) return { error: 'ErrorEmptyDownload' };
 
     let bestQuality;
-    if (Number(quality) < Number(resolutionMatch[variants[0].resolution.width])) {
+    if (quality < resolutionMatch[variants[0].resolution.width]) {
         bestQuality = variants.find(v =>
-            (Number(quality) === Number(resolutionMatch[v.resolution.width]))
+            (quality === resolutionMatch[v.resolution.width])
         );
     }
     if (!bestQuality) bestQuality = variants[0];
@@ -71,6 +72,8 @@ export default async function(obj) {
             urls,
             expandLink(audioPath)
         ]
+    } else if (obj.isAudioOnly) {
+        return { error: 'ErrorEmptyDownload' };
     }
 
     return {

From de7df94271e23e3cf5f7e3a0bf7400e7a1b4e166 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Sun, 23 Jun 2024 23:26:43 +0600
Subject: [PATCH 12/67] vimeo: use proper local variable name in expandLink

---
 src/modules/processing/services/vimeo.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/modules/processing/services/vimeo.js b/src/modules/processing/services/vimeo.js
index 2198bae7..a80561f9 100644
--- a/src/modules/processing/services/vimeo.js
+++ b/src/modules/processing/services/vimeo.js
@@ -60,8 +60,8 @@ export default async function(obj) {
     }
     if (!bestQuality) bestQuality = variants[0];
 
-    const expandLink = (url) => {
-        return new URL(url, urlMasterHLS).toString();
+    const expandLink = (path) => {
+        return new URL(path, urlMasterHLS).toString();
     };
 
     let urls = expandLink(bestQuality.uri);

From b51bcc2a7cd247b88bd73cf20624ec4023de7eb9 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Sun, 23 Jun 2024 23:35:29 +0600
Subject: [PATCH 13/67] vimeo: added more checks to avoid exceptions

---
 src/modules/processing/services/vimeo.js | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/modules/processing/services/vimeo.js b/src/modules/processing/services/vimeo.js
index a80561f9..d79d2e92 100644
--- a/src/modules/processing/services/vimeo.js
+++ b/src/modules/processing/services/vimeo.js
@@ -29,6 +29,7 @@ export default async function(obj) {
     const api = await fetch(url)
                     .then(r => r.json())
                     .catch(() => {});
+
     if (!api) return { error: 'ErrorCouldntFetch' };
 
     const fileMetadata = {
@@ -36,10 +37,12 @@ export default async function(obj) {
         artist: cleanString(api.video.owner.name.trim()),
     }
 
-    if (api.video.duration > env.durationLimit)
+    if (api.video?.duration > env.durationLimit)
         return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
 
-    const urlMasterHLS = api.request.files.hls.cdns.akfire_interconnect_quic.url;
+    const urlMasterHLS = api.request?.files?.hls?.cdns?.akfire_interconnect_quic?.url;
+
+    if (!urlMasterHLS) return { error: 'ErrorCouldntFetch' }
 
     const masterHLS = await fetch(urlMasterHLS)
                             .then(r => r.text())
@@ -47,17 +50,19 @@ export default async function(obj) {
 
     if (!masterHLS) return { error: 'ErrorCouldntFetch' };
 
-    const variants = HLS.parse(masterHLS).variants.sort(
+    const variants = HLS.parse(masterHLS)?.variants?.sort(
         (a, b) => Number(b.bandwidth) - Number(a.bandwidth)
     );
-    if (!variants) return { error: 'ErrorEmptyDownload' };
+    if (!variants || variants.length === 0) return { error: 'ErrorEmptyDownload' };
 
     let bestQuality;
-    if (quality < resolutionMatch[variants[0].resolution.width]) {
+
+    if (quality < resolutionMatch[variants[0]?.resolution?.width]) {
         bestQuality = variants.find(v =>
             (quality === resolutionMatch[v.resolution.width])
         );
     }
+
     if (!bestQuality) bestQuality = variants[0];
 
     const expandLink = (path) => {

From ed905fd60be554bc9da20db7cb1b8c18b88078ac Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Mon, 24 Jun 2024 17:42:29 +0000
Subject: [PATCH 14/67] stream/manage: inherit controller from parent istream
 if it exists

---
 src/modules/stream/manage.js | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js
index 031d6711..06c73a6b 100644
--- a/src/modules/stream/manage.js
+++ b/src/modules/stream/manage.js
@@ -1,6 +1,7 @@
 import NodeCache from "node-cache";
 import { randomBytes } from "crypto";
 import { nanoid } from "nanoid";
+import { setMaxListeners } from "node:events";
 
 import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js";
 import { env } from "../config.js";
@@ -79,7 +80,13 @@ export function createInternalStream(url, obj = {}) {
     }
 
     const streamID = nanoid();
-    const controller = new AbortController();
+    let controller = obj.controller;
+
+    if (!controller) {
+        controller = new AbortController(); 
+        setMaxListeners(Infinity, controller.signal);
+    }
+
     internalStreamCache[streamID] = {
         url,
         service: obj.service,

From af80db36341955e49ef42514fc05b99e42457d35 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Sat, 29 Jun 2024 18:59:19 +0600
Subject: [PATCH 15/67] github: add issue template for service request

---
 .github/ISSUE_TEMPLATE/service-request.md | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE/service-request.md

diff --git a/.github/ISSUE_TEMPLATE/service-request.md b/.github/ISSUE_TEMPLATE/service-request.md
new file mode 100644
index 00000000..61824423
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/service-request.md
@@ -0,0 +1,18 @@
+---
+name: service request
+about: request service support in cobalt
+title: 'add support for [service name]'
+labels: service request
+assignees: ''
+
+---
+
+**service name & description**
+provide the service name and brief description of what it is.
+
+**link samples for the service you'd like cobalt to support**
+list of links that cobalt should recognize. 
+could be regular video link, shared video link, mobile video link, shortened link, etc.
+
+**additional context**
+any additional context or screenshots should go here. if there aren't any, just remove this part.

From 5f60b8274e867c8ff434ac5cbe209ab3b0b652eb Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Sat, 29 Jun 2024 19:04:45 +0600
Subject: [PATCH 16/67] github: add an issue template for hosting help

---
 .github/ISSUE_TEMPLATE/hosting-help.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE/hosting-help.md

diff --git a/.github/ISSUE_TEMPLATE/hosting-help.md b/.github/ISSUE_TEMPLATE/hosting-help.md
new file mode 100644
index 00000000..e340062d
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/hosting-help.md
@@ -0,0 +1,12 @@
+---
+name: instance hosting help
+about: ask any question regarding cobalt instance hosting
+title: '[short description of the problem]'
+labels: instance hosting help
+assignees: ''
+
+---
+
+**problem description**
+describe what issue you're having, clearly and concisely. 
+support your description with screenshots/links/etc when needed.

From 40b24d30b31f116589ef20c6fdcb8eec74df6893 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Sat, 29 Jun 2024 19:11:13 +0600
Subject: [PATCH 17/67] github: update bug report template

improved formatting & clarity
---
 .github/ISSUE_TEMPLATE/bug-report.md | 32 ++++++++++++++++------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md
index e5429401..4370171c 100644
--- a/.github/ISSUE_TEMPLATE/bug-report.md
+++ b/.github/ISSUE_TEMPLATE/bug-report.md
@@ -1,32 +1,36 @@
 ---
 name: bug report
-about: report an issue with downloads or something else
-title: ''
+about: report a global issue with the cobalt codebase
+title: '[short description of the bug]'
 labels: bug
 assignees: ''
 
 ---
 
-**bug description**
-a clear and concise description of what the bug is.
+### bug description
+clear and concise description of what the issue is.
 
-**reproduction steps**
-steps to reproduce the behavior:
+### reproduction steps
+steps to reproduce the described behavior.  
+here's an example of what it could look like:
 1. go to '...'
 2. click on '....'
-3. download this video: **[link here]**
+3. download [media type] from [service]
 4. see error
 
-**screenshots**
-if applicable, add screenshots or screen recordings to help explain your problem.
+### screenshots
+if applicable, add screenshots or screen recordings to support your explanation.  
+if not, remove this section.
 
-**links**
-if applicable, add links that cause the issue. more = better.
+### links
+if applicable, add links that cause the issue. more = better.  
+if not, remove this section.
 
-**platform**
+### platform information
 - OS [e.g. iOS, windows]
 - browser [e.g. chrome, safari, firefox]
 - version [e.g. 115]
 
-**additional context**
-add any other context about the problem here.
+### additional context
+add any other context about the problem here if applicable.  
+if not, remove this section.

From 59eabe2ada33f2d6e313344efe4c04e081735cc5 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Sat, 29 Jun 2024 19:11:52 +0600
Subject: [PATCH 18/67] github: update formatting in hosting help template

---
 .github/ISSUE_TEMPLATE/hosting-help.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/ISSUE_TEMPLATE/hosting-help.md b/.github/ISSUE_TEMPLATE/hosting-help.md
index e340062d..d10a677b 100644
--- a/.github/ISSUE_TEMPLATE/hosting-help.md
+++ b/.github/ISSUE_TEMPLATE/hosting-help.md
@@ -7,6 +7,6 @@ assignees: ''
 
 ---
 
-**problem description**
+### problem description
 describe what issue you're having, clearly and concisely. 
 support your description with screenshots/links/etc when needed.

From c27d1bbbeb077c4dea30cd31de116cc74f553ef9 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Sat, 29 Jun 2024 19:12:19 +0600
Subject: [PATCH 19/67] github: update formatting in service request template

---
 .github/ISSUE_TEMPLATE/service-request.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/service-request.md b/.github/ISSUE_TEMPLATE/service-request.md
index 61824423..ca948656 100644
--- a/.github/ISSUE_TEMPLATE/service-request.md
+++ b/.github/ISSUE_TEMPLATE/service-request.md
@@ -7,12 +7,12 @@ assignees: ''
 
 ---
 
-**service name & description**
+### service name & description
 provide the service name and brief description of what it is.
 
-**link samples for the service you'd like cobalt to support**
+### link samples for the service you'd like cobalt to support
 list of links that cobalt should recognize. 
 could be regular video link, shared video link, mobile video link, shortened link, etc.
 
-**additional context**
+### additional context
 any additional context or screenshots should go here. if there aren't any, just remove this part.

From 94f512f76808489bfcc01e9d9acb85f3b15f6358 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Sat, 29 Jun 2024 19:17:48 +0600
Subject: [PATCH 20/67] github: update feature request template

better clarity & formatting
---
 .github/ISSUE_TEMPLATE/feature-request.md | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md
index 18307f4f..806b2bdf 100644
--- a/.github/ISSUE_TEMPLATE/feature-request.md
+++ b/.github/ISSUE_TEMPLATE/feature-request.md
@@ -1,17 +1,15 @@
 ---
 name: feature request
 about: suggest a feature for cobalt
-title: ''
+title: '[short feature request description]'
 labels: feature request
 assignees: ''
 
 ---
 
-**describe the feature you'd like to see**
-a clear and concise description of what you want to happen.
+### describe the feature you'd like to see
+clear and concise description of the feature you want to see in cobalt.
 
-**describe alternatives you've considered**
-a clear and concise description of any alternative solutions or features you've considered.
-
-**additional context**
-add any other context or screenshots about the feature request here.
+### additional context
+if applicable, add any other context or screenshots related to the feature request here.  
+if not, remove this section.

From ec786f2babf97974dd357ba2d9c7cc558a138576 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Sat, 29 Jun 2024 19:20:48 +0600
Subject: [PATCH 21/67] github: add an issue template for bugs only on main
 instance

---
 .github/ISSUE_TEMPLATE/bug-main-instance.md | 36 +++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE/bug-main-instance.md

diff --git a/.github/ISSUE_TEMPLATE/bug-main-instance.md b/.github/ISSUE_TEMPLATE/bug-main-instance.md
new file mode 100644
index 00000000..088811b3
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug-main-instance.md
@@ -0,0 +1,36 @@
+---
+name: main instance bug report
+about: report an issue with cobalt.tools or api.cobalt.tools
+title: '[short description of the bug]'
+labels: main instance issue
+assignees: ''
+
+---
+
+### bug description
+clear and concise description of what the issue is.
+
+### reproduction steps
+steps to reproduce the described behavior.  
+here's an example of what it could look like:
+1. go to '...'
+2. click on '....'
+3. download [media type] from [service]
+4. see error
+
+### screenshots
+if applicable, add screenshots or screen recordings to support your explanation.  
+if not, remove this section.
+
+### links
+if applicable, add links that cause the issue. more = better.  
+if not, remove this section.
+
+### platform information
+- OS [e.g. iOS, windows]
+- browser [e.g. chrome, safari, firefox]
+- version [e.g. 115]
+
+### additional context
+add any other context about the problem here if applicable.  
+if not, remove this section.

From 8276e51dbcc7afb8b3da7538a0185b6bb29db40a Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Thu, 4 Jul 2024 11:28:17 +0000
Subject: [PATCH 22/67] ci: add fast-forward merge action

---
 .github/workflows/fast-forward.yml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 .github/workflows/fast-forward.yml

diff --git a/.github/workflows/fast-forward.yml b/.github/workflows/fast-forward.yml
new file mode 100644
index 00000000..adda8f35
--- /dev/null
+++ b/.github/workflows/fast-forward.yml
@@ -0,0 +1,22 @@
+name: fast-forward
+on:
+  issue_comment:
+    types: [created, edited]
+jobs:
+  fast-forward:
+    # Only run if the comment contains the /fast-forward command.
+    if: ${{ contains(github.event.comment.body, '/fast-forward')
+            && github.event.issue.pull_request }}
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+      pull-requests: write
+      issues: write
+
+    steps:
+      - name: Fast forwarding
+        uses: sequoia-pgp/fast-forward@v1
+        with:
+          merge: true
+          comment: 'on-error'
\ No newline at end of file

From 0fefc4ac27a980befd047426c6069c7ba3a81a94 Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Sat, 6 Jul 2024 08:23:20 +0000
Subject: [PATCH 23/67] services/ok: fix video data extraction

closes #589
---
 src/modules/processing/services/ok.js | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/modules/processing/services/ok.js b/src/modules/processing/services/ok.js
index 295d5b81..97bbcf82 100644
--- a/src/modules/processing/services/ok.js
+++ b/src/modules/processing/services/ok.js
@@ -20,14 +20,15 @@ export default async function(o) {
     }).then(r => r.text()).catch(() => {});
 
     if (!html) return { error: 'ErrorCouldntFetch' };
-    if (!html.includes(`<div data-module="OKVideo" data-options="{`)) {
+
+    let videoData = html.match(/<div data-module="OKVideo" .*? data-options="({.*?})"( .*?)>/)
+                        ?.[1]
+                        ?.replaceAll("&quot;", '"');
+
+    if (!videoData) {
         return { error: 'ErrorEmptyDownload' };
     }
 
-    let videoData = html.split(`<div data-module="OKVideo" data-options="`)[1]
-                        .split('" data-')[0]
-                        .replaceAll("&quot;", '"');
-
     videoData = JSON.parse(JSON.parse(videoData).flashvars.metadata);
 
     if (videoData.provider !== "UPLOADED_ODKL")

From 4daeaa31074b27a727f4014d1f27cb326f4c3f7c Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Sat, 6 Jul 2024 08:33:02 +0000
Subject: [PATCH 24/67] services/soundcloud: properly check script hostname

---
 src/modules/processing/services/soundcloud.js | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/modules/processing/services/soundcloud.js b/src/modules/processing/services/soundcloud.js
index 389eaf6c..c36ec61d 100644
--- a/src/modules/processing/services/soundcloud.js
+++ b/src/modules/processing/services/soundcloud.js
@@ -12,17 +12,19 @@ async function findClientID() {
         let scVersion = String(sc.match(/<script>window\.__sc_version="[0-9]{10}"<\/script>/)[0].match(/[0-9]{10}/));
 
         if (cachedID.version === scVersion) return cachedID.id;
-        
+
         let scripts = sc.matchAll(/<script.+src="(.+)">/g);
         let clientid;
         for (let script of scripts) {
             let url = script[1];
-    
-            if (url && !url.startsWith('https://a-v2.sndcdn.com')) return;
-    
+
+            if (!url?.startsWith('https://a-v2.sndcdn.com/')) {
+                return;
+            }
+
             let scrf = await fetch(url).then(r => r.text()).catch(() => {});
             let id = scrf.match(/\("client_id=[A-Za-z0-9]{32}"\)/);
-    
+
             if (id && typeof id[0] === 'string') {
                 clientid = id[0].match(/[A-Za-z0-9]{32}/)[0];
                 break;

From 479e14778de6822eff4ee9421ce2db5120e3e60b Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Sat, 6 Jul 2024 08:55:48 +0000
Subject: [PATCH 25/67] services/pinterest: stricter hostname check on video
 regex

---
 src/modules/processing/services/pinterest.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/modules/processing/services/pinterest.js b/src/modules/processing/services/pinterest.js
index f823d6bb..3e9e1d32 100644
--- a/src/modules/processing/services/pinterest.js
+++ b/src/modules/processing/services/pinterest.js
@@ -1,6 +1,6 @@
 import { genericUserAgent } from "../../config.js";
 
-const videoRegex = /"url":"(https:\/\/v1.pinimg.com\/videos\/.*?)"/g;
+const videoRegex = /"url":"(https:\/\/v1\.pinimg\.com\/videos\/.*?)"/g;
 const imageRegex = /src="(https:\/\/i\.pinimg\.com\/.*\.(jpg|gif))"/g;
 
 export default async function(o) {

From 315ddb17c1de9c90da4d43788b12e42301731448 Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Sat, 6 Jul 2024 08:59:02 +0000
Subject: [PATCH 26/67] web/onDemand: check blockId is string, early return if
 not

---
 src/core/web.js | 69 +++++++++++++++++++++++++------------------------
 1 file changed, 35 insertions(+), 34 deletions(-)

diff --git a/src/core/web.js b/src/core/web.js
index 9a281c47..d892c56c 100644
--- a/src/core/web.js
+++ b/src/core/web.js
@@ -26,45 +26,46 @@ export async function runWeb(express, app, gitCommit, gitBranch, __dirname) {
 
     app.get('/onDemand', (req, res) => {
         try {
-            if (req.query.blockId) {
-                let blockId = req.query.blockId.slice(0, 3);
-                let blockData;
-                switch(blockId) {
-                    // changelog history
-                    case "0":
-                        let history = changelogHistory();
-                        if (history) {
-                            blockData = createResponse("success", { t: history })
-                        } else {
-                            blockData = createResponse("error", {
-                                t: "couldn't render this block, please try again!"
-                            })
-                        }
-                        break;
-                    // celebrations emoji
-                    case "1":
-                        let celebration = celebrationsEmoji();
-                        if (celebration) {
-                            blockData = createResponse("success", { t: celebration })
-                        }
-                        break;
-                    default:
-                        blockData = createResponse("error", {
-                            t: "couldn't find a block with this id"
-                        })
-                        break;
-                }
-                if (blockData?.body) {
-                    return res.status(blockData.status).json(blockData.body);
-                } else {
-                    return res.status(204).end();
-                }
-            } else {
+            if (typeof req.query.blockId !== 'string') {
                 return res.status(400).json({
                     status: "error",
                     text: "couldn't render this block, please try again!"
                 });
             }
+
+            let blockId = req.query.blockId.slice(0, 3);
+            let blockData;
+            switch(blockId) {
+                // changelog history
+                case "0":
+                    let history = changelogHistory();
+                    if (history) {
+                        blockData = createResponse("success", { t: history })
+                    } else {
+                        blockData = createResponse("error", {
+                            t: "couldn't render this block, please try again!"
+                        })
+                    }
+                    break;
+                // celebrations emoji
+                case "1":
+                    let celebration = celebrationsEmoji();
+                    if (celebration) {
+                        blockData = createResponse("success", { t: celebration })
+                    }
+                    break;
+                default:
+                    blockData = createResponse("error", {
+                        t: "couldn't find a block with this id"
+                    })
+                    break;
+            }
+
+            if (blockData?.body) {
+                return res.status(blockData.status).json(blockData.body);
+            } else {
+                return res.status(204).end();
+            }
         } catch {
             return res.status(400).json({
                 status: "error",

From 3096bc9df03e6f493de27f679a34b3e1343b3992 Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Sat, 6 Jul 2024 11:36:25 +0000
Subject: [PATCH 27/67] api: use Map for internal stream headers instead of
 plain object

---
 src/core/api.js              | 8 ++++----
 src/modules/stream/manage.js | 7 ++++++-
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/core/api.js b/src/core/api.js
index a366ad09..cf5b51a5 100644
--- a/src/core/api.js
+++ b/src/core/api.js
@@ -196,10 +196,10 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
             return res.sendStatus(404);
         }
 
-        streamInfo.headers = {
-            ...streamInfo.headers,
-            ...req.headers
-        };
+        streamInfo.headers = new Map([
+            streamInfo.headers || {},
+            req.headers
+        ]);
 
         return stream(res, { type: 'internal', ...streamInfo });
     })
diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js
index 06c73a6b..af041a67 100644
--- a/src/modules/stream/manage.js
+++ b/src/modules/stream/manage.js
@@ -87,10 +87,15 @@ export function createInternalStream(url, obj = {}) {
         setMaxListeners(Infinity, controller.signal);
     }
 
+    let headers;
+    if (obj.headers) {
+        headers = new Map(Object.entries(obj.headers));
+    }
+
     internalStreamCache[streamID] = {
         url,
         service: obj.service,
-        headers: obj.headers,
+        headers,
         controller,
         dispatcher
     };

From 92f3d2366cf8d974220e233449631cfb5adc5c84 Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Sat, 6 Jul 2024 11:40:41 +0000
Subject: [PATCH 28/67] api: use spread operator for merging headers

---
 src/core/api.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/core/api.js b/src/core/api.js
index cf5b51a5..8eb4cb40 100644
--- a/src/core/api.js
+++ b/src/core/api.js
@@ -197,8 +197,8 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
         }
 
         streamInfo.headers = new Map([
-            streamInfo.headers || {},
-            req.headers
+            ...(streamInfo.headers || []),
+            ...Object.entries(req.headers)
         ]);
 
         return stream(res, { type: 'internal', ...streamInfo });

From eb65e816af2fa9ce40c686da709130d27739d4c7 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Sat, 6 Jul 2024 23:15:15 +0600
Subject: [PATCH 29/67] stream/internal: convert headers to iterable object

---
 src/modules/stream/internal.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/modules/stream/internal.js b/src/modules/stream/internal.js
index 8ab2ec76..7a155a9a 100644
--- a/src/modules/stream/internal.js
+++ b/src/modules/stream/internal.js
@@ -85,7 +85,7 @@ async function handleGenericStream(streamInfo, res) {
     try {
         const req = await request(streamInfo.url, {
             headers: {
-                ...streamInfo.headers,
+                ...Object.fromEntries(streamInfo.headers),
                 host: undefined
             },
             dispatcher: streamInfo.dispatcher,

From 216529bfdfcef3c45471769f2bc41b5e63490705 Mon Sep 17 00:00:00 2001
From: jj <log@riseup.net>
Date: Sun, 7 Jul 2024 14:25:17 +0200
Subject: [PATCH 30/67] vimeo: use api to get video info, direct download if
 possible (#612)

---
 src/modules/processing/services/vimeo.js | 121 ++++++++++++++++++-----
 src/modules/sub/utils.js                 |  14 +++
 src/util/tests.json                      |   2 +-
 3 files changed, 111 insertions(+), 26 deletions(-)

diff --git a/src/modules/processing/services/vimeo.js b/src/modules/processing/services/vimeo.js
index d79d2e92..30ee3368 100644
--- a/src/modules/processing/services/vimeo.js
+++ b/src/modules/processing/services/vimeo.js
@@ -1,5 +1,5 @@
 import { env } from "../../config.js";
-import { cleanString } from '../../sub/utils.js';
+import { cleanString, merge } from '../../sub/utils.js';
 
 import HLS from "hls-parser";
 
@@ -16,32 +16,70 @@ const resolutionMatch = {
     "426": 240
 }
 
-export default async function(obj) {
-    let quality = obj.quality === "max" ? 9000 : Number(obj.quality);
-    if (quality < 240) quality = 240;
-    if (!quality || obj.isAudioOnly) quality = 9000;
-
-    const url = new URL(`https://player.vimeo.com/video/${obj.id}/config`);
-    if (obj.password) {
-        url.searchParams.set('h', obj.password);
+const requestApiInfo = (videoId, password) => {
+    if (password) {
+        videoId += `:${password}`
     }
 
-    const api = await fetch(url)
+    return fetch(
+        `https://api.vimeo.com/videos/${videoId}`,
+        {
+            headers: {
+                Accept: 'application/vnd.vimeo.*+json; version=3.4.2',
+                'User-Agent': 'Vimeo/10.19.0 (com.vimeo; build:101900.57.0; iOS 17.5.1) Alamofire/5.9.0 VimeoNetworking/5.0.0',
+                Authorization: 'Basic MTMxNzViY2Y0NDE0YTQ5YzhjZTc0YmU0NjVjNDQxYzNkYWVjOWRlOTpHKzRvMmgzVUh4UkxjdU5FRW80cDNDbDhDWGR5dVJLNUJZZ055dHBHTTB4V1VzaG41bEx1a2hiN0NWYWNUcldSSW53dzRUdFRYZlJEZmFoTTArOTBUZkJHS3R4V2llYU04Qnl1bERSWWxUdXRidjNqR2J4SHFpVmtFSUcyRktuQw==',
+                'Accept-Language': 'en'
+            }
+        }
+    )
+    .then(a => a.json())
+    .catch(() => {});
+}
+
+const compareQuality = (rendition, requestedQuality) => {
+    const quality = parseInt(rendition);
+    return Math.abs(quality - requestedQuality);
+}
+
+const getDirectLink = (data, quality) => {
+    if (!data.files) return;
+
+    const match = data.files
+        .filter(f => f.rendition?.endsWith('p'))
+        .reduce((prev, next) => {
+            const delta = {
+                prev: compareQuality(prev.rendition, quality),
+                next: compareQuality(next.rendition, quality)
+            };
+
+            return delta.prev < delta.next ? prev : next;
+        });
+
+    if (!match) return;
+
+    return {
+        urls: match.link,
+        filenameAttributes: {
+            resolution: `${match.width}x${match.height}`,
+            qualityLabel: match.rendition,
+            extension: "mp4"
+        }
+    }
+}
+
+const getHLS = async (configURL, obj) => {
+    if (!configURL) return;
+
+    const api = await fetch(configURL)
                     .then(r => r.json())
                     .catch(() => {});
-
     if (!api) return { error: 'ErrorCouldntFetch' };
 
-    const fileMetadata = {
-        title: cleanString(api.video.title.trim()),
-        artist: cleanString(api.video.owner.name.trim()),
+    if (api.video?.duration > env.durationLimit) {
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
     }
 
-    if (api.video?.duration > env.durationLimit)
-        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
-
     const urlMasterHLS = api.request?.files?.hls?.cdns?.akfire_interconnect_quic?.url;
-
     if (!urlMasterHLS) return { error: 'ErrorCouldntFetch' }
 
     const masterHLS = await fetch(urlMasterHLS)
@@ -57,9 +95,9 @@ export default async function(obj) {
 
     let bestQuality;
 
-    if (quality < resolutionMatch[variants[0]?.resolution?.width]) {
+    if (obj.quality < resolutionMatch[variants[0]?.resolution?.width]) {
         bestQuality = variants.find(v =>
-            (quality === resolutionMatch[v.resolution.width])
+            (obj.quality === resolutionMatch[v.resolution.width])
         );
     }
 
@@ -84,15 +122,48 @@ export default async function(obj) {
     return {
         urls,
         isM3U8: true,
-        fileMetadata: fileMetadata,
         filenameAttributes: {
-            service: "vimeo",
-            id: obj.id,
-            title: fileMetadata.title,
-            author: fileMetadata.artist,
             resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`,
             qualityLabel: `${resolutionMatch[bestQuality.resolution.width]}p`,
             extension: "mp4"
         }
     }
 }
+
+export default async function(obj) {
+    let quality = obj.quality === "max" ? 9000 : Number(obj.quality);
+    if (quality < 240) quality = 240;
+    if (!quality || obj.isAudioOnly) quality = 9000;
+
+    const info = await requestApiInfo(obj.id, obj.password);
+    let response;
+
+    if (obj.isAudioOnly) {
+        response = await getHLS(info.config_url, { ...obj, quality });
+    }
+
+    if (!response) response = getDirectLink(info, quality);
+    if (!response) response = { error: 'ErrorEmptyDownload' };
+
+    if (response.error) {
+        return response;
+    }
+
+    const fileMetadata = {
+        title: cleanString(info.name),
+        artist: cleanString(info.user.name),
+    };
+
+    return merge(
+        {
+            fileMetadata,
+            filenameAttributes: {
+                service: "vimeo",
+                id: obj.id,
+                title: fileMetadata.title,
+                author: fileMetadata.artist,
+            }
+        },
+        response
+    );
+}
diff --git a/src/modules/sub/utils.js b/src/modules/sub/utils.js
index 678eb70e..4a75bc0f 100644
--- a/src/modules/sub/utils.js
+++ b/src/modules/sub/utils.js
@@ -44,3 +44,17 @@ export function cleanHTML(html) {
     clean = clean.replace(/\n/g, '');
     return clean
 }
+
+export function merge(a, b) {
+    for (const k of Object.keys(b)) {
+        if (Array.isArray(b[k])) {
+            a[k] = [...(a[k] ?? []), ...b[k]];
+        } else if (typeof b[k] === 'object') {
+            a[k] = merge(a[k], b[k]);
+        } else {
+            a[k] = b[k];
+        }
+    }
+
+    return a;
+}
diff --git a/src/util/tests.json b/src/util/tests.json
index 501ac2c0..982d99cb 100644
--- a/src/util/tests.json
+++ b/src/util/tests.json
@@ -674,7 +674,7 @@
         "params": {},
         "expected": {
             "code": 200,
-            "status": "stream"
+            "status": "redirect"
         }
     }],
     "reddit": [{

From 5d122be4515018a3cc3ede0a356749e627127127 Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Sun, 7 Jul 2024 13:11:04 +0000
Subject: [PATCH 31/67] repo: add CONTRIBUTING.md file

Co-authored-by: wukko <me@wukko.me>
---
 CONTRIBUTING.md | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 CONTRIBUTING.md

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 00000000..dd2140eb
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,39 @@
+# contributing to cobalt
+if you're reading this, you are probably interested in contributing to cobalt, which we are very thankful for :3
+
+this document serves as a guide to help you make contributions that we can merge into the cobalt codebase.
+
+## translations
+currently, we are **not accepting** translations of cobalt. this is because we are making significant changes to the frontend, and the currently used localization structure is being completely reworked. if this changes, this document will be updated.
+
+## adding features or support for services
+before putting in the effort to implement a feature, it's worth considering whether it would be appropriate to add it to cobalt. the cobalt api is built to assist people **only with downloading freely accessible content**. other functionality, such as:
+- downloading paid / not publicly accessible content
+- downloading content protected by DRM
+- scraping unrelated information & exposing it outside of file metadata
+
+will not be reviewed or merged.
+
+if you plan on adding a feature or support for a service, but are unsure whether it would be appropriate, it's best to open an issue and discuss it beforehand.
+
+## git
+when contributing code to cobalt, there are a few guidelines in place to ensure that the code history is readable and comprehensible.
+
+### clean commit messages
+internally, we use a format similar to [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) - the first part signifies which part of the code you are changing (the *scope*), and the second part explains the change. for inspiration on how to write appropriate commit titles, you can take a look at the [commit history](https://github.com/imputnet/cobalt/commits/).
+
+the scope is not strictly defined, you can write whatever you find most fitting for the particular change. suppose you are changing a small part of a more significant part of the codebase. in that case, you can specify both the larger and smaller scopes in the commit message for clarity (e.g., if you were changing something in internal streams, the commit could be something like `stream/internal: fix object not being handled properly`).
+
+if you think a change deserves further explanation, we encourage you to write a short explanation in the commit message ([example](https://github.com/imputnet/cobalt/commit/d2e5b6542f71f3809ba94d56c26f382b5cb62762)), which will save both you and us time having to enquire about the change, and you explaining the reason behind it.
+
+if your contribution has uninformative commit messages, you may be asked to interactively rebase your branch and amend each commit to include a meaningful message.
+
+### clean commit history
+if your branch is out of date and/or has some merge conflicts with the `current` branch, you should **rebase** it instead of merging. this prevents meaningless merge commits from being included in your branch, which would then end up in the cobalt git history.
+
+if you find a mistake or bug in your code before it's merged or reviewed, instead of making a brand new commit to fix it, it would be preferable to amend that specific commit where the mistake was first introduced. this also helps us easily revert a commit if we discover that it introduced a bug or some unwanted behavior.
+- if the commit you are fixing is the latest one, you can add your files to staging and then use `git commit --amend` to apply the change.
+- if the commit is somewhere deeper in your branch, you can use `git commit --fixup=HASH`, where *`HASH`* is the commit you are fixing.
+    - afterward, you must interactively rebase your branch with `git rebase -i current --autosquash`.
+      this will open up an editor, but you don't need to do anything else except save the file and exit.
+- once you do either of these things, you will need to do a **force push** to your branch with `git push --force-with-lease`.

From 2f4a013a2a1c401e9ea1dc590e1920e5666951f6 Mon Sep 17 00:00:00 2001
From: jj <log@riseup.net>
Date: Sun, 7 Jul 2024 15:31:55 +0200
Subject: [PATCH 32/67] docs/api: use js-specific type for `cors` in serverInfo
 response

there is no such thing as an int in js
---
 docs/api.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/api.md b/docs/api.md
index 87ef2489..b63c9216 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -73,5 +73,5 @@ response body type: `application/json`
 | `branch`    | `string` | git branch        |
 | `name`      | `string` | server name       |
 | `url`       | `string` | server url        |
-| `cors`      | `int`    | cors status       |
+| `cors`      | `number` | cors status       |
 | `startTime` | `string` | server start time |

From ab1b07fe44a72715b5e573410f7e7a5eaf5c1fdc Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Sun, 7 Jul 2024 15:14:12 +0000
Subject: [PATCH 33/67] rutube: pick closest quality to requested quality

---
 src/modules/processing/services/rutube.js | 24 ++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/src/modules/processing/services/rutube.js b/src/modules/processing/services/rutube.js
index a8d0abbe..325e9ccc 100644
--- a/src/modules/processing/services/rutube.js
+++ b/src/modules/processing/services/rutube.js
@@ -10,6 +10,8 @@ async function requestJSON(url) {
     } catch {}
 }
 
+const delta = (a, b) => Math.abs(a - b);
+
 export default async function(obj) {
     if (obj.yappyId) {
         const yappy = await requestJSON(
@@ -25,7 +27,7 @@ export default async function(obj) {
         }
     }
 
-    const quality = obj.quality === "max" ? "9000" : obj.quality;
+    const quality = Number(obj.quality) || 9000;
 
     const requestURL = new URL(`https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2`);
     if (obj.key) requestURL.searchParams.set('p', obj.key);
@@ -45,12 +47,16 @@ export default async function(obj) {
 
     if (!m3u8) return { error: 'ErrorCouldntFetch' };
 
-    m3u8 = HLS.parse(m3u8).variants.sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth));
+    m3u8 = HLS.parse(m3u8).variants;
 
-    let bestQuality = m3u8[0];
-    if (Number(quality) < bestQuality.resolution.height) {
-        bestQuality = m3u8.find((i) => (Number(quality) === i.resolution.height));
-    }
+    const matchingQuality = m3u8.reduce((prev, next) => {
+        const diff = {
+            prev: delta(quality, prev.resolution.height),
+            next: delta(quality, next.resolution.height)
+        };
+
+        return diff.prev < diff.next ? prev : next;
+    });
 
     const fileMetadata = {
         title: cleanString(play.title.trim()),
@@ -58,15 +64,15 @@ export default async function(obj) {
     }
 
     return {
-        urls: bestQuality.uri,
+        urls: matchingQuality.uri,
         isM3U8: true,
         filenameAttributes: {
             service: "rutube",
             id: obj.id,
             title: fileMetadata.title,
             author: fileMetadata.artist,
-            resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`,
-            qualityLabel: `${bestQuality.resolution.height}p`,
+            resolution: `${matchingQuality.resolution.width}x${matchingQuality.resolution.height}`,
+            qualityLabel: `${matchingQuality.resolution.height}p`,
             extension: "mp4"
         },
         fileMetadata: fileMetadata

From 87783a4c862e980d0c964943e909c1f21aed2d8a Mon Sep 17 00:00:00 2001
From: jj <log@riseup.net>
Date: Sun, 7 Jul 2024 17:23:42 +0200
Subject: [PATCH 34/67] CONTRIBUTING: replace "message" with "title" when
 talking about amends

---
 CONTRIBUTING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index dd2140eb..2668242b 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -26,7 +26,7 @@ the scope is not strictly defined, you can write whatever you find most fitting
 
 if you think a change deserves further explanation, we encourage you to write a short explanation in the commit message ([example](https://github.com/imputnet/cobalt/commit/d2e5b6542f71f3809ba94d56c26f382b5cb62762)), which will save both you and us time having to enquire about the change, and you explaining the reason behind it.
 
-if your contribution has uninformative commit messages, you may be asked to interactively rebase your branch and amend each commit to include a meaningful message.
+if your contribution has uninformative commit titles, you may be asked to interactively rebase your branch and amend each commit to include a meaningful title.
 
 ### clean commit history
 if your branch is out of date and/or has some merge conflicts with the `current` branch, you should **rebase** it instead of merging. this prevents meaningless merge commits from being included in your branch, which would then end up in the cobalt git history.

From cfce04bbd00e36ca222aed1d0b7d997ed791c726 Mon Sep 17 00:00:00 2001
From: hyperdefined <contact@hyper.lol>
Date: Mon, 8 Jul 2024 18:03:03 -0400
Subject: [PATCH 35/67] tests: fix broken twitter links (#604)

---
 src/util/tests.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/util/tests.json b/src/util/tests.json
index 982d99cb..ba79970c 100644
--- a/src/util/tests.json
+++ b/src/util/tests.json
@@ -1,7 +1,7 @@
 {
     "twitter": [{
         "name": "regular video",
-        "url": "https://twitter.com/TwitterSpaces/status/1526955853743546372?s=20",
+        "url": "https://twitter.com/X/status/1697304622749086011",
         "params": {
             "aFormat": "mp3",
             "isAudioOnly": false,
@@ -14,7 +14,7 @@
     },
     {
         "name": "video with mobile web mediaviewer",
-        "url": "https://twitter.com/XSpaces/status/1526955853743546372/mediaViewer?currentTweet=1526955853743546372&currentTweetUser=XSpaces&currentTweet=1526955853743546372&currentTweetUser=XSpaces",
+        "url": "https://twitter.com/X/status/1697304622749086011/mediaViewer?currentTweet=1697304622749086011&currentTweetUser=X&currentTweet=1697304622749086011&currentTweetUser=X",
         "params": {},
         "expected": {
             "code": 200,
@@ -34,7 +34,7 @@
         }
     }, {
         "name": "mixed media (image + gif)",
-        "url": "https://twitter.com/Twitter/status/1580661436132757506?s=20",
+        "url": "https://twitter.com/sky_mj26/status/1807756010712428565",
         "params": {
             "aFormat": "mp3",
             "isAudioOnly": false,

From bcb8ab101f170fb870bc74528f6f39b33b0763e7 Mon Sep 17 00:00:00 2001
From: ihatespawn <168680471+ihatespawn@users.noreply.github.com>
Date: Tue, 9 Jul 2024 15:17:09 +0200
Subject: [PATCH 36/67] tests: fix broken links, correct expected responses
 (#618)

---
 src/util/tests.json | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/util/tests.json b/src/util/tests.json
index ba79970c..d3e5ec07 100644
--- a/src/util/tests.json
+++ b/src/util/tests.json
@@ -650,23 +650,23 @@
         }
     }, {
         "name": "1080p dash parcel",
-        "url": "https://vimeo.com/774694040",
+        "url": "https://vimeo.com/967252742",
         "params": {
             "vQuality": "1440"
         },
         "expected": {
             "code": 200,
-            "status": "stream"
+            "status": "redirect"
         }
     }, {
         "name": "720p dash parcel",
-        "url": "https://vimeo.com/774694040",
+        "url": "https://vimeo.com/967252742",
         "params": {
             "vQuality": "360"
         },
         "expected": {
             "code": 200,
-            "status": "stream"
+            "status": "redirect"
         }
     }, {
         "name": "private video",
@@ -947,7 +947,7 @@
     }],
     "streamable": [{
         "name": "regular video",
-        "url": "https://streamable.com/03r3c2",
+        "url": "https://streamable.com/p9cln4",
         "params": {},
         "expected": {
             "code": 200,
@@ -963,7 +963,7 @@
         }
     }, {
         "name": "regular video (isAudioOnly)",
-        "url": "https://streamable.com/03r3c2",
+        "url": "https://streamable.com/p9cln4",
         "params": {
             "isAudioOnly": true
         },
@@ -973,7 +973,7 @@
         }
     }, {
         "name": "regular video (isAudioMuted)",
-        "url": "https://streamable.com/03r3c2",
+        "url": "https://streamable.com/p9cln4",
         "params": {
             "isAudioMuted": true
         },
@@ -1042,8 +1042,8 @@
         "url": "https://rutube.ru/video/b521653b4f71ece57b8ff54e57ca9b82/",
         "params": {},
         "expected": {
-            "code": 200,
-            "status": "stream"
+            "code": 400,
+            "status": "error"
         }
     }, {
         "name": "vertical video",
@@ -1055,7 +1055,7 @@
         }
     }, {
         "name": "yappy",
-        "url": "https://rutube.ru/yappy/f1771e86294748eea8ecb2ac88e55740/",
+        "url": "https://rutube.ru/yappy/a06b1bf53bce403b9a069107f23c47eb/",
         "params": {},
         "expected": {
             "code": 200,
@@ -1157,8 +1157,8 @@
             "isAudioOnly": true
         },
         "expected": {
-            "code": 200,
-            "status": "stream"
+            "code": 400,
+            "status": "error"
         }
     }]
 }

From 33fa653ee54e0cade6090a85e5081ba3374b6b97 Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Tue, 9 Jul 2024 13:55:29 +0000
Subject: [PATCH 37/67] package: bump version to 7.14.6

---
 package-lock.json | 4 ++--
 package.json      | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 9c5434d9..6e4ea4f3 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
     "name": "cobalt",
-    "version": "7.14.5",
+    "version": "7.14.6",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "cobalt",
-            "version": "7.14.5",
+            "version": "7.14.6",
             "license": "AGPL-3.0",
             "dependencies": {
                 "content-disposition-header": "0.6.0",
diff --git a/package.json b/package.json
index b2224a50..75c8228c 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
 {
     "name": "cobalt",
     "description": "save what you love",
-    "version": "7.14.5",
+    "version": "7.14.6",
     "author": "imput",
     "exports": "./src/cobalt.js",
     "type": "module",

From 404cad711f5f0da12919e1198650a54762c99048 Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Wed, 10 Jul 2024 14:13:56 +0000
Subject: [PATCH 38/67] youtube: bump youtubei.js to v10.1.0

---
 package-lock.json                          | 22 +++++++--------
 package.json                               |  2 +-
 src/modules/processing/services/youtube.js | 33 ++++++++++++++--------
 3 files changed, 33 insertions(+), 24 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 6e4ea4f3..4917de2d 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -24,7 +24,7 @@
                 "set-cookie-parser": "2.6.0",
                 "undici": "^5.19.1",
                 "url-pattern": "1.0.3",
-                "youtubei.js": "^9.3.0"
+                "youtubei.js": "^10.1.0"
             },
             "engines": {
                 "node": ">=18"
@@ -73,9 +73,9 @@
             }
         },
         "node_modules/acorn": {
-            "version": "8.11.3",
-            "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
-            "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==",
+            "version": "8.12.1",
+            "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz",
+            "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==",
             "bin": {
                 "acorn": "bin/acorn"
             },
@@ -683,9 +683,9 @@
             }
         },
         "node_modules/jintr": {
-            "version": "1.1.0",
-            "resolved": "https://registry.npmjs.org/jintr/-/jintr-1.1.0.tgz",
-            "integrity": "sha512-Tu9wk3BpN2v+kb8yT6YBtue+/nbjeLFv4vvVC4PJ7oCidHKbifWhvORrAbQfxVIQZG+67am/mDagpiGSVtvrZg==",
+            "version": "2.0.0",
+            "resolved": "https://registry.npmjs.org/jintr/-/jintr-2.0.0.tgz",
+            "integrity": "sha512-RiVlevxttZ4eHEYB2dXKXDXluzHfRuw0DJQGsYuKCc5IvZj5/GbOakeqVX+Bar/G9kTty9xDJREcxukurkmYLA==",
             "funding": [
                 "https://github.com/sponsors/LuanRT"
             ],
@@ -1123,14 +1123,14 @@
             }
         },
         "node_modules/youtubei.js": {
-            "version": "9.4.0",
-            "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-9.4.0.tgz",
-            "integrity": "sha512-8plCOZD2WabqWSEgZU3RjzigIIeR7sF028EERJENYrC9xO/6awpLMZfeoE1gNrNEbKcA+bzbMvonqlvBdxGdKg==",
+            "version": "10.1.0",
+            "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-10.1.0.tgz",
+            "integrity": "sha512-MokZMAnpWH11VYvWuW6qjPiiPmgRl5rfDgPQOpif9qXcVHoVw1hi8ePuRSD0AZSZ+uvWGe8rvas2dzp+Jv5JKQ==",
             "funding": [
                 "https://github.com/sponsors/LuanRT"
             ],
             "dependencies": {
-                "jintr": "^1.1.0",
+                "jintr": "^2.0.0",
                 "tslib": "^2.5.0",
                 "undici": "^5.19.1"
             }
diff --git a/package.json b/package.json
index 75c8228c..bcba1894 100644
--- a/package.json
+++ b/package.json
@@ -40,7 +40,7 @@
         "set-cookie-parser": "2.6.0",
         "undici": "^5.19.1",
         "url-pattern": "1.0.3",
-        "youtubei.js": "^9.3.0"
+        "youtubei.js": "^10.1.0"
     },
     "optionalDependencies": {
         "freebind": "^0.2.2"
diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js
index 1c86b4ed..b3a405e7 100644
--- a/src/modules/processing/services/youtube.js
+++ b/src/modules/processing/services/youtube.js
@@ -28,20 +28,24 @@ const transformSessionData = (cookie) => {
     if (!cookie)
         return;
 
-    const values = cookie.values();
+    const values = { ...cookie.values() };
     const REQUIRED_VALUES = [
         'access_token', 'refresh_token',
-        'client_id', 'client_secret',
-        'expires'
+        'client_id', 'client_secret'
     ];
 
     if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) {
         return;
     }
-    return {
-        ...values,
-        expires: new Date(values.expires),
-    };
+
+    if (values.expires) {
+        values.expiry_date = values.expires;
+        delete values.expires;
+    } else if (!values.expiry_date) {
+        return;
+    }
+
+    return values;
 }
 
 const cloneInnertube = async (customFetch) => {
@@ -70,14 +74,19 @@ const cloneInnertube = async (customFetch) => {
     }
 
     if (session.logged_in) {
-        await session.oauth.refreshIfRequired();
-        const oldExpiry = new Date(cookie.values().expires);
-        const newExpiry = session.oauth.credentials.expires;
+        if (session.oauth.shouldRefreshToken()) {
+            await session.oauth.refreshAccessToken();
+        }
+
+        const cookieValues = cookie.values();
+        const oldExpiry = new Date(cookieValues.expiry_date);
+        const newExpiry = new Date(session.oauth.oauth2_tokens.expiry_date);
 
         if (oldExpiry.getTime() !== newExpiry.getTime()) {
             updateCookieValues(cookie, {
-                ...session.oauth.credentials,
-                expires: session.oauth.credentials.expires.toISOString()
+                ...session.oauth.client_id,
+                ...session.oauth.oauth2_tokens,
+                expiry_date: newExpiry.toISOString()
             });
         }
     }

From 20c409cdb308bcdd7e3e5ddffa97cc026ee2ab47 Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Wed, 10 Jul 2024 15:14:28 +0000
Subject: [PATCH 39/67] generate-youtube-tokens: update response format

---
 src/util/generate-youtube-tokens.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/util/generate-youtube-tokens.js b/src/util/generate-youtube-tokens.js
index 924afb13..ab877f96 100644
--- a/src/util/generate-youtube-tokens.js
+++ b/src/util/generate-youtube-tokens.js
@@ -20,9 +20,9 @@ tube.session.once(
 );
 
 tube.session.once('auth-error', (err) => bail('An error occurred:', err));
-tube.session.once('auth', ({ status, credentials, ...rest }) => {
-    if (status !== 'SUCCESS') {
-        bail('something went wrong', rest);
+tube.session.once('auth', ({ credentials }) => {
+    if (!credentials.access_token) {
+        bail('something went wrong');
     }
 
     console.log(

From 2f4e43f78f84ca53458c7e14d9a0a96fd5b1f7c1 Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Thu, 11 Jul 2024 07:56:13 +0000
Subject: [PATCH 40/67] youtube: client_id/client_secret is optional in session
 data

---
 src/modules/processing/services/youtube.js | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js
index b3a405e7..7beca2b4 100644
--- a/src/modules/processing/services/youtube.js
+++ b/src/modules/processing/services/youtube.js
@@ -29,10 +29,7 @@ const transformSessionData = (cookie) => {
         return;
 
     const values = { ...cookie.values() };
-    const REQUIRED_VALUES = [
-        'access_token', 'refresh_token',
-        'client_id', 'client_secret'
-    ];
+    const REQUIRED_VALUES = [ 'access_token', 'refresh_token' ];
 
     if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) {
         return;

From d68ce2f490a8cfaf35499150844b81c29e516a43 Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Fri, 12 Jul 2024 00:01:18 +0000
Subject: [PATCH 41/67] stream/types: only use `nice` if parsed
 `processingPriority` is a number

for some reason, isNaN(true) -> false, which is technically correct,
but what the fuck...
---
 src/modules/stream/types.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js
index af4aa2e5..071cb6d0 100644
--- a/src/modules/stream/types.js
+++ b/src/modules/stream/types.js
@@ -25,7 +25,7 @@ function killProcess(p) {
 }
 
 function getCommand(args) {
-    if (!isNaN(env.processingPriority)) {
+    if (typeof env.processingPriority === 'number' && !isNaN(env.processingPriority)) {
         return ['nice', ['-n', env.processingPriority.toString(), ffmpeg, ...args]]
     }
     return [ffmpeg, args]

From 31e1fa5c5ce47bb603d3286e3d6eb77cbad0783a Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Sat, 20 Jul 2024 12:53:59 +0200
Subject: [PATCH 42/67] run-an-instance: remove slash from end of CORS_URL
 example

it's somewhat misleading, since this specifies the origin
(https://html.spec.whatwg.org/multipage/browsers.html#concept-origin-tuple) and not a full URL
---
 docs/run-an-instance.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md
index a440d11d..1faafb91 100644
--- a/docs/run-an-instance.md
+++ b/docs/run-an-instance.md
@@ -57,7 +57,7 @@ sudo service nscd start
 | `API_URL`             | ➖        | `https://api.cobalt.tools/` | changes url from which api server is accessible. <br> ***REQUIRED TO RUN THE API***. |
 | `API_NAME`            | `unknown` | `ams-1`                 | api server name that is shown in `/api/serverInfo`. |
 | `CORS_WILDCARD`       | `1`       | `0`                     | toggles cross-origin resource sharing. <br> `0`: disabled. `1`: enabled. |
-| `CORS_URL`            | not used  | `https://cobalt.tools/` | cross-origin resource sharing url. api will be available only from this url if `CORS_WILDCARD` is set to `0`. |
+| `CORS_URL`            | not used  | `https://cobalt.tools`  | cross-origin resource sharing url. api will be available only from this url if `CORS_WILDCARD` is set to `0`. |
 | `COOKIE_PATH`         | not used  | `/cookies.json`         | path for cookie file relative to main folder. |
 | `PROCESSING_PRIORITY` | not used  | `10`                    | changes `nice` value* for ffmpeg subprocess. available only on unix systems. |
 | `FREEBIND_CIDR`       | ➖        | `2001:db8::/32`         | IPv6 prefix used for randomly assigning addresses to cobalt requests. only supported on linux systems. see below for more info. |

From c77ee2eb449bb4ca1726cc5846eb85a34590f453 Mon Sep 17 00:00:00 2001
From: Brama Udi <brama@udi.my.id>
Date: Wed, 24 Jul 2024 22:05:21 +0700
Subject: [PATCH 43/67] services: add facebook support (#403)

* feat: add facebook support

* chore: fix fail check

* chore: minor fix

* chore: add service in README.md

* chore: cleaning post-merge code

* facebook: add shared link pattern

* chore: clean up removing unnecessarily code

* fix: facebook shared link pattern

* matchActionDecider: redirect to facebook video instead of rendering

* facebook: pass sourceUrl in object

* url: fix botched lint

* fix: facebook shared link pattern with clean up

* test: change facebook test response to redirect

---------

Co-authored-by: dumbmoron <log@riseup.net>
---
 README.md                                     |  2 +
 src/modules/processing/match.js               |  6 ++
 src/modules/processing/matchActionDecider.js  |  1 +
 src/modules/processing/services/facebook.js   | 62 +++++++++++++++++++
 src/modules/processing/servicesConfig.json    | 13 ++++
 .../processing/servicesPatternTesters.js      | 10 ++-
 src/modules/processing/url.js                 | 10 +++
 src/util/tests.json                           | 57 +++++++++++++++++
 8 files changed, 159 insertions(+), 2 deletions(-)
 create mode 100644 src/modules/processing/services/facebook.js

diff --git a/README.md b/README.md
index d2bb064c..9d6bb9a9 100644
--- a/README.md
+++ b/README.md
@@ -19,6 +19,7 @@ this list is not final and keeps expanding over time. if support for a service y
 | bilibili.com & bilibili.tv     | ✅            | ✅         | ✅         | ➖         | ➖              |
 | dailymotion                    | ✅            | ✅         | ✅         | ✅         | ✅              |
 | instagram posts & reels        | ✅            | ✅         | ✅         | ➖         | ➖              |
+| facebook videos                | ✅            | ❌         | ❌         | ➖         | ➖              |
 | loom                           | ✅            | ❌         | ✅         | ✅         | ➖              |
 | ok video                       | ✅            | ❌         | ✅         | ✅         | ✅              |
 | pinterest                      | ✅            | ✅         | ✅         | ➖         | ➖              |
@@ -45,6 +46,7 @@ this list is not final and keeps expanding over time. if support for a service y
 | service    | notes or features                                                                                                    |
 | :--------  | :-----                                                                                                               |
 | instagram  | supports reels, photos, and videos. lets you pick what to save from multi-media posts.                               |
+| facebook   | supports public accessible videos content only.                                                                      |
 | pinterest  | supports photos, gifs, videos and stories.                                                                           |
 | reddit     | supports gifs and videos.                                                                                            |
 | rutube     | supports yappy & private links.                                                                                      |
diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js
index 3e38c4db..59bc2dd9 100644
--- a/src/modules/processing/match.js
+++ b/src/modules/processing/match.js
@@ -25,6 +25,7 @@ import twitch from "./services/twitch.js";
 import rutube from "./services/rutube.js";
 import dailymotion from "./services/dailymotion.js";
 import loom from "./services/loom.js";
+import facebook from "./services/facebook.js";
 
 let freebind;
 
@@ -192,6 +193,11 @@ export default async function(host, patternMatch, lang, obj) {
                 r = await loom({
                     id: patternMatch.id
                 });
+            case "facebook":
+                r = await facebook({
+                    ...patternMatch,
+                    sourceUrl: url.href
+                });
                 break;
             default:
                 return createResponse("error", {
diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js
index 74f0f8c7..74f0ec49 100644
--- a/src/modules/processing/matchActionDecider.js
+++ b/src/modules/processing/matchActionDecider.js
@@ -130,6 +130,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
                     params = { type: "bridge" };
                     break;
 
+                case "facebook":
                 case "vine":
                 case "instagram":
                 case "tumblr":
diff --git a/src/modules/processing/services/facebook.js b/src/modules/processing/services/facebook.js
new file mode 100644
index 00000000..45d31b5f
--- /dev/null
+++ b/src/modules/processing/services/facebook.js
@@ -0,0 +1,62 @@
+import { genericUserAgent } from "../../config.js";
+
+const headers = {
+    'User-Agent': genericUserAgent,
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+    'Accept-Encoding': 'gzip, deflate, br',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+}
+
+function resolveUrl(url) {
+    return fetch(url, { headers })
+        .then(r => {
+            if (r.headers.get('location')) {
+                return decodeURIComponent(r.headers.get('location'))
+            }
+            if (r.headers.get('link')) {
+                const linkMatch = r.headers.get('link').match(/<(.*?)\/>/)
+                return decodeURIComponent(linkMatch[1])
+            }
+            return false
+        })
+        .catch(() => false)
+}
+
+export default async function({ sourceUrl, shortLink, username, id }) {
+    const isShortLink = !!shortLink?.length
+    const isSharedLink = !!sourceUrl.match(/\/share\/\w\//)?.length
+
+    let url = isShortLink
+        ? `https://fb.watch/${shortLink}`
+        : `https://web.facebook.com/${username}/videos/${id}`
+
+    if (isShortLink) url = await resolveUrl(url)
+    if (isSharedLink) url = sourceUrl
+
+    const html = await fetch(url, { headers })
+        .then(r => r.text())
+        .catch(() => false)
+
+    if (!html) return { error: 'ErrorCouldntFetch' };
+
+    const urls = []
+    const hd = html.match('"browser_native_hd_url":(".*?")')
+    const sd = html.match('"browser_native_sd_url":(".*?")')
+
+    if (hd?.[1]) urls.push(JSON.parse(hd[1]))
+    if (sd?.[1]) urls.push(JSON.parse(sd[1]))
+
+    if (!urls.length) {
+        return { error: 'ErrorEmptyDownload' };
+    }
+
+    let filename = `facebook_${id || shortLink}.mp4`
+
+    return {
+        urls: urls[0],
+        filename,
+        audioFilename: `${filename.slice(0, -4)}_audio`,
+    };
+}
\ No newline at end of file
diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json
index d727b9a5..b6c62cf2 100644
--- a/src/modules/processing/servicesConfig.json
+++ b/src/modules/processing/servicesConfig.json
@@ -118,6 +118,19 @@
             "alias": "loom videos",
             "patterns": ["share/:id"],
             "enabled": true
+        },
+        "facebook": {
+            "alias": "facebook videos",
+            "altDomains": ["fb.watch"],
+            "subdomains": ["web"],
+            "patterns": [
+                "_shortLink/:shortLink",
+                ":username/videos/:caption/:id",
+                ":username/videos/:id",
+                "reel/:id",
+                "share/:shortLink/:id"
+            ],
+            "enabled": true
         }
     }
 }
diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js
index ddeea31f..0fed8723 100644
--- a/src/modules/processing/servicesPatternTesters.js
+++ b/src/modules/processing/servicesPatternTesters.js
@@ -1,5 +1,5 @@
 export const testers = {
-    "bilibili": (patternMatch) => 
+    "bilibili": (patternMatch) =>
         patternMatch.comId?.length <= 12 || patternMatch.comShortLink?.length <= 16
         || patternMatch.tvId?.length <= 24,
 
@@ -27,7 +27,7 @@ export const testers = {
         patternMatch.id?.length === 32 || patternMatch.yappyId?.length === 32,
 
     "soundcloud": (patternMatch) =>
-        (patternMatch.author?.length <= 255 && patternMatch.song?.length <= 255) 
+        (patternMatch.author?.length <= 255 && patternMatch.song?.length <= 255)
         || patternMatch.shortLink?.length <= 32,
 
     "streamable": (patternMatch) =>
@@ -58,4 +58,10 @@ export const testers = {
 
     "youtube": (patternMatch) =>
         patternMatch.id?.length <= 11,
+
+    "facebook": (patternMatch) =>
+        patternMatch.shortLink?.length <= 11
+        || patternMatch.username?.length <= 30
+        || patternMatch.caption?.length <= 255
+        || patternMatch.id?.length <= 20,
 }
diff --git a/src/modules/processing/url.js b/src/modules/processing/url.js
index 111f1f6f..180ec6ee 100644
--- a/src/modules/processing/url.js
+++ b/src/modules/processing/url.js
@@ -64,7 +64,17 @@ function aliasURL(url) {
             if (url.hostname === 'dai.ly' && parts.length === 2) {
                 url = new URL(`https://dailymotion.com/video/${parts[1]}`)
             }
+
+        case "facebook":
+        case "fb":
+            if (url.searchParams.get('v')) {
+                url = new URL(`https://web.facebook.com/user/videos/${url.searchParams.get('v')}`)
+            }
+            if (url.hostname === 'fb.watch') {
+                url = new URL(`https://web.facebook.com/_shortLink/${parts[1]}`)
+            }
             break;
+
         case "ddinstagram":
             if (services.instagram.altDomains.includes(host.domain) && [null, 'd', 'g'].includes(host.subdomain)) {
                 url.hostname = 'instagram.com';
diff --git a/src/util/tests.json b/src/util/tests.json
index d3e5ec07..89e02daf 100644
--- a/src/util/tests.json
+++ b/src/util/tests.json
@@ -1160,5 +1160,62 @@
             "code": 400,
             "status": "error"
         }
+    }],
+    "facebook": [{
+        "name": "direct video with username and id",
+        "url": "https://web.facebook.com/100048111287134/videos/1157798148685638/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "direct video with id as query param",
+        "url": "https://web.facebook.com/watch/?v=883839773514682&ref=sharing",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "direct video with caption",
+        "url": "https://web.facebook.com/wood57/videos/𝐒𝐞𝐛𝐚𝐬𝐤𝐨𝐦-𝐟𝐮𝐥𝐥/883839773514682",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "shortlink video",
+        "url": "https://fb.watch/r1K6XHMfGT/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "reel video",
+        "url": "https://web.facebook.com/reel/730293269054758",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "shared video link",
+        "url": "https://www.facebook.com/share/v/NEf87jbPTvFE8LsL/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "shared video link v2",
+        "url": "https://web.facebook.com/share/r/JFZfPVgLkiJQmWrr/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
     }]
 }

From 4080cd45812240c0aee621ae9784aea4eaed15ea Mon Sep 17 00:00:00 2001
From: Snazzah <7025343+Snazzah@users.noreply.github.com>
Date: Wed, 24 Jul 2024 10:06:10 -0500
Subject: [PATCH 44/67] services: add snapchat support (#429)

* feat: snapchat support

* chore: remove redundancy

* chore: a bit of better matching

* chore: update readme

* refactor(snapchat): refactor story matching to use pickers

* fix: small fix to directly linked stories

* fix(snapchat): fix filenames

* chore: update readme

* ref(snapchat): rewrite service, new test, split redirects into a util

* fix(snapchat): small fixes

* chore: deepscan error fixed

* fix: remove debug logging

* fix(snapchat): fix merge, clean up code with new utils

* fix(snapchat): update with suggested changes

---------

Signed-off-by: Snazzah <7025343+Snazzah@users.noreply.github.com>
Co-authored-by: jj <log@riseup.net>
---
 README.md                                     |  2 +
 src/modules/processing/match.js               |  9 ++
 src/modules/processing/matchActionDecider.js  |  2 +
 src/modules/processing/services/snapchat.js   | 96 +++++++++++++++++++
 src/modules/processing/servicesConfig.json    |  6 ++
 .../processing/servicesPatternTesters.js      |  5 +
 src/modules/sub/utils.js                      |  7 ++
 src/util/tests.json                           | 27 +++++-
 8 files changed, 153 insertions(+), 1 deletion(-)
 create mode 100644 src/modules/processing/services/snapchat.js

diff --git a/README.md b/README.md
index 9d6bb9a9..d5fe8164 100644
--- a/README.md
+++ b/README.md
@@ -25,6 +25,7 @@ this list is not final and keeps expanding over time. if support for a service y
 | pinterest                      | ✅            | ✅         | ✅         | ➖         | ➖              |
 | reddit                         | ✅            | ✅         | ✅         | ❌         | ❌              |
 | rutube                         | ✅            | ✅         | ✅         | ✅         | ✅              |
+| snapchat stories & spotlights  | ✅            | ✅         | ✅         | ➖         | ➖              |
 | soundcloud                     | ➖            | ✅         | ➖         | ✅         | ✅              |
 | streamable                     | ✅            | ✅         | ✅         | ➖         | ➖              |
 | tiktok                         | ✅            | ✅         | ✅         | ❌         | ❌              |
@@ -49,6 +50,7 @@ this list is not final and keeps expanding over time. if support for a service y
 | facebook   | supports public accessible videos content only.                                                                      |
 | pinterest  | supports photos, gifs, videos and stories.                                                                           |
 | reddit     | supports gifs and videos.                                                                                            |
+| snapchat   | supports spotlights and stories. lets you pick what to save from stories.                                            |
 | rutube     | supports yappy & private links.                                                                                      |
 | soundcloud | supports private links.                                                                                              |
 | tiktok     | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios.      |
diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js
index 59bc2dd9..3c16a75b 100644
--- a/src/modules/processing/match.js
+++ b/src/modules/processing/match.js
@@ -24,6 +24,7 @@ import streamable from "./services/streamable.js";
 import twitch from "./services/twitch.js";
 import rutube from "./services/rutube.js";
 import dailymotion from "./services/dailymotion.js";
+import snapchat from "./services/snapchat.js";
 import loom from "./services/loom.js";
 import facebook from "./services/facebook.js";
 
@@ -189,6 +190,14 @@ export default async function(host, patternMatch, lang, obj) {
             case "dailymotion":
                 r = await dailymotion(patternMatch);
                 break;
+            case "snapchat":
+                r = await snapchat({
+                    url,
+                    username: patternMatch.username,
+                    storyId: patternMatch.storyId,
+                    spotlightId: patternMatch.spotlightId,
+                    shortLink: patternMatch.shortLink || false
+                });
             case "loom":
                 r = await loom({
                     id: patternMatch.id
diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js
index 74f0ec49..7643d491 100644
--- a/src/modules/processing/matchActionDecider.js
+++ b/src/modules/processing/matchActionDecider.js
@@ -73,6 +73,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
             switch (host) {
                 case "instagram":
                 case "twitter":
+                case "snapchat":
                     params = { picker: r.picker };
                     break;
                 case "tiktok":
@@ -136,6 +137,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
                 case "tumblr":
                 case "pinterest":
                 case "streamable":
+                case "snapchat":
                 case "loom":
                     responseType = "redirect";
                     break;
diff --git a/src/modules/processing/services/snapchat.js b/src/modules/processing/services/snapchat.js
new file mode 100644
index 00000000..44a2b84e
--- /dev/null
+++ b/src/modules/processing/services/snapchat.js
@@ -0,0 +1,96 @@
+import { genericUserAgent } from "../../config.js";
+import { getRedirectingURL } from "../../sub/utils.js";
+import { extract, normalizeURL } from "../url.js";
+
+const SPOTLIGHT_VIDEO_REGEX = /<link data-react-helmet="true" rel="preload" href="(https:\/\/cf-st\.sc-cdn\.net\/d\/[\w.?=]+&amp;uc=\d+)" as="video"\/>/;
+const NEXT_DATA_REGEX = /<script id="__NEXT_DATA__" type="application\/json">({.+})<\/script><\/body><\/html>$/;
+
+async function getSpotlight(id) {
+    const html = await fetch(`https://www.snapchat.com/spotlight/${id}`, {
+        headers: { 'User-Agent': genericUserAgent }
+    }).then((r) => r.text()).catch(() => null);
+    if (!html) {  
+        return { error: 'ErrorCouldntFetch' };  
+    }
+
+    const videoURL = html.match(SPOTLIGHT_VIDEO_REGEX)?.[1];
+    if (videoURL) {
+        return {
+            urls: videoURL,
+            filename: `snapchat_${id}.mp4`,
+            audioFilename: `snapchat_${id}_audio`
+        }
+    }
+}
+
+async function getStory(username, storyId) {
+    const html = await fetch(`https://www.snapchat.com/add/${username}${storyId ? `/${storyId}` : ''}`, {
+        headers: { 'User-Agent': genericUserAgent }
+    }).then((r) => r.text()).catch(() => null);
+    if (!html) {  
+        return { error: 'ErrorCouldntFetch' };  
+    }  
+
+    const nextDataString = html.match(NEXT_DATA_REGEX)?.[1];
+    if (nextDataString) {
+        const data = JSON.parse(nextDataString);
+        const storyIdParam = data.query.profileParams[1];
+
+        if (storyIdParam && data.props.pageProps.story) {
+            const story = data.props.pageProps.story.snapList.find((snap) => snap.snapId.value === storyIdParam);
+            if (story) {
+                if (story.snapMediaType === 0) {
+                    return {
+                        urls: story.snapUrls.mediaUrl,
+                        isPhoto: true
+                    }
+                }
+
+                return {
+                    urls: story.snapUrls.mediaUrl,
+                    filename: `snapchat_${storyId}.mp4`,
+                    audioFilename: `snapchat_${storyId}_audio`
+                }
+            }
+        }
+
+        const defaultStory = data.props.pageProps.curatedHighlights[0];
+        if (defaultStory) {
+            return {
+                picker: defaultStory.snapList.map((snap) => ({
+                    type: snap.snapMediaType === 0 ? 'photo' : 'video',
+                    url: snap.snapUrls.mediaUrl,
+                    thumb: snap.snapUrls.mediaPreviewUrl.value
+                }))
+            }
+        }
+    }
+}
+
+export default async function(obj) {
+    let params = obj;
+    if (obj.url.hostname === 't.snapchat.com' && obj.shortLink) {
+        const link = await getRedirectingURL(`https://t.snapchat.com/${obj.shortLink}`);
+    
+        if (!link?.startsWith('https://www.snapchat.com/')) {  
+            return { error: 'ErrorCouldntFetch' };  
+        }  
+
+        const extractResult = extract(normalizeURL(link));
+        if (extractResult?.host !== 'snapchat') {  
+            return { error: 'ErrorCouldntFetch' };  
+        }
+
+        params = extractResult.patternMatch;
+    }
+
+    if (params.spotlightId) {
+        const result = await getSpotlight(params.spotlightId);
+        if (result) return result;
+    } else if (params.username) {
+        const result = await getStory(params.username, params.storyId);
+        if (result) return result;
+    }
+
+    return { error: 'ErrorCouldntFetch' };
+}
diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json
index b6c62cf2..9161117a 100644
--- a/src/modules/processing/servicesConfig.json
+++ b/src/modules/processing/servicesConfig.json
@@ -114,6 +114,12 @@
             "patterns": ["video/:id"],
             "enabled": true
         },
+        "snapchat": {
+            "alias": "snapchat stories & spotlights",
+            "subdomains": ["t", "story"],
+            "patterns": [":shortLink", "spotlight/:spotlightId", "add/:username/:storyId", "u/:username/:storyId", "add/:username", "u/:username"],
+            "enabled": true
+        },
         "loom": {
             "alias": "loom videos",
             "patterns": ["share/:id"],
diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js
index 0fed8723..9f4e0f91 100644
--- a/src/modules/processing/servicesPatternTesters.js
+++ b/src/modules/processing/servicesPatternTesters.js
@@ -30,6 +30,11 @@ export const testers = {
         (patternMatch.author?.length <= 255 && patternMatch.song?.length <= 255)
         || patternMatch.shortLink?.length <= 32,
 
+    "snapchat": (patternMatch) =>
+        (patternMatch.username?.length <= 32 && (!patternMatch.storyId || patternMatch.storyId?.length <= 255)) 
+        || patternMatch.spotlightId?.length <= 255 
+        || patternMatch.shortLink?.length <= 16,
+
     "streamable": (patternMatch) =>
         patternMatch.id?.length === 6,
 
diff --git a/src/modules/sub/utils.js b/src/modules/sub/utils.js
index 4a75bc0f..05192d97 100644
--- a/src/modules/sub/utils.js
+++ b/src/modules/sub/utils.js
@@ -45,6 +45,13 @@ export function cleanHTML(html) {
     return clean
 }
 
+export function getRedirectingURL(url) {
+    return fetch(url, { redirect: 'manual' }).then((r) => {
+        if ([301, 302, 303].includes(r.status) && r.headers.has('location'))
+            return r.headers.get('location');
+    }).catch(() => null);
+}
+
 export function merge(a, b) {
     for (const k of Object.keys(b)) {
         if (Array.isArray(b[k])) {
diff --git a/src/util/tests.json b/src/util/tests.json
index 89e02daf..b34783bd 100644
--- a/src/util/tests.json
+++ b/src/util/tests.json
@@ -1132,6 +1132,31 @@
             "status": "stream"
         }
     }],
+    "snapchat": [{
+        "name": "spotlight",
+        "url": "https://www.snapchat.com/spotlight/W7_EDlXWTBiXAEEniNoMPwAAYdWxucG9pZmNqAY46j0a5AY46j0YbAAAAAQ",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "shortlinked spotlight",
+        "url": "https://t.snapchat.com/4ZsiBLDi",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "story",
+        "url": "https://www.snapchat.com/add/bazerkmakane",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    }],
     "loom": [{
         "name": "1080p video",
         "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761",
@@ -1218,4 +1243,4 @@
             "status": "redirect"
         }
     }]
-}
+}
\ No newline at end of file

From b516033f09ad6e38f1c076a3364979b69e98dfa5 Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Wed, 24 Jul 2024 15:17:58 +0000
Subject: [PATCH 45/67] match: clean up snapchat call & add missing `break`s

---
 src/modules/processing/match.js | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js
index 3c16a75b..b7d46333 100644
--- a/src/modules/processing/match.js
+++ b/src/modules/processing/match.js
@@ -193,15 +193,14 @@ export default async function(host, patternMatch, lang, obj) {
             case "snapchat":
                 r = await snapchat({
                     url,
-                    username: patternMatch.username,
-                    storyId: patternMatch.storyId,
-                    spotlightId: patternMatch.spotlightId,
-                    shortLink: patternMatch.shortLink || false
+                    ...patternMatch
                 });
+                break;
             case "loom":
                 r = await loom({
                     id: patternMatch.id
                 });
+                break;
             case "facebook":
                 r = await facebook({
                     ...patternMatch,

From 85e376bffd85e4ec2f8a8d2c99eedf611be61085 Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Wed, 24 Jul 2024 17:27:26 +0200
Subject: [PATCH 46/67] api: move accept header check into handler, simplify
 error handling (#614)

---
 src/core/api.js                    | 37 +++++++++++-------------------
 src/localization/languages/en.json |  3 ++-
 2 files changed, 15 insertions(+), 25 deletions(-)

diff --git a/src/core/api.js b/src/core/api.js
index 8eb4cb40..aaad0887 100644
--- a/src/core/api.js
+++ b/src/core/api.js
@@ -26,7 +26,7 @@ const corsConfig = env.corsWildcard ? {} : {
 export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
     const startTime = new Date();
     const startTimestamp = startTime.getTime();
-    
+
     const serverInfo = {
         version: version,
         commit: gitCommit,
@@ -81,38 +81,23 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
     app.use((req, res, next) => {
         try {
             decodeURIComponent(req.path)
-        } catch { 
+        } catch {
             return res.redirect('/')
         }
         next();
     })
 
-    app.use('/api/json', express.json({
-        verify: (req, res, buf) => {
-            if (String(req.header('Accept')) === "application/json") {
-                if (buf.length > 720) throw new Error();
-                JSON.parse(buf);
-            } else {
-                throw new Error();
-            }
-        }
-    }))
-
-    // handle express.json errors properly (https://github.com/expressjs/express/issues/4065)
-    app.use('/api/json', (err, req, res, next) => {
-        let errorText = "invalid json body";
-        const acceptHeader = String(req.header('Accept')) !== "application/json";
-
-        if (err || acceptHeader) {
-            if (acceptHeader) errorText = "invalid accept header";
+    app.use('/api/json', express.json({ limit: 1024 }));
+    app.use('/api/json', (err, _, res, next) => {
+        if (err) {
             return res.status(400).json({
                 status: "error",
-                text: errorText
+                text: "invalid json body"
             });
-        } else {
-            next();
         }
-    })
+
+        next();
+    });
 
     app.post('/api/json', async (req, res) => {
         const request = req.body;
@@ -123,6 +108,10 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
             res.status(status).json(body);
         }
 
+        if (!acceptRegex.test(req.header('Accept'))) {
+            return fail('ErrorInvalidAcceptHeader');
+        }
+
         if (!acceptRegex.test(req.header('Content-Type'))) {
             return fail('ErrorInvalidContentType');
         }
diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json
index eecd9ac1..2b10f41d 100644
--- a/src/localization/languages/en.json
+++ b/src/localization/languages/en.json
@@ -159,6 +159,7 @@
         "UpdateOneMillion": "1 million users and blazing speed",
         "ErrorYTAgeRestrict": "this youtube video is age-restricted, so i can't see it. try another one!",
         "ErrorYTLogin": "couldn't get this youtube video because it requires an account to view.\n\nthis limitation is done by google to seemingly stop scraping, affecting all 3rd party tools and even their own clients.\n\ntry again, but if issue persists, {ContactLink}.",
-        "ErrorYTRateLimit": "i got rate limited by youtube. try again in a few seconds, but if issue persists, {ContactLink}."
+        "ErrorYTRateLimit": "i got rate limited by youtube. try again in a few seconds, but if issue persists, {ContactLink}.",
+        "ErrorInvalidAcceptHeader": "invalid accept header"
     }
 }

From a6623567e29c4dd6ce1d385d14eba16322bf79de Mon Sep 17 00:00:00 2001
From: alina sireneva <alina@tei.su>
Date: Wed, 24 Jul 2024 08:49:47 -0700
Subject: [PATCH 47/67] feat: support external proxies (#588)

---
 docs/run-an-instance.md | 1 +
 src/core/api.js         | 9 +++++++++
 src/modules/config.js   | 4 +++-
 3 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md
index 1faafb91..d31a67f5 100644
--- a/docs/run-an-instance.md
+++ b/docs/run-an-instance.md
@@ -56,6 +56,7 @@ sudo service nscd start
 | `API_LISTEN_ADDRESS`  | `0.0.0.0` | `127.0.0.1`             | changes address from which api server is accessible. **if you are using docker, you usually don't need to configure this.** |
 | `API_URL`             | ➖        | `https://api.cobalt.tools/` | changes url from which api server is accessible. <br> ***REQUIRED TO RUN THE API***. |
 | `API_NAME`            | `unknown` | `ams-1`                 | api server name that is shown in `/api/serverInfo`. |
+| `API_EXTERNAL_PROXY`  | ➖        | `http://user:password@127.0.0.1:8080`| url of the proxy that will be passed to [`ProxyAgent`](https://undici.nodejs.org/#/docs/api/ProxyAgent) and used for all external requests. HTTP(S) only. |
 | `CORS_WILDCARD`       | `1`       | `0`                     | toggles cross-origin resource sharing. <br> `0`: disabled. `1`: enabled. |
 | `CORS_URL`            | not used  | `https://cobalt.tools`  | cross-origin resource sharing url. api will be available only from this url if `CORS_WILDCARD` is set to `0`. |
 | `COOKIE_PATH`         | not used  | `/cookies.json`         | path for cookie file relative to main folder. |
diff --git a/src/core/api.js b/src/core/api.js
index aaad0887..5f4ee804 100644
--- a/src/core/api.js
+++ b/src/core/api.js
@@ -1,5 +1,6 @@
 import cors from "cors";
 import rateLimit from "express-rate-limit";
+import { setGlobalDispatcher, ProxyAgent } from "undici";
 
 import { env, version } from "../modules/config.js";
 
@@ -208,6 +209,14 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
     randomizeCiphers();
     setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes
 
+    if (env.externalProxy) {
+        if (env.freebindCIDR) {
+            throw new Error('Freebind is not available when external proxy is enabled')
+        }
+
+        setGlobalDispatcher(new ProxyAgent(env.externalProxy))
+    }
+
     app.listen(env.apiPort, env.listenAddress, () => {
         console.log(`\n` +
             `${Cyan("cobalt")} API ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\n` +
diff --git a/src/modules/config.js b/src/modules/config.js
index 530c5f0b..662d8b05 100644
--- a/src/modules/config.js
+++ b/src/modules/config.js
@@ -48,7 +48,9 @@ const
 
         processingPriority: process.platform !== 'win32'
                                 && process.env.PROCESSING_PRIORITY
-                                && parseInt(process.env.PROCESSING_PRIORITY)
+                                && parseInt(process.env.PROCESSING_PRIORITY),
+
+        externalProxy: process.env.API_EXTERNAL_PROXY,
     }
 
 export const

From 0fe85b2ae94766630aee72f91c73f3b9039ef274 Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Wed, 24 Jul 2024 15:52:09 +0000
Subject: [PATCH 48/67] services/ok: fallback to compilation title if author
 name is missing

---
 src/modules/processing/services/ok.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/modules/processing/services/ok.js b/src/modules/processing/services/ok.js
index 97bbcf82..33847cd8 100644
--- a/src/modules/processing/services/ok.js
+++ b/src/modules/processing/services/ok.js
@@ -45,7 +45,7 @@ export default async function(o) {
 
     let fileMetadata = {
         title: cleanString(videoData.movie.title.trim()),
-        author: cleanString(videoData.author.name.trim()),
+        author: cleanString((videoData.author?.name || videoData.compilationTitle).trim()),
     }
 
     if (bestVideo) return {

From cdb2b401ced4b357c882e81b02db684ccc20934d Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Tue, 9 Jul 2024 18:30:24 +0000
Subject: [PATCH 49/67] ci: test services in github actions

---
 .github/workflows/test.yml | 24 +++++++++++
 src/modules/test.js        | 42 ++++++++++++++++++++
 src/util/test-ci.js        | 81 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 147 insertions(+)
 create mode 100644 src/modules/test.js
 create mode 100644 src/util/test-ci.js

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 193ddd01..4ac2daf3 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -36,3 +36,27 @@ jobs:
         uses: actions/checkout@v4
       - name: Run test script
         run: .github/test.sh api
+
+  check-services:
+    name: test service functionality
+    runs-on: ubuntu-latest
+    outputs:
+      services: ${{ steps.checkServices.outputs.service_list }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - id: checkServices
+        run: npm ci && echo "service_list=$(node src/util/test-ci get-services)" >> "$GITHUB_OUTPUT"
+
+  test-services:
+    needs: check-services
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        service: ${{ fromJson(needs.check-services.outputs.services) }}
+    name: "test service: ${{ matrix.service }}"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - run: npm ci && node src/util/test-ci run-tests-for ${{ matrix.service }}
\ No newline at end of file
diff --git a/src/modules/test.js b/src/modules/test.js
new file mode 100644
index 00000000..75bdab26
--- /dev/null
+++ b/src/modules/test.js
@@ -0,0 +1,42 @@
+import { normalizeRequest } from "../modules/processing/request.js";
+import match from "./processing/match.js";
+import { extract } from "./processing/url.js";
+
+export async function runTest(url, params, expect) {
+    const normalized = normalizeRequest({ url, ...params });
+    if (!normalized) {
+        throw "invalid request";
+    }
+
+    const parsed = extract(normalized.url);
+    if (parsed === null) {
+        throw `invalid url: ${normalized.url}`;
+    }
+
+    const result = await match(
+        parsed.host, parsed.patternMatch, "en", normalized
+    );
+
+    let error = [];
+    if (expect.status !== result.body.status) {
+        const detail = `${expect.status} (expected) != ${result.body.status} (actual)`;
+        error.push(`status mismatch: ${detail}`);
+    }
+
+    if (expect.code !== result.status) {
+        const detail = `${expect.code} (expected) != ${result.status} (actual)`;
+        error.push(`status code mismatch: ${detail}`);
+    }
+
+    if (error.length) {
+        if (result.body.text) {
+            error.push(`error message: ${result.body.text}`);
+        }
+
+        throw error.join('\n');
+    }
+
+    if (result.body.status === 'stream') {
+        // TODO: stream testing
+    }
+}
\ No newline at end of file
diff --git a/src/util/test-ci.js b/src/util/test-ci.js
new file mode 100644
index 00000000..ed6962c7
--- /dev/null
+++ b/src/util/test-ci.js
@@ -0,0 +1,81 @@
+import { env } from "../modules/config.js";
+import { runTest } from "../modules/test.js";
+import { loadLoc } from "../localization/manager.js";
+import { loadJSON } from "../modules/sub/loadFromFs.js";
+import { Red, Bright } from "../modules/sub/consoleText.js";
+
+const tests = loadJSON('./src/util/tests.json');
+const services = loadJSON('./src/modules/processing/servicesConfig.json');
+
+// services that are known to frequently fail due to external
+// factors (e.g. rate limiting)
+const finnicky = new Set(['bilibili', 'instagram', 'youtube'])
+
+const action = process.argv[2];
+switch (action) {
+    case "get-services":
+        const fromConfig = Object.keys(services.config);
+
+        const missingTests = fromConfig.filter(
+            service => !tests[service] || tests[service].length === 0
+        );
+
+        if (missingTests.length) {
+            console.error('services have no tests:', missingTests);
+            console.log('[]');
+            process.exitCode = 1;
+            break;
+        }
+
+        console.log(JSON.stringify(fromConfig));
+        break;
+
+    case "run-tests-for":
+        const service = process.argv[3];
+        let failed = false;
+
+        if (!tests[service]) {
+            console.error('no such service:', service);
+        }
+
+        await loadLoc();
+        env.streamLifespan = 10000;
+        env.apiURL = 'http://x';
+
+        for (const test of tests[service]) {
+            const { name, url, params, expected } = test;
+            const canFail = test.canFail || finnicky.has(service);
+
+            try {
+                await runTest(url, params, expected);
+                console.log(`${service}/${name}: ok`);
+
+            } catch(e) {
+                failed = !canFail;
+
+                let failText = canFail ? `${Red('FAIL')} (ignored)` : Bright(Red('FAIL'));
+                if (canFail && process.env.GITHUB_ACTION) {
+                    console.log(`::warning title=${service}/${name.replace(/,/g, ';')}::failed and was ignored`);
+                }
+
+                console.error(`${service}/${name}: ${failText}`);
+                const errorString = e.toString().split('\n');
+                let c = '┃';
+                errorString.forEach((line, index) => {
+                    line = line.replace('!=', Red('!='));
+
+                    if (index === errorString.length - 1) {
+                        c = '┗';
+                    }
+
+                    console.error(`   ${c}`, line);
+                });
+            }
+        }
+
+        process.exitCode = Number(failed);
+        break;
+    default:
+        console.error('invalid action:', action);
+        process.exitCode = 1;
+}
\ No newline at end of file

From 93c27f262b34585d473cdaf1691707b15ff15a17 Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Tue, 9 Jul 2024 18:45:16 +0000
Subject: [PATCH 50/67] tests/twitter: fix incorrect status, expect age
 restricted test to fail

---
 src/util/tests.json | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/util/tests.json b/src/util/tests.json
index b34783bd..902ec00a 100644
--- a/src/util/tests.json
+++ b/src/util/tests.json
@@ -42,7 +42,7 @@
         },
         "expected": {
             "code": 200,
-            "status": "redirect"
+            "status": "picker"
         }
     }, {
         "name": "picker: mixed media (3 videos)",
@@ -113,9 +113,10 @@
             "status": "redirect"
         }
     }, {
-        "name": "age-restricted video",
+        "name": "age restricted video",
         "url": "https://twitter.com/FckyeahCharli/status/1650987582749065220",
         "params": {},
+        "canFail": true,
         "expected": {
             "code": 200,
             "status": "redirect"

From 27a6310357da8dd474be4a48498672d6b0f77357 Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Tue, 9 Jul 2024 19:04:52 +0000
Subject: [PATCH 51/67] ci/test: update basic test to use more reliable example
 link

---
 .github/test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/test.sh b/.github/test.sh
index 85683b91..80de1fcd 100755
--- a/.github/test.sh
+++ b/.github/test.sh
@@ -18,7 +18,7 @@ test_api() {
          -X POST \
          -H "Accept: application/json" \
          -H "Content-Type: application/json" \
-         -d '{"url":"https://www.youtube.com/watch?v=jNQXAC9IVRw"}')
+         -d '{"url":"https://vine.co/v/huwVJIEJW50", "isAudioOnly": true}')
 
     echo "$API_RESPONSE"
     STATUS=$(echo "$API_RESPONSE" | jq -r .status)

From 9bf6cd36b57417d5a7321ab4c5f704eaee185a0e Mon Sep 17 00:00:00 2001
From: dumbmoron <log@riseup.net>
Date: Wed, 24 Jul 2024 15:56:50 +0000
Subject: [PATCH 52/67] tests: add canFail to facebook shortlink test

---
 src/util/tests.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/util/tests.json b/src/util/tests.json
index 902ec00a..39ca17e1 100644
--- a/src/util/tests.json
+++ b/src/util/tests.json
@@ -1214,6 +1214,7 @@
     }, {
         "name": "shortlink video",
         "url": "https://fb.watch/r1K6XHMfGT/",
+        "canFail": true,
         "params": {},
         "expected": {
             "code": 200,
@@ -1244,4 +1245,4 @@
             "status": "redirect"
         }
     }]
-}
\ No newline at end of file
+}

From faeb96848bd8d90bce0565011bc5a6979c11b50c Mon Sep 17 00:00:00 2001
From: ihatespawn <168680471+ihatespawn@users.noreply.github.com>
Date: Wed, 24 Jul 2024 19:46:29 +0200
Subject: [PATCH 53/67] tests: add mature video vimeo test (#628)

---
 src/util/tests.json | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/util/tests.json b/src/util/tests.json
index 39ca17e1..ac65bf0b 100644
--- a/src/util/tests.json
+++ b/src/util/tests.json
@@ -677,6 +677,14 @@
             "code": 200,
             "status": "redirect"
         }
+    }, {
+        "name": "mature video",
+        "url": "https://vimeo.com/973212054",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
     }],
     "reddit": [{
         "name": "video with audio",

From 83af16bb12c19371b458947d7a49f634eb980b63 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Thu, 25 Jul 2024 11:57:02 +0600
Subject: [PATCH 54/67] facebook: do not request the source url & clean up

fixed a vulnerability, removed useless variables, and cleaned up

thankfully we haven't built the image yesterday
---
 src/modules/processing/match.js               |  3 +-
 src/modules/processing/services/facebook.js   | 46 ++++++++-----------
 src/modules/processing/servicesConfig.json    |  2 +-
 .../processing/servicesPatternTesters.js      |  3 +-
 4 files changed, 24 insertions(+), 30 deletions(-)

diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js
index b7d46333..ec052e60 100644
--- a/src/modules/processing/match.js
+++ b/src/modules/processing/match.js
@@ -203,8 +203,7 @@ export default async function(host, patternMatch, lang, obj) {
                 break;
             case "facebook":
                 r = await facebook({
-                    ...patternMatch,
-                    sourceUrl: url.href
+                    ...patternMatch
                 });
                 break;
             default:
diff --git a/src/modules/processing/services/facebook.js b/src/modules/processing/services/facebook.js
index 45d31b5f..17dedab4 100644
--- a/src/modules/processing/services/facebook.js
+++ b/src/modules/processing/services/facebook.js
@@ -4,59 +4,53 @@ const headers = {
     'User-Agent': genericUserAgent,
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
     'Accept-Language': 'en-US,en;q=0.5',
-    'Accept-Encoding': 'gzip, deflate, br',
     'Sec-Fetch-Mode': 'navigate',
     'Sec-Fetch-Site': 'none',
 }
 
-function resolveUrl(url) {
+const resolveUrl = (url) => {
     return fetch(url, { headers })
         .then(r => {
             if (r.headers.get('location')) {
-                return decodeURIComponent(r.headers.get('location'))
+                return decodeURIComponent(r.headers.get('location'));
             }
             if (r.headers.get('link')) {
-                const linkMatch = r.headers.get('link').match(/<(.*?)\/>/)
-                return decodeURIComponent(linkMatch[1])
+                const linkMatch = r.headers.get('link').match(/<(.*?)\/>/);
+                return decodeURIComponent(linkMatch[1]);
             }
-            return false
+            return false;
         })
-        .catch(() => false)
+        .catch(() => false);
 }
 
-export default async function({ sourceUrl, shortLink, username, id }) {
-    const isShortLink = !!shortLink?.length
-    const isSharedLink = !!sourceUrl.match(/\/share\/\w\//)?.length
+export default async function({ id, shareType, shortLink }) {
+    let url = `https://web.facebook.com/i/videos/${id}`;
 
-    let url = isShortLink
-        ? `https://fb.watch/${shortLink}`
-        : `https://web.facebook.com/${username}/videos/${id}`
-
-    if (isShortLink) url = await resolveUrl(url)
-    if (isSharedLink) url = sourceUrl
+    if (shareType) url = `https://web.facebook.com/share/${shareType}/${id}`;
+    if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`);
 
     const html = await fetch(url, { headers })
         .then(r => r.text())
-        .catch(() => false)
+        .catch(() => false);
 
     if (!html) return { error: 'ErrorCouldntFetch' };
 
-    const urls = []
-    const hd = html.match('"browser_native_hd_url":(".*?")')
-    const sd = html.match('"browser_native_sd_url":(".*?")')
+    const urls = [];
+    const hd = html.match('"browser_native_hd_url":(".*?")');
+    const sd = html.match('"browser_native_sd_url":(".*?")');
 
-    if (hd?.[1]) urls.push(JSON.parse(hd[1]))
-    if (sd?.[1]) urls.push(JSON.parse(sd[1]))
+    if (hd?.[1]) urls.push(JSON.parse(hd[1]));
+    if (sd?.[1]) urls.push(JSON.parse(sd[1]));
 
     if (!urls.length) {
         return { error: 'ErrorEmptyDownload' };
     }
 
-    let filename = `facebook_${id || shortLink}.mp4`
+    const baseFilename = `facebook_${id || shortLink}`;
 
     return {
         urls: urls[0],
-        filename,
-        audioFilename: `${filename.slice(0, -4)}_audio`,
+        filename: `${baseFilename}.mp4`,
+        audioFilename: `${baseFilename}_audio`,
     };
-}
\ No newline at end of file
+}
diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json
index 9161117a..6955a946 100644
--- a/src/modules/processing/servicesConfig.json
+++ b/src/modules/processing/servicesConfig.json
@@ -134,7 +134,7 @@
                 ":username/videos/:caption/:id",
                 ":username/videos/:id",
                 "reel/:id",
-                "share/:shortLink/:id"
+                "share/:shareType/:id"
             ],
             "enabled": true
         }
diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js
index 9f4e0f91..48f953c1 100644
--- a/src/modules/processing/servicesPatternTesters.js
+++ b/src/modules/processing/servicesPatternTesters.js
@@ -68,5 +68,6 @@ export const testers = {
         patternMatch.shortLink?.length <= 11
         || patternMatch.username?.length <= 30
         || patternMatch.caption?.length <= 255
-        || patternMatch.id?.length <= 20,
+        || patternMatch.id?.length <= 20 && !patternMatch.shareType
+        || patternMatch.id?.length <= 20 && patternMatch.shareType?.length === 1,
 }

From a388839e15dd1002cb781dcec12bf2112ac4617e Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Thu, 25 Jul 2024 11:58:43 +0600
Subject: [PATCH 55/67] snapchat: clean up

---
 src/modules/processing/match.js             |  2 +-
 src/modules/processing/services/snapchat.js | 26 ++++++++++-----------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js
index ec052e60..6e600cf0 100644
--- a/src/modules/processing/match.js
+++ b/src/modules/processing/match.js
@@ -192,7 +192,7 @@ export default async function(host, patternMatch, lang, obj) {
                 break;
             case "snapchat":
                 r = await snapchat({
-                    url,
+                    hostname: url.hostname,
                     ...patternMatch
                 });
                 break;
diff --git a/src/modules/processing/services/snapchat.js b/src/modules/processing/services/snapchat.js
index 44a2b84e..a93f0933 100644
--- a/src/modules/processing/services/snapchat.js
+++ b/src/modules/processing/services/snapchat.js
@@ -9,8 +9,8 @@ async function getSpotlight(id) {
     const html = await fetch(`https://www.snapchat.com/spotlight/${id}`, {
         headers: { 'User-Agent': genericUserAgent }
     }).then((r) => r.text()).catch(() => null);
-    if (!html) {  
-        return { error: 'ErrorCouldntFetch' };  
+    if (!html) {
+        return { error: 'ErrorCouldntFetch' };
     }
 
     const videoURL = html.match(SPOTLIGHT_VIDEO_REGEX)?.[1];
@@ -27,9 +27,9 @@ async function getStory(username, storyId) {
     const html = await fetch(`https://www.snapchat.com/add/${username}${storyId ? `/${storyId}` : ''}`, {
         headers: { 'User-Agent': genericUserAgent }
     }).then((r) => r.text()).catch(() => null);
-    if (!html) {  
-        return { error: 'ErrorCouldntFetch' };  
-    }  
+    if (!html) {
+        return { error: 'ErrorCouldntFetch' };
+    }
 
     const nextDataString = html.match(NEXT_DATA_REGEX)?.[1];
     if (nextDataString) {
@@ -67,18 +67,18 @@ async function getStory(username, storyId) {
     }
 }
 
-export default async function(obj) {
+export default async function (obj) {
     let params = obj;
-    if (obj.url.hostname === 't.snapchat.com' && obj.shortLink) {
+    if (obj.hostname === 't.snapchat.com' && obj.shortLink) {
         const link = await getRedirectingURL(`https://t.snapchat.com/${obj.shortLink}`);
-    
-        if (!link?.startsWith('https://www.snapchat.com/')) {  
-            return { error: 'ErrorCouldntFetch' };  
-        }  
+
+        if (!link?.startsWith('https://www.snapchat.com/')) {
+            return { error: 'ErrorCouldntFetch' };
+        }
 
         const extractResult = extract(normalizeURL(link));
-        if (extractResult?.host !== 'snapchat') {  
-            return { error: 'ErrorCouldntFetch' };  
+        if (extractResult?.host !== 'snapchat') {
+            return { error: 'ErrorCouldntFetch' };
         }
 
         params = extractResult.patternMatch;

From 8ab0661f1aa7e4229fbc2bfc02b46ef633e7c589 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Thu, 25 Jul 2024 12:08:37 +0600
Subject: [PATCH 56/67] tests: update rutube yappy link

---
 src/util/tests.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/util/tests.json b/src/util/tests.json
index ac65bf0b..6399d8ac 100644
--- a/src/util/tests.json
+++ b/src/util/tests.json
@@ -1064,7 +1064,7 @@
         }
     }, {
         "name": "yappy",
-        "url": "https://rutube.ru/yappy/a06b1bf53bce403b9a069107f23c47eb/",
+        "url": "https://rutube.ru/yappy/c8c32bf7aee04412837656ea26c2b25b/",
         "params": {},
         "expected": {
             "code": 200,

From cb2bde5b5175089bdd32c0d3d1552637393c0547 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Thu, 25 Jul 2024 12:14:25 +0600
Subject: [PATCH 57/67] tests: allow facebook reel test to fail

---
 src/util/tests.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/util/tests.json b/src/util/tests.json
index 6399d8ac..1093d75c 100644
--- a/src/util/tests.json
+++ b/src/util/tests.json
@@ -1231,6 +1231,7 @@
     }, {
         "name": "reel video",
         "url": "https://web.facebook.com/reel/730293269054758",
+        "canFail": true,
         "params": {},
         "expected": {
             "code": 200,

From 339a932e7fa402855ce5f8bf3139d1a8ccccbb01 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Fri, 26 Jul 2024 09:50:27 +0600
Subject: [PATCH 58/67] package: update youtubei.js

---
 package-lock.json | 9 +++++----
 package.json      | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 4917de2d..34d4ec7d 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -24,7 +24,7 @@
                 "set-cookie-parser": "2.6.0",
                 "undici": "^5.19.1",
                 "url-pattern": "1.0.3",
-                "youtubei.js": "^10.1.0"
+                "youtubei.js": "^10.2.0"
             },
             "engines": {
                 "node": ">=18"
@@ -1123,12 +1123,13 @@
             }
         },
         "node_modules/youtubei.js": {
-            "version": "10.1.0",
-            "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-10.1.0.tgz",
-            "integrity": "sha512-MokZMAnpWH11VYvWuW6qjPiiPmgRl5rfDgPQOpif9qXcVHoVw1hi8ePuRSD0AZSZ+uvWGe8rvas2dzp+Jv5JKQ==",
+            "version": "10.2.0",
+            "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-10.2.0.tgz",
+            "integrity": "sha512-JLKW9AHQ1qrTwBbre1aDkH8UJFmNcc4+kOSaVou5jSY7AzfFPFJK0yvX6afnLst0UVC9wfXHrLiNx93sutVErA==",
             "funding": [
                 "https://github.com/sponsors/LuanRT"
             ],
+            "license": "MIT",
             "dependencies": {
                 "jintr": "^2.0.0",
                 "tslib": "^2.5.0",
diff --git a/package.json b/package.json
index bcba1894..9a4a4063 100644
--- a/package.json
+++ b/package.json
@@ -40,7 +40,7 @@
         "set-cookie-parser": "2.6.0",
         "undici": "^5.19.1",
         "url-pattern": "1.0.3",
-        "youtubei.js": "^10.1.0"
+        "youtubei.js": "^10.2.0"
     },
     "optionalDependencies": {
         "freebind": "^0.2.2"

From b718b79ccd0a2afaf2a2a67f6286387f3019a66f Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Fri, 26 Jul 2024 09:52:55 +0600
Subject: [PATCH 59/67] package: bump version to 7.15

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 9a4a4063..58fdec83 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
 {
     "name": "cobalt",
     "description": "save what you love",
-    "version": "7.14.6",
+    "version": "7.15",
     "author": "imput",
     "exports": "./src/cobalt.js",
     "type": "module",

From b308db4a5964c995f2d5f49ceb14a9887464830a Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Fri, 26 Jul 2024 09:55:41 +0600
Subject: [PATCH 60/67] package: update lock

---
 package-lock.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 34d4ec7d..87e11661 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
     "name": "cobalt",
-    "version": "7.14.6",
+    "version": "7.15",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "cobalt",
-            "version": "7.14.6",
+            "version": "7.15",
             "license": "AGPL-3.0",
             "dependencies": {
                 "content-disposition-header": "0.6.0",

From b2385873240f2fd6897fe2e5078065af4f2d1b1b Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Fri, 26 Jul 2024 09:56:22 +0600
Subject: [PATCH 61/67] tests: allow rutube yappy test to fail

---
 src/util/tests.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/util/tests.json b/src/util/tests.json
index 1093d75c..d447b348 100644
--- a/src/util/tests.json
+++ b/src/util/tests.json
@@ -1065,6 +1065,7 @@
     }, {
         "name": "yappy",
         "url": "https://rutube.ru/yappy/c8c32bf7aee04412837656ea26c2b25b/",
+        "canFail": true,
         "params": {},
         "expected": {
             "code": 200,

From 0895c695e7e94fa9f5837eef5517d2cd0ff4afe7 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Fri, 26 Jul 2024 13:45:00 +0600
Subject: [PATCH 62/67] repo: use global org funding file

---
 .github/FUNDING.yml | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 .github/FUNDING.yml

diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml
deleted file mode 100644
index 89951fe0..00000000
--- a/.github/FUNDING.yml
+++ /dev/null
@@ -1 +0,0 @@
-custom: https://boosty.to/wukko/donate

From 707115758007fe80433a41026121554096ffcfb3 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Fri, 26 Jul 2024 19:54:19 +0600
Subject: [PATCH 63/67] youtube: fix audio track selection for dubbed videos &
 clean up

closes #642
---
 src/modules/processing/services/youtube.js | 109 ++++++++++++++-------
 1 file changed, 73 insertions(+), 36 deletions(-)

diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js
index 7beca2b4..2a0d25c5 100644
--- a/src/modules/processing/services/youtube.js
+++ b/src/modules/processing/services/youtube.js
@@ -1,25 +1,27 @@
-import { Innertube, Session } from 'youtubei.js';
-import { env } from '../../config.js';
-import { cleanString } from '../../sub/utils.js';
-import { fetch } from 'undici'
-import { getCookie, updateCookieValues } from '../cookie/manager.js'
+import { fetch } from "undici";
+
+import { Innertube, Session } from "youtubei.js";
+
+import { env } from "../../config.js";
+import { cleanString } from "../../sub/utils.js";
+import { getCookie, updateCookieValues } from "../cookie/manager.js";
 
 const ytBase = Innertube.create().catch(e => e);
 
 const codecMatch = {
     h264: {
-        codec: "avc1",
-        aCodec: "mp4a",
+        videoCodec: "avc1",
+        audioCodec: "mp4a",
         container: "mp4"
     },
     av1: {
-        codec: "av01",
-        aCodec: "mp4a",
+        videoCodec: "av01",
+        audioCodec: "mp4a",
         container: "mp4"
     },
     vp9: {
-        codec: "vp9",
-        aCodec: "opus",
+        videoCodec: "vp9",
+        audioCodec: "opus",
         container: "webm"
     }
 }
@@ -94,11 +96,16 @@ const cloneInnertube = async (customFetch) => {
 
 export default async function(o) {
     const yt = await cloneInnertube(
-        (input, init) => fetch(input, { ...init, dispatcher: o.dispatcher })
+        (input, init) => fetch(input, {
+            ...init,
+            dispatcher: o.dispatcher
+        })
     );
 
-    let info, isDubbed, format = o.format || "h264";
-    let quality = o.quality === "max" ? "9000" : o.quality; // 9000(p) - max quality
+    const quality = o.quality === "max" ? "9000" : o.quality;
+
+    let info, isDubbed,
+        format = o.format || "h264";
 
     function qual(i) {
         if (!i.quality_label) {
@@ -121,6 +128,7 @@ export default async function(o) {
     if (!info) return { error: 'ErrorCantConnectToServiceAPI' };
 
     const playability = info.playability_status;
+    const basicInfo = info.basic_info;
 
     if (playability.status === 'LOGIN_REQUIRED') {
         if (playability.reason.endsWith('bot')) {
@@ -135,11 +143,11 @@ export default async function(o) {
     }
 
     if (playability.status !== 'OK') return { error: 'ErrorYTUnavailable' };
-    if (info.basic_info.is_live) return { error: 'ErrorLiveVideo' };
+    if (basicInfo.is_live) return { error: 'ErrorLiveVideo' };
 
     // return a critical error if returned video is "Video Not Available"
     // or a similar stub by youtube
-    if (info.basic_info.id !== o.id) {
+    if (basicInfo.id !== o.id) {
         return {
             error: 'ErrorCantConnectToServiceAPI',
             critical: true
@@ -148,11 +156,16 @@ export default async function(o) {
 
     let bestQuality, hasAudio;
 
-    const filterByCodec = (formats) => formats.filter(e => 
-        e.mime_type.includes(codecMatch[format].codec) || e.mime_type.includes(codecMatch[format].aCodec)
-    ).sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
+    const filterByCodec = (formats) =>
+        formats
+        .filter(e =>
+            e.mime_type.includes(codecMatch[format].videoCodec)
+            || e.mime_type.includes(codecMatch[format].audioCodec)
+        )
+        .sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
 
     let adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats);
+
     if (adaptive_formats.length === 0 && format === "vp9") {
         format = "h264"
         adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats)
@@ -162,27 +175,43 @@ export default async function(o) {
     hasAudio = adaptive_formats.find(i => i.has_audio && i.content_length);
 
     if (bestQuality) bestQuality = qual(bestQuality);
-    if (!bestQuality && !o.isAudioOnly || !hasAudio) return { error: 'ErrorYTTryOtherCodec' };
-    if (info.basic_info.duration > env.durationLimit) return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
 
-    let checkBestAudio = (i) => (i.has_audio && !i.has_video),
-        audio = adaptive_formats.find(i => checkBestAudio(i) && !i.is_dubbed);
+    if (!bestQuality && !o.isAudioOnly || !hasAudio)
+        return { error: 'ErrorYTTryOtherCodec' };
+
+    if (basicInfo.duration > env.durationLimit)
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
+
+    const checkBestAudio = (i) => (i.has_audio && !i.has_video);
+
+    let audio = adaptive_formats.find(i =>
+        checkBestAudio(i) && i.is_original
+    );
 
     if (o.dubLang) {
         let dubbedAudio = adaptive_formats.find(i =>
-            checkBestAudio(i) && i.language === o.dubLang && i.audio_track && !i.audio_track.audio_is_default
-        );
+            checkBestAudio(i)
+            && i.language === o.dubLang
+            && i.audio_track
+        )
+
         if (dubbedAudio) {
             audio = dubbedAudio;
-            isDubbed = true
+            isDubbed = true;
         }
     }
-    let fileMetadata = {
-        title: cleanString(info.basic_info.title.trim()),
-        artist: cleanString(info.basic_info.author.replace("- Topic", "").trim()),
+
+    if (!audio) {
+        audio = adaptive_formats.find(i => checkBestAudio(i));
     }
-    if (info.basic_info.short_description && info.basic_info.short_description.startsWith("Provided to YouTube by")) {
-        let descItems = info.basic_info.short_description.split("\n\n");
+
+    let fileMetadata = {
+        title: cleanString(basicInfo.title.trim()),
+        artist: cleanString(basicInfo.author.replace("- Topic", "").trim()),
+    }
+
+    if (basicInfo?.short_description?.startsWith("Provided to YouTube by")) {
+        let descItems = basicInfo.short_description.split("\n\n");
         fileMetadata.album = descItems[2];
         fileMetadata.copyright = descItems[3];
         if (descItems[4].startsWith("Released on:")) {
@@ -204,13 +233,17 @@ export default async function(o) {
         urls: audio.decipher(yt.session.player),
         filenameAttributes: filenameAttributes,
         fileMetadata: fileMetadata,
-        bestAudio: format === "h264" ? 'm4a' : 'opus'
+        bestAudio: format === "h264" ? "m4a" : "opus"
     }
+
     const matchingQuality = Number(quality) > Number(bestQuality) ? bestQuality : quality,
-        checkSingle = i => qual(i) === matchingQuality && i.mime_type.includes(codecMatch[format].codec),
-        checkRender = i => qual(i) === matchingQuality && i.has_video && !i.has_audio;
+        checkSingle = i =>
+            qual(i) === matchingQuality && i.mime_type.includes(codecMatch[format].videoCodec),
+        checkRender = i =>
+            qual(i) === matchingQuality && i.has_video && !i.has_audio;
 
     let match, type, urls;
+
     if (!o.isAudioOnly && !o.isAudioMuted && format === 'h264') {
         match = info.streaming_data.formats.find(checkSingle);
         type = "bridge";
@@ -218,10 +251,14 @@ export default async function(o) {
     }
 
     const video = adaptive_formats.find(checkRender);
+
     if (!match && video) {
         match = video;
         type = "render";
-        urls = [video.decipher(yt.session.player), audio.decipher(yt.session.player)];
+        urls = [
+            video.decipher(yt.session.player),
+            audio.decipher(yt.session.player)
+        ]
     }
 
     if (match) {
@@ -232,7 +269,7 @@ export default async function(o) {
         return {
             type,
             urls,
-            filenameAttributes, 
+            filenameAttributes,
             fileMetadata
         }
     }

From 57051968f3f35f271929b486c2fc22eadc949da8 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Fri, 26 Jul 2024 20:18:59 +0600
Subject: [PATCH 64/67] youtube: better audio checking

---
 src/modules/processing/services/youtube.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js
index 2a0d25c5..5aa201ce 100644
--- a/src/modules/processing/services/youtube.js
+++ b/src/modules/processing/services/youtube.js
@@ -176,7 +176,7 @@ export default async function(o) {
 
     if (bestQuality) bestQuality = qual(bestQuality);
 
-    if (!bestQuality && !o.isAudioOnly || !hasAudio)
+    if ((!bestQuality && !o.isAudioOnly) || !hasAudio)
         return { error: 'ErrorYTTryOtherCodec' };
 
     if (basicInfo.duration > env.durationLimit)
@@ -227,7 +227,7 @@ export default async function(o) {
         youtubeDubName: isDubbed ? o.dubLang : false
     }
 
-    if (hasAudio && o.isAudioOnly) return {
+    if (audio && o.isAudioOnly) return {
         type: "render",
         isAudioOnly: true,
         urls: audio.decipher(yt.session.player),
@@ -252,7 +252,7 @@ export default async function(o) {
 
     const video = adaptive_formats.find(checkRender);
 
-    if (!match && video) {
+    if (!match && video && audio) {
         match = video;
         type = "render";
         urls = [

From dad13548e6e176a48793f91e56a149da1ba96b32 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Fri, 26 Jul 2024 21:17:37 +0600
Subject: [PATCH 65/67] twitter: add support for downloading photos

---
 src/modules/processing/services/twitter.js | 27 ++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/src/modules/processing/services/twitter.js b/src/modules/processing/services/twitter.js
index 36a8669b..30681636 100644
--- a/src/modules/processing/services/twitter.js
+++ b/src/modules/processing/services/twitter.js
@@ -147,7 +147,6 @@ export default async function({ id, index, toGif, dispatcher }) {
     }
 
     let media = (repostedTweet?.media || baseTweet?.extended_entities?.media);
-    media = media?.filter(m => m.video_info?.variants?.length);
 
     // check if there's a video at given index (/video/<index>)
     if (index >= 0 && index < media?.length) {
@@ -159,18 +158,38 @@ export default async function({ id, index, toGif, dispatcher }) {
         case 0:
             return { error: 'ErrorNoVideosInTweet' };
         case 1:
+            if (media[0].type === "photo") {
+                return {
+                    type: "normal",
+                    isPhoto: true,
+                    urls: `${media[0].media_url_https}?name=4096x4096`
+                }
+            }
+
             return {
                 type: needsFixing(media[0]) ? "remux" : "normal",
                 urls: bestQuality(media[0].video_info.variants),
                 filename: `twitter_${id}.mp4`,
                 audioFilename: `twitter_${id}_audio`,
                 isGif: media[0].type === "animated_gif"
-            };
+            }
         default:
             const picker = media.map((content, i) => {
+                if (content.type === "photo") {
+                    let url = `${content.media_url_https}?name=4096x4096`;
+                    return {
+                        type: "photo",
+                        url,
+                        thumb: url,
+                    }
+                }
+
                 let url = bestQuality(content.video_info.variants);
                 const shouldRenderGif = content.type === 'animated_gif' && toGif;
 
+                let type = "video";
+                if (shouldRenderGif) type = "gif";
+
                 if (needsFixing(content) || shouldRenderGif) {
                     url = createStream({
                         service: 'twitter',
@@ -181,9 +200,9 @@ export default async function({ id, index, toGif, dispatcher }) {
                 }
 
                 return {
-                    type: 'video',
+                    type,
                     url,
-                    thumb: content.media_url_https,
+                    thumb: content.media_url_https
                 }
             });
             return { picker };

From 9d4f7e666f8d67e6ce20d1053b6ef372d055aaaa Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Fri, 26 Jul 2024 21:22:24 +0600
Subject: [PATCH 66/67] tests: add twitter image tests

---
 src/util/tests.json | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/util/tests.json b/src/util/tests.json
index d447b348..ab6a1e6a 100644
--- a/src/util/tests.json
+++ b/src/util/tests.json
@@ -11,8 +11,7 @@
             "code": 200,
             "status": "redirect"
         }
-    },
-    {
+    }, {
         "name": "video with mobile web mediaviewer",
         "url": "https://twitter.com/X/status/1697304622749086011/mediaViewer?currentTweet=1697304622749086011&currentTweetUser=X&currentTweet=1697304622749086011&currentTweetUser=X",
         "params": {},
@@ -137,6 +136,22 @@
             "code": 200,
             "status": "redirect"
         }
+    }, {
+        "name": "post with 1 image",
+        "url": "https://x.com/PopCrave/status/1815960083475423235",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "post with 4 images",
+        "url": "https://x.com/PopCrave/status/1816260887147114696",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
     }, {
         "name": "retweeted video, isAudioOnly",
         "url": "https://twitter.com/hugekiwinuts/status/1618671150829309953?s=46&t=gItGzgwGQQJJaJrO6qc1Pg",

From f8a28c6c5f2fd0cee45daf3fbdd521d22f6a0e7f Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Fri, 26 Jul 2024 22:02:59 +0600
Subject: [PATCH 67/67] docs/api: update info about picker item types

---
 docs/api.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/api.md b/docs/api.md
index b63c9216..6cd66bba 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -49,9 +49,9 @@ item type: `object`
 
 | key     | type     | variables                                               | description                            |
 |:--------|:---------|:--------------------------------------------------------|:---------------------------------------|
-| `type`  | `string` | `video`                                                 | used only if `pickerType`is `various`. |
+| `type`  | `string` | `video / photo / gif`                                   | used only if `pickerType` is `various` |
 | `url`   | `string` | direct link to a file or a link to cobalt's live render |                                        |
-| `thumb` | `string` | item thumbnail that's displayed in the picker           | used only for `video` type.            |
+| `thumb` | `string` | item thumbnail that's displayed in the picker           | used for `video` and `gif` types       |
 
 ## GET: `/api/stream`
 cobalt's live render (or stream) endpoint. usually, you will receive a url to this endpoint