Merge branch 'wukko:current' into current

package.json

@@ -1,7 +1,7 @@
 {
     "name": "cobalt",
     "description": "save what you love",
-    "version": "7.12.6",
+    "version": "7.13",
     "author": "wukko",
     "exports": "./src/cobalt.js",
     "type": "module",
@@ -40,6 +40,6 @@
         "set-cookie-parser": "2.6.0",
         "undici": "^6.7.0",
         "url-pattern": "1.0.3",
-        "youtubei.js": "^9.2.0"
+        "youtubei.js": "^9.3.0"
     }
 }

src/core/api.js

@@ -11,7 +11,7 @@ import { Bright, Cyan } from "../modules/sub/consoleText.js";
 import stream from "../modules/stream/stream.js";
 import loc from "../localization/manager.js";
 import { generateHmac } from "../modules/sub/crypto.js";
-import { verifyStream } from "../modules/stream/manage.js";
+import { verifyStream, getInternalStream } from "../modules/stream/manage.js";
 
 export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
     const corsConfig = process.env.CORS_WILDCARD === '0' ? {
@@ -123,13 +123,13 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
 
     app.get('/api/:type', (req, res) => {
         try {
+            let j;
             switch (req.params.type) {
                 case 'stream':
                     const q = req.query;
                     const checkQueries = q.t && q.e && q.h && q.s && q.i;
                     const checkBaseLength = q.t.length === 21 && q.e.length === 13;
                     const checkSafeLength = q.h.length === 43 && q.s.length === 43 && q.i.length === 22;
-
                     if (checkQueries && checkBaseLength && checkSafeLength) {
                         let streamInfo = verifyStream(q.t, q.h, q.e, q.s, q.i);
                         if (streamInfo.error) {
@@ -141,12 +141,23 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
                             });
                         }
                         return stream(res, streamInfo);
-                    } else {
+                    } 
-                        let j = apiJSON(0, {
+
-                            t: "bad request. stream link may be incomplete or corrupted."
+                    j = apiJSON(0, {
-                        })
+                        t: "bad request. stream link may be incomplete or corrupted."
-                        return res.status(j.status).json(j.body);
+                    })
-                    }
+                    return res.status(j.status).json(j.body);
+                case 'istream':
+                    if (!req.ip.endsWith('127.0.0.1'))
+                        return res.sendStatus(403);
+                    if (('' + req.query.t).length !== 21)
+                        return res.sendStatus(400);
+    
+                    let streamInfo = getInternalStream(req.query.t);
+                    if (!streamInfo) return res.sendStatus(404);
+                    streamInfo.headers = req.headers;
+    
+                    return stream(res, { type: 'internal', ...streamInfo });
                 case 'serverInfo':
                     return res.status(200).json({
                         version: version,
@@ -158,7 +169,7 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
                         startTime: `${startTimestamp}`
                     });
                 default:
-                    let j = apiJSON(0, {
+                    j = apiJSON(0, {
                         t: "unknown response type"
                     })
                     return res.status(j.status).json(j.body);

src/front/manifest.webmanifest

@@ -18,6 +18,48 @@
             "sizes": "512x512",
             "type": "image/png",
             "purpose": "any"
+        },
+        {
+            "src": "/icons/maskable/48.png",
+            "sizes": "48x48",
+            "type": "image/png",
+            "purpose": "maskable"
+        },
+        {
+            "src": "/icons/maskable/72.png",
+            "sizes": "72x72",
+            "type": "image/png",
+            "purpose": "maskable"
+        },
+        {
+            "src": "/icons/maskable/96.png",
+            "sizes": "96x96",
+            "type": "image/png",
+            "purpose": "maskable"
+        },
+        {
+            "src": "/icons/maskable/128.png",
+            "sizes": "128x128",
+            "type": "image/png",
+            "purpose": "maskable"
+        },
+        {
+            "src": "/icons/maskable/192.png",
+            "sizes": "192x192",
+            "type": "image/png",
+            "purpose": "maskable"
+        },
+        {
+            "src": "/icons/maskable/384.png",
+            "sizes": "384x384",
+            "type": "image/png",
+            "purpose": "maskable"
+        },
+        {
+            "src": "/icons/maskable/512.png",
+            "sizes": "512x512",
+            "type": "image/png",
+            "purpose": "maskable"
         }
     ],
     "share_target": {

src/localization/languages/en.json

@@ -101,10 +101,10 @@
         "FollowSupport": "keep in touch with cobalt for news, support, and more:",
         "SourceCode": "explore source code, report issues, star or fork the repo:",
         "PrivacyPolicy": "cobalt's privacy policy is simple: no data about you is ever collected or stored. zero, zilch, nada, nothing.\nwhat you download is solely your business, not mine or anyone else's.\n\nif your download requires rendering, then data about requested content is encrypted and temporarily stored in server's RAM. it's necessary for this feature to function.\n\nencrypted data is stored for <span class=\"text-backdrop\">90 seconds</span> and then permanently removed.\n\nstored data is only possible to decrypt with unique encryption keys from your download link. furthermore, the official cobalt codebase doesn't provide a way to read temporarily stored data outside of processing functions.\n\nyou can check cobalt's <a class=\"text-backdrop link\" href=\"{repo}\" target=\"_blank\">source code</a> yourself and see that everything is as stated.",
-        "ErrorYTUnavailable": "this youtube video is unavailable, it could be region or age restricted. try another one!",
+        "ErrorYTUnavailable": "this youtube video is unavailable. it could be age or region restricted. try another one!",
-        "ErrorYTTryOtherCodec": "i couldn't find anything to download with your settings. try another codec or quality!\n\nsometimes youtube api acts unexpectedly. try again or try another settings.",
+        "ErrorYTTryOtherCodec": "i couldn't find anything to download with your settings. try another codec or quality in settings!",
         "SettingsCodecSubtitle": "youtube codec",
-        "SettingsCodecDescription": "h264: generally better player support, but quality tops out at 1080p.\nav1: poor player support, but supports 8k & HDR.\nvp9: usually highest bitrate, preserves most detail. supports 4k & HDR.\n\npick h264 if you want best editor/player/social media compatibility.",
+        "SettingsCodecDescription": "h264: best support across apps/platforms, average detail level. max quality is 1080p.\nav1: best quality, small file size, most detail. supports 8k & HDR.\nvp9: same quality as av1, but file is x2 bigger. supports 4k & HDR.\n\npick h264 if you want best compatibility.\n\npick av1 if you want best quality and efficiency.",
         "SettingsAudioDub": "youtube audio track",
         "SettingsAudioDubDescription": "defines which audio track will be used. if dubbed track isn't available, original video language is used instead.\n\noriginal: original video language is used.\nauto: default browser (and cobalt) language is used.",
         "SettingsDubDefault": "original",
@@ -113,7 +113,6 @@
         "SettingsVimeoPreferDescription": "progressive: direct file link to vimeo's cdn. max quality is 1080p.\ndash: video and audio are merged by cobalt into one file. max quality is 4k.\n\npick \"progressive\" if you want best editor/player/social media compatibility. if progressive download isn't available, dash is used instead.",
         "ShareURL": "share",
         "ErrorTweetUnavailable": "couldn't find anything about this tweet. this could be because its visibility is limited. try another one!",
-        "ErrorTwitterRIP": "twitter has restricted access to any content to unauthenticated users. while there's a way to get regular tweets, spaces are, unfortunately, impossible to get at this time. i am looking into possible solutions.",
         "PopupCloseDone": "done",
         "Accessibility": "accessibility",
         "SettingsReduceTransparency": "reduce transparency",

src/localization/languages/ru.json

@@ -102,11 +102,11 @@
         "FollowSupport": "подписывайся на соц.сети кобальта для новостей и поддержки:",
         "SourceCode": "шарься в исходнике, пиши о проблемах, или же форкай репозиторий:",
         "PrivacyPolicy": "политика конфиденциальности кобальта довольно проста: никакие данные о тебе никогда не собираются и не хранятся. нуль, ноль, нада, ничего.\nто, что ты скачиваешь, - твоё личное дело, а не чьё-либо ещё.\n\nесли твоей загрузке требуется рендер, то зашифрованные данные о ней временно хранятся в ОЗУ сервера. это необходимо для работы данной функции.\n\nзашифрованные данные хранятся в течение <span class=\"text-backdrop\">90 секунд</span> и затем безвозвратно удаляются.\n\ncохранённые данные можно расшифровать только с помощью уникальных ключей шифрования из твоей ссылки на скачивание. кроме того, официальная кодовая база кобальта не предусматривает возможности чтения эти данные вне функций обработки.\n\nты всегда можешь посмотреть <a class=\"text-backdrop link\" href=\"{repo}\" target=\"_blank\">исходный код кобальта</a> и убедиться, что всё так, как заявлено.",
-        "ErrorYTUnavailable": "это видео недоступно, возможно оно ограничено по региону или доступу. попробуй другое!",
+        "ErrorYTUnavailable": "это видео недоступно. возможно оно ограничено по доступу или региону. попробуй другое!",
-        "ErrorYTTryOtherCodec": "я не нашёл того, что мог бы скачать с твоими настройками. попробуй другой кодек или качество!",
+        "ErrorYTTryOtherCodec": "я не нашёл того, что мог бы скачать с твоими настройками. попробуй другой кодек или качество в настройках!",
-        "SettingsCodecSubtitle": "кодек для видео с youtube",
+        "SettingsCodecSubtitle": "кодек для youtube видео",
-        "SettingsCodecDescription": "h264: обширная поддержка плеерами, но макс. качество всего лишь 1080p.\nav1: слабая поддержка плеерами, но поддерживает 8k и HDR.\nvp9: обычно наиболее высокий битрейт, лучше сохраняется качество видео. поддерживает 4k и HDR.\n\nвыбирай h264, если тебе нужна наилучшая совместимость с плеерами/редакторами/соцсетями.",
+        "SettingsCodecDescription": "h264: лучшая совместимость, средний уровень детализированности. максимальное качество - 1080p.\nav1: лучшее качество, маленький размер файла, наибольшее количество деталей. поддерживает 8k и HDR.\nvp9: такая же детализированность, как и у av1, но файл в 2 раза больше. поддерживает 4k и HDR.\n\nвыбирай h264, если тебе нужна наилучшая совместимость.\nвыбирай av1, если ты хочешь лучшее качество и эффективность.",
-        "SettingsAudioDub": "звуковая дорожка для видео с youtube",
+        "SettingsAudioDub": "звуковая дорожка для youtube видео",
         "SettingsAudioDubDescription": "определяет, какая звуковая дорожка используется при скачивании видео. если дублированная дорожка недоступна, то вместо неё используется оригинальная.\n\nоригинал: используется оригинальная дорожка.\nавто: используется язык браузера и интерфейса кобальта.",
         "SettingsDubDefault": "оригинал",
         "SettingsDubAuto": "авто",
@@ -114,7 +114,6 @@
         "SettingsVimeoPreferDescription": "progressive: прямая ссылка на файл с сервера vimeo. максимальное качество: 1080p.\ndash: кобальт совмещает видео и аудио в один файл. максимальное качество: 4k.\n\nвыбирай \"progressive\", если тебе нужна наилучшая совместимость с плеерами/редакторами/соцсетями. если \"progressive\" файл недоступен, кобальт скачает \"dash\".",
         "ShareURL": "поделиться",
         "ErrorTweetUnavailable": "не смог найти что-либо об этом твите. возможно его видимость ограничена. попробуй другой!",
-        "ErrorTwitterRIP": "твиттер ограничил доступ к любому контенту на сайте для пользователей без аккаунтов. я нашёл лазейку, чтобы доставать обычные твиты, а для spaces, к сожалению, нет. я ищу возможные варианты выхода из ситуации.",
         "PopupCloseDone": "готово",
         "Accessibility": "общедоступность",
         "SettingsReduceTransparency": "уменьшить прозрачность",

src/modules/processing/services/pinterest.js

@@ -1,22 +1,16 @@
 import { genericUserAgent } from "../../config.js";
 
-const videoLinkBase = {
+const linkRegex = /"url":"(https:\/\/v1.pinimg.com\/videos\/.*?)"/g;
-    "regular": "https://v1.pinimg.com/videos/mc/720p/",
-    "story": "https://v1.pinimg.com/videos/mc/720p/"
-}
 
 export default async function(o) {
-    let id = o.id, type = "regular";
+    let id = o.id;
 
     if (!o.id && o.shortLink) {
         id = await fetch(`https://api.pinterest.com/url_shortener/${o.shortLink}/redirect/`, { redirect: "manual" }).then((r) => {
             return r.headers.get("location").split('pin/')[1].split('/')[0]
         }).catch(() => {});
     }
-    if (id.includes("--")) {
+    if (id.includes("--")) id = id.split("--")[1];
-        id = id.split("--")[1];
-        type = "story";
-    }
     if (!id) return { error: 'ErrorCouldntFetch' };
 
     let html = await fetch(`https://www.pinterest.com/pin/${id}/`, {
@@ -25,11 +19,14 @@ export default async function(o) {
 
     if (!html) return { error: 'ErrorCouldntFetch' };
 
-    let videoLink = html.split(`"url":"${videoLinkBase[type]}`)[1]?.split('"')[0];
+    let videoLink = [...html.matchAll(linkRegex)]
-    if (!html.includes(videoLink)) return { error: 'ErrorEmptyDownload' };
+                    .map(([, link]) => link)
+                    .filter(a => a.endsWith('.mp4') && a.includes('720p'))[0];
+
+    if (!videoLink) return { error: 'ErrorEmptyDownload' };
 
     return {
-        urls: `${videoLinkBase[type]}${videoLink}`,
+        urls: videoLink,
         filename: `pinterest_${o.id}.mp4`,
         audioFilename: `pinterest_${o.id}_audio`
     }

src/modules/processing/services/youtube.js

@@ -23,7 +23,9 @@ const c = {
 }
 
 export default async function(o) {
-    let info, isDubbed, quality = o.quality === "max" ? "9000" : o.quality; //set quality 9000(p) to be interpreted as max
+    let info, isDubbed,
+        quality = o.quality === "max" ? "9000" : o.quality; // 9000(p) - max quality
+
     function qual(i) {
         if (!i.quality_label) {
             return;
@@ -33,7 +35,7 @@ export default async function(o) {
     }
 
     try {
-        info = await yt.getBasicInfo(o.id, 'ANDROID');
+        info = await yt.getBasicInfo(o.id, 'WEB');
     } catch (e) {
         return { error: 'ErrorCantConnectToServiceAPI' };
     }
@@ -43,7 +45,18 @@ export default async function(o) {
     if (info.playability_status.status !== 'OK') return { error: 'ErrorYTUnavailable' };
     if (info.basic_info.is_live) return { error: 'ErrorLiveVideo' };
 
-    let bestQuality, hasAudio, adaptive_formats = info.streaming_data.adaptive_formats.filter(e => 
+    // return a critical error if returned video is "Video Not Available"
+    // or a similar stub by youtube
+    if (info.basic_info.id !== o.id) {
+        return {
+            error: 'ErrorCantConnectToServiceAPI',
+            critical: true
+        }
+    }
+
+    let bestQuality, hasAudio;
+
+    let adaptive_formats = info.streaming_data.adaptive_formats.filter(e => 
         e.mime_type.includes(c[o.format].codec) || e.mime_type.includes(c[o.format].aCodec)
     ).sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
 
@@ -87,16 +100,10 @@ export default async function(o) {
         youtubeDubName: isDubbed ? o.dubLang : false
     }
 
-    if (filenameAttributes.title === "Video Not Available" && filenameAttributes.author === "YouTube Viewers")
-        return {
-            error: 'ErrorCantConnectToServiceAPI',
-            critical: true
-        }
-
     if (hasAudio && o.isAudioOnly) return {
         type: "render",
         isAudioOnly: true,
-        urls: audio.url,
+        urls: audio.decipher(yt.session.player),
         filenameAttributes: filenameAttributes,
         fileMetadata: fileMetadata
     }
@@ -108,14 +115,14 @@ export default async function(o) {
     if (!o.isAudioOnly && !o.isAudioMuted && o.format === 'h264') {
         match = info.streaming_data.formats.find(checkSingle);
         type = "bridge";
-        urls = match?.url;
+        urls = match?.decipher(yt.session.player);
     }
 
     const video = adaptive_formats.find(checkRender);
     if (!match && video) {
         match = video;
         type = "render";
-        urls = [video.url, audio.url];
+        urls = [video.decipher(yt.session.player), audio.decipher(yt.session.player)];
     }
 
     if (match) {

src/modules/processing/servicesConfig.json

@@ -7,6 +7,7 @@
                 "video/:comId", "_shortLink/:comShortLink",
                 "_tv/:lang/video/:tvId", "_tv/video/:tvId"
             ],
+            "subdomains": ["m"],
             "enabled": true
         },
         "reddit": {
@@ -66,7 +67,7 @@
             "enabled": false
         },
         "vimeo": {
-            "patterns": [":id", "video/:id", ":id/:password"],
+            "patterns": [":id", "video/:id", ":id/:password", "/channels/:user/:id"],
             "enabled": true,
             "bestAudio": "mp3"
         },

src/modules/stream/internal.js

@@ -0,0 +1,101 @@
+import { request } from 'undici';
+import { Readable } from 'node:stream';
+import { assert } from 'console';
+import { getHeaders } from './shared.js';
+
+const CHUNK_SIZE = BigInt(8e6); // 8 MB
+const min = (a, b) => a < b ? a : b;
+
+async function* readChunks(streamInfo, size) {
+    let read = 0n;
+    while (read < size) {
+        if (streamInfo.controller.signal.aborted) {
+            throw new Error("controller aborted");
+        }
+
+        const chunk = await request(streamInfo.url, {
+            headers: {
+                ...getHeaders('youtube'),
+                Range: `bytes=${read}-${read + CHUNK_SIZE}`
+            },
+            signal: streamInfo.controller.signal
+        });
+
+        const expected = min(CHUNK_SIZE, size - read);
+        const received = BigInt(chunk.headers['content-length']);
+
+        if (received < expected / 2n) {
+            streamInfo.controller.abort();
+        }
+        
+        for await (const data of chunk.body) {
+            yield data;
+        }
+
+        read += received;
+    }
+} 
+
+function chunkedStream(streamInfo, size) {
+    assert(streamInfo.controller instanceof AbortController);
+    const stream = Readable.from(readChunks(streamInfo, size));
+    return stream;
+}
+
+async function handleYoutubeStream(streamInfo, res) {
+    try {
+        const req = await fetch(streamInfo.url, {
+            headers: getHeaders('youtube'),
+            method: 'HEAD',
+            signal: streamInfo.controller.signal
+        });
+
+        streamInfo.url = req.url;
+        const size = BigInt(req.headers.get('content-length'));
+
+        if (req.status !== 200 || !size)
+            return res.destroy();
+
+        const stream = chunkedStream(streamInfo, size);
+
+        for (const headerName of ['content-type', 'content-length']) {
+            const headerValue = req.headers.get(headerName);
+            if (headerValue) res.setHeader(headerName, headerValue);
+        }
+
+        stream.pipe(res);
+        stream.on('error', () => res.destroy());
+    } catch {
+        res.destroy();
+    }
+}
+
+export async function internalStream(streamInfo, res) {
+    if (streamInfo.service === 'youtube') {
+        return handleYoutubeStream(streamInfo, res);
+    }
+
+    try {
+        const req = await request(streamInfo.url, {
+            headers: {
+                ...streamInfo.headers,
+                host: undefined
+            },
+            signal: streamInfo.controller.signal,
+            maxRedirections: 16
+        });
+
+        res.status(req.statusCode);
+
+        for (const [ name, value ] of Object.entries(req.headers))
+            res.setHeader(name, value)
+
+        if (req.statusCode < 200 || req.statusCode > 299)
+            return res.destroy();
+
+        req.body.pipe(res);
+        req.body.on('error', () => res.destroy());
+    } catch {
+        streamInfo.controller.abort();
+    }
+}

src/modules/stream/manage.js

@@ -4,6 +4,7 @@ import { nanoid } from 'nanoid';
 
 import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js";
 import { streamLifespan } from "../config.js";
+import { strict as assert } from "assert";
 
 const streamNoAccess = {
     error: "i couldn't verify if you have access to this stream. go back and try again!",
@@ -24,6 +25,7 @@ streamCache.on("expired", (key) => {
     streamCache.del(key);
 })
 
+const internalStreamCache = {};
 const hmacSalt = randomBytes(64).toString('hex');
 
 export function createStream(obj) {
@@ -67,6 +69,35 @@ export function createStream(obj) {
     return streamLink.toString();
 }
 
+export function getInternalStream(id) {
+    return internalStreamCache[id];
+}
+
+export function createInternalStream(obj = {}) {
+    assert(typeof obj.url === 'string');
+
+    const streamID = nanoid();
+    internalStreamCache[streamID] = {
+        url: obj.url,
+        service: obj.service,
+        controller: new AbortController()
+    };
+
+    let streamLink = new URL('/api/istream', `http://127.0.0.1:${process.env.API_PORT}`);
+    streamLink.searchParams.set('t', streamID);
+    return streamLink.toString();
+}
+
+export function destroyInternalStream(url) {
+    const id = new URL(url).searchParams.get('t');
+    assert(id);
+
+    if (internalStreamCache[id]) {
+        internalStreamCache[id].controller.abort();
+        delete internalStreamCache[id];
+    }
+}
+
 export function verifyStream(id, hmac, exp, secret, iv) {
     try {
         const ghmac = generateHmac(`${id},${exp},${iv},${secret}`, hmacSalt);
@@ -82,9 +113,27 @@ export function verifyStream(id, hmac, exp, secret, iv) {
         if (Number(exp) <= new Date().getTime())
             return streamNoExist;
 
+        if (!streamInfo.originalUrls) {
+            streamInfo.originalUrls = streamInfo.urls;
+        }        
+
+        if (typeof streamInfo.originalUrls === 'string') {
+            streamInfo.urls = createInternalStream({
+                url: streamInfo.originalUrls,
+                ...streamInfo
+            });
+        } else if (Array.isArray(streamInfo.originalUrls)) {
+            for (const idx in streamInfo.originalUrls) {
+                streamInfo.originalUrls[idx] = createInternalStream({
+                    url: streamInfo.originalUrls[idx],
+                    ...streamInfo
+                });
+            }
+        } else throw 'invalid urls';
+
         return streamInfo;
     }
-    catch (e) {
+    catch {
         return {
             error: "something went wrong and i couldn't verify this stream. go back and try again!",
             status: 500

src/modules/stream/shared.js

@@ -0,0 +1,21 @@
+import { genericUserAgent } from "../config.js";
+
+const defaultHeaders = {
+    'user-agent': genericUserAgent
+}
+
+const serviceHeaders = {
+    bilibili: {
+        referer: 'https://www.bilibili.com/'
+    },
+    youtube: {
+        accept: '*/*',
+        origin: 'https://www.youtube.com',
+        referer: 'https://www.youtube.com',
+        DNT: '?1'
+    }
+}
+
+export function getHeaders(service) {
+    return { ...defaultHeaders, ...serviceHeaders[service] }
+}

src/modules/stream/stream.js

@@ -1,4 +1,5 @@
 import { streamAudioOnly, streamDefault, streamLiveRender, streamVideoOnly, convertToGif } from "./types.js";
+import { internalStream } from './internal.js';
 
 export default async function(res, streamInfo) {
     try {
@@ -7,6 +8,8 @@ export default async function(res, streamInfo) {
             return;
         }
         switch (streamInfo.type) {
+            case "internal":
+                return await internalStream(streamInfo, res);
             case "render":
                 await streamLiveRender(streamInfo, res);
                 break;
@@ -21,7 +24,7 @@ export default async function(res, streamInfo) {
                 await streamDefault(streamInfo, res);
                 break;
         }
-    } catch (e) {
+    } catch {
         res.status(500).json({ status: "error", text: "Internal Server Error" });
     }
 }

src/modules/stream/types.js

@@ -1,10 +1,19 @@
-import { spawn } from "child_process";
-import ffmpeg from "ffmpeg-static";
-import { ffmpegArgs, genericUserAgent } from "../config.js";
-import { metadataManager } from "../sub/utils.js";
 import { request } from "undici";
+import ffmpeg from "ffmpeg-static";
+import { spawn } from "child_process";
 import { create as contentDisposition } from "content-disposition-header";
 
+import { metadataManager } from "../sub/utils.js";
+import { destroyInternalStream } from "./manage.js";
+import { ffmpegArgs } from "../config.js";
+import { getHeaders } from "./shared.js";
+
+function toRawHeaders(headers) {
+    return Object.entries(headers)
+                 .map(([key, value]) => `${key}: ${value}\r\n`)
+                 .join('');
+}
+
 function closeRequest(controller) {
     try { controller.abort() } catch {}
 }
@@ -43,7 +52,11 @@ function getCommand(args) {
 
 export async function streamDefault(streamInfo, res) {
     const abortController = new AbortController();
-    const shutdown = () => (closeRequest(abortController), closeResponse(res));
+    const shutdown = () => (
+        closeRequest(abortController),
+        closeResponse(res),
+        destroyInternalStream(streamInfo.urls)
+    );
 
     try {
         let filename = streamInfo.filename;
@@ -53,13 +66,16 @@ export async function streamDefault(streamInfo, res) {
         res.setHeader('Content-disposition', contentDisposition(filename));
 
         const { body: stream, headers } = await request(streamInfo.urls, {
-            headers: { 'user-agent': genericUserAgent },
+            headers: getHeaders(streamInfo.service),
             signal: abortController.signal,
             maxRedirections: 16
         });
 
-        res.setHeader('content-type', headers['content-type']);
+        for (const headerName of ['content-type', 'content-length']) {
-        res.setHeader('content-length', headers['content-length']);
+            if (headers[headerName]) {
+                res.setHeader(headerName, headers[headerName]);
+            }
+        }
 
         pipe(stream, res, shutdown);
     } catch {
@@ -67,68 +83,52 @@ export async function streamDefault(streamInfo, res) {
     }
 }
 
-export async function streamLiveRender(streamInfo, res) {
+export function streamLiveRender(streamInfo, res) {
-    let abortController = new AbortController(), process;
+    let process;
     const shutdown = () => (
-        closeRequest(abortController),
         killProcess(process),
-        closeResponse(res)
+        closeResponse(res),
+        streamInfo.urls.map(destroyInternalStream)
     );
 
+    const headers = getHeaders(streamInfo.service);
+    const rawHeaders = toRawHeaders(headers);
+
     try {
         if (streamInfo.urls.length !== 2) return shutdown();
 
-        const { body: audio } = await request(streamInfo.urls[1], {
-            maxRedirections: 16, signal: abortController.signal,
-            headers: {
-                'user-agent': genericUserAgent,
-                referer: streamInfo.service === 'bilibili'
-                            ? 'https://www.bilibili.com/'
-                            : undefined,
-            }
-        });
-
         const format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1];
+
         let args = [
             '-loglevel', '-8',
-            '-user_agent', genericUserAgent
+            '-headers', rawHeaders,
-        ];
-
-        if (streamInfo.service === 'bilibili') {
-            args.push(
-                '-headers', 'Referer: https://www.bilibili.com/\r\n',
-            )
-        }
-
-        args.push(
             '-i', streamInfo.urls[0],
-            '-i', 'pipe:3',
+            '-i', streamInfo.urls[1],
             '-map', '0:v',
             '-map', '1:a',
-        );
+        ]
 
         args = args.concat(ffmpegArgs[format]);
+
         if (streamInfo.metadata) {
             args = args.concat(metadataManager(streamInfo.metadata))
         }
-        args.push('-f', format, 'pipe:4');
+
+        args.push('-f', format, 'pipe:3');
 
         process = spawn(...getCommand(args), {
             windowsHide: true,
             stdio: [
                 'inherit', 'inherit', 'inherit',
-                'pipe', 'pipe'
+                'pipe'
             ],
         });
 
-        const [,,, audioInput, muxOutput] = process.stdio;
+        const [,,, muxOutput] = process.stdio;
 
         res.setHeader('Connection', 'keep-alive');
         res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename));
 
-        audio.on('error', shutdown);
-        audioInput.on('error', shutdown);
-        audio.pipe(audioInput);
         pipe(muxOutput, res, shutdown);
 
         process.on('close', shutdown);
@@ -140,18 +140,20 @@ export async function streamLiveRender(streamInfo, res) {
 
 export function streamAudioOnly(streamInfo, res) {
     let process;
-    const shutdown = () => (killProcess(process), closeResponse(res));
+    const shutdown = () => (
+        killProcess(process),
+        closeResponse(res),
+        destroyInternalStream(streamInfo.urls)
+    );
 
     try {
         let args = [
             '-loglevel', '-8',
-            '-user_agent', genericUserAgent
+            '-headers', toRawHeaders(getHeaders(streamInfo.service)),
-        ];
+        ]
 
         if (streamInfo.service === "twitter") {
             args.push('-seekable', '0');
-        } else if (streamInfo.service === 'bilibili') {
-            args.push('-headers', 'Referer: https://www.bilibili.com/\r\n');
         }
 
         args.push(
@@ -162,12 +164,12 @@ export function streamAudioOnly(streamInfo, res) {
         if (streamInfo.metadata) {
             args = args.concat(metadataManager(streamInfo.metadata))
         }
-        let arg = streamInfo.copy ? ffmpegArgs["copy"] : ffmpegArgs["audio"];
-        args = args.concat(arg);
 
+        args = args.concat(ffmpegArgs[streamInfo.copy ? 'copy' : 'audio']);
         if (ffmpegArgs[streamInfo.audioFormat]) {
             args = args.concat(ffmpegArgs[streamInfo.audioFormat])
         }
+
         args.push('-f', streamInfo.audioFormat === "m4a" ? "ipod" : streamInfo.audioFormat, 'pipe:3');
 
         process = spawn(...getCommand(args), {
@@ -192,17 +194,20 @@ export function streamAudioOnly(streamInfo, res) {
 
 export function streamVideoOnly(streamInfo, res) {
     let process;
-    const shutdown = () => (killProcess(process), closeResponse(res));
+    const shutdown = () => (
+        killProcess(process),
+        closeResponse(res),
+        destroyInternalStream(streamInfo.urls)
+    );
 
     try {
         let args = [
-            '-loglevel', '-8'
+            '-loglevel', '-8',
+            '-headers', toRawHeaders(getHeaders(streamInfo.service)),
         ]
 
         if (streamInfo.service === "twitter") {
             args.push('-seekable', '0')
-        } else if (streamInfo.service === 'bilibili') {
-            args.push('-headers', 'Referer: https://www.bilibili.com/\r\n')
         }
 
         args.push(
@@ -222,6 +227,7 @@ export function streamVideoOnly(streamInfo, res) {
         if (format === "mp4") {
             args.push('-movflags', 'faststart+frag_keyframe+empty_moov')
         }
+
         args.push('-f', format, 'pipe:3');
 
         process = spawn(...getCommand(args), {
@@ -254,10 +260,12 @@ export function convertToGif(streamInfo, res) {
         let args = [
             '-loglevel', '-8'
         ]
+
         if (streamInfo.service === "twitter") {
             args.push('-seekable', '0')
         }
-        args.push('-i', streamInfo.urls)
+
+        args.push('-i', streamInfo.urls);
         args = args.concat(ffmpegArgs["gif"]);
         args.push('-f', "gif", 'pipe:3');