From 9899f778a9221fa82fe36a7671f453e098070ab4 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Mon, 25 Dec 2023 12:38:51 +0000 Subject: [PATCH 01/51] package.json: use punycode.js version of psl from git --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e839fe3..a191465 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "hls-parser": "^0.10.7", "nanoid": "^4.0.2", "node-cache": "^5.1.2", - "psl": "1.9.0", + "psl": "https://github.com/lupomontero/psl#5eadae91361d8289d582700f90582b0d0cb73155", "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", From cbfcfcfa18a1ebdc85cb283a6d670d68798e9832 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Wed, 21 Feb 2024 11:48:00 +0000 Subject: [PATCH 02/51] tumblr: rewrite & fix audio support closes #342 --- src/modules/processing/matchActionDecider.js | 12 ++-- src/modules/processing/services/tumblr.js | 74 +++++++++++++++----- src/test/tests.json | 2 +- 3 files changed, 62 insertions(+), 26 deletions(-) diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index 7aa154b..2d8840a 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -143,7 +143,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di const isBestHostAudio = services[host]["bestAudio"] && (audioFormat === services[host]["bestAudio"]); const isTikTok = host === "tiktok" || host === "douyin"; - const isTumblr = host === "tumblr" && !r.filename; + const isTumblrAudio = host === "tumblr" && !r.filename; const isSoundCloud = host === "soundcloud"; if (isTikTok && services.tiktok.audioFormats.includes(audioFormat)) { @@ -168,11 +168,6 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di } } - if (isTumblr && isBestOrMp3) { - audioFormat = "mp3"; - processType = "bridge" - } - if (isBestAudioDefined || isBestHostAudio) { audioFormat = services[host]["bestAudio"]; processType = "bridge"; @@ -181,6 +176,11 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di copy = true } + if (isTumblrAudio && isBestOrMp3) { + audioFormat = "mp3"; + processType = "bridge" + } + if (r.isM3U8 || host === "vimeo") { copy = false; processType = "render" diff --git a/src/modules/processing/services/tumblr.js b/src/modules/processing/services/tumblr.js index 75d354e..05c7fd8 100644 --- a/src/modules/processing/services/tumblr.js +++ b/src/modules/processing/services/tumblr.js @@ -1,8 +1,26 @@ import psl from "psl"; import { genericUserAgent } from "../../config.js"; -export default async function(obj) { - let { subdomain } = psl.parse(obj.url.hostname); +const API_KEY = 'jrsCWX1XDuVxAFO4GkK147syAoN8BJZ5voz8tS80bPcj26Vc5Z'; +const API_BASE = 'https://api-http2.tumblr.com'; + +function request(domain, id) { + const url = new URL(`/v2/blog/${domain}/posts/${id}/permalink`, API_BASE); + url.searchParams.set('api_key', API_KEY); + url.searchParams.set('fields[blogs]', 'uuid,name,avatar,?description,?can_message,?can_be_followed,?is_adult,?reply_conditions,' + + '?theme,?title,?url,?is_blocked_from_primary,?placement_id,?primary,?updated,?followed,' + + '?ask,?can_subscribe,?paywall_access,?subscription_plan,?is_blogless_advertiser,?tumblrmart_accessories'); + + return fetch(url, { + headers: { + 'User-Agent': 'Tumblr/iPhone/33.3/333010/17.3.1/tumblr', + 'X-Version': 'iPhone/33.3/333010/17.3.1/tumblr' + } + }).then(a => a.json()).catch(() => {}); +} + +export default async function(input) { + let { subdomain } = psl.parse(input.url.hostname); if (subdomain?.includes('.')) { return { error: ['ErrorBrokenLink', 'tumblr'] } @@ -10,26 +28,44 @@ export default async function(obj) { subdomain = undefined } - let html = await fetch(`https://${subdomain ?? obj.user}.tumblr.com/post/${obj.id}`, { - headers: { "user-agent": genericUserAgent } - }).then((r) => { return r.text() }).catch(() => { return false }); + const domain = `${subdomain ?? input.user}.tumblr.com`; + const data = await request(domain, input.id); - if (!html) return { error: 'ErrorCouldntFetch' }; + const element = data?.response?.timeline?.elements?.[0]; + if (!element) return { error: 'ErrorEmptyDownload' }; - let r; - if (html.includes('property="og:video" content="https://va.media.tumblr.com/')) { - r = { - urls: `https://va.media.tumblr.com/${html.split('property="og:video" content="https://va.media.tumblr.com/')[1].split('"')[0]}`, - filename: `tumblr_${obj.id}.mp4`, - audioFilename: `tumblr_${obj.id}_audio` - } - } else if (html.includes('property="og:audio" content="https://a.tumblr.com/')) { - r = { - urls: `https://a.tumblr.com/${html.split('property="og:audio" content="https://a.tumblr.com/')[1].split('"')[0]}`, - audioFilename: `tumblr_${obj.id}`, + const contents = [ + ...element.content, + ...element?.trail?.map(t => t.content).flat() + ] + + const audio = contents.find(c => c.type === 'audio'); + if (audio && audio.provider === 'tumblr') { + const fileMetadata = { + title: audio?.title, + artist: audio?.artist + }; + + return { + urls: audio.media.url, + filenameAttributes: { + service: 'tumblr', + id: input.id, + title: fileMetadata.title, + author: fileMetadata.artist + }, isAudioOnly: true } - } else r = { error: 'ErrorEmptyDownload' }; + } - return r + const video = contents.find(c => c.type === 'video'); + if (video && video.provider === 'tumblr') { + return { + urls: video.media.url, + filename: `tumblr_${input.id}.mp4`, + audioFilename: `tumblr_${input.id}_audio` + } + } + + return { error: 'ErrorEmptyDownload' } } diff --git a/src/test/tests.json b/src/test/tests.json index a298d15..3b9a9ed 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -773,7 +773,7 @@ } }, { "name": "tumblr audio", - "url": "https://rf9weu8hjf789234hf9.tumblr.com/post/172006661342/everyone-thats-made-a-video-out-of-this-without", + "url": "https://www.tumblr.com/zedneon/737815079301562368/zedneon-ft-mr-sauceman-tech-n9ne-speed-of?source=share", "params": {}, "expected": { "code": 200, From 06b808852646503e53c696e3df80b128a3f1537c Mon Sep 17 00:00:00 2001 From: DrWarpMan <36279265+DrWarpMan@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:17:19 +0100 Subject: [PATCH 03/51] cookies: add all necessary instagram cookies to example Signed-off-by: DrWarpMan <36279265+DrWarpMan@users.noreply.github.com> --- docs/examples/cookies.example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/cookies.example.json b/docs/examples/cookies.example.json index faaeb56..9b488c2 100644 --- a/docs/examples/cookies.example.json +++ b/docs/examples/cookies.example.json @@ -1,5 +1,5 @@ { "instagram": [ - "mid=replace; ig_did=this; csrftoken=cookie" + "mid=; ig_did=; csrftoken=; ds_user_id=; sessionid=" ] } From 6227b7a38a6f2036c650dffacea2cb583f131327 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Wed, 21 Feb 2024 20:23:54 +0000 Subject: [PATCH 04/51] cookies: add example for reddit --- docs/examples/cookies.example.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/examples/cookies.example.json b/docs/examples/cookies.example.json index 9b488c2..5ebdb63 100644 --- a/docs/examples/cookies.example.json +++ b/docs/examples/cookies.example.json @@ -1,5 +1,8 @@ { "instagram": [ "mid=; ig_did=; csrftoken=; ds_user_id=; sessionid=" + ], + "reddit": [ + "client_id=; client_secret=; refresh_token=" ] } From d70754238ee6712a7ffd634b7165bc1f26ecc9b3 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Thu, 15 Feb 2024 01:27:19 +0000 Subject: [PATCH 05/51] stream: fix bilibili downloads fixes #302 --- src/modules/stream/types.js | 43 ++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index cdfb4a0..7dcfd74 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -80,17 +80,33 @@ export async function streamLiveRender(streamInfo, res) { if (streamInfo.urls.length !== 2) return shutdown(); const { body: audio } = await request(streamInfo.urls[1], { - maxRedirections: 16, signal: abortController.signal + maxRedirections: 16, signal: abortController.signal, + headers: { + 'user-agent': genericUserAgent, + referer: streamInfo.service === 'bilibili' + ? 'https://www.bilibili.com/' + : undefined, + } }); - let format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1], - args = [ + const format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1]; + let args = [ '-loglevel', '-8', + '-user_agent', genericUserAgent + ]; + + if (streamInfo.service === 'bilibili') { + args.push( + '-headers', 'Referer: https://www.bilibili.com/\r\n', + ) + } + + args.push( '-i', streamInfo.urls[0], '-i', 'pipe:3', '-map', '0:v', '-map', '1:a', - ]; + ); args = args.concat(ffmpegArgs[format]); if (streamInfo.metadata) { @@ -129,11 +145,16 @@ export function streamAudioOnly(streamInfo, res) { try { let args = [ - '-loglevel', '-8' - ] + '-loglevel', '-8', + '-user_agent', genericUserAgent + ]; + if (streamInfo.service === "twitter") { - args.push('-seekable', '0') + args.push('-seekable', '0'); + } else if (streamInfo.service === 'bilibili') { + args.push('-headers', 'Referer: https://www.bilibili.com/\r\n'); } + args.push( '-i', streamInfo.urls, '-vn' @@ -178,17 +199,23 @@ export function streamVideoOnly(streamInfo, res) { let args = [ '-loglevel', '-8' ] + if (streamInfo.service === "twitter") { args.push('-seekable', '0') + } else if (streamInfo.service === 'bilibili') { + args.push('-headers', 'Referer: https://www.bilibili.com/\r\n') } + args.push( '-i', streamInfo.urls, '-c', 'copy' ) + if (streamInfo.mute) { args.push('-an') } - if (streamInfo.service === "vimeo" || streamInfo.service === "rutube") { + + if (['vimeo', 'rutube'].includes(streamInfo.service)) { args.push('-bsf:a', 'aac_adtstoasc') } From 6e1eddad82cc8fe21183e61f29fbf7f8ca3aa782 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Thu, 15 Feb 2024 01:42:15 +0000 Subject: [PATCH 06/51] bilibili: add support for b23.tv links resolves #320 --- src/modules/processing/match.js | 4 +--- src/modules/processing/services/bilibili.js | 23 +++++++++++++++---- src/modules/processing/servicesConfig.json | 2 +- .../processing/servicesPatternTesters.js | 2 +- src/modules/processing/url.js | 8 +++++++ src/test/tests.json | 8 +++++++ 6 files changed, 38 insertions(+), 9 deletions(-) diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index cc2516f..95558c8 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -56,9 +56,7 @@ export default async function(host, patternMatch, url, lang, obj) { }); break; case "bilibili": - r = await bilibili({ - id: patternMatch.id.slice(0, 12) - }); + r = await bilibili(patternMatch); break; case "youtube": let fetchInfo = { diff --git a/src/modules/processing/services/bilibili.js b/src/modules/processing/services/bilibili.js index 0194ee4..5664e8c 100644 --- a/src/modules/processing/services/bilibili.js +++ b/src/modules/processing/services/bilibili.js @@ -1,8 +1,23 @@ import { genericUserAgent, maxVideoDuration } from "../../config.js"; // TO-DO: quality picking, bilibili.tv support, and higher quality downloads (currently requires an account) -export default async function(obj) { - let html = await fetch(`https://bilibili.com/video/${obj.id}`, { +export default async function({ id, shortLink }) { + if (shortLink) { + id = await fetch(`https://b23.tv/${shortLink}`, { redirect: 'manual' }) + .then(r => r.status > 300 && r.status < 400 && r.headers.get('location')) + .then(url => { + const path = new URL(url).pathname; + if (path.startsWith('/video/')) + return path.split('/')[2]; + }) + .catch(() => {}) + } + + if (!id) { + return { error: 'ErrorCouldntFetch' }; + } + + let html = await fetch(`https://bilibili.com/video/${id}`, { headers: { "user-agent": genericUserAgent } }).then((r) => { return r.text() }).catch(() => { return false }); if (!html) return { error: 'ErrorCouldntFetch' }; @@ -21,7 +36,7 @@ export default async function(obj) { return { urls: [video[0]["baseUrl"], audio[0]["baseUrl"]], - audioFilename: `bilibili_${obj.id}_audio`, - filename: `bilibili_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.mp4` + audioFilename: `bilibili_${id}_audio`, + filename: `bilibili_${id}_${video[0]["width"]}x${video[0]["height"]}.mp4` }; } diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index 804f597..bec1dab 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -3,7 +3,7 @@ "config": { "bilibili": { "alias": "bilibili.com videos", - "patterns": ["video/:id"], + "patterns": ["video/:id", "_shortLink/:shortLink"], "enabled": true }, "reddit": { diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index 970e8f4..f341844 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -1,6 +1,6 @@ export const testers = { "bilibili": (patternMatch) => - patternMatch.id?.length <= 12, + patternMatch.id?.length <= 12 || patternMatch.shortLink?.length <= 16, "instagram": (patternMatch) => patternMatch.postId?.length <= 12 diff --git a/src/modules/processing/url.js b/src/modules/processing/url.js index 9c87889..753e9d5 100644 --- a/src/modules/processing/url.js +++ b/src/modules/processing/url.js @@ -16,6 +16,7 @@ export function aliasURL(url) { url.search = `?v=${encodeURIComponent(parts[2])}` } break; + case "youtu": if (url.hostname === 'youtu.be' && parts.length >= 2) { /* youtu.be urls can be weird, e.g. https://youtu.be///asdasd// still works @@ -25,6 +26,7 @@ export function aliasURL(url) { }`) } break; + case "pin": if (url.hostname === 'pin.it' && parts.length === 2) { url = new URL(`https://pinterest.com/url_shortener/${ @@ -46,6 +48,12 @@ export function aliasURL(url) { url = new URL(`https://twitch.tv/_/clip/${parts[1]}`); } break; + + case "b23": + if (url.hostname === 'b23.tv' && parts.length === 2) { + url = new URL(`https://bilibili.com/_shortLink/${parts[1]}`) + } + break; } return url diff --git a/src/test/tests.json b/src/test/tests.json index a298d15..3161b00 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -746,6 +746,14 @@ "code": 200, "status": "stream" } + }, { + "name": "b23.tv shortlink", + "url": "https://b23.tv/lbMyOI9", + "params": {}, + "expected": { + "code": 200, + "status": "stream" + } }], "tumblr": [{ "name": "at.tumblr link", From 0852ade1be4c84a188e35f8ee2b962fde35292e7 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Thu, 15 Feb 2024 18:11:18 +0000 Subject: [PATCH 07/51] bilibili: add support for bilibili.tv links closes #319 --- src/modules/processing/services/bilibili.js | 101 ++++++++++++++---- src/modules/processing/servicesConfig.json | 7 +- .../processing/servicesPatternTesters.js | 5 +- src/modules/processing/url.js | 5 + src/test/tests.json | 9 ++ 5 files changed, 104 insertions(+), 23 deletions(-) diff --git a/src/modules/processing/services/bilibili.js b/src/modules/processing/services/bilibili.js index 5664e8c..6da110b 100644 --- a/src/modules/processing/services/bilibili.js +++ b/src/modules/processing/services/bilibili.js @@ -1,42 +1,105 @@ import { genericUserAgent, maxVideoDuration } from "../../config.js"; -// TO-DO: quality picking, bilibili.tv support, and higher quality downloads (currently requires an account) -export default async function({ id, shortLink }) { - if (shortLink) { - id = await fetch(`https://b23.tv/${shortLink}`, { redirect: 'manual' }) +// TO-DO: higher quality downloads (currently requires an account) + +function com_resolveShortlink(shortId) { + return fetch(`https://b23.tv/${shortId}`, { redirect: 'manual' }) .then(r => r.status > 300 && r.status < 400 && r.headers.get('location')) .then(url => { + if (!url) return; const path = new URL(url).pathname; if (path.startsWith('/video/')) return path.split('/')[2]; }) .catch(() => {}) - } +} - if (!id) { - return { error: 'ErrorCouldntFetch' }; - } +function getBest(content) { + return content?.filter(v => v.baseUrl || v.url) + .map(v => (v.baseUrl = v.baseUrl || v.url, v)) + .reduce((a, b) => a?.bandwidth > b?.bandwidth ? a : b); +} +function extractBestQuality(dashData) { + const bestVideo = getBest(dashData.video), + bestAudio = getBest(dashData.audio); + + if (!bestVideo || !bestAudio) return []; + return [ bestVideo, bestAudio ]; +} + +async function com_download(id) { let html = await fetch(`https://bilibili.com/video/${id}`, { headers: { "user-agent": genericUserAgent } }).then((r) => { return r.text() }).catch(() => { return false }); if (!html) return { error: 'ErrorCouldntFetch' }; - if (!(html.includes('')[0]); - if (streamData.data.timelength > maxVideoDuration) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + if (streamData.data.timelength > maxVideoDuration) { + return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + } - let video = streamData["data"]["dash"]["video"].filter(v => - !v["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/") - ).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth)); - - let audio = streamData["data"]["dash"]["audio"].filter(a => - !a["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/") - ).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth)); + const [ video, audio ] = extractBestQuality(streamData.data.dash); + if (!video || !audio) { + return { error: 'ErrorEmptyDownload' }; + } return { - urls: [video[0]["baseUrl"], audio[0]["baseUrl"]], + urls: [video.baseUrl, audio.baseUrl], audioFilename: `bilibili_${id}_audio`, - filename: `bilibili_${id}_${video[0]["width"]}x${video[0]["height"]}.mp4` + filename: `bilibili_${id}_${video.width}x${video.height}.mp4` }; } + +async function tv_download(id) { + const url = new URL( + 'https://api.bilibili.tv/intl/gateway/web/playurl' + + '?s_locale=en_US&platform=web&qn=64&type=0&device=wap' + + '&tf=0&spm_id=bstar-web.ugc-video-detail.0.0&from_spm_id=' + ); + + url.searchParams.set('aid', id); + + const { data } = await fetch(url).then(a => a.json()); + if (!data?.playurl?.video) { + return { error: 'ErrorEmptyDownload' }; + } + + const [ video, audio ] = extractBestQuality({ + video: data.playurl.video.map(s => s.video_resource) + .filter(s => s.codecs.includes('avc1')), + audio: data.playurl.audio_resource + }); + + if (!video || !audio) { + return { error: 'ErrorEmptyDownload' }; + } + + if (video.duration > maxVideoDuration) { + return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + } + + return { + urls: [video.url, audio.url], + audioFilename: `bilibili_tv_${id}_audio`, + filename: `bilibili_tv_${id}.mp4` + }; +} + +export default async function({ comId, tvId, comShortLink }) { + if (comShortLink) { + comId = await com_resolveShortlink(comShortLink); + } + + if (comId) { + return com_download(comId); + } else if (tvId) { + return tv_download(tvId); + } + + return { error: 'ErrorCouldntFetch' }; +} diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index bec1dab..d6e137e 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -2,8 +2,11 @@ "audioIgnore": ["vk", "ok"], "config": { "bilibili": { - "alias": "bilibili.com videos", - "patterns": ["video/:id", "_shortLink/:shortLink"], + "alias": "bilibili.com and bilibili.tv videos", + "patterns": [ + "video/:comId", "_shortLink/:comShortLink", + "_tv/:lang/video/:tvId", "_tv/video/:tvId" + ], "enabled": true }, "reddit": { diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index f341844..30892f6 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -1,6 +1,7 @@ export const testers = { - "bilibili": (patternMatch) => - patternMatch.id?.length <= 12 || patternMatch.shortLink?.length <= 16, + "bilibili": (patternMatch) => + patternMatch.comId?.length <= 12 || patternMatch.comShortLink?.length <= 16 + || patternMatch.tvId?.length <= 24, "instagram": (patternMatch) => patternMatch.postId?.length <= 12 diff --git a/src/modules/processing/url.js b/src/modules/processing/url.js index 753e9d5..5e6bd15 100644 --- a/src/modules/processing/url.js +++ b/src/modules/processing/url.js @@ -49,6 +49,11 @@ export function aliasURL(url) { } break; + case "bilibili": + if (host.tld === 'tv') { + url = new URL(`https://bilibili.com/_tv${url.pathname}`); + } + break; case "b23": if (url.hostname === 'b23.tv' && parts.length === 2) { url = new URL(`https://bilibili.com/_shortLink/${parts[1]}`) diff --git a/src/test/tests.json b/src/test/tests.json index 3161b00..42f75f4 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -754,6 +754,15 @@ "code": 200, "status": "stream" } + }, + { + "name": "bilibili.tv link", + "url": "https://www.bilibili.tv/en/video/4789599404426256", + "params": {}, + "expected": { + "code": 200, + "status": "stream" + } }], "tumblr": [{ "name": "at.tumblr link", From decedd77369a573016d662b3ea8228f9b3898606 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 5 Mar 2024 12:39:49 +0600 Subject: [PATCH 08/51] package: use latest version of undici --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a191465..dfbe009 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "node-cache": "^5.1.2", "psl": "https://github.com/lupomontero/psl#5eadae91361d8289d582700f90582b0d0cb73155", "set-cookie-parser": "2.6.0", - "undici": "^5.19.1", + "undici": "^6.7.0", "url-pattern": "1.0.3", "youtubei.js": "^6.4.1" } From b1bc7126a49ab7e5d06684cf9627d62c266772ac Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 5 Mar 2024 13:38:46 +0600 Subject: [PATCH 09/51] servicesConfig: update bilibili alias Signed-off-by: wukko --- src/modules/processing/servicesConfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index d6e137e..5277053 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -2,7 +2,7 @@ "audioIgnore": ["vk", "ok"], "config": { "bilibili": { - "alias": "bilibili.com and bilibili.tv videos", + "alias": "bilibili videos", "patterns": [ "video/:comId", "_shortLink/:comShortLink", "_tv/:lang/video/:tvId", "_tv/video/:tvId" From e282a9183fcecfcf2a969cc9fe47927b3ccc7a32 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 5 Mar 2024 18:14:26 +0600 Subject: [PATCH 10/51] stream: encrypt cached stream data & clean up related modules also limited CORS methods to GET and POST --- src/core/api.js | 26 ++++++++++---- src/modules/stream/manage.js | 69 +++++++++++++++++++++++------------- src/modules/sub/crypto.js | 25 +++++++++++-- src/modules/sub/utils.js | 1 - 4 files changed, 88 insertions(+), 33 deletions(-) diff --git a/src/core/api.js b/src/core/api.js index 5f91031..8b3dec0 100644 --- a/src/core/api.js +++ b/src/core/api.js @@ -51,7 +51,11 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { app.set('trust proxy', ['loopback', 'uniquelocal']); - app.use('/api/:type', cors(corsConfig)); + app.use('/api/:type', cors({ + methods: ['GET', 'POST'], + ...corsConfig + })); + app.use('/api/json', apiLimiter); app.use('/api/stream', apiLimiterStream); app.use('/api/onDemand', apiLimiter); @@ -60,6 +64,7 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { try { decodeURIComponent(req.path) } catch (e) { return res.redirect('/') } next(); }); + app.use('/api/json', express.json({ verify: (req, res, buf) => { let acceptCon = String(req.header('Accept')) === "application/json"; @@ -71,6 +76,7 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { } } })); + // handle express.json errors properly (https://github.com/expressjs/express/issues/4065) app.use('/api/json', (err, req, res, next) => { let errorText = "invalid json body"; @@ -86,6 +92,7 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { next(); } }); + app.post('/api/json', async (req, res) => { try { let lang = languageCode(req); @@ -118,13 +125,17 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { try { switch (req.params.type) { case 'stream': - if (req.query.t && req.query.h && req.query.e && req.query.t.toString().length === 21 - && req.query.h.toString().length === 64 && req.query.e.toString().length === 13) { - let streamInfo = verifyStream(req.query.t, req.query.h, req.query.e); + const q = req.query; + const checkQueries = q.t && q.e && q.h && q.s && q.i; + const checkBaseLength = q.t.length === 21 && q.e.length === 13; + const checkSafeLength = q.h.length === 44 && q.s.length === 344 && q.i.length === 24; + + if (checkQueries && checkBaseLength && checkSafeLength) { + let streamInfo = verifyStream(q.t, q.h, q.e, q.s, q.i); if (streamInfo.error) { return res.status(streamInfo.status).json(apiJSON(0, { t: streamInfo.error }).body); } - if (req.query.p) { + if (q.p) { return res.status(200).json({ status: "continue" }); @@ -132,7 +143,7 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { return stream(res, streamInfo); } else { let j = apiJSON(0, { - t: "stream token, hmac, or expiry timestamp is missing" + t: "bad request. stream link may be incomplete or corrupted." }) return res.status(j.status).json(j.body); } @@ -159,12 +170,15 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { }); } }); + app.get('/api/status', (req, res) => { res.status(200).end() }); + app.get('/favicon.ico', (req, res) => { res.sendFile(`${__dirname}/src/front/icons/favicon.ico`) }); + app.get('/*', (req, res) => { res.redirect('/api/json') }); diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js index 459e5a6..b034268 100644 --- a/src/modules/stream/manage.js +++ b/src/modules/stream/manage.js @@ -2,7 +2,7 @@ import NodeCache from "node-cache"; import { randomBytes } from "crypto"; import { nanoid } from 'nanoid'; -import { sha256 } from "../sub/crypto.js"; +import { decryptStream, encryptStream, sha256 } from "../sub/crypto.js"; import { streamLifespan } from "../config.js"; const streamCache = new NodeCache({ @@ -15,48 +15,69 @@ streamCache.on("expired", (key) => { streamCache.del(key); }) -const streamSalt = randomBytes(64).toString('hex'); +const hmacSalt = randomBytes(64).toString('hex'); export function createStream(obj) { - let streamID = nanoid(), + const streamID = nanoid(), + iv = randomBytes(16).toString('base64'), + secret = randomBytes(256).toString('base64'), exp = Math.floor(new Date().getTime()) + streamLifespan, - ghmac = sha256(`${streamID},${obj.service},${exp}`, streamSalt); - - if (!streamCache.has(streamID)) { - streamCache.set(streamID, { - id: streamID, + hmac = sha256(`${streamID},${exp},${iv},${secret}`, hmacSalt), + streamData = { service: obj.service, type: obj.type, urls: obj.u, filename: obj.filename, - hmac: ghmac, exp: exp, - isAudioOnly: !!obj.isAudioOnly, audioFormat: obj.audioFormat, - time: obj.time ? obj.time : false, + isAudioOnly: !!obj.isAudioOnly, + time: obj.time || false, copy: !!obj.copy, mute: !!obj.mute, - metadata: obj.fileMetadata ? obj.fileMetadata : false - }); - } else { - let streamInfo = streamCache.get(streamID); - exp = streamInfo.exp; - ghmac = streamInfo.hmac; + metadata: obj.fileMetadata || false + }; + + streamCache.set( + streamID, + encryptStream(streamData, iv, secret) + ) + + let streamLink = new URL('/api/stream', process.env.apiURL); + + const params = { + 't': streamID, + 'e': exp, + 'h': hmac, + 's': secret, + 'i': iv } - return `${process.env.apiURL}api/stream?t=${streamID}&e=${exp}&h=${ghmac}`; + + for (const [key, value] of Object.entries(params)) { + streamLink.searchParams.append(key, value); + } + + return streamLink.toString(); } -export function verifyStream(id, hmac, exp) { +export function verifyStream(id, hmac, exp, secret, iv) { try { - let streamInfo = streamCache.get(id.toString()); + const ghmac = sha256(`${id},${exp},${iv},${secret}`, hmacSalt); + + if (ghmac !== String(hmac)) { + return { + error: "i couldn't verify if you have access to this stream. go back and try again!", + status: 401 + } + } + + const streamInfo = JSON.parse(decryptStream(streamCache.get(id.toString()), iv, secret)); + if (!streamInfo) return { error: "this download link has expired or doesn't exist. go back and try again!", status: 400 } - let ghmac = sha256(`${id},${streamInfo.service},${exp}`, streamSalt); - if (String(hmac) === ghmac && String(exp) === String(streamInfo.exp) && ghmac === String(streamInfo.hmac) - && Number(exp) > Math.floor(new Date().getTime())) { + if (String(exp) === String(streamInfo.exp) && Number(exp) > Math.floor(new Date().getTime())) { return streamInfo; } return { @@ -64,6 +85,6 @@ export function verifyStream(id, hmac, exp) { status: 401 } } catch (e) { - return { status: 500, body: { status: "error", text: "Internal Server Error" } }; + return { status: 500, body: { status: "error", text: "couldn't verify this stream. request a new one!" } }; } } diff --git a/src/modules/sub/crypto.js b/src/modules/sub/crypto.js index e8bf2f9..70484f4 100644 --- a/src/modules/sub/crypto.js +++ b/src/modules/sub/crypto.js @@ -1,5 +1,26 @@ -import { createHmac } from "crypto"; +import { createHmac, createCipheriv, createDecipheriv, scryptSync } from "crypto"; + +const algorithm = "aes256" +const keyLength = 32; export function sha256(str, salt) { - return createHmac("sha256", salt).update(str).digest("hex"); + return createHmac("sha256", salt).update(str).digest("base64"); +} + +export function encryptStream(str, iv, secret) { + const buff = Buffer.from(JSON.stringify(str), "utf-8"); + + const key = scryptSync(Buffer.from(secret, "base64"), "salt", keyLength); + const cipher = createCipheriv(algorithm, key, Buffer.from(iv, "base64")); + + return Buffer.from(cipher.update(buff, "utf8", "binary") + cipher.final("binary"), "binary"); +} + +export function decryptStream(buf, iv, secret) { + const buff = Buffer.from(buf, "binary"); + + const key = scryptSync(Buffer.from(secret, "base64"), "salt", keyLength); + const decipher = createDecipheriv(algorithm, key, Buffer.from(iv, "base64")); + + return decipher.update(buff, "binary", "utf8") + decipher.final("utf8"); } diff --git a/src/modules/sub/utils.js b/src/modules/sub/utils.js index 8f8678d..bb21bbb 100644 --- a/src/modules/sub/utils.js +++ b/src/modules/sub/utils.js @@ -11,7 +11,6 @@ const apiVar = { }, booleanOnly: ["isAudioOnly", "isNoTTWatermark", "isTTFullAudio", "isAudioMuted", "dubLang", "vimeoDash", "disableMetadata", "twitterGif"] } -const forbiddenChars = ['}', '{', '(', ')', '\\', '>', '<', '^', '*', '!', '~', ';', ':', ',', '`', '[', ']', '#', '$', '"', "'", "@", '==']; const forbiddenCharsString = ['}', '{', '%', '>', '<', '^', ';', '`', '$', '"', "@", '=']; export function apiJSON(type, obj) { From 5222d93c69b17319549f1ecc5af2e85ec79f73b9 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 5 Mar 2024 18:16:05 +0600 Subject: [PATCH 11/51] servicesConfig: add support for embed links from ok.ru --- src/modules/processing/servicesConfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index 5277053..384ca1b 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -35,7 +35,7 @@ "ok": { "alias": "ok video", "tld": "ru", - "patterns": ["video/:id"], + "patterns": ["video/:id", "videoembed/:id"], "enabled": true }, "youtube": { From 77df90412b0e413e3ca96730523f1374b840e85f Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 5 Mar 2024 18:16:27 +0600 Subject: [PATCH 12/51] package: bump version to 7.11 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6aedc42..4378bcf 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "7.10.4", + "version": "7.11", "author": "wukko", "exports": "./src/cobalt.js", "type": "module", From e16ee6c1d32e3c29df77728917403320e599712c Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 5 Mar 2024 19:08:59 +0600 Subject: [PATCH 13/51] env: readable environment variables in all files apiPort -> API_PORT apiURL -> API_URL apiName -> API_NAME cors -> ENABLE_CORS cookiePath -> COOKIE_PATH webPort -> WEB_PORT webURL -> WEB_URL showSponsors -> SHOW_SPONSORS isBeta -> IS_BETA --- src/cobalt.js | 4 +-- src/core/api.js | 14 +++++------ src/core/web.js | 6 ++--- src/modules/pageRender/elements.js | 2 +- src/modules/pageRender/page.js | 8 +++--- src/modules/processing/cookie/manager.js | 2 +- src/modules/setup.js | 32 ++++++++++++------------ src/modules/stream/manage.js | 2 +- 8 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/cobalt.js b/src/cobalt.js index 2d90e07..40bca04 100644 --- a/src/cobalt.js +++ b/src/cobalt.js @@ -21,8 +21,8 @@ app.disable('x-powered-by'); await loadLoc(); -const apiMode = process.env.apiURL && !process.env.webURL; -const webMode = process.env.webURL && process.env.apiURL; +const apiMode = process.env.API_URL && !process.env.WEB_URL; +const webMode = process.env.WEB_URL && process.env.API_URL; if (apiMode) { const { runAPI } = await import('./core/api.js'); diff --git a/src/core/api.js b/src/core/api.js index 5f91031..8e26dbd 100644 --- a/src/core/api.js +++ b/src/core/api.js @@ -14,7 +14,7 @@ import { sha256 } from "../modules/sub/crypto.js"; import { verifyStream } from "../modules/stream/manage.js"; export function runAPI(express, app, gitCommit, gitBranch, __dirname) { - const corsConfig = process.env.cors === '0' ? { + const corsConfig = process.env.ENABLE_CORS === '0' ? { origin: process.env.CORS_URL, optionsSuccessStatus: 200 } : {}; @@ -141,9 +141,9 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { version: version, commit: gitCommit, branch: gitBranch, - name: process.env.apiName || "unknown", - url: process.env.apiURL, - cors: process.env?.cors === "0" ? 0 : 1, + name: process.env.API_NAME || "unknown", + url: process.env.API_URL, + cors: process.env?.ENABLE_CORS === "0" ? 0 : 1, startTime: `${startTimestamp}` }); default: @@ -169,12 +169,12 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { res.redirect('/api/json') }); - app.listen(process.env.apiPort || 9000, () => { + app.listen(process.env.API_PORT || 9000, () => { console.log(`\n` + `${Cyan("cobalt")} API ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\n` + `Start time: ${Bright(`${startTime.toUTCString()} (${startTimestamp})`)}\n\n` + - `URL: ${Cyan(`${process.env.apiURL}`)}\n` + - `Port: ${process.env.apiPort || 9000}\n` + `URL: ${Cyan(`${process.env.API_URL}`)}\n` + + `Port: ${process.env.API_PORT || 9000}\n` ) }); } diff --git a/src/core/web.js b/src/core/web.js index 08a6ffe..7c0cbf3 100644 --- a/src/core/web.js +++ b/src/core/web.js @@ -76,12 +76,12 @@ export async function runWeb(express, app, gitCommit, gitBranch, __dirname) { return res.redirect('/') }); - app.listen(process.env.webPort || 9001, () => { + app.listen(process.env.WEB_PORT || 9001, () => { console.log(`\n` + `${Cyan("cobalt")} WEB ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\n` + `Start time: ${Bright(`${startTime.toUTCString()} (${startTimestamp})`)}\n\n` + - `URL: ${Cyan(`${process.env.webURL}`)}\n` + - `Port: ${process.env.webPort || 9001}\n` + `URL: ${Cyan(`${process.env.WEB_URL}`)}\n` + + `Port: ${process.env.WEB_PORT || 9001}\n` ) }) } diff --git a/src/modules/pageRender/elements.js b/src/modules/pageRender/elements.js index a677d2b..53ad3c4 100644 --- a/src/modules/pageRender/elements.js +++ b/src/modules/pageRender/elements.js @@ -264,5 +264,5 @@ export function sponsoredList() { } export function betaTag() { - return process.env.isBeta ? 'β' : '' + return process.env.IS_BETA ? 'β' : '' } diff --git a/src/modules/pageRender/page.js b/src/modules/pageRender/page.js index 81e6d51..77b87f2 100644 --- a/src/modules/pageRender/page.js +++ b/src/modules/pageRender/page.js @@ -48,10 +48,10 @@ export default function(obj) { ${t("AppTitleCobalt")} - + - + @@ -165,7 +165,7 @@ export default function(obj) { body: t("FairUse") }]) }, - ...(process.env.showSponsors ? + ...(process.env.SHOW_SPONSORS ? [{ text: t("SponsoredBy"), classes: ["sponsored-by-text"], @@ -627,7 +627,7 @@ export default function(obj) {