From 9899f778a9221fa82fe36a7671f453e098070ab4 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Mon, 25 Dec 2023 12:38:51 +0000 Subject: [PATCH 01/27] package.json: use punycode.js version of psl from git --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e839fe34..a1914654 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "hls-parser": "^0.10.7", "nanoid": "^4.0.2", "node-cache": "^5.1.2", - "psl": "1.9.0", + "psl": "https://github.com/lupomontero/psl#5eadae91361d8289d582700f90582b0d0cb73155", "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", From cbfcfcfa18a1ebdc85cb283a6d670d68798e9832 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Wed, 21 Feb 2024 11:48:00 +0000 Subject: [PATCH 02/27] tumblr: rewrite & fix audio support closes #342 --- src/modules/processing/matchActionDecider.js | 12 ++-- src/modules/processing/services/tumblr.js | 74 +++++++++++++++----- src/test/tests.json | 2 +- 3 files changed, 62 insertions(+), 26 deletions(-) diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index 7aa154bd..2d8840a3 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -143,7 +143,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di const isBestHostAudio = services[host]["bestAudio"] && (audioFormat === services[host]["bestAudio"]); const isTikTok = host === "tiktok" || host === "douyin"; - const isTumblr = host === "tumblr" && !r.filename; + const isTumblrAudio = host === "tumblr" && !r.filename; const isSoundCloud = host === "soundcloud"; if (isTikTok && services.tiktok.audioFormats.includes(audioFormat)) { @@ -168,11 +168,6 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di } } - if (isTumblr && isBestOrMp3) { - audioFormat = "mp3"; - processType = "bridge" - } - if (isBestAudioDefined || isBestHostAudio) { audioFormat = services[host]["bestAudio"]; processType = "bridge"; @@ -181,6 +176,11 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di copy = true } + if (isTumblrAudio && isBestOrMp3) { + audioFormat = "mp3"; + processType = "bridge" + } + if (r.isM3U8 || host === "vimeo") { copy = false; processType = "render" diff --git a/src/modules/processing/services/tumblr.js b/src/modules/processing/services/tumblr.js index 75d354e7..05c7fd84 100644 --- a/src/modules/processing/services/tumblr.js +++ b/src/modules/processing/services/tumblr.js @@ -1,8 +1,26 @@ import psl from "psl"; import { genericUserAgent } from "../../config.js"; -export default async function(obj) { - let { subdomain } = psl.parse(obj.url.hostname); +const API_KEY = 'jrsCWX1XDuVxAFO4GkK147syAoN8BJZ5voz8tS80bPcj26Vc5Z'; +const API_BASE = 'https://api-http2.tumblr.com'; + +function request(domain, id) { + const url = new URL(`/v2/blog/${domain}/posts/${id}/permalink`, API_BASE); + url.searchParams.set('api_key', API_KEY); + url.searchParams.set('fields[blogs]', 'uuid,name,avatar,?description,?can_message,?can_be_followed,?is_adult,?reply_conditions,' + + '?theme,?title,?url,?is_blocked_from_primary,?placement_id,?primary,?updated,?followed,' + + '?ask,?can_subscribe,?paywall_access,?subscription_plan,?is_blogless_advertiser,?tumblrmart_accessories'); + + return fetch(url, { + headers: { + 'User-Agent': 'Tumblr/iPhone/33.3/333010/17.3.1/tumblr', + 'X-Version': 'iPhone/33.3/333010/17.3.1/tumblr' + } + }).then(a => a.json()).catch(() => {}); +} + +export default async function(input) { + let { subdomain } = psl.parse(input.url.hostname); if (subdomain?.includes('.')) { return { error: ['ErrorBrokenLink', 'tumblr'] } @@ -10,26 +28,44 @@ export default async function(obj) { subdomain = undefined } - let html = await fetch(`https://${subdomain ?? obj.user}.tumblr.com/post/${obj.id}`, { - headers: { "user-agent": genericUserAgent } - }).then((r) => { return r.text() }).catch(() => { return false }); + const domain = `${subdomain ?? input.user}.tumblr.com`; + const data = await request(domain, input.id); - if (!html) return { error: 'ErrorCouldntFetch' }; + const element = data?.response?.timeline?.elements?.[0]; + if (!element) return { error: 'ErrorEmptyDownload' }; - let r; - if (html.includes('property="og:video" content="https://va.media.tumblr.com/')) { - r = { - urls: `https://va.media.tumblr.com/${html.split('property="og:video" content="https://va.media.tumblr.com/')[1].split('"')[0]}`, - filename: `tumblr_${obj.id}.mp4`, - audioFilename: `tumblr_${obj.id}_audio` - } - } else if (html.includes('property="og:audio" content="https://a.tumblr.com/')) { - r = { - urls: `https://a.tumblr.com/${html.split('property="og:audio" content="https://a.tumblr.com/')[1].split('"')[0]}`, - audioFilename: `tumblr_${obj.id}`, + const contents = [ + ...element.content, + ...element?.trail?.map(t => t.content).flat() + ] + + const audio = contents.find(c => c.type === 'audio'); + if (audio && audio.provider === 'tumblr') { + const fileMetadata = { + title: audio?.title, + artist: audio?.artist + }; + + return { + urls: audio.media.url, + filenameAttributes: { + service: 'tumblr', + id: input.id, + title: fileMetadata.title, + author: fileMetadata.artist + }, isAudioOnly: true } - } else r = { error: 'ErrorEmptyDownload' }; + } - return r + const video = contents.find(c => c.type === 'video'); + if (video && video.provider === 'tumblr') { + return { + urls: video.media.url, + filename: `tumblr_${input.id}.mp4`, + audioFilename: `tumblr_${input.id}_audio` + } + } + + return { error: 'ErrorEmptyDownload' } } diff --git a/src/test/tests.json b/src/test/tests.json index a298d152..3b9a9ede 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -773,7 +773,7 @@ } }, { "name": "tumblr audio", - "url": "https://rf9weu8hjf789234hf9.tumblr.com/post/172006661342/everyone-thats-made-a-video-out-of-this-without", + "url": "https://www.tumblr.com/zedneon/737815079301562368/zedneon-ft-mr-sauceman-tech-n9ne-speed-of?source=share", "params": {}, "expected": { "code": 200, From 06b808852646503e53c696e3df80b128a3f1537c Mon Sep 17 00:00:00 2001 From: DrWarpMan <36279265+DrWarpMan@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:17:19 +0100 Subject: [PATCH 03/27] cookies: add all necessary instagram cookies to example Signed-off-by: DrWarpMan <36279265+DrWarpMan@users.noreply.github.com> --- docs/examples/cookies.example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/cookies.example.json b/docs/examples/cookies.example.json index faaeb569..9b488c29 100644 --- a/docs/examples/cookies.example.json +++ b/docs/examples/cookies.example.json @@ -1,5 +1,5 @@ { "instagram": [ - "mid=replace; ig_did=this; csrftoken=cookie" + "mid=; ig_did=; csrftoken=; ds_user_id=; sessionid=" ] } From 6227b7a38a6f2036c650dffacea2cb583f131327 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Wed, 21 Feb 2024 20:23:54 +0000 Subject: [PATCH 04/27] cookies: add example for reddit --- docs/examples/cookies.example.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/examples/cookies.example.json b/docs/examples/cookies.example.json index 9b488c29..5ebdb635 100644 --- a/docs/examples/cookies.example.json +++ b/docs/examples/cookies.example.json @@ -1,5 +1,8 @@ { "instagram": [ "mid=; ig_did=; csrftoken=; ds_user_id=; sessionid=" + ], + "reddit": [ + "client_id=; client_secret=; refresh_token=" ] } From d70754238ee6712a7ffd634b7165bc1f26ecc9b3 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Thu, 15 Feb 2024 01:27:19 +0000 Subject: [PATCH 05/27] stream: fix bilibili downloads fixes #302 --- src/modules/stream/types.js | 43 ++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index cdfb4a05..7dcfd74a 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -80,17 +80,33 @@ export async function streamLiveRender(streamInfo, res) { if (streamInfo.urls.length !== 2) return shutdown(); const { body: audio } = await request(streamInfo.urls[1], { - maxRedirections: 16, signal: abortController.signal + maxRedirections: 16, signal: abortController.signal, + headers: { + 'user-agent': genericUserAgent, + referer: streamInfo.service === 'bilibili' + ? 'https://www.bilibili.com/' + : undefined, + } }); - let format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1], - args = [ + const format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1]; + let args = [ '-loglevel', '-8', + '-user_agent', genericUserAgent + ]; + + if (streamInfo.service === 'bilibili') { + args.push( + '-headers', 'Referer: https://www.bilibili.com/\r\n', + ) + } + + args.push( '-i', streamInfo.urls[0], '-i', 'pipe:3', '-map', '0:v', '-map', '1:a', - ]; + ); args = args.concat(ffmpegArgs[format]); if (streamInfo.metadata) { @@ -129,11 +145,16 @@ export function streamAudioOnly(streamInfo, res) { try { let args = [ - '-loglevel', '-8' - ] + '-loglevel', '-8', + '-user_agent', genericUserAgent + ]; + if (streamInfo.service === "twitter") { - args.push('-seekable', '0') + args.push('-seekable', '0'); + } else if (streamInfo.service === 'bilibili') { + args.push('-headers', 'Referer: https://www.bilibili.com/\r\n'); } + args.push( '-i', streamInfo.urls, '-vn' @@ -178,17 +199,23 @@ export function streamVideoOnly(streamInfo, res) { let args = [ '-loglevel', '-8' ] + if (streamInfo.service === "twitter") { args.push('-seekable', '0') + } else if (streamInfo.service === 'bilibili') { + args.push('-headers', 'Referer: https://www.bilibili.com/\r\n') } + args.push( '-i', streamInfo.urls, '-c', 'copy' ) + if (streamInfo.mute) { args.push('-an') } - if (streamInfo.service === "vimeo" || streamInfo.service === "rutube") { + + if (['vimeo', 'rutube'].includes(streamInfo.service)) { args.push('-bsf:a', 'aac_adtstoasc') } From 6e1eddad82cc8fe21183e61f29fbf7f8ca3aa782 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Thu, 15 Feb 2024 01:42:15 +0000 Subject: [PATCH 06/27] bilibili: add support for b23.tv links resolves #320 --- src/modules/processing/match.js | 4 +--- src/modules/processing/services/bilibili.js | 23 +++++++++++++++---- src/modules/processing/servicesConfig.json | 2 +- .../processing/servicesPatternTesters.js | 2 +- src/modules/processing/url.js | 8 +++++++ src/test/tests.json | 8 +++++++ 6 files changed, 38 insertions(+), 9 deletions(-) diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index cc2516f2..95558c8b 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -56,9 +56,7 @@ export default async function(host, patternMatch, url, lang, obj) { }); break; case "bilibili": - r = await bilibili({ - id: patternMatch.id.slice(0, 12) - }); + r = await bilibili(patternMatch); break; case "youtube": let fetchInfo = { diff --git a/src/modules/processing/services/bilibili.js b/src/modules/processing/services/bilibili.js index 0194ee46..5664e8cb 100644 --- a/src/modules/processing/services/bilibili.js +++ b/src/modules/processing/services/bilibili.js @@ -1,8 +1,23 @@ import { genericUserAgent, maxVideoDuration } from "../../config.js"; // TO-DO: quality picking, bilibili.tv support, and higher quality downloads (currently requires an account) -export default async function(obj) { - let html = await fetch(`https://bilibili.com/video/${obj.id}`, { +export default async function({ id, shortLink }) { + if (shortLink) { + id = await fetch(`https://b23.tv/${shortLink}`, { redirect: 'manual' }) + .then(r => r.status > 300 && r.status < 400 && r.headers.get('location')) + .then(url => { + const path = new URL(url).pathname; + if (path.startsWith('/video/')) + return path.split('/')[2]; + }) + .catch(() => {}) + } + + if (!id) { + return { error: 'ErrorCouldntFetch' }; + } + + let html = await fetch(`https://bilibili.com/video/${id}`, { headers: { "user-agent": genericUserAgent } }).then((r) => { return r.text() }).catch(() => { return false }); if (!html) return { error: 'ErrorCouldntFetch' }; @@ -21,7 +36,7 @@ export default async function(obj) { return { urls: [video[0]["baseUrl"], audio[0]["baseUrl"]], - audioFilename: `bilibili_${obj.id}_audio`, - filename: `bilibili_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.mp4` + audioFilename: `bilibili_${id}_audio`, + filename: `bilibili_${id}_${video[0]["width"]}x${video[0]["height"]}.mp4` }; } diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index 804f5978..bec1dab2 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -3,7 +3,7 @@ "config": { "bilibili": { "alias": "bilibili.com videos", - "patterns": ["video/:id"], + "patterns": ["video/:id", "_shortLink/:shortLink"], "enabled": true }, "reddit": { diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index 970e8f40..f3418445 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -1,6 +1,6 @@ export const testers = { "bilibili": (patternMatch) => - patternMatch.id?.length <= 12, + patternMatch.id?.length <= 12 || patternMatch.shortLink?.length <= 16, "instagram": (patternMatch) => patternMatch.postId?.length <= 12 diff --git a/src/modules/processing/url.js b/src/modules/processing/url.js index 9c87889d..753e9d5d 100644 --- a/src/modules/processing/url.js +++ b/src/modules/processing/url.js @@ -16,6 +16,7 @@ export function aliasURL(url) { url.search = `?v=${encodeURIComponent(parts[2])}` } break; + case "youtu": if (url.hostname === 'youtu.be' && parts.length >= 2) { /* youtu.be urls can be weird, e.g. https://youtu.be///asdasd// still works @@ -25,6 +26,7 @@ export function aliasURL(url) { }`) } break; + case "pin": if (url.hostname === 'pin.it' && parts.length === 2) { url = new URL(`https://pinterest.com/url_shortener/${ @@ -46,6 +48,12 @@ export function aliasURL(url) { url = new URL(`https://twitch.tv/_/clip/${parts[1]}`); } break; + + case "b23": + if (url.hostname === 'b23.tv' && parts.length === 2) { + url = new URL(`https://bilibili.com/_shortLink/${parts[1]}`) + } + break; } return url diff --git a/src/test/tests.json b/src/test/tests.json index a298d152..3161b00d 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -746,6 +746,14 @@ "code": 200, "status": "stream" } + }, { + "name": "b23.tv shortlink", + "url": "https://b23.tv/lbMyOI9", + "params": {}, + "expected": { + "code": 200, + "status": "stream" + } }], "tumblr": [{ "name": "at.tumblr link", From 0852ade1be4c84a188e35f8ee2b962fde35292e7 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Thu, 15 Feb 2024 18:11:18 +0000 Subject: [PATCH 07/27] bilibili: add support for bilibili.tv links closes #319 --- src/modules/processing/services/bilibili.js | 101 ++++++++++++++---- src/modules/processing/servicesConfig.json | 7 +- .../processing/servicesPatternTesters.js | 5 +- src/modules/processing/url.js | 5 + src/test/tests.json | 9 ++ 5 files changed, 104 insertions(+), 23 deletions(-) diff --git a/src/modules/processing/services/bilibili.js b/src/modules/processing/services/bilibili.js index 5664e8cb..6da110bf 100644 --- a/src/modules/processing/services/bilibili.js +++ b/src/modules/processing/services/bilibili.js @@ -1,42 +1,105 @@ import { genericUserAgent, maxVideoDuration } from "../../config.js"; -// TO-DO: quality picking, bilibili.tv support, and higher quality downloads (currently requires an account) -export default async function({ id, shortLink }) { - if (shortLink) { - id = await fetch(`https://b23.tv/${shortLink}`, { redirect: 'manual' }) +// TO-DO: higher quality downloads (currently requires an account) + +function com_resolveShortlink(shortId) { + return fetch(`https://b23.tv/${shortId}`, { redirect: 'manual' }) .then(r => r.status > 300 && r.status < 400 && r.headers.get('location')) .then(url => { + if (!url) return; const path = new URL(url).pathname; if (path.startsWith('/video/')) return path.split('/')[2]; }) .catch(() => {}) - } +} - if (!id) { - return { error: 'ErrorCouldntFetch' }; - } +function getBest(content) { + return content?.filter(v => v.baseUrl || v.url) + .map(v => (v.baseUrl = v.baseUrl || v.url, v)) + .reduce((a, b) => a?.bandwidth > b?.bandwidth ? a : b); +} +function extractBestQuality(dashData) { + const bestVideo = getBest(dashData.video), + bestAudio = getBest(dashData.audio); + + if (!bestVideo || !bestAudio) return []; + return [ bestVideo, bestAudio ]; +} + +async function com_download(id) { let html = await fetch(`https://bilibili.com/video/${id}`, { headers: { "user-agent": genericUserAgent } }).then((r) => { return r.text() }).catch(() => { return false }); if (!html) return { error: 'ErrorCouldntFetch' }; - if (!(html.includes('')[0]); - if (streamData.data.timelength > maxVideoDuration) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + if (streamData.data.timelength > maxVideoDuration) { + return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + } - let video = streamData["data"]["dash"]["video"].filter(v => - !v["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/") - ).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth)); - - let audio = streamData["data"]["dash"]["audio"].filter(a => - !a["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/") - ).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth)); + const [ video, audio ] = extractBestQuality(streamData.data.dash); + if (!video || !audio) { + return { error: 'ErrorEmptyDownload' }; + } return { - urls: [video[0]["baseUrl"], audio[0]["baseUrl"]], + urls: [video.baseUrl, audio.baseUrl], audioFilename: `bilibili_${id}_audio`, - filename: `bilibili_${id}_${video[0]["width"]}x${video[0]["height"]}.mp4` + filename: `bilibili_${id}_${video.width}x${video.height}.mp4` }; } + +async function tv_download(id) { + const url = new URL( + 'https://api.bilibili.tv/intl/gateway/web/playurl' + + '?s_locale=en_US&platform=web&qn=64&type=0&device=wap' + + '&tf=0&spm_id=bstar-web.ugc-video-detail.0.0&from_spm_id=' + ); + + url.searchParams.set('aid', id); + + const { data } = await fetch(url).then(a => a.json()); + if (!data?.playurl?.video) { + return { error: 'ErrorEmptyDownload' }; + } + + const [ video, audio ] = extractBestQuality({ + video: data.playurl.video.map(s => s.video_resource) + .filter(s => s.codecs.includes('avc1')), + audio: data.playurl.audio_resource + }); + + if (!video || !audio) { + return { error: 'ErrorEmptyDownload' }; + } + + if (video.duration > maxVideoDuration) { + return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + } + + return { + urls: [video.url, audio.url], + audioFilename: `bilibili_tv_${id}_audio`, + filename: `bilibili_tv_${id}.mp4` + }; +} + +export default async function({ comId, tvId, comShortLink }) { + if (comShortLink) { + comId = await com_resolveShortlink(comShortLink); + } + + if (comId) { + return com_download(comId); + } else if (tvId) { + return tv_download(tvId); + } + + return { error: 'ErrorCouldntFetch' }; +} diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index bec1dab2..d6e137e9 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -2,8 +2,11 @@ "audioIgnore": ["vk", "ok"], "config": { "bilibili": { - "alias": "bilibili.com videos", - "patterns": ["video/:id", "_shortLink/:shortLink"], + "alias": "bilibili.com and bilibili.tv videos", + "patterns": [ + "video/:comId", "_shortLink/:comShortLink", + "_tv/:lang/video/:tvId", "_tv/video/:tvId" + ], "enabled": true }, "reddit": { diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index f3418445..30892f62 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -1,6 +1,7 @@ export const testers = { - "bilibili": (patternMatch) => - patternMatch.id?.length <= 12 || patternMatch.shortLink?.length <= 16, + "bilibili": (patternMatch) => + patternMatch.comId?.length <= 12 || patternMatch.comShortLink?.length <= 16 + || patternMatch.tvId?.length <= 24, "instagram": (patternMatch) => patternMatch.postId?.length <= 12 diff --git a/src/modules/processing/url.js b/src/modules/processing/url.js index 753e9d5d..5e6bd15a 100644 --- a/src/modules/processing/url.js +++ b/src/modules/processing/url.js @@ -49,6 +49,11 @@ export function aliasURL(url) { } break; + case "bilibili": + if (host.tld === 'tv') { + url = new URL(`https://bilibili.com/_tv${url.pathname}`); + } + break; case "b23": if (url.hostname === 'b23.tv' && parts.length === 2) { url = new URL(`https://bilibili.com/_shortLink/${parts[1]}`) diff --git a/src/test/tests.json b/src/test/tests.json index 3161b00d..42f75f40 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -754,6 +754,15 @@ "code": 200, "status": "stream" } + }, + { + "name": "bilibili.tv link", + "url": "https://www.bilibili.tv/en/video/4789599404426256", + "params": {}, + "expected": { + "code": 200, + "status": "stream" + } }], "tumblr": [{ "name": "at.tumblr link", From decedd77369a573016d662b3ea8228f9b3898606 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 5 Mar 2024 12:39:49 +0600 Subject: [PATCH 08/27] package: use latest version of undici --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a1914654..dfbe009e 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "node-cache": "^5.1.2", "psl": "https://github.com/lupomontero/psl#5eadae91361d8289d582700f90582b0d0cb73155", "set-cookie-parser": "2.6.0", - "undici": "^5.19.1", + "undici": "^6.7.0", "url-pattern": "1.0.3", "youtubei.js": "^6.4.1" } From b1bc7126a49ab7e5d06684cf9627d62c266772ac Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 5 Mar 2024 13:38:46 +0600 Subject: [PATCH 09/27] servicesConfig: update bilibili alias Signed-off-by: wukko --- src/modules/processing/servicesConfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index d6e137e9..5277053e 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -2,7 +2,7 @@ "audioIgnore": ["vk", "ok"], "config": { "bilibili": { - "alias": "bilibili.com and bilibili.tv videos", + "alias": "bilibili videos", "patterns": [ "video/:comId", "_shortLink/:comShortLink", "_tv/:lang/video/:tvId", "_tv/video/:tvId" From 5222d93c69b17319549f1ecc5af2e85ec79f73b9 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 5 Mar 2024 18:16:05 +0600 Subject: [PATCH 10/27] servicesConfig: add support for embed links from ok.ru --- src/modules/processing/servicesConfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index 5277053e..384ca1b6 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -35,7 +35,7 @@ "ok": { "alias": "ok video", "tld": "ru", - "patterns": ["video/:id"], + "patterns": ["video/:id", "videoembed/:id"], "enabled": true }, "youtube": { From 77df90412b0e413e3ca96730523f1374b840e85f Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 5 Mar 2024 18:16:27 +0600 Subject: [PATCH 11/27] package: bump version to 7.11 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6aedc426..4378bcff 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "7.10.4", + "version": "7.11", "author": "wukko", "exports": "./src/cobalt.js", "type": "module", From e16ee6c1d32e3c29df77728917403320e599712c Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 5 Mar 2024 19:08:59 +0600 Subject: [PATCH 12/27] env: readable environment variables in all files apiPort -> API_PORT apiURL -> API_URL apiName -> API_NAME cors -> ENABLE_CORS cookiePath -> COOKIE_PATH webPort -> WEB_PORT webURL -> WEB_URL showSponsors -> SHOW_SPONSORS isBeta -> IS_BETA --- src/cobalt.js | 4 +-- src/core/api.js | 14 +++++------ src/core/web.js | 6 ++--- src/modules/pageRender/elements.js | 2 +- src/modules/pageRender/page.js | 8 +++--- src/modules/processing/cookie/manager.js | 2 +- src/modules/setup.js | 32 ++++++++++++------------ src/modules/stream/manage.js | 2 +- 8 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/cobalt.js b/src/cobalt.js index 2d90e07e..40bca04b 100644 --- a/src/cobalt.js +++ b/src/cobalt.js @@ -21,8 +21,8 @@ app.disable('x-powered-by'); await loadLoc(); -const apiMode = process.env.apiURL && !process.env.webURL; -const webMode = process.env.webURL && process.env.apiURL; +const apiMode = process.env.API_URL && !process.env.WEB_URL; +const webMode = process.env.WEB_URL && process.env.API_URL; if (apiMode) { const { runAPI } = await import('./core/api.js'); diff --git a/src/core/api.js b/src/core/api.js index 5f910315..8e26dbd0 100644 --- a/src/core/api.js +++ b/src/core/api.js @@ -14,7 +14,7 @@ import { sha256 } from "../modules/sub/crypto.js"; import { verifyStream } from "../modules/stream/manage.js"; export function runAPI(express, app, gitCommit, gitBranch, __dirname) { - const corsConfig = process.env.cors === '0' ? { + const corsConfig = process.env.ENABLE_CORS === '0' ? { origin: process.env.CORS_URL, optionsSuccessStatus: 200 } : {}; @@ -141,9 +141,9 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { version: version, commit: gitCommit, branch: gitBranch, - name: process.env.apiName || "unknown", - url: process.env.apiURL, - cors: process.env?.cors === "0" ? 0 : 1, + name: process.env.API_NAME || "unknown", + url: process.env.API_URL, + cors: process.env?.ENABLE_CORS === "0" ? 0 : 1, startTime: `${startTimestamp}` }); default: @@ -169,12 +169,12 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { res.redirect('/api/json') }); - app.listen(process.env.apiPort || 9000, () => { + app.listen(process.env.API_PORT || 9000, () => { console.log(`\n` + `${Cyan("cobalt")} API ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\n` + `Start time: ${Bright(`${startTime.toUTCString()} (${startTimestamp})`)}\n\n` + - `URL: ${Cyan(`${process.env.apiURL}`)}\n` + - `Port: ${process.env.apiPort || 9000}\n` + `URL: ${Cyan(`${process.env.API_URL}`)}\n` + + `Port: ${process.env.API_PORT || 9000}\n` ) }); } diff --git a/src/core/web.js b/src/core/web.js index 08a6ffed..7c0cbf33 100644 --- a/src/core/web.js +++ b/src/core/web.js @@ -76,12 +76,12 @@ export async function runWeb(express, app, gitCommit, gitBranch, __dirname) { return res.redirect('/') }); - app.listen(process.env.webPort || 9001, () => { + app.listen(process.env.WEB_PORT || 9001, () => { console.log(`\n` + `${Cyan("cobalt")} WEB ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\n` + `Start time: ${Bright(`${startTime.toUTCString()} (${startTimestamp})`)}\n\n` + - `URL: ${Cyan(`${process.env.webURL}`)}\n` + - `Port: ${process.env.webPort || 9001}\n` + `URL: ${Cyan(`${process.env.WEB_URL}`)}\n` + + `Port: ${process.env.WEB_PORT || 9001}\n` ) }) } diff --git a/src/modules/pageRender/elements.js b/src/modules/pageRender/elements.js index a677d2bc..53ad3c42 100644 --- a/src/modules/pageRender/elements.js +++ b/src/modules/pageRender/elements.js @@ -264,5 +264,5 @@ export function sponsoredList() { } export function betaTag() { - return process.env.isBeta ? 'β' : '' + return process.env.IS_BETA ? 'β' : '' } diff --git a/src/modules/pageRender/page.js b/src/modules/pageRender/page.js index 81e6d514..77b87f29 100644 --- a/src/modules/pageRender/page.js +++ b/src/modules/pageRender/page.js @@ -48,10 +48,10 @@ export default function(obj) { ${t("AppTitleCobalt")} - + - + @@ -165,7 +165,7 @@ export default function(obj) { body: t("FairUse") }]) }, - ...(process.env.showSponsors ? + ...(process.env.SHOW_SPONSORS ? [{ text: t("SponsoredBy"), classes: ["sponsored-by-text"], @@ -627,7 +627,7 @@ export default function(obj) {