From 0378a1ae15ed6d1cefdded8efbd7e5a993b01491 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 20 Jan 2025 12:37:36 +0000 Subject: [PATCH 01/28] api/youtube: fix error when downloading stuff from WEB --- api/src/processing/services/youtube.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index 6559978c..e0dc8594 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -491,12 +491,12 @@ export default async function (o) { filenameAttributes.resolution = `${video.width}x${video.height}`; filenameAttributes.extension = codecList[codec].container; - video = video.url; - audio = audio.url; - if (innertubeClient === "WEB" && innertube) { video = video.decipher(innertube.session.player); audio = audio.decipher(innertube.session.player); + } else { + video = video.url; + audio = audio.url; } } From ec0d7737926d5d4f7c4db0dc71964c9291452770 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 20 Jan 2025 12:38:12 +0000 Subject: [PATCH 02/28] api/youtube: use Math.min instead of ternary operator --- api/src/processing/services/youtube.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index e0dc8594..e16e86e7 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -240,7 +240,7 @@ export default async function (o) { const quality = o.quality === "max" ? 9000 : Number(o.quality); const normalizeQuality = res => { - const shortestSide = res.height > res.width ? res.width : res.height; + const shortestSide = Math.min(res.height, res.width); return videoQualities.find(qual => qual >= shortestSide); } From 9bdcb9d8216940aea0b1d499e80478477683b182 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 18:51:37 +0600 Subject: [PATCH 03/28] api/utils: update getRedirectingURL to accept more statuses & dispatcher --- api/src/misc/utils.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/api/src/misc/utils.js b/api/src/misc/utils.js index fd497d18..6bc72176 100644 --- a/api/src/misc/utils.js +++ b/api/src/misc/utils.js @@ -1,8 +1,16 @@ -export function getRedirectingURL(url) { - return fetch(url, { redirect: 'manual' }).then((r) => { - if ([301, 302, 303].includes(r.status) && r.headers.has('location')) +const redirectStatuses = [301, 302, 303, 307, 308]; + +export async function getRedirectingURL(url, dispatcher) { + const location = await fetch(url, { + redirect: 'manual', + dispatcher, + }).then((r) => { + if (redirectStatuses.includes(r.status) && r.headers.has('location')) { return r.headers.get('location'); + } }).catch(() => null); + + return location; } export function merge(a, b) { From 63b2681017df728f1aeeffae393f5da5a663805a Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:04:31 +0600 Subject: [PATCH 04/28] api/match-action: always proxy photos --- api/src/processing/match-action.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/processing/match-action.js b/api/src/processing/match-action.js index 64f86836..1cd36eb2 100644 --- a/api/src/processing/match-action.js +++ b/api/src/processing/match-action.js @@ -47,7 +47,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab }); case "photo": - responseType = "redirect"; + params = { type: "proxy" }; break; case "gif": From ed8f4353ea3346982d69eb2aadc4cfa5a8c26855 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:10:02 +0600 Subject: [PATCH 05/28] api/processing: add support for xiaohongshu --- api/src/processing/match-action.js | 2 + api/src/processing/match.js | 10 ++ api/src/processing/service-config.js | 8 ++ api/src/processing/service-patterns.js | 4 + api/src/processing/services/xiaohongshu.js | 123 +++++++++++++++++++++ api/src/processing/url.js | 26 +++-- 6 files changed, 165 insertions(+), 8 deletions(-) create mode 100644 api/src/processing/services/xiaohongshu.js diff --git a/api/src/processing/match-action.js b/api/src/processing/match-action.js index 1cd36eb2..0d87b5a8 100644 --- a/api/src/processing/match-action.js +++ b/api/src/processing/match-action.js @@ -83,6 +83,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab case "twitter": case "snapchat": case "bsky": + case "xiaohongshu": params = { picker: r.picker }; break; @@ -143,6 +144,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab case "ok": case "vk": case "tiktok": + case "xiaohongshu": params = { type: "proxy" }; break; diff --git a/api/src/processing/match.js b/api/src/processing/match.js index 57f04b36..9fabf379 100644 --- a/api/src/processing/match.js +++ b/api/src/processing/match.js @@ -28,6 +28,7 @@ import snapchat from "./services/snapchat.js"; import loom from "./services/loom.js"; import facebook from "./services/facebook.js"; import bluesky from "./services/bluesky.js"; +import xiaohongshu from "./services/xiaohongshu.js"; let freebind; @@ -239,6 +240,15 @@ export default async function({ host, patternMatch, params }) { }); break; + case "xiaohongshu": + r = await xiaohongshu({ + ...patternMatch, + h265: params.tiktokH265, + isAudioOnly, + dispatcher, + }); + break; + default: return createResponse("error", { code: "error.api.service.unsupported" diff --git a/api/src/processing/service-config.js b/api/src/processing/service-config.js index 81afaf39..86352f9a 100644 --- a/api/src/processing/service-config.js +++ b/api/src/processing/service-config.js @@ -166,6 +166,14 @@ export const services = { subdomains: ["m"], altDomains: ["vkvideo.ru", "vk.ru"], }, + xiaohongshu: { + patterns: [ + "explore/:id?xsec_token=:token", + "discovery/item/:id?xsec_token=:token", + "a/:shareId" + ], + altDomains: ["xhslink.com"], + }, youtube: { patterns: [ "watch?v=:id", diff --git a/api/src/processing/service-patterns.js b/api/src/processing/service-patterns.js index e8c46639..42f64d26 100644 --- a/api/src/processing/service-patterns.js +++ b/api/src/processing/service-patterns.js @@ -71,4 +71,8 @@ export const testers = { "bsky": pattern => pattern.user?.length <= 128 && pattern.post?.length <= 128, + + "xiaohongshu": pattern => + pattern.id?.length <= 24 && pattern.token?.length <= 64 + || pattern.shareId?.length <= 12, } diff --git a/api/src/processing/services/xiaohongshu.js b/api/src/processing/services/xiaohongshu.js new file mode 100644 index 00000000..d428aaf9 --- /dev/null +++ b/api/src/processing/services/xiaohongshu.js @@ -0,0 +1,123 @@ +import { extract, normalizeURL } from "../url.js"; +import { genericUserAgent } from "../../config.js"; +import { createStream } from "../../stream/manage.js"; +import { getRedirectingURL } from "../../misc/utils.js"; + +const https = (url) => { + return url.replace(/^http:/i, 'https:'); +} + +export default async function ({ id, token, shareId, h265, isAudioOnly, dispatcher }) { + let noteId = id; + let xsecToken = token; + + if (!noteId) { + const extractedURL = await getRedirectingURL( + `https://xhslink.com/a/${shareId}`, + dispatcher + ); + + if (extractedURL) { + const { patternMatch } = extract(normalizeURL(extractedURL)); + + if (patternMatch) { + noteId = patternMatch.id; + xsecToken = patternMatch.token; + } + } + } + + if (!noteId || !xsecToken) return { error: "fetch.short_link" }; + + const res = await fetch(`https://www.xiaohongshu.com/explore/${noteId}?xsec_token=${xsecToken}`, { + headers: { + "user-agent": genericUserAgent, + }, + dispatcher, + }); + + const html = await res.text(); + + let note; + try { + const initialState = html + .split('')[0] + .replace(/:undefined/g, ":null"); + + const data = JSON.parse(initialState); + + const noteInfo = data?.note?.noteDetailMap; + if (!noteInfo) throw "no note detail map"; + + const currentNote = noteInfo[noteId]; + if (!currentNote) throw "no current note in detail map"; + + note = currentNote.note; + } catch { + return { error: "fetch.empty" }; + } + + if (!note) return { error: "fetch.empty" }; + + const video = note.video; + const images = note.imageList; + + const filenameBase = `xiaohongshu_${noteId}`; + + if (video) { + const videoFilename = `${filenameBase}.mp4`; + const audioFilename = `${filenameBase}_audio`; + + let videoURL; + + if (h265 && !isAudioOnly && video.consumer?.originVideoKey) { + videoURL = `https://sns-video-bd.xhscdn.com/${video.consumer.originVideoKey}`; + } + + if (!videoURL) { + const h264Streams = video.media?.stream?.h264; + if (!h264Streams) return { error: "fetch.empty" }; + + if (h264Streams.length > 1) { + videoURL = h264Streams.reduce((a, b) => Number(a?.videoBitrate) > Number(b?.videoBitrate) ? a : b).masterUrl; + } else { + videoURL = h264Streams[0].masterUrl; + } + } + + if (!videoURL) return { error: "fetch.empty" }; + + return { + urls: https(videoURL), + filename: videoFilename, + audioFilename: audioFilename, + } + } + + if (!images || images.length === 0) { + return { error: "fetch.empty" }; + } + + if (images.length === 1) { + return { + isPhoto: true, + urls: https(images[0].urlDefault), + filename: `${filenameBase}.jpg`, + } + } + + const picker = images.map((image, i) => { + return { + type: "photo", + url: createStream({ + service: "xiaohongshu", + type: "proxy", + url: https(image.urlDefault), + filename: `${filenameBase}_${i + 1}.jpg`, + }) + } + }); + + return { picker }; +} diff --git a/api/src/processing/url.js b/api/src/processing/url.js index 8f0e7dc2..cfbbecc0 100644 --- a/api/src/processing/url.js +++ b/api/src/processing/url.js @@ -92,9 +92,14 @@ function aliasURL(url) { url.hostname = 'vk.com'; } break; + + case "xhslink": + if (url.hostname === 'xhslink.com' && parts.length === 3) { + url = new URL(`https://www.xiaohongshu.com/a/${parts[2]}`); + } } - return url + return url; } function cleanURL(url) { @@ -114,36 +119,41 @@ function cleanURL(url) { break; case "vk": if (url.pathname.includes('/clip') && url.searchParams.get('z')) { - limitQuery('z') + limitQuery('z'); } break; case "youtube": if (url.searchParams.get('v')) { - limitQuery('v') + limitQuery('v'); } break; case "rutube": if (url.searchParams.get('p')) { - limitQuery('p') + limitQuery('p'); } break; case "twitter": if (url.searchParams.get('post_id')) { - limitQuery('post_id') + limitQuery('post_id'); + } + break; + case "xiaohongshu": + if (url.searchParams.get('xsec_token')) { + limitQuery('xsec_token'); } break; } if (stripQuery) { - url.search = '' + url.search = ''; } - url.username = url.password = url.port = url.hash = '' + url.username = url.password = url.port = url.hash = ''; if (url.pathname.endsWith('/')) url.pathname = url.pathname.slice(0, -1); - return url + return url; } function getHostIfValid(url) { From ad6f29a3c83047de5b7f007d6cd103b03f9aac34 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:21:44 +0600 Subject: [PATCH 06/28] api/tests: add xiaohongshu tests --- api/src/util/tests/xiaohongshu.json | 58 +++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 api/src/util/tests/xiaohongshu.json diff --git a/api/src/util/tests/xiaohongshu.json b/api/src/util/tests/xiaohongshu.json new file mode 100644 index 00000000..425a6a2e --- /dev/null +++ b/api/src/util/tests/xiaohongshu.json @@ -0,0 +1,58 @@ +[ + { + "name": "long link video", + "url": "https://www.xiaohongshu.com/discovery/item/6789065900000000210035fc?source=webshare&xhsshare=pc_web&xsec_token=CBustnz_Twf1BSybpe5-D-BzUb-Bx28DPLb418TN9S9Kk&xsec_source=pc_share", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "picker with multiple live photos", + "url": "https://www.xiaohongshu.com/explore/67847fa1000000000203e6ed?xsec_token=CBzyP7Y44PPpsM20lgxqrIIJMHqOLemusDsRcmsX0cTpk", + "params": {}, + "expected": { + "code": 200, + "status": "picker" + } + }, + { + "name": "one photo", + "url": "https://www.xiaohongshu.com/explore/6788b56200000000210008c8?xsec_token=CBSDiWU4N-DgirHrOVbIWrlKfUNFHKwm-Wsjqz7dIMc_k", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "short link, might expire eventually", + "url": "https://xhslink.com/a/czn4z6c1tic4", + "canFail": true, + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "wrong note id", + "url": "https://www.xiaohongshu.com/discovery/item/6789065911100000210035fc?source=webshare&xhsshare=pc_web&xsec_token=CBustnz_Twf1BSybpe5-D-BzUb-Bx28DPLb418TN9S9Kk&xsec_source=pc_share", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "short link, wrong id", + "url": "https://xhslink.com/a/aaaaaa", + "canFail": true, + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + } +] From cd466a418ac18b85d3685a3af34a0885ea138ba5 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:24:12 +0600 Subject: [PATCH 07/28] api/tests/bsky: fix expected photo test status --- api/src/util/tests/bsky.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/util/tests/bsky.json b/api/src/util/tests/bsky.json index 840f1169..5a03538d 100644 --- a/api/src/util/tests/bsky.json +++ b/api/src/util/tests/bsky.json @@ -54,7 +54,7 @@ "params": {}, "expected": { "code": 200, - "status": "redirect" + "status": "tunnel" } }, { From cd0a2a47c9eb6a4a29c75db1cbcdfe55133d4b37 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:28:35 +0600 Subject: [PATCH 08/28] api/tests/pinterest: update expected photo status --- api/src/util/tests/pinterest.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/src/util/tests/pinterest.json b/api/src/util/tests/pinterest.json index 2f15fb0b..6308adb4 100644 --- a/api/src/util/tests/pinterest.json +++ b/api/src/util/tests/pinterest.json @@ -54,7 +54,7 @@ "params": {}, "expected": { "code": 200, - "status": "redirect" + "status": "tunnel" } }, { @@ -63,7 +63,7 @@ "params": {}, "expected": { "code": 200, - "status": "redirect" + "status": "tunnel" } }, { @@ -72,7 +72,7 @@ "params": {}, "expected": { "code": 200, - "status": "redirect" + "status": "tunnel" } }, { @@ -81,7 +81,7 @@ "params": {}, "expected": { "code": 200, - "status": "redirect" + "status": "tunnel" } } ] \ No newline at end of file From de5eca19a5c3d82462b691b8831034985c98a847 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:30:11 +0600 Subject: [PATCH 09/28] api/utils: replace redirectStatuses array with a set Co-authored-by: jj --- api/src/misc/utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/misc/utils.js b/api/src/misc/utils.js index 6bc72176..9978ceed 100644 --- a/api/src/misc/utils.js +++ b/api/src/misc/utils.js @@ -1,11 +1,11 @@ -const redirectStatuses = [301, 302, 303, 307, 308]; +const redirectStatuses = new Set([301, 302, 303, 307, 308]); export async function getRedirectingURL(url, dispatcher) { const location = await fetch(url, { redirect: 'manual', dispatcher, }).then((r) => { - if (redirectStatuses.includes(r.status) && r.headers.has('location')) { + if (redirectStatuses.has(r.status) && r.headers.has('location')) { return r.headers.get('location'); } }).catch(() => null); From 3cbed87c3ebaae58b3bab01a67c852d0f0c495cd Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:35:53 +0600 Subject: [PATCH 10/28] api/xiaohongshu: update initial state extraction regex Co-authored-by: jj --- api/src/processing/services/xiaohongshu.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/processing/services/xiaohongshu.js b/api/src/processing/services/xiaohongshu.js index d428aaf9..4637358e 100644 --- a/api/src/processing/services/xiaohongshu.js +++ b/api/src/processing/services/xiaohongshu.js @@ -43,7 +43,7 @@ export default async function ({ id, token, shareId, h265, isAudioOnly, dispatch const initialState = html .split('')[0] - .replace(/:undefined/g, ":null"); + .replace(/:\s*undefined/g, ":null"); const data = JSON.parse(initialState); From 4963c9f128fc71cf822dca661e34f94f3dca9417 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:37:23 +0600 Subject: [PATCH 11/28] api/xiaohongshu: remove duplicated extraction error Co-authored-by: jj --- api/src/processing/services/xiaohongshu.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api/src/processing/services/xiaohongshu.js b/api/src/processing/services/xiaohongshu.js index 4637358e..6e141f5a 100644 --- a/api/src/processing/services/xiaohongshu.js +++ b/api/src/processing/services/xiaohongshu.js @@ -54,9 +54,7 @@ export default async function ({ id, token, shareId, h265, isAudioOnly, dispatch if (!currentNote) throw "no current note in detail map"; note = currentNote.note; - } catch { - return { error: "fetch.empty" }; - } + } catch {} if (!note) return { error: "fetch.empty" }; From e39b0ae7b3b09e1a0444810a6f35bf703f1ed604 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:41:02 +0600 Subject: [PATCH 12/28] api/xiaohongshu: deduplicate h264 stream extraction reduce() isn't called on 1 item arrays, so this is just fine Co-authored-by: jj --- api/src/processing/services/xiaohongshu.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/api/src/processing/services/xiaohongshu.js b/api/src/processing/services/xiaohongshu.js index 6e141f5a..45d7acc5 100644 --- a/api/src/processing/services/xiaohongshu.js +++ b/api/src/processing/services/xiaohongshu.js @@ -75,12 +75,9 @@ export default async function ({ id, token, shareId, h265, isAudioOnly, dispatch if (!videoURL) { const h264Streams = video.media?.stream?.h264; - if (!h264Streams) return { error: "fetch.empty" }; - if (h264Streams.length > 1) { + if (h264Streams?.length) { videoURL = h264Streams.reduce((a, b) => Number(a?.videoBitrate) > Number(b?.videoBitrate) ? a : b).masterUrl; - } else { - videoURL = h264Streams[0].masterUrl; } } From 7488c74fafb92b73ac58772f661e5601e2bb5cac Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:46:12 +0600 Subject: [PATCH 13/28] api/xiaohongshu: clean up the h265-h264 if statement Co-authored-by: jj --- api/src/processing/services/xiaohongshu.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api/src/processing/services/xiaohongshu.js b/api/src/processing/services/xiaohongshu.js index 45d7acc5..bbb53ab1 100644 --- a/api/src/processing/services/xiaohongshu.js +++ b/api/src/processing/services/xiaohongshu.js @@ -71,9 +71,7 @@ export default async function ({ id, token, shareId, h265, isAudioOnly, dispatch if (h265 && !isAudioOnly && video.consumer?.originVideoKey) { videoURL = `https://sns-video-bd.xhscdn.com/${video.consumer.originVideoKey}`; - } - - if (!videoURL) { + } else { const h264Streams = video.media?.stream?.h264; if (h264Streams?.length) { From 9f0f885ae6b0b46fa129d5e4b7ade67750ff35c4 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:59:59 +0600 Subject: [PATCH 14/28] web/settings/video: update h265 toggle strings because now it also applies to xiaohongshu --- web/i18n/en/settings.json | 6 +++--- web/src/routes/settings/video/+page.svelte | 17 +++++++++-------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/web/i18n/en/settings.json b/web/i18n/en/settings.json index c450b4b9..418410bf 100644 --- a/web/i18n/en/settings.json +++ b/web/i18n/en/settings.json @@ -40,9 +40,9 @@ "video.twitter.gif.title": "convert looping videos to GIF", "video.twitter.gif.description": "GIF conversion is inefficient, converted file may be obnoxiously big and low quality.", - "video.tiktok.h265": "tiktok", - "video.tiktok.h265.title": "prefer HEVC/H265 format", - "video.tiktok.h265.description": "allows downloading videos in 1080p at cost of compatibility.", + "video.h265": "high efficiency video codec", + "video.h265.title": "allow h265 for videos", + "video.h265.description": "allows downloading videos from platforms like tiktok and xiaohongshu in higher quality at cost of compatibility.", "audio.format": "audio format", "audio.format.best": "best", diff --git a/web/src/routes/settings/video/+page.svelte b/web/src/routes/settings/video/+page.svelte index 9897d7eb..88f9a5ca 100644 --- a/web/src/routes/settings/video/+page.svelte +++ b/web/src/routes/settings/video/+page.svelte @@ -69,6 +69,15 @@ /> + + + + - - - From 73f458a99913c794ac5600dfef824b71985e3449 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 20:01:55 +0600 Subject: [PATCH 15/28] docs/api: update tiktokH265 description --- docs/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api.md b/docs/api.md index 6da93a44..fb1a1450 100644 --- a/docs/api.md +++ b/docs/api.md @@ -68,7 +68,7 @@ Content-Type: application/json | `alwaysProxy` | `boolean` | `true / false` | `false` | tunnels all downloads through the processing server, even when not necessary. | | `disableMetadata` | `boolean` | `true / false` | `false` | disables file metadata when set to `true`. | | `tiktokFullAudio` | `boolean` | `true / false` | `false` | enables download of original sound used in a tiktok video. | -| `tiktokH265` | `boolean` | `true / false` | `false` | changes whether 1080p h265 videos are preferred or not. | +| `tiktokH265` | `boolean` | `true / false` | `false` | allows h265 videos when enabled. applies to tiktok & xiaohongshu. | | `twitterGif` | `boolean` | `true / false` | `true` | changes whether twitter gifs are converted to .gif | | `youtubeHLS` | `boolean` | `true / false` | `false` | specifies whether to use HLS for downloading video or audio from youtube. | From 035825bc0555fa2e1c2084a407ee14d04be97445 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 20 Jan 2025 14:38:55 +0000 Subject: [PATCH 16/28] api: cache original request parameters in stream --- api/src/processing/match-action.js | 3 ++- api/src/stream/manage.js | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/api/src/processing/match-action.js b/api/src/processing/match-action.js index 64f86836..19896ceb 100644 --- a/api/src/processing/match-action.js +++ b/api/src/processing/match-action.js @@ -15,7 +15,8 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab filename: r.filenameAttributes ? createFilename(r.filenameAttributes, filenameStyle, isAudioOnly, isAudioMuted) : r.filename, fileMetadata: !disableMetadata ? r.fileMetadata : false, - requestIP + requestIP, + originalRequest: r.originalRequest }, params = {}; diff --git a/api/src/stream/manage.js b/api/src/stream/manage.js index 79b5c1db..3323ce5d 100644 --- a/api/src/stream/manage.js +++ b/api/src/stream/manage.js @@ -40,6 +40,7 @@ export function createStream(obj) { audioFormat: obj.audioFormat, isHLS: obj.isHLS || false, + originalRequest: obj.parameters }; // FIXME: this is now a Promise, but it is not awaited From 7767a5f5bb49d05b3e4d6f5a9fdf86e4a038c3a1 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 20 Jan 2025 14:46:55 +0000 Subject: [PATCH 17/28] api/youtube: add support for pinning client/itag --- api/src/processing/services/youtube.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index e16e86e7..3ee673b9 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -149,7 +149,7 @@ export default async function (o) { useHLS = false; } - let innertubeClient = "ANDROID"; + let innertubeClient = o.innertubeClient || "ANDROID"; if (cookie) { useHLS = false; @@ -245,7 +245,7 @@ export default async function (o) { } let video, audio, dubbedLanguage, - codec = o.format || "h264"; + codec = o.format || "h264", itag = o.itag; if (useHLS) { const hlsManifest = info.streaming_data.hls_manifest_url; @@ -351,17 +351,21 @@ export default async function (o) { Number(b.bitrate) - Number(a.bitrate) ).forEach(format => { Object.keys(codecList).forEach(yCodec => { + const matchingItag = slot => !itag || itag[slot] === format.itag; const sorted = sorted_formats[yCodec]; const goodFormat = checkFormat(format, yCodec); if (!goodFormat) return; - if (format.has_video) { + if (format.has_video && matchingItag('video')) { sorted.video.push(format); - if (!sorted.bestVideo) sorted.bestVideo = format; + if (!sorted.bestVideo) + sorted.bestVideo = format; } - if (format.has_audio) { + + if (format.has_audio && matchingItag('audio')) { sorted.audio.push(format); - if (!sorted.bestAudio) sorted.bestAudio = format; + if (!sorted.bestAudio) + sorted.bestAudio = format; } }) }); From 19ade7c9053dc9323c35ccee25dffb25a6e19f27 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 20 Jan 2025 14:47:09 +0000 Subject: [PATCH 18/28] api/youtube: return internal metadata for replaying request --- api/src/processing/services/youtube.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index 3ee673b9..ff4a95cf 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -452,6 +452,18 @@ export default async function (o) { youtubeDubName: dubbedLanguage || false, } + itag = { + video: video.itag, + audio: audio.itag + }; + + const originalRequest = { + ...o, + dispatcher: undefined, + itag, + innertubeClient + }; + if (audio && o.isAudioOnly) { let bestAudio = codec === "h264" ? "m4a" : "opus"; let urls = audio.url; @@ -473,6 +485,7 @@ export default async function (o) { fileMetadata, bestAudio, isHLS: useHLS, + originalRequest } } @@ -516,6 +529,7 @@ export default async function (o) { filenameAttributes, fileMetadata, isHLS: useHLS, + originalRequest } } From 39752b2c5f00175eb07c700f97f886679f938853 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 21:26:55 +0600 Subject: [PATCH 19/28] web/Omnibox: improve pasting links from clipboard - `text/uri-list` type is now accepted (such as clipboard data from bluesky) - http links are now allowed (such as those from rednote) - rednote share link is properly extracted --- web/src/components/save/Omnibox.svelte | 25 ++++++++++++++----------- web/src/lib/clipboard.ts | 17 +++++++++++++++++ 2 files changed, 31 insertions(+), 11 deletions(-) create mode 100644 web/src/lib/clipboard.ts diff --git a/web/src/components/save/Omnibox.svelte b/web/src/components/save/Omnibox.svelte index 8da051d4..c10db574 100644 --- a/web/src/components/save/Omnibox.svelte +++ b/web/src/components/save/Omnibox.svelte @@ -11,6 +11,7 @@ import dialogs from "$lib/state/dialogs"; import { link } from "$lib/state/omnibox"; import { updateSetting } from "$lib/state/settings"; + import { pasteLinkFromClipboard } from "$lib/clipboard"; import { turnstileEnabled, turnstileSolved } from "$lib/state/turnstile"; import type { Optional } from "$lib/types/generic"; @@ -41,7 +42,7 @@ const validLink = (url: string) => { try { - return /^https:/i.test(new URL(url).protocol); + return /^https?\:/i.test(new URL(url).protocol); } catch {} }; @@ -59,22 +60,24 @@ goto("/", { replaceState: true }); } - const pasteClipboard = () => { + const pasteClipboard = async () => { if ($dialogs.length > 0 || isDisabled || isLoading) { return; } - navigator.clipboard.readText().then(async (text: string) => { - let matchLink = text.match(/https:\/\/[^\s]+/g); - if (matchLink) { - $link = matchLink[0]; + const pastedData = await pasteLinkFromClipboard(); + if (!pastedData) return; - if (!isBotCheckOngoing) { - await tick(); // wait for button to render - downloadButton.download($link); - } + const linkMatch = pastedData.match(/https?\:\/\/[^\s]+/g); + + if (linkMatch) { + $link = linkMatch[0].split('，')[0]; + + if (!isBotCheckOngoing) { + await tick(); // wait for button to render + downloadButton.download($link); } - }); + } }; const changeDownloadMode = (mode: DownloadModeOption) => { diff --git a/web/src/lib/clipboard.ts b/web/src/lib/clipboard.ts new file mode 100644 index 00000000..221d17ae --- /dev/null +++ b/web/src/lib/clipboard.ts @@ -0,0 +1,17 @@ +const allowedLinkTypes = new Set(["text/plain", "text/uri-list"]); + +export const pasteLinkFromClipboard = async () => { + const clipboard = await navigator.clipboard.read(); + + if (clipboard?.length) { + const clipboardItem = clipboard[0]; + for (const type of clipboardItem.types) { + if (allowedLinkTypes.has(type)) { + const blob = await clipboardItem.getType(type); + const blobText = await blob.text(); + + return blobText; + } + } + } +} From c07940bfa4fc5b5c26d2892074b901ba928a5184 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 20 Jan 2025 15:46:03 +0000 Subject: [PATCH 20/28] api/itunnel: pass itunnel object by reference --- api/src/core/api.js | 2 +- api/src/stream/stream.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/core/api.js b/api/src/core/api.js index 153f2ca6..e4d3dfcf 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -313,7 +313,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { ...Object.entries(req.headers) ]); - return stream(res, { type: 'internal', ...streamInfo }); + return stream(res, { type: 'internal', data: streamInfo }); }; app.get('/itunnel', itunnelHandler); diff --git a/api/src/stream/stream.js b/api/src/stream/stream.js index a6d41200..c7cf7b56 100644 --- a/api/src/stream/stream.js +++ b/api/src/stream/stream.js @@ -10,7 +10,7 @@ export default async function(res, streamInfo) { return await stream.proxy(streamInfo, res); case "internal": - return internalStream(streamInfo, res); + return internalStream(streamInfo.data, res); case "merge": return stream.merge(streamInfo, res); From 600c7691414f07568e038cca7877426b4ab9057d Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 20 Jan 2025 15:55:26 +0000 Subject: [PATCH 21/28] api/stream: implement itunnel transplants --- api/src/misc/utils.js | 4 +++ api/src/stream/internal.js | 12 ++++++- api/src/stream/manage.js | 71 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 83 insertions(+), 4 deletions(-) diff --git a/api/src/misc/utils.js b/api/src/misc/utils.js index fd497d18..e15690b0 100644 --- a/api/src/misc/utils.js +++ b/api/src/misc/utils.js @@ -29,3 +29,7 @@ export function splitFilenameExtension(filename) { return [ parts.join('.'), ext ] } } + +export function zip(a, b) { + return a.map((value, i) => [ value, b[i] ]); +} diff --git a/api/src/stream/internal.js b/api/src/stream/internal.js index 7d8bf4c9..8c94c485 100644 --- a/api/src/stream/internal.js +++ b/api/src/stream/internal.js @@ -7,7 +7,7 @@ const CHUNK_SIZE = BigInt(8e6); // 8 MB const min = (a, b) => a < b ? a : b; async function* readChunks(streamInfo, size) { - let read = 0n; + let read = 0n, chunksSinceTransplant = 0; while (read < size) { if (streamInfo.controller.signal.aborted) { throw new Error("controller aborted"); @@ -22,6 +22,16 @@ async function* readChunks(streamInfo, size) { signal: streamInfo.controller.signal }); + if (chunk.statusCode === 403 && chunksSinceTransplant >= 3 && streamInfo.transplant) { + chunksSinceTransplant = 0; + try { + await streamInfo.transplant(streamInfo.dispatcher); + continue; + } catch {} + } + + chunksSinceTransplant++; + const expected = min(CHUNK_SIZE, size - read); const received = BigInt(chunk.headers['content-length']); diff --git a/api/src/stream/manage.js b/api/src/stream/manage.js index 3323ce5d..ebb5c6c7 100644 --- a/api/src/stream/manage.js +++ b/api/src/stream/manage.js @@ -9,6 +9,7 @@ import { env } from "../config.js"; import { closeRequest } from "./shared.js"; import { decryptStream, encryptStream } from "../misc/crypto.js"; import { hashHmac } from "../security/secrets.js"; +import { zip } from "../misc/utils.js"; // optional dependency const freebind = env.freebindCIDR && await import('freebind').catch(() => {}); @@ -40,7 +41,7 @@ export function createStream(obj) { audioFormat: obj.audioFormat, isHLS: obj.isHLS || false, - originalRequest: obj.parameters + originalRequest: obj.originalRequest }; // FIXME: this is now a Promise, but it is not awaited @@ -101,6 +102,7 @@ export function createInternalStream(url, obj = {}) { controller, dispatcher, isHLS: obj.isHLS, + transplant: obj.transplant }); let streamLink = new URL('/itunnel', `http://127.0.0.1:${env.tunnelPort}`); @@ -116,13 +118,17 @@ export function createInternalStream(url, obj = {}) { return streamLink.toString(); } -export function destroyInternalStream(url) { +function getInternalTunnelId(url) { url = new URL(url); if (url.hostname !== '127.0.0.1') { return; } - const id = url.searchParams.get('id'); + return url.searchParams.get('id'); +} + +export function destroyInternalStream(url) { + const id = getInternalTunnelId(url); if (internalStreamCache.has(id)) { closeRequest(getInternalStream(id)?.controller); @@ -130,9 +136,68 @@ export function destroyInternalStream(url) { } } +const transplantInternalTunnels = function(tunnelUrls, transplantUrls) { + if (tunnelUrls.length !== transplantUrls.length) { + return; + } + + for (const [ tun, url ] of zip(tunnelUrls, transplantUrls)) { + const id = getInternalTunnelId(tun); + const itunnel = getInternalStream(id); + + if (!itunnel) continue; + itunnel.url = url; + } +} + +const transplantTunnel = async function (dispatcher) { + if (this.pendingTransplant) { + await this.pendingTransplant; + return; + } + + let finished; + this.pendingTransplant = new Promise(r => finished = r); + + try { + const handler = await import(`../processing/services/${this.service}.js`); + const response = await handler.default({ + ...this.originalRequest, + dispatcher + }); + + if (!response.urls) { + return; + } + + response.urls = [response.urls].flat(); + if (this.originalRequest.isAudioOnly && response.urls.length > 1) { + response.urls = [response.urls[1]]; + } else if (this.originalRequest.isAudioMuted) { + response.urls = [response.urls[0]]; + } + + const tunnels = [this.urls].flat(); + if (tunnels.length !== response.urls.length) { + return; + } + + transplantInternalTunnels(tunnels, response.urls); + } + catch {} + finally { + finished(); + delete this.pendingTransplant; + } +} + function wrapStream(streamInfo) { const url = streamInfo.urls; + if (streamInfo.originalRequest) { + streamInfo.transplant = transplantTunnel.bind(streamInfo); + } + if (typeof url === 'string') { streamInfo.urls = createInternalStream(url, streamInfo); } else if (Array.isArray(url)) { From ee3ef60a20701adf40fa5b4bb89c6db37cd33af9 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 20 Jan 2025 20:12:21 +0000 Subject: [PATCH 22/28] api/youtube: expect one of itags to be empty --- api/src/processing/services/youtube.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index ff4a95cf..f0766b3f 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -351,7 +351,7 @@ export default async function (o) { Number(b.bitrate) - Number(a.bitrate) ).forEach(format => { Object.keys(codecList).forEach(yCodec => { - const matchingItag = slot => !itag || itag[slot] === format.itag; + const matchingItag = slot => !itag?.[slot] || itag[slot] === format.itag; const sorted = sorted_formats[yCodec]; const goodFormat = checkFormat(format, yCodec); if (!goodFormat) return; @@ -453,8 +453,8 @@ export default async function (o) { } itag = { - video: video.itag, - audio: audio.itag + video: video?.itag, + audio: audio?.itag }; const originalRequest = { From 36d4608ee58b076912d5611594f2922fa77ab90a Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 21 Jan 2025 17:18:49 +0600 Subject: [PATCH 23/28] api/bluesky: add support for tenor gifs --- api/src/processing/services/bluesky.js | 61 ++++++++++++++++++++------ api/src/util/tests/bsky.json | 18 ++++++++ 2 files changed, 65 insertions(+), 14 deletions(-) diff --git a/api/src/processing/services/bluesky.js b/api/src/processing/services/bluesky.js index bc887437..598e9739 100644 --- a/api/src/processing/services/bluesky.js +++ b/api/src/processing/services/bluesky.js @@ -71,6 +71,24 @@ const extractImages = ({ getPost, filename, alwaysProxy }) => { return { picker }; } +const extractGif = ({ url, filename }) => { + const gifUrl = new URL(url); + + if (!gifUrl || gifUrl.hostname !== "media.tenor.com") { + return { error: "fetch.empty" }; + } + + // remove downscaling params from gif url + // such as "?hh=498&ww=498" + gifUrl.search = ""; + + return { + urls: gifUrl, + isPhoto: true, + filename: `${filename}.gif`, + } +} + export default async function ({ user, post, alwaysProxy, dispatcher }) { const apiEndpoint = new URL("https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread?depth=0&parentHeight=0"); apiEndpoint.searchParams.set( @@ -102,22 +120,37 @@ export default async function ({ user, post, alwaysProxy, dispatcher }) { const embedType = getPost?.thread?.post?.embed?.$type; const filename = `bluesky_${user}_${post}`; - if (embedType === "app.bsky.embed.video#view") { - return extractVideo({ - media: getPost.thread?.post?.embed, - filename, - }) - } + switch (embedType) { + case "app.bsky.embed.video#view": + return extractVideo({ + media: getPost.thread?.post?.embed, + filename, + }); - if (embedType === "app.bsky.embed.recordWithMedia#view") { - return extractVideo({ - media: getPost.thread?.post?.embed?.media, - filename, - }) - } + case "app.bsky.embed.images#view": + return extractImages({ + getPost, + filename, + alwaysProxy + }); - if (embedType === "app.bsky.embed.images#view") { - return extractImages({ getPost, filename, alwaysProxy }); + case "app.bsky.embed.external#view": + return extractGif({ + url: getPost?.thread?.post?.embed?.external?.uri, + filename, + }); + + case "app.bsky.embed.recordWithMedia#view": + if (getPost?.thread?.post?.embed?.media?.$type === "app.bsky.embed.external#view") { + return extractGif({ + url: getPost?.thread?.post?.embed?.media?.external?.uri, + filename, + }); + } + return extractVideo({ + media: getPost.thread?.post?.embed?.media, + filename, + }); } return { error: "fetch.empty" }; diff --git a/api/src/util/tests/bsky.json b/api/src/util/tests/bsky.json index 5a03538d..6e1d6b2b 100644 --- a/api/src/util/tests/bsky.json +++ b/api/src/util/tests/bsky.json @@ -57,6 +57,24 @@ "status": "tunnel" } }, + { + "name": "gif with a quoted post", + "url": "https://bsky.app/profile/imlunahey.com/post/3lgajpn5dtk2t", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "gif alone in a post", + "url": "https://bsky.app/profile/imlunahey.com/post/3lgah3ovxnc2q", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, { "name": "several images", "url": "https://bsky.app/profile/did:plc:rai7s6su2sy22ss7skouedl7/post/3kzxuxbiul626", From cecb8a4c5343b4b2cc166e8995dfb599fe222743 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 21 Jan 2025 17:25:45 +0600 Subject: [PATCH 24/28] api/package: bump version to 10.6 --- api/package.json | 4 ++-- pnpm-lock.yaml | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/package.json b/api/package.json index fccbddff..829106c3 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "10.5.4", + "version": "10.6", "author": "imput", "exports": "./src/cobalt.js", "type": "module", @@ -39,7 +39,7 @@ "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", - "youtubei.js": "^12.2.0", + "youtubei.js": "^13.0.0", "zod": "^3.23.8" }, "optionalDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c7f3b712..9e12913a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,8 +56,8 @@ importers: specifier: 1.0.3 version: 1.0.3 youtubei.js: - specifier: ^12.2.0 - version: 12.2.0 + specifier: ^13.0.0 + version: 13.0.0 zod: specifier: ^3.23.8 version: 3.23.8 @@ -2286,8 +2286,8 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} - youtubei.js@12.2.0: - resolution: {integrity: sha512-G+50qrbJCToMYhu8jbaHiS3Vf+RRul+CcDbz3hEGwHkGPh+zLiWwD6SS+YhYF+2/op4ZU5zDYQJrGqJ+wKh7Gw==} + youtubei.js@13.0.0: + resolution: {integrity: sha512-b1QkN9bfgphK+5tI4qteSK54kNxmPhoedvMw0jl4uSn+L8gbDbJ4z52amNuYNcOdp4X/SI3JuUb+f5V0DPJ8Vw==} zod@3.23.8: resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} @@ -4242,7 +4242,7 @@ snapshots: yocto-queue@0.1.0: {} - youtubei.js@12.2.0: + youtubei.js@13.0.0: dependencies: '@bufbuild/protobuf': 2.1.0 jintr: 3.2.0 From 8d3db909d95b798e9a051cc2f023d08fe439baa2 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 21 Jan 2025 17:25:55 +0600 Subject: [PATCH 25/28] web/package: bump version to 10.6 --- web/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/package.json b/web/package.json index 37a7bb49..0c621f02 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "@imput/cobalt-web", - "version": "10.5.1", + "version": "10.6", "type": "module", "private": true, "scripts": { From 99265d594bc171b2881312782cb9551241f3882b Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 22 Jan 2025 14:41:44 +0600 Subject: [PATCH 26/28] api/readme: update list of supported services & list of dependencies --- api/README.md | 53 ++++++++++++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/api/README.md b/api/README.md index 84c534ea..70d85de6 100644 --- a/api/README.md +++ b/api/README.md @@ -11,12 +11,9 @@ we recommend [deploying your own instance](/docs/run-an-instance.md) if you wish you can read [the api documentation here](/docs/api.md). -> [!WARNING] -> the v7 public api (/api/json) will be shut down on **november 11th, 2024**. -> you can access documentation for it [here](https://github.com/imputnet/cobalt/blob/7/docs/api.md). - ## supported services -this list is not final and keeps expanding over time. if support for a service you want is missing, create an issue (or a pull request 👀). +this list is not final and keeps expanding over time! +if the desired service isn't supported yet, feel free to create an appropriate issue (or a pull request 👀). | service | video + audio | only audio | only video | metadata | rich file names | | :-------- | :-----------: | :--------: | :--------: | :------: | :-------------: | @@ -39,12 +36,13 @@ this list is not final and keeps expanding over time. if support for a service y | twitter/x | ✅ | ✅ | ✅ | ➖ | ➖ | | vimeo | ✅ | ✅ | ✅ | ✅ | ✅ | | vk videos & clips | ✅ | ❌ | ✅ | ✅ | ✅ | +| xiaohongshu | ✅ | ✅ | ✅ | ➖ | ➖ | | youtube | ✅ | ✅ | ✅ | ✅ | ✅ | | emoji | meaning | | :-----: | :---------------------- | | ✅ | supported | -| ➖ | impossible/unreasonable | +| ➖ | unreasonable/impossible | | ❌ | not supported | ### additional notes or features (per service) @@ -71,36 +69,35 @@ as long as you: - provide a link to the license and indicate if changes to the code were made, and - release the code under the **same license** -## acknowledgements +## open source acknowledgements ### ffmpeg -cobalt heavily relies on ffmpeg for converting and merging media files. it's an absolutely amazing piece of software offered for anyone for free, yet doesn't receive as much credit as it should. +cobalt relies on ffmpeg for muxing and encoding media files. ffmpeg is absolutely spectacular and we're privileged to have an ability to use it for free, just like anyone else. we believe it should be way more recognized. you can [support ffmpeg here](https://ffmpeg.org/donations.html)! -#### ffmpeg-static -we use [ffmpeg-static](https://github.com/eugeneware/ffmpeg-static) to get binaries for ffmpeg depending on the platform. - -you can support the developer via various methods listed on their github page! (linked above) - ### youtube.js -cobalt relies on [youtube.js](https://github.com/LuanRT/YouTube.js) for interacting with the innertube api, it wouldn't have been possible without it. +cobalt relies on **[youtube.js](https://github.com/LuanRT/YouTube.js)** for interacting with youtube's innertube api, it wouldn't have been possible without this package. -you can support the developer via various methods listed on their github page! (linked above) +you can support the developer via various methods listed on their github page! +(linked above) ### many others -cobalt also depends on: +cobalt-api also depends on: -- [content-disposition-header](https://www.npmjs.com/package/content-disposition-header) to simplify the provision of `content-disposition` headers. -- [cors](https://www.npmjs.com/package/cors) to manage cross-origin resource sharing within expressjs. -- [dotenv](https://www.npmjs.com/package/dotenv) to load environment variables from the `.env` file. -- [express](https://www.npmjs.com/package/express) as the backbone of cobalt servers. -- [express-rate-limit](https://www.npmjs.com/package/express-rate-limit) to rate limit api endpoints. -- [hls-parser](https://www.npmjs.com/package/hls-parser) to parse `m3u8` playlists for certain services. -- [ipaddr.js](https://www.npmjs.com/package/ipaddr.js) to parse ip addresses (for rate limiting). -- [nanoid](https://www.npmjs.com/package/nanoid) to generate unique (temporary) identifiers for each requested stream. -- [psl](https://www.npmjs.com/package/psl) as the domain name parser. -- [set-cookie-parser](https://www.npmjs.com/package/set-cookie-parser) to parse cookies that cobalt receives from certain services. -- [undici](https://www.npmjs.com/package/undici) for making http requests. -- [url-pattern](https://www.npmjs.com/package/url-pattern) to match provided links with supported patterns. +- **[content-disposition-header](https://www.npmjs.com/package/content-disposition-header)** to simplify the provision of `content-disposition` headers. +- **[cors](https://www.npmjs.com/package/cors)** to manage cross-origin resource sharing within expressjs. +- **[dotenv](https://www.npmjs.com/package/dotenv)** to load environment variables from the `.env` file. +- **[express](https://www.npmjs.com/package/express)** as the backbone of cobalt servers. +- **[express-rate-limit](https://www.npmjs.com/package/express-rate-limit)** to rate limit api endpoints. +- **[ffmpeg-static](https://www.npmjs.com/package/ffmpeg-static)** to get binaries for ffmpeg depending on the platform. +- **[hls-parser](https://www.npmjs.com/package/hls-parser)** to parse HLS playlists according to spec (very impressive stuff). +- **[ipaddr.js](https://www.npmjs.com/package/ipaddr.js)** to parse ip addresses (used for rate limiting). +- **[nanoid](https://www.npmjs.com/package/nanoid)** to generate unique identifiers for each requested tunnel. +- **[set-cookie-parser](https://www.npmjs.com/package/set-cookie-parser)** to parse cookies that cobalt receives from certain services. +- **[undici](https://www.npmjs.com/package/undici)** for making http requests. +- **[url-pattern](https://www.npmjs.com/package/url-pattern)** to match provided links with supported patterns. +- **[zod](https://www.npmjs.com/package/zod)** to lock down the api request schema. +- **[@datastructures-js/priority-queue](https://www.npmjs.com/package/@datastructures-js/priority-queue)** for sorting stream caches for future clean up (without redis). +- **[@imput/psl](https://www.npmjs.com/package/@imput/psl)** as the domain name parser, our fork of [psl](https://www.npmjs.com/package/psl). ...and many other packages that these packages rely on. From 3be98a14b369289b4b50b2ec9147d76287c3f2e3 Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 22 Jan 2025 14:46:09 +0600 Subject: [PATCH 27/28] readme: update some phrasing & add a link to bluesky --- README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8168a4df..9a5a05e7 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,13 @@ 💬 community discord server +
🐦 twitter + + 🦋 bluesky +


@@ -34,11 +38,10 @@ this monorepo includes source code for api, frontend, and related packages: it also includes documentation in the [docs tree](/docs/): - [cobalt api documentation](/docs/api.md) - [how to run a cobalt instance](/docs/run-an-instance.md) -- [how to protect a cobalt instance](/docs/protect-an-instance.md) -- [how to configure a cobalt instance for youtube](/docs/configure-for-youtube.md) +- [how to protect a cobalt instance](/docs/protect-an-instance.md) (recommended if you host a public instance) ### thank you -cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt) and the main processing servers are hosted on their network. we really appreciate their kindness and support! +cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt). a part of our infrastructure is hosted on their network. we really appreciate their kindness and support! ### ethics cobalt is a tool that makes downloading public content easier. it takes **zero liability**. @@ -50,7 +53,7 @@ it can only download free & publicly accessible content. same content can be downloaded via dev tools of any modern web browser. ### contributing -thank you for considering making a contribution to cobalt! please check the [contributing guidelines here](/CONTRIBUTING.md) before making a pull request. +if you're considering contributing to cobalt, first of all, thank you! check the [contribution guidelines here](/CONTRIBUTING.md) before getting started, they'll help you do your best right away. ### licenses for relevant licensing information, see the [api](api/README.md) and [web](web/README.md) READMEs. From 899d1efdea2d1b23cb88d9d33ca50245ff094a69 Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 22 Jan 2025 14:46:30 +0600 Subject: [PATCH 28/28] web/about/general: update infra partner phrasing --- web/i18n/en/about/general.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/i18n/en/about/general.md b/web/i18n/en/about/general.md index 2334ab8e..1f5e53fd 100644 --- a/web/i18n/en/about/general.md +++ b/web/i18n/en/about/general.md @@ -25,7 +25,7 @@ no ads, trackers, paywalls, or other nonsense. just a convenient web app that wo cobalt was created for public benefit, to protect people from ads and malware pushed by its alternatives. we believe that the best software is safe, open, and accessible. -it's possible to keep the main instances up thanks to our long-standing infrastructure partner, [royalehosting.net]({partners.royalehosting})! +a part of our infrastructure is provided by our long-standing partner, [royalehosting.net]({partners.royalehosting})!