From ef687750b41af5807c84280aa12ee9bb9947ce85 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 18 Jan 2025 17:02:24 +0600 Subject: [PATCH 01/26] api/tiktok: update domain because dns records for main one are gone closes #1057 --- api/src/processing/services/tiktok.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/processing/services/tiktok.js b/api/src/processing/services/tiktok.js index 6978e071..6fec01d8 100644 --- a/api/src/processing/services/tiktok.js +++ b/api/src/processing/services/tiktok.js @@ -30,7 +30,7 @@ export default async function(obj) { if (!postId) return { error: "fetch.short_link" }; // should always be /video/, even for photos - const res = await fetch(`https://tiktok.com/@i/video/${postId}`, { + const res = await fetch(`https://www.tiktok.com/@i/video/${postId}`, { headers: { "user-agent": genericUserAgent, cookie, From 0378a1ae15ed6d1cefdded8efbd7e5a993b01491 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 20 Jan 2025 12:37:36 +0000 Subject: [PATCH 02/26] api/youtube: fix error when downloading stuff from WEB --- api/src/processing/services/youtube.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index 6559978c..e0dc8594 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -491,12 +491,12 @@ export default async function (o) { filenameAttributes.resolution = `${video.width}x${video.height}`; filenameAttributes.extension = codecList[codec].container; - video = video.url; - audio = audio.url; - if (innertubeClient === "WEB" && innertube) { video = video.decipher(innertube.session.player); audio = audio.decipher(innertube.session.player); + } else { + video = video.url; + audio = audio.url; } } From ec0d7737926d5d4f7c4db0dc71964c9291452770 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 20 Jan 2025 12:38:12 +0000 Subject: [PATCH 03/26] api/youtube: use Math.min instead of ternary operator --- api/src/processing/services/youtube.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index e0dc8594..e16e86e7 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -240,7 +240,7 @@ export default async function (o) { const quality = o.quality === "max" ? 9000 : Number(o.quality); const normalizeQuality = res => { - const shortestSide = res.height > res.width ? res.width : res.height; + const shortestSide = Math.min(res.height, res.width); return videoQualities.find(qual => qual >= shortestSide); } From 9bdcb9d8216940aea0b1d499e80478477683b182 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 18:51:37 +0600 Subject: [PATCH 04/26] api/utils: update getRedirectingURL to accept more statuses & dispatcher --- api/src/misc/utils.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/api/src/misc/utils.js b/api/src/misc/utils.js index fd497d18..6bc72176 100644 --- a/api/src/misc/utils.js +++ b/api/src/misc/utils.js @@ -1,8 +1,16 @@ -export function getRedirectingURL(url) { - return fetch(url, { redirect: 'manual' }).then((r) => { - if ([301, 302, 303].includes(r.status) && r.headers.has('location')) +const redirectStatuses = [301, 302, 303, 307, 308]; + +export async function getRedirectingURL(url, dispatcher) { + const location = await fetch(url, { + redirect: 'manual', + dispatcher, + }).then((r) => { + if (redirectStatuses.includes(r.status) && r.headers.has('location')) { return r.headers.get('location'); + } }).catch(() => null); + + return location; } export function merge(a, b) { From 63b2681017df728f1aeeffae393f5da5a663805a Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:04:31 +0600 Subject: [PATCH 05/26] api/match-action: always proxy photos --- api/src/processing/match-action.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/processing/match-action.js b/api/src/processing/match-action.js index 64f86836..1cd36eb2 100644 --- a/api/src/processing/match-action.js +++ b/api/src/processing/match-action.js @@ -47,7 +47,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab }); case "photo": - responseType = "redirect"; + params = { type: "proxy" }; break; case "gif": From ed8f4353ea3346982d69eb2aadc4cfa5a8c26855 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:10:02 +0600 Subject: [PATCH 06/26] api/processing: add support for xiaohongshu --- api/src/processing/match-action.js | 2 + api/src/processing/match.js | 10 ++ api/src/processing/service-config.js | 8 ++ api/src/processing/service-patterns.js | 4 + api/src/processing/services/xiaohongshu.js | 123 +++++++++++++++++++++ api/src/processing/url.js | 26 +++-- 6 files changed, 165 insertions(+), 8 deletions(-) create mode 100644 api/src/processing/services/xiaohongshu.js diff --git a/api/src/processing/match-action.js b/api/src/processing/match-action.js index 1cd36eb2..0d87b5a8 100644 --- a/api/src/processing/match-action.js +++ b/api/src/processing/match-action.js @@ -83,6 +83,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab case "twitter": case "snapchat": case "bsky": + case "xiaohongshu": params = { picker: r.picker }; break; @@ -143,6 +144,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab case "ok": case "vk": case "tiktok": + case "xiaohongshu": params = { type: "proxy" }; break; diff --git a/api/src/processing/match.js b/api/src/processing/match.js index 57f04b36..9fabf379 100644 --- a/api/src/processing/match.js +++ b/api/src/processing/match.js @@ -28,6 +28,7 @@ import snapchat from "./services/snapchat.js"; import loom from "./services/loom.js"; import facebook from "./services/facebook.js"; import bluesky from "./services/bluesky.js"; +import xiaohongshu from "./services/xiaohongshu.js"; let freebind; @@ -239,6 +240,15 @@ export default async function({ host, patternMatch, params }) { }); break; + case "xiaohongshu": + r = await xiaohongshu({ + ...patternMatch, + h265: params.tiktokH265, + isAudioOnly, + dispatcher, + }); + break; + default: return createResponse("error", { code: "error.api.service.unsupported" diff --git a/api/src/processing/service-config.js b/api/src/processing/service-config.js index 81afaf39..86352f9a 100644 --- a/api/src/processing/service-config.js +++ b/api/src/processing/service-config.js @@ -166,6 +166,14 @@ export const services = { subdomains: ["m"], altDomains: ["vkvideo.ru", "vk.ru"], }, + xiaohongshu: { + patterns: [ + "explore/:id?xsec_token=:token", + "discovery/item/:id?xsec_token=:token", + "a/:shareId" + ], + altDomains: ["xhslink.com"], + }, youtube: { patterns: [ "watch?v=:id", diff --git a/api/src/processing/service-patterns.js b/api/src/processing/service-patterns.js index e8c46639..42f64d26 100644 --- a/api/src/processing/service-patterns.js +++ b/api/src/processing/service-patterns.js @@ -71,4 +71,8 @@ export const testers = { "bsky": pattern => pattern.user?.length <= 128 && pattern.post?.length <= 128, + + "xiaohongshu": pattern => + pattern.id?.length <= 24 && pattern.token?.length <= 64 + || pattern.shareId?.length <= 12, } diff --git a/api/src/processing/services/xiaohongshu.js b/api/src/processing/services/xiaohongshu.js new file mode 100644 index 00000000..d428aaf9 --- /dev/null +++ b/api/src/processing/services/xiaohongshu.js @@ -0,0 +1,123 @@ +import { extract, normalizeURL } from "../url.js"; +import { genericUserAgent } from "../../config.js"; +import { createStream } from "../../stream/manage.js"; +import { getRedirectingURL } from "../../misc/utils.js"; + +const https = (url) => { + return url.replace(/^http:/i, 'https:'); +} + +export default async function ({ id, token, shareId, h265, isAudioOnly, dispatcher }) { + let noteId = id; + let xsecToken = token; + + if (!noteId) { + const extractedURL = await getRedirectingURL( + `https://xhslink.com/a/${shareId}`, + dispatcher + ); + + if (extractedURL) { + const { patternMatch } = extract(normalizeURL(extractedURL)); + + if (patternMatch) { + noteId = patternMatch.id; + xsecToken = patternMatch.token; + } + } + } + + if (!noteId || !xsecToken) return { error: "fetch.short_link" }; + + const res = await fetch(`https://www.xiaohongshu.com/explore/${noteId}?xsec_token=${xsecToken}`, { + headers: { + "user-agent": genericUserAgent, + }, + dispatcher, + }); + + const html = await res.text(); + + let note; + try { + const initialState = html + .split('')[0] + .replace(/:undefined/g, ":null"); + + const data = JSON.parse(initialState); + + const noteInfo = data?.note?.noteDetailMap; + if (!noteInfo) throw "no note detail map"; + + const currentNote = noteInfo[noteId]; + if (!currentNote) throw "no current note in detail map"; + + note = currentNote.note; + } catch { + return { error: "fetch.empty" }; + } + + if (!note) return { error: "fetch.empty" }; + + const video = note.video; + const images = note.imageList; + + const filenameBase = `xiaohongshu_${noteId}`; + + if (video) { + const videoFilename = `${filenameBase}.mp4`; + const audioFilename = `${filenameBase}_audio`; + + let videoURL; + + if (h265 && !isAudioOnly && video.consumer?.originVideoKey) { + videoURL = `https://sns-video-bd.xhscdn.com/${video.consumer.originVideoKey}`; + } + + if (!videoURL) { + const h264Streams = video.media?.stream?.h264; + if (!h264Streams) return { error: "fetch.empty" }; + + if (h264Streams.length > 1) { + videoURL = h264Streams.reduce((a, b) => Number(a?.videoBitrate) > Number(b?.videoBitrate) ? a : b).masterUrl; + } else { + videoURL = h264Streams[0].masterUrl; + } + } + + if (!videoURL) return { error: "fetch.empty" }; + + return { + urls: https(videoURL), + filename: videoFilename, + audioFilename: audioFilename, + } + } + + if (!images || images.length === 0) { + return { error: "fetch.empty" }; + } + + if (images.length === 1) { + return { + isPhoto: true, + urls: https(images[0].urlDefault), + filename: `${filenameBase}.jpg`, + } + } + + const picker = images.map((image, i) => { + return { + type: "photo", + url: createStream({ + service: "xiaohongshu", + type: "proxy", + url: https(image.urlDefault), + filename: `${filenameBase}_${i + 1}.jpg`, + }) + } + }); + + return { picker }; +} diff --git a/api/src/processing/url.js b/api/src/processing/url.js index 8f0e7dc2..cfbbecc0 100644 --- a/api/src/processing/url.js +++ b/api/src/processing/url.js @@ -92,9 +92,14 @@ function aliasURL(url) { url.hostname = 'vk.com'; } break; + + case "xhslink": + if (url.hostname === 'xhslink.com' && parts.length === 3) { + url = new URL(`https://www.xiaohongshu.com/a/${parts[2]}`); + } } - return url + return url; } function cleanURL(url) { @@ -114,36 +119,41 @@ function cleanURL(url) { break; case "vk": if (url.pathname.includes('/clip') && url.searchParams.get('z')) { - limitQuery('z') + limitQuery('z'); } break; case "youtube": if (url.searchParams.get('v')) { - limitQuery('v') + limitQuery('v'); } break; case "rutube": if (url.searchParams.get('p')) { - limitQuery('p') + limitQuery('p'); } break; case "twitter": if (url.searchParams.get('post_id')) { - limitQuery('post_id') + limitQuery('post_id'); + } + break; + case "xiaohongshu": + if (url.searchParams.get('xsec_token')) { + limitQuery('xsec_token'); } break; } if (stripQuery) { - url.search = '' + url.search = ''; } - url.username = url.password = url.port = url.hash = '' + url.username = url.password = url.port = url.hash = ''; if (url.pathname.endsWith('/')) url.pathname = url.pathname.slice(0, -1); - return url + return url; } function getHostIfValid(url) { From ad6f29a3c83047de5b7f007d6cd103b03f9aac34 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:21:44 +0600 Subject: [PATCH 07/26] api/tests: add xiaohongshu tests --- api/src/util/tests/xiaohongshu.json | 58 +++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 api/src/util/tests/xiaohongshu.json diff --git a/api/src/util/tests/xiaohongshu.json b/api/src/util/tests/xiaohongshu.json new file mode 100644 index 00000000..425a6a2e --- /dev/null +++ b/api/src/util/tests/xiaohongshu.json @@ -0,0 +1,58 @@ +[ + { + "name": "long link video", + "url": "https://www.xiaohongshu.com/discovery/item/6789065900000000210035fc?source=webshare&xhsshare=pc_web&xsec_token=CBustnz_Twf1BSybpe5-D-BzUb-Bx28DPLb418TN9S9Kk&xsec_source=pc_share", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "picker with multiple live photos", + "url": "https://www.xiaohongshu.com/explore/67847fa1000000000203e6ed?xsec_token=CBzyP7Y44PPpsM20lgxqrIIJMHqOLemusDsRcmsX0cTpk", + "params": {}, + "expected": { + "code": 200, + "status": "picker" + } + }, + { + "name": "one photo", + "url": "https://www.xiaohongshu.com/explore/6788b56200000000210008c8?xsec_token=CBSDiWU4N-DgirHrOVbIWrlKfUNFHKwm-Wsjqz7dIMc_k", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "short link, might expire eventually", + "url": "https://xhslink.com/a/czn4z6c1tic4", + "canFail": true, + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "wrong note id", + "url": "https://www.xiaohongshu.com/discovery/item/6789065911100000210035fc?source=webshare&xhsshare=pc_web&xsec_token=CBustnz_Twf1BSybpe5-D-BzUb-Bx28DPLb418TN9S9Kk&xsec_source=pc_share", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "short link, wrong id", + "url": "https://xhslink.com/a/aaaaaa", + "canFail": true, + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + } +] From cd466a418ac18b85d3685a3af34a0885ea138ba5 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:24:12 +0600 Subject: [PATCH 08/26] api/tests/bsky: fix expected photo test status --- api/src/util/tests/bsky.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/util/tests/bsky.json b/api/src/util/tests/bsky.json index 840f1169..5a03538d 100644 --- a/api/src/util/tests/bsky.json +++ b/api/src/util/tests/bsky.json @@ -54,7 +54,7 @@ "params": {}, "expected": { "code": 200, - "status": "redirect" + "status": "tunnel" } }, { From cd0a2a47c9eb6a4a29c75db1cbcdfe55133d4b37 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:28:35 +0600 Subject: [PATCH 09/26] api/tests/pinterest: update expected photo status --- api/src/util/tests/pinterest.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/src/util/tests/pinterest.json b/api/src/util/tests/pinterest.json index 2f15fb0b..6308adb4 100644 --- a/api/src/util/tests/pinterest.json +++ b/api/src/util/tests/pinterest.json @@ -54,7 +54,7 @@ "params": {}, "expected": { "code": 200, - "status": "redirect" + "status": "tunnel" } }, { @@ -63,7 +63,7 @@ "params": {}, "expected": { "code": 200, - "status": "redirect" + "status": "tunnel" } }, { @@ -72,7 +72,7 @@ "params": {}, "expected": { "code": 200, - "status": "redirect" + "status": "tunnel" } }, { @@ -81,7 +81,7 @@ "params": {}, "expected": { "code": 200, - "status": "redirect" + "status": "tunnel" } } ] \ No newline at end of file From de5eca19a5c3d82462b691b8831034985c98a847 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:30:11 +0600 Subject: [PATCH 10/26] api/utils: replace redirectStatuses array with a set Co-authored-by: jj --- api/src/misc/utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/misc/utils.js b/api/src/misc/utils.js index 6bc72176..9978ceed 100644 --- a/api/src/misc/utils.js +++ b/api/src/misc/utils.js @@ -1,11 +1,11 @@ -const redirectStatuses = [301, 302, 303, 307, 308]; +const redirectStatuses = new Set([301, 302, 303, 307, 308]); export async function getRedirectingURL(url, dispatcher) { const location = await fetch(url, { redirect: 'manual', dispatcher, }).then((r) => { - if (redirectStatuses.includes(r.status) && r.headers.has('location')) { + if (redirectStatuses.has(r.status) && r.headers.has('location')) { return r.headers.get('location'); } }).catch(() => null); From 3cbed87c3ebaae58b3bab01a67c852d0f0c495cd Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:35:53 +0600 Subject: [PATCH 11/26] api/xiaohongshu: update initial state extraction regex Co-authored-by: jj --- api/src/processing/services/xiaohongshu.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/processing/services/xiaohongshu.js b/api/src/processing/services/xiaohongshu.js index d428aaf9..4637358e 100644 --- a/api/src/processing/services/xiaohongshu.js +++ b/api/src/processing/services/xiaohongshu.js @@ -43,7 +43,7 @@ export default async function ({ id, token, shareId, h265, isAudioOnly, dispatch const initialState = html .split('')[0] - .replace(/:undefined/g, ":null"); + .replace(/:\s*undefined/g, ":null"); const data = JSON.parse(initialState); From 4963c9f128fc71cf822dca661e34f94f3dca9417 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:37:23 +0600 Subject: [PATCH 12/26] api/xiaohongshu: remove duplicated extraction error Co-authored-by: jj --- api/src/processing/services/xiaohongshu.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api/src/processing/services/xiaohongshu.js b/api/src/processing/services/xiaohongshu.js index 4637358e..6e141f5a 100644 --- a/api/src/processing/services/xiaohongshu.js +++ b/api/src/processing/services/xiaohongshu.js @@ -54,9 +54,7 @@ export default async function ({ id, token, shareId, h265, isAudioOnly, dispatch if (!currentNote) throw "no current note in detail map"; note = currentNote.note; - } catch { - return { error: "fetch.empty" }; - } + } catch {} if (!note) return { error: "fetch.empty" }; From e39b0ae7b3b09e1a0444810a6f35bf703f1ed604 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:41:02 +0600 Subject: [PATCH 13/26] api/xiaohongshu: deduplicate h264 stream extraction reduce() isn't called on 1 item arrays, so this is just fine Co-authored-by: jj --- api/src/processing/services/xiaohongshu.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/api/src/processing/services/xiaohongshu.js b/api/src/processing/services/xiaohongshu.js index 6e141f5a..45d7acc5 100644 --- a/api/src/processing/services/xiaohongshu.js +++ b/api/src/processing/services/xiaohongshu.js @@ -75,12 +75,9 @@ export default async function ({ id, token, shareId, h265, isAudioOnly, dispatch if (!videoURL) { const h264Streams = video.media?.stream?.h264; - if (!h264Streams) return { error: "fetch.empty" }; - if (h264Streams.length > 1) { + if (h264Streams?.length) { videoURL = h264Streams.reduce((a, b) => Number(a?.videoBitrate) > Number(b?.videoBitrate) ? a : b).masterUrl; - } else { - videoURL = h264Streams[0].masterUrl; } } From 7488c74fafb92b73ac58772f661e5601e2bb5cac Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:46:12 +0600 Subject: [PATCH 14/26] api/xiaohongshu: clean up the h265-h264 if statement Co-authored-by: jj --- api/src/processing/services/xiaohongshu.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api/src/processing/services/xiaohongshu.js b/api/src/processing/services/xiaohongshu.js index 45d7acc5..bbb53ab1 100644 --- a/api/src/processing/services/xiaohongshu.js +++ b/api/src/processing/services/xiaohongshu.js @@ -71,9 +71,7 @@ export default async function ({ id, token, shareId, h265, isAudioOnly, dispatch if (h265 && !isAudioOnly && video.consumer?.originVideoKey) { videoURL = `https://sns-video-bd.xhscdn.com/${video.consumer.originVideoKey}`; - } - - if (!videoURL) { + } else { const h264Streams = video.media?.stream?.h264; if (h264Streams?.length) { From 9f0f885ae6b0b46fa129d5e4b7ade67750ff35c4 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:59:59 +0600 Subject: [PATCH 15/26] web/settings/video: update h265 toggle strings because now it also applies to xiaohongshu --- web/i18n/en/settings.json | 6 +++--- web/src/routes/settings/video/+page.svelte | 17 +++++++++-------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/web/i18n/en/settings.json b/web/i18n/en/settings.json index c450b4b9..418410bf 100644 --- a/web/i18n/en/settings.json +++ b/web/i18n/en/settings.json @@ -40,9 +40,9 @@ "video.twitter.gif.title": "convert looping videos to GIF", "video.twitter.gif.description": "GIF conversion is inefficient, converted file may be obnoxiously big and low quality.", - "video.tiktok.h265": "tiktok", - "video.tiktok.h265.title": "prefer HEVC/H265 format", - "video.tiktok.h265.description": "allows downloading videos in 1080p at cost of compatibility.", + "video.h265": "high efficiency video codec", + "video.h265.title": "allow h265 for videos", + "video.h265.description": "allows downloading videos from platforms like tiktok and xiaohongshu in higher quality at cost of compatibility.", "audio.format": "audio format", "audio.format.best": "best", diff --git a/web/src/routes/settings/video/+page.svelte b/web/src/routes/settings/video/+page.svelte index 9897d7eb..88f9a5ca 100644 --- a/web/src/routes/settings/video/+page.svelte +++ b/web/src/routes/settings/video/+page.svelte @@ -69,6 +69,15 @@ /> + + + + - - - From 73f458a99913c794ac5600dfef824b71985e3449 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 20:01:55 +0600 Subject: [PATCH 16/26] docs/api: update tiktokH265 description --- docs/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api.md b/docs/api.md index 6da93a44..fb1a1450 100644 --- a/docs/api.md +++ b/docs/api.md @@ -68,7 +68,7 @@ Content-Type: application/json | `alwaysProxy` | `boolean` | `true / false` | `false` | tunnels all downloads through the processing server, even when not necessary. | | `disableMetadata` | `boolean` | `true / false` | `false` | disables file metadata when set to `true`. | | `tiktokFullAudio` | `boolean` | `true / false` | `false` | enables download of original sound used in a tiktok video. | -| `tiktokH265` | `boolean` | `true / false` | `false` | changes whether 1080p h265 videos are preferred or not. | +| `tiktokH265` | `boolean` | `true / false` | `false` | allows h265 videos when enabled. applies to tiktok & xiaohongshu. | | `twitterGif` | `boolean` | `true / false` | `true` | changes whether twitter gifs are converted to .gif | | `youtubeHLS` | `boolean` | `true / false` | `false` | specifies whether to use HLS for downloading video or audio from youtube. | From 035825bc0555fa2e1c2084a407ee14d04be97445 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 20 Jan 2025 14:38:55 +0000 Subject: [PATCH 17/26] api: cache original request parameters in stream --- api/src/processing/match-action.js | 3 ++- api/src/stream/manage.js | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/api/src/processing/match-action.js b/api/src/processing/match-action.js index 64f86836..19896ceb 100644 --- a/api/src/processing/match-action.js +++ b/api/src/processing/match-action.js @@ -15,7 +15,8 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab filename: r.filenameAttributes ? createFilename(r.filenameAttributes, filenameStyle, isAudioOnly, isAudioMuted) : r.filename, fileMetadata: !disableMetadata ? r.fileMetadata : false, - requestIP + requestIP, + originalRequest: r.originalRequest }, params = {}; diff --git a/api/src/stream/manage.js b/api/src/stream/manage.js index 79b5c1db..3323ce5d 100644 --- a/api/src/stream/manage.js +++ b/api/src/stream/manage.js @@ -40,6 +40,7 @@ export function createStream(obj) { audioFormat: obj.audioFormat, isHLS: obj.isHLS || false, + originalRequest: obj.parameters }; // FIXME: this is now a Promise, but it is not awaited From 7767a5f5bb49d05b3e4d6f5a9fdf86e4a038c3a1 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 20 Jan 2025 14:46:55 +0000 Subject: [PATCH 18/26] api/youtube: add support for pinning client/itag --- api/src/processing/services/youtube.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index e16e86e7..3ee673b9 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -149,7 +149,7 @@ export default async function (o) { useHLS = false; } - let innertubeClient = "ANDROID"; + let innertubeClient = o.innertubeClient || "ANDROID"; if (cookie) { useHLS = false; @@ -245,7 +245,7 @@ export default async function (o) { } let video, audio, dubbedLanguage, - codec = o.format || "h264"; + codec = o.format || "h264", itag = o.itag; if (useHLS) { const hlsManifest = info.streaming_data.hls_manifest_url; @@ -351,17 +351,21 @@ export default async function (o) { Number(b.bitrate) - Number(a.bitrate) ).forEach(format => { Object.keys(codecList).forEach(yCodec => { + const matchingItag = slot => !itag || itag[slot] === format.itag; const sorted = sorted_formats[yCodec]; const goodFormat = checkFormat(format, yCodec); if (!goodFormat) return; - if (format.has_video) { + if (format.has_video && matchingItag('video')) { sorted.video.push(format); - if (!sorted.bestVideo) sorted.bestVideo = format; + if (!sorted.bestVideo) + sorted.bestVideo = format; } - if (format.has_audio) { + + if (format.has_audio && matchingItag('audio')) { sorted.audio.push(format); - if (!sorted.bestAudio) sorted.bestAudio = format; + if (!sorted.bestAudio) + sorted.bestAudio = format; } }) }); From 19ade7c9053dc9323c35ccee25dffb25a6e19f27 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 20 Jan 2025 14:47:09 +0000 Subject: [PATCH 19/26] api/youtube: return internal metadata for replaying request --- api/src/processing/services/youtube.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index 3ee673b9..ff4a95cf 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -452,6 +452,18 @@ export default async function (o) { youtubeDubName: dubbedLanguage || false, } + itag = { + video: video.itag, + audio: audio.itag + }; + + const originalRequest = { + ...o, + dispatcher: undefined, + itag, + innertubeClient + }; + if (audio && o.isAudioOnly) { let bestAudio = codec === "h264" ? "m4a" : "opus"; let urls = audio.url; @@ -473,6 +485,7 @@ export default async function (o) { fileMetadata, bestAudio, isHLS: useHLS, + originalRequest } } @@ -516,6 +529,7 @@ export default async function (o) { filenameAttributes, fileMetadata, isHLS: useHLS, + originalRequest } } From 39752b2c5f00175eb07c700f97f886679f938853 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 21:26:55 +0600 Subject: [PATCH 20/26] web/Omnibox: improve pasting links from clipboard - `text/uri-list` type is now accepted (such as clipboard data from bluesky) - http links are now allowed (such as those from rednote) - rednote share link is properly extracted --- web/src/components/save/Omnibox.svelte | 25 ++++++++++++++----------- web/src/lib/clipboard.ts | 17 +++++++++++++++++ 2 files changed, 31 insertions(+), 11 deletions(-) create mode 100644 web/src/lib/clipboard.ts diff --git a/web/src/components/save/Omnibox.svelte b/web/src/components/save/Omnibox.svelte index 8da051d4..c10db574 100644 --- a/web/src/components/save/Omnibox.svelte +++ b/web/src/components/save/Omnibox.svelte @@ -11,6 +11,7 @@ import dialogs from "$lib/state/dialogs"; import { link } from "$lib/state/omnibox"; import { updateSetting } from "$lib/state/settings"; + import { pasteLinkFromClipboard } from "$lib/clipboard"; import { turnstileEnabled, turnstileSolved } from "$lib/state/turnstile"; import type { Optional } from "$lib/types/generic"; @@ -41,7 +42,7 @@ const validLink = (url: string) => { try { - return /^https:/i.test(new URL(url).protocol); + return /^https?\:/i.test(new URL(url).protocol); } catch {} }; @@ -59,22 +60,24 @@ goto("/", { replaceState: true }); } - const pasteClipboard = () => { + const pasteClipboard = async () => { if ($dialogs.length > 0 || isDisabled || isLoading) { return; } - navigator.clipboard.readText().then(async (text: string) => { - let matchLink = text.match(/https:\/\/[^\s]+/g); - if (matchLink) { - $link = matchLink[0]; + const pastedData = await pasteLinkFromClipboard(); + if (!pastedData) return; - if (!isBotCheckOngoing) { - await tick(); // wait for button to render - downloadButton.download($link); - } + const linkMatch = pastedData.match(/https?\:\/\/[^\s]+/g); + + if (linkMatch) { + $link = linkMatch[0].split('，')[0]; + + if (!isBotCheckOngoing) { + await tick(); // wait for button to render + downloadButton.download($link); } - }); + } }; const changeDownloadMode = (mode: DownloadModeOption) => { diff --git a/web/src/lib/clipboard.ts b/web/src/lib/clipboard.ts new file mode 100644 index 00000000..221d17ae --- /dev/null +++ b/web/src/lib/clipboard.ts @@ -0,0 +1,17 @@ +const allowedLinkTypes = new Set(["text/plain", "text/uri-list"]); + +export const pasteLinkFromClipboard = async () => { + const clipboard = await navigator.clipboard.read(); + + if (clipboard?.length) { + const clipboardItem = clipboard[0]; + for (const type of clipboardItem.types) { + if (allowedLinkTypes.has(type)) { + const blob = await clipboardItem.getType(type); + const blobText = await blob.text(); + + return blobText; + } + } + } +} From c07940bfa4fc5b5c26d2892074b901ba928a5184 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 20 Jan 2025 15:46:03 +0000 Subject: [PATCH 21/26] api/itunnel: pass itunnel object by reference --- api/src/core/api.js | 2 +- api/src/stream/stream.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/core/api.js b/api/src/core/api.js index 153f2ca6..e4d3dfcf 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -313,7 +313,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { ...Object.entries(req.headers) ]); - return stream(res, { type: 'internal', ...streamInfo }); + return stream(res, { type: 'internal', data: streamInfo }); }; app.get('/itunnel', itunnelHandler); diff --git a/api/src/stream/stream.js b/api/src/stream/stream.js index a6d41200..c7cf7b56 100644 --- a/api/src/stream/stream.js +++ b/api/src/stream/stream.js @@ -10,7 +10,7 @@ export default async function(res, streamInfo) { return await stream.proxy(streamInfo, res); case "internal": - return internalStream(streamInfo, res); + return internalStream(streamInfo.data, res); case "merge": return stream.merge(streamInfo, res); From 600c7691414f07568e038cca7877426b4ab9057d Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 20 Jan 2025 15:55:26 +0000 Subject: [PATCH 22/26] api/stream: implement itunnel transplants --- api/src/misc/utils.js | 4 +++ api/src/stream/internal.js | 12 ++++++- api/src/stream/manage.js | 71 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 83 insertions(+), 4 deletions(-) diff --git a/api/src/misc/utils.js b/api/src/misc/utils.js index fd497d18..e15690b0 100644 --- a/api/src/misc/utils.js +++ b/api/src/misc/utils.js @@ -29,3 +29,7 @@ export function splitFilenameExtension(filename) { return [ parts.join('.'), ext ] } } + +export function zip(a, b) { + return a.map((value, i) => [ value, b[i] ]); +} diff --git a/api/src/stream/internal.js b/api/src/stream/internal.js index 7d8bf4c9..8c94c485 100644 --- a/api/src/stream/internal.js +++ b/api/src/stream/internal.js @@ -7,7 +7,7 @@ const CHUNK_SIZE = BigInt(8e6); // 8 MB const min = (a, b) => a < b ? a : b; async function* readChunks(streamInfo, size) { - let read = 0n; + let read = 0n, chunksSinceTransplant = 0; while (read < size) { if (streamInfo.controller.signal.aborted) { throw new Error("controller aborted"); @@ -22,6 +22,16 @@ async function* readChunks(streamInfo, size) { signal: streamInfo.controller.signal }); + if (chunk.statusCode === 403 && chunksSinceTransplant >= 3 && streamInfo.transplant) { + chunksSinceTransplant = 0; + try { + await streamInfo.transplant(streamInfo.dispatcher); + continue; + } catch {} + } + + chunksSinceTransplant++; + const expected = min(CHUNK_SIZE, size - read); const received = BigInt(chunk.headers['content-length']); diff --git a/api/src/stream/manage.js b/api/src/stream/manage.js index 3323ce5d..ebb5c6c7 100644 --- a/api/src/stream/manage.js +++ b/api/src/stream/manage.js @@ -9,6 +9,7 @@ import { env } from "../config.js"; import { closeRequest } from "./shared.js"; import { decryptStream, encryptStream } from "../misc/crypto.js"; import { hashHmac } from "../security/secrets.js"; +import { zip } from "../misc/utils.js"; // optional dependency const freebind = env.freebindCIDR && await import('freebind').catch(() => {}); @@ -40,7 +41,7 @@ export function createStream(obj) { audioFormat: obj.audioFormat, isHLS: obj.isHLS || false, - originalRequest: obj.parameters + originalRequest: obj.originalRequest }; // FIXME: this is now a Promise, but it is not awaited @@ -101,6 +102,7 @@ export function createInternalStream(url, obj = {}) { controller, dispatcher, isHLS: obj.isHLS, + transplant: obj.transplant }); let streamLink = new URL('/itunnel', `http://127.0.0.1:${env.tunnelPort}`); @@ -116,13 +118,17 @@ export function createInternalStream(url, obj = {}) { return streamLink.toString(); } -export function destroyInternalStream(url) { +function getInternalTunnelId(url) { url = new URL(url); if (url.hostname !== '127.0.0.1') { return; } - const id = url.searchParams.get('id'); + return url.searchParams.get('id'); +} + +export function destroyInternalStream(url) { + const id = getInternalTunnelId(url); if (internalStreamCache.has(id)) { closeRequest(getInternalStream(id)?.controller); @@ -130,9 +136,68 @@ export function destroyInternalStream(url) { } } +const transplantInternalTunnels = function(tunnelUrls, transplantUrls) { + if (tunnelUrls.length !== transplantUrls.length) { + return; + } + + for (const [ tun, url ] of zip(tunnelUrls, transplantUrls)) { + const id = getInternalTunnelId(tun); + const itunnel = getInternalStream(id); + + if (!itunnel) continue; + itunnel.url = url; + } +} + +const transplantTunnel = async function (dispatcher) { + if (this.pendingTransplant) { + await this.pendingTransplant; + return; + } + + let finished; + this.pendingTransplant = new Promise(r => finished = r); + + try { + const handler = await import(`../processing/services/${this.service}.js`); + const response = await handler.default({ + ...this.originalRequest, + dispatcher + }); + + if (!response.urls) { + return; + } + + response.urls = [response.urls].flat(); + if (this.originalRequest.isAudioOnly && response.urls.length > 1) { + response.urls = [response.urls[1]]; + } else if (this.originalRequest.isAudioMuted) { + response.urls = [response.urls[0]]; + } + + const tunnels = [this.urls].flat(); + if (tunnels.length !== response.urls.length) { + return; + } + + transplantInternalTunnels(tunnels, response.urls); + } + catch {} + finally { + finished(); + delete this.pendingTransplant; + } +} + function wrapStream(streamInfo) { const url = streamInfo.urls; + if (streamInfo.originalRequest) { + streamInfo.transplant = transplantTunnel.bind(streamInfo); + } + if (typeof url === 'string') { streamInfo.urls = createInternalStream(url, streamInfo); } else if (Array.isArray(url)) { From ee3ef60a20701adf40fa5b4bb89c6db37cd33af9 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 20 Jan 2025 20:12:21 +0000 Subject: [PATCH 23/26] api/youtube: expect one of itags to be empty --- api/src/processing/services/youtube.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index ff4a95cf..f0766b3f 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -351,7 +351,7 @@ export default async function (o) { Number(b.bitrate) - Number(a.bitrate) ).forEach(format => { Object.keys(codecList).forEach(yCodec => { - const matchingItag = slot => !itag || itag[slot] === format.itag; + const matchingItag = slot => !itag?.[slot] || itag[slot] === format.itag; const sorted = sorted_formats[yCodec]; const goodFormat = checkFormat(format, yCodec); if (!goodFormat) return; @@ -453,8 +453,8 @@ export default async function (o) { } itag = { - video: video.itag, - audio: audio.itag + video: video?.itag, + audio: audio?.itag }; const originalRequest = { From 36d4608ee58b076912d5611594f2922fa77ab90a Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 21 Jan 2025 17:18:49 +0600 Subject: [PATCH 24/26] api/bluesky: add support for tenor gifs --- api/src/processing/services/bluesky.js | 61 ++++++++++++++++++++------ api/src/util/tests/bsky.json | 18 ++++++++ 2 files changed, 65 insertions(+), 14 deletions(-) diff --git a/api/src/processing/services/bluesky.js b/api/src/processing/services/bluesky.js index bc887437..598e9739 100644 --- a/api/src/processing/services/bluesky.js +++ b/api/src/processing/services/bluesky.js @@ -71,6 +71,24 @@ const extractImages = ({ getPost, filename, alwaysProxy }) => { return { picker }; } +const extractGif = ({ url, filename }) => { + const gifUrl = new URL(url); + + if (!gifUrl || gifUrl.hostname !== "media.tenor.com") { + return { error: "fetch.empty" }; + } + + // remove downscaling params from gif url + // such as "?hh=498&ww=498" + gifUrl.search = ""; + + return { + urls: gifUrl, + isPhoto: true, + filename: `${filename}.gif`, + } +} + export default async function ({ user, post, alwaysProxy, dispatcher }) { const apiEndpoint = new URL("https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread?depth=0&parentHeight=0"); apiEndpoint.searchParams.set( @@ -102,22 +120,37 @@ export default async function ({ user, post, alwaysProxy, dispatcher }) { const embedType = getPost?.thread?.post?.embed?.$type; const filename = `bluesky_${user}_${post}`; - if (embedType === "app.bsky.embed.video#view") { - return extractVideo({ - media: getPost.thread?.post?.embed, - filename, - }) - } + switch (embedType) { + case "app.bsky.embed.video#view": + return extractVideo({ + media: getPost.thread?.post?.embed, + filename, + }); - if (embedType === "app.bsky.embed.recordWithMedia#view") { - return extractVideo({ - media: getPost.thread?.post?.embed?.media, - filename, - }) - } + case "app.bsky.embed.images#view": + return extractImages({ + getPost, + filename, + alwaysProxy + }); - if (embedType === "app.bsky.embed.images#view") { - return extractImages({ getPost, filename, alwaysProxy }); + case "app.bsky.embed.external#view": + return extractGif({ + url: getPost?.thread?.post?.embed?.external?.uri, + filename, + }); + + case "app.bsky.embed.recordWithMedia#view": + if (getPost?.thread?.post?.embed?.media?.$type === "app.bsky.embed.external#view") { + return extractGif({ + url: getPost?.thread?.post?.embed?.media?.external?.uri, + filename, + }); + } + return extractVideo({ + media: getPost.thread?.post?.embed?.media, + filename, + }); } return { error: "fetch.empty" }; diff --git a/api/src/util/tests/bsky.json b/api/src/util/tests/bsky.json index 5a03538d..6e1d6b2b 100644 --- a/api/src/util/tests/bsky.json +++ b/api/src/util/tests/bsky.json @@ -57,6 +57,24 @@ "status": "tunnel" } }, + { + "name": "gif with a quoted post", + "url": "https://bsky.app/profile/imlunahey.com/post/3lgajpn5dtk2t", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "gif alone in a post", + "url": "https://bsky.app/profile/imlunahey.com/post/3lgah3ovxnc2q", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, { "name": "several images", "url": "https://bsky.app/profile/did:plc:rai7s6su2sy22ss7skouedl7/post/3kzxuxbiul626", From cecb8a4c5343b4b2cc166e8995dfb599fe222743 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 21 Jan 2025 17:25:45 +0600 Subject: [PATCH 25/26] api/package: bump version to 10.6 --- api/package.json | 4 ++-- pnpm-lock.yaml | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/package.json b/api/package.json index fccbddff..829106c3 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "10.5.4", + "version": "10.6", "author": "imput", "exports": "./src/cobalt.js", "type": "module", @@ -39,7 +39,7 @@ "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", - "youtubei.js": "^12.2.0", + "youtubei.js": "^13.0.0", "zod": "^3.23.8" }, "optionalDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c7f3b712..9e12913a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,8 +56,8 @@ importers: specifier: 1.0.3 version: 1.0.3 youtubei.js: - specifier: ^12.2.0 - version: 12.2.0 + specifier: ^13.0.0 + version: 13.0.0 zod: specifier: ^3.23.8 version: 3.23.8 @@ -2286,8 +2286,8 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} - youtubei.js@12.2.0: - resolution: {integrity: sha512-G+50qrbJCToMYhu8jbaHiS3Vf+RRul+CcDbz3hEGwHkGPh+zLiWwD6SS+YhYF+2/op4ZU5zDYQJrGqJ+wKh7Gw==} + youtubei.js@13.0.0: + resolution: {integrity: sha512-b1QkN9bfgphK+5tI4qteSK54kNxmPhoedvMw0jl4uSn+L8gbDbJ4z52amNuYNcOdp4X/SI3JuUb+f5V0DPJ8Vw==} zod@3.23.8: resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} @@ -4242,7 +4242,7 @@ snapshots: yocto-queue@0.1.0: {} - youtubei.js@12.2.0: + youtubei.js@13.0.0: dependencies: '@bufbuild/protobuf': 2.1.0 jintr: 3.2.0 From 8d3db909d95b798e9a051cc2f023d08fe439baa2 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 21 Jan 2025 17:25:55 +0600 Subject: [PATCH 26/26] web/package: bump version to 10.6 --- web/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/package.json b/web/package.json index 37a7bb49..0c621f02 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "@imput/cobalt-web", - "version": "10.5.1", + "version": "10.6", "type": "module", "private": true, "scripts": {