From ed8f4353ea3346982d69eb2aadc4cfa5a8c26855 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 20 Jan 2025 19:10:02 +0600 Subject: [PATCH] api/processing: add support for xiaohongshu --- api/src/processing/match-action.js | 2 + api/src/processing/match.js | 10 ++ api/src/processing/service-config.js | 8 ++ api/src/processing/service-patterns.js | 4 + api/src/processing/services/xiaohongshu.js | 123 +++++++++++++++++++++ api/src/processing/url.js | 26 +++-- 6 files changed, 165 insertions(+), 8 deletions(-) create mode 100644 api/src/processing/services/xiaohongshu.js diff --git a/api/src/processing/match-action.js b/api/src/processing/match-action.js index 1cd36eb2..0d87b5a8 100644 --- a/api/src/processing/match-action.js +++ b/api/src/processing/match-action.js @@ -83,6 +83,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab case "twitter": case "snapchat": case "bsky": + case "xiaohongshu": params = { picker: r.picker }; break; @@ -143,6 +144,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab case "ok": case "vk": case "tiktok": + case "xiaohongshu": params = { type: "proxy" }; break; diff --git a/api/src/processing/match.js b/api/src/processing/match.js index 57f04b36..9fabf379 100644 --- a/api/src/processing/match.js +++ b/api/src/processing/match.js @@ -28,6 +28,7 @@ import snapchat from "./services/snapchat.js"; import loom from "./services/loom.js"; import facebook from "./services/facebook.js"; import bluesky from "./services/bluesky.js"; +import xiaohongshu from "./services/xiaohongshu.js"; let freebind; @@ -239,6 +240,15 @@ export default async function({ host, patternMatch, params }) { }); break; + case "xiaohongshu": + r = await xiaohongshu({ + ...patternMatch, + h265: params.tiktokH265, + isAudioOnly, + dispatcher, + }); + break; + default: return createResponse("error", { code: "error.api.service.unsupported" diff --git a/api/src/processing/service-config.js b/api/src/processing/service-config.js index 81afaf39..86352f9a 100644 --- a/api/src/processing/service-config.js +++ b/api/src/processing/service-config.js @@ -166,6 +166,14 @@ export const services = { subdomains: ["m"], altDomains: ["vkvideo.ru", "vk.ru"], }, + xiaohongshu: { + patterns: [ + "explore/:id?xsec_token=:token", + "discovery/item/:id?xsec_token=:token", + "a/:shareId" + ], + altDomains: ["xhslink.com"], + }, youtube: { patterns: [ "watch?v=:id", diff --git a/api/src/processing/service-patterns.js b/api/src/processing/service-patterns.js index e8c46639..42f64d26 100644 --- a/api/src/processing/service-patterns.js +++ b/api/src/processing/service-patterns.js @@ -71,4 +71,8 @@ export const testers = { "bsky": pattern => pattern.user?.length <= 128 && pattern.post?.length <= 128, + + "xiaohongshu": pattern => + pattern.id?.length <= 24 && pattern.token?.length <= 64 + || pattern.shareId?.length <= 12, } diff --git a/api/src/processing/services/xiaohongshu.js b/api/src/processing/services/xiaohongshu.js new file mode 100644 index 00000000..d428aaf9 --- /dev/null +++ b/api/src/processing/services/xiaohongshu.js @@ -0,0 +1,123 @@ +import { extract, normalizeURL } from "../url.js"; +import { genericUserAgent } from "../../config.js"; +import { createStream } from "../../stream/manage.js"; +import { getRedirectingURL } from "../../misc/utils.js"; + +const https = (url) => { + return url.replace(/^http:/i, 'https:'); +} + +export default async function ({ id, token, shareId, h265, isAudioOnly, dispatcher }) { + let noteId = id; + let xsecToken = token; + + if (!noteId) { + const extractedURL = await getRedirectingURL( + `https://xhslink.com/a/${shareId}`, + dispatcher + ); + + if (extractedURL) { + const { patternMatch } = extract(normalizeURL(extractedURL)); + + if (patternMatch) { + noteId = patternMatch.id; + xsecToken = patternMatch.token; + } + } + } + + if (!noteId || !xsecToken) return { error: "fetch.short_link" }; + + const res = await fetch(`https://www.xiaohongshu.com/explore/${noteId}?xsec_token=${xsecToken}`, { + headers: { + "user-agent": genericUserAgent, + }, + dispatcher, + }); + + const html = await res.text(); + + let note; + try { + const initialState = html + .split('')[0] + .replace(/:undefined/g, ":null"); + + const data = JSON.parse(initialState); + + const noteInfo = data?.note?.noteDetailMap; + if (!noteInfo) throw "no note detail map"; + + const currentNote = noteInfo[noteId]; + if (!currentNote) throw "no current note in detail map"; + + note = currentNote.note; + } catch { + return { error: "fetch.empty" }; + } + + if (!note) return { error: "fetch.empty" }; + + const video = note.video; + const images = note.imageList; + + const filenameBase = `xiaohongshu_${noteId}`; + + if (video) { + const videoFilename = `${filenameBase}.mp4`; + const audioFilename = `${filenameBase}_audio`; + + let videoURL; + + if (h265 && !isAudioOnly && video.consumer?.originVideoKey) { + videoURL = `https://sns-video-bd.xhscdn.com/${video.consumer.originVideoKey}`; + } + + if (!videoURL) { + const h264Streams = video.media?.stream?.h264; + if (!h264Streams) return { error: "fetch.empty" }; + + if (h264Streams.length > 1) { + videoURL = h264Streams.reduce((a, b) => Number(a?.videoBitrate) > Number(b?.videoBitrate) ? a : b).masterUrl; + } else { + videoURL = h264Streams[0].masterUrl; + } + } + + if (!videoURL) return { error: "fetch.empty" }; + + return { + urls: https(videoURL), + filename: videoFilename, + audioFilename: audioFilename, + } + } + + if (!images || images.length === 0) { + return { error: "fetch.empty" }; + } + + if (images.length === 1) { + return { + isPhoto: true, + urls: https(images[0].urlDefault), + filename: `${filenameBase}.jpg`, + } + } + + const picker = images.map((image, i) => { + return { + type: "photo", + url: createStream({ + service: "xiaohongshu", + type: "proxy", + url: https(image.urlDefault), + filename: `${filenameBase}_${i + 1}.jpg`, + }) + } + }); + + return { picker }; +} diff --git a/api/src/processing/url.js b/api/src/processing/url.js index 8f0e7dc2..cfbbecc0 100644 --- a/api/src/processing/url.js +++ b/api/src/processing/url.js @@ -92,9 +92,14 @@ function aliasURL(url) { url.hostname = 'vk.com'; } break; + + case "xhslink": + if (url.hostname === 'xhslink.com' && parts.length === 3) { + url = new URL(`https://www.xiaohongshu.com/a/${parts[2]}`); + } } - return url + return url; } function cleanURL(url) { @@ -114,36 +119,41 @@ function cleanURL(url) { break; case "vk": if (url.pathname.includes('/clip') && url.searchParams.get('z')) { - limitQuery('z') + limitQuery('z'); } break; case "youtube": if (url.searchParams.get('v')) { - limitQuery('v') + limitQuery('v'); } break; case "rutube": if (url.searchParams.get('p')) { - limitQuery('p') + limitQuery('p'); } break; case "twitter": if (url.searchParams.get('post_id')) { - limitQuery('post_id') + limitQuery('post_id'); + } + break; + case "xiaohongshu": + if (url.searchParams.get('xsec_token')) { + limitQuery('xsec_token'); } break; } if (stripQuery) { - url.search = '' + url.search = ''; } - url.username = url.password = url.port = url.hash = '' + url.username = url.password = url.port = url.hash = ''; if (url.pathname.endsWith('/')) url.pathname = url.pathname.slice(0, -1); - return url + return url; } function getHostIfValid(url) {