From a0f227d68b2b49ab52b6139291457c4f6fd453bf Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 10 Feb 2025 00:17:48 +0600 Subject: [PATCH 1/5] api/reddit: add support for mobile links & bunch of other links (#1098) * api/reddit: extract params from a mobile share link * api/reddit: add support for a bunch of links & update the api endpoint also fixed "undefined" in a filename when downloading a user post * api/service-patterns: fix reddit id pattern --- api/src/processing/match.js | 5 ++--- api/src/processing/service-config.js | 15 ++++++++++++- api/src/processing/service-patterns.js | 6 +++-- api/src/processing/services/reddit.js | 31 +++++++++++++++++++------- 4 files changed, 43 insertions(+), 14 deletions(-) diff --git a/api/src/processing/match.js b/api/src/processing/match.js index 9fabf379..b7be8456 100644 --- a/api/src/processing/match.js +++ b/api/src/processing/match.js @@ -120,9 +120,8 @@ export default async function({ host, patternMatch, params }) { case "reddit": r = await reddit({ - sub: patternMatch.sub, - id: patternMatch.id, - user: patternMatch.user + ...patternMatch, + dispatcher, }); break; diff --git a/api/src/processing/service-config.js b/api/src/processing/service-config.js index 68301adc..3d382c69 100644 --- a/api/src/processing/service-config.js +++ b/api/src/processing/service-config.js @@ -65,8 +65,21 @@ export const services = { }, reddit: { patterns: [ + "comments/:id", + + "r/:sub/comments/:id", "r/:sub/comments/:id/:title", - "user/:user/comments/:id/:title" + "r/:sub/comments/:id/comment/:commentId", + + "user/:user/comments/:id", + "user/:user/comments/:id/:title", + "user/:user/comments/:id/comment/:commentId", + + "r/u_:user/comments/:id", + "r/u_:user/comments/:id/:title", + "r/u_:user/comments/:id/comment/:commentId", + + "r/:sub/s/:shareId" ], subdomains: "*", }, diff --git a/api/src/processing/service-patterns.js b/api/src/processing/service-patterns.js index cf89d574..8735f123 100644 --- a/api/src/processing/service-patterns.js +++ b/api/src/processing/service-patterns.js @@ -20,8 +20,10 @@ export const testers = { pattern.id?.length <= 128 || pattern.shortLink?.length <= 32, "reddit": pattern => - (pattern.sub?.length <= 22 && pattern.id?.length <= 10) - || (pattern.user?.length <= 22 && pattern.id?.length <= 10), + pattern.id?.length <= 16 && !pattern.sub && !pattern.user + || (pattern.sub?.length <= 22 && pattern.id?.length <= 16) + || (pattern.user?.length <= 22 && pattern.id?.length <= 16) + || (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16), "rutube": pattern => (pattern.id?.length === 32 && pattern.key?.length <= 32) || diff --git a/api/src/processing/services/reddit.js b/api/src/processing/services/reddit.js index 701db236..50c78d35 100644 --- a/api/src/processing/services/reddit.js +++ b/api/src/processing/services/reddit.js @@ -1,3 +1,4 @@ +import { resolveRedirectingURL } from "../url.js"; import { genericUserAgent, env } from "../../config.js"; import { getCookie, updateCookieValues } from "../cookie/manager.js"; @@ -48,12 +49,20 @@ async function getAccessToken() { } export default async function(obj) { - let url = new URL(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}.json`); + let params = obj; - if (obj.user) { - url.pathname = `/user/${obj.user}/comments/${obj.id}.json`; + if (!params.id && params.shareId) { + params = await resolveRedirectingURL( + `https://www.reddit.com/r/${params.sub}/s/${params.shareId}`, + obj.dispatcher, + genericUserAgent + ); } + if (!params?.id) return { error: "fetch.short_link" }; + + const url = new URL(`https://www.reddit.com/comments/${params.id}.json`); + const accessToken = await getAccessToken(); if (accessToken) url.hostname = 'oauth.reddit.com'; @@ -73,12 +82,17 @@ export default async function(obj) { data = data[0]?.data?.children[0]?.data; - const id = `${String(obj.sub).toLowerCase()}_${obj.id}`; + let sourceId; + if (params.sub || params.user) { + sourceId = `${String(params.sub || params.user).toLowerCase()}_${params.id}`; + } else { + sourceId = params.id; + } if (data?.url?.endsWith('.gif')) return { typeId: "redirect", urls: data.url, - filename: `reddit_${id}.gif`, + filename: `reddit_${sourceId}.gif`, } if (!data.secure_media?.reddit_video) @@ -87,8 +101,9 @@ export default async function(obj) { if (data.secure_media?.reddit_video?.duration > env.durationLimit) return { error: "content.too_long" }; + const video = data.secure_media?.reddit_video?.fallback_url?.split('?')[0]; + let audio = false, - video = data.secure_media?.reddit_video?.fallback_url?.split('?')[0], audioFileLink = `${data.secure_media?.reddit_video?.fallback_url?.split('DASH')[0]}audio`; if (video.match('.mp4')) { @@ -121,7 +136,7 @@ export default async function(obj) { typeId: "tunnel", type: "merge", urls: [video, audioFileLink], - audioFilename: `reddit_${id}_audio`, - filename: `reddit_${id}.mp4` + audioFilename: `reddit_${sourceId}_audio`, + filename: `reddit_${sourceId}.mp4` } } From 09706160a9e1036534316362b18ae06adae48fb7 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 10 Feb 2025 00:33:23 +0600 Subject: [PATCH 2/5] api/snapchat: allow profile params to be missing fixes broken story extraction --- api/src/processing/services/snapchat.js | 6 +++--- api/src/util/tests/snapchat.json | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/api/src/processing/services/snapchat.js b/api/src/processing/services/snapchat.js index 10359a03..f5d66136 100644 --- a/api/src/processing/services/snapchat.js +++ b/api/src/processing/services/snapchat.js @@ -40,9 +40,9 @@ async function getStory(username, storyId, alwaysProxy) { const nextDataString = html.match(NEXT_DATA_REGEX)?.[1]; if (nextDataString) { const data = JSON.parse(nextDataString); - const storyIdParam = data.query.profileParams[1]; + const storyIdParam = data?.query?.profileParams?.[1]; - if (storyIdParam && data.props.pageProps.story) { + if (storyIdParam && data?.props?.pageProps?.story) { const story = data.props.pageProps.story.snapList.find((snap) => snap.snapId.value === storyIdParam); if (story) { if (story.snapMediaType === 0) { @@ -61,7 +61,7 @@ async function getStory(username, storyId, alwaysProxy) { } } - const defaultStory = data.props.pageProps.curatedHighlights[0]; + const defaultStory = data?.props?.pageProps?.curatedHighlights?.[0]; if (defaultStory) { return { picker: defaultStory.snapList.map(snap => { diff --git a/api/src/util/tests/snapchat.json b/api/src/util/tests/snapchat.json index 36acfbc6..bf0c9da3 100644 --- a/api/src/util/tests/snapchat.json +++ b/api/src/util/tests/snapchat.json @@ -20,7 +20,6 @@ { "name": "story", "url": "https://www.snapchat.com/add/bazerkmakane", - "canFail": true, "params": {}, "expected": { "code": 200, From 8ab5e3239053f271d341de0591032ef3ae1e9fa9 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 10 Feb 2025 00:57:19 +0600 Subject: [PATCH 3/5] api/package: bump version to 10.7.1 --- api/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/package.json b/api/package.json index 8f049e9c..93cce930 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "10.7", + "version": "10.7.1", "author": "imput", "exports": "./src/cobalt.js", "type": "module", From de69989bbeb4f7fd87d74963eb1d03c2cb5fb20f Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 10 Feb 2025 11:53:37 +0600 Subject: [PATCH 4/5] api/service-config/instagram: add support for more share links --- api/src/processing/service-config.js | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/api/src/processing/service-config.js b/api/src/processing/service-config.js index 3d382c69..1dc8bf30 100644 --- a/api/src/processing/service-config.js +++ b/api/src/processing/service-config.js @@ -35,14 +35,25 @@ export const services = { }, instagram: { patterns: [ - "reels/:postId", - ":username/reel/:postId", - "reel/:postId", "p/:postId", - ":username/p/:postId", "tv/:postId", + "reel/:postId", + "reels/:postId", "stories/:username/:storyId", - "share/:shareId" + + /* + share & username links use the same url pattern, + so we test the share pattern first, cuz id type is different. + however, if someone has the "share" username and the user + somehow gets a link of this ancient style, it's joever. + */ + + "share/:shareId", + "share/p/:shareId", + "share/reel/:shareId", + + ":username/p/:postId", + ":username/reel/:postId", ], altDomains: ["ddinstagram.com"], }, From 07947882c44249cbb3306f0d150dbba5b8195bcd Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 10 Feb 2025 11:53:49 +0600 Subject: [PATCH 5/5] api/package: bump version to 10.7.2 --- api/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/package.json b/api/package.json index 93cce930..b46f0a6a 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "10.7.1", + "version": "10.7.2", "author": "imput", "exports": "./src/cobalt.js", "type": "module",