Merge 5dfc16b76c into 080fc043ea

also improved tiktok audio file naming scheme. full audio now has the "_audio_original" tag. audio extracted from video is simply "_audio".

README.md

@@ -21,6 +21,7 @@ this list is not final and keeps expanding over time. if support for a service y
 | pinterest                      | ✅            | ✅         | ✅         | ➖         | ➖              |
 | reddit                         | ✅            | ✅         | ✅         | ❌         | ❌              |
 | rutube                         | ✅            | ✅         | ✅         | ✅         | ✅              |
+| snapchat stories & spotlights  | ✅            | ✅         | ✅         | ➖         | ➖              |
 | soundcloud                     | ➖            | ✅         | ➖         | ✅         | ✅              |
 | streamable                     | ✅            | ✅         | ✅         | ➖         | ➖              |
 | tiktok                         | ✅            | ✅         | ✅         | ❌         | ❌              |
@@ -44,6 +45,7 @@ this list is not final and keeps expanding over time. if support for a service y
 | instagram  | supports photos, videos, and stories. lets you pick what to save from multi-media posts.                             |
 | pinterest  | supports videos and stories.                                                                                         |
 | reddit     | supports gifs and videos.                                                                                            |
+| snapchat   | supports spotlights and stories. stories will need to be directly linked to a video.                                 |
 | soundcloud | supports private links.                                                                                              |
 | tiktok     | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios.      |
 | twitter/x  | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management.               |

src/modules/processing/match.js

@@ -25,6 +25,7 @@ import streamable from "./services/streamable.js";
 import twitch from "./services/twitch.js";
 import rutube from "./services/rutube.js";
 import dailymotion from "./services/dailymotion.js";
+import snapchat from "./services/snapchat.js";
 
 export default async function(host, patternMatch, url, lang, obj) {
     assert(url instanceof URL);
@@ -159,6 +160,15 @@ export default async function(host, patternMatch, url, lang, obj) {
             case "dailymotion":
                 r = await dailymotion(patternMatch);
                 break;
+            case "snapchat":
+                r = await snapchat({
+                    url,
+                    username: patternMatch.username,
+                    storyId: patternMatch.storyId,
+                    spotlightId: patternMatch.spotlightId,
+                    shortLink: patternMatch.shortLink || false
+                });
+                break;
             default:
                 return apiJSON(0, { t: errorUnsupported(lang) });
         }

src/modules/processing/matchActionDecider.js

@@ -120,6 +120,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
                 case "tumblr":
                 case "pinterest":
                 case "streamable":
+                case "snapchat":
                     responseType = 1;
                     break;
             }
@@ -137,40 +138,22 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
                 audioFormat = "best"
             }
 
+            const serviceBestAudio = r.bestAudio || services[host]["bestAudio"];
             const isBestAudio = audioFormat === "best";
-            const isBestOrMp3 = audioFormat === "mp3" || isBestAudio;
-            const isBestAudioDefined = isBestAudio && services[host]["bestAudio"];
-            const isBestHostAudio = services[host]["bestAudio"] && (audioFormat === services[host]["bestAudio"]);
+            const isBestOrMp3 = isBestAudio || audioFormat === "mp3";
+            const isBestAudioDefined = isBestAudio && serviceBestAudio;
+            const isBestHostAudio = serviceBestAudio && (audioFormat === serviceBestAudio);
 
-            const isTikTok = host === "tiktok" || host === "douyin";
             const isTumblrAudio = host === "tumblr" && !r.filename;
             const isSoundCloud = host === "soundcloud";
 
-            if (isTikTok && services.tiktok.audioFormats.includes(audioFormat)) {
-                if (r.isMp3 && isBestOrMp3) {
-                    audioFormat = "mp3";
-                    processType = "bridge"
-                } else if (isBestAudio) {
-                    audioFormat = "m4a";
-                    processType = "bridge"
-                }
-            }
-
-            if (isSoundCloud && services.soundcloud.audioFormats.includes(audioFormat)) {
-                if (r.isMp3 && isBestOrMp3) {
-                    audioFormat = "mp3";
-                    processType = "render"
-                    copy = true
-                } else if (isBestAudio || audioFormat === "opus") {
-                    audioFormat = "opus";
-                    processType = "render"
-                    copy = true
-                }
-            }
-
             if (isBestAudioDefined || isBestHostAudio) {
-                audioFormat = services[host]["bestAudio"];
+                audioFormat = serviceBestAudio;
                 processType = "bridge";
+                if (isSoundCloud) {
+                    processType = "render"
+                    copy = true
+                }
             } else if (isBestAudio && !isSoundCloud) {
                 audioFormat = "m4a";
                 copy = true

src/modules/processing/services/snapchat.js

@@ -0,0 +1,54 @@
+import { genericUserAgent } from "../../config.js";
+
+const SPOTLIGHT_VIDEO_REGEX = /<link data-react-helmet="true" rel="preload" href="(https:\/\/cf-st\.sc-cdn\.net\/d\/[\w.?=]+&amp;uc=\d+)" as="video"\/>/;
+
+export default async function(obj) {
+    let link;
+    if (obj.url.hostname === 't.snapchat.com' && obj.shortLink) {
+        link = await fetch(`https://t.snapchat.com/${obj.shortLink}`, { redirect: "manual" }).then((r) => {
+            if (r.status === 303 && r.headers.get("location").startsWith("https://www.snapchat.com/")) {
+                return r.headers.get("location").split('?', 1)[0]
+            }
+        }).catch(() => {});
+    }
+
+    if (!link && obj.username && obj.storyId) {
+        link = `https://www.snapchat.com/add/${obj.username}/${obj.storyId}`
+    } else if (!link && obj.spotlightId) {
+        link = `https://www.snapchat.com/spotlight/${obj.spotlightId}`
+    }
+
+    const path = new URL(link).pathname;
+
+    if (path.startsWith('/spotlight/')) {
+        const html = await fetch(link, {
+            headers: { "user-agent": genericUserAgent }
+        }).then((r) => { return r.text() }).catch(() => { return false });
+        if (!html) return { error: 'ErrorCouldntFetch' };
+
+        const id = path.split('/')[2];
+        const videoURL = html.match(SPOTLIGHT_VIDEO_REGEX)?.[1];
+        if (videoURL) return {
+            urls: videoURL,
+            filename: `snapchat_${id}.mp4`,
+            audioFilename: `snapchat_${id}_audio`
+        }
+    } else if (path.startsWith('/add/')) {
+        const html = await fetch(link, {
+            headers: { "user-agent": genericUserAgent }
+        }).then((r) => { return r.text() }).catch(() => { return false });
+        if (!html) return { error: 'ErrorCouldntFetch' };
+
+        const id = path.split('/')[3];
+        const storyVideoRegex = new RegExp(`"snapId":{"value":"${id}"},"snapMediaType":1,"snapUrls":{"mediaUrl":"(https:\\/\\/bolt-gcdn\\.sc-cdn\\.net\\/3\/[^"]+)","mediaPreviewUrl"`);
+        const videoURL = html.match(storyVideoRegex)?.[1];
+        if (videoURL) return {
+            urls: videoURL,
+            filename: `snapchat_${id}.mp4`,
+            audioFilename: `snapchat_${id}_audio`
+        }
+    }
+
+
+    return { error: 'ErrorCouldntFetch' };
+}

src/modules/processing/services/soundcloud.js

@@ -4,7 +4,7 @@ import { cleanString } from "../../sub/utils.js";
 const cachedID = {
     version: '',
     id: ''
-};
+}
 
 async function findClientID() {
     try {
@@ -32,9 +32,7 @@ async function findClientID() {
         cachedID.id = clientid;
 
         return clientid;
-    } catch (e) {
-        return false;
-    }
+    } catch {}
 }
 
 export default async function(obj) {
@@ -58,27 +56,30 @@ export default async function(obj) {
 
     let json = await fetch(`https://api-v2.soundcloud.com/resolve?url=${link}&client_id=${clientId}`).then((r) => {
         return r.status === 200 ? r.json() : false
-    }).catch(() => { return false });
+    }).catch(() => {});
+
     if (!json) return { error: 'ErrorCouldntFetch' };
 
     if (!json["media"]["transcodings"]) return { error: 'ErrorEmptyDownload' };
 
-    let isMp3,
-        selectedStream = json.media.transcodings.filter(v => v.preset === "opus_0_0")
+    let bestAudio = "opus",
+        selectedStream = json.media.transcodings.find(v => v.preset === "opus_0_0");
 
     // fall back to mp3 if no opus is available
     if (selectedStream.length === 0) {
-        selectedStream = json.media.transcodings.filter(v => v.preset === "mp3_0_0")
-        isMp3 = true
+        selectedStream = json.media.transcodings.find(v => v.preset === "mp3_0_0");
+        bestAudio = "mp3"
     }
-    let fileUrlBase = selectedStream[0]["url"];
+
+    let fileUrlBase = selectedStream.url;
     let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`;
 
     if (fileUrl.substring(0, 54) !== "https://api-v2.soundcloud.com/media/soundcloud:tracks:") return { error: 'ErrorEmptyDownload' };
 
-    if (json.duration > maxVideoDuration) return { error: ['ErrorLengthAudioConvert', maxVideoDuration / 60000] };
+    if (json.duration > maxVideoDuration)
+        return { error: ['ErrorLengthAudioConvert', maxVideoDuration / 60000] };
 
-    let file = await fetch(fileUrl).then(async (r) => { return (await r.json()).url }).catch(() => { return false });
+    let file = await fetch(fileUrl).then(async (r) => { return (await r.json()).url }).catch(() => {});
     if (!file) return { error: 'ErrorCouldntFetch' };
 
     let fileMetadata = {
@@ -94,7 +95,7 @@ export default async function(obj) {
             title: fileMetadata.title,
             author: fileMetadata.artist
         },
-        isMp3,
+        bestAudio,
         fileMetadata
     }
 }

src/modules/processing/services/tiktok.js

@@ -41,8 +41,9 @@ export default async function(obj) {
     detail = detail?.aweme_list?.find(v => v.aweme_id === postId);
     if (!detail) return { error: 'ErrorCouldntFetch' };
 
-    let video, videoFilename, audioFilename, isMp3, audio, images,
-        filenameBase = `tiktok_${detail.author.unique_id}_${postId}`;
+    let video, videoFilename, audioFilename, audio, images,
+        filenameBase = `tiktok_${detail.author.unique_id}_${postId}`,
+        bestAudio = 'm4a';
 
     images = detail.image_post_info?.images;
 
@@ -56,12 +57,12 @@ export default async function(obj) {
     } else {
         let fallback = playAddr.url_list[0];
         audio = fallback;
-        audioFilename = `${filenameBase}_audio_fv`;  // fv - from video
+        audioFilename = `${filenameBase}_audio`;
         if (obj.fullAudio || fallback.includes("music")) {
             audio = detail.music.play_url.url_list[0]
-            audioFilename = `${filenameBase}_audio`
+            audioFilename = `${filenameBase}_audio_original`
         }
-        if (audio.slice(-4) === ".mp3") isMp3 = true;
+        if (audio.slice(-4) === ".mp3") bestAudio = 'mp3';
     }
 
     if (video) return {
@@ -72,7 +73,7 @@ export default async function(obj) {
         urls: audio,
         audioFilename: audioFilename,
         isAudioOnly: true,
-        isMp3: isMp3
+        bestAudio
     }
     if (images) {
         let imageLinks = [];
@@ -86,13 +87,13 @@ export default async function(obj) {
             urls: audio,
             audioFilename: audioFilename,
             isAudioOnly: true,
-            isMp3: isMp3
+            bestAudio
         }
     }
     if (audio) return {
         urls: audio,
         audioFilename: audioFilename,
         isAudioOnly: true,
-        isMp3: isMp3
+        bestAudio
     }
 }

src/modules/processing/services/youtube.js

@@ -4,7 +4,7 @@ import { cleanString } from '../../sub/utils.js';
 
 const yt = await Innertube.create();
 
-const c = {
+const codecMatch = {
     h264: {
         codec: "avc1",
         aCodec: "mp4a",
@@ -23,8 +23,8 @@ const c = {
 }
 
 export default async function(o) {
-    let info, isDubbed,
-        quality = o.quality === "max" ? "9000" : o.quality; // 9000(p) - max quality
+    let info, isDubbed, format = o.format || "h264";
+    let quality = o.quality === "max" ? "9000" : o.quality; // 9000(p) - max quality
 
     function qual(i) {
         if (!i.quality_label) {
@@ -56,10 +56,16 @@ export default async function(o) {
 
     let bestQuality, hasAudio;
 
-    let adaptive_formats = info.streaming_data.adaptive_formats.filter(e => 
-        e.mime_type.includes(c[o.format].codec) || e.mime_type.includes(c[o.format].aCodec)
+    const filterByCodec = (formats) => formats.filter(e => 
+        e.mime_type.includes(codecMatch[format].codec) || e.mime_type.includes(codecMatch[format].aCodec)
     ).sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
 
+    let adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats);
+    if (adaptive_formats.length === 0 && format === "vp9") {
+        format = "h264"
+        adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats)
+    }
+
     bestQuality = adaptive_formats.find(i => i.has_video);
     hasAudio = adaptive_formats.find(i => i.has_audio);
 
@@ -105,14 +111,15 @@ export default async function(o) {
         isAudioOnly: true,
         urls: audio.decipher(yt.session.player),
         filenameAttributes: filenameAttributes,
-        fileMetadata: fileMetadata
+        fileMetadata: fileMetadata,
+        bestAudio: format === "h264" ? 'm4a' : 'opus'
     }
     const matchingQuality = Number(quality) > Number(bestQuality) ? bestQuality : quality,
-        checkSingle = i => qual(i) === matchingQuality && i.mime_type.includes(c[o.format].codec),
+        checkSingle = i => qual(i) === matchingQuality && i.mime_type.includes(codecMatch[format].codec),
         checkRender = i => qual(i) === matchingQuality && i.has_video && !i.has_audio;
 
     let match, type, urls;
-    if (!o.isAudioOnly && !o.isAudioMuted && o.format === 'h264') {
+    if (!o.isAudioOnly && !o.isAudioMuted && format === 'h264') {
         match = info.streaming_data.formats.find(checkSingle);
         type = "bridge";
         urls = match?.decipher(yt.session.player);
@@ -128,8 +135,8 @@ export default async function(o) {
     if (match) {
         filenameAttributes.qualityLabel = match.quality_label;
         filenameAttributes.resolution = `${match.width}x${match.height}`;
-        filenameAttributes.extension = c[o.format].container;
-        filenameAttributes.youtubeFormat = o.format;
+        filenameAttributes.extension = codecMatch[format].container;
+        filenameAttributes.youtubeFormat = format;
         return {
             type,
             urls,

src/modules/processing/servicesConfig.json

@@ -57,7 +57,6 @@
             "alias": "tiktok videos, photos & audio",
             "patterns": [":user/video/:postId", ":id", "t/:id", ":user/photo/:postId"],
             "subdomains": ["vt", "vm"],
-            "audioFormats": ["best", "m4a", "mp3"],
             "enabled": true
         },
         "douyin": {
@@ -74,7 +73,6 @@
         "soundcloud": {
             "patterns": [":author/:song/s-:accessKey", ":author/:song", ":shortLink"],
             "subdomains": ["on", "m"],
-            "audioFormats": ["best", "opus", "mp3"],
             "enabled": true
         },
         "instagram": {
@@ -118,6 +116,12 @@
             "alias": "dailymotion videos",
             "patterns": ["video/:id"],
             "enabled": true
+        },
+        "snapchat": {
+            "alias": "snapchat stories & spotlights",
+            "subdomains": ["t", "story"],
+            "patterns": [":shortLink", "spotlight/:spotlightId", "add/:username/:storyId", "u/:username/:storyId"],
+            "enabled": true
         }
     }
 }

src/modules/processing/servicesPatternTesters.js

@@ -25,6 +25,11 @@ export const testers = {
         (patternMatch.author?.length <= 255 && patternMatch.song?.length <= 255) 
         || patternMatch.shortLink?.length <= 32,
 
+    "snapchat": (patternMatch) =>
+        (patternMatch.username?.length <= 32 && patternMatch.storyId?.length <= 255) 
+        || patternMatch.spotlightId?.length <= 255 
+        || patternMatch.shortLink?.length <= 16,
+
     "streamable": (patternMatch) =>
         patternMatch.id?.length === 6,
     

src/test/tests.json

@@ -1155,5 +1155,22 @@
             "code": 200,
             "status": "stream"
         }
+    }],
+    "snapchat": [{
+        "name": "spotlight",
+        "url": "https://www.snapchat.com/spotlight/W7_EDlXWTBiXAEEniNoMPwAAYdWxucG9pZmNqAY46j0a5AY46j0YbAAAAAQ",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "shortlinked spotlight",
+        "url": "https://t.snapchat.com/4ZsiBLDi",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
     }]
-}
+}