tiktok: use webapp-based downloading method (#503)

Signed-off-by: Damir Modyarov <otomir@yandex.ru>
Co-authored-by: wukko <me@wukko.me>

src/modules/processing/matchActionDecider.js

@@ -2,6 +2,7 @@ import { audioIgnore, services, supportedAudio } from "../config.js";
 import { createResponse } from "../processing/request.js";
 import loc from "../../localization/manager.js";
 import createFilename from "./createFilename.js";
+import { createStream } from "../stream/manage.js";
 
 export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern, toGif, requestIP) {
     let action,
@@ -41,7 +42,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
         case "photo":
             responseType = "redirect";
             break;
-        
+
         case "gif":
             params = { type: "gif" }
             break;
@@ -68,16 +69,22 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
                     params = { picker: r.picker };
                     break;
                 case "tiktok":
-                    let pickerType = "render";
+                    let audioStreamType = "render";
-                    if (audioFormat === "mp3" || audioFormat === "best") {
+                    if (r.bestAudio === "mp3" && (audioFormat === "mp3" || audioFormat === "best")) {
                         audioFormat = "mp3";
-                        pickerType = "bridge"
+                        audioStreamType = "bridge"
                     }
                     params = {
-                        type: pickerType,
                         picker: r.picker,
-                        u: Array.isArray(r.urls) ? r.urls[1] : r.urls,
+                        u: createStream({
-                        copy: audioFormat === "best" ? true : false
+                            service: "tiktok",
+                            type: audioStreamType,
+                            u: r.urls,
+                            filename: r.audioFilename,
+                            isAudioOnly: true,
+                            audioFormat,
+                        }),
+                        copy: audioFormat === "best"
                     }
             }
             break;
@@ -101,7 +108,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
                         responseType = "redirect";
                     }
                     break;
-                
+
                 case "twitter":
                     if (r.type === "remux") {
                         params = { type: r.type };
@@ -125,7 +132,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
             }
             break;
 
-        case "audio": 
+        case "audio":
             if (audioIgnore.includes(host)
                 || (host === "reddit" && r.typeId === "redirect")) {
                 return createResponse("error", { t: loc(lang, 'ErrorEmptyDownload') })
@@ -133,7 +140,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
 
             let processType = "render",
                 copy = false;
-            
+
             if (!supportedAudio.includes(audioFormat)) {
                 audioFormat = "best"
             }
@@ -146,11 +153,12 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
 
             const isTumblrAudio = host === "tumblr" && !r.filename;
             const isSoundCloud = host === "soundcloud";
+            const isTiktok = host === "tiktok";
 
             if (isBestAudioDefined || isBestHostAudio) {
                 audioFormat = serviceBestAudio;
                 processType = "bridge";
-                if (isSoundCloud) {
+                if (isSoundCloud || (isTiktok && audioFormat === "m4a")) {
                     processType = "render"
                     copy = true
                 }

src/modules/processing/services/tiktok.js

@@ -1,13 +1,13 @@
-import { genericUserAgent, env } from "../../config.js";
+import { genericUserAgent } from "../../config.js";
+import { updateCookie } from "../cookie/manager.js";
+import { extract } from "../url.js";
+import Cookie from "../cookie/cookie.js";
 
 const shortDomain = "https://vt.tiktok.com/";
-const apiPath = "https://api22-normal-c-alisg.tiktokv.com/aweme/v1/feed/?region=US&carrier_region=US";
+export const cookie = new Cookie({});
-const apiUserAgent = "TikTok/338014 CFNetwork/1410.1 Darwin/22.6.0";
 
 export default async function(obj) {
-    let postId = obj.postId ? obj.postId : false;
+    let postId = obj.postId;
-
-    if (!env.tiktokDeviceInfo) return { error: 'ErrorCouldntFetch' };
 
     if (!postId) {
         let html = await fetch(`${shortDomain}${obj.id}`, {
@@ -19,54 +19,59 @@ export default async function(obj) {
 
         if (!html) return { error: 'ErrorCouldntFetch' };
 
-        if (html.slice(0, 17) === '<a href="https://') {
+        if (html.startsWith('<a href="https://')) {
-            postId = html.split('<a href="https://')[1].split('?')[0].split('/')[3]
+            const { patternMatch } = extract(html.split('<a href="https://')[1].split('?')[0]);
-        } else if (html.slice(0, 32) === '<a href="https://m.tiktok.com/v/' && html.includes('/v/')) {
+            postId = patternMatch.postId
-            postId = html.split('/v/')[1].split('.html')[0].replace("/", '')
         }
     }
     if (!postId) return { error: 'ErrorCantGetID' };
 
-    let deviceInfo = new URLSearchParams(env.tiktokDeviceInfo).toString();
+    // should always be /video/, even for photos
-
+    const res = await fetch(`https://tiktok.com/@i/video/${postId}`, {
-    let apiURL = new URL(apiPath);
-        apiURL.searchParams.append("aweme_id", postId);
-
-    let detail = await fetch(`${apiURL.href}&${deviceInfo}`, {
         headers: {
-            "user-agent": apiUserAgent
+            "user-agent": genericUserAgent,
+            cookie,
         }
-    }).then(r => r.json()).catch(() => {});
+    })
+    updateCookie(cookie, res.headers);
 
-    detail = detail?.aweme_list?.find(v => v.aweme_id === postId);
+    const html = await res.text();
-    if (!detail) return { error: 'ErrorCouldntFetch' };
+
+    let detail;
+    try {
+        const json = html
+            .split('<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">')[1]
+            .split('</script>')[0]
+        const data = JSON.parse(json)
+        detail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]
+    } catch {
+        return { error: 'ErrorCouldntFetch' };
+    }
 
     let video, videoFilename, audioFilename, audio, images,
-        filenameBase = `tiktok_${detail.author.unique_id}_${postId}`,
+        filenameBase = `tiktok_${detail.author.uniqueId}_${postId}`,
         bestAudio = 'm4a';
 
-    images = detail.image_post_info?.images;
+    images = detail.imagePost?.images;
 
-    let playAddr = detail.video.play_addr_h264;
+    let playAddr = detail.video.playAddr;
     if (obj.h265) {
-        playAddr = detail.video.bit_rate[0].play_addr
+        const h265PlayAddr = detail.video.bitrateInfo.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0]
-    }
+        playAddr = h265PlayAddr || playAddr
-    if (!playAddr && detail.video.play_addr) {
-        playAddr = detail.video.play_addr
     }
 
     if (!obj.isAudioOnly && !images) {
-        video = playAddr.url_list[0];
+        video = playAddr;
         videoFilename = `${filenameBase}.mp4`;
     } else {
-        let fallback = playAddr.url_list[0];
+        audio = playAddr;
-        audio = fallback;
         audioFilename = `${filenameBase}_audio`;
-        if (obj.fullAudio || fallback.includes("music")) {
+
-            audio = detail.music.play_url.url_list[0]
+        if (obj.fullAudio || !audio) {
-            audioFilename = `${filenameBase}_audio_original`
+            audio = detail.music.playUrl;
+            audioFilename += `_original`
         }
-        if (audio.slice(-4) === ".mp3") bestAudio = 'mp3';
+        if (audio.includes("mime_type=audio_mpeg")) bestAudio = 'mp3';
     }
 
     if (video) return {
@@ -80,12 +85,9 @@ export default async function(obj) {
         bestAudio
     }
     if (images) {
-        let imageLinks = [];
+        let imageLinks = images
-        for (let i in images) {
+            .map(i => i.imageURL.urlList.find(p => p.includes(".jpeg?")))
-            let sel = images[i].display_image.url_list;
+            .map(url => ({ url }))
-            sel = sel.filter(p => p.includes(".jpeg?"))
-            imageLinks.push({url: sel[0]})
-        }
         return {
             picker: imageLinks,
             urls: audio,

src/modules/stream/shared.js

@@ -1,4 +1,5 @@
 import { genericUserAgent } from "../config.js";
+import { cookie as tiktokCookie } from "../processing/services/tiktok.js";
 
 const defaultHeaders = {
     'user-agent': genericUserAgent
@@ -13,6 +14,9 @@ const serviceHeaders = {
         origin: 'https://www.youtube.com',
         referer: 'https://www.youtube.com',
         DNT: '?1'
+    },
+    tiktok: {
+        cookie: tiktokCookie
     }
 }
 
@@ -22,5 +26,7 @@ export function closeResponse(res) {
 }
 
 export function getHeaders(service) {
-    return { ...defaultHeaders, ...serviceHeaders[service] }
+    // Converting all header values to strings
+    return Object.entries({ ...defaultHeaders, ...serviceHeaders[service] })
+        .reduce((p, [key, val]) => ({ ...p, [key]: String(val) }), {})
 }