From a1c5a4da7204f75e7e37eadfb2b454ef1320e89e Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 29 Mar 2024 06:01:04 +0600 Subject: [PATCH 1/9] tiktok: update domain & force device info --- src/modules/processing/services/tiktok.js | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/modules/processing/services/tiktok.js b/src/modules/processing/services/tiktok.js index 1818daf..daea8c4 100644 --- a/src/modules/processing/services/tiktok.js +++ b/src/modules/processing/services/tiktok.js @@ -4,8 +4,8 @@ const userAgent = genericUserAgent.split(' Chrome/1')[0], config = { tiktok: { short: "https://vt.tiktok.com/", - api: "https://api22-normal-c-useast2a.tiktokv.com/aweme/v1/feed/?aweme_id={postId}&version_code=262&app_name=musical_ly&channel=App&device_id=null&os_version=14.4.2&device_platform=iphone&device_type=iPhone9®ion=US&carrier_region=US", - userAgent: "TikTok 26.2.0 rv:262018 (iPhone; iOS 14.4.2; en_US) Cronet" + api: "https://api22-normal-c-alisg.tiktokv.com/aweme/v1/feed/?aweme_id={postId}®ion=US&carrier_region=US", + userAgent: "TikTok/338014 CFNetwork/1410.1 Darwin/22.6.0" }, douyin: { short: "https://v.douyin.com/", @@ -32,6 +32,8 @@ function selector(j, h, id) { export default async function(obj) { let postId = obj.postId ? obj.postId : false; + if (!process.env.TIKTOK_DEVICE_INFO) return { error: 'ErrorCouldntFetch' }; + if (!postId) { let html = await fetch(`${config[obj.host].short}${obj.id}`, { redirect: "manual", @@ -48,12 +50,15 @@ export default async function(obj) { } if (!postId) return { error: 'ErrorCantGetID' }; + let deviceInfo = JSON.parse(process.env.TIKTOK_DEVICE_INFO); + deviceInfo = new URLSearchParams(deviceInfo).toString(); + let detail; - detail = await fetch(config[obj.host].api.replace("{postId}", postId), { - headers: { - "user-agent": config[obj.host].userAgent - } - }).then((r) => { return r.json() }).catch(() => { return false }); + detail = await fetch(`${config[obj.host].api.replace("{postId}", postId)}&${deviceInfo}`, { + headers: { + "user-agent": config[obj.host].userAgent + } + }).then((r) => { return r.json() }).catch(() => { return false }); detail = selector(detail, obj.host, postId); if (!detail) return { error: 'ErrorCouldntFetch' }; From c2c62c1bc29bff24f2cbd762a4c584baf8c760c6 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 29 Mar 2024 06:01:47 +0600 Subject: [PATCH 2/9] docs/run-an-instance: add TIKTOK_DEVICE_INFO description --- docs/run-an-instance.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md index 819e021..01675e2 100644 --- a/docs/run-an-instance.md +++ b/docs/run-an-instance.md @@ -59,9 +59,32 @@ sudo service nscd start | `CORS_URL` | not used | `https://cobalt.tools/` | cross-origin resource sharing url. api will be available only from this url if `CORS_WILDCARD` is set to `0`. | | `COOKIE_PATH` | not used | `/cookies.json` | path for cookie file relative to main folder. | | `PROCESSING_PRIORITY` | not used | `10` | changes `nice` value* for ffmpeg subprocess. available only on unix systems. | +| `TIKTOK_DEVICE_INFO` | ➖ | *see below* | device info (including `iid` and `device_id`) for tiktok functionality. required for tiktok to work. | \* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)). +#### TIKTOK_DEVICE_INFO +you need to get your own device info for tiktok functionality to work. this can be done by proxying the app through any request-intercepting proxy (such as [mitmproxy](https://mitmproxy.org)). you need to disable ssl pinning to see requests. there will be no assistance provided by cobalt for this. + +example config (replace **ALL** values with ones you got from mitm): +```json +'{ + "iid": "", + "device_id": "", + "channel": "googleplay", + "app_name": "musical_ly", + "version_code": "310503", + "device_platform": "android", + "device_type": "Redmi+7", + "os_version": "13" +}' +``` + +you can compress the json to save space. if you're using a `.env` file then the line would would look like this (***note the quotes***): +``` +TIKTOK_DEVICE_INFO='{"iid":"","device_id":"","channel":"googleplay","app_name":"musical_ly","version_code":"310503","device_platform":"android","device_type":"Redmi+7","os_version":"13"}' +``` + ### variables for web | variable name | default | example | description | |:---------------------|:---------------------|:------------------------|:--------------------------------------------------------------------------------------| From bcd8ca14b0bcaa8d2390a6cc0b6dba2f1a197c35 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 29 Mar 2024 06:03:45 +0600 Subject: [PATCH 3/9] loc/en: remove extra comma in ContactLink --- src/localization/languages/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json index 43239ad..8450e11 100644 --- a/src/localization/languages/en.json +++ b/src/localization/languages/en.json @@ -1,7 +1,7 @@ { "name": "english", "substrings": { - "ContactLink": "check the status page or create an issue on github." + "ContactLink": "check the status page or create an issue on github" }, "strings": { "AppTitleCobalt": "cobalt", From 5438eb4405d5ebac64cafbc2bc46339a43655370 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 29 Mar 2024 06:04:16 +0600 Subject: [PATCH 4/9] cobalt.css: stretch error popup content to full width --- src/front/cobalt.css | 1 + 1 file changed, 1 insertion(+) diff --git a/src/front/cobalt.css b/src/front/cobalt.css index 90b9718..67516ed 100644 --- a/src/front/cobalt.css +++ b/src/front/cobalt.css @@ -452,6 +452,7 @@ button:active, display: flex; flex-direction: column; gap: 18px; + width: 100%; } .popup.small.visible { transform: translate(-50%, -50%); From 79772d45d7a8e65fe67ee9324d4255ed08181fe3 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 29 Mar 2024 06:04:34 +0600 Subject: [PATCH 5/9] package: bump version to 7.12.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index feff48f..99f6c74 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "7.12.1", + "version": "7.12.2", "author": "wukko", "exports": "./src/cobalt.js", "type": "module", From 18545e7c910976d1c376713e10bdf93d289effbd Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 29 Mar 2024 06:28:52 +0600 Subject: [PATCH 6/9] tiktok: clean up --- src/modules/processing/match.js | 2 - src/modules/processing/services/tiktok.js | 63 +++++++---------------- 2 files changed, 19 insertions(+), 46 deletions(-) diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 4169057..6f715ef 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -83,10 +83,8 @@ export default async function(host, patternMatch, url, lang, obj) { id: patternMatch.id }); break; - case "douyin": case "tiktok": r = await tiktok({ - host: host, postId: patternMatch.postId, id: patternMatch.id, fullAudio: obj.isTTFullAudio, diff --git a/src/modules/processing/services/tiktok.js b/src/modules/processing/services/tiktok.js index daea8c4..ef74c0b 100644 --- a/src/modules/processing/services/tiktok.js +++ b/src/modules/processing/services/tiktok.js @@ -1,33 +1,8 @@ import { genericUserAgent } from "../../config.js"; -const userAgent = genericUserAgent.split(' Chrome/1')[0], - config = { - tiktok: { - short: "https://vt.tiktok.com/", - api: "https://api22-normal-c-alisg.tiktokv.com/aweme/v1/feed/?aweme_id={postId}®ion=US&carrier_region=US", - userAgent: "TikTok/338014 CFNetwork/1410.1 Darwin/22.6.0" - }, - douyin: { - short: "https://v.douyin.com/", - api: "https://www.iesdouyin.com/aweme/v1/web/aweme/detail/?aweme_id={postId}", - userAgent: "TikTok 26.2.0 rv:262018 (iPhone; iOS 14.4.2; en_US) Cronet" - } -} - -function selector(j, h, id) { - if (!j) return false; - let t; - switch (h) { - case "tiktok": - t = j.aweme_list.filter(v => v.aweme_id === id)[0]; - break; - case "douyin": - t = j.aweme_detail; - break; - } - if (t?.length < 3) return false; - return t; -} +const shortDomain = "https://vt.tiktok.com/"; +const apiPath = "https://api22-normal-c-alisg.tiktokv.com/aweme/v1/feed/?region=US&carrier_region=US"; +const apiUserAgent = "TikTok/338014 CFNetwork/1410.1 Darwin/22.6.0"; export default async function(obj) { let postId = obj.postId ? obj.postId : false; @@ -35,9 +10,11 @@ export default async function(obj) { if (!process.env.TIKTOK_DEVICE_INFO) return { error: 'ErrorCouldntFetch' }; if (!postId) { - let html = await fetch(`${config[obj.host].short}${obj.id}`, { + let html = await fetch(`${shortDomain}${obj.id}`, { redirect: "manual", - headers: { "user-agent": userAgent } + headers: { + "user-agent": genericUserAgent.split(' Chrome/1')[0] + } }).then((r) => { return r.text() }).catch(() => { return false }); if (!html) return { error: 'ErrorCouldntFetch' }; @@ -53,24 +30,22 @@ export default async function(obj) { let deviceInfo = JSON.parse(process.env.TIKTOK_DEVICE_INFO); deviceInfo = new URLSearchParams(deviceInfo).toString(); - let detail; - detail = await fetch(`${config[obj.host].api.replace("{postId}", postId)}&${deviceInfo}`, { - headers: { - "user-agent": config[obj.host].userAgent - } - }).then((r) => { return r.json() }).catch(() => { return false }); + let apiURL = new URL(apiPath); + apiURL.searchParams.append("aweme_id", postId); - detail = selector(detail, obj.host, postId); + let detail = await fetch(`${apiURL.href}&${deviceInfo}`, { + headers: { + "user-agent": apiUserAgent + } + }).then((r) => { return r.json() }).catch(() => { return false }); + + detail = detail?.aweme_list?.filter(v => v.aweme_id === postId)[0]; if (!detail) return { error: 'ErrorCouldntFetch' }; let video, videoFilename, audioFilename, isMp3, audio, images, - filenameBase = `${obj.host}_${detail.author.unique_id}_${postId}`; + filenameBase = `tiktok_${detail.author.unique_id}_${postId}`; - if (obj.host === "tiktok") { - images = detail.image_post_info ? detail.image_post_info.images : false - } else { - images = detail.images ? detail.images : false - } + images = detail.image_post_info ? detail.image_post_info.images : false; if (!obj.isAudioOnly && !images) { video = detail.video.play_addr.url_list[0]; @@ -99,7 +74,7 @@ export default async function(obj) { if (images) { let imageLinks = []; for (let i in images) { - let sel = obj.host === "tiktok" ? images[i].display_image.url_list : images[i].url_list; + let sel = images[i].display_image.url_list; sel = sel.filter(p => p.includes(".jpeg?")) imageLinks.push({url: sel[0]}) } From 299b46e940bf98fd45ad5b0865dcf058f72cbc31 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 29 Mar 2024 06:42:01 +0600 Subject: [PATCH 7/9] tiktok: replace filter with find --- src/modules/processing/services/tiktok.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/services/tiktok.js b/src/modules/processing/services/tiktok.js index ef74c0b..c764521 100644 --- a/src/modules/processing/services/tiktok.js +++ b/src/modules/processing/services/tiktok.js @@ -39,7 +39,7 @@ export default async function(obj) { } }).then((r) => { return r.json() }).catch(() => { return false }); - detail = detail?.aweme_list?.filter(v => v.aweme_id === postId)[0]; + detail = detail?.aweme_list?.find(v => v.aweme_id === postId); if (!detail) return { error: 'ErrorCouldntFetch' }; let video, videoFilename, audioFilename, isMp3, audio, images, From 5dcbe397dc06f936b44a5307befb6b410b83a4f9 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 29 Mar 2024 06:44:07 +0600 Subject: [PATCH 8/9] tiktok: default to h264 but fall back to h265 --- src/modules/processing/services/tiktok.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/modules/processing/services/tiktok.js b/src/modules/processing/services/tiktok.js index c764521..046b08b 100644 --- a/src/modules/processing/services/tiktok.js +++ b/src/modules/processing/services/tiktok.js @@ -47,11 +47,15 @@ export default async function(obj) { images = detail.image_post_info ? detail.image_post_info.images : false; + let playAddr = detail.video.play_addr_h264; + + if (!playAddr) playAddr = detail.video.play_addr; + if (!obj.isAudioOnly && !images) { - video = detail.video.play_addr.url_list[0]; + video = playAddr.url_list[0]; videoFilename = `${filenameBase}.mp4`; } else { - let fallback = detail.video.play_addr.url_list[0]; + let fallback = playAddr.url_list[0]; audio = fallback; audioFilename = `${filenameBase}_audio_fv`; // fv - from video if (obj.fullAudio || fallback.includes("music")) { From acdb22c41bc03f2b6f900c711cab3b4bdf89a865 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 29 Mar 2024 06:51:55 +0600 Subject: [PATCH 9/9] tiktok: more cleaning up --- src/modules/processing/services/tiktok.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/modules/processing/services/tiktok.js b/src/modules/processing/services/tiktok.js index 046b08b..f362910 100644 --- a/src/modules/processing/services/tiktok.js +++ b/src/modules/processing/services/tiktok.js @@ -15,7 +15,7 @@ export default async function(obj) { headers: { "user-agent": genericUserAgent.split(' Chrome/1')[0] } - }).then((r) => { return r.text() }).catch(() => { return false }); + }).then(r => r.text()).catch(() => {}); if (!html) return { error: 'ErrorCouldntFetch' }; @@ -37,7 +37,7 @@ export default async function(obj) { headers: { "user-agent": apiUserAgent } - }).then((r) => { return r.json() }).catch(() => { return false }); + }).then(r => r.json()).catch(() => {}); detail = detail?.aweme_list?.find(v => v.aweme_id === postId); if (!detail) return { error: 'ErrorCouldntFetch' }; @@ -45,7 +45,7 @@ export default async function(obj) { let video, videoFilename, audioFilename, isMp3, audio, images, filenameBase = `tiktok_${detail.author.unique_id}_${postId}`; - images = detail.image_post_info ? detail.image_post_info.images : false; + images = detail.image_post_info?.images; let playAddr = detail.video.play_addr_h264;