added ability to download full audios from tiktok (3.3.5)

- it's now possible to download full audios from tiktok videos, you just have to turn that on in settings.
- tiktok audios are better in quality when it's possible to get exact audio used in video and not the full version of it.
- cleaned up the way user preference stuff is passed over between modules, should be way more flexible now.
- added audio ignore list to services config json instead of hardcoding it.

package.json

@@ -1,7 +1,7 @@
 {
     "name": "cobalt",
     "description": "save what you love",
-    "version": "3.3",
+    "version": "3.3.5",
     "author": "wukko",
     "exports": "./src/cobalt.js",
     "type": "module",

src/cobalt.js

@@ -61,16 +61,15 @@ if (fs.existsSync('./.env')) {
             switch (req.params.type) {
                 case 'json':
                     if (req.query.url && req.query.url.length < 150) {
-                        let j = await getJSON(
+                        let j = await getJSON(req.query.url.trim(), languageCode(req), {
-                            req.query.url.trim(),
+                                ip: req.header('x-forwarded-for') ? req.header('x-forwarded-for') : req.ip,
-                            req.header('x-forwarded-for') ? req.header('x-forwarded-for') : req.ip,
+                                format: req.query.format ? req.query.format.slice(0, 5) : "webm",
-                            languageCode(req),
+                                quality: req.query.quality ? req.query.quality.slice(0, 3) : "max",
-                            req.query.format ? req.query.format.slice(0, 5) : "webm",
+                                audioFormat: req.query.audioFormat ? req.query.audioFormat.slice(0, 4) : false,
-                            req.query.quality ? req.query.quality.slice(0, 3) : "max",
+                                isAudioOnly: req.query.audio ? true : false,
-                            req.query.audioFormat ? req.query.audioFormat.slice(0, 4) : false,
+                                noWatermark: req.query.nw ? true : false,
-                            req.query.audio ? true : false,
+                                fullAudio: req.query.ttfull ? true : false,
-                            req.query.nw ? true : false
+                        })
-                        )
                         res.status(j.status).json(j.body);
                     } else {
                         let j = apiJSON(3, { t: loc(languageCode(req), 'ErrorNoLink', process.env.selfURL) })

src/front/cobalt.js

@@ -7,6 +7,7 @@ let switchers = {
     "quality": ["max", "hig", "mid", "low"],
     "audioFormat": ["best", "mp3", "ogg", "wav", "opus"]
 }
+let checkboxes = ["disableTikTokWatermark", "fullTikTokAudio"]
 let exceptions = { // used solely for ios devices, because they're less capable than everything else.
     "ytFormat": "mp4",
     "audioFormat": "mp3"
@@ -176,8 +177,8 @@ function loadSettings() {
     if (!sGet("audioMode")) {
         toggle("audioMode")
     }
-    if (sGet("disableTikTokWatermark") == "true") {
+    for (let i = 0; i < checkboxes.length; i++) {
-        eid("disableTikTokWatermark").checked = true;
+        if (sGet(checkboxes[i]) == "true") eid(checkboxes[i]).checked = true;
     }
     updateToggle("audioMode", sGet("audioMode"));
     for (let i in switchers) {
@@ -226,6 +227,7 @@ async function download(url) {
         }
     } else {
         format = `&nw=true`
+        if (sGet("fullTikTokAudio") == "true") format += `&ttfull=true`
     }
     let mode = (sGet("audioMode") == "true") ? `audio=true` : `quality=${sGet("quality")}`
     fetch(`/api/json?audioFormat=${sGet("audioFormat")}&${mode}${format}&url=${encodeURIComponent(url)}`).then(async (response) => {

src/localization/languages/en.json

@@ -6,7 +6,7 @@
     },
     "strings": {
         "ChangelogContentTitle": "soundcloud and better usability (3.3)",
-        "ChangelogContent": "- full support for soundcloud is here. you now can save your favorite songs from there, if you want to.\n- did you know that there's an audio download mode in cobalt? if you didn't, there's now a tooltip that shows you how to switch between modes.\n- added length limit to conversion of audios, because converting a 3 hour audio to wav will give you a 4gb file, and that's just unreasonable. you can still download audio in original (best) format without any limits.\n- if best and preferred audio format match, cobalt won't needlessly convert the audio anymore.\n- fixed format override for ios, you still might have to toggle between them once.\n- increased input area length limit on frontend because some reddit and soundcloud links wouldn't fit.\n- version in settings now opens current commit page on github, instead of general commits page. it also opens in a new tab instead of replacing the current one.\n- fixed some localization stuff in english, russian, and ukrainian. it's now easier to understand what mode is on, and general cobalt description in russian doesn't sound awkward anymore.",
+        "ChangelogContent": "- full support for soundcloud is here. you now can save your favorite songs from there, if you want to.\n- added ability to download full audios from tiktok/douyin, and made tiktok audio downloads better in general.\n- did you know that there's an audio download mode in cobalt? if you didn't, there's now a tooltip that shows you how to switch between modes.\n- added length limit to conversion of audios, because converting a 3 hour audio to wav will give you a 4gb file, and that's just unreasonable. you can still download audio in original (best) format without any limits.\n- if best and preferred audio format match, cobalt won't needlessly convert the audio anymore.\n- fixed format override for ios, you still might have to toggle between them once.\n- increased input area length limit on frontend because some reddit and soundcloud links wouldn't fit.\n- version in settings now opens current commit page on github, instead of general commits page. it also opens in a new tab instead of replacing the current one.\n- fixed some localization stuff in english, russian, and ukrainian. it's now easier to understand what mode is on, and general cobalt description in russian doesn't sound awkward anymore.",
 
         "FollowTwitter": "follow cobalt's twitter account for polls, updates, and more: <a class=\"text-backdrop\" href=\"https://twitter.com/justusecobalt\" target=\"_blank\">@justusecobalt</a>",
         "LinkInput": "paste the link here",
@@ -94,6 +94,8 @@
         "ModeToggle": "mode",
         "ModeToggleSmart": "smart",
         "PressToChange": "press to change",
-        "ErrorLengthAudioConvert": "current length limit for audio conversion is {s} minutes. pick \"best\" format instead!"
+        "ErrorLengthAudioConvert": "current length limit for audio conversion is {s} minutes. pick \"best\" format instead!",
+        "SettingsAudioFullTikTok": "download full audio",
+        "SettingsAudioFullTikTokDescription": "this audio is most often music or original sound used in video. aka audio without voiceover, tts, or trimming will be downloaded, if it's available, of course."
     }
 }

src/localization/languages/ru.json

@@ -90,6 +90,8 @@
         "ModeToggle": "режим",
         "ModeToggleSmart": "умный",
         "PressToChange": "нажми, чтобы изменить",
-        "ErrorLengthAudioConvert": "я не могу конвертировать аудио дольше чем {s} минут(ы). выбери \"лучший\" формат аудио, чтобы скачать аудио такой продолжительности."
+        "ErrorLengthAudioConvert": "я не могу конвертировать аудио дольше чем {s} минут(ы). выбери \"лучший\" формат аудио, чтобы скачать аудио такой продолжительности.",
+        "SettingsAudioFullTikTok": "скачивать полное аудио",
+        "SettingsAudioFullTikTokDescription": "обычно такое аудио - оригинальный звук или песня, которое используется в видео. то есть, это аудио без обрезаний, голоса за кадром, и чего-либо подобного."
     }
 }

src/localization/languages/uk.json

@@ -90,6 +90,8 @@
         "ModeToggle": "режим",
         "ModeToggleSmart": "розумний",
         "PressToChange": "натисни, щоб змінити",
-        "ErrorLengthAudioConvert": "я не можу конвертувати аудіо довше ніж {s} хвилин (и). вибери \"найкращий\" формат аудіо, щоб завантажити аудіо такої тривалості."
+        "ErrorLengthAudioConvert": "я не можу конвертувати аудіо довше ніж {s} хвилин (и). вибери \"найкращий\" формат аудіо, щоб завантажити аудіо такої тривалості.",
+        "SettingsAudioFullTikTok": "завантажувати повне аудіо",
+        "SettingsAudioFullTikTokDescription": "зазвичай таке аудіо-оригінальний звук або пісня, яке використовується в відео. тобто, це аудіо без обрізань, голосу за кадром, і чого-небудь подібного."
     }
 }

src/modules/api.js

@@ -7,7 +7,7 @@ import { errorUnsupported } from "./sub/errors.js";
 import loc from "../localization/manager.js";
 import match from "./match.js";
 
-export async function getJSON(originalURL, ip, lang, format, quality, audioFormat, isAudioOnly, noWatermark) {
+export async function getJSON(originalURL, lang, obj) {
     try {
         let url = decodeURI(originalURL);
         if (!url.includes('http://')) {
@@ -32,7 +32,7 @@ export async function getJSON(originalURL, ip, lang, format, quality, audioForma
                     if (patternMatch) break;
                 }
                 if (patternMatch) {
-                    return await match(host, patternMatch, url, ip, lang, format, quality, audioFormat, isAudioOnly, noWatermark);
+                    return await match(host, patternMatch, url, lang, obj);
                 } return apiJSON(0, { t: errorUnsupported(lang) })
             } return apiJSON(0, { t: errorUnsupported(lang) })
         } else {

src/modules/config.js

@@ -1,9 +1,11 @@
 import loadJson from "./sub/loadJSON.js";
 const config = loadJson("./src/config.json");
 const packageJson = loadJson("./package.json");
+const servicesConfigJson = loadJson("./src/modules/servicesConfig.json");
 
 export const
-    services = loadJson("./src/modules/servicesConfig.json"),
+    services = servicesConfigJson.config,
+    audioIgnore = servicesConfigJson.audioIgnore,
     appName = packageJson.name,
     version = packageJson.version,
     streamLifespan = config.streamLifespan,

src/modules/match.js

@@ -15,7 +15,7 @@ import matchActionDecider from "./sub/matchActionDecider.js";
 import vimeo from "./services/vimeo.js";
 import soundcloud from "./services/soundcloud.js";
 
-export default async function (host, patternMatch, url, ip, lang, format, quality, audioFormat, isAudioOnly, noWatermark) {
+export default async function (host, patternMatch, url, lang, obj) {
     try {
         if (!testers[host]) return apiJSON(0, { t: errorUnsupported(lang) });
         if (!(testers[host](patternMatch))) throw Error();
@@ -32,7 +32,7 @@ export default async function (host, patternMatch, url, ip, lang, format, qualit
                 r = await vk({
                     userId: patternMatch["userId"],
                     videoId: patternMatch["videoId"],
-                    lang: lang, quality: quality
+                    lang: lang, quality: obj.quality
                 });
                 break;
             case "bilibili":
@@ -44,11 +44,11 @@ export default async function (host, patternMatch, url, ip, lang, format, qualit
             case "youtube":
                 let fetchInfo = {
                     id: patternMatch["id"].slice(0, 11),
-                    lang: lang, quality: quality,
+                    lang: lang, quality: obj.quality,
                     format: "webm"
                 };
-                if (url.match('music.youtube.com') || isAudioOnly == true) format = "audio";
+                if (url.match('music.youtube.com') || obj.isAudioOnly == true) obj.format = "audio";
-                switch (format) {
+                switch (obj.format) {
                     case "mp4":
                         fetchInfo["format"] = "mp4";
                         break;
@@ -56,7 +56,7 @@ export default async function (host, patternMatch, url, ip, lang, format, qualit
                         fetchInfo["format"] = "webm";
                         fetchInfo["isAudioOnly"] = true;
                         fetchInfo["quality"] = "max";
-                        isAudioOnly = true;
+                        obj.isAudioOnly = true;
                         break;
                 }
                 r = await youtube(fetchInfo);
@@ -71,13 +71,17 @@ export default async function (host, patternMatch, url, ip, lang, format, qualit
             case "tiktok":
                 r = await tiktok({
                     postId: patternMatch["postId"],
-                    id: patternMatch["id"], lang: lang, noWatermark: noWatermark
+                    id: patternMatch["id"], lang: lang,
+                    noWatermark: obj.noWatermark, fullAudio: obj.fullAudio,
+                    isAudioOnly: obj.isAudioOnly
                 });
                 break;
             case "douyin":
                 r = await douyin({
                     postId: patternMatch["postId"],
-                    id: patternMatch["id"], lang: lang, noWatermark: noWatermark
+                    id: patternMatch["id"], lang: lang,
+                    noWatermark: obj.noWatermark, fullAudio: obj.fullAudio,
+                    isAudioOnly: obj.isAudioOnly
                 });
                 break;
             case "tumblr":
@@ -88,23 +92,23 @@ export default async function (host, patternMatch, url, ip, lang, format, qualit
                 break;
             case "vimeo":
                 r = await vimeo({
-                    id: patternMatch["id"].slice(0, 11), quality: quality,
+                    id: patternMatch["id"].slice(0, 11), quality: obj.quality,
                     lang: lang
                 });
                 break;
             case "soundcloud":
-                isAudioOnly = true;
+                obj.isAudioOnly = true;
                 r = await soundcloud({
                     author: patternMatch["author"], song: patternMatch["song"], url: url,
                     shortLink: patternMatch["shortLink"] ? patternMatch["shortLink"] : false,
-                    format: audioFormat,
+                    format: obj.audioFormat,
                     lang: lang
                 });
                 break;
             default:
                 return apiJSON(0, { t: errorUnsupported(lang) });
         }
-        return matchActionDecider(r, host, ip, audioFormat, isAudioOnly)
+        return matchActionDecider(r, host, obj.ip, obj.audioFormat, obj.isAudioOnly)
     } catch (e) {
         return apiJSON(0, { t: genericError(lang, host) })
     }

src/modules/pageRender/page.js

@@ -212,6 +212,10 @@ export default function(obj) {
                         explanation: loc(obj.lang, 'SettingsAudioFormatDescription'),
                         items: audioFormats
                     })
+                }) + settingsCategory({
+                    name: "tiktok",
+                    title: "tiktok & douyin",
+                    body: checkbox("fullTikTokAudio", loc(obj.lang, 'SettingsAudioFullTikTok'), loc(obj.lang, 'SettingsAudioFullTikTok')) + `<div class="explanation">${loc(obj.lang, 'SettingsAudioFullTikTokDescription')}</div>`
                 })
             }, {
                 name: "other",

src/modules/services/douyin.js

@@ -29,16 +29,31 @@ export default async function(obj) {
             return { error: loc(obj.lang, 'ErrorCantConnectToServiceAPI', 'douyin') };
         });
         iteminfo = JSON.parse(iteminfo.body);
-        if (iteminfo['item_list'][0]['video']['play_addr']['url_list'][0]) {
+        let video = iteminfo['item_list'][0]['video']['play_addr']['url_list'][0];
+        let audio = obj.isAudioOnly ? iteminfo['item_list'][0]["music"]["play_url"]["url_list"][0] : false;
+        if (audio && obj.fullAudio) {
+            return {
+                urls: audio,
+                audioFilename: `douyin_${obj.postId}_audio_full`,
+                isAudio: true
+            }
+        } else if (audio && audio.slice(-4) == ".mp3") {
+            return {
+                urls: audio,
+                audioFilename: `douyin_${obj.postId}_audio`,
+                isAudio: true,
+                isMp3: true,
+            };
+        } else if (video) {
             if (!obj.noWatermark) {
                 return {
-                    urls: iteminfo['item_list'][0]['video']['play_addr']['url_list'][0],
+                    urls: video,
                     audioFilename: `douyin_${obj.postId}_audio`, 
                     filename: `douyin_${obj.postId}.mp4`
                 };
             } else {
                 return {
-                    urls: iteminfo['item_list'][0]['video']['play_addr']['url_list'][0].replace("playwm", "play"),
+                    urls: video.replace("playwm", "play"),
                     audioFilename: `douyin_${obj.postId}_audio`,
                     filename: `douyin_${obj.postId}_nw.mp4`
                 };

src/modules/services/tiktok.js

@@ -17,14 +17,18 @@ export default async function(obj) {
                 obj.postId = html.split('aweme/detail/')[1].split('?')[0]
             }
         }
-        if (!obj.noWatermark) {
+        if (!obj.noWatermark && !obj.isAudioOnly) {
             let html = await got.get(`https://tiktok.com/@video/video/${obj.postId}`, { headers: { "user-agent": genericUserAgent } });
             html.on('error', (err) => {
                 return { error: loc(obj.lang, 'ErrorCantConnectToServiceAPI', 'tiktok') };
             });
             html = html.body;
             if (html.includes(',"preloadList":[{"url":"')) {
-                return { urls: unicodeDecode(html.split(',"preloadList":[{"url":"')[1].split('","id":"')[0].trim()), audioFilename: `tiktok_${obj.postId}_audio`, filename: `tiktok_${obj.postId}.mp4` };
+                return {
+                    urls: unicodeDecode(html.split(',"preloadList":[{"url":"')[1].split('","id":"')[0].trim()),
+                    audioFilename: `tiktok_${obj.postId}_audio`,
+                    filename: `tiktok_${obj.postId}.mp4`
+                };
             } else {
                 return { error: loc(obj.lang, 'ErrorEmptyDownload') };
             }
@@ -34,8 +38,27 @@ export default async function(obj) {
                 return { error: loc(obj.lang, 'ErrorCantConnectToServiceAPI', 'tiktok') };
             });
             detail = JSON.parse(detail.body);
-            if (detail["aweme_detail"]["video"]["play_addr"]["url_list"][0]) {
+            let video = detail["aweme_detail"]["video"]["play_addr"]["url_list"][0];
-                return { urls: detail["aweme_detail"]["video"]["play_addr"]["url_list"][0], audioFilename: `tiktok_${obj.postId}_audio`, filename: `tiktok_${obj.postId}_nw.mp4` };
+            let audio = obj.isAudioOnly ? detail["aweme_detail"]["music"]["play_url"]["url_list"][0] : false;
+            if (audio && obj.fullAudio) {
+                return {
+                    urls: audio,
+                    audioFilename: `tiktok_${obj.postId}_audio_full`,
+                    isAudio: true
+                }
+            } else if (audio && audio.slice(-4) == ".mp3") {
+                return {
+                    urls: audio,
+                    audioFilename: `tiktok_${obj.postId}_audio`,
+                    isAudio: true,
+                    isMp3: true,
+                };
+            } else if (video) {
+                return {
+                    urls: video,
+                    audioFilename: `tiktok_${obj.postId}_audio`,
+                    filename: `tiktok_${obj.postId}_nw.mp4`
+                };
             } else {
                 return { error: loc(obj.lang, 'ErrorEmptyDownload') };
             }

src/modules/servicesConfig.json

@@ -1,76 +1,79 @@
 {
-    "bilibili": {
+    "audioIgnore": ["vk", "vimeo"],
-        "alias": "bilibili.com",
+    "config": {
-        "patterns": ["video/:id"],
+        "bilibili": {
-        "quality_match": ["2160", "1440", "1080", "720", "480", "360", "240", "144"],
+            "alias": "bilibili.com",
-        "enabled": true
+            "patterns": ["video/:id"],
-    },
+            "quality_match": ["2160", "1440", "1080", "720", "480", "360", "240", "144"],
-    "reddit": {
+            "enabled": true
-        "patterns": ["r/:sub/comments/:id/:title"],
+        },
-        "enabled": true
+        "reddit": {
-    },
+            "patterns": ["r/:sub/comments/:id/:title"],
-    "twitter": {
+            "enabled": true
-        "patterns": [":user/status/:id"],
+        },
-        "quality_match": ["1080", "720", "480", "360", "240", "144"],
+        "twitter": {
-        "enabled": true,
+            "patterns": [":user/status/:id"],
-        "api": "api.twitter.com",
+            "quality_match": ["1080", "720", "480", "360", "240", "144"],
-        "token": "AAAAAAAAAAAAAAAAAAAAAIK1zgAAAAAA2tUWuhGZ2JceoId5GwYWU5GspY4%3DUq7gzFoCZs1QfwGoVdvSac3IniczZEYXIcDyumCauIXpcAPorE",
+            "enabled": true,
-        "apiURLs": {
+            "api": "api.twitter.com",
-            "activate": "1.1/guest/activate.json",
+            "token": "AAAAAAAAAAAAAAAAAAAAAIK1zgAAAAAA2tUWuhGZ2JceoId5GwYWU5GspY4%3DUq7gzFoCZs1QfwGoVdvSac3IniczZEYXIcDyumCauIXpcAPorE",
-            "status_show": "1.1/statuses/show.json"
+            "apiURLs": {
+                "activate": "1.1/guest/activate.json",
+                "status_show": "1.1/statuses/show.json"
+            }
+        },
+        "vk": {
+            "patterns": ["video-:userId_:videoId"],
+            "quality_match": {
+                "2160": 7,
+                "1440": 6,
+                "1080": 5,
+                "720": 3,
+                "480": 2,
+                "360": 1,
+                "240": 0,
+                "144": 4
+            },
+            "quality": {
+                "1080": "hig",
+                "720": "mid",
+                "480": "low"
+            },
+            "enabled": true
+        },
+        "youtube": {
+            "alias": "youtube, youtube music",
+            "patterns": ["watch?v=:id"],
+            "quality_match": ["2160", "1440", "1080", "720", "480", "360", "240", "144"],
+            "bestAudio": "opus",
+            "quality": {
+                "1080": "hig",
+                "720": "mid",
+                "480": "low"
+            },
+            "enabled": true
+        },
+        "tumblr": {
+            "patterns": ["post/:id", "blog/view/:user/:id"],
+            "enabled": true
+        },
+        "tiktok": {
+            "patterns": [":user/video/:postId", ":id", "t/:id"],
+            "enabled": true
+        },
+        "douyin": {
+            "patterns": ["video/:postId", ":id"],
+            "enabled": true
+        },
+        "vimeo": {
+            "patterns": [":id"],
+            "enabled": true
+        },
+        "soundcloud": {
+            "patterns": [":author/:song", ":shortLink"],
+            "bestAudio": "mp3",
+            "clientid": "lnFbWHXluNwOkW7TxTYUXrrse0qj1C72",
+            "enabled": true
         }
-    },
-    "vk": {
-        "patterns": ["video-:userId_:videoId"],
-        "quality_match": {
-            "2160": 7,
-            "1440": 6,
-            "1080": 5,
-            "720": 3,
-            "480": 2,
-            "360": 1,
-            "240": 0,
-            "144": 4
-        },
-        "quality": {
-            "1080": "hig",
-            "720": "mid",
-            "480": "low"
-        },
-        "enabled": true
-    },
-    "youtube": {
-        "alias": "youtube, youtube music",
-        "patterns": ["watch?v=:id"],
-        "quality_match": ["2160", "1440", "1080", "720", "480", "360", "240", "144"],
-        "bestAudio": "opus",
-        "quality": {
-            "1080": "hig",
-            "720": "mid",
-            "480": "low"
-        },
-        "enabled": true
-    },
-    "tumblr": {
-        "patterns": ["post/:id", "blog/view/:user/:id"],
-        "enabled": true
-    },
-    "tiktok": {
-        "patterns": [":user/video/:postId", ":id", "t/:id"],
-        "enabled": true
-    },
-    "douyin": {
-        "patterns": ["video/:postId", ":id"],
-        "enabled": true
-    },
-    "vimeo": {
-        "patterns": [":id"],
-        "enabled": true
-    },
-    "soundcloud": {
-        "patterns": [":author/:song", ":shortLink"],
-        "bestAudio": "mp3",
-        "clientid": "lnFbWHXluNwOkW7TxTYUXrrse0qj1C72",
-        "enabled": true
     }    
 }

src/modules/sub/matchActionDecider.js

@@ -1,4 +1,4 @@
-import { services, supportedAudio } from "../config.js"
+import { audioIgnore, services, supportedAudio } from "../config.js"
 import { apiJSON } from "./utils.js"
 
 export default function(r, host, ip, audioFormat, isAudioOnly) {
@@ -55,7 +55,17 @@ export default function(r, host, ip, audioFormat, isAudioOnly) {
                 audioFormat = "m4a"
                 copy = true
             }
-            if (host == "reddit" && r.typeId == 1 || host == "vk" || host == "vimeo") return apiJSON(0, { t: r.audioFilename });
+            if ((host == "tiktok" || host == "douyin") && r.isAudio) {
+                if (r.isMp3) {
+                    audioFormat = "mp3"
+                    type = "bridge"
+                    copy = false
+                } else {
+                    type = "bridge"
+                    copy = false
+                }
+            }
+            if (host == "reddit" && r.typeId == 1 || audioIgnore.includes(host)) return apiJSON(0, { t: r.audioFilename });
             return apiJSON(2, {
                 type: type,
                 u: Array.isArray(r.urls) ? r.urls[1] : r.urls, service: host, ip: ip,