added ability to download full audios from tiktok (3.3.5)

- it's now possible to download full audios from tiktok videos, you just have to turn that on in settings.
- tiktok audios are better in quality when it's possible to get exact audio used in video and not the full version of it.
- cleaned up the way user preference stuff is passed over between modules, should be way more flexible now.
- added audio ignore list to services config json instead of hardcoding it.
This commit is contained in:
wukko 2022-08-23 20:43:56 +06:00
parent 189ecf8fe7
commit a8b5555a1b
14 changed files with 182 additions and 114 deletions

View file

@ -1,7 +1,7 @@
{ {
"name": "cobalt", "name": "cobalt",
"description": "save what you love", "description": "save what you love",
"version": "3.3", "version": "3.3.5",
"author": "wukko", "author": "wukko",
"exports": "./src/cobalt.js", "exports": "./src/cobalt.js",
"type": "module", "type": "module",

View file

@ -61,16 +61,15 @@ if (fs.existsSync('./.env')) {
switch (req.params.type) { switch (req.params.type) {
case 'json': case 'json':
if (req.query.url && req.query.url.length < 150) { if (req.query.url && req.query.url.length < 150) {
let j = await getJSON( let j = await getJSON(req.query.url.trim(), languageCode(req), {
req.query.url.trim(), ip: req.header('x-forwarded-for') ? req.header('x-forwarded-for') : req.ip,
req.header('x-forwarded-for') ? req.header('x-forwarded-for') : req.ip, format: req.query.format ? req.query.format.slice(0, 5) : "webm",
languageCode(req), quality: req.query.quality ? req.query.quality.slice(0, 3) : "max",
req.query.format ? req.query.format.slice(0, 5) : "webm", audioFormat: req.query.audioFormat ? req.query.audioFormat.slice(0, 4) : false,
req.query.quality ? req.query.quality.slice(0, 3) : "max", isAudioOnly: req.query.audio ? true : false,
req.query.audioFormat ? req.query.audioFormat.slice(0, 4) : false, noWatermark: req.query.nw ? true : false,
req.query.audio ? true : false, fullAudio: req.query.ttfull ? true : false,
req.query.nw ? true : false })
)
res.status(j.status).json(j.body); res.status(j.status).json(j.body);
} else { } else {
let j = apiJSON(3, { t: loc(languageCode(req), 'ErrorNoLink', process.env.selfURL) }) let j = apiJSON(3, { t: loc(languageCode(req), 'ErrorNoLink', process.env.selfURL) })

View file

@ -7,6 +7,7 @@ let switchers = {
"quality": ["max", "hig", "mid", "low"], "quality": ["max", "hig", "mid", "low"],
"audioFormat": ["best", "mp3", "ogg", "wav", "opus"] "audioFormat": ["best", "mp3", "ogg", "wav", "opus"]
} }
let checkboxes = ["disableTikTokWatermark", "fullTikTokAudio"]
let exceptions = { // used solely for ios devices, because they're less capable than everything else. let exceptions = { // used solely for ios devices, because they're less capable than everything else.
"ytFormat": "mp4", "ytFormat": "mp4",
"audioFormat": "mp3" "audioFormat": "mp3"
@ -176,8 +177,8 @@ function loadSettings() {
if (!sGet("audioMode")) { if (!sGet("audioMode")) {
toggle("audioMode") toggle("audioMode")
} }
if (sGet("disableTikTokWatermark") == "true") { for (let i = 0; i < checkboxes.length; i++) {
eid("disableTikTokWatermark").checked = true; if (sGet(checkboxes[i]) == "true") eid(checkboxes[i]).checked = true;
} }
updateToggle("audioMode", sGet("audioMode")); updateToggle("audioMode", sGet("audioMode"));
for (let i in switchers) { for (let i in switchers) {
@ -226,6 +227,7 @@ async function download(url) {
} }
} else { } else {
format = `&nw=true` format = `&nw=true`
if (sGet("fullTikTokAudio") == "true") format += `&ttfull=true`
} }
let mode = (sGet("audioMode") == "true") ? `audio=true` : `quality=${sGet("quality")}` let mode = (sGet("audioMode") == "true") ? `audio=true` : `quality=${sGet("quality")}`
fetch(`/api/json?audioFormat=${sGet("audioFormat")}&${mode}${format}&url=${encodeURIComponent(url)}`).then(async (response) => { fetch(`/api/json?audioFormat=${sGet("audioFormat")}&${mode}${format}&url=${encodeURIComponent(url)}`).then(async (response) => {

View file

@ -6,7 +6,7 @@
}, },
"strings": { "strings": {
"ChangelogContentTitle": "soundcloud and better usability (3.3)", "ChangelogContentTitle": "soundcloud and better usability (3.3)",
"ChangelogContent": "- full support for soundcloud is here. you now can save your favorite songs from there, if you want to.\n- did you know that there's an audio download mode in cobalt? if you didn't, there's now a tooltip that shows you how to switch between modes.\n- added length limit to conversion of audios, because converting a 3 hour audio to wav will give you a 4gb file, and that's just unreasonable. you can still download audio in original (best) format without any limits.\n- if best and preferred audio format match, cobalt won't needlessly convert the audio anymore.\n- fixed format override for ios, you still might have to toggle between them once.\n- increased input area length limit on frontend because some reddit and soundcloud links wouldn't fit.\n- version in settings now opens current commit page on github, instead of general commits page. it also opens in a new tab instead of replacing the current one.\n- fixed some localization stuff in english, russian, and ukrainian. it's now easier to understand what mode is on, and general cobalt description in russian doesn't sound awkward anymore.", "ChangelogContent": "- full support for soundcloud is here. you now can save your favorite songs from there, if you want to.\n- added ability to download full audios from tiktok/douyin, and made tiktok audio downloads better in general.\n- did you know that there's an audio download mode in cobalt? if you didn't, there's now a tooltip that shows you how to switch between modes.\n- added length limit to conversion of audios, because converting a 3 hour audio to wav will give you a 4gb file, and that's just unreasonable. you can still download audio in original (best) format without any limits.\n- if best and preferred audio format match, cobalt won't needlessly convert the audio anymore.\n- fixed format override for ios, you still might have to toggle between them once.\n- increased input area length limit on frontend because some reddit and soundcloud links wouldn't fit.\n- version in settings now opens current commit page on github, instead of general commits page. it also opens in a new tab instead of replacing the current one.\n- fixed some localization stuff in english, russian, and ukrainian. it's now easier to understand what mode is on, and general cobalt description in russian doesn't sound awkward anymore.",
"FollowTwitter": "follow cobalt's twitter account for polls, updates, and more: <a class=\"text-backdrop\" href=\"https://twitter.com/justusecobalt\" target=\"_blank\">@justusecobalt</a>", "FollowTwitter": "follow cobalt's twitter account for polls, updates, and more: <a class=\"text-backdrop\" href=\"https://twitter.com/justusecobalt\" target=\"_blank\">@justusecobalt</a>",
"LinkInput": "paste the link here", "LinkInput": "paste the link here",
@ -94,6 +94,8 @@
"ModeToggle": "mode", "ModeToggle": "mode",
"ModeToggleSmart": "smart", "ModeToggleSmart": "smart",
"PressToChange": "press to change", "PressToChange": "press to change",
"ErrorLengthAudioConvert": "current length limit for audio conversion is {s} minutes. pick \"best\" format instead!" "ErrorLengthAudioConvert": "current length limit for audio conversion is {s} minutes. pick \"best\" format instead!",
"SettingsAudioFullTikTok": "download full audio",
"SettingsAudioFullTikTokDescription": "this audio is most often music or original sound used in video. aka audio without voiceover, tts, or trimming will be downloaded, if it's available, of course."
} }
} }

View file

@ -90,6 +90,8 @@
"ModeToggle": "режим", "ModeToggle": "режим",
"ModeToggleSmart": "умный", "ModeToggleSmart": "умный",
"PressToChange": "нажми, чтобы изменить", "PressToChange": "нажми, чтобы изменить",
"ErrorLengthAudioConvert": "я не могу конвертировать аудио дольше чем {s} минут(ы). выбери \"лучший\" формат аудио, чтобы скачать аудио такой продолжительности." "ErrorLengthAudioConvert": "я не могу конвертировать аудио дольше чем {s} минут(ы). выбери \"лучший\" формат аудио, чтобы скачать аудио такой продолжительности.",
"SettingsAudioFullTikTok": "скачивать полное аудио",
"SettingsAudioFullTikTokDescription": "обычно такое аудио - оригинальный звук или песня, которое используется в видео. то есть, это аудио без обрезаний, голоса за кадром, и чего-либо подобного."
} }
} }

View file

@ -90,6 +90,8 @@
"ModeToggle": "режим", "ModeToggle": "режим",
"ModeToggleSmart": "розумний", "ModeToggleSmart": "розумний",
"PressToChange": "натисни, щоб змінити", "PressToChange": "натисни, щоб змінити",
"ErrorLengthAudioConvert": "я не можу конвертувати аудіо довше ніж {s} хвилин (и). вибери \"найкращий\" формат аудіо, щоб завантажити аудіо такої тривалості." "ErrorLengthAudioConvert": "я не можу конвертувати аудіо довше ніж {s} хвилин (и). вибери \"найкращий\" формат аудіо, щоб завантажити аудіо такої тривалості.",
"SettingsAudioFullTikTok": "завантажувати повне аудіо",
"SettingsAudioFullTikTokDescription": "зазвичай таке аудіо-оригінальний звук або пісня, яке використовується в відео. тобто, це аудіо без обрізань, голосу за кадром, і чого-небудь подібного."
} }
} }

View file

@ -7,7 +7,7 @@ import { errorUnsupported } from "./sub/errors.js";
import loc from "../localization/manager.js"; import loc from "../localization/manager.js";
import match from "./match.js"; import match from "./match.js";
export async function getJSON(originalURL, ip, lang, format, quality, audioFormat, isAudioOnly, noWatermark) { export async function getJSON(originalURL, lang, obj) {
try { try {
let url = decodeURI(originalURL); let url = decodeURI(originalURL);
if (!url.includes('http://')) { if (!url.includes('http://')) {
@ -32,7 +32,7 @@ export async function getJSON(originalURL, ip, lang, format, quality, audioForma
if (patternMatch) break; if (patternMatch) break;
} }
if (patternMatch) { if (patternMatch) {
return await match(host, patternMatch, url, ip, lang, format, quality, audioFormat, isAudioOnly, noWatermark); return await match(host, patternMatch, url, lang, obj);
} return apiJSON(0, { t: errorUnsupported(lang) }) } return apiJSON(0, { t: errorUnsupported(lang) })
} return apiJSON(0, { t: errorUnsupported(lang) }) } return apiJSON(0, { t: errorUnsupported(lang) })
} else { } else {

View file

@ -1,9 +1,11 @@
import loadJson from "./sub/loadJSON.js"; import loadJson from "./sub/loadJSON.js";
const config = loadJson("./src/config.json"); const config = loadJson("./src/config.json");
const packageJson = loadJson("./package.json"); const packageJson = loadJson("./package.json");
const servicesConfigJson = loadJson("./src/modules/servicesConfig.json");
export const export const
services = loadJson("./src/modules/servicesConfig.json"), services = servicesConfigJson.config,
audioIgnore = servicesConfigJson.audioIgnore,
appName = packageJson.name, appName = packageJson.name,
version = packageJson.version, version = packageJson.version,
streamLifespan = config.streamLifespan, streamLifespan = config.streamLifespan,

View file

@ -15,7 +15,7 @@ import matchActionDecider from "./sub/matchActionDecider.js";
import vimeo from "./services/vimeo.js"; import vimeo from "./services/vimeo.js";
import soundcloud from "./services/soundcloud.js"; import soundcloud from "./services/soundcloud.js";
export default async function (host, patternMatch, url, ip, lang, format, quality, audioFormat, isAudioOnly, noWatermark) { export default async function (host, patternMatch, url, lang, obj) {
try { try {
if (!testers[host]) return apiJSON(0, { t: errorUnsupported(lang) }); if (!testers[host]) return apiJSON(0, { t: errorUnsupported(lang) });
if (!(testers[host](patternMatch))) throw Error(); if (!(testers[host](patternMatch))) throw Error();
@ -32,7 +32,7 @@ export default async function (host, patternMatch, url, ip, lang, format, qualit
r = await vk({ r = await vk({
userId: patternMatch["userId"], userId: patternMatch["userId"],
videoId: patternMatch["videoId"], videoId: patternMatch["videoId"],
lang: lang, quality: quality lang: lang, quality: obj.quality
}); });
break; break;
case "bilibili": case "bilibili":
@ -44,11 +44,11 @@ export default async function (host, patternMatch, url, ip, lang, format, qualit
case "youtube": case "youtube":
let fetchInfo = { let fetchInfo = {
id: patternMatch["id"].slice(0, 11), id: patternMatch["id"].slice(0, 11),
lang: lang, quality: quality, lang: lang, quality: obj.quality,
format: "webm" format: "webm"
}; };
if (url.match('music.youtube.com') || isAudioOnly == true) format = "audio"; if (url.match('music.youtube.com') || obj.isAudioOnly == true) obj.format = "audio";
switch (format) { switch (obj.format) {
case "mp4": case "mp4":
fetchInfo["format"] = "mp4"; fetchInfo["format"] = "mp4";
break; break;
@ -56,7 +56,7 @@ export default async function (host, patternMatch, url, ip, lang, format, qualit
fetchInfo["format"] = "webm"; fetchInfo["format"] = "webm";
fetchInfo["isAudioOnly"] = true; fetchInfo["isAudioOnly"] = true;
fetchInfo["quality"] = "max"; fetchInfo["quality"] = "max";
isAudioOnly = true; obj.isAudioOnly = true;
break; break;
} }
r = await youtube(fetchInfo); r = await youtube(fetchInfo);
@ -71,13 +71,17 @@ export default async function (host, patternMatch, url, ip, lang, format, qualit
case "tiktok": case "tiktok":
r = await tiktok({ r = await tiktok({
postId: patternMatch["postId"], postId: patternMatch["postId"],
id: patternMatch["id"], lang: lang, noWatermark: noWatermark id: patternMatch["id"], lang: lang,
noWatermark: obj.noWatermark, fullAudio: obj.fullAudio,
isAudioOnly: obj.isAudioOnly
}); });
break; break;
case "douyin": case "douyin":
r = await douyin({ r = await douyin({
postId: patternMatch["postId"], postId: patternMatch["postId"],
id: patternMatch["id"], lang: lang, noWatermark: noWatermark id: patternMatch["id"], lang: lang,
noWatermark: obj.noWatermark, fullAudio: obj.fullAudio,
isAudioOnly: obj.isAudioOnly
}); });
break; break;
case "tumblr": case "tumblr":
@ -88,23 +92,23 @@ export default async function (host, patternMatch, url, ip, lang, format, qualit
break; break;
case "vimeo": case "vimeo":
r = await vimeo({ r = await vimeo({
id: patternMatch["id"].slice(0, 11), quality: quality, id: patternMatch["id"].slice(0, 11), quality: obj.quality,
lang: lang lang: lang
}); });
break; break;
case "soundcloud": case "soundcloud":
isAudioOnly = true; obj.isAudioOnly = true;
r = await soundcloud({ r = await soundcloud({
author: patternMatch["author"], song: patternMatch["song"], url: url, author: patternMatch["author"], song: patternMatch["song"], url: url,
shortLink: patternMatch["shortLink"] ? patternMatch["shortLink"] : false, shortLink: patternMatch["shortLink"] ? patternMatch["shortLink"] : false,
format: audioFormat, format: obj.audioFormat,
lang: lang lang: lang
}); });
break; break;
default: default:
return apiJSON(0, { t: errorUnsupported(lang) }); return apiJSON(0, { t: errorUnsupported(lang) });
} }
return matchActionDecider(r, host, ip, audioFormat, isAudioOnly) return matchActionDecider(r, host, obj.ip, obj.audioFormat, obj.isAudioOnly)
} catch (e) { } catch (e) {
return apiJSON(0, { t: genericError(lang, host) }) return apiJSON(0, { t: genericError(lang, host) })
} }

View file

@ -212,6 +212,10 @@ export default function(obj) {
explanation: loc(obj.lang, 'SettingsAudioFormatDescription'), explanation: loc(obj.lang, 'SettingsAudioFormatDescription'),
items: audioFormats items: audioFormats
}) })
}) + settingsCategory({
name: "tiktok",
title: "tiktok & douyin",
body: checkbox("fullTikTokAudio", loc(obj.lang, 'SettingsAudioFullTikTok'), loc(obj.lang, 'SettingsAudioFullTikTok')) + `<div class="explanation">${loc(obj.lang, 'SettingsAudioFullTikTokDescription')}</div>`
}) })
}, { }, {
name: "other", name: "other",

View file

@ -29,16 +29,31 @@ export default async function(obj) {
return { error: loc(obj.lang, 'ErrorCantConnectToServiceAPI', 'douyin') }; return { error: loc(obj.lang, 'ErrorCantConnectToServiceAPI', 'douyin') };
}); });
iteminfo = JSON.parse(iteminfo.body); iteminfo = JSON.parse(iteminfo.body);
if (iteminfo['item_list'][0]['video']['play_addr']['url_list'][0]) { let video = iteminfo['item_list'][0]['video']['play_addr']['url_list'][0];
let audio = obj.isAudioOnly ? iteminfo['item_list'][0]["music"]["play_url"]["url_list"][0] : false;
if (audio && obj.fullAudio) {
return {
urls: audio,
audioFilename: `douyin_${obj.postId}_audio_full`,
isAudio: true
}
} else if (audio && audio.slice(-4) == ".mp3") {
return {
urls: audio,
audioFilename: `douyin_${obj.postId}_audio`,
isAudio: true,
isMp3: true,
};
} else if (video) {
if (!obj.noWatermark) { if (!obj.noWatermark) {
return { return {
urls: iteminfo['item_list'][0]['video']['play_addr']['url_list'][0], urls: video,
audioFilename: `douyin_${obj.postId}_audio`, audioFilename: `douyin_${obj.postId}_audio`,
filename: `douyin_${obj.postId}.mp4` filename: `douyin_${obj.postId}.mp4`
}; };
} else { } else {
return { return {
urls: iteminfo['item_list'][0]['video']['play_addr']['url_list'][0].replace("playwm", "play"), urls: video.replace("playwm", "play"),
audioFilename: `douyin_${obj.postId}_audio`, audioFilename: `douyin_${obj.postId}_audio`,
filename: `douyin_${obj.postId}_nw.mp4` filename: `douyin_${obj.postId}_nw.mp4`
}; };

View file

@ -17,14 +17,18 @@ export default async function(obj) {
obj.postId = html.split('aweme/detail/')[1].split('?')[0] obj.postId = html.split('aweme/detail/')[1].split('?')[0]
} }
} }
if (!obj.noWatermark) { if (!obj.noWatermark && !obj.isAudioOnly) {
let html = await got.get(`https://tiktok.com/@video/video/${obj.postId}`, { headers: { "user-agent": genericUserAgent } }); let html = await got.get(`https://tiktok.com/@video/video/${obj.postId}`, { headers: { "user-agent": genericUserAgent } });
html.on('error', (err) => { html.on('error', (err) => {
return { error: loc(obj.lang, 'ErrorCantConnectToServiceAPI', 'tiktok') }; return { error: loc(obj.lang, 'ErrorCantConnectToServiceAPI', 'tiktok') };
}); });
html = html.body; html = html.body;
if (html.includes(',"preloadList":[{"url":"')) { if (html.includes(',"preloadList":[{"url":"')) {
return { urls: unicodeDecode(html.split(',"preloadList":[{"url":"')[1].split('","id":"')[0].trim()), audioFilename: `tiktok_${obj.postId}_audio`, filename: `tiktok_${obj.postId}.mp4` }; return {
urls: unicodeDecode(html.split(',"preloadList":[{"url":"')[1].split('","id":"')[0].trim()),
audioFilename: `tiktok_${obj.postId}_audio`,
filename: `tiktok_${obj.postId}.mp4`
};
} else { } else {
return { error: loc(obj.lang, 'ErrorEmptyDownload') }; return { error: loc(obj.lang, 'ErrorEmptyDownload') };
} }
@ -34,8 +38,27 @@ export default async function(obj) {
return { error: loc(obj.lang, 'ErrorCantConnectToServiceAPI', 'tiktok') }; return { error: loc(obj.lang, 'ErrorCantConnectToServiceAPI', 'tiktok') };
}); });
detail = JSON.parse(detail.body); detail = JSON.parse(detail.body);
if (detail["aweme_detail"]["video"]["play_addr"]["url_list"][0]) { let video = detail["aweme_detail"]["video"]["play_addr"]["url_list"][0];
return { urls: detail["aweme_detail"]["video"]["play_addr"]["url_list"][0], audioFilename: `tiktok_${obj.postId}_audio`, filename: `tiktok_${obj.postId}_nw.mp4` }; let audio = obj.isAudioOnly ? detail["aweme_detail"]["music"]["play_url"]["url_list"][0] : false;
if (audio && obj.fullAudio) {
return {
urls: audio,
audioFilename: `tiktok_${obj.postId}_audio_full`,
isAudio: true
}
} else if (audio && audio.slice(-4) == ".mp3") {
return {
urls: audio,
audioFilename: `tiktok_${obj.postId}_audio`,
isAudio: true,
isMp3: true,
};
} else if (video) {
return {
urls: video,
audioFilename: `tiktok_${obj.postId}_audio`,
filename: `tiktok_${obj.postId}_nw.mp4`
};
} else { } else {
return { error: loc(obj.lang, 'ErrorEmptyDownload') }; return { error: loc(obj.lang, 'ErrorEmptyDownload') };
} }

View file

@ -1,4 +1,6 @@
{ {
"audioIgnore": ["vk", "vimeo"],
"config": {
"bilibili": { "bilibili": {
"alias": "bilibili.com", "alias": "bilibili.com",
"patterns": ["video/:id"], "patterns": ["video/:id"],
@ -73,4 +75,5 @@
"clientid": "lnFbWHXluNwOkW7TxTYUXrrse0qj1C72", "clientid": "lnFbWHXluNwOkW7TxTYUXrrse0qj1C72",
"enabled": true "enabled": true
} }
}
} }

View file

@ -1,4 +1,4 @@
import { services, supportedAudio } from "../config.js" import { audioIgnore, services, supportedAudio } from "../config.js"
import { apiJSON } from "./utils.js" import { apiJSON } from "./utils.js"
export default function(r, host, ip, audioFormat, isAudioOnly) { export default function(r, host, ip, audioFormat, isAudioOnly) {
@ -55,7 +55,17 @@ export default function(r, host, ip, audioFormat, isAudioOnly) {
audioFormat = "m4a" audioFormat = "m4a"
copy = true copy = true
} }
if (host == "reddit" && r.typeId == 1 || host == "vk" || host == "vimeo") return apiJSON(0, { t: r.audioFilename }); if ((host == "tiktok" || host == "douyin") && r.isAudio) {
if (r.isMp3) {
audioFormat = "mp3"
type = "bridge"
copy = false
} else {
type = "bridge"
copy = false
}
}
if (host == "reddit" && r.typeId == 1 || audioIgnore.includes(host)) return apiJSON(0, { t: r.audioFilename });
return apiJSON(2, { return apiJSON(2, {
type: type, type: type,
u: Array.isArray(r.urls) ? r.urls[1] : r.urls, service: host, ip: ip, u: Array.isArray(r.urls) ? r.urls[1] : r.urls, service: host, ip: ip,