diff --git a/.gitignore b/.gitignore index 01d4a695..97df23a8 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,4 @@ package-lock.json .env # esbuild -min \ No newline at end of file +min diff --git a/package.json b/package.json index f3387dcf..581536df 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "4.9-dev", + "version": "5.0-dev1", "author": "wukko", "exports": "./src/cobalt.js", "type": "module", @@ -10,7 +10,8 @@ }, "scripts": { "start": "node src/cobalt", - "setup": "node src/modules/setup" + "setup": "node src/modules/setup", + "test": "node src/test/test" }, "repository": { "type": "git", diff --git a/src/cobalt.js b/src/cobalt.js index 7512db78..e8173cca 100644 --- a/src/cobalt.js +++ b/src/cobalt.js @@ -1,11 +1,11 @@ -import "dotenv/config" +import "dotenv/config"; import express from "express"; import cors from "cors"; import * as fs from "fs"; import rateLimit from "express-rate-limit"; -import { shortCommit } from "./modules/sub/currentCommit.js"; +import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js"; import { appName, genericUserAgent, version } from "./modules/config.js"; import { getJSON } from "./modules/api.js"; import renderPage from "./modules/pageRender/page.js"; @@ -18,13 +18,14 @@ import { changelogHistory } from "./modules/pageRender/onDemand.js"; import { sha256 } from "./modules/sub/crypto.js"; const commitHash = shortCommit(); +const branch = getCurrentBranch(); const app = express(); app.disable('x-powered-by'); if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt && process.env.port) { const apiLimiter = rateLimit({ - windowMs: 1 * 60 * 1000, + windowMs: 60000, max: 12, standardHeaders: true, legacyHeaders: false, @@ -33,7 +34,7 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt && } }); const apiLimiterStream = rateLimit({ - windowMs: 1 * 60 * 1000, + windowMs: 60000, max: 12, standardHeaders: true, legacyHeaders: false, @@ -63,7 +64,6 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt && } next(); }); - app.use('/api/json', express.json({ verify: (req, res, buf) => { try { @@ -76,6 +76,7 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt && } } })); + app.post('/api/:type', cors({ origin: process.env.selfURL, optionsSuccessStatus: 200 }), async (req, res) => { try { let ip = sha256(req.header('x-forwarded-for') ? req.header('x-forwarded-for') : req.ip.replace('::ffff:', ''), process.env.streamSalt); @@ -89,10 +90,10 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt && let j = await getJSON(chck["url"], languageCode(req), chck) res.status(j.status).json(j.body); } else if (request.url && !chck) { - let j = apiJSON(3, { t: loc(languageCode(req), 'ErrorCouldntFetch') }); + let j = apiJSON(0, { t: loc(languageCode(req), 'ErrorCouldntFetch') }); res.status(j.status).json(j.body); } else { - let j = apiJSON(3, { t: loc(languageCode(req), 'ErrorNoLink') }) + let j = apiJSON(0, { t: loc(languageCode(req), 'ErrorNoLink') }) res.status(j.status).json(j.body); } } catch (e) { @@ -108,12 +109,16 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt && res.status(500).json({ 'status': 'error', 'text': loc(languageCode(req), 'ErrorCantProcess') }) } }); + app.get('/api/:type', cors({ origin: process.env.selfURL, optionsSuccessStatus: 200 }), (req, res) => { try { let ip = sha256(req.header('x-forwarded-for') ? req.header('x-forwarded-for') : req.ip.replace('::ffff:', ''), process.env.streamSalt); switch (req.params.type) { case 'json': - res.status(405).json({ 'status': 'error', 'text': 'GET method for this request has been deprecated. see https://github.com/wukko/cobalt/blob/current/docs/API.md for up-to-date API documentation.' }); + res.status(405).json({ + 'status': 'error', + 'text': 'GET method for this endpoint has been deprecated. see https://github.com/wukko/cobalt/blob/current/docs/API.md for up-to-date API documentation.' + }); break; case 'stream': if (req.query.p) { @@ -153,6 +158,7 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt && res.status(500).json({ 'status': 'error', 'text': loc(languageCode(req), 'ErrorCantProcess') }) } }); + app.get("/api", (req, res) => { res.redirect('/api/json') }); @@ -161,7 +167,8 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt && "hash": commitHash, "type": "default", "lang": languageCode(req), - "useragent": req.header('user-agent') ? req.header('user-agent') : genericUserAgent + "useragent": req.header('user-agent') ? req.header('user-agent') : genericUserAgent, + "branch": branch })) }); app.get("/favicon.ico", (req, res) => { @@ -170,9 +177,10 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt && app.get("/*", (req, res) => { res.redirect('/') }); + app.listen(process.env.port, () => { let startTime = new Date(); - console.log(`\n${Cyan(appName)} ${Bright(`v.${version}-${commitHash}`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(new Date().getTime())})`)}\n\nURL: ${Cyan(`${process.env.selfURL}`)}\nPort: ${process.env.port}\n`) + console.log(`\n${Cyan(appName)} ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(new Date().getTime())})`)}\n\nURL: ${Cyan(`${process.env.selfURL}`)}\nPort: ${process.env.port}\n`) }); } else { console.log(Red(`cobalt hasn't been configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`)) diff --git a/src/front/cobalt.js b/src/front/cobalt.js index 0628d355..13333ae9 100644 --- a/src/front/cobalt.js +++ b/src/front/cobalt.js @@ -273,7 +273,7 @@ function toggle(toggl) { } function loadSettings() { try { - if (typeof(navigator.clipboard.readText) == "undefined") throw new Error(); + if (typeof(navigator.clipboard.readText) === undefined) throw new Error(); } catch (err) { eid("pasteFromClipboard").style.display = "none" } diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json index 3e70c30b..384c1695 100644 --- a/src/localization/languages/en.json +++ b/src/localization/languages/en.json @@ -25,7 +25,7 @@ "ErrorBrokenLink": "{s} is supported, but something is wrong with your link. maybe you didn't copy it fully?", "ErrorNoLink": "i can't guess what you want to download! please give me a link.", "ErrorPageRenderFail": "something went wrong and page couldn't render. if it's a recurring or critical issue, please {ContactLink}. it'd be useful if you provided current commit hash ({s}) and error recreation steps. thank you in advance :D", - "ErrorRateLimit": "you're making too many requests. calm down and try again in a bit.", + "ErrorRateLimit": "you're making too many requests. try again in a minute!", "ErrorCouldntFetch": "couldn't get any info about your link. check if it's correct and try again.", "ErrorLengthLimit": "current length limit is {s} minutes. video that you tried to download is longer than {s} minutes. pick something else!", "ErrorBadFetch": "an error occurred when i tried to get info about your link. are you sure it works? check if it does, and try again.", diff --git a/src/localization/languages/ru.json b/src/localization/languages/ru.json index e09b016e..2a312065 100644 --- a/src/localization/languages/ru.json +++ b/src/localization/languages/ru.json @@ -7,7 +7,7 @@ "LinkInput": "вставь ссылку сюда", "AboutSummary": "{appName} — твой друг при скачивании контента из соц. сетей. никакой рекламы или трекеров. вставляешь ссылку и получаешь файл. ничего лишнего.", "EmbedBriefDescription": "сохраняй что хочешь, без мороки и вторжения в личное пространство", - "MadeWithLove": "сделано с <3 wukko", + "MadeWithLove": "сделано wukko, с <3", "AccessibilityInputArea": "зона вставки ссылки", "AccessibilityOpenAbout": "открыть окно с инфой", "AccessibilityDownloadButton": "кнопка скачивания", @@ -23,9 +23,9 @@ "ErrorSomethingWentWrong": "что-то пошло совсем не так, и у меня не получилось ничего для тебя достать. ты можешь попробовать ещё раз, но если так и не получится, {ContactLink}.", "ErrorUnsupported": "с твоей ссылкой что-то не так, или же этот сервис ещё не поддерживается. может быть, ты вставил не ту ссылку?", "ErrorBrokenLink": "{s} поддерживается, но с твоей ссылкой что-то не так. может быть, ты её не полностью скопировал?", - "ErrorNoLink": "я не гадалка и не могу угадывать, что ты хочешь скачать. попробуй в следующий раз вставить ссылку.", + "ErrorNoLink": "пока что я не умею угадывать, что ты хочешь скачать. дай мне, пожалуйста, ссылку :(", "ErrorPageRenderFail": "что-то пошло не так и у меня не получилось срендерить страницу. если это повторится ещё раз, пожалуйста, {ContactLink}. также приложи хэш текущего коммита ({s}) с действиями для повторения этой ошибки. можно на русском языке. спасибо :)", - "ErrorRateLimit": "ты делаешь слишком много запросов. успокойся и попробуй ещё раз через несколько минут.", + "ErrorRateLimit": "ты делаешь слишком много запросов. попробуй ещё раз через минуту!", "ErrorCouldntFetch": "мне не удалось получить инфу о твоей ссылке. проверь её и попробуй ещё раз.", "ErrorLengthLimit": "твоё видео длиннее чем {s} минут(ы). это превышает текущий лимит. скачай что-нибудь покороче, а не экранизацию \"войны и мира\".", "ErrorBadFetch": "произошла ошибка при получении инфы о твоей ссылке. ты уверен, что она работает? проверь её, и попробуй ещё раз.", diff --git a/src/modules/api.js b/src/modules/api.js index 9a6db6c8..1e0d9a92 100644 --- a/src/modules/api.js +++ b/src/modules/api.js @@ -31,7 +31,7 @@ export async function getJSON(originalURL, lang, obj) { break; case "tumblr": if (!url.includes("blog/view")) { - if (url.slice(-1) == '/') url = url.slice(0, -1); + if (url.slice(-1) === '/') url = url.slice(0, -1); url = url.replace(url.split('/')[5], ''); } break; diff --git a/src/modules/pageRender/elements.js b/src/modules/pageRender/elements.js index 909927ac..f6300f23 100644 --- a/src/modules/pageRender/elements.js +++ b/src/modules/pageRender/elements.js @@ -2,7 +2,7 @@ import { celebrations } from "../config.js"; export function switcher(obj) { let items = ``; - if (obj.name == "download") { + if (obj.name === "download") { items = obj.items; } else { for (let i = 0; i < obj.items.length; i++) { diff --git a/src/modules/pageRender/page.js b/src/modules/pageRender/page.js index 87d4a0ad..0c90a756 100644 --- a/src/modules/pageRender/page.js +++ b/src/modules/pageRender/page.js @@ -186,7 +186,7 @@ export default function(obj) { closeAria: t('AccessibilityClosePopup'), header: { aboveTitle: { - text: `v.${version}-${obj.hash}`, + text: `v.${version}-${obj.hash} (${obj.branch})`, url: `${repo}/commit/${obj.hash}` }, title: `${emoji("⚙️", 30)} ${t('TitlePopupSettings')}` diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index ff150b83..654a9f5a 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -5,23 +5,24 @@ import loc from "../../localization/manager.js"; import { testers } from "./servicesPatternTesters.js"; -import bilibili from "../services/bilibili.js"; -import reddit from "../services/reddit.js"; -import twitter from "../services/twitter.js"; -import youtube from "../services/youtube.js"; -import vk from "../services/vk.js"; -import tiktok from "../services/tiktok.js"; -import tumblr from "../services/tumblr.js"; +import bilibili from "./services/bilibili.js"; +import reddit from "./services/reddit.js"; +import twitter from "./services/twitter.js"; +import youtube from "./services/youtube.js"; +import vk from "./services/vk.js"; +import tiktok from "./services/tiktok.js"; +import tumblr from "./services/tumblr.js"; import matchActionDecider from "./matchActionDecider.js"; -import vimeo from "../services/vimeo.js"; -import soundcloud from "../services/soundcloud.js"; +import vimeo from "./services/vimeo.js"; +import soundcloud from "./services/soundcloud.js"; export default async function (host, patternMatch, url, lang, obj) { try { - if (!testers[host]) return apiJSON(0, { t: errorUnsupported(lang) }); - if (!(testers[host](patternMatch))) throw Error(); + let r, isAudioOnly = !!obj.isAudioOnly; + + if (!testers[host]) return apiJSON(0, { t: errorUnsupported(lang) }); + if (!(testers[host](patternMatch))) return apiJSON(0, { t: brokenLink(lang) }); - let r; switch (host) { case "twitter": r = await twitter({ @@ -29,14 +30,14 @@ export default async function (host, patternMatch, url, lang, obj) { spaceId: patternMatch["spaceId"] ? patternMatch["spaceId"] : false, lang: lang }); - if (r.isAudioOnly) obj.isAudioOnly = true break; case "vk": r = await vk({ url: url, userId: patternMatch["userId"], videoId: patternMatch["videoId"], - lang: lang, quality: obj.vQuality + lang: lang, + quality: obj.vQuality }); break; case "bilibili": @@ -48,10 +49,11 @@ export default async function (host, patternMatch, url, lang, obj) { case "youtube": let fetchInfo = { id: patternMatch["id"].slice(0, 11), - lang: lang, quality: obj.vQuality, + lang: lang, + quality: obj.vQuality, format: "webm" }; - if (url.match('music.youtube.com') || obj.isAudioOnly == true) obj.vFormat = "audio"; + if (url.match('music.youtube.com') || isAudioOnly === true) obj.vFormat = "audio"; switch (obj.vFormat) { case "mp4": fetchInfo["format"] = "mp4"; @@ -60,7 +62,7 @@ export default async function (host, patternMatch, url, lang, obj) { fetchInfo["format"] = "webm"; fetchInfo["isAudioOnly"] = true; fetchInfo["quality"] = "max"; - obj.isAudioOnly = true; + isAudioOnly = true; break; } r = await youtube(fetchInfo); @@ -69,7 +71,8 @@ export default async function (host, patternMatch, url, lang, obj) { r = await reddit({ sub: patternMatch["sub"], id: patternMatch["id"], - title: patternMatch["title"], lang: lang, + title: patternMatch["title"], + lang: lang, }); break; case "douyin": @@ -77,28 +80,33 @@ export default async function (host, patternMatch, url, lang, obj) { r = await tiktok({ host: host, postId: patternMatch["postId"], - id: patternMatch["id"], lang: lang, - noWatermark: obj.isNoTTWatermark, fullAudio: obj.isTTFullAudio, - isAudioOnly: obj.isAudioOnly + id: patternMatch["id"], + lang: lang, + noWatermark: obj.isNoTTWatermark, + fullAudio: obj.isTTFullAudio, + isAudioOnly: isAudioOnly }); - if (r.isAudioOnly) obj.isAudioOnly = true; break; case "tumblr": r = await tumblr({ - id: patternMatch["id"], url: url, user: patternMatch["user"] ? patternMatch["user"] : false, + id: patternMatch["id"], + url: url, + user: patternMatch["user"] ? patternMatch["user"] : false, lang: lang }); break; case "vimeo": r = await vimeo({ - id: patternMatch["id"].slice(0, 11), quality: obj.vQuality, + id: patternMatch["id"].slice(0, 11), + quality: obj.vQuality, lang: lang }); break; case "soundcloud": - obj.isAudioOnly = true; + isAudioOnly = true; r = await soundcloud({ - author: patternMatch["author"], song: patternMatch["song"], url: url, + author: patternMatch["author"], + song: patternMatch["song"], url: url, shortLink: patternMatch["shortLink"] ? patternMatch["shortLink"] : false, accessKey: patternMatch["accessKey"] ? patternMatch["accessKey"] : false, format: obj.aFormat, @@ -108,10 +116,15 @@ export default async function (host, patternMatch, url, lang, obj) { default: return apiJSON(0, { t: errorUnsupported(lang) }); } - return !r.error ? matchActionDecider(r, host, obj.ip, obj.aFormat, obj.isAudioOnly, lang, obj.isAudioMuted) : apiJSON(0, { - t: Array.isArray(r.error) ? loc(lang, r.error[0], r.error[1]) : loc(lang, r.error) - }); + + if (r.isAudioOnly) isAudioOnly = true; + let isAudioMuted = isAudioOnly ? false : obj.isAudioMuted; + + if (r.error) return apiJSON(0, { t: Array.isArray(r.error) ? loc(lang, r.error[0], r.error[1]) : loc(lang, r.error) }); + + return matchActionDecider(r, host, obj.ip, obj.aFormat, isAudioOnly, lang, isAudioMuted); } catch (e) { + console.log(e) return apiJSON(0, { t: genericError(lang, host) }) } } diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index a8070cde..7e76d602 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -1,121 +1,137 @@ -import { audioIgnore, services, supportedAudio } from "../config.js" -import { apiJSON } from "../sub/utils.js" +import { audioIgnore, services, supportedAudio } from "../config.js"; +import { apiJSON } from "../sub/utils.js"; import loc from "../../localization/manager.js"; export default function(r, host, ip, audioFormat, isAudioOnly, lang, isAudioMuted) { - if (!isAudioOnly && !r.picker && !isAudioMuted) { - switch (host) { - case "twitter": - return apiJSON(1, { u: r.urls }); - case "vk": - return apiJSON(2, { - type: "bridge", u: r.urls, service: host, ip: ip, - filename: r.filename, - }); - case "bilibili": - return apiJSON(2, { - type: "render", u: r.urls, service: host, ip: ip, - filename: r.filename, - time: r.time - }); - case "youtube": - return apiJSON(2, { - type: r.type, u: r.urls, service: host, ip: ip, - filename: r.filename, - time: r.time, - }); - case "reddit": - return apiJSON(r.typeId, { - type: r.type, u: r.urls, service: host, ip: ip, - filename: r.filename, - }); - case "tiktok": - return apiJSON(2, { - type: "bridge", u: r.urls, service: host, ip: ip, - filename: r.filename, - }); - case "douyin": - return apiJSON(2, { - type: "bridge", u: r.urls, service: host, ip: ip, - filename: r.filename, - }); - case "tumblr": - return apiJSON(1, { u: r.urls }); - case "vimeo": - if (Array.isArray(r.urls)) { - return apiJSON(2, { - type: "render", u: r.urls, service: host, ip: ip, - filename: r.filename - }); - } else { - return apiJSON(1, { u: r.urls }); - } - } - } else if (isAudioMuted && !isAudioOnly) { - let isSplit = Array.isArray(r.urls); - return apiJSON(2, { - type: isSplit ? "bridge" : "mute", - u: isSplit ? r.urls[0] : r.urls, + let action, + responseType = 2, + defaultParams = { + u: r.urls, service: host, ip: ip, filename: r.filename, - mute: true, - }); - } else if (r.picker) { - switch (host) { - case "douyin": - case "tiktok": - let type = "render"; - if (audioFormat === "mp3" || audioFormat === "best") { - audioFormat = "mp3" - type = "bridge" - } - return apiJSON(5, { - type: type, - picker: r.picker, - u: Array.isArray(r.urls) ? r.urls[1] : r.urls, service: host, ip: ip, - filename: r.audioFilename, isAudioOnly: true, audioFormat: audioFormat, copy: audioFormat === "best" ? true : false, - }) - case "twitter": - return apiJSON(5, { - picker: r.picker, service: host - }) - } - } else if (isAudioOnly) { - if ((host === "reddit" && r.typeId === 1) || (host === "vimeo" && !r.filename) || audioIgnore.includes(host)) return apiJSON(0, { t: loc(lang, 'ErrorEmptyDownload') }); - let type = "render"; - let copy = false; - - if (!supportedAudio.includes(audioFormat)) audioFormat = "best"; - if ((host == "tiktok" || host == "douyin") && services.tiktok.audioFormats.includes(audioFormat)) { - if (r.isMp3) { - if (audioFormat === "mp3" || audioFormat === "best") { - audioFormat = "mp3" - type = "bridge" - } - } else if (audioFormat === "best") { - audioFormat = "m4a" - type = "bridge" - } - } - if ((audioFormat === "best" && services[host]["bestAudio"]) || services[host]["bestAudio"] && (audioFormat === services[host]["bestAudio"])) { - audioFormat = services[host]["bestAudio"] - type = "bridge" - } else if (audioFormat === "best") { - audioFormat = "m4a" - copy = true - if (r.audioFilename.includes("twitterspaces")) { - audioFormat = "mp3" - copy = false - } - } - return apiJSON(2, { - type: type, - u: Array.isArray(r.urls) ? r.urls[1] : r.urls, service: host, ip: ip, - filename: r.audioFilename, isAudioOnly: true, - audioFormat: audioFormat, copy: copy, fileMetadata: r.fileMetadata ? r.fileMetadata : false - }) - } else { - return apiJSON(0, { t: loc(lang, 'ErrorSomethingWentWrong') }); + }, + params = {} + + if (isAudioMuted) action = "muteVideo"; + if (!isAudioOnly && !r.picker && !isAudioMuted) action = "video"; + if (r.picker) action = "picker"; + if (isAudioOnly) action = "audio"; + + if (action === "picker" || action === "audio") { + defaultParams.filename = r.audioFilename; + defaultParams.isAudioOnly = true; + defaultParams.audioFormat = audioFormat; } + + switch (action) { + case "video": + switch (host) { + case "bilibili": + params = { type: "render", time: r.time }; + break; + case "youtube": + params = { type: r.type, time: r.time }; + break; + case "reddit": + responseType = r.typeId; + params = { type: r.type }; + break; + case "vimeo": + if (Array.isArray(r.urls)) { + params = { type: "render" } + } else { + responseType = 1; + } + break; + + case "vk": + case "douyin": + case "tiktok": + params = { type: "bridge" }; + break; + + case "tumblr": + case "twitter": + responseType = 1; + break; + } + break; + + case "muteVideo": + params = { + type: Array.isArray(r.urls) ? "bridge" : "mute", + u: Array.isArray(r.urls) ? r.urls[0] : r.urls, + mute: true + } + break; + + case "picker": + responseType = 5; + switch (host) { + case "twitter": + params = { picker: r.picker }; + break; + case "douyin": + case "tiktok": + let pickerType = "render"; + if (audioFormat === "mp3" || audioFormat === "best") { + audioFormat = "mp3"; + pickerType = "bridge" + } + params = { + type: pickerType, + picker: r.picker, + u: Array.isArray(r.urls) ? r.urls[1] : r.urls, + copy: audioFormat === "best" ? true : false + } + } + break; + + case "audio": + if ((host === "reddit" && r.typeId === 1) || (host === "vimeo" && !r.filename) || audioIgnore.includes(host)) return apiJSON(0, { t: loc(lang, 'ErrorEmptyDownload') }); + + let processType = "render"; + let copy = false; + + if (!supportedAudio.includes(audioFormat)) audioFormat = "best"; + + if ((host === "tiktok" || host === "douyin") && services.tiktok.audioFormats.includes(audioFormat)) { + if (r.isMp3) { + if (audioFormat === "mp3" || audioFormat === "best") { + audioFormat = "mp3"; + processType = "bridge" + } + } else if (audioFormat === "best") { + audioFormat = "m4a"; + processType = "bridge" + } + } + + if ((audioFormat === "best" && services[host]["bestAudio"]) + || services[host]["bestAudio"] && (audioFormat === services[host]["bestAudio"])) { + audioFormat = services[host]["bestAudio"]; + processType = "bridge" + } else if (audioFormat === "best") { + audioFormat = "m4a"; + copy = true; + if (r.audioFilename.includes("twitterspaces")) { + audioFormat = "mp3" + copy = false + } + } + + params = { + type: processType, + u: Array.isArray(r.urls) ? r.urls[1] : r.urls, + audioFormat: audioFormat, + copy: copy, + fileMetadata: r.fileMetadata ? r.fileMetadata : false + } + break; + default: + return apiJSON(0, { t: loc(lang, 'ErrorEmptyDownload') }); + } + + return apiJSON(responseType, {...defaultParams, ...params}) } diff --git a/src/modules/processing/services/bilibili.js b/src/modules/processing/services/bilibili.js new file mode 100644 index 00000000..8f9ab286 --- /dev/null +++ b/src/modules/processing/services/bilibili.js @@ -0,0 +1,28 @@ +import { genericUserAgent, maxVideoDuration } from "../../config.js"; + +// TO-DO: quality picking & bilibili.tv support +export default async function(obj) { + let html = await fetch(`https://bilibili.com/video/${obj.id}`, { + headers: { "user-agent": genericUserAgent } + }).then((r) => { return r.text() }).catch(() => { return false }); + if (!html) return { error: 'ErrorCouldntFetch' }; + if (!(html.includes('')[0]); + if (streamData.data.timelength > maxVideoDuration) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + + let video = streamData["data"]["dash"]["video"].filter((v) => { + if (!v["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/")) return true; + }).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth)); + + let audio = streamData["data"]["dash"]["audio"].filter((a) => { + if (!a["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/")) return true; + }).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth)); + + return { + urls: [video[0]["baseUrl"], audio[0]["baseUrl"]], + time: streamData.data.timelength, + audioFilename: `bilibili_${obj.id}_audio`, + filename: `bilibili_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.mp4` + }; +} diff --git a/src/modules/processing/services/reddit.js b/src/modules/processing/services/reddit.js new file mode 100644 index 00000000..4465ec04 --- /dev/null +++ b/src/modules/processing/services/reddit.js @@ -0,0 +1,28 @@ +import { maxVideoDuration } from "../../config.js"; + +export default async function(obj) { + let data = await fetch(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}/${obj.name}.json`).then((r) => { return r.json() }).catch(() => { return false }); + if (!data) return { error: 'ErrorCouldntFetch' }; + + data = data[0]["data"]["children"][0]["data"]; + + if (data.url.endsWith('.gif')) return { typeId: 1, urls: data.url }; + + if (!"reddit_video" in data["secure_media"]) return { error: 'ErrorEmptyDownload' }; + if (data["secure_media"]["reddit_video"]["duration"] * 1000 > maxVideoDuration) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + + let video = data["secure_media"]["reddit_video"]["fallback_url"].split('?')[0], + audio = video.match('.mp4') ? `${video.split('_')[0]}_audio.mp4` : `${data["secure_media"]["reddit_video"]["fallback_url"].split('DASH')[0]}audio`; + await fetch(audio, { method: "HEAD" }).then((r) => {if (r.status != 200) audio = ''}).catch(() => {audio = ''}); + + let id = data["secure_media"]["reddit_video"]["fallback_url"].split('/')[3]; + if (!audio.length > 0) return { typeId: 1, urls: video }; + + return { + typeId: 2, + type: "render", + urls: [video, audio], + audioFilename: `reddit_${id}_audio`, + filename: `reddit_${id}.mp4` + }; +} diff --git a/src/modules/processing/services/soundcloud.js b/src/modules/processing/services/soundcloud.js new file mode 100644 index 00000000..312f60ef --- /dev/null +++ b/src/modules/processing/services/soundcloud.js @@ -0,0 +1,74 @@ +import { maxAudioDuration } from "../../config.js"; + +let cachedID = {}; + +async function findClientID() { + try { + let sc = await fetch('https://soundcloud.com/').then((r) => { return r.text() }).catch(() => { return false }); + let scVersion = String(sc.match(/')[0]) + if (!json["media"]["transcodings"]) return { error: 'ErrorEmptyDownload' }; + + let clientId = await findClientID(); + if (!clientId) return { error: 'ErrorSoundCloudNoClientId' }; + + let fileUrlBase = json.media.transcodings[0]["url"].replace("/hls", "/progressive") + let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`; + if (!fileUrl.substring(0, 54) === "https://api-v2.soundcloud.com/media/soundcloud:tracks:") return { error: 'ErrorEmptyDownload' }; + + if (json.duration > maxAudioDuration) return { error: ['ErrorLengthAudioConvert', maxAudioDuration / 60000] }; + + let file = await fetch(fileUrl).then(async (r) => { return (await r.json()).url }).catch(() => { return false }); + if (!file) return { error: 'ErrorCouldntFetch' }; + + return { + urls: file, + audioFilename: `soundcloud_${json.id}`, + fileMetadata: { + title: json.title, + artist: json.user.username, + } + } +} diff --git a/src/modules/processing/services/tiktok.js b/src/modules/processing/services/tiktok.js new file mode 100644 index 00000000..4ac0d19c --- /dev/null +++ b/src/modules/processing/services/tiktok.js @@ -0,0 +1,112 @@ +import { genericUserAgent } from "../../config.js"; + +let userAgent = genericUserAgent.split(' Chrome/1')[0], + config = { + tiktok: { + short: "https://vt.tiktok.com/", + api: "https://api2.musical.ly/aweme/v1/feed/?aweme_id={postId}&version_code=262&app_name=musical_ly&channel=App&device_id=null&os_version=14.4.2&device_platform=iphone&device_type=iPhone9®ion=US&carrier_region=US", + }, + douyin: { + short: "https://v.douyin.com/", + api: "https://www.iesdouyin.com/web/api/v2/aweme/iteminfo/?item_ids={postId}", + } +} + +function selector(j, h, id) { + if (!j) return false; + let t; + switch (h) { + case "tiktok": + t = j["aweme_list"].filter((v) => { if (v["aweme_id"] === id) return true }); + break; + case "douyin": + t = j['item_list'].filter((v) => { if (v["aweme_id"] === id) return true }); + break; + } + if (!t.length > 0) return false; + return t[0]; +} + +export default async function(obj) { + if (!obj.postId) { + let html = await fetch(`${config[obj.host]["short"]}${obj.id}`, { + redirect: "manual", + headers: { "user-agent": userAgent } + }).then((r) => { return r.text() }).catch(() => { return false }); + if (!html) return { error: 'ErrorCouldntFetch' }; + + if (html.slice(0, 17) === ' { return r.json() }).catch(() => { return false }); + + detail = selector(detail, obj.host, obj.postId); + if (!detail) return { error: 'ErrorCouldntFetch' }; + + let video, videoFilename, audioFilename, isMp3, audio, images, filenameBase = `${obj.host}_${obj.postId}`; + if (obj.host === "tiktok") { + images = detail["image_post_info"] ? detail["image_post_info"]["images"] : false + } else { + images = detail["images"] ? detail["images"] : false + } + + if (!obj.isAudioOnly && !images) { + video = obj.host === "tiktok" ? detail["video"]["download_addr"]["url_list"][0] : detail['video']['play_addr']['url_list'][0]; + videoFilename = `${filenameBase}_video.mp4`; + if (obj.noWatermark) { + video = obj.host === "tiktok" ? detail["video"]["play_addr"]["url_list"][0] : detail["video"]["play_addr"]["url_list"][0].replace("playwm", "play"); + videoFilename = `${filenameBase}_video_nw.mp4` // nw - no watermark + } + } else { + let fallback = obj.host === "douyin" ? detail["video"]["play_addr"]["url_list"][0].replace("playwm", "play") : detail["video"]["play_addr"]["url_list"][0]; + audio = fallback; + audioFilename = `${filenameBase}_audio_fv`; // fv - from video + if (obj.fullAudio || fallback.includes("music")) { + audio = detail["music"]["play_url"]["url_list"][0] + audioFilename = `${filenameBase}_audio` + } + if (audio.slice(-4) === ".mp3") isMp3 = true; + } + + if (video) return { + urls: video, + filename: videoFilename + } + if (images && obj.isAudioOnly) { + return { + urls: audio, + audioFilename: audioFilename, + isAudioOnly: true, + isMp3: isMp3, + } + } + if (images) { + let imageLinks = []; + for (let i in images) { + let sel = obj.host === "tiktok" ? images[i]["display_image"]["url_list"] : images[i]["url_list"]; + sel = sel.filter((p) => { if (p.includes(".jpeg?")) return true; }) + imageLinks.push({url: sel[0]}) + } + return { + picker: imageLinks, + urls: audio, + audioFilename: audioFilename, + isAudioOnly: true, + isMp3: isMp3, + } + } + if (audio) return { + urls: audio, + audioFilename: audioFilename, + isAudioOnly: true, + isMp3: isMp3, + } +} diff --git a/src/modules/processing/services/tumblr.js b/src/modules/processing/services/tumblr.js new file mode 100644 index 00000000..45291f07 --- /dev/null +++ b/src/modules/processing/services/tumblr.js @@ -0,0 +1,14 @@ +import { genericUserAgent } from "../../config.js"; + +export default async function(obj) { + let html = await fetch(`https://${ + obj.user ? obj.user : obj.url.split('.')[0].replace('https://', '') + }.tumblr.com/post/${obj.id}`, { + headers: { "user-agent": genericUserAgent } + }).then((r) => { return r.text() }).catch(() => { return false }); + + if (!html) return { error: 'ErrorCouldntFetch' }; + if (!html.includes('property="og:video" content="https://va.media.tumblr.com/')) return { error: 'ErrorEmptyDownload' }; + + return { urls: `https://va.media.tumblr.com/${html.split('property="og:video" content="https://va.media.tumblr.com/')[1].split('"')[0]}`, audioFilename: `tumblr_${obj.id}_audio` } +} diff --git a/src/modules/processing/services/twitter.js b/src/modules/processing/services/twitter.js new file mode 100644 index 00000000..f46c63a4 --- /dev/null +++ b/src/modules/processing/services/twitter.js @@ -0,0 +1,101 @@ +import { genericUserAgent } from "../../config.js"; + +function bestQuality(arr) { + return arr.filter((v) => { if (v["content_type"] === "video/mp4") return true }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate))[0]["url"].split("?")[0] +} +const apiURL = "https://api.twitter.com/1.1" + +// TO-DO: move from 1.1 api to graphql +export default async function(obj) { + let _headers = { + "user-agent": genericUserAgent, + "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA", + // ^ no explicit content, but with multi media support + "host": "api.twitter.com" + }; + let req_act = await fetch(`${apiURL}/guest/activate.json`, { + method: "POST", + headers: _headers + }).then((r) => { return r.status === 200 ? r.json() : false }).catch(() => { return false }); + if (!req_act) return { error: 'ErrorCouldntFetch' }; + + _headers["x-guest-token"] = req_act["guest_token"]; + let showURL = `${apiURL}/statuses/show/${obj.id}.json?tweet_mode=extended&include_user_entities=0&trim_user=1&include_entities=0&cards_platform=Web-12&include_cards=1`; + + if (!obj.spaceId) { + let req_status = await fetch(showURL, { headers: _headers }).then((r) => { return r.status === 200 ? r.json() : false }).catch((e) => { return false }); + if (!req_status) { + _headers.authorization = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"; + // ^ explicit content, but no multi media support + delete _headers["x-guest-token"] + + req_act = await fetch(`${apiURL}/guest/activate.json`, { + method: "POST", + headers: _headers + }).then((r) => { return r.status === 200 ? r.json() : false}).catch(() => { return false }); + if (!req_act) return { error: 'ErrorCouldntFetch' }; + + _headers["x-guest-token"] = req_act["guest_token"]; + req_status = await fetch(showURL, { headers: _headers }).then((r) => { return r.status === 200 ? r.json() : false }).catch(() => { return false }); + } + if (!req_status) return { error: 'ErrorCouldntFetch' }; + if (!req_status["extended_entities"] || !req_status["extended_entities"]["media"]) return { error: 'ErrorNoVideosInTweet' }; + + let single, multiple = [], media = req_status["extended_entities"]["media"]; + media = media.filter((i) => { if (i["type"] === "video" || i["type"] === "animated_gif") return true }) + if (media.length > 1) { + for (let i in media) { multiple.push({type: "video", thumb: media[i]["media_url_https"], url: bestQuality(media[i]["video_info"]["variants"])}) } + } else if (media.length === 1) { + single = bestQuality(media[0]["video_info"]["variants"]) + } else { + return { error: 'ErrorNoVideosInTweet' } + } + + if (single) { + return { urls: single, filename: `twitter_${obj.id}.mp4`, audioFilename: `twitter_${obj.id}_audio` } + } else if (multiple) { + return { picker: multiple } + } else { + return { error: 'ErrorNoVideosInTweet' } + } + } else { + _headers["host"] = "twitter.com" + _headers["content-type"] = "application/json" + + let query = { + variables: {"id": obj.spaceId,"isMetatagsQuery":true,"withSuperFollowsUserFields":true,"withDownvotePerspective":false,"withReactionsMetadata":false,"withReactionsPerspective":false,"withSuperFollowsTweetFields":true,"withReplays":true}, + features: {"spaces_2022_h2_clipping":true,"spaces_2022_h2_spaces_communities":true,"verified_phone_label_enabled":false,"tweetypie_unmention_optimization_enabled":true,"responsive_web_uc_gql_enabled":true,"vibe_api_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":false,"interactive_text_enabled":true,"responsive_web_text_conversations_enabled":false,"responsive_web_enhance_cards_enabled":true} + } + query.variables = new URLSearchParams(JSON.stringify(query.variables)).toString().slice(0, -1); + query.features = new URLSearchParams(JSON.stringify(query.features)).toString().slice(0, -1); + query = `https://twitter.com/i/api/graphql/wJ5g4zf7v8qPHSQbaozYuw/AudioSpaceById?variables=${query.variables}&features=${query.features}` + + let AudioSpaceById = await fetch(query, { headers: _headers }).then((r) => {return r.status === 200 ? r.json() : false}).catch((e) => { return false }); + if (!AudioSpaceById) return { error: 'ErrorEmptyDownload' }; + + if (!AudioSpaceById.data.audioSpace.metadata) return { error: 'ErrorEmptyDownload' }; + if (!AudioSpaceById.data.audioSpace.metadata.is_space_available_for_replay === true) return { error: 'TwitterSpaceWasntRecorded' }; + + let streamStatus = await fetch( + `https://twitter.com/i/api/1.1/live_video_stream/status/${AudioSpaceById.data.audioSpace.metadata.media_key}`, { headers: _headers } + ).then((r) =>{ return r.status === 200 ? r.json() : false }).catch(() => { return false }); + if (!streamStatus) return { error: 'ErrorCouldntFetch' }; + + let participants = AudioSpaceById.data.audioSpace.participants.speakers; + let listOfParticipants = `Twitter Space speakers: `; + for (let i in participants) { listOfParticipants += `@${participants[i]["twitter_screen_name"]}, ` } + listOfParticipants = listOfParticipants.slice(0, -2); + + return { + urls: streamStatus.source.noRedirectPlaybackUrl, + audioFilename: `twitterspaces_${obj.spaceId}`, + isAudioOnly: true, + fileMetadata: { + title: AudioSpaceById.data.audioSpace.metadata.title, + artist: `Twitter Space by @${AudioSpaceById.data.audioSpace.metadata.creator_results.result.legacy.screen_name}`, + comment: listOfParticipants, + // cover: AudioSpaceById.data.audioSpace.metadata.creator_results.result.legacy.profile_image_url_https.replace("_normal", "") + } + } + } +} diff --git a/src/modules/processing/services/vimeo.js b/src/modules/processing/services/vimeo.js new file mode 100644 index 00000000..534ce05d --- /dev/null +++ b/src/modules/processing/services/vimeo.js @@ -0,0 +1,77 @@ +import { maxVideoDuration, quality, services } from "../../config.js"; + +export default async function(obj) { + let api = await fetch(`https://player.vimeo.com/video/${obj.id}/config`).then((r) => { return r.json() }).catch(() => { return false }); + if (!api) return { error: 'ErrorCouldntFetch' }; + + let downloadType = "dash"; + if (JSON.stringify(api).includes('"progressive":[{')) downloadType = "progressive"; + + switch(downloadType) { + case "progressive": + let all = api["request"]["files"]["progressive"].sort((a, b) => Number(b.width) - Number(a.width)); + let best = all[0]; + + try { + if (obj.quality != "max") { + let pref = parseInt(quality[obj.quality], 10) + for (let i in all) { + let currQuality = parseInt(all[i]["quality"].replace('p', ''), 10) + if (currQuality === pref) { + best = all[i]; + break + } + if (currQuality < pref) { + best = all[i-1]; + break + } + } + } + } catch (e) { + best = all[0] + } + + return { urls: best["url"], filename: `tumblr_${obj.id}.mp4` }; + case "dash": + if (api.video.duration > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + + let masterJSONURL = api["request"]["files"]["dash"]["cdns"]["akfire_interconnect_quic"]["url"]; + let masterJSON = await fetch(masterJSONURL).then((r) => { return r.json() }).catch(() => { return false }); + + if (!masterJSON) return { error: 'ErrorCouldntFetch' }; + if (!masterJSON.video) return { error: 'ErrorEmptyDownload' }; + + let type = "parcel"; + if (masterJSON.base_url === "../") type = "chop"; + + let masterJSON_Video = masterJSON.video.sort((a, b) => Number(b.width) - Number(a.width)); + let masterJSON_Audio = masterJSON.audio.sort((a, b) => Number(b.bitrate) - Number(a.bitrate)).filter((a)=> {if (a['mime_type'] === "audio/mp4") return true;}); + let bestVideo = masterJSON_Video[0], bestAudio = masterJSON_Audio[0]; + + switch (type) { + case "parcel": + if (obj.quality != "max") { + let pref = parseInt(quality[obj.quality], 10) + for (let i in masterJSON_Video) { + let currQuality = parseInt(services.vimeo.resolutionMatch[masterJSON_Video[i]["width"]], 10) + if (currQuality < pref) { + break; + } else if (String(currQuality) === String(pref)) { + bestVideo = masterJSON_Video[i] + } + } + } + + let baseUrl = masterJSONURL.split("/sep/")[0]; + let videoUrl = `${baseUrl}/parcel/video/${bestVideo.index_segment.split('?')[0]}`, + audioUrl = `${baseUrl}/parcel/audio/${bestAudio.index_segment.split('?')[0]}`; + + return { urls: [videoUrl, audioUrl], audioFilename: `vimeo_${obj.id}_audio`, filename: `vimeo_${obj.id}_${bestVideo["width"]}x${bestVideo["height"]}.mp4` } + case "chop": // TO-DO: support for chop stream type + default: + return { error: 'ErrorEmptyDownload' } + } + default: + return { error: 'ErrorEmptyDownload' } + } +} diff --git a/src/modules/processing/services/vk.js b/src/modules/processing/services/vk.js new file mode 100644 index 00000000..2ab7feb7 --- /dev/null +++ b/src/modules/processing/services/vk.js @@ -0,0 +1,45 @@ +import { xml2json } from "xml-js"; +import { genericUserAgent, maxVideoDuration, services } from "../../config.js"; +import selectQuality from "../../stream/selectQuality.js"; + +export default async function(obj) { + let html; + html = await fetch(`https://vk.com/video-${obj.userId}_${obj.videoId}`, { + headers: { "user-agent": genericUserAgent } + }).then((r) => { return r.text() }).catch(() => { return false }); + if (!html) return { error: 'ErrorCouldntFetch' }; + if (!html.includes(`{"lang":`)) return { error: 'ErrorEmptyDownload' }; + + let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]); + + if (!Number(js["mvData"]["is_active_live"]) === 0) return { error: 'ErrorLiveVideo' }; + if (js["mvData"]["duration"] > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + + let mpd = JSON.parse(xml2json(js["player"]["params"][0]["manifest"], { compact: true, spaces: 4 })); + let repr = mpd["MPD"]["Period"]["AdaptationSet"]["Representation"]; + if (!mpd["MPD"]["Period"]["AdaptationSet"]["Representation"]) repr = mpd["MPD"]["Period"]["AdaptationSet"][0]["Representation"]; + + let selectedQuality, + attr = repr[repr.length - 1]["_attributes"], + qualities = Object.keys(services.vk.quality_match); + for (let i in qualities) { + if (qualities[i] === attr["height"]) { + selectedQuality = `url${attr["height"]}`; + break + } + if (qualities[i] === attr["width"]) { + selectedQuality = `url${attr["width"]}`; + break + } + } + + let maxQuality = js["player"]["params"][0][selectedQuality].split('type=')[1].slice(0, 1); + let userQuality = selectQuality('vk', obj.quality, Object.entries(services.vk.quality_match).reduce((r, [k, v]) => { r[v] = k; return r; })[maxQuality]); + let userRepr = repr[services.vk.representation_match[userQuality]]["_attributes"]; + if (!selectedQuality in js["player"]["params"][0]) return { error: 'ErrorEmptyDownload' }; + + return { + urls: js["player"]["params"][0][`url${userQuality}`], + filename: `vk_${obj.userId}_${obj.videoId}_${userRepr["width"]}x${userRepr['height']}.mp4` + } +} diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js new file mode 100644 index 00000000..8ec87453 --- /dev/null +++ b/src/modules/processing/services/youtube.js @@ -0,0 +1,88 @@ +import ytdl from "better-ytdl-core"; +import { maxVideoDuration, quality as mq } from "../../config.js"; +import selectQuality from "../../stream/selectQuality.js"; + +export default async function(obj) { + let isAudioOnly = !!obj.isAudioOnly, + infoInitial = await ytdl.getInfo(obj.id); + if (!infoInitial) return { error: 'ErrorCantConnectToServiceAPI' }; + + let info = infoInitial.formats; + if (info[0]["isLive"]) return { error: 'ErrorLiveVideo' }; + + let videoMatch = [], fullVideoMatch = [], video = [], + audio = info.filter((a) => { + if (!a["isHLS"] && !a["isDashMPD"] && a["hasAudio"] && !a["hasVideo"] && a["container"] == obj.format) return true + }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate)); + + if (audio.length === 0) return { error: 'ErrorBadFetch' }; + if (audio[0]["approxDurationMs"] > maxVideoDuration) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + + if (!isAudioOnly) { + video = info.filter((a) => { + if (!a["isHLS"] && !a["isDashMPD"] && a["hasVideo"] && a["container"] == obj.format) { + if (obj.quality != "max") { + if (a["hasAudio"] && mq[obj.quality] == a["height"]) { + fullVideoMatch.push(a) + } else if (!a["hasAudio"] && mq[obj.quality] == a["height"]) { + videoMatch.push(a) + } + } + return true + } + }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate)); + + if (obj.quality != "max") { + if (videoMatch.length === 0) { + let ss = selectQuality("youtube", obj.quality, video[0]["qualityLabel"].slice(0, 5).replace('p', '').trim()); + videoMatch = video.filter((a) => { + if (a["qualityLabel"].slice(0, 5).replace('p', '').trim() == ss) return true + }) + } else if (fullVideoMatch.length > 0) { + videoMatch = [fullVideoMatch[0]] + } + } else videoMatch = [video[0]]; + if (obj.quality === "los") videoMatch = [video[video.length - 1]]; + } + if (video.length === 0) isAudioOnly = true; + + if (isAudioOnly) { + let r = { + type: "render", + isAudioOnly: true, + urls: audio[0]["url"], + audioFilename: `youtube_${obj.id}_audio`, + fileMetadata: { + title: infoInitial.videoDetails.title, + artist: infoInitial.videoDetails.ownerChannelName.replace("- Topic", "").trim(), + } + } + if (infoInitial.videoDetails.description) { + let isAutoGenAudio = infoInitial.videoDetails.description.startsWith("Provided to YouTube by"); + if (isAutoGenAudio) { + let descItems = infoInitial.videoDetails.description.split("\n\n") + r.fileMetadata.album = descItems[2] + r.fileMetadata.copyright = descItems[3] + if (descItems[4].startsWith("Released on:")) r.fileMetadata.date = descItems[4].replace("Released on: ", '').trim(); + } + } + return r + } + let singleTest; + if (videoMatch.length > 0) { + singleTest = videoMatch[0]["hasVideo"] && videoMatch[0]["hasAudio"]; + return { + type: singleTest ? "bridge" : "render", + urls: singleTest ? videoMatch[0]["url"] : [videoMatch[0]["url"], audio[0]["url"]], + time: videoMatch[0]["approxDurationMs"], + filename: `youtube_${obj.id}_${videoMatch[0]["width"]}x${videoMatch[0]["height"]}.${obj.format}` + } + } + singleTest = video[0]["hasVideo"] && video[0]["hasAudio"]; + return { + type: singleTest ? "bridge" : "render", + urls: singleTest ? video[0]["url"] : [video[0]["url"], audio[0]["url"]], + time: video[0]["approxDurationMs"], + filename: `youtube_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.${obj.format}` + } +} diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index fa1fd968..25c6f3f5 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -8,6 +8,7 @@ "enabled": true }, "reddit": { + "alias": "reddit videos & gifs", "patterns": ["r/:sub/comments/:id/:title"], "enabled": true }, diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index 56ca2b28..13695714 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -1,27 +1,28 @@ export const testers = { - "twitter": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length < 20) || (patternMatch["spaceId"] && patternMatch["spaceId"].length === 13), + "twitter": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length < 20) + || (patternMatch["spaceId"] && patternMatch["spaceId"].length === 13), - "vk": (patternMatch) => (patternMatch["userId"] && patternMatch["videoId"] && - patternMatch["userId"].length <= 10 && patternMatch["videoId"].length === 9), + "vk": (patternMatch) => (patternMatch["userId"] && patternMatch["videoId"] + && patternMatch["userId"].length <= 10 && patternMatch["videoId"].length === 9), "bilibili": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length >= 12), "youtube": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length >= 11), - "reddit": (patternMatch) => (patternMatch["sub"] && patternMatch["id"] && patternMatch["title"] && - patternMatch["sub"].length <= 22 && patternMatch["id"].length <= 10 && patternMatch["title"].length <= 96), + "reddit": (patternMatch) => (patternMatch["sub"] && patternMatch["id"] && patternMatch["title"] + && patternMatch["sub"].length <= 22 && patternMatch["id"].length <= 10 && patternMatch["title"].length <= 96), - "tiktok": (patternMatch) => ((patternMatch["user"] && patternMatch["postId"] && patternMatch["postId"].length <= 21) || - (patternMatch["id"] && patternMatch["id"].length <= 13)), + "tiktok": (patternMatch) => ((patternMatch["user"] && patternMatch["postId"] && patternMatch["postId"].length <= 21) + || (patternMatch["id"] && patternMatch["id"].length <= 13)), - "douyin": (patternMatch) => ((patternMatch["postId"] && patternMatch["postId"].length <= 21) || - (patternMatch["id"] && patternMatch["id"].length <= 13)), + "douyin": (patternMatch) => ((patternMatch["postId"] && patternMatch["postId"].length <= 21) + || (patternMatch["id"] && patternMatch["id"].length <= 13)), - "tumblr": (patternMatch) => ((patternMatch["id"] && patternMatch["id"].length < 21) || - (patternMatch["id"] && patternMatch["id"].length < 21 && patternMatch["user"] && patternMatch["user"].length <= 32)), + "tumblr": (patternMatch) => ((patternMatch["id"] && patternMatch["id"].length < 21) + || (patternMatch["id"] && patternMatch["id"].length < 21 && patternMatch["user"] && patternMatch["user"].length <= 32)), "vimeo": (patternMatch) => ((patternMatch["id"] && patternMatch["id"].length <= 11)), - "soundcloud": (patternMatch) => ((patternMatch["author"] && patternMatch["song"] && (patternMatch["author"].length + patternMatch["song"].length) <= 96) || - (patternMatch["shortLink"] && patternMatch["shortLink"].length <= 32)) -}; \ No newline at end of file + "soundcloud": (patternMatch) => ((patternMatch["author"] && patternMatch["song"] + && (patternMatch["author"].length + patternMatch["song"].length) <= 96) || (patternMatch["shortLink"] && patternMatch["shortLink"].length <= 32)) +} diff --git a/src/modules/services/bilibili.js b/src/modules/services/bilibili.js deleted file mode 100644 index 17d12f15..00000000 --- a/src/modules/services/bilibili.js +++ /dev/null @@ -1,38 +0,0 @@ -import { genericUserAgent, maxVideoDuration } from "../config.js"; - -// TO-DO: quality picking -export default async function(obj) { - try { - let html = await fetch(`https://bilibili.com/video/${obj.id}`, { - headers: {"user-agent": genericUserAgent} - }).then((r) => { return r.text() }).catch(() => { return false }); - if (!html) { - return { error: 'ErrorCouldntFetch' }; - } - - if (!(html.includes('')[0]); - if (streamData.data.timelength > maxVideoDuration) { - return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; - } - - let video = streamData["data"]["dash"]["video"].filter((v) => { - if (!v["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/")) return true; - }).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth)); - - let audio = streamData["data"]["dash"]["audio"].filter((a) => { - if (!a["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/")) return true; - }).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth)); - - return { - urls: [video[0]["baseUrl"], audio[0]["baseUrl"]], - time: streamData.data.timelength, - audioFilename: `bilibili_${obj.id}_audio`, - filename: `bilibili_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.mp4` - }; - } catch (e) { - return { error: 'ErrorBadFetch' }; - } -} diff --git a/src/modules/services/reddit.js b/src/modules/services/reddit.js deleted file mode 100644 index 3a8fe25f..00000000 --- a/src/modules/services/reddit.js +++ /dev/null @@ -1,40 +0,0 @@ -import { maxVideoDuration } from "../config.js"; - -// TO-DO: add support for gifs (#80) -export default async function(obj) { - try { - let data = await fetch(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}/${obj.name}.json`).then((r) => { return r.json() }).catch(() => { return false }); - if (!data) { - return { error: 'ErrorCouldntFetch' }; - } - data = data[0]["data"]["children"][0]["data"]; - - if (!"reddit_video" in data["secure_media"]) { - return { error: 'ErrorEmptyDownload' }; - } - if (data["secure_media"]["reddit_video"]["duration"] * 1000 > maxVideoDuration) { - return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; - } - let video = data["secure_media"]["reddit_video"]["fallback_url"].split('?')[0], - audio = video.match('.mp4') - ? `${video.split('_')[0]}_audio.mp4` - : `${data["secure_media"]["reddit_video"]["fallback_url"].split('DASH')[0]}audio`; - - await fetch(audio, {method: "HEAD"}).then((r) => {if (r.status != 200) audio = ''}).catch(() => {audio = ''}); - - let id = data["secure_media"]["reddit_video"]["fallback_url"].split('/')[3]; - - if (!audio.length > 0) { - return { typeId: 1, urls: video }; - } - return { - typeId: 2, - type: "render", - urls: [video, audio], - audioFilename: `reddit_${id}_audio`, - filename: `reddit_${id}.mp4` - }; - } catch (err) { - return { error: 'ErrorBadFetch' }; - } -} diff --git a/src/modules/services/soundcloud.js b/src/modules/services/soundcloud.js deleted file mode 100644 index e6026cb5..00000000 --- a/src/modules/services/soundcloud.js +++ /dev/null @@ -1,90 +0,0 @@ -import { genericUserAgent, maxAudioDuration } from "../config.js"; - -let cachedID = {} - -async function findClientID() { - try { - let sc = await fetch('https://soundcloud.com/').then((r) => { return r.text() }).catch(() => { return false }); - let scVersion = String(sc.match(/')[0]) - if (!json["media"]["transcodings"]) { - return { error: 'ErrorEmptyDownload' } - } - let clientId = await findClientID(); - if (!clientId) { - return { error: 'ErrorSoundCloudNoClientId' } - } - let fileUrlBase = json.media.transcodings[0]["url"].replace("/hls", "/progressive") - let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`; - if (!fileUrl.substring(0, 54) === "https://api-v2.soundcloud.com/media/soundcloud:tracks:") { - return { error: 'ErrorEmptyDownload' } - } - if (json.duration > maxAudioDuration) { - return { error: ['ErrorLengthAudioConvert', maxAudioDuration / 60000] } - } - let file = await fetch(fileUrl).then(async (r) => { return (await r.json()).url }).catch(() => { return false }); - if (!file) { - return { error: 'ErrorCouldntFetch' }; - } - return { - urls: file, - audioFilename: `soundcloud_${json.id}`, - fileMetadata: { - title: json.title, - artist: json.user.username, - } - } - } catch (e) { - return { error: 'ErrorBadFetch' }; - } -} diff --git a/src/modules/services/tiktok.js b/src/modules/services/tiktok.js deleted file mode 100644 index ab30311f..00000000 --- a/src/modules/services/tiktok.js +++ /dev/null @@ -1,117 +0,0 @@ -import { genericUserAgent } from "../config.js"; - -let userAgent = genericUserAgent.split(' Chrome/1')[0] -let config = { - tiktok: { - short: "https://vt.tiktok.com/", - api: "https://api2.musical.ly/aweme/v1/feed/?aweme_id={postId}&version_code=262&app_name=musical_ly&channel=App&device_id=null&os_version=14.4.2&device_platform=iphone&device_type=iPhone9®ion=US&carrier_region=US", - }, - douyin: { - short: "https://v.douyin.com/", - api: "https://www.iesdouyin.com/web/api/v2/aweme/iteminfo/?item_ids={postId}", - } -} -function selector(j, h, id) { - if (!j) return false - let t; - switch (h) { - case "tiktok": - t = j["aweme_list"].filter((v) => { if (v["aweme_id"] == id) return true }) - break; - case "douyin": - t = j['item_list'].filter((v) => { if (v["aweme_id"] == id) return true }) - break; - } - if (!t.length > 0) return false - return t[0] -} - -export default async function(obj) { - try { - if (!obj.postId) { - let html = await fetch(`${config[obj.host]["short"]}${obj.id}`, { - redirect: "manual", - headers: { "user-agent": userAgent } - }).then((r) => { return r.text() }).catch(() => { return false }); - if (!html) return { error: 'ErrorCouldntFetch' }; - - if (html.slice(0, 17) === ' { return r.json() }).catch(() => { return false }); - - detail = selector(detail, obj.host, obj.postId); - - if (!detail) return { error: 'ErrorCouldntFetch' } - - let video, videoFilename, audioFilename, isMp3, audio, images, - filenameBase = `${obj.host}_${obj.postId}`; - if (obj.host == "tiktok") { - images = detail["image_post_info"] ? detail["image_post_info"]["images"] : false - } else { - images = detail["images"] ? detail["images"] : false - } - if (!obj.isAudioOnly && !images) { - video = obj.host === "tiktok" ? detail["video"]["download_addr"]["url_list"][0] : detail['video']['play_addr']['url_list'][0] - videoFilename = `${filenameBase}_video.mp4` - if (obj.noWatermark) { - video = obj.host === "tiktok" ? detail["video"]["play_addr"]["url_list"][0] : detail["video"]["play_addr"]["url_list"][0].replace("playwm", "play"); - videoFilename = `${filenameBase}_video_nw.mp4` // nw - no watermark - } - } else { - let fallback = obj.host === "douyin" ? detail["video"]["play_addr"]["url_list"][0].replace("playwm", "play") : detail["video"]["play_addr"]["url_list"][0]; - audio = fallback; - audioFilename = `${filenameBase}_audio_fv`; // fv - from video - if (obj.fullAudio || fallback.includes("music")) { - audio = detail["music"]["play_url"]["url_list"][0] - audioFilename = `${filenameBase}_audio` - } - if (audio.slice(-4) === ".mp3") isMp3 = true; - } - if (video) return { - urls: video, - filename: videoFilename - } - if (images && obj.isAudioOnly) { - return { - urls: audio, - audioFilename: audioFilename, - isAudioOnly: true, - isMp3: isMp3, - } - } - if (images) { - let imageLinks = []; - for (let i in images) { - let sel = obj.host == "tiktok" ? images[i]["display_image"]["url_list"] : images[i]["url_list"]; - sel = sel.filter((p) => { if (p.includes(".jpeg?")) return true; }) - imageLinks.push({url: sel[0]}) - } - return { - picker: imageLinks, - urls: audio, - audioFilename: audioFilename, - isAudioOnly: true, - isMp3: isMp3, - } - } - if (audio) return { - urls: audio, - audioFilename: audioFilename, - isAudioOnly: true, - isMp3: isMp3, - } - } catch (e) { - return { error: 'ErrorBadFetch' }; - } -} diff --git a/src/modules/services/tumblr.js b/src/modules/services/tumblr.js deleted file mode 100644 index 372388ac..00000000 --- a/src/modules/services/tumblr.js +++ /dev/null @@ -1,17 +0,0 @@ -import { genericUserAgent } from "../config.js"; - -export default async function(obj) { - try { - let user = obj.user ? obj.user : obj.url.split('.')[0].replace('https://', ''); - let html = await fetch(`https://${user}.tumblr.com/post/${obj.id}`, { - headers: {"user-agent": genericUserAgent} - }).then((r) => { return r.text() }).catch(() => { return false }); - if (!html) return { error: 'ErrorCouldntFetch' }; - if (!html.includes('property="og:video" content="https://va.media.tumblr.com/')) { - return { error: 'ErrorEmptyDownload' } - } - return { urls: `https://va.media.tumblr.com/${html.split('property="og:video" content="https://va.media.tumblr.com/')[1].split('"')[0]}`, audioFilename: `tumblr_${obj.id}_audio` } - } catch (e) { - return { error: 'ErrorBadFetch' }; - } -} diff --git a/src/modules/services/twitter.js b/src/modules/services/twitter.js deleted file mode 100644 index 32e7918a..00000000 --- a/src/modules/services/twitter.js +++ /dev/null @@ -1,100 +0,0 @@ -import { genericUserAgent } from "../config.js"; - -function bestQuality(arr) { - return arr.filter((v) => { if (v["content_type"] === "video/mp4") return true; }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate))[0]["url"].split("?")[0] -} -const apiURL = "https://api.twitter.com/1.1" - -export default async function(obj) { - try { - let _headers = { - "user-agent": genericUserAgent, - "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA", - "host": "api.twitter.com" - }; - let req_act = await fetch(`${apiURL}/guest/activate.json`, { - method: "POST", - headers: _headers - }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); - - if (!req_act) return { error: 'ErrorCouldntFetch' }; - _headers["x-guest-token"] = req_act["guest_token"]; - let showURL = `${apiURL}/statuses/show/${obj.id}.json?tweet_mode=extended&include_user_entities=0&trim_user=1&include_entities=0&cards_platform=Web-12&include_cards=1` - if (!obj.spaceId) { - let req_status = await fetch(showURL, { headers: _headers }).then((r) => { return r.status == 200 ? r.json() : false;}).catch((e) => { return false}); - if (!req_status) { - _headers.authorization = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"; - delete _headers["x-guest-token"] - - req_act = await fetch(`${apiURL}/guest/activate.json`, { - method: "POST", - headers: _headers - }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); - if (!req_act) return { error: 'ErrorCouldntFetch' }; - - _headers["x-guest-token"] = req_act["guest_token"]; - req_status = await fetch(showURL, { headers: _headers }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); - } - if (!req_status) return { error: 'ErrorCouldntFetch' } - if (!req_status["extended_entities"] && req_status["extended_entities"]["media"]) { - return { error: 'ErrorNoVideosInTweet' } - } - let single, multiple = [], media = req_status["extended_entities"]["media"]; - media = media.filter((i) => { if (i["type"] === "video" || i["type"] === "animated_gif") return true }) - if (media.length > 1) { - for (let i in media) { multiple.push({type: "video", thumb: media[i]["media_url_https"], url: bestQuality(media[i]["video_info"]["variants"])}) } - } else if (media.length === 1) { - single = bestQuality(media[0]["video_info"]["variants"]) - } else { - return { error: 'ErrorNoVideosInTweet' } - } - if (single) { - return { urls: single, filename: `twitter_${obj.id}.mp4`, audioFilename: `twitter_${obj.id}_audio` } - } else if (multiple) { - return { picker: multiple } - } else { - return { error: 'ErrorNoVideosInTweet' } - } - } else { - _headers["host"] = "twitter.com" - _headers["content-type"] = "application/json" - let query = { - variables: {"id": obj.spaceId,"isMetatagsQuery":true,"withSuperFollowsUserFields":true,"withDownvotePerspective":false,"withReactionsMetadata":false,"withReactionsPerspective":false,"withSuperFollowsTweetFields":true,"withReplays":true}, features: {"spaces_2022_h2_clipping":true,"spaces_2022_h2_spaces_communities":true,"verified_phone_label_enabled":false,"tweetypie_unmention_optimization_enabled":true,"responsive_web_uc_gql_enabled":true,"vibe_api_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":false,"interactive_text_enabled":true,"responsive_web_text_conversations_enabled":false,"responsive_web_enhance_cards_enabled":true} - } - - let AudioSpaceById = await fetch(`https://twitter.com/i/api/graphql/wJ5g4zf7v8qPHSQbaozYuw/AudioSpaceById?variables=${new URLSearchParams(JSON.stringify(query.variables)).toString().slice(0, -1)}&features=${new URLSearchParams(JSON.stringify(query.features)).toString().slice(0, -1)}`, { headers: _headers }).then((r) => { - return r.status == 200 ? r.json() : false; - }).catch((e) => {return false}); - - if (!AudioSpaceById) { - return { error: 'ErrorEmptyDownload' } - } - if (!AudioSpaceById.data.audioSpace.metadata.is_space_available_for_replay === true) { - return { error: 'TwitterSpaceWasntRecorded' }; - } - let streamStatus = await fetch(`https://twitter.com/i/api/1.1/live_video_stream/status/${AudioSpaceById.data.audioSpace.metadata.media_key}`, - { headers: _headers }).then((r) =>{return r.status == 200 ? r.json() : false;}).catch(() => {return false;}); - if (!streamStatus) return { error: 'ErrorCouldntFetch' }; - - let participants = AudioSpaceById.data.audioSpace.participants.speakers - let listOfParticipants = `Twitter Space speakers: ` - for (let i in participants) { - listOfParticipants += `@${participants[i]["twitter_screen_name"]}, ` - } - listOfParticipants = listOfParticipants.slice(0, -2); - return { - urls: streamStatus.source.noRedirectPlaybackUrl, - audioFilename: `twitterspaces_${obj.spaceId}`, - isAudioOnly: true, - fileMetadata: { - title: AudioSpaceById.data.audioSpace.metadata.title, - artist: `Twitter Space by @${AudioSpaceById.data.audioSpace.metadata.creator_results.result.legacy.screen_name}`, - comment: listOfParticipants, - // cover: AudioSpaceById.data.audioSpace.metadata.creator_results.result.legacy.profile_image_url_https.replace("_normal", "") - } - } - } - } catch (err) { - return { error: 'ErrorBadFetch' }; - } -} diff --git a/src/modules/services/vimeo.js b/src/modules/services/vimeo.js deleted file mode 100644 index fe1af72f..00000000 --- a/src/modules/services/vimeo.js +++ /dev/null @@ -1,84 +0,0 @@ -import { maxVideoDuration, quality, services } from "../config.js"; - -export default async function(obj) { - try { - let api = await fetch(`https://player.vimeo.com/video/${obj.id}/config`).then((r) => {return r.json()}).catch(() => {return false}); - if (!api) return { error: 'ErrorCouldntFetch' }; - - let downloadType = "dash"; - if (JSON.stringify(api).includes('"progressive":[{')) { - downloadType = "progressive"; - } - - switch(downloadType) { - case "progressive": - let all = api["request"]["files"]["progressive"].sort((a, b) => Number(b.width) - Number(a.width)); - let best = all[0] - try { - if (obj.quality != "max") { - let pref = parseInt(quality[obj.quality], 10) - for (let i in all) { - let currQuality = parseInt(all[i]["quality"].replace('p', ''), 10) - if (currQuality === pref) { - best = all[i]; - break - } - if (currQuality < pref) { - best = all[i-1]; - break - } - } - } - } catch (e) { - best = all[0] - } - return { urls: best["url"], filename: `tumblr_${obj.id}.mp4` }; - case "dash": - if (api.video.duration > maxVideoDuration / 1000) { - return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; - } - let masterJSONURL = api["request"]["files"]["dash"]["cdns"]["akfire_interconnect_quic"]["url"]; - let masterJSON = await fetch(masterJSONURL).then((r) => {return r.json()}).catch(() => {return false}); - - if (!masterJSON) return { error: 'ErrorCouldntFetch' }; - if (!masterJSON.video) { - return { error: 'ErrorEmptyDownload' } - } - let type = "parcel"; - if (masterJSON.base_url == "../") { - type = "chop" - } - let masterJSON_Video = masterJSON.video.sort((a, b) => Number(b.width) - Number(a.width)); - let masterJSON_Audio = masterJSON.audio.sort((a, b) => Number(b.bitrate) - Number(a.bitrate)).filter((a)=> {if (a['mime_type'] === "audio/mp4") return true;}); - - let bestVideo = masterJSON_Video[0] - let bestAudio = masterJSON_Audio[0] - switch (type) { - case "parcel": - if (obj.quality != "max") { - let pref = parseInt(quality[obj.quality], 10) - for (let i in masterJSON_Video) { - let currQuality = parseInt(services.vimeo.resolutionMatch[masterJSON_Video[i]["width"]], 10) - if (currQuality < pref) { - break; - } else if (currQuality == pref) { - bestVideo = masterJSON_Video[i] - } - } - } - let baseUrl = masterJSONURL.split("/sep/")[0] - let videoUrl = `${baseUrl}/parcel/video/${bestVideo.index_segment.split('?')[0]}`; - let audioUrl = `${baseUrl}/parcel/audio/${bestAudio.index_segment.split('?')[0]}`; - - return { urls: [videoUrl, audioUrl], audioFilename: `vimeo_${obj.id}_audio`, filename: `vimeo_${obj.id}_${bestVideo["width"]}x${bestVideo["height"]}.mp4` } - case "chop": // TO-DO: support chop type of streams - default: - return { error: 'ErrorEmptyDownload' } - } - default: - return { error: 'ErrorEmptyDownload' } - } - } catch (e) { - return { error: 'ErrorBadFetch' }; - } -} diff --git a/src/modules/services/vk.js b/src/modules/services/vk.js deleted file mode 100644 index 0c4872be..00000000 --- a/src/modules/services/vk.js +++ /dev/null @@ -1,54 +0,0 @@ -import { xml2json } from "xml-js"; -import { genericUserAgent, maxVideoDuration, services } from "../config.js"; -import selectQuality from "../stream/selectQuality.js"; - -export default async function(obj) { - try { - let html; - html = await fetch(`https://vk.com/video-${obj.userId}_${obj.videoId}`, { - headers: {"user-agent": genericUserAgent} - }).then((r) => {return r.text()}).catch(() => {return false}); - if (!html) return { error: 'ErrorCouldntFetch' }; - if (!html.includes(`{"lang":`)) { - return { error: 'ErrorEmptyDownload' }; - } - let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]); - if (!js["mvData"]["is_active_live"] == '0') { - return { error: 'ErrorLiveVideo' }; - } - if (js["mvData"]["duration"] > maxVideoDuration / 1000) { - return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; - } - let mpd = JSON.parse(xml2json(js["player"]["params"][0]["manifest"], { compact: true, spaces: 4 })); - - let repr = mpd["MPD"]["Period"]["AdaptationSet"]["Representation"]; - if (!mpd["MPD"]["Period"]["AdaptationSet"]["Representation"]) { - repr = mpd["MPD"]["Period"]["AdaptationSet"][0]["Representation"]; - } - let attr = repr[repr.length - 1]["_attributes"]; - let selectedQuality; - let qualities = Object.keys(services.vk.quality_match); - for (let i in qualities) { - if (qualities[i] == attr["height"]) { - selectedQuality = `url${attr["height"]}`; - break; - } - if (qualities[i] == attr["width"]) { - selectedQuality = `url${attr["width"]}`; - break; - } - } - let maxQuality = js["player"]["params"][0][selectedQuality].split('type=')[1].slice(0, 1) - let userQuality = selectQuality('vk', obj.quality, Object.entries(services.vk.quality_match).reduce((r, [k, v]) => { r[v] = k; return r; })[maxQuality]); - let userRepr = repr[services.vk.representation_match[userQuality]]["_attributes"]; - if (!selectedQuality in js["player"]["params"][0]) { - return { error: 'ErrorEmptyDownload' }; - } - return { - urls: js["player"]["params"][0][`url${userQuality}`], - filename: `vk_${obj.userId}_${obj.videoId}_${userRepr["width"]}x${userRepr['height']}.mp4` - }; - } catch (err) { - return { error: 'ErrorBadFetch' }; - } -} diff --git a/src/modules/services/youtube.js b/src/modules/services/youtube.js deleted file mode 100644 index dfba7583..00000000 --- a/src/modules/services/youtube.js +++ /dev/null @@ -1,93 +0,0 @@ -import ytdl from "better-ytdl-core"; -import { maxVideoDuration, quality as mq } from "../config.js"; -import selectQuality from "../stream/selectQuality.js"; - -export default async function(obj) { - try { - let infoInitial = await ytdl.getInfo(obj.id); - if (!infoInitial) { - return { error: 'ErrorCantConnectToServiceAPI' }; - } - let info = infoInitial.formats; - if (info[0]["isLive"]) { - return { error: 'ErrorLiveVideo' }; - } - let videoMatch = [], fullVideoMatch = [], video = [], audio = info.filter((a) => { - if (!a["isHLS"] && !a["isDashMPD"] && a["hasAudio"] && !a["hasVideo"] && a["container"] == obj.format) return true; - }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate)); - if (!obj.isAudioOnly) { - video = info.filter((a) => { - if (!a["isHLS"] && !a["isDashMPD"] && a["hasVideo"] && a["container"] == obj.format) { - if (obj.quality != "max") { - if (a["hasAudio"] && mq[obj.quality] == a["height"]) { - fullVideoMatch.push(a) - } else if (!a["hasAudio"] && mq[obj.quality] == a["height"]) { - videoMatch.push(a); - } - } - return true - } - }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate)); - if (obj.quality != "max") { - if (videoMatch.length == 0) { - let ss = selectQuality("youtube", obj.quality, video[0]["qualityLabel"].slice(0, 5).replace('p', '').trim()) - videoMatch = video.filter((a) => { - if (a["qualityLabel"].slice(0, 5).replace('p', '').trim() == ss) return true; - }) - } else if (fullVideoMatch.length > 0) { - videoMatch = [fullVideoMatch[0]] - } - } else videoMatch = [video[0]]; - if (obj.quality == "los") videoMatch = [video[video.length - 1]]; - } - let generalMeta = { - title: infoInitial.videoDetails.title, - artist: infoInitial.videoDetails.ownerChannelName.replace("- Topic", "").trim(), - } - if (audio[0]["approxDurationMs"] > maxVideoDuration) { - return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; - } - if (!obj.isAudioOnly && videoMatch.length > 0) { - if (video.length === 0 && audio.length === 0) { - return { error: 'ErrorBadFetch' }; - } - if (videoMatch[0]["hasVideo"] && videoMatch[0]["hasAudio"]) { - return { - type: "bridge", urls: videoMatch[0]["url"], time: videoMatch[0]["approxDurationMs"], - filename: `youtube_${obj.id}_${videoMatch[0]["width"]}x${videoMatch[0]["height"]}.${obj.format}` - }; - } - return { - type: "render", urls: [videoMatch[0]["url"], audio[0]["url"]], time: videoMatch[0]["approxDurationMs"], - filename: `youtube_${obj.id}_${videoMatch[0]["width"]}x${videoMatch[0]["height"]}.${obj.format}` - }; - } else if (!obj.isAudioOnly) { - return { - type: "render", urls: [video[0]["url"], audio[0]["url"]], time: video[0]["approxDurationMs"], - filename: `youtube_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.${video[0]["container"]}` - }; - } else if (audio.length > 0) { - let r = { - type: "render", - isAudioOnly: true, - urls: audio[0]["url"], - audioFilename: `youtube_${obj.id}_audio`, - fileMetadata: generalMeta - }; - if (infoInitial.videoDetails.description) { - let isAutoGenAudio = infoInitial.videoDetails.description.startsWith("Provided to YouTube by"); - if (isAutoGenAudio) { - let descItems = infoInitial.videoDetails.description.split("\n\n") - r.fileMetadata.album = descItems[2] - r.fileMetadata.copyright = descItems[3] - if (descItems[4].startsWith("Released on:")) r.fileMetadata.date = descItems[4].replace("Released on: ", '').trim(); - } - } - return r - } else { - return { error: 'ErrorBadFetch' }; - } - } catch (e) { - return { error: 'ErrorBadFetch' }; - } -} diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js index 065c8c2d..5da23c3c 100644 --- a/src/modules/stream/manage.js +++ b/src/modules/stream/manage.js @@ -47,7 +47,8 @@ export function verifyStream(ip, id, hmac, exp) { return { error: 'this stream token does not exist', status: 400 }; } let ghmac = sha256(`${id},${streamInfo.service},${ip},${exp}`, salt); - if (hmac == ghmac && exp.toString() == streamInfo.exp && ghmac == streamInfo.hmac && ip == streamInfo.ip && exp > Math.floor(new Date().getTime())) { + if (String(hmac) === ghmac && String(exp) === String(streamInfo.exp) && ghmac === String(streamInfo.hmac) + && String(ip) === streamInfo.ip && Number(exp) > Math.floor(new Date().getTime())) { return streamInfo; } return { error: 'Unauthorized', status: 401 }; diff --git a/src/modules/stream/selectQuality.js b/src/modules/stream/selectQuality.js index 9ddf1fdc..56244482 100644 --- a/src/modules/stream/selectQuality.js +++ b/src/modules/stream/selectQuality.js @@ -8,12 +8,12 @@ function closest(goal, array) { } export default function(service, quality, maxQuality) { - if (quality == "max") return maxQuality; + if (quality === "max") return maxQuality; quality = parseInt(mq[quality], 10) maxQuality = parseInt(maxQuality, 10) - if (quality >= maxQuality || quality == maxQuality) return maxQuality; + if (quality >= maxQuality || quality === maxQuality) return maxQuality; if (quality < maxQuality) { if (!services[service]["quality"][quality]) { diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index 50ce1389..5160d83f 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -116,7 +116,7 @@ export function streamVideoOnly(streamInfo, res) { '-i', streamInfo.urls, '-c', 'copy', '-an' ] - if (format == "mp4") args.push('-movflags', 'faststart+frag_keyframe+empty_moov') + if (format === "mp4") args.push('-movflags', 'faststart+frag_keyframe+empty_moov') args.push('-f', format, 'pipe:3'); const ffmpegProcess = spawn(ffmpeg, args, { windowsHide: true, diff --git a/src/modules/sub/currentCommit.js b/src/modules/sub/currentCommit.js index 7dfb1f9b..f3c145f5 100644 --- a/src/modules/sub/currentCommit.js +++ b/src/modules/sub/currentCommit.js @@ -1,10 +1,23 @@ import { execSync } from "child_process"; +let commit, commitInfo, branch; + export function shortCommit() { - return execSync('git rev-parse --short HEAD').toString().trim() + if (commit) return commit; + let c = execSync('git rev-parse --short HEAD').toString().trim(); + commit = c; + return c } export function getCommitInfo() { - let d = execSync(`git show -s --format='%s;;;%B'`).toString().trim().replace(/[\r\n]/gm, '\n').split(';;;') - d[1] = d[1].replace(d[0], '').trim().toString().replace(/[\r\n]/gm, '
') + if (commitInfo) return commitInfo; + let d = execSync(`git show -s --format='%s;;;%B'`).toString().trim().replace(/[\r\n]/gm, '\n').split(';;;'); + d[1] = d[1].replace(d[0], '').trim().toString().replace(/[\r\n]/gm, '
'); + commitInfo = d; return d } +export function getCurrentBranch() { + if (branch) return branch; + let b = execSync('git branch --show-current').toString().trim(); + branch = b; + return b +} diff --git a/src/modules/sub/errors.js b/src/modules/sub/errors.js index d4570ae3..c73ffe85 100644 --- a/src/modules/sub/errors.js +++ b/src/modules/sub/errors.js @@ -3,6 +3,9 @@ import loc from "../../localization/manager.js"; export function errorUnsupported(lang) { return loc(lang, 'ErrorUnsupported'); } -export function genericError(lang, host) { +export function brokenLink(lang, host) { return loc(lang, 'ErrorBrokenLink', host); } +export function genericError(lang, host) { + return loc(lang, 'ErrorBadFetch', host); +} diff --git a/src/modules/sub/utils.js b/src/modules/sub/utils.js index 78efaf14..e8bcd9d2 100644 --- a/src/modules/sub/utils.js +++ b/src/modules/sub/utils.js @@ -103,10 +103,9 @@ export function checkJSONPost(obj) { } try { let objKeys = Object.keys(obj); - if (!(objKeys.length < 8 && obj.url)) { - return false - } + if (!(objKeys.length <= 8 && obj.url)) return false; let defKeys = Object.keys(def); + for (let i in objKeys) { if (String(objKeys[i]) !== "url" && defKeys.includes(objKeys[i])) { if (apiVar.booleanOnly.includes(objKeys[i])) { @@ -116,12 +115,14 @@ export function checkJSONPost(obj) { } } } - obj["url"] = decodeURIComponent(String(obj["url"])) + + obj["url"] = decodeURIComponent(String(obj["url"])); let hostname = obj["url"].replace("https://", "").replace(' ', '').split('&')[0].split("/")[0].split("."), - host = hostname[hostname.length - 2] - def["url"] = encodeURIComponent(cleanURL(obj["url"], host)) + host = hostname[hostname.length - 2]; + def["url"] = encodeURIComponent(cleanURL(obj["url"], host)); + return def } catch (e) { - return false; + return false } } diff --git a/src/test/services.json b/src/test/services.json new file mode 100644 index 00000000..fcf38416 --- /dev/null +++ b/src/test/services.json @@ -0,0 +1,256 @@ +{ + "twitter": [{ + "name": "regular video", + "url": "https://twitter.com/TwitterSpaces/status/1526955853743546372?s=20", + "params": { + "aFormat": "mp3", + "isAudioOnly": false, + "isAudioMuted": false + }, + "expected": { + "code": 200, + "status": "redirect" + } + }, { + "name": "embedded twitter video", + "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", + "params": { + "aFormat": "mp3", + "isAudioOnly": false, + "isAudioMuted": false + }, + "expected": { + "code": 200, + "status": "redirect" + } + }, { + "name": "mixed media (image + gif)", + "url": "https://twitter.com/Twitter/status/1580661436132757506?s=20", + "params": { + "aFormat": "mp3", + "isAudioOnly": false, + "isAudioMuted": false + }, + "expected": { + "code": 200, + "status": "redirect" + } + }, { + "name": "picker: mixed media (3 gifs + image)", + "url": "https://twitter.com/emerald_pedrod/status/1582418163521581063?s=20", + "params": { + "aFormat": "mp3", + "isAudioOnly": false, + "isAudioMuted": false + }, + "expected": { + "code": 200, + "status": "picker" + } + }, { + "name": "audio from embedded twitter video (mp3, isAudioOnly)", + "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", + "params": { + "aFormat": "mp3", + "isAudioOnly": true, + "isAudioMuted": false + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "audio from embedded twitter video (best, isAudioOnly)", + "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", + "params": { + "aFormat": "best", + "isAudioOnly": true, + "isAudioMuted": false + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "audio from embedded twitter video (ogg, isAudioOnly, isAudioMuted)", + "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", + "params": { + "aFormat": "best", + "isAudioOnly": true, + "isAudioMuted": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "muted embedded twitter video", + "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", + "params": { + "aFormat": "mp3", + "isAudioOnly": false, + "isAudioMuted": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "inexistent post", + "url": "https://twitter.com/test/status/9487653", + "params": { + "aFormat": "best", + "isAudioOnly": false, + "isAudioMuted": false + }, + "expected": { + "code": 400, + "status": "error" + } + }, { + "name": "post with no media content", + "url": "https://twitter.com/elonmusk/status/1604617643973124097?s=20", + "params": { + "aFormat": "best", + "isAudioOnly": false, + "isAudioMuted": false + }, + "expected": { + "code": 400, + "status": "error" + } + }, { + "name": "recorded space by nyc (best)", + "url": "https://twitter.com/i/spaces/1gqxvyLoYQkJB", + "params": { + "aFormat": "best", + "isAudioOnly": false, + "isAudioMuted": false + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "recorded space by nyc (mp3)", + "url": "https://twitter.com/i/spaces/1gqxvyLoYQkJB", + "params": { + "aFormat": "mp3", + "isAudioOnly": false, + "isAudioMuted": false + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "recorded space by nyc (wav, isAudioMuted)", + "url": "https://twitter.com/i/spaces/1gqxvyLoYQkJB", + "params": { + "aFormat": "wav", + "isAudioOnly": false, + "isAudioMuted": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "recorded space by service95 & dualipa (mp3, isAudioMuted, isAudioOnly)", + "url": "https://twitter.com/i/spaces/1nAJErvvVXgxL", + "params": { + "aFormat": "mp3", + "isAudioOnly": true, + "isAudioMuted": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "unavailable space", + "url": "https://twitter.com/i/spaces/1OwGWwjRjVVGQ?s=20", + "params": { + "aFormat": "mp3", + "isAudioOnly": false, + "isAudioMuted": false + }, + "expected": { + "code": 400, + "status": "error" + } + }, { + "name": "inexistent space", + "url": "https://twitter.com/i/spaces/10Wkie2j29iiI", + "params": { + "aFormat": "mp3", + "isAudioOnly": false, + "isAudioMuted": false + }, + "expected": { + "code": 400, + "status": "error" + } + }], + "soundcloud": [{ + "name": "public song (best)", + "url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing", + "params": { + "aFormat": "best", + "isAudioOnly": false, + "isAudioMuted": false + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "public song (mp3, isAudioMuted)", + "url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing", + "params": { + "aFormat": "mp3", + "isAudioOnly": false, + "isAudioMuted": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "private song", + "url": "https://soundcloud.com/4kayy/unhappy-new-year-prod4kay/s-9bKbvwLdRWG", + "params": { + "aFormat": "mp3", + "isAudioOnly": false, + "isAudioMuted": false + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "private song (wav, isAudioMuted)", + "url": "https://soundcloud.com/4kayy/unhappy-new-year-prod4kay/s-9bKbvwLdRWG", + "params": { + "aFormat": "wav", + "isAudioOnly": false, + "isAudioMuted": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "private song (ogg, isAudioMuted, isAudioOnly)", + "url": "https://soundcloud.com/4kayy/unhappy-new-year-prod4kay/s-9bKbvwLdRWG", + "params": { + "aFormat": "ogg", + "isAudioOnly": true, + "isAudioMuted": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }] +} \ No newline at end of file diff --git a/src/test/test.js b/src/test/test.js new file mode 100644 index 00000000..ff4dc95e --- /dev/null +++ b/src/test/test.js @@ -0,0 +1,66 @@ +import "dotenv/config"; + +import { getJSON } from "../modules/api.js"; +import { services } from "../modules/config.js"; +import loadJSON from "../modules/sub/loadJSON.js"; +import { checkJSONPost } from "../modules/sub/utils.js"; + +let tests = loadJSON('./src/test/services.json'); + +let noTest = []; +let failed = []; +let success = 0; + +function addToFail(service, testName, url, response) { + failed.push({ + service: service, + name: testName, + url: url, + response: response + }) +} +for (let i in services) { + if (tests[i]) { + console.log(`\nRunning tests for ${i}...\n`) + for (let k = 0; k < tests[i].length; k++) { + let test = tests[i][k]; + + console.log(`Running test ${k+1}: ${test.name}`); + console.log('params:'); + let params = {...{url: test.url}, ...test.params}; + console.log(params); + + let chck = checkJSONPost(params); + if (chck) { + chck["ip"] = "d21ec524bc2ade41bef569c0361ac57728c69e2764b5cb3cb310fe36568ca53f"; // random sha256 + let j = await getJSON(chck["url"], "en", chck); + console.log('\nReceived:'); + console.log(j) + if (j.status === test.expected.code && j.body.status === test.expected.status) { + console.log("\n✅ Success.\n"); + success++ + } else { + console.log(`\n❌ Fail. Expected: ${test.expected.code} & ${test.expected.status}, received: ${j.status} & ${j.body.status}\n`); + addToFail(i, test.name, test.url, j) + } + } else { + console.log("\n❌ couldn't validate the request JSON.\n"); + addToFail(i, test.name, test.url, {}) + } + } + console.log("\n\n") + } else { + console.warn(`No tests found for ${i}.`); + noTest.push(i) + } +} + +console.log(`\n✅ ${success} tests succeeded.`); +console.log(`❌ ${failed.length} tests failed.`); +console.log(`❔ ${noTest.length} services weren't tested.`); + +console.log(`\nFailed tests:`); +console.log(failed) + +console.log(`\nMissing tests:`); +console.log(noTest)