Nikurasu/cobalt
1
0
Fork 0
Code Pull requests Activity

5.0-dev1

Browse source 
- rewrote and/or optimized all service modules
- rewrote matching and processing modules to optimize readability and performance
- added support for reddit gifs
- fixed various issues with twitter error explanations
- code optimizations and enhancements (such as finally getting rid of ==, prettier and more readable formatting, etc)
- added branch information
- all functions in currentCommit submodule run only once and cache received data
- added a test script. only twitter and soundcloud are 100% covered and tested atm, will add tests (and probably fixes) for the rest of services in next commits
- changed some localization strings for russian
- added more clarity to rate limit message
- moved services folder into processing folder
This commit is contained in:
wukko 2023-02-12 13:40:49 +06:00
parent 3432c91482
commit dacaaf5b27
39 changed files with 1139 additions and 825 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
package.json
View file

@ -1,7 +1,7 @@
{
"name": "cobalt",
"description": "save what you love",
"version": "4.9-dev",
"version": "5.0-dev1",
"author": "wukko",
"exports": "./src/cobalt.js",
"type": "module",
@ -10,7 +10,8 @@
},
"scripts": {
"start": "node src/cobalt",
"setup": "node src/modules/setup"
"setup": "node src/modules/setup",
"test": "node src/test/test"
},
"repository": {
"type": "git",

28
src/cobalt.js
View file

@ -1,11 +1,11 @@
import "dotenv/config"
import "dotenv/config";
import express from "express";
import cors from "cors";
import * as fs from "fs";
import rateLimit from "express-rate-limit";
import { shortCommit } from "./modules/sub/currentCommit.js";
import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js";
import { appName, genericUserAgent, version } from "./modules/config.js";
import { getJSON } from "./modules/api.js";
import renderPage from "./modules/pageRender/page.js";
@ -18,13 +18,14 @@ import { changelogHistory } from "./modules/pageRender/onDemand.js";
import { sha256 } from "./modules/sub/crypto.js";
const commitHash = shortCommit();
const branch = getCurrentBranch();
const app = express();
app.disable('x-powered-by');
if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt && process.env.port) {
const apiLimiter = rateLimit({
windowMs: 1 * 60 * 1000,
windowMs: 60000,
max: 12,
standardHeaders: true,
legacyHeaders: false,
@ -33,7 +34,7 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt &&
}
});
const apiLimiterStream = rateLimit({
windowMs: 1 * 60 * 1000,
windowMs: 60000,
max: 12,
standardHeaders: true,
legacyHeaders: false,
@ -63,7 +64,6 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt &&
}
next();
});
app.use('/api/json', express.json({
verify: (req, res, buf) => {
try {
@ -76,6 +76,7 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt &&
}
}
}));
app.post('/api/:type', cors({ origin: process.env.selfURL, optionsSuccessStatus: 200 }), async (req, res) => {
try {
let ip = sha256(req.header('x-forwarded-for') ? req.header('x-forwarded-for') : req.ip.replace('::ffff:', ''), process.env.streamSalt);
@ -89,10 +90,10 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt &&
let j = await getJSON(chck["url"], languageCode(req), chck)
res.status(j.status).json(j.body);
} else if (request.url && !chck) {
let j = apiJSON(3, { t: loc(languageCode(req), 'ErrorCouldntFetch') });
let j = apiJSON(0, { t: loc(languageCode(req), 'ErrorCouldntFetch') });
res.status(j.status).json(j.body);
} else {
let j = apiJSON(3, { t: loc(languageCode(req), 'ErrorNoLink') })
let j = apiJSON(0, { t: loc(languageCode(req), 'ErrorNoLink') })
res.status(j.status).json(j.body);
}
} catch (e) {
@ -108,12 +109,16 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt &&
res.status(500).json({ 'status': 'error', 'text': loc(languageCode(req), 'ErrorCantProcess') })
}
});
app.get('/api/:type', cors({ origin: process.env.selfURL, optionsSuccessStatus: 200 }), (req, res) => {
try {
let ip = sha256(req.header('x-forwarded-for') ? req.header('x-forwarded-for') : req.ip.replace('::ffff:', ''), process.env.streamSalt);
switch (req.params.type) {
case 'json':
res.status(405).json({ 'status': 'error', 'text': 'GET method for this request has been deprecated. see https://github.com/wukko/cobalt/blob/current/docs/API.md for up-to-date API documentation.' });
res.status(405).json({
'status': 'error',
'text': 'GET method for this endpoint has been deprecated. see https://github.com/wukko/cobalt/blob/current/docs/API.md for up-to-date API documentation.'
});
break;
case 'stream':
if (req.query.p) {
@ -153,6 +158,7 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt &&
res.status(500).json({ 'status': 'error', 'text': loc(languageCode(req), 'ErrorCantProcess') })
}
});
app.get("/api", (req, res) => {
res.redirect('/api/json')
});
@ -161,7 +167,8 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt &&
"hash": commitHash,
"type": "default",
"lang": languageCode(req),
"useragent": req.header('user-agent') ? req.header('user-agent') : genericUserAgent
"useragent": req.header('user-agent') ? req.header('user-agent') : genericUserAgent,
"branch": branch
}))
});
app.get("/favicon.ico", (req, res) => {
@ -170,9 +177,10 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt &&
app.get("/*", (req, res) => {
res.redirect('/')
});
app.listen(process.env.port, () => {
let startTime = new Date();
console.log(`\n${Cyan(appName)} ${Bright(`v.${version}-${commitHash}`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(new Date().getTime())})`)}\n\nURL: ${Cyan(`${process.env.selfURL}`)}\nPort: ${process.env.port}\n`)
console.log(`\n${Cyan(appName)} ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(new Date().getTime())})`)}\n\nURL: ${Cyan(`${process.env.selfURL}`)}\nPort: ${process.env.port}\n`)
});
} else {
console.log(Red(`cobalt hasn't been configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`))

2
src/front/cobalt.js
View file

@ -273,7 +273,7 @@ function toggle(toggl) {
}
function loadSettings() {
try {
if (typeof(navigator.clipboard.readText) == "undefined") throw new Error();
if (typeof(navigator.clipboard.readText) === undefined) throw new Error();
} catch (err) {
eid("pasteFromClipboard").style.display = "none"
}

2
src/localization/languages/en.json
View file

@ -25,7 +25,7 @@
"ErrorBrokenLink": "{s} is supported, but something is wrong with your link. maybe you didn't copy it fully?",
"ErrorNoLink": "i can't guess what you want to download! please give me a link.",
"ErrorPageRenderFail": "something went wrong and page couldn't render. if it's a recurring or critical issue, please {ContactLink}. it'd be useful if you provided current commit hash ({s}) and error recreation steps. thank you in advance :D",
"ErrorRateLimit": "you're making too many requests. calm down and try again in a bit.",
"ErrorRateLimit": "you're making too many requests. try again in a minute!",
"ErrorCouldntFetch": "couldn't get any info about your link. check if it's correct and try again.",
"ErrorLengthLimit": "current length limit is {s} minutes. video that you tried to download is longer than {s} minutes. pick something else!",
"ErrorBadFetch": "an error occurred when i tried to get info about your link. are you sure it works? check if it does, and try again.",

6
src/localization/languages/ru.json
View file

@ -7,7 +7,7 @@
"LinkInput": "вставь ссылку сюда",
"AboutSummary": "{appName} — твой друг при скачивании контента из соц. сетей. никакой рекламы или трекеров. вставляешь ссылку и получаешь файл. ничего лишнего.",
"EmbedBriefDescription": "сохраняй что хочешь, без мороки и вторжения в личное пространство",
"MadeWithLove": "сделано с <3 wukko",
"MadeWithLove": "сделано wukko, с <3",
"AccessibilityInputArea": "зона вставки ссылки",
"AccessibilityOpenAbout": "открыть окно с инфой",
"AccessibilityDownloadButton": "кнопка скачивания",
@ -23,9 +23,9 @@
"ErrorSomethingWentWrong": "что-то пошло совсем не так, и у меня не получилось ничего для тебя достать. ты можешь попробовать ещё раз, но если так и не получится, {ContactLink}.",
"ErrorUnsupported": "с твоей ссылкой что-то не так, или же этот сервис ещё не поддерживается. может быть, ты вставил не ту ссылку?",
"ErrorBrokenLink": "{s} поддерживается, но с твоей ссылкой что-то не так. может быть, ты её не полностью скопировал?",
"ErrorNoLink": "я не гадалка и не могу угадывать, что ты хочешь скачать. попробуй в следующий раз вставить ссылку.",
"ErrorNoLink": "пока что я не умею угадывать, что ты хочешь скачать. дай мне, пожалуйста, ссылку :(",
"ErrorPageRenderFail": "что-то пошло не так и у меня не получилось срендерить страницу. если это повторится ещё раз, пожалуйста, {ContactLink}. также приложи хэш текущего коммита ({s}) с действиями для повторения этой ошибки. можно на русском языке. спасибо :)",
"ErrorRateLimit": "ты делаешь слишком много запросов. успокойся и попробуй ещё раз через несколько минут.",
"ErrorRateLimit": "ты делаешь слишком много запросов. попробуй ещё раз через минуту!",
"ErrorCouldntFetch": "мне не удалось получить инфу о твоей ссылке. проверь её и попробуй ещё раз.",
"ErrorLengthLimit": "твоё видео длиннее чем {s} минут(ы). это превышает текущий лимит. скачай что-нибудь покороче, а не экранизацию \"войны и мира\".",
"ErrorBadFetch": "произошла ошибка при получении инфы о твоей ссылке. ты уверен, что она работает? проверь её, и попробуй ещё раз.",

2
src/modules/api.js
View file

@ -31,7 +31,7 @@ export async function getJSON(originalURL, lang, obj) {
break;
case "tumblr":
if (!url.includes("blog/view")) {
if (url.slice(-1) == '/') url = url.slice(0, -1);
if (url.slice(-1) === '/') url = url.slice(0, -1);
url = url.replace(url.split('/')[5], '');
}
break;

2
src/modules/pageRender/elements.js
View file

@ -2,7 +2,7 @@ import { celebrations } from "../config.js";
export function switcher(obj) {
let items = ``;
if (obj.name == "download") {
if (obj.name === "download") {
items = obj.items;
} else {
for (let i = 0; i < obj.items.length; i++) {

2
src/modules/pageRender/page.js
View file

@ -186,7 +186,7 @@ export default function(obj) {
closeAria: t('AccessibilityClosePopup'),
header: {
aboveTitle: {
text: `v.${version}-${obj.hash}`,
text: `v.${version}-${obj.hash} (${obj.branch})`,
url: `${repo}/commit/${obj.hash}`
},
title: `${emoji("⚙️", 30)} ${t('TitlePopupSettings')}`

71
src/modules/processing/match.js
View file

@ -5,23 +5,24 @@ import loc from "../../localization/manager.js";
import { testers } from "./servicesPatternTesters.js";
import bilibili from "../services/bilibili.js";
import reddit from "../services/reddit.js";
import twitter from "../services/twitter.js";
import youtube from "../services/youtube.js";
import vk from "../services/vk.js";
import tiktok from "../services/tiktok.js";
import tumblr from "../services/tumblr.js";
import bilibili from "./services/bilibili.js";
import reddit from "./services/reddit.js";
import twitter from "./services/twitter.js";
import youtube from "./services/youtube.js";
import vk from "./services/vk.js";
import tiktok from "./services/tiktok.js";
import tumblr from "./services/tumblr.js";
import matchActionDecider from "./matchActionDecider.js";
import vimeo from "../services/vimeo.js";
import soundcloud from "../services/soundcloud.js";
import vimeo from "./services/vimeo.js";
import soundcloud from "./services/soundcloud.js";
export default async function (host, patternMatch, url, lang, obj) {
try {
if (!testers[host]) return apiJSON(0, { t: errorUnsupported(lang) });
if (!(testers[host](patternMatch))) throw Error();
let r, isAudioOnly = !!obj.isAudioOnly;
if (!testers[host]) return apiJSON(0, { t: errorUnsupported(lang) });
if (!(testers[host](patternMatch))) return apiJSON(0, { t: brokenLink(lang) });
let r;
switch (host) {
case "twitter":
r = await twitter({
@ -29,14 +30,14 @@ export default async function (host, patternMatch, url, lang, obj) {
spaceId: patternMatch["spaceId"] ? patternMatch["spaceId"] : false,
lang: lang
});
if (r.isAudioOnly) obj.isAudioOnly = true
break;
case "vk":
r = await vk({
url: url,
userId: patternMatch["userId"],
videoId: patternMatch["videoId"],
lang: lang, quality: obj.vQuality
lang: lang,
quality: obj.vQuality
});
break;
case "bilibili":
@ -48,10 +49,11 @@ export default async function (host, patternMatch, url, lang, obj) {
case "youtube":
let fetchInfo = {
id: patternMatch["id"].slice(0, 11),
lang: lang, quality: obj.vQuality,
lang: lang,
quality: obj.vQuality,
format: "webm"
};
if (url.match('music.youtube.com') || obj.isAudioOnly == true) obj.vFormat = "audio";
if (url.match('music.youtube.com') || isAudioOnly === true) obj.vFormat = "audio";
switch (obj.vFormat) {
case "mp4":
fetchInfo["format"] = "mp4";
@ -60,7 +62,7 @@ export default async function (host, patternMatch, url, lang, obj) {
fetchInfo["format"] = "webm";
fetchInfo["isAudioOnly"] = true;
fetchInfo["quality"] = "max";
obj.isAudioOnly = true;
isAudioOnly = true;
break;
}
r = await youtube(fetchInfo);
@ -69,7 +71,8 @@ export default async function (host, patternMatch, url, lang, obj) {
r = await reddit({
sub: patternMatch["sub"],
id: patternMatch["id"],
title: patternMatch["title"], lang: lang,
title: patternMatch["title"],
lang: lang,
});
break;
case "douyin":
@ -77,28 +80,33 @@ export default async function (host, patternMatch, url, lang, obj) {
r = await tiktok({
host: host,
postId: patternMatch["postId"],
id: patternMatch["id"], lang: lang,
noWatermark: obj.isNoTTWatermark, fullAudio: obj.isTTFullAudio,
isAudioOnly: obj.isAudioOnly
id: patternMatch["id"],
lang: lang,
noWatermark: obj.isNoTTWatermark,
fullAudio: obj.isTTFullAudio,
isAudioOnly: isAudioOnly
});
if (r.isAudioOnly) obj.isAudioOnly = true;
break;
case "tumblr":
r = await tumblr({
id: patternMatch["id"], url: url, user: patternMatch["user"] ? patternMatch["user"] : false,
id: patternMatch["id"],
url: url,
user: patternMatch["user"] ? patternMatch["user"] : false,
lang: lang
});
break;
case "vimeo":
r = await vimeo({
id: patternMatch["id"].slice(0, 11), quality: obj.vQuality,
id: patternMatch["id"].slice(0, 11),
quality: obj.vQuality,
lang: lang
});
break;
case "soundcloud":
obj.isAudioOnly = true;
isAudioOnly = true;
r = await soundcloud({
author: patternMatch["author"], song: patternMatch["song"], url: url,
author: patternMatch["author"],
song: patternMatch["song"], url: url,
shortLink: patternMatch["shortLink"] ? patternMatch["shortLink"] : false,
accessKey: patternMatch["accessKey"] ? patternMatch["accessKey"] : false,
format: obj.aFormat,
@ -108,10 +116,15 @@ export default async function (host, patternMatch, url, lang, obj) {
default:
return apiJSON(0, { t: errorUnsupported(lang) });
}
return !r.error ? matchActionDecider(r, host, obj.ip, obj.aFormat, obj.isAudioOnly, lang, obj.isAudioMuted) : apiJSON(0, {
t: Array.isArray(r.error) ? loc(lang, r.error[0], r.error[1]) : loc(lang, r.error)
});
if (r.isAudioOnly) isAudioOnly = true;
let isAudioMuted = isAudioOnly ? false : obj.isAudioMuted;
if (r.error) return apiJSON(0, { t: Array.isArray(r.error) ? loc(lang, r.error[0], r.error[1]) : loc(lang, r.error) });
return matchActionDecider(r, host, obj.ip, obj.aFormat, isAudioOnly, lang, isAudioMuted);
} catch (e) {
console.log(e)
return apiJSON(0, { t: genericError(lang, host) })
}
}

240
src/modules/processing/matchActionDecider.js
View file

@ -1,121 +1,137 @@
import { audioIgnore, services, supportedAudio } from "../config.js"
import { apiJSON } from "../sub/utils.js"
import { audioIgnore, services, supportedAudio } from "../config.js";
import { apiJSON } from "../sub/utils.js";
import loc from "../../localization/manager.js";
export default function(r, host, ip, audioFormat, isAudioOnly, lang, isAudioMuted) {
if (!isAudioOnly && !r.picker && !isAudioMuted) {
switch (host) {
case "twitter":
return apiJSON(1, { u: r.urls });
case "vk":
return apiJSON(2, {
type: "bridge", u: r.urls, service: host, ip: ip,
filename: r.filename,
});
case "bilibili":
return apiJSON(2, {
type: "render", u: r.urls, service: host, ip: ip,
filename: r.filename,
time: r.time
});
case "youtube":
return apiJSON(2, {
type: r.type, u: r.urls, service: host, ip: ip,
filename: r.filename,
time: r.time,
});
case "reddit":
return apiJSON(r.typeId, {
type: r.type, u: r.urls, service: host, ip: ip,
filename: r.filename,
});
case "tiktok":
return apiJSON(2, {
type: "bridge", u: r.urls, service: host, ip: ip,
filename: r.filename,
});
case "douyin":
return apiJSON(2, {
type: "bridge", u: r.urls, service: host, ip: ip,
filename: r.filename,
});
case "tumblr":
return apiJSON(1, { u: r.urls });
case "vimeo":
if (Array.isArray(r.urls)) {
return apiJSON(2, {
type: "render", u: r.urls, service: host, ip: ip,
filename: r.filename
});
} else {
return apiJSON(1, { u: r.urls });
}
}
} else if (isAudioMuted && !isAudioOnly) {
let isSplit = Array.isArray(r.urls);
return apiJSON(2, {
type: isSplit ? "bridge" : "mute",
u: isSplit ? r.urls[0] : r.urls,
let action,
responseType = 2,
defaultParams = {
u: r.urls,
service: host,
ip: ip,
filename: r.filename,
mute: true,
});
} else if (r.picker) {
switch (host) {
case "douyin":
case "tiktok":
let type = "render";
if (audioFormat === "mp3" || audioFormat === "best") {
audioFormat = "mp3"
type = "bridge"
}
return apiJSON(5, {
type: type,
picker: r.picker,
u: Array.isArray(r.urls) ? r.urls[1] : r.urls, service: host, ip: ip,
filename: r.audioFilename, isAudioOnly: true, audioFormat: audioFormat, copy: audioFormat === "best" ? true : false,
})
case "twitter":
return apiJSON(5, {
picker: r.picker, service: host
})
}
} else if (isAudioOnly) {
if ((host === "reddit" && r.typeId === 1) || (host === "vimeo" && !r.filename) || audioIgnore.includes(host)) return apiJSON(0, { t: loc(lang, 'ErrorEmptyDownload') });
let type = "render";
let copy = false;
},
params = {}
if (!supportedAudio.includes(audioFormat)) audioFormat = "best";
if ((host == "tiktok" || host == "douyin") && services.tiktok.audioFormats.includes(audioFormat)) {
if (r.isMp3) {
if (audioFormat === "mp3" || audioFormat === "best") {
audioFormat = "mp3"
type = "bridge"
}
} else if (audioFormat === "best") {
audioFormat = "m4a"
type = "bridge"
}
}
if ((audioFormat === "best" && services[host]["bestAudio"]) || services[host]["bestAudio"] && (audioFormat === services[host]["bestAudio"])) {
audioFormat = services[host]["bestAudio"]
type = "bridge"
} else if (audioFormat === "best") {
audioFormat = "m4a"
copy = true
if (r.audioFilename.includes("twitterspaces")) {
audioFormat = "mp3"
copy = false
}
}
return apiJSON(2, {
type: type,
u: Array.isArray(r.urls) ? r.urls[1] : r.urls, service: host, ip: ip,
filename: r.audioFilename, isAudioOnly: true,
audioFormat: audioFormat, copy: copy, fileMetadata: r.fileMetadata ? r.fileMetadata : false
})
} else {
return apiJSON(0, { t: loc(lang, 'ErrorSomethingWentWrong') });
if (isAudioMuted) action = "muteVideo";
if (!isAudioOnly && !r.picker && !isAudioMuted) action = "video";
if (r.picker) action = "picker";
if (isAudioOnly) action = "audio";
if (action === "picker" || action === "audio") {
defaultParams.filename = r.audioFilename;
defaultParams.isAudioOnly = true;
defaultParams.audioFormat = audioFormat;
}
switch (action) {
case "video":
switch (host) {
case "bilibili":
params = { type: "render", time: r.time };
break;
case "youtube":
params = { type: r.type, time: r.time };
break;
case "reddit":
responseType = r.typeId;
params = { type: r.type };
break;
case "vimeo":
if (Array.isArray(r.urls)) {
params = { type: "render" }
} else {
responseType = 1;
}
break;
case "vk":
case "douyin":
case "tiktok":
params = { type: "bridge" };
break;
case "tumblr":
case "twitter":
responseType = 1;
break;
}
break;
case "muteVideo":
params = {
type: Array.isArray(r.urls) ? "bridge" : "mute",
u: Array.isArray(r.urls) ? r.urls[0] : r.urls,
mute: true
}
break;
case "picker":
responseType = 5;
switch (host) {
case "twitter":
params = { picker: r.picker };
break;
case "douyin":
case "tiktok":
let pickerType = "render";
if (audioFormat === "mp3" || audioFormat === "best") {
audioFormat = "mp3";
pickerType = "bridge"
}
params = {
type: pickerType,
picker: r.picker,
u: Array.isArray(r.urls) ? r.urls[1] : r.urls,
copy: audioFormat === "best" ? true : false
}
}
break;
case "audio":
if ((host === "reddit" && r.typeId === 1) || (host === "vimeo" && !r.filename) || audioIgnore.includes(host)) return apiJSON(0, { t: loc(lang, 'ErrorEmptyDownload') });
let processType = "render";
let copy = false;
if (!supportedAudio.includes(audioFormat)) audioFormat = "best";
if ((host === "tiktok" || host === "douyin") && services.tiktok.audioFormats.includes(audioFormat)) {
if (r.isMp3) {
if (audioFormat === "mp3" || audioFormat === "best") {
audioFormat = "mp3";
processType = "bridge"
}
} else if (audioFormat === "best") {
audioFormat = "m4a";
processType = "bridge"
}
}
if ((audioFormat === "best" && services[host]["bestAudio"])
|| services[host]["bestAudio"] && (audioFormat === services[host]["bestAudio"])) {
audioFormat = services[host]["bestAudio"];
processType = "bridge"
} else if (audioFormat === "best") {
audioFormat = "m4a";
copy = true;
if (r.audioFilename.includes("twitterspaces")) {
audioFormat = "mp3"
copy = false
}
}
params = {
type: processType,
u: Array.isArray(r.urls) ? r.urls[1] : r.urls,
audioFormat: audioFormat,
copy: copy,
fileMetadata: r.fileMetadata ? r.fileMetadata : false
}
break;
default:
return apiJSON(0, { t: loc(lang, 'ErrorEmptyDownload') });
}
return apiJSON(responseType, {...defaultParams, ...params})
}

28
src/modules/processing/services/bilibili.js Normal file
View file

@ -0,0 +1,28 @@
import { genericUserAgent, maxVideoDuration } from "../../config.js";
// TO-DO: quality picking & bilibili.tv support
export default async function(obj) {
let html = await fetch(`https://bilibili.com/video/${obj.id}`, {
headers: { "user-agent": genericUserAgent }
}).then((r) => { return r.text() }).catch(() => { return false });
if (!html) return { error: 'ErrorCouldntFetch' };
if (!(html.includes('<script>window.__playinfo__=') && html.includes('"video_codecid"'))) return { error: 'ErrorEmptyDownload' };
let streamData = JSON.parse(html.split('<script>window.__playinfo__=')[1].split('</script>')[0]);
if (streamData.data.timelength > maxVideoDuration) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
let video = streamData["data"]["dash"]["video"].filter((v) => {
if (!v["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/")) return true;
}).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth));
let audio = streamData["data"]["dash"]["audio"].filter((a) => {
if (!a["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/")) return true;
}).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth));
return {
urls: [video[0]["baseUrl"], audio[0]["baseUrl"]],
time: streamData.data.timelength,
audioFilename: `bilibili_${obj.id}_audio`,
filename: `bilibili_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.mp4`
};
}

28
src/modules/processing/services/reddit.js Normal file
View file

@ -0,0 +1,28 @@
import { maxVideoDuration } from "../../config.js";
export default async function(obj) {
let data = await fetch(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}/${obj.name}.json`).then((r) => { return r.json() }).catch(() => { return false });
if (!data) return { error: 'ErrorCouldntFetch' };
data = data[0]["data"]["children"][0]["data"];
if (data.url.endsWith('.gif')) return { typeId: 1, urls: data.url };
if (!"reddit_video" in data["secure_media"]) return { error: 'ErrorEmptyDownload' };
if (data["secure_media"]["reddit_video"]["duration"] * 1000 > maxVideoDuration) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
let video = data["secure_media"]["reddit_video"]["fallback_url"].split('?')[0],
audio = video.match('.mp4') ? `${video.split('_')[0]}_audio.mp4` : `${data["secure_media"]["reddit_video"]["fallback_url"].split('DASH')[0]}audio`;
await fetch(audio, { method: "HEAD" }).then((r) => {if (r.status != 200) audio = ''}).catch(() => {audio = ''});
let id = data["secure_media"]["reddit_video"]["fallback_url"].split('/')[3];
if (!audio.length > 0) return { typeId: 1, urls: video };
return {
typeId: 2,
type: "render",
urls: [video, audio],
audioFilename: `reddit_${id}_audio`,
filename: `reddit_${id}.mp4`
};
}

74
src/modules/processing/services/soundcloud.js Normal file
View file

@ -0,0 +1,74 @@
import { maxAudioDuration } from "../../config.js";
let cachedID = {};
async function findClientID() {
try {
let sc = await fetch('https://soundcloud.com/').then((r) => { return r.text() }).catch(() => { return false });
let scVersion = String(sc.match(/<script>window\.__sc_version="[0-9]{10}"<\/script>/)[0].match(/[0-9]{10}/));
if (cachedID.version === scVersion) return cachedID.id;
let scripts = sc.matchAll(/<script.+src="(.+)">/g);
let clientid;
for (let script of scripts) {
let url = script[1];
if (url && !url.startsWith('https://a-v2.sndcdn.com')) return;
let scrf = await fetch(url).then((r) => {return r.text()}).catch(() => { return false });
let id = scrf.match(/\("client_id=[A-Za-z0-9]{32}"\)/);
if (id && typeof id[0] === 'string') {
clientid = id[0].match(/[A-Za-z0-9]{32}/)[0];
break;
}
}
cachedID.version = scVersion;
cachedID.id = clientid;
return clientid;
} catch (e) {
return false;
}
}
export default async function(obj) {
let html;
if (!obj.author && !obj.song && obj.shortLink) {
html = await fetch(`https://soundcloud.app.goo.gl/${obj.shortLink}/`).then((r) => { return r.text() }).catch(() => { return false });
}
if (obj.author && obj.song) {
html = await fetch(`https://soundcloud.com/${obj.author}/${obj.song}${obj.accessKey ? `/s-${obj.accessKey}` : ''}`).then((r) => { return r.text() }).catch(() => { return false });
}
if (!html) return { error: 'ErrorCouldntFetch'};
if (!(html.includes('<script>window.__sc_hydration = ')
&& html.includes('"format":{"protocol":"progressive","mime_type":"audio/mpeg"},')
&& html.includes('{"hydratable":"sound","data":'))) {
return { error: ['ErrorBrokenLink', 'soundcloud'] }
}
let json = JSON.parse(html.split('{"hydratable":"sound","data":')[1].split('}];</script>')[0])
if (!json["media"]["transcodings"]) return { error: 'ErrorEmptyDownload' };
let clientId = await findClientID();
if (!clientId) return { error: 'ErrorSoundCloudNoClientId' };
let fileUrlBase = json.media.transcodings[0]["url"].replace("/hls", "/progressive")
let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`;
if (!fileUrl.substring(0, 54) === "https://api-v2.soundcloud.com/media/soundcloud:tracks:") return { error: 'ErrorEmptyDownload' };
if (json.duration > maxAudioDuration) return { error: ['ErrorLengthAudioConvert', maxAudioDuration / 60000] };
let file = await fetch(fileUrl).then(async (r) => { return (await r.json()).url }).catch(() => { return false });
if (!file) return { error: 'ErrorCouldntFetch' };
return {
urls: file,
audioFilename: `soundcloud_${json.id}`,
fileMetadata: {
title: json.title,
artist: json.user.username,
}
}
}

112
src/modules/processing/services/tiktok.js Normal file
View file

@ -0,0 +1,112 @@
import { genericUserAgent } from "../../config.js";
let userAgent = genericUserAgent.split(' Chrome/1')[0],
config = {
tiktok: {
short: "https://vt.tiktok.com/",
api: "https://api2.musical.ly/aweme/v1/feed/?aweme_id={postId}&version_code=262&app_name=musical_ly&channel=App&device_id=null&os_version=14.4.2&device_platform=iphone&device_type=iPhone9&region=US&carrier_region=US",
},
douyin: {
short: "https://v.douyin.com/",
api: "https://www.iesdouyin.com/web/api/v2/aweme/iteminfo/?item_ids={postId}",
}
}
function selector(j, h, id) {
if (!j) return false;
let t;
switch (h) {
case "tiktok":
t = j["aweme_list"].filter((v) => { if (v["aweme_id"] === id) return true });
break;
case "douyin":
t = j['item_list'].filter((v) => { if (v["aweme_id"] === id) return true });
break;
}
if (!t.length > 0) return false;
return t[0];
}
export default async function(obj) {
if (!obj.postId) {
let html = await fetch(`${config[obj.host]["short"]}${obj.id}`, {
redirect: "manual",
headers: { "user-agent": userAgent }
}).then((r) => { return r.text() }).catch(() => { return false });
if (!html) return { error: 'ErrorCouldntFetch' };
if (html.slice(0, 17) === '<a href="https://' && html.includes('/video/')) {
obj.postId = html.split('/video/')[1].split('?')[0].replace("/", '')
} else if (html.slice(0, 32) === '<a href="https://m.tiktok.com/v/' && html.includes('/v/')) {
obj.postId = html.split('/v/')[1].split('.html')[0].replace("/", '')
}
}
if (!obj.postId) return { error: 'ErrorCantGetID' };
let detail;
detail = await fetch(config[obj.host]["api"].replace("{postId}", obj.postId), {
headers: {"user-agent": "TikTok 26.2.0 rv:262018 (iPhone; iOS 14.4.2; en_US) Cronet"}
}).then((r) => { return r.json() }).catch(() => { return false });
detail = selector(detail, obj.host, obj.postId);
if (!detail) return { error: 'ErrorCouldntFetch' };
let video, videoFilename, audioFilename, isMp3, audio, images, filenameBase = `${obj.host}_${obj.postId}`;
if (obj.host === "tiktok") {
images = detail["image_post_info"] ? detail["image_post_info"]["images"] : false
} else {
images = detail["images"] ? detail["images"] : false
}
if (!obj.isAudioOnly && !images) {
video = obj.host === "tiktok" ? detail["video"]["download_addr"]["url_list"][0] : detail['video']['play_addr']['url_list'][0];
videoFilename = `${filenameBase}_video.mp4`;
if (obj.noWatermark) {
video = obj.host === "tiktok" ? detail["video"]["play_addr"]["url_list"][0] : detail["video"]["play_addr"]["url_list"][0].replace("playwm", "play");
videoFilename = `${filenameBase}_video_nw.mp4` // nw - no watermark
}
} else {
let fallback = obj.host === "douyin" ? detail["video"]["play_addr"]["url_list"][0].replace("playwm", "play") : detail["video"]["play_addr"]["url_list"][0];
audio = fallback;
audioFilename = `${filenameBase}_audio_fv`; // fv - from video
if (obj.fullAudio || fallback.includes("music")) {
audio = detail["music"]["play_url"]["url_list"][0]
audioFilename = `${filenameBase}_audio`
}
if (audio.slice(-4) === ".mp3") isMp3 = true;
}
if (video) return {
urls: video,
filename: videoFilename
}
if (images && obj.isAudioOnly) {
return {
urls: audio,
audioFilename: audioFilename,
isAudioOnly: true,
isMp3: isMp3,
}
}
if (images) {
let imageLinks = [];
for (let i in images) {
let sel = obj.host === "tiktok" ? images[i]["display_image"]["url_list"] : images[i]["url_list"];
sel = sel.filter((p) => { if (p.includes(".jpeg?")) return true; })
imageLinks.push({url: sel[0]})
}
return {
picker: imageLinks,
urls: audio,
audioFilename: audioFilename,
isAudioOnly: true,
isMp3: isMp3,
}
}
if (audio) return {
urls: audio,
audioFilename: audioFilename,
isAudioOnly: true,
isMp3: isMp3,
}
}

14
src/modules/processing/services/tumblr.js Normal file
View file

@ -0,0 +1,14 @@
import { genericUserAgent } from "../../config.js";
export default async function(obj) {
let html = await fetch(`https://${
obj.user ? obj.user : obj.url.split('.')[0].replace('https://', '')
}.tumblr.com/post/${obj.id}`, {
headers: { "user-agent": genericUserAgent }
}).then((r) => { return r.text() }).catch(() => { return false });
if (!html) return { error: 'ErrorCouldntFetch' };
if (!html.includes('property="og:video" content="https://va.media.tumblr.com/')) return { error: 'ErrorEmptyDownload' };
return { urls: `https://va.media.tumblr.com/${html.split('property="og:video" content="https://va.media.tumblr.com/')[1].split('"')[0]}`, audioFilename: `tumblr_${obj.id}_audio` }
}

101
src/modules/processing/services/twitter.js Normal file
View file

@ -0,0 +1,101 @@
import { genericUserAgent } from "../../config.js";
function bestQuality(arr) {
return arr.filter((v) => { if (v["content_type"] === "video/mp4") return true }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate))[0]["url"].split("?")[0]
}
const apiURL = "https://api.twitter.com/1.1"
// TO-DO: move from 1.1 api to graphql
export default async function(obj) {
let _headers = {
"user-agent": genericUserAgent,
"authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
// ^ no explicit content, but with multi media support
"host": "api.twitter.com"
};
let req_act = await fetch(`${apiURL}/guest/activate.json`, {
method: "POST",
headers: _headers
}).then((r) => { return r.status === 200 ? r.json() : false }).catch(() => { return false });
if (!req_act) return { error: 'ErrorCouldntFetch' };
_headers["x-guest-token"] = req_act["guest_token"];
let showURL = `${apiURL}/statuses/show/${obj.id}.json?tweet_mode=extended&include_user_entities=0&trim_user=1&include_entities=0&cards_platform=Web-12&include_cards=1`;
if (!obj.spaceId) {
let req_status = await fetch(showURL, { headers: _headers }).then((r) => { return r.status === 200 ? r.json() : false }).catch((e) => { return false });
if (!req_status) {
_headers.authorization = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw";
// ^ explicit content, but no multi media support
delete _headers["x-guest-token"]
req_act = await fetch(`${apiURL}/guest/activate.json`, {
method: "POST",
headers: _headers
}).then((r) => { return r.status === 200 ? r.json() : false}).catch(() => { return false });
if (!req_act) return { error: 'ErrorCouldntFetch' };
_headers["x-guest-token"] = req_act["guest_token"];
req_status = await fetch(showURL, { headers: _headers }).then((r) => { return r.status === 200 ? r.json() : false }).catch(() => { return false });
}
if (!req_status) return { error: 'ErrorCouldntFetch' };
if (!req_status["extended_entities"] || !req_status["extended_entities"]["media"]) return { error: 'ErrorNoVideosInTweet' };
let single, multiple = [], media = req_status["extended_entities"]["media"];
media = media.filter((i) => { if (i["type"] === "video" || i["type"] === "animated_gif") return true })
if (media.length > 1) {
for (let i in media) { multiple.push({type: "video", thumb: media[i]["media_url_https"], url: bestQuality(media[i]["video_info"]["variants"])}) }
} else if (media.length === 1) {
single = bestQuality(media[0]["video_info"]["variants"])
} else {
return { error: 'ErrorNoVideosInTweet' }
}
if (single) {
return { urls: single, filename: `twitter_${obj.id}.mp4`, audioFilename: `twitter_${obj.id}_audio` }
} else if (multiple) {
return { picker: multiple }
} else {
return { error: 'ErrorNoVideosInTweet' }
}
} else {
_headers["host"] = "twitter.com"
_headers["content-type"] = "application/json"
let query = {
variables: {"id": obj.spaceId,"isMetatagsQuery":true,"withSuperFollowsUserFields":true,"withDownvotePerspective":false,"withReactionsMetadata":false,"withReactionsPerspective":false,"withSuperFollowsTweetFields":true,"withReplays":true},
features: {"spaces_2022_h2_clipping":true,"spaces_2022_h2_spaces_communities":true,"verified_phone_label_enabled":false,"tweetypie_unmention_optimization_enabled":true,"responsive_web_uc_gql_enabled":true,"vibe_api_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":false,"interactive_text_enabled":true,"responsive_web_text_conversations_enabled":false,"responsive_web_enhance_cards_enabled":true}
}
query.variables = new URLSearchParams(JSON.stringify(query.variables)).toString().slice(0, -1);
query.features = new URLSearchParams(JSON.stringify(query.features)).toString().slice(0, -1);
query = `https://twitter.com/i/api/graphql/wJ5g4zf7v8qPHSQbaozYuw/AudioSpaceById?variables=${query.variables}&features=${query.features}`
let AudioSpaceById = await fetch(query, { headers: _headers }).then((r) => {return r.status === 200 ? r.json() : false}).catch((e) => { return false });
if (!AudioSpaceById) return { error: 'ErrorEmptyDownload' };
if (!AudioSpaceById.data.audioSpace.metadata) return { error: 'ErrorEmptyDownload' };
if (!AudioSpaceById.data.audioSpace.metadata.is_space_available_for_replay === true) return { error: 'TwitterSpaceWasntRecorded' };
let streamStatus = await fetch(
`https://twitter.com/i/api/1.1/live_video_stream/status/${AudioSpaceById.data.audioSpace.metadata.media_key}`, { headers: _headers }
).then((r) =>{ return r.status === 200 ? r.json() : false }).catch(() => { return false });
if (!streamStatus) return { error: 'ErrorCouldntFetch' };
let participants = AudioSpaceById.data.audioSpace.participants.speakers;
let listOfParticipants = `Twitter Space speakers: `;
for (let i in participants) { listOfParticipants += `@${participants[i]["twitter_screen_name"]}, ` }
listOfParticipants = listOfParticipants.slice(0, -2);
return {
urls: streamStatus.source.noRedirectPlaybackUrl,
audioFilename: `twitterspaces_${obj.spaceId}`,
isAudioOnly: true,
fileMetadata: {
title: AudioSpaceById.data.audioSpace.metadata.title,
artist: `Twitter Space by @${AudioSpaceById.data.audioSpace.metadata.creator_results.result.legacy.screen_name}`,
comment: listOfParticipants,
// cover: AudioSpaceById.data.audioSpace.metadata.creator_results.result.legacy.profile_image_url_https.replace("_normal", "")
}
}
}
}

77
src/modules/processing/services/vimeo.js Normal file
View file

@ -0,0 +1,77 @@
import { maxVideoDuration, quality, services } from "../../config.js";
export default async function(obj) {
let api = await fetch(`https://player.vimeo.com/video/${obj.id}/config`).then((r) => { return r.json() }).catch(() => { return false });
if (!api) return { error: 'ErrorCouldntFetch' };
let downloadType = "dash";
if (JSON.stringify(api).includes('"progressive":[{')) downloadType = "progressive";
switch(downloadType) {
case "progressive":
let all = api["request"]["files"]["progressive"].sort((a, b) => Number(b.width) - Number(a.width));
let best = all[0];
try {
if (obj.quality != "max") {
let pref = parseInt(quality[obj.quality], 10)
for (let i in all) {
let currQuality = parseInt(all[i]["quality"].replace('p', ''), 10)
if (currQuality === pref) {
best = all[i];
break
}
if (currQuality < pref) {
best = all[i-1];
break
}
}
}
} catch (e) {
best = all[0]
}
return { urls: best["url"], filename: `tumblr_${obj.id}.mp4` };
case "dash":
if (api.video.duration > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
let masterJSONURL = api["request"]["files"]["dash"]["cdns"]["akfire_interconnect_quic"]["url"];
let masterJSON = await fetch(masterJSONURL).then((r) => { return r.json() }).catch(() => { return false });
if (!masterJSON) return { error: 'ErrorCouldntFetch' };
if (!masterJSON.video) return { error: 'ErrorEmptyDownload' };
let type = "parcel";
if (masterJSON.base_url === "../") type = "chop";
let masterJSON_Video = masterJSON.video.sort((a, b) => Number(b.width) - Number(a.width));
let masterJSON_Audio = masterJSON.audio.sort((a, b) => Number(b.bitrate) - Number(a.bitrate)).filter((a)=> {if (a['mime_type'] === "audio/mp4") return true;});
let bestVideo = masterJSON_Video[0], bestAudio = masterJSON_Audio[0];
switch (type) {
case "parcel":
if (obj.quality != "max") {
let pref = parseInt(quality[obj.quality], 10)
for (let i in masterJSON_Video) {
let currQuality = parseInt(services.vimeo.resolutionMatch[masterJSON_Video[i]["width"]], 10)
if (currQuality < pref) {
break;
} else if (String(currQuality) === String(pref)) {
bestVideo = masterJSON_Video[i]
}
}
}
let baseUrl = masterJSONURL.split("/sep/")[0];
let videoUrl = `${baseUrl}/parcel/video/${bestVideo.index_segment.split('?')[0]}`,
audioUrl = `${baseUrl}/parcel/audio/${bestAudio.index_segment.split('?')[0]}`;
return { urls: [videoUrl, audioUrl], audioFilename: `vimeo_${obj.id}_audio`, filename: `vimeo_${obj.id}_${bestVideo["width"]}x${bestVideo["height"]}.mp4` }
case "chop": // TO-DO: support for chop stream type
default:
return { error: 'ErrorEmptyDownload' }
}
default:
return { error: 'ErrorEmptyDownload' }
}
}

45
src/modules/processing/services/vk.js Normal file
View file

@ -0,0 +1,45 @@
import { xml2json } from "xml-js";
import { genericUserAgent, maxVideoDuration, services } from "../../config.js";
import selectQuality from "../../stream/selectQuality.js";
export default async function(obj) {
let html;
html = await fetch(`https://vk.com/video-${obj.userId}_${obj.videoId}`, {
headers: { "user-agent": genericUserAgent }
}).then((r) => { return r.text() }).catch(() => { return false });
if (!html) return { error: 'ErrorCouldntFetch' };
if (!html.includes(`{"lang":`)) return { error: 'ErrorEmptyDownload' };
let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]);
if (!Number(js["mvData"]["is_active_live"]) === 0) return { error: 'ErrorLiveVideo' };
if (js["mvData"]["duration"] > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
let mpd = JSON.parse(xml2json(js["player"]["params"][0]["manifest"], { compact: true, spaces: 4 }));
let repr = mpd["MPD"]["Period"]["AdaptationSet"]["Representation"];
if (!mpd["MPD"]["Period"]["AdaptationSet"]["Representation"]) repr = mpd["MPD"]["Period"]["AdaptationSet"][0]["Representation"];
let selectedQuality,
attr = repr[repr.length - 1]["_attributes"],
qualities = Object.keys(services.vk.quality_match);
for (let i in qualities) {
if (qualities[i] === attr["height"]) {
selectedQuality = `url${attr["height"]}`;
break
}
if (qualities[i] === attr["width"]) {
selectedQuality = `url${attr["width"]}`;
break
}
}
let maxQuality = js["player"]["params"][0][selectedQuality].split('type=')[1].slice(0, 1);
let userQuality = selectQuality('vk', obj.quality, Object.entries(services.vk.quality_match).reduce((r, [k, v]) => { r[v] = k; return r; })[maxQuality]);
let userRepr = repr[services.vk.representation_match[userQuality]]["_attributes"];
if (!selectedQuality in js["player"]["params"][0]) return { error: 'ErrorEmptyDownload' };
return {
urls: js["player"]["params"][0][`url${userQuality}`],
filename: `vk_${obj.userId}_${obj.videoId}_${userRepr["width"]}x${userRepr['height']}.mp4`
}
}

88
src/modules/processing/services/youtube.js Normal file
View file

@ -0,0 +1,88 @@
import ytdl from "better-ytdl-core";
import { maxVideoDuration, quality as mq } from "../../config.js";
import selectQuality from "../../stream/selectQuality.js";
export default async function(obj) {
let isAudioOnly = !!obj.isAudioOnly,
infoInitial = await ytdl.getInfo(obj.id);
if (!infoInitial) return { error: 'ErrorCantConnectToServiceAPI' };
let info = infoInitial.formats;
if (info[0]["isLive"]) return { error: 'ErrorLiveVideo' };
let videoMatch = [], fullVideoMatch = [], video = [],
audio = info.filter((a) => {
if (!a["isHLS"] && !a["isDashMPD"] && a["hasAudio"] && !a["hasVideo"] && a["container"] == obj.format) return true
}).sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
if (audio.length === 0) return { error: 'ErrorBadFetch' };
if (audio[0]["approxDurationMs"] > maxVideoDuration) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
if (!isAudioOnly) {
video = info.filter((a) => {
if (!a["isHLS"] && !a["isDashMPD"] && a["hasVideo"] && a["container"] == obj.format) {
if (obj.quality != "max") {
if (a["hasAudio"] && mq[obj.quality] == a["height"]) {
fullVideoMatch.push(a)
} else if (!a["hasAudio"] && mq[obj.quality] == a["height"]) {
videoMatch.push(a)
}
}
return true
}
}).sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
if (obj.quality != "max") {
if (videoMatch.length === 0) {
let ss = selectQuality("youtube", obj.quality, video[0]["qualityLabel"].slice(0, 5).replace('p', '').trim());
videoMatch = video.filter((a) => {
if (a["qualityLabel"].slice(0, 5).replace('p', '').trim() == ss) return true
})
} else if (fullVideoMatch.length > 0) {
videoMatch = [fullVideoMatch[0]]
}
} else videoMatch = [video[0]];
if (obj.quality === "los") videoMatch = [video[video.length - 1]];
}
if (video.length === 0) isAudioOnly = true;
if (isAudioOnly) {
let r = {
type: "render",
isAudioOnly: true,
urls: audio[0]["url"],
audioFilename: `youtube_${obj.id}_audio`,
fileMetadata: {
title: infoInitial.videoDetails.title,
artist: infoInitial.videoDetails.ownerChannelName.replace("- Topic", "").trim(),
}
}
if (infoInitial.videoDetails.description) {
let isAutoGenAudio = infoInitial.videoDetails.description.startsWith("Provided to YouTube by");
if (isAutoGenAudio) {
let descItems = infoInitial.videoDetails.description.split("\n\n")
r.fileMetadata.album = descItems[2]
r.fileMetadata.copyright = descItems[3]
if (descItems[4].startsWith("Released on:")) r.fileMetadata.date = descItems[4].replace("Released on: ", '').trim();
}
}
return r
}
let singleTest;
if (videoMatch.length > 0) {
singleTest = videoMatch[0]["hasVideo"] && videoMatch[0]["hasAudio"];
return {
type: singleTest ? "bridge" : "render",
urls: singleTest ? videoMatch[0]["url"] : [videoMatch[0]["url"], audio[0]["url"]],
time: videoMatch[0]["approxDurationMs"],
filename: `youtube_${obj.id}_${videoMatch[0]["width"]}x${videoMatch[0]["height"]}.${obj.format}`
}
}
singleTest = video[0]["hasVideo"] && video[0]["hasAudio"];
return {
type: singleTest ? "bridge" : "render",
urls: singleTest ? video[0]["url"] : [video[0]["url"], audio[0]["url"]],
time: video[0]["approxDurationMs"],
filename: `youtube_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.${obj.format}`
}
}

1
src/modules/processing/servicesConfig.json
View file

@ -8,6 +8,7 @@
"enabled": true
},
"reddit": {
"alias": "reddit videos & gifs",
"patterns": ["r/:sub/comments/:id/:title"],
"enabled": true
},

29
src/modules/processing/servicesPatternTesters.js
View file

@ -1,27 +1,28 @@
export const testers = {
"twitter": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length < 20) || (patternMatch["spaceId"] && patternMatch["spaceId"].length === 13),
"twitter": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length < 20)
|| (patternMatch["spaceId"] && patternMatch["spaceId"].length === 13),
"vk": (patternMatch) => (patternMatch["userId"] && patternMatch["videoId"] &&
patternMatch["userId"].length <= 10 && patternMatch["videoId"].length === 9),
"vk": (patternMatch) => (patternMatch["userId"] && patternMatch["videoId"]
&& patternMatch["userId"].length <= 10 && patternMatch["videoId"].length === 9),
"bilibili": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length >= 12),
"youtube": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length >= 11),
"reddit": (patternMatch) => (patternMatch["sub"] && patternMatch["id"] && patternMatch["title"] &&
patternMatch["sub"].length <= 22 && patternMatch["id"].length <= 10 && patternMatch["title"].length <= 96),
"reddit": (patternMatch) => (patternMatch["sub"] && patternMatch["id"] && patternMatch["title"]
&& patternMatch["sub"].length <= 22 && patternMatch["id"].length <= 10 && patternMatch["title"].length <= 96),
"tiktok": (patternMatch) => ((patternMatch["user"] && patternMatch["postId"] && patternMatch["postId"].length <= 21) ||
(patternMatch["id"] && patternMatch["id"].length <= 13)),
"tiktok": (patternMatch) => ((patternMatch["user"] && patternMatch["postId"] && patternMatch["postId"].length <= 21)
|| (patternMatch["id"] && patternMatch["id"].length <= 13)),
"douyin": (patternMatch) => ((patternMatch["postId"] && patternMatch["postId"].length <= 21) ||
(patternMatch["id"] && patternMatch["id"].length <= 13)),
"douyin": (patternMatch) => ((patternMatch["postId"] && patternMatch["postId"].length <= 21)
|| (patternMatch["id"] && patternMatch["id"].length <= 13)),
"tumblr": (patternMatch) => ((patternMatch["id"] && patternMatch["id"].length < 21) ||
(patternMatch["id"] && patternMatch["id"].length < 21 && patternMatch["user"] && patternMatch["user"].length <= 32)),
"tumblr": (patternMatch) => ((patternMatch["id"] && patternMatch["id"].length < 21)
|| (patternMatch["id"] && patternMatch["id"].length < 21 && patternMatch["user"] && patternMatch["user"].length <= 32)),
"vimeo": (patternMatch) => ((patternMatch["id"] && patternMatch["id"].length <= 11)),
"soundcloud": (patternMatch) => ((patternMatch["author"] && patternMatch["song"] && (patternMatch["author"].length + patternMatch["song"].length) <= 96) ||
(patternMatch["shortLink"] && patternMatch["shortLink"].length <= 32))
};
"soundcloud": (patternMatch) => ((patternMatch["author"] && patternMatch["song"]
&& (patternMatch["author"].length + patternMatch["song"].length) <= 96) || (patternMatch["shortLink"] && patternMatch["shortLink"].length <= 32))
}

38
src/modules/services/bilibili.js
View file

@ -1,38 +0,0 @@
import { genericUserAgent, maxVideoDuration } from "../config.js";
// TO-DO: quality picking
export default async function(obj) {
try {
let html = await fetch(`https://bilibili.com/video/${obj.id}`, {
headers: {"user-agent": genericUserAgent}
}).then((r) => { return r.text() }).catch(() => { return false });
if (!html) {
return { error: 'ErrorCouldntFetch' };
}
if (!(html.includes('<script>window.__playinfo__=') && html.includes('"video_codecid"'))) {
return { error: 'ErrorEmptyDownload' };
}
let streamData = JSON.parse(html.split('<script>window.__playinfo__=')[1].split('</script>')[0]);
if (streamData.data.timelength > maxVideoDuration) {
return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
}
let video = streamData["data"]["dash"]["video"].filter((v) => {
if (!v["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/")) return true;
}).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth));
let audio = streamData["data"]["dash"]["audio"].filter((a) => {
if (!a["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/")) return true;
}).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth));
return {
urls: [video[0]["baseUrl"], audio[0]["baseUrl"]],
time: streamData.data.timelength,
audioFilename: `bilibili_${obj.id}_audio`,
filename: `bilibili_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.mp4`
};
} catch (e) {
return { error: 'ErrorBadFetch' };
}
}

40
src/modules/services/reddit.js
View file

@ -1,40 +0,0 @@
import { maxVideoDuration } from "../config.js";
// TO-DO: add support for gifs (#80)
export default async function(obj) {
try {
let data = await fetch(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}/${obj.name}.json`).then((r) => { return r.json() }).catch(() => { return false });
if (!data) {
return { error: 'ErrorCouldntFetch' };
}
data = data[0]["data"]["children"][0]["data"];
if (!"reddit_video" in data["secure_media"]) {
return { error: 'ErrorEmptyDownload' };
}
if (data["secure_media"]["reddit_video"]["duration"] * 1000 > maxVideoDuration) {
return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
}
let video = data["secure_media"]["reddit_video"]["fallback_url"].split('?')[0],
audio = video.match('.mp4')
? `${video.split('_')[0]}_audio.mp4`
: `${data["secure_media"]["reddit_video"]["fallback_url"].split('DASH')[0]}audio`;
await fetch(audio, {method: "HEAD"}).then((r) => {if (r.status != 200) audio = ''}).catch(() => {audio = ''});
let id = data["secure_media"]["reddit_video"]["fallback_url"].split('/')[3];
if (!audio.length > 0) {
return { typeId: 1, urls: video };
}
return {
typeId: 2,
type: "render",
urls: [video, audio],
audioFilename: `reddit_${id}_audio`,
filename: `reddit_${id}.mp4`
};
} catch (err) {
return { error: 'ErrorBadFetch' };
}
}

90
src/modules/services/soundcloud.js
View file

@ -1,90 +0,0 @@
import { genericUserAgent, maxAudioDuration } from "../config.js";
let cachedID = {}
async function findClientID() {
try {
let sc = await fetch('https://soundcloud.com/').then((r) => { return r.text() }).catch(() => { return false });
let scVersion = String(sc.match(/<script>window\.__sc_version="[0-9]{10}"<\/script>/)[0].match(/[0-9]{10}/));
if (cachedID.version == scVersion) {
return cachedID.id
}
let scripts = sc.matchAll(/<script.+src="(.+)">/g);
let clientid;
for (let script of scripts) {
let url = script[1];
if (url && !url.startsWith('https://a-v2.sndcdn.com')) return;
let scrf = await fetch(url).then((r) => {return r.text()}).catch(() => { return false });
let id = scrf.match(/\("client_id=[A-Za-z0-9]{32}"\)/);
if (id && typeof id[0] === 'string') {
clientid = id[0].match(/[A-Za-z0-9]{32}/)[0];
break;
}
}
cachedID.version = scVersion;
cachedID.id = clientid;
return clientid;
} catch (e) {
return false;
}
}
export default async function(obj) {
try {
let html;
if (!obj.author && !obj.song && obj.shortLink) {
html = await fetch(`https://soundcloud.app.goo.gl/${obj.shortLink}/`, {
headers: {"user-agent": genericUserAgent}
}).then((r) => { return r.text() }).catch(() => { return false });
}
if (obj.author && obj.song) {
html = await fetch(`https://soundcloud.com/${obj.author}/${obj.song}${obj.accessKey ? `/s-${obj.accessKey}` : ''}`, {
headers: {"user-agent": genericUserAgent}
}).then((r) => { return r.text() }).catch(() => { return false });
}
if (!html) {
return { error: 'ErrorCouldntFetch'};
}
if (!(html.includes('<script>window.__sc_hydration = ')
&& html.includes('"format":{"protocol":"progressive","mime_type":"audio/mpeg"},')
&& html.includes('{"hydratable":"sound","data":'))) {
return { error: ['ErrorBrokenLink', 'soundcloud'] }
}
let json = JSON.parse(html.split('{"hydratable":"sound","data":')[1].split('}];</script>')[0])
if (!json["media"]["transcodings"]) {
return { error: 'ErrorEmptyDownload' }
}
let clientId = await findClientID();
if (!clientId) {
return { error: 'ErrorSoundCloudNoClientId' }
}
let fileUrlBase = json.media.transcodings[0]["url"].replace("/hls", "/progressive")
let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`;
if (!fileUrl.substring(0, 54) === "https://api-v2.soundcloud.com/media/soundcloud:tracks:") {
return { error: 'ErrorEmptyDownload' }
}
if (json.duration > maxAudioDuration) {
return { error: ['ErrorLengthAudioConvert', maxAudioDuration / 60000] }
}
let file = await fetch(fileUrl).then(async (r) => { return (await r.json()).url }).catch(() => { return false });
if (!file) {
return { error: 'ErrorCouldntFetch' };
}
return {
urls: file,
audioFilename: `soundcloud_${json.id}`,
fileMetadata: {
title: json.title,
artist: json.user.username,
}
}
} catch (e) {
return { error: 'ErrorBadFetch' };
}
}

117
src/modules/services/tiktok.js
View file

@ -1,117 +0,0 @@
import { genericUserAgent } from "../config.js";
let userAgent = genericUserAgent.split(' Chrome/1')[0]
let config = {
tiktok: {
short: "https://vt.tiktok.com/",
api: "https://api2.musical.ly/aweme/v1/feed/?aweme_id={postId}&version_code=262&app_name=musical_ly&channel=App&device_id=null&os_version=14.4.2&device_platform=iphone&device_type=iPhone9&region=US&carrier_region=US",
},
douyin: {
short: "https://v.douyin.com/",
api: "https://www.iesdouyin.com/web/api/v2/aweme/iteminfo/?item_ids={postId}",
}
}
function selector(j, h, id) {
if (!j) return false
let t;
switch (h) {
case "tiktok":
t = j["aweme_list"].filter((v) => { if (v["aweme_id"] == id) return true })
break;
case "douyin":
t = j['item_list'].filter((v) => { if (v["aweme_id"] == id) return true })
break;
}
if (!t.length > 0) return false
return t[0]
}
export default async function(obj) {
try {
if (!obj.postId) {
let html = await fetch(`${config[obj.host]["short"]}${obj.id}`, {
redirect: "manual",
headers: { "user-agent": userAgent }
}).then((r) => { return r.text() }).catch(() => { return false });
if (!html) return { error: 'ErrorCouldntFetch' };
if (html.slice(0, 17) === '<a href="https://' && html.includes('/video/')) {
obj.postId = html.split('/video/')[1].split('?')[0].replace("/", '')
} else if (html.slice(0, 32) === '<a href="https://m.tiktok.com/v/' && html.includes('/v/')) {
obj.postId = html.split('/v/')[1].split('.html')[0].replace("/", '')
}
}
if (!obj.postId) {
return { error: 'ErrorCantGetID' };
}
let detail;
detail = await fetch(config[obj.host]["api"].replace("{postId}", obj.postId), {
headers: {"user-agent": "TikTok 26.2.0 rv:262018 (iPhone; iOS 14.4.2; en_US) Cronet"}
}).then((r) => { return r.json() }).catch(() => { return false });
detail = selector(detail, obj.host, obj.postId);
if (!detail) return { error: 'ErrorCouldntFetch' }
let video, videoFilename, audioFilename, isMp3, audio, images,
filenameBase = `${obj.host}_${obj.postId}`;
if (obj.host == "tiktok") {
images = detail["image_post_info"] ? detail["image_post_info"]["images"] : false
} else {
images = detail["images"] ? detail["images"] : false
}
if (!obj.isAudioOnly && !images) {
video = obj.host === "tiktok" ? detail["video"]["download_addr"]["url_list"][0] : detail['video']['play_addr']['url_list'][0]
videoFilename = `${filenameBase}_video.mp4`
if (obj.noWatermark) {
video = obj.host === "tiktok" ? detail["video"]["play_addr"]["url_list"][0] : detail["video"]["play_addr"]["url_list"][0].replace("playwm", "play");
videoFilename = `${filenameBase}_video_nw.mp4` // nw - no watermark
}
} else {
let fallback = obj.host === "douyin" ? detail["video"]["play_addr"]["url_list"][0].replace("playwm", "play") : detail["video"]["play_addr"]["url_list"][0];
audio = fallback;
audioFilename = `${filenameBase}_audio_fv`; // fv - from video
if (obj.fullAudio || fallback.includes("music")) {
audio = detail["music"]["play_url"]["url_list"][0]
audioFilename = `${filenameBase}_audio`
}
if (audio.slice(-4) === ".mp3") isMp3 = true;
}
if (video) return {
urls: video,
filename: videoFilename
}
if (images && obj.isAudioOnly) {
return {
urls: audio,
audioFilename: audioFilename,
isAudioOnly: true,
isMp3: isMp3,
}
}
if (images) {
let imageLinks = [];
for (let i in images) {
let sel = obj.host == "tiktok" ? images[i]["display_image"]["url_list"] : images[i]["url_list"];
sel = sel.filter((p) => { if (p.includes(".jpeg?")) return true; })
imageLinks.push({url: sel[0]})
}
return {
picker: imageLinks,
urls: audio,
audioFilename: audioFilename,
isAudioOnly: true,
isMp3: isMp3,
}
}
if (audio) return {
urls: audio,
audioFilename: audioFilename,
isAudioOnly: true,
isMp3: isMp3,
}
} catch (e) {
return { error: 'ErrorBadFetch' };
}
}

17
src/modules/services/tumblr.js
View file

@ -1,17 +0,0 @@
import { genericUserAgent } from "../config.js";
export default async function(obj) {
try {
let user = obj.user ? obj.user : obj.url.split('.')[0].replace('https://', '');
let html = await fetch(`https://${user}.tumblr.com/post/${obj.id}`, {
headers: {"user-agent": genericUserAgent}
}).then((r) => { return r.text() }).catch(() => { return false });
if (!html) return { error: 'ErrorCouldntFetch' };
if (!html.includes('property="og:video" content="https://va.media.tumblr.com/')) {
return { error: 'ErrorEmptyDownload' }
}
return { urls: `https://va.media.tumblr.com/${html.split('property="og:video" content="https://va.media.tumblr.com/')[1].split('"')[0]}`, audioFilename: `tumblr_${obj.id}_audio` }
} catch (e) {
return { error: 'ErrorBadFetch' };
}
}

100
src/modules/services/twitter.js
View file

@ -1,100 +0,0 @@
import { genericUserAgent } from "../config.js";
function bestQuality(arr) {
return arr.filter((v) => { if (v["content_type"] === "video/mp4") return true; }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate))[0]["url"].split("?")[0]
}
const apiURL = "https://api.twitter.com/1.1"
export default async function(obj) {
try {
let _headers = {
"user-agent": genericUserAgent,
"authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
"host": "api.twitter.com"
};
let req_act = await fetch(`${apiURL}/guest/activate.json`, {
method: "POST",
headers: _headers
}).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false});
if (!req_act) return { error: 'ErrorCouldntFetch' };
_headers["x-guest-token"] = req_act["guest_token"];
let showURL = `${apiURL}/statuses/show/${obj.id}.json?tweet_mode=extended&include_user_entities=0&trim_user=1&include_entities=0&cards_platform=Web-12&include_cards=1`
if (!obj.spaceId) {
let req_status = await fetch(showURL, { headers: _headers }).then((r) => { return r.status == 200 ? r.json() : false;}).catch((e) => { return false});
if (!req_status) {
_headers.authorization = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw";
delete _headers["x-guest-token"]
req_act = await fetch(`${apiURL}/guest/activate.json`, {
method: "POST",
headers: _headers
}).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false});
if (!req_act) return { error: 'ErrorCouldntFetch' };
_headers["x-guest-token"] = req_act["guest_token"];
req_status = await fetch(showURL, { headers: _headers }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false});
}
if (!req_status) return { error: 'ErrorCouldntFetch' }
if (!req_status["extended_entities"] && req_status["extended_entities"]["media"]) {
return { error: 'ErrorNoVideosInTweet' }
}
let single, multiple = [], media = req_status["extended_entities"]["media"];
media = media.filter((i) => { if (i["type"] === "video" || i["type"] === "animated_gif") return true })
if (media.length > 1) {
for (let i in media) { multiple.push({type: "video", thumb: media[i]["media_url_https"], url: bestQuality(media[i]["video_info"]["variants"])}) }
} else if (media.length === 1) {
single = bestQuality(media[0]["video_info"]["variants"])
} else {
return { error: 'ErrorNoVideosInTweet' }
}
if (single) {
return { urls: single, filename: `twitter_${obj.id}.mp4`, audioFilename: `twitter_${obj.id}_audio` }
} else if (multiple) {
return { picker: multiple }
} else {
return { error: 'ErrorNoVideosInTweet' }
}
} else {
_headers["host"] = "twitter.com"
_headers["content-type"] = "application/json"
let query = {
variables: {"id": obj.spaceId,"isMetatagsQuery":true,"withSuperFollowsUserFields":true,"withDownvotePerspective":false,"withReactionsMetadata":false,"withReactionsPerspective":false,"withSuperFollowsTweetFields":true,"withReplays":true}, features: {"spaces_2022_h2_clipping":true,"spaces_2022_h2_spaces_communities":true,"verified_phone_label_enabled":false,"tweetypie_unmention_optimization_enabled":true,"responsive_web_uc_gql_enabled":true,"vibe_api_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":false,"interactive_text_enabled":true,"responsive_web_text_conversations_enabled":false,"responsive_web_enhance_cards_enabled":true}
}
let AudioSpaceById = await fetch(`https://twitter.com/i/api/graphql/wJ5g4zf7v8qPHSQbaozYuw/AudioSpaceById?variables=${new URLSearchParams(JSON.stringify(query.variables)).toString().slice(0, -1)}&features=${new URLSearchParams(JSON.stringify(query.features)).toString().slice(0, -1)}`, { headers: _headers }).then((r) => {
return r.status == 200 ? r.json() : false;
}).catch((e) => {return false});
if (!AudioSpaceById) {
return { error: 'ErrorEmptyDownload' }
}
if (!AudioSpaceById.data.audioSpace.metadata.is_space_available_for_replay === true) {
return { error: 'TwitterSpaceWasntRecorded' };
}
let streamStatus = await fetch(`https://twitter.com/i/api/1.1/live_video_stream/status/${AudioSpaceById.data.audioSpace.metadata.media_key}`,
{ headers: _headers }).then((r) =>{return r.status == 200 ? r.json() : false;}).catch(() => {return false;});
if (!streamStatus) return { error: 'ErrorCouldntFetch' };
let participants = AudioSpaceById.data.audioSpace.participants.speakers
let listOfParticipants = `Twitter Space speakers: `
for (let i in participants) {
listOfParticipants += `@${participants[i]["twitter_screen_name"]}, `
}
listOfParticipants = listOfParticipants.slice(0, -2);
return {
urls: streamStatus.source.noRedirectPlaybackUrl,
audioFilename: `twitterspaces_${obj.spaceId}`,
isAudioOnly: true,
fileMetadata: {
title: AudioSpaceById.data.audioSpace.metadata.title,
artist: `Twitter Space by @${AudioSpaceById.data.audioSpace.metadata.creator_results.result.legacy.screen_name}`,
comment: listOfParticipants,
// cover: AudioSpaceById.data.audioSpace.metadata.creator_results.result.legacy.profile_image_url_https.replace("_normal", "")
}
}
}
} catch (err) {
return { error: 'ErrorBadFetch' };
}
}

84
src/modules/services/vimeo.js
View file

@ -1,84 +0,0 @@
import { maxVideoDuration, quality, services } from "../config.js";
export default async function(obj) {
try {
let api = await fetch(`https://player.vimeo.com/video/${obj.id}/config`).then((r) => {return r.json()}).catch(() => {return false});
if (!api) return { error: 'ErrorCouldntFetch' };
let downloadType = "dash";
if (JSON.stringify(api).includes('"progressive":[{')) {
downloadType = "progressive";
}
switch(downloadType) {
case "progressive":
let all = api["request"]["files"]["progressive"].sort((a, b) => Number(b.width) - Number(a.width));
let best = all[0]
try {
if (obj.quality != "max") {
let pref = parseInt(quality[obj.quality], 10)
for (let i in all) {
let currQuality = parseInt(all[i]["quality"].replace('p', ''), 10)
if (currQuality === pref) {
best = all[i];
break
}
if (currQuality < pref) {
best = all[i-1];
break
}
}
}
} catch (e) {
best = all[0]
}
return { urls: best["url"], filename: `tumblr_${obj.id}.mp4` };
case "dash":
if (api.video.duration > maxVideoDuration / 1000) {
return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
}
let masterJSONURL = api["request"]["files"]["dash"]["cdns"]["akfire_interconnect_quic"]["url"];
let masterJSON = await fetch(masterJSONURL).then((r) => {return r.json()}).catch(() => {return false});
if (!masterJSON) return { error: 'ErrorCouldntFetch' };
if (!masterJSON.video) {
return { error: 'ErrorEmptyDownload' }
}
let type = "parcel";
if (masterJSON.base_url == "../") {
type = "chop"
}
let masterJSON_Video = masterJSON.video.sort((a, b) => Number(b.width) - Number(a.width));
let masterJSON_Audio = masterJSON.audio.sort((a, b) => Number(b.bitrate) - Number(a.bitrate)).filter((a)=> {if (a['mime_type'] === "audio/mp4") return true;});
let bestVideo = masterJSON_Video[0]
let bestAudio = masterJSON_Audio[0]
switch (type) {
case "parcel":
if (obj.quality != "max") {
let pref = parseInt(quality[obj.quality], 10)
for (let i in masterJSON_Video) {
let currQuality = parseInt(services.vimeo.resolutionMatch[masterJSON_Video[i]["width"]], 10)
if (currQuality < pref) {
break;
} else if (currQuality == pref) {
bestVideo = masterJSON_Video[i]
}
}
}
let baseUrl = masterJSONURL.split("/sep/")[0]
let videoUrl = `${baseUrl}/parcel/video/${bestVideo.index_segment.split('?')[0]}`;
let audioUrl = `${baseUrl}/parcel/audio/${bestAudio.index_segment.split('?')[0]}`;
return { urls: [videoUrl, audioUrl], audioFilename: `vimeo_${obj.id}_audio`, filename: `vimeo_${obj.id}_${bestVideo["width"]}x${bestVideo["height"]}.mp4` }
case "chop": // TO-DO: support chop type of streams
default:
return { error: 'ErrorEmptyDownload' }
}
default:
return { error: 'ErrorEmptyDownload' }
}
} catch (e) {
return { error: 'ErrorBadFetch' };
}
}

54
src/modules/services/vk.js
View file

@ -1,54 +0,0 @@
import { xml2json } from "xml-js";
import { genericUserAgent, maxVideoDuration, services } from "../config.js";
import selectQuality from "../stream/selectQuality.js";
export default async function(obj) {
try {
let html;
html = await fetch(`https://vk.com/video-${obj.userId}_${obj.videoId}`, {
headers: {"user-agent": genericUserAgent}
}).then((r) => {return r.text()}).catch(() => {return false});
if (!html) return { error: 'ErrorCouldntFetch' };
if (!html.includes(`{"lang":`)) {
return { error: 'ErrorEmptyDownload' };
}
let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]);
if (!js["mvData"]["is_active_live"] == '0') {
return { error: 'ErrorLiveVideo' };
}
if (js["mvData"]["duration"] > maxVideoDuration / 1000) {
return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
}
let mpd = JSON.parse(xml2json(js["player"]["params"][0]["manifest"], { compact: true, spaces: 4 }));
let repr = mpd["MPD"]["Period"]["AdaptationSet"]["Representation"];
if (!mpd["MPD"]["Period"]["AdaptationSet"]["Representation"]) {
repr = mpd["MPD"]["Period"]["AdaptationSet"][0]["Representation"];
}
let attr = repr[repr.length - 1]["_attributes"];
let selectedQuality;
let qualities = Object.keys(services.vk.quality_match);
for (let i in qualities) {
if (qualities[i] == attr["height"]) {
selectedQuality = `url${attr["height"]}`;
break;
}
if (qualities[i] == attr["width"]) {
selectedQuality = `url${attr["width"]}`;
break;
}
}
let maxQuality = js["player"]["params"][0][selectedQuality].split('type=')[1].slice(0, 1)
let userQuality = selectQuality('vk', obj.quality, Object.entries(services.vk.quality_match).reduce((r, [k, v]) => { r[v] = k; return r; })[maxQuality]);
let userRepr = repr[services.vk.representation_match[userQuality]]["_attributes"];
if (!selectedQuality in js["player"]["params"][0]) {
return { error: 'ErrorEmptyDownload' };
}
return {
urls: js["player"]["params"][0][`url${userQuality}`],
filename: `vk_${obj.userId}_${obj.videoId}_${userRepr["width"]}x${userRepr['height']}.mp4`
};
} catch (err) {
return { error: 'ErrorBadFetch' };
}
}

93
src/modules/services/youtube.js
View file

@ -1,93 +0,0 @@
import ytdl from "better-ytdl-core";
import { maxVideoDuration, quality as mq } from "../config.js";
import selectQuality from "../stream/selectQuality.js";
export default async function(obj) {
try {
let infoInitial = await ytdl.getInfo(obj.id);
if (!infoInitial) {
return { error: 'ErrorCantConnectToServiceAPI' };
}
let info = infoInitial.formats;
if (info[0]["isLive"]) {
return { error: 'ErrorLiveVideo' };
}
let videoMatch = [], fullVideoMatch = [], video = [], audio = info.filter((a) => {
if (!a["isHLS"] && !a["isDashMPD"] && a["hasAudio"] && !a["hasVideo"] && a["container"] == obj.format) return true;
}).sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
if (!obj.isAudioOnly) {
video = info.filter((a) => {
if (!a["isHLS"] && !a["isDashMPD"] && a["hasVideo"] && a["container"] == obj.format) {
if (obj.quality != "max") {
if (a["hasAudio"] && mq[obj.quality] == a["height"]) {
fullVideoMatch.push(a)
} else if (!a["hasAudio"] && mq[obj.quality] == a["height"]) {
videoMatch.push(a);
}
}
return true
}
}).sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
if (obj.quality != "max") {
if (videoMatch.length == 0) {
let ss = selectQuality("youtube", obj.quality, video[0]["qualityLabel"].slice(0, 5).replace('p', '').trim())
videoMatch = video.filter((a) => {
if (a["qualityLabel"].slice(0, 5).replace('p', '').trim() == ss) return true;
})
} else if (fullVideoMatch.length > 0) {
videoMatch = [fullVideoMatch[0]]
}
} else videoMatch = [video[0]];
if (obj.quality == "los") videoMatch = [video[video.length - 1]];
}
let generalMeta = {
title: infoInitial.videoDetails.title,
artist: infoInitial.videoDetails.ownerChannelName.replace("- Topic", "").trim(),
}
if (audio[0]["approxDurationMs"] > maxVideoDuration) {
return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
}
if (!obj.isAudioOnly && videoMatch.length > 0) {
if (video.length === 0 && audio.length === 0) {
return { error: 'ErrorBadFetch' };
}
if (videoMatch[0]["hasVideo"] && videoMatch[0]["hasAudio"]) {
return {
type: "bridge", urls: videoMatch[0]["url"], time: videoMatch[0]["approxDurationMs"],
filename: `youtube_${obj.id}_${videoMatch[0]["width"]}x${videoMatch[0]["height"]}.${obj.format}`
};
}
return {
type: "render", urls: [videoMatch[0]["url"], audio[0]["url"]], time: videoMatch[0]["approxDurationMs"],
filename: `youtube_${obj.id}_${videoMatch[0]["width"]}x${videoMatch[0]["height"]}.${obj.format}`
};
} else if (!obj.isAudioOnly) {
return {
type: "render", urls: [video[0]["url"], audio[0]["url"]], time: video[0]["approxDurationMs"],
filename: `youtube_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.${video[0]["container"]}`
};
} else if (audio.length > 0) {
let r = {
type: "render",
isAudioOnly: true,
urls: audio[0]["url"],
audioFilename: `youtube_${obj.id}_audio`,
fileMetadata: generalMeta
};
if (infoInitial.videoDetails.description) {
let isAutoGenAudio = infoInitial.videoDetails.description.startsWith("Provided to YouTube by");
if (isAutoGenAudio) {
let descItems = infoInitial.videoDetails.description.split("\n\n")
r.fileMetadata.album = descItems[2]
r.fileMetadata.copyright = descItems[3]
if (descItems[4].startsWith("Released on:")) r.fileMetadata.date = descItems[4].replace("Released on: ", '').trim();
}
}
return r
} else {
return { error: 'ErrorBadFetch' };
}
} catch (e) {
return { error: 'ErrorBadFetch' };
}
}

3
src/modules/stream/manage.js
View file

@ -47,7 +47,8 @@ export function verifyStream(ip, id, hmac, exp) {
return { error: 'this stream token does not exist', status: 400 };
}
let ghmac = sha256(`${id},${streamInfo.service},${ip},${exp}`, salt);
if (hmac == ghmac && exp.toString() == streamInfo.exp && ghmac == streamInfo.hmac && ip == streamInfo.ip && exp > Math.floor(new Date().getTime())) {
if (String(hmac) === ghmac && String(exp) === String(streamInfo.exp) && ghmac === String(streamInfo.hmac)
&& String(ip) === streamInfo.ip && Number(exp) > Math.floor(new Date().getTime())) {
return streamInfo;
}
return { error: 'Unauthorized', status: 401 };

4
src/modules/stream/selectQuality.js
View file

@ -8,12 +8,12 @@ function closest(goal, array) {
}
export default function(service, quality, maxQuality) {
if (quality == "max") return maxQuality;
if (quality === "max") return maxQuality;
quality = parseInt(mq[quality], 10)
maxQuality = parseInt(maxQuality, 10)
if (quality >= maxQuality || quality == maxQuality) return maxQuality;
if (quality >= maxQuality || quality === maxQuality) return maxQuality;
if (quality < maxQuality) {
if (!services[service]["quality"][quality]) {

2
src/modules/stream/types.js
View file

@ -116,7 +116,7 @@ export function streamVideoOnly(streamInfo, res) {
'-i', streamInfo.urls,
'-c', 'copy', '-an'
]
if (format == "mp4") args.push('-movflags', 'faststart+frag_keyframe+empty_moov')
if (format === "mp4") args.push('-movflags', 'faststart+frag_keyframe+empty_moov')
args.push('-f', format, 'pipe:3');
const ffmpegProcess = spawn(ffmpeg, args, {
windowsHide: true,

19
src/modules/sub/currentCommit.js
View file

@ -1,10 +1,23 @@
import { execSync } from "child_process";
let commit, commitInfo, branch;
export function shortCommit() {
return execSync('git rev-parse --short HEAD').toString().trim()
if (commit) return commit;
let c = execSync('git rev-parse --short HEAD').toString().trim();
commit = c;
return c
}
export function getCommitInfo() {
let d = execSync(`git show -s --format='%s;;;%B'`).toString().trim().replace(/[\r\n]/gm, '\n').split(';;;')
d[1] = d[1].replace(d[0], '').trim().toString().replace(/[\r\n]/gm, '<br>')
if (commitInfo) return commitInfo;
let d = execSync(`git show -s --format='%s;;;%B'`).toString().trim().replace(/[\r\n]/gm, '\n').split(';;;');
d[1] = d[1].replace(d[0], '').trim().toString().replace(/[\r\n]/gm, '<br>');
commitInfo = d;
return d
}
export function getCurrentBranch() {
if (branch) return branch;
let b = execSync('git branch --show-current').toString().trim();
branch = b;
return b
}

5
src/modules/sub/errors.js
View file

@ -3,6 +3,9 @@ import loc from "../../localization/manager.js";
export function errorUnsupported(lang) {
return loc(lang, 'ErrorUnsupported');
}
export function genericError(lang, host) {
export function brokenLink(lang, host) {
return loc(lang, 'ErrorBrokenLink', host);
}
export function genericError(lang, host) {
return loc(lang, 'ErrorBadFetch', host);
}

15
src/modules/sub/utils.js
View file

@ -103,10 +103,9 @@ export function checkJSONPost(obj) {
}
try {
let objKeys = Object.keys(obj);
if (!(objKeys.length < 8 && obj.url)) {
return false
}
if (!(objKeys.length <= 8 && obj.url)) return false;
let defKeys = Object.keys(def);
for (let i in objKeys) {
if (String(objKeys[i]) !== "url" && defKeys.includes(objKeys[i])) {
if (apiVar.booleanOnly.includes(objKeys[i])) {
@ -116,12 +115,14 @@ export function checkJSONPost(obj) {
}
}
}
obj["url"] = decodeURIComponent(String(obj["url"]))
obj["url"] = decodeURIComponent(String(obj["url"]));
let hostname = obj["url"].replace("https://", "").replace(' ', '').split('&')[0].split("/")[0].split("."),
host = hostname[hostname.length - 2]
def["url"] = encodeURIComponent(cleanURL(obj["url"], host))
host = hostname[hostname.length - 2];
def["url"] = encodeURIComponent(cleanURL(obj["url"], host));
return def
} catch (e) {
return false;
return false
}
}

256
src/test/services.json Normal file
View file

@ -0,0 +1,256 @@
{
"twitter": [{
"name": "regular video",
"url": "https://twitter.com/TwitterSpaces/status/1526955853743546372?s=20",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "redirect"
}
}, {
"name": "embedded twitter video",
"url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "redirect"
}
}, {
"name": "mixed media (image + gif)",
"url": "https://twitter.com/Twitter/status/1580661436132757506?s=20",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "redirect"
}
}, {
"name": "picker: mixed media (3 gifs + image)",
"url": "https://twitter.com/emerald_pedrod/status/1582418163521581063?s=20",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "picker"
}
}, {
"name": "audio from embedded twitter video (mp3, isAudioOnly)",
"url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
"params": {
"aFormat": "mp3",
"isAudioOnly": true,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "audio from embedded twitter video (best, isAudioOnly)",
"url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
"params": {
"aFormat": "best",
"isAudioOnly": true,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "audio from embedded twitter video (ogg, isAudioOnly, isAudioMuted)",
"url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
"params": {
"aFormat": "best",
"isAudioOnly": true,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "muted embedded twitter video",
"url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "inexistent post",
"url": "https://twitter.com/test/status/9487653",
"params": {
"aFormat": "best",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 400,
"status": "error"
}
}, {
"name": "post with no media content",
"url": "https://twitter.com/elonmusk/status/1604617643973124097?s=20",
"params": {
"aFormat": "best",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 400,
"status": "error"
}
}, {
"name": "recorded space by nyc (best)",
"url": "https://twitter.com/i/spaces/1gqxvyLoYQkJB",
"params": {
"aFormat": "best",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "recorded space by nyc (mp3)",
"url": "https://twitter.com/i/spaces/1gqxvyLoYQkJB",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "recorded space by nyc (wav, isAudioMuted)",
"url": "https://twitter.com/i/spaces/1gqxvyLoYQkJB",
"params": {
"aFormat": "wav",
"isAudioOnly": false,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "recorded space by service95 & dualipa (mp3, isAudioMuted, isAudioOnly)",
"url": "https://twitter.com/i/spaces/1nAJErvvVXgxL",
"params": {
"aFormat": "mp3",
"isAudioOnly": true,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "unavailable space",
"url": "https://twitter.com/i/spaces/1OwGWwjRjVVGQ?s=20",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 400,
"status": "error"
}
}, {
"name": "inexistent space",
"url": "https://twitter.com/i/spaces/10Wkie2j29iiI",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 400,
"status": "error"
}
}],
"soundcloud": [{
"name": "public song (best)",
"url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing",
"params": {
"aFormat": "best",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "public song (mp3, isAudioMuted)",
"url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "private song",
"url": "https://soundcloud.com/4kayy/unhappy-new-year-prod4kay/s-9bKbvwLdRWG",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "private song (wav, isAudioMuted)",
"url": "https://soundcloud.com/4kayy/unhappy-new-year-prod4kay/s-9bKbvwLdRWG",
"params": {
"aFormat": "wav",
"isAudioOnly": false,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "private song (ogg, isAudioMuted, isAudioOnly)",
"url": "https://soundcloud.com/4kayy/unhappy-new-year-prod4kay/s-9bKbvwLdRWG",
"params": {
"aFormat": "ogg",
"isAudioOnly": true,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}]
}

66
src/test/test.js Normal file
View file

@ -0,0 +1,66 @@
import "dotenv/config";
import { getJSON } from "../modules/api.js";
import { services } from "../modules/config.js";
import loadJSON from "../modules/sub/loadJSON.js";
import { checkJSONPost } from "../modules/sub/utils.js";
let tests = loadJSON('./src/test/services.json');
let noTest = [];
let failed = [];
let success = 0;
function addToFail(service, testName, url, response) {
failed.push({
service: service,
name: testName,
url: url,
response: response
})
}
for (let i in services) {
if (tests[i]) {
console.log(`\nRunning tests for ${i}...\n`)
for (let k = 0; k < tests[i].length; k++) {
let test = tests[i][k];
console.log(`Running test ${k+1}: ${test.name}`);
console.log('params:');
let params = {...{url: test.url}, ...test.params};
console.log(params);
let chck = checkJSONPost(params);
if (chck) {
chck["ip"] = "d21ec524bc2ade41bef569c0361ac57728c69e2764b5cb3cb310fe36568ca53f"; // random sha256
let j = await getJSON(chck["url"], "en", chck);
console.log('\nReceived:');
console.log(j)
if (j.status === test.expected.code && j.body.status === test.expected.status) {
console.log("\n✅ Success.\n");
success++
} else {
console.log(`\n❌ Fail. Expected: ${test.expected.code} & ${test.expected.status}, received: ${j.status} & ${j.body.status}\n`);
addToFail(i, test.name, test.url, j)
}
} else {
console.log("\n❌ couldn't validate the request JSON.\n");
addToFail(i, test.name, test.url, {})
}
}
console.log("\n\n")
} else {
console.warn(`No tests found for ${i}.`);
noTest.push(i)
}
}
console.log(`\n✅ ${success} tests succeeded.`);
console.log(`❌ ${failed.length} tests failed.`);
console.log(`❔ ${noTest.length} services weren't tested.`);
console.log(`\nFailed tests:`);
console.log(failed)
console.log(`\nMissing tests:`);
console.log(noTest)