Nikurasu/cobalt
1
0
Fork 0
Code Pull requests Activity

merge 5.0 from wukko/develop

Browse source 
stable 5.0
This commit is contained in:
wukko 2023-02-13 20:44:27 +06:00 committed by GitHub
commit 7911a35eb6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
47 changed files with 1771 additions and 936 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.deepsource.toml
View file

@ -1,5 +1,11 @@
version = 1
test_patterns = [
"src/test/test.js"
]
[[analyzers]]
name = "javascript"
enabled = true
[analyzers.meta]
environment = ["nodejs"]

5
package.json
View file

@ -1,7 +1,7 @@
{
"name": "cobalt",
"description": "save what you love",
"version": "4.8",
"version": "5.0",
"author": "wukko",
"exports": "./src/cobalt.js",
"type": "module",
@ -10,7 +10,8 @@
},
"scripts": {
"start": "node src/cobalt",
"setup": "node src/modules/setup"
"setup": "node src/modules/setup",
"test": "node src/test/test"
},
"repository": {
"type": "git",

47
src/cobalt.js
View file

@ -1,12 +1,12 @@
import "dotenv/config"
import "dotenv/config";
import express from "express";
import cors from "cors";
import * as fs from "fs";
import rateLimit from "express-rate-limit";
import { shortCommit } from "./modules/sub/currentCommit.js";
import { appName, genericUserAgent, version, internetExplorerRedirect } from "./modules/config.js";
import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js";
import { appName, genericUserAgent, version } from "./modules/config.js";
import { getJSON } from "./modules/api.js";
import renderPage from "./modules/pageRender/page.js";
import { apiJSON, checkJSONPost, languageCode } from "./modules/sub/utils.js";
@ -18,13 +18,14 @@ import { changelogHistory } from "./modules/pageRender/onDemand.js";
import { sha256 } from "./modules/sub/crypto.js";
const commitHash = shortCommit();
const branch = getCurrentBranch();
const app = express();
app.disable('x-powered-by');
if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt && process.env.port) {
const apiLimiter = rateLimit({
windowMs: 1 * 60 * 1000,
windowMs: 60000,
max: 12,
standardHeaders: true,
legacyHeaders: false,
@ -33,7 +34,7 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt &&
}
});
const apiLimiterStream = rateLimit({
windowMs: 1 * 60 * 1000,
windowMs: 60000,
max: 12,
standardHeaders: true,
legacyHeaders: false,
@ -57,18 +58,25 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt &&
}
next();
});
app.use((req, res, next) => {
if (req.header("user-agent") && req.header("user-agent").includes("Trident")) {
res.destroy()
}
next();
});
app.use('/api/json', express.json({
verify: (req, res, buf) => {
try {
JSON.parse(buf);
if (buf.length > 720) throw new Error();
if (req.header('Content-Type') != "application/json") res.status(500).json({ 'status': 'error', 'text': 'invalid content type header' })
if (req.header('Accept') != "application/json") res.status(500).json({ 'status': 'error', 'text': 'invalid accept header' })
if (String(req.header('Content-Type')) !== "application/json") res.status(500).json({ 'status': 'error', 'text': 'invalid content type header' })
if (String(req.header('Accept')) !== "application/json") res.status(500).json({ 'status': 'error', 'text': 'invalid accept header' })
} catch(e) {
res.status(500).json({ 'status': 'error', 'text': 'invalid json body.' })
}
}
}));
app.post('/api/:type', cors({ origin: process.env.selfURL, optionsSuccessStatus: 200 }), async (req, res) => {
try {
let ip = sha256(req.header('x-forwarded-for') ? req.header('x-forwarded-for') : req.ip.replace('::ffff:', ''), process.env.streamSalt);
@ -82,10 +90,10 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt &&
let j = await getJSON(chck["url"], languageCode(req), chck)
res.status(j.status).json(j.body);
} else if (request.url && !chck) {
let j = apiJSON(3, { t: loc(languageCode(req), 'ErrorCouldntFetch') });
let j = apiJSON(0, { t: loc(languageCode(req), 'ErrorCouldntFetch') });
res.status(j.status).json(j.body);
} else {
let j = apiJSON(3, { t: loc(languageCode(req), 'ErrorNoLink') })
let j = apiJSON(0, { t: loc(languageCode(req), 'ErrorNoLink') })
res.status(j.status).json(j.body);
}
} catch (e) {
@ -101,12 +109,16 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt &&
res.status(500).json({ 'status': 'error', 'text': loc(languageCode(req), 'ErrorCantProcess') })
}
});
app.get('/api/:type', cors({ origin: process.env.selfURL, optionsSuccessStatus: 200 }), (req, res) => {
try {
let ip = sha256(req.header('x-forwarded-for') ? req.header('x-forwarded-for') : req.ip.replace('::ffff:', ''), process.env.streamSalt);
switch (req.params.type) {
case 'json':
res.status(405).json({ 'status': 'error', 'text': 'GET method for this request has been deprecated. see https://github.com/wukko/cobalt/blob/current/docs/API.md for up-to-date API documentation.' });
res.status(405).json({
'status': 'error',
'text': 'GET method for this endpoint has been deprecated. see https://github.com/wukko/cobalt/blob/current/docs/API.md for up-to-date API documentation.'
});
break;
case 'stream':
if (req.query.p) {
@ -146,24 +158,18 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt &&
res.status(500).json({ 'status': 'error', 'text': loc(languageCode(req), 'ErrorCantProcess') })
}
});
app.get("/api", (req, res) => {
res.redirect('/api/json')
});
app.get("/", (req, res) => {
if (req.header("user-agent") && req.header("user-agent").includes("Trident")) {
if (internetExplorerRedirect.newNT.includes(req.header("user-agent").split('NT ')[1].split(';')[0])) {
res.redirect(internetExplorerRedirect.new)
} else {
res.redirect(internetExplorerRedirect.old)
}
} else {
res.send(renderPage({
"hash": commitHash,
"type": "default",
"lang": languageCode(req),
"useragent": req.header('user-agent') ? req.header('user-agent') : genericUserAgent
"useragent": req.header('user-agent') ? req.header('user-agent') : genericUserAgent,
"branch": branch
}))
}
});
app.get("/favicon.ico", (req, res) => {
res.redirect('/icons/favicon.ico');
@ -171,9 +177,10 @@ if (fs.existsSync('./.env') && process.env.selfURL && process.env.streamSalt &&
app.get("/*", (req, res) => {
res.redirect('/')
});
app.listen(process.env.port, () => {
let startTime = new Date();
console.log(`\n${Cyan(appName)} ${Bright(`v.${version}-${commitHash}`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(new Date().getTime())})`)}\n\nURL: ${Cyan(`${process.env.selfURL}`)}\nPort: ${process.env.port}\n`)
console.log(`\n${Cyan(appName)} ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(new Date().getTime())})`)}\n\nURL: ${Cyan(`${process.env.selfURL}`)}\nPort: ${process.env.port}\n`)
});
} else {
console.log(Red(`cobalt hasn't been configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`))

7
src/config.json
View file

@ -2,7 +2,7 @@
"streamLifespan": 120000,
"maxVideoDuration": 7500000,
"maxAudioDuration": 7500000,
"genericUserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36",
"genericUserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36",
"authorInfo": {
"name": "wukko",
"link": "https://wukko.me/",
@ -18,11 +18,6 @@
}
}
},
"internetExplorerRedirect": {
"newNT": ["6.1", "6.2", "6.3", "10.0"],
"old": "https://mypal-browser.org/",
"new": "https://www.mozilla.org/firefox/new/"
},
"donations": {
"crypto": {
"bitcoin": "bc1q59jyyjvrzj4c22rkk3ljeecq6jmpyscgz9spnd",

2
src/front/cobalt.css
View file

@ -278,7 +278,7 @@ input[type="checkbox"] {
.italic {
font-style: italic;
}
.social-link {
.cobalt-support-link {
display: flex;
flex-direction: row;
justify-content: flex-start;

BIN
src/front/updateBanners/valentines.webp Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 126 KiB

2
src/localization/languages/en.json
View file

@ -25,7 +25,7 @@
"ErrorBrokenLink": "{s} is supported, but something is wrong with your link. maybe you didn't copy it fully?",
"ErrorNoLink": "i can't guess what you want to download! please give me a link.",
"ErrorPageRenderFail": "something went wrong and page couldn't render. if it's a recurring or critical issue, please {ContactLink}. it'd be useful if you provided current commit hash ({s}) and error recreation steps. thank you in advance :D",
"ErrorRateLimit": "you're making too many requests. calm down and try again in a bit.",
"ErrorRateLimit": "you're making too many requests. try again in a minute!",
"ErrorCouldntFetch": "couldn't get any info about your link. check if it's correct and try again.",
"ErrorLengthLimit": "current length limit is {s} minutes. video that you tried to download is longer than {s} minutes. pick something else!",
"ErrorBadFetch": "an error occurred when i tried to get info about your link. are you sure it works? check if it does, and try again.",

6
src/localization/languages/ru.json
View file

@ -7,7 +7,7 @@
"LinkInput": "вставь ссылку сюда",
"AboutSummary": "{appName} — твой друг при скачивании контента из соц. сетей. никакой рекламы или трекеров. вставляешь ссылку и получаешь файл. ничего лишнего.",
"EmbedBriefDescription": "сохраняй что хочешь, без мороки и вторжения в личное пространство",
"MadeWithLove": "сделано с <3 wukko",
"MadeWithLove": "сделано wukko, с <3",
"AccessibilityInputArea": "зона вставки ссылки",
"AccessibilityOpenAbout": "открыть окно с инфой",
"AccessibilityDownloadButton": "кнопка скачивания",
@ -23,9 +23,9 @@
"ErrorSomethingWentWrong": "что-то пошло совсем не так, и у меня не получилось ничего для тебя достать. ты можешь попробовать ещё раз, но если так и не получится, {ContactLink}.",
"ErrorUnsupported": "с твоей ссылкой что-то не так, или же этот сервис ещё не поддерживается. может быть, ты вставил не ту ссылку?",
"ErrorBrokenLink": "{s} поддерживается, но с твоей ссылкой что-то не так. может быть, ты её не полностью скопировал?",
"ErrorNoLink": "я не гадалка и не могу угадывать, что ты хочешь скачать. попробуй в следующий раз вставить ссылку.",
"ErrorNoLink": "пока что я не умею угадывать, что ты хочешь скачать. дай мне, пожалуйста, ссылку :(",
"ErrorPageRenderFail": "что-то пошло не так и у меня не получилось срендерить страницу. если это повторится ещё раз, пожалуйста, {ContactLink}. также приложи хэш текущего коммита ({s}) с действиями для повторения этой ошибки. можно на русском языке. спасибо :)",
"ErrorRateLimit": "ты делаешь слишком много запросов. успокойся и попробуй ещё раз через несколько минут.",
"ErrorRateLimit": "ты делаешь слишком много запросов. попробуй ещё раз через минуту!",
"ErrorCouldntFetch": "мне не удалось получить инфу о твоей ссылке. проверь её и попробуй ещё раз.",
"ErrorLengthLimit": "твоё видео длиннее чем {s} минут(ы). это превышает текущий лимит. скачай что-нибудь покороче, а не экранизацию \"войны и мира\".",
"ErrorBadFetch": "произошла ошибка при получении инфы о твоей ссылке. ты уверен, что она работает? проверь её, и попробуй ещё раз.",

31
src/modules/api.js
View file

@ -10,32 +10,41 @@ import match from "./processing/match.js";
export async function getJSON(originalURL, lang, obj) {
try {
let url = decodeURIComponent(originalURL);
if (!url.includes('http://')) {
if (url.startsWith('http://')) {
return apiJSON(0, { t: errorUnsupported(lang) });
}
let hostname = url.replace("https://", "").replace(' ', '').split('&')[0].split("/")[0].split("."),
host = hostname[hostname.length - 2],
patternMatch;
if (host === "youtu") {
// TO-DO: bring all tests into one unified module instead of placing them in several places
switch(host) {
case "youtu":
host = "youtube";
url = `https://youtube.com/watch?v=${url.replace("youtu.be/", "").replace("https://", "")}`;
}
if (host === "goo" && url.substring(0, 30) === "https://soundcloud.app.goo.gl/") {
break;
case "goo":
if (url.substring(0, 30) === "https://soundcloud.app.goo.gl/"){
host = "soundcloud"
url = `https://soundcloud.com/${url.replace("https://soundcloud.app.goo.gl/", "").split('/')[0]}`
}
if (host === "tumblr" && !url.includes("blog/view")) {
if (url.slice(-1) == '/') url = url.slice(0, -1);
break;
case "tumblr":
if (!url.includes("blog/view")) {
if (url.slice(-1) === '/') url = url.slice(0, -1);
url = url.replace(url.split('/')[5], '');
}
if (host && host.length < 20 && host in patterns && patterns[host]["enabled"]) {
break;
}
if (!(host && host.length < 20 && host in patterns && patterns[host]["enabled"])) return apiJSON(0, { t: errorUnsupported(lang) });
for (let i in patterns[host]["patterns"]) {
patternMatch = new UrlPattern(patterns[host]["patterns"][i]).match(cleanURL(url, host).split(".com/")[1]);
if (patternMatch) break;
}
if (patternMatch) {
if (!patternMatch) return apiJSON(0, { t: errorUnsupported(lang) });
return await match(host, patternMatch, url, lang, obj);
} else return apiJSON(0, { t: errorUnsupported(lang) });
} else return apiJSON(0, { t: errorUnsupported(lang) });
} else return apiJSON(0, { t: errorUnsupported(lang) });
} catch (e) {
return apiJSON(0, { t: loc(lang, 'ErrorSomethingWentWrong') });
}

4
src/modules/build.js
View file

@ -4,9 +4,9 @@ export async function buildFront() {
try {
await esbuild.build({
entryPoints: ['src/front/cobalt.js', 'src/front/cobalt.css'],
outdir: `min/`,
outdir: 'min/',
minify: true,
loader: { ".js": "js", ".css": "css" }
loader: { '.js': 'js', '.css': 'css' }
})
} catch (e) {
return;

1
src/modules/config.js
View file

@ -15,7 +15,6 @@ export const
repo = packageJson["bugs"]["url"].replace('/issues', ''),
authorInfo = config.authorInfo,
quality = config.quality,
internetExplorerRedirect = config.internetExplorerRedirect,
donations = config.donations,
ffmpegArgs = config.ffmpegArgs,
supportedAudio = config.supportedAudio,

4
src/modules/pageRender/elements.js
View file

@ -2,7 +2,7 @@ import { celebrations } from "../config.js";
export function switcher(obj) {
let items = ``;
if (obj.name == "download") {
if (obj.name === "download") {
items = obj.items;
} else {
for (let i = 0; i < obj.items.length; i++) {
@ -129,7 +129,7 @@ export function backdropLink(link, text) {
return `<a class="text-backdrop italic" href="${link}" target="_blank">${text}</a>`
}
export function socialLink(emoji, name, handle, url) {
return `<div class="social-link">${emoji} ${name}: <a class="text-backdrop italic" href="${url}" target="_blank">${handle}</a></div>`
return `<div class="cobalt-support-link">${emoji} ${name}: <a class="text-backdrop italic" href="${url}" target="_blank">${handle}</a></div>`
}
export function settingsCategory(obj) {
return `<div id="settings-${obj.name}" class="settings-category">

2
src/modules/pageRender/onDemand.js
View file

@ -6,7 +6,7 @@ export function changelogHistory() { // blockId 0
let historyLen = history.length
for (let i in history) {
let separator = (i != 0 && i != historyLen) ? '<div class="separator"></div>' : ''
let separator = (i !== 0 && i !== historyLen) ? '<div class="separator"></div>' : ''
render += `${separator}${history[i]["banner"] ? `<div class="changelog-banner"><img class="changelog-img" src="${history[i]["banner"]}" onerror="this.style.display='none'"></img></div>` : ''}<div id="popup-desc" class="changelog-subtitle">${history[i]["title"]}</div><div id="popup-desc" class="desc-padding">${history[i]["content"]}</div>`
}
return render;

2
src/modules/pageRender/page.js
View file

@ -186,7 +186,7 @@ export default function(obj) {
closeAria: t('AccessibilityClosePopup'),
header: {
aboveTitle: {
text: `v.${version}-${obj.hash}`,
text: `v.${version}-${obj.hash} (${obj.branch})`,
url: `${repo}/commit/${obj.hash}`
},
title: `${emoji("⚙️", 30)} ${t('TitlePopupSettings')}`

72
src/modules/processing/match.js
View file

@ -1,27 +1,28 @@
import { apiJSON } from "../sub/utils.js";
import { errorUnsupported, genericError } from "../sub/errors.js";
import { errorUnsupported, genericError, brokenLink } from "../sub/errors.js";
import loc from "../../localization/manager.js";
import { testers } from "./servicesPatternTesters.js";
import bilibili from "../services/bilibili.js";
import reddit from "../services/reddit.js";
import twitter from "../services/twitter.js";
import youtube from "../services/youtube.js";
import vk from "../services/vk.js";
import tiktok from "../services/tiktok.js";
import tumblr from "../services/tumblr.js";
import bilibili from "./services/bilibili.js";
import reddit from "./services/reddit.js";
import twitter from "./services/twitter.js";
import youtube from "./services/youtube.js";
import vk from "./services/vk.js";
import tiktok from "./services/tiktok.js";
import tumblr from "./services/tumblr.js";
import matchActionDecider from "./matchActionDecider.js";
import vimeo from "../services/vimeo.js";
import soundcloud from "../services/soundcloud.js";
import vimeo from "./services/vimeo.js";
import soundcloud from "./services/soundcloud.js";
export default async function (host, patternMatch, url, lang, obj) {
try {
if (!testers[host]) return apiJSON(0, { t: errorUnsupported(lang) });
if (!(testers[host](patternMatch))) throw Error();
let r, isAudioOnly = !!obj.isAudioOnly;
if (!testers[host]) return apiJSON(0, { t: errorUnsupported(lang) });
if (!(testers[host](patternMatch))) return apiJSON(0, { t: brokenLink(lang) });
let r;
switch (host) {
case "twitter":
r = await twitter({
@ -29,14 +30,14 @@ export default async function (host, patternMatch, url, lang, obj) {
spaceId: patternMatch["spaceId"] ? patternMatch["spaceId"] : false,
lang: lang
});
if (r.isAudioOnly) obj.isAudioOnly = true
break;
case "vk":
r = await vk({
url: url,
userId: patternMatch["userId"],
videoId: patternMatch["videoId"],
lang: lang, quality: obj.vQuality
lang: lang,
quality: obj.vQuality
});
break;
case "bilibili":
@ -48,10 +49,11 @@ export default async function (host, patternMatch, url, lang, obj) {
case "youtube":
let fetchInfo = {
id: patternMatch["id"].slice(0, 11),
lang: lang, quality: obj.vQuality,
lang: lang,
quality: obj.vQuality,
format: "webm"
};
if (url.match('music.youtube.com') || obj.isAudioOnly == true) obj.vFormat = "audio";
if (url.match('music.youtube.com') || isAudioOnly === true) obj.vFormat = "audio";
switch (obj.vFormat) {
case "mp4":
fetchInfo["format"] = "mp4";
@ -60,7 +62,7 @@ export default async function (host, patternMatch, url, lang, obj) {
fetchInfo["format"] = "webm";
fetchInfo["isAudioOnly"] = true;
fetchInfo["quality"] = "max";
obj.isAudioOnly = true;
isAudioOnly = true;
break;
}
r = await youtube(fetchInfo);
@ -69,7 +71,8 @@ export default async function (host, patternMatch, url, lang, obj) {
r = await reddit({
sub: patternMatch["sub"],
id: patternMatch["id"],
title: patternMatch["title"], lang: lang,
title: patternMatch["title"],
lang: lang,
});
break;
case "douyin":
@ -77,28 +80,33 @@ export default async function (host, patternMatch, url, lang, obj) {
r = await tiktok({
host: host,
postId: patternMatch["postId"],
id: patternMatch["id"], lang: lang,
noWatermark: obj.isNoTTWatermark, fullAudio: obj.isTTFullAudio,
isAudioOnly: obj.isAudioOnly
id: patternMatch["id"],
lang: lang,
noWatermark: obj.isNoTTWatermark,
fullAudio: obj.isTTFullAudio,
isAudioOnly: isAudioOnly
});
if (r.isAudioOnly) obj.isAudioOnly = true;
break;
case "tumblr":
r = await tumblr({
id: patternMatch["id"], url: url, user: patternMatch["user"] ? patternMatch["user"] : false,
id: patternMatch["id"],
url: url,
user: patternMatch["user"] ? patternMatch["user"] : false,
lang: lang
});
break;
case "vimeo":
r = await vimeo({
id: patternMatch["id"].slice(0, 11), quality: obj.vQuality,
id: patternMatch["id"].slice(0, 11),
quality: obj.vQuality,
lang: lang
});
break;
case "soundcloud":
obj.isAudioOnly = true;
isAudioOnly = true;
r = await soundcloud({
author: patternMatch["author"], song: patternMatch["song"], url: url,
author: patternMatch["author"],
song: patternMatch["song"], url: url,
shortLink: patternMatch["shortLink"] ? patternMatch["shortLink"] : false,
accessKey: patternMatch["accessKey"] ? patternMatch["accessKey"] : false,
format: obj.aFormat,
@ -108,9 +116,13 @@ export default async function (host, patternMatch, url, lang, obj) {
default:
return apiJSON(0, { t: errorUnsupported(lang) });
}
return !r.error ? matchActionDecider(r, host, obj.ip, obj.aFormat, obj.isAudioOnly, lang, obj.isAudioMuted) : apiJSON(0, {
t: Array.isArray(r.error) ? loc(lang, r.error[0], r.error[1]) : loc(lang, r.error)
});
if (r.isAudioOnly) isAudioOnly = true;
let isAudioMuted = isAudioOnly ? false : obj.isAudioMuted;
if (r.error) return apiJSON(0, { t: Array.isArray(r.error) ? loc(lang, r.error[0], r.error[1]) : loc(lang, r.error) });
return matchActionDecider(r, host, obj.ip, obj.aFormat, isAudioOnly, lang, isAudioMuted);
} catch (e) {
return apiJSON(0, { t: genericError(lang, host) })
}

200
src/modules/processing/matchActionDecider.js
View file

@ -1,121 +1,137 @@
import { audioIgnore, services, supportedAudio } from "../config.js"
import { apiJSON } from "../sub/utils.js"
import { audioIgnore, services, supportedAudio } from "../config.js";
import { apiJSON } from "../sub/utils.js";
import loc from "../../localization/manager.js";
export default function(r, host, ip, audioFormat, isAudioOnly, lang, isAudioMuted) {
if (!isAudioOnly && !r.picker && !isAudioMuted) {
switch (host) {
case "twitter":
return apiJSON(1, { u: r.urls });
case "vk":
return apiJSON(2, {
type: "bridge", u: r.urls, service: host, ip: ip,
filename: r.filename,
});
case "bilibili":
return apiJSON(2, {
type: "render", u: r.urls, service: host, ip: ip,
filename: r.filename,
time: r.time
});
case "youtube":
return apiJSON(2, {
type: r.type, u: r.urls, service: host, ip: ip,
filename: r.filename,
time: r.time,
});
case "reddit":
return apiJSON(r.typeId, {
type: r.type, u: r.urls, service: host, ip: ip,
filename: r.filename,
});
case "tiktok":
return apiJSON(2, {
type: "bridge", u: r.urls, service: host, ip: ip,
filename: r.filename,
});
case "douyin":
return apiJSON(2, {
type: "bridge", u: r.urls, service: host, ip: ip,
filename: r.filename,
});
case "tumblr":
return apiJSON(1, { u: r.urls });
case "vimeo":
if (Array.isArray(r.urls)) {
return apiJSON(2, {
type: "render", u: r.urls, service: host, ip: ip,
filename: r.filename
});
} else {
return apiJSON(1, { u: r.urls });
}
}
} else if (isAudioMuted && !isAudioOnly) {
let isSplit = Array.isArray(r.urls);
return apiJSON(2, {
type: isSplit ? "bridge" : "mute",
u: isSplit ? r.urls[0] : r.urls,
let action,
responseType = 2,
defaultParams = {
u: r.urls,
service: host,
ip: ip,
filename: r.filename,
mute: true,
});
} else if (r.picker) {
},
params = {}
if (!isAudioOnly && !r.picker && !isAudioMuted) action = "video";
if (isAudioOnly && !r.picker) action = "audio";
if (r.picker) action = "picker";
if (isAudioMuted) action = "muteVideo";
if (action === "picker" || action === "audio") {
defaultParams.filename = r.audioFilename;
defaultParams.isAudioOnly = true;
defaultParams.audioFormat = audioFormat;
}
switch (action) {
case "video":
switch (host) {
case "bilibili":
params = { type: "render", time: r.time };
break;
case "youtube":
params = { type: r.type, time: r.time };
break;
case "reddit":
responseType = r.typeId;
params = { type: r.type };
break;
case "vimeo":
if (Array.isArray(r.urls)) {
params = { type: "render" }
} else {
responseType = 1;
}
break;
case "vk":
case "douyin":
case "tiktok":
let type = "render";
if (audioFormat === "mp3" || audioFormat === "best") {
audioFormat = "mp3"
type = "bridge"
}
return apiJSON(5, {
type: type,
picker: r.picker,
u: Array.isArray(r.urls) ? r.urls[1] : r.urls, service: host, ip: ip,
filename: r.audioFilename, isAudioOnly: true, audioFormat: audioFormat, copy: audioFormat === "best" ? true : false,
})
params = { type: "bridge" };
break;
case "tumblr":
case "twitter":
return apiJSON(5, {
picker: r.picker, service: host
})
responseType = 1;
break;
}
} else if (isAudioOnly) {
break;
case "muteVideo":
params = {
type: Array.isArray(r.urls) ? "bridge" : "mute",
u: Array.isArray(r.urls) ? r.urls[0] : r.urls,
mute: true
}
break;
case "picker":
responseType = 5;
switch (host) {
case "twitter":
params = { picker: r.picker };
break;
case "douyin":
case "tiktok":
let pickerType = "render";
if (audioFormat === "mp3" || audioFormat === "best") {
audioFormat = "mp3";
pickerType = "bridge"
}
params = {
type: pickerType,
picker: r.picker,
u: Array.isArray(r.urls) ? r.urls[1] : r.urls,
copy: audioFormat === "best" ? true : false
}
}
break;
case "audio":
if ((host === "reddit" && r.typeId === 1) || (host === "vimeo" && !r.filename) || audioIgnore.includes(host)) return apiJSON(0, { t: loc(lang, 'ErrorEmptyDownload') });
let type = "render";
let processType = "render";
let copy = false;
if (!supportedAudio.includes(audioFormat)) audioFormat = "best";
if ((host == "tiktok" || host == "douyin") && services.tiktok.audioFormats.includes(audioFormat)) {
if ((host === "tiktok" || host === "douyin") && services.tiktok.audioFormats.includes(audioFormat)) {
if (r.isMp3) {
if (audioFormat === "mp3" || audioFormat === "best") {
audioFormat = "mp3"
type = "bridge"
audioFormat = "mp3";
processType = "bridge"
}
} else if (audioFormat === "best") {
audioFormat = "m4a"
type = "bridge"
audioFormat = "m4a";
processType = "bridge"
}
}
if ((audioFormat === "best" && services[host]["bestAudio"]) || services[host]["bestAudio"] && (audioFormat === services[host]["bestAudio"])) {
audioFormat = services[host]["bestAudio"]
type = "bridge"
if ((audioFormat === "best" && services[host]["bestAudio"])
|| services[host]["bestAudio"] && (audioFormat === services[host]["bestAudio"])) {
audioFormat = services[host]["bestAudio"];
processType = "bridge"
} else if (audioFormat === "best") {
audioFormat = "m4a"
copy = true
audioFormat = "m4a";
copy = true;
if (r.audioFilename.includes("twitterspaces")) {
audioFormat = "mp3"
copy = false
}
}
return apiJSON(2, {
type: type,
u: Array.isArray(r.urls) ? r.urls[1] : r.urls, service: host, ip: ip,
filename: r.audioFilename, isAudioOnly: true,
audioFormat: audioFormat, copy: copy, fileMetadata: r.fileMetadata ? r.fileMetadata : false
})
} else {
return apiJSON(0, { t: loc(lang, 'ErrorSomethingWentWrong') });
params = {
type: processType,
u: Array.isArray(r.urls) ? r.urls[1] : r.urls,
audioFormat: audioFormat,
copy: copy,
fileMetadata: r.fileMetadata ? r.fileMetadata : false
}
break;
default:
return apiJSON(0, { t: loc(lang, 'ErrorEmptyDownload') });
}
return apiJSON(responseType, {...defaultParams, ...params})
}

28
src/modules/processing/services/bilibili.js Normal file
View file

@ -0,0 +1,28 @@
import { genericUserAgent, maxVideoDuration } from "../../config.js";
// TO-DO: quality picking, bilibili.tv support, and higher quality downloads (currently requires an account)
export default async function(obj) {
let html = await fetch(`https://bilibili.com/video/${obj.id}`, {
headers: { "user-agent": genericUserAgent }
}).then((r) => { return r.text() }).catch(() => { return false });
if (!html) return { error: 'ErrorCouldntFetch' };
if (!(html.includes('<script>window.__playinfo__=') && html.includes('"video_codecid"'))) return { error: 'ErrorEmptyDownload' };
let streamData = JSON.parse(html.split('<script>window.__playinfo__=')[1].split('</script>')[0]);
if (streamData.data.timelength > maxVideoDuration) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
let video = streamData["data"]["dash"]["video"].filter((v) => {
if (!v["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/")) return true;
}).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth));
let audio = streamData["data"]["dash"]["audio"].filter((a) => {
if (!a["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/")) return true;
}).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth));
return {
urls: [video[0]["baseUrl"], audio[0]["baseUrl"]],
time: streamData.data.timelength,
audioFilename: `bilibili_${obj.id}_audio`,
filename: `bilibili_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.mp4`
};
}

28
src/modules/processing/services/reddit.js Normal file
View file

@ -0,0 +1,28 @@
import { maxVideoDuration } from "../../config.js";
export default async function(obj) {
let data = await fetch(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}/${obj.name}.json`).then((r) => { return r.json() }).catch(() => { return false });
if (!data) return { error: 'ErrorCouldntFetch' };
data = data[0]["data"]["children"][0]["data"];
if (data.url.endsWith('.gif')) return { typeId: 1, urls: data.url };
if (!("reddit_video" in data["secure_media"])) return { error: 'ErrorEmptyDownload' };
if (data["secure_media"]["reddit_video"]["duration"] * 1000 > maxVideoDuration) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
let video = data["secure_media"]["reddit_video"]["fallback_url"].split('?')[0],
audio = video.match('.mp4') ? `${video.split('_')[0]}_audio.mp4` : `${data["secure_media"]["reddit_video"]["fallback_url"].split('DASH')[0]}audio`;
await fetch(audio, { method: "HEAD" }).then((r) => {if (Number(r.status) !== 200) audio = ''}).catch(() => {audio = ''});
let id = data["secure_media"]["reddit_video"]["fallback_url"].split('/')[3];
if (!audio.length > 0) return { typeId: 1, urls: video };
return {
typeId: 2,
type: "render",
urls: [video, audio],
audioFilename: `reddit_${id}_audio`,
filename: `reddit_${id}.mp4`
};
}

74
src/modules/processing/services/soundcloud.js Normal file
View file

@ -0,0 +1,74 @@
import { maxAudioDuration } from "../../config.js";
let cachedID = {};
async function findClientID() {
try {
let sc = await fetch('https://soundcloud.com/').then((r) => { return r.text() }).catch(() => { return false });
let scVersion = String(sc.match(/<script>window\.__sc_version="[0-9]{10}"<\/script>/)[0].match(/[0-9]{10}/));
if (cachedID.version === scVersion) return cachedID.id;
let scripts = sc.matchAll(/<script.+src="(.+)">/g);
let clientid;
for (let script of scripts) {
let url = script[1];
if (url && !url.startsWith('https://a-v2.sndcdn.com')) return;
let scrf = await fetch(url).then((r) => {return r.text()}).catch(() => { return false });
let id = scrf.match(/\("client_id=[A-Za-z0-9]{32}"\)/);
if (id && typeof id[0] === 'string') {
clientid = id[0].match(/[A-Za-z0-9]{32}/)[0];
break;
}
}
cachedID.version = scVersion;
cachedID.id = clientid;
return clientid;
} catch (e) {
return false;
}
}
export default async function(obj) {
let html;
if (!obj.author && !obj.song && obj.shortLink) {
html = await fetch(`https://soundcloud.app.goo.gl/${obj.shortLink}/`).then((r) => { return r.text() }).catch(() => { return false });
}
if (obj.author && obj.song) {
html = await fetch(`https://soundcloud.com/${obj.author}/${obj.song}${obj.accessKey ? `/s-${obj.accessKey}` : ''}`).then((r) => { return r.text() }).catch(() => { return false });
}
if (!html) return { error: 'ErrorCouldntFetch'};
if (!(html.includes('<script>window.__sc_hydration = ')
&& html.includes('"format":{"protocol":"progressive","mime_type":"audio/mpeg"},')
&& html.includes('{"hydratable":"sound","data":'))) {
return { error: ['ErrorBrokenLink', 'soundcloud'] }
}
let json = JSON.parse(html.split('{"hydratable":"sound","data":')[1].split('}];</script>')[0])
if (!json["media"]["transcodings"]) return { error: 'ErrorEmptyDownload' };
let clientId = await findClientID();
if (!clientId) return { error: 'ErrorSoundCloudNoClientId' };
let fileUrlBase = json.media.transcodings[0]["url"].replace("/hls", "/progressive")
let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`;
if (fileUrl.substring(0, 54) !== "https://api-v2.soundcloud.com/media/soundcloud:tracks:") return { error: 'ErrorEmptyDownload' };
if (json.duration > maxAudioDuration) return { error: ['ErrorLengthAudioConvert', maxAudioDuration / 60000] };
let file = await fetch(fileUrl).then(async (r) => { return (await r.json()).url }).catch(() => { return false });
if (!file) return { error: 'ErrorCouldntFetch' };
return {
urls: file,
audioFilename: `soundcloud_${json.id}`,
fileMetadata: {
title: json.title,
artist: json.user.username,
}
}
}

112
src/modules/processing/services/tiktok.js Normal file
View file

@ -0,0 +1,112 @@
import { genericUserAgent } from "../../config.js";
let userAgent = genericUserAgent.split(' Chrome/1')[0],
config = {
tiktok: {
short: "https://vt.tiktok.com/",
api: "https://api2.musical.ly/aweme/v1/feed/?aweme_id={postId}&version_code=262&app_name=musical_ly&channel=App&device_id=null&os_version=14.4.2&device_platform=iphone&device_type=iPhone9&region=US&carrier_region=US",
},
douyin: {
short: "https://v.douyin.com/",
api: "https://www.iesdouyin.com/aweme/v1/web/aweme/detail/?aweme_id={postId}",
}
}
function selector(j, h, id) {
if (!j) return false;
let t;
switch (h) {
case "tiktok":
t = j["aweme_list"].filter((v) => { if (v["aweme_id"] === id) return true })[0];
break;
case "douyin":
t = j['aweme_detail'];
break;
}
if (t.length < 3) return false;
return t;
}
export default async function(obj) {
let postId = obj.postId ? obj.postId : false;
if (!postId) {
let html = await fetch(`${config[obj.host]["short"]}${obj.id}`, {
redirect: "manual",
headers: { "user-agent": userAgent }
}).then((r) => { return r.text() }).catch(() => { return false });
if (!html) return { error: 'ErrorCouldntFetch' };
if (html.slice(0, 17) === '<a href="https://' && html.includes('/video/')) {
postId = html.split('/video/')[1].split('?')[0].replace("/", '')
} else if (html.slice(0, 32) === '<a href="https://m.tiktok.com/v/' && html.includes('/v/')) {
postId = html.split('/v/')[1].split('.html')[0].replace("/", '')
}
}
if (!postId) return { error: 'ErrorCantGetID' };
let detail;
detail = await fetch(config[obj.host]["api"].replace("{postId}", postId), {
headers: {"user-agent": "TikTok 26.2.0 rv:262018 (iPhone; iOS 14.4.2; en_US) Cronet"}
}).then((r) => { return r.json() }).catch(() => { return false });
detail = selector(detail, obj.host, postId);
if (!detail) return { error: 'ErrorCouldntFetch' };
let video, videoFilename, audioFilename, isMp3, audio, images, filenameBase = `${obj.host}_${postId}`;
if (obj.host === "tiktok") {
images = detail["image_post_info"] ? detail["image_post_info"]["images"] : false
} else {
images = detail["images"] ? detail["images"] : false
}
if (!obj.isAudioOnly && !images) {
video = obj.host === "tiktok" ? detail["video"]["download_addr"]["url_list"][0] : detail["video"]["play_addr"]["url_list"][2].replace("/play/", "/playwm/");
videoFilename = `${filenameBase}_video.mp4`;
if (obj.noWatermark) {
video = obj.host === "tiktok" ? detail["video"]["play_addr"]["url_list"][0] : detail["video"]["play_addr"]["url_list"][0];
videoFilename = `${filenameBase}_video_nw.mp4` // nw - no watermark
}
} else {
let fallback = obj.host === "douyin" ? detail["video"]["play_addr"]["url_list"][0].replace("playwm", "play") : detail["video"]["play_addr"]["url_list"][0];
audio = fallback;
audioFilename = `${filenameBase}_audio_fv`; // fv - from video
if (obj.fullAudio || fallback.includes("music")) {
audio = detail["music"]["play_url"]["url_list"][0]
audioFilename = `${filenameBase}_audio`
}
if (audio.slice(-4) === ".mp3") isMp3 = true;
}
if (video) return {
urls: video,
filename: videoFilename
}
if (images && obj.isAudioOnly) return {
urls: audio,
audioFilename: audioFilename,
isAudioOnly: true,
isMp3: isMp3
}
if (images) {
let imageLinks = [];
for (let i in images) {
let sel = obj.host === "tiktok" ? images[i]["display_image"]["url_list"] : images[i]["url_list"];
sel = sel.filter((p) => { if (p.includes(".jpeg?")) return true; })
imageLinks.push({url: sel[0]})
}
return {
picker: imageLinks,
urls: audio,
audioFilename: audioFilename,
isAudioOnly: true,
isMp3: isMp3
}
}
if (audio) return {
urls: audio,
audioFilename: audioFilename,
isAudioOnly: true,
isMp3: isMp3
}
}

14
src/modules/processing/services/tumblr.js Normal file
View file

@ -0,0 +1,14 @@
import { genericUserAgent } from "../../config.js";
export default async function(obj) {
let html = await fetch(`https://${
obj.user ? obj.user : obj.url.split('.')[0].replace('https://', '')
}.tumblr.com/post/${obj.id}`, {
headers: { "user-agent": genericUserAgent }
}).then((r) => { return r.text() }).catch(() => { return false });
if (!html) return { error: 'ErrorCouldntFetch' };
if (!html.includes('property="og:video" content="https://va.media.tumblr.com/')) return { error: 'ErrorEmptyDownload' };
return { urls: `https://va.media.tumblr.com/${html.split('property="og:video" content="https://va.media.tumblr.com/')[1].split('"')[0]}`, audioFilename: `tumblr_${obj.id}_audio` }
}

101
src/modules/processing/services/twitter.js Normal file
View file

@ -0,0 +1,101 @@
import { genericUserAgent } from "../../config.js";
function bestQuality(arr) {
return arr.filter((v) => { if (v["content_type"] === "video/mp4") return true }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate))[0]["url"].split("?")[0]
}
const apiURL = "https://api.twitter.com/1.1"
// TO-DO: move from 1.1 api to graphql
export default async function(obj) {
let _headers = {
"user-agent": genericUserAgent,
"authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
// ^ no explicit content, but with multi media support
"host": "api.twitter.com"
};
let req_act = await fetch(`${apiURL}/guest/activate.json`, {
method: "POST",
headers: _headers
}).then((r) => { return r.status === 200 ? r.json() : false }).catch(() => { return false });
if (!req_act) return { error: 'ErrorCouldntFetch' };
_headers["x-guest-token"] = req_act["guest_token"];
let showURL = `${apiURL}/statuses/show/${obj.id}.json?tweet_mode=extended&include_user_entities=0&trim_user=1&include_entities=0&cards_platform=Web-12&include_cards=1`;
if (!obj.spaceId) {
let req_status = await fetch(showURL, { headers: _headers }).then((r) => { return r.status === 200 ? r.json() : false }).catch((e) => { return false });
if (!req_status) {
_headers.authorization = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw";
// ^ explicit content, but no multi media support
delete _headers["x-guest-token"]
req_act = await fetch(`${apiURL}/guest/activate.json`, {
method: "POST",
headers: _headers
}).then((r) => { return r.status === 200 ? r.json() : false}).catch(() => { return false });
if (!req_act) return { error: 'ErrorCouldntFetch' };
_headers["x-guest-token"] = req_act["guest_token"];
req_status = await fetch(showURL, { headers: _headers }).then((r) => { return r.status === 200 ? r.json() : false }).catch(() => { return false });
}
if (!req_status) return { error: 'ErrorCouldntFetch' };
if (!req_status["extended_entities"] || !req_status["extended_entities"]["media"]) return { error: 'ErrorNoVideosInTweet' };
let single, multiple = [], media = req_status["extended_entities"]["media"];
media = media.filter((i) => { if (i["type"] === "video" || i["type"] === "animated_gif") return true })
if (media.length > 1) {
for (let i in media) { multiple.push({type: "video", thumb: media[i]["media_url_https"], url: bestQuality(media[i]["video_info"]["variants"])}) }
} else if (media.length === 1) {
single = bestQuality(media[0]["video_info"]["variants"])
} else {
return { error: 'ErrorNoVideosInTweet' }
}
if (single) {
return { urls: single, filename: `twitter_${obj.id}.mp4`, audioFilename: `twitter_${obj.id}_audio` }
} else if (multiple) {
return { picker: multiple }
} else {
return { error: 'ErrorNoVideosInTweet' }
}
} else {
_headers["host"] = "twitter.com"
_headers["content-type"] = "application/json"
let query = {
variables: {"id": obj.spaceId,"isMetatagsQuery":true,"withSuperFollowsUserFields":true,"withDownvotePerspective":false,"withReactionsMetadata":false,"withReactionsPerspective":false,"withSuperFollowsTweetFields":true,"withReplays":true},
features: {"spaces_2022_h2_clipping":true,"spaces_2022_h2_spaces_communities":true,"verified_phone_label_enabled":false,"tweetypie_unmention_optimization_enabled":true,"responsive_web_uc_gql_enabled":true,"vibe_api_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":false,"interactive_text_enabled":true,"responsive_web_text_conversations_enabled":false,"responsive_web_enhance_cards_enabled":true}
}
query.variables = new URLSearchParams(JSON.stringify(query.variables)).toString().slice(0, -1);
query.features = new URLSearchParams(JSON.stringify(query.features)).toString().slice(0, -1);
query = `https://twitter.com/i/api/graphql/wJ5g4zf7v8qPHSQbaozYuw/AudioSpaceById?variables=${query.variables}&features=${query.features}`
let AudioSpaceById = await fetch(query, { headers: _headers }).then((r) => {return r.status === 200 ? r.json() : false}).catch((e) => { return false });
if (!AudioSpaceById) return { error: 'ErrorEmptyDownload' };
if (!AudioSpaceById.data.audioSpace.metadata) return { error: 'ErrorEmptyDownload' };
if (AudioSpaceById.data.audioSpace.metadata.is_space_available_for_replay !== true) return { error: 'TwitterSpaceWasntRecorded' };
let streamStatus = await fetch(
`https://twitter.com/i/api/1.1/live_video_stream/status/${AudioSpaceById.data.audioSpace.metadata.media_key}`, { headers: _headers }
).then((r) =>{ return r.status === 200 ? r.json() : false }).catch(() => { return false });
if (!streamStatus) return { error: 'ErrorCouldntFetch' };
let participants = AudioSpaceById.data.audioSpace.participants.speakers;
let listOfParticipants = `Twitter Space speakers: `;
for (let i in participants) { listOfParticipants += `@${participants[i]["twitter_screen_name"]}, ` }
listOfParticipants = listOfParticipants.slice(0, -2);
return {
urls: streamStatus.source.noRedirectPlaybackUrl,
audioFilename: `twitterspaces_${obj.spaceId}`,
isAudioOnly: true,
fileMetadata: {
title: AudioSpaceById.data.audioSpace.metadata.title,
artist: `Twitter Space by @${AudioSpaceById.data.audioSpace.metadata.creator_results.result.legacy.screen_name}`,
comment: listOfParticipants,
// cover: AudioSpaceById.data.audioSpace.metadata.creator_results.result.legacy.profile_image_url_https.replace("_normal", "")
}
}
}
}

77
src/modules/processing/services/vimeo.js Normal file
View file

@ -0,0 +1,77 @@
import { maxVideoDuration, quality, services } from "../../config.js";
export default async function(obj) {
let api = await fetch(`https://player.vimeo.com/video/${obj.id}/config`).then((r) => { return r.json() }).catch(() => { return false });
if (!api) return { error: 'ErrorCouldntFetch' };
let downloadType = "dash";
if (JSON.stringify(api).includes('"progressive":[{')) downloadType = "progressive";
switch(downloadType) {
case "progressive":
let all = api["request"]["files"]["progressive"].sort((a, b) => Number(b.width) - Number(a.width));
let best = all[0];
try {
if (obj.quality !== "max") {
let pref = parseInt(quality[obj.quality], 10)
for (let i in all) {
let currQuality = parseInt(all[i]["quality"].replace('p', ''), 10)
if (currQuality === pref) {
best = all[i];
break
}
if (currQuality < pref) {
best = all[i-1];
break
}
}
}
} catch (e) {
best = all[0]
}
return { urls: best["url"], filename: `tumblr_${obj.id}.mp4` };
case "dash":
if (api.video.duration > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
let masterJSONURL = api["request"]["files"]["dash"]["cdns"]["akfire_interconnect_quic"]["url"];
let masterJSON = await fetch(masterJSONURL).then((r) => { return r.json() }).catch(() => { return false });
if (!masterJSON) return { error: 'ErrorCouldntFetch' };
if (!masterJSON.video) return { error: 'ErrorEmptyDownload' };
let type = "parcel";
if (masterJSON.base_url === "../") type = "chop";
let masterJSON_Video = masterJSON.video.sort((a, b) => Number(b.width) - Number(a.width));
let masterJSON_Audio = masterJSON.audio.sort((a, b) => Number(b.bitrate) - Number(a.bitrate)).filter((a)=> {if (a['mime_type'] === "audio/mp4") return true;});
let bestVideo = masterJSON_Video[0], bestAudio = masterJSON_Audio[0];
switch (type) {
case "parcel":
if (obj.quality !== "max") {
let pref = parseInt(quality[obj.quality], 10)
for (let i in masterJSON_Video) {
let currQuality = parseInt(services.vimeo.resolutionMatch[masterJSON_Video[i]["width"]], 10)
if (currQuality < pref) {
break;
} else if (String(currQuality) === String(pref)) {
bestVideo = masterJSON_Video[i]
}
}
}
let baseUrl = masterJSONURL.split("/sep/")[0];
let videoUrl = `${baseUrl}/parcel/video/${bestVideo.index_segment.split('?')[0]}`,
audioUrl = `${baseUrl}/parcel/audio/${bestAudio.index_segment.split('?')[0]}`;
return { urls: [videoUrl, audioUrl], audioFilename: `vimeo_${obj.id}_audio`, filename: `vimeo_${obj.id}_${bestVideo["width"]}x${bestVideo["height"]}.mp4` }
case "chop": // TO-DO: support for chop stream type
default:
return { error: 'ErrorEmptyDownload' }
}
default:
return { error: 'ErrorEmptyDownload' }
}
}

45
src/modules/processing/services/vk.js Normal file
View file

@ -0,0 +1,45 @@
import { xml2json } from "xml-js";
import { genericUserAgent, maxVideoDuration, services } from "../../config.js";
import selectQuality from "../../stream/selectQuality.js";
export default async function(obj) {
let html;
html = await fetch(`https://vk.com/video-${obj.userId}_${obj.videoId}`, {
headers: { "user-agent": genericUserAgent }
}).then((r) => { return r.text() }).catch(() => { return false });
if (!html) return { error: 'ErrorCouldntFetch' };
if (!html.includes(`{"lang":`)) return { error: 'ErrorEmptyDownload' };
let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]);
if (Number(js["mvData"]["is_active_live"]) !== 0) return { error: 'ErrorLiveVideo' };
if (js["mvData"]["duration"] > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
let mpd = JSON.parse(xml2json(js["player"]["params"][0]["manifest"], { compact: true, spaces: 4 }));
let repr = mpd["MPD"]["Period"]["AdaptationSet"]["Representation"];
if (!mpd["MPD"]["Period"]["AdaptationSet"]["Representation"]) repr = mpd["MPD"]["Period"]["AdaptationSet"][0]["Representation"];
let selectedQuality,
attr = repr[repr.length - 1]["_attributes"],
qualities = Object.keys(services.vk.quality_match);
for (let i in qualities) {
if (qualities[i] === attr["height"]) {
selectedQuality = `url${attr["height"]}`;
break
}
if (qualities[i] === attr["width"]) {
selectedQuality = `url${attr["width"]}`;
break
}
}
let maxQuality = js["player"]["params"][0][selectedQuality].split('type=')[1].slice(0, 1);
let userQuality = selectQuality('vk', obj.quality, Object.entries(services.vk.quality_match).reduce((r, [k, v]) => { r[v] = k; return r; })[maxQuality]);
let userRepr = repr[services.vk.representation_match[userQuality]]["_attributes"];
if (!(selectedQuality in js["player"]["params"][0])) return { error: 'ErrorEmptyDownload' };
return {
urls: js["player"]["params"][0][`url${userQuality}`],
filename: `vk_${obj.userId}_${obj.videoId}_${userRepr["width"]}x${userRepr['height']}.mp4`
}
}

88
src/modules/processing/services/youtube.js Normal file
View file

@ -0,0 +1,88 @@
import ytdl from "better-ytdl-core";
import { maxVideoDuration, quality as mq } from "../../config.js";
import selectQuality from "../../stream/selectQuality.js";
export default async function(obj) {
let isAudioOnly = !!obj.isAudioOnly,
infoInitial = await ytdl.getInfo(obj.id);
if (!infoInitial) return { error: 'ErrorCantConnectToServiceAPI' };
let info = infoInitial.formats;
if (info[0]["isLive"]) return { error: 'ErrorLiveVideo' };
let videoMatch = [], fullVideoMatch = [], video = [],
audio = info.filter((a) => {
if (!a["isHLS"] && !a["isDashMPD"] && a["hasAudio"] && !a["hasVideo"] && a["container"] === obj.format) return true
}).sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
if (audio.length === 0) return { error: 'ErrorBadFetch' };
if (audio[0]["approxDurationMs"] > maxVideoDuration) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
if (!isAudioOnly) {
video = info.filter((a) => {
if (!a["isHLS"] && !a["isDashMPD"] && a["hasVideo"] && a["container"] === obj.format) {
if (obj.quality !== "max") {
if (a["hasAudio"] && String(mq[obj.quality]) === String(a["height"])) {
fullVideoMatch.push(a)
} else if (!a["hasAudio"] && String(mq[obj.quality]) === String(a["height"])) {
videoMatch.push(a)
}
}
return true
}
}).sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
if (obj.quality !== "max") {
if (videoMatch.length === 0) {
let ss = selectQuality("youtube", obj.quality, video[0]["qualityLabel"].slice(0, 5).replace('p', '').trim());
videoMatch = video.filter((a) => {
if (a["qualityLabel"].slice(0, 5).replace('p', '').trim() === String(ss)) return true
})
} else if (fullVideoMatch.length > 0) {
videoMatch = [fullVideoMatch[0]]
}
} else videoMatch = [video[0]];
if (obj.quality === "los") videoMatch = [video[video.length - 1]];
}
if (video.length === 0) isAudioOnly = true;
if (isAudioOnly) {
let r = {
type: "render",
isAudioOnly: true,
urls: audio[0]["url"],
audioFilename: `youtube_${obj.id}_audio`,
fileMetadata: {
title: infoInitial.videoDetails.title,
artist: infoInitial.videoDetails.ownerChannelName.replace("- Topic", "").trim(),
}
}
if (infoInitial.videoDetails.description) {
let isAutoGenAudio = infoInitial.videoDetails.description.startsWith("Provided to YouTube by");
if (isAutoGenAudio) {
let descItems = infoInitial.videoDetails.description.split("\n\n")
r.fileMetadata.album = descItems[2]
r.fileMetadata.copyright = descItems[3]
if (descItems[4].startsWith("Released on:")) r.fileMetadata.date = descItems[4].replace("Released on: ", '').trim();
}
}
return r
}
let singleTest;
if (videoMatch.length > 0) {
singleTest = videoMatch[0]["hasVideo"] && videoMatch[0]["hasAudio"];
return {
type: singleTest ? "bridge" : "render",
urls: singleTest ? videoMatch[0]["url"] : [videoMatch[0]["url"], audio[0]["url"]],
time: videoMatch[0]["approxDurationMs"],
filename: `youtube_${obj.id}_${videoMatch[0]["width"]}x${videoMatch[0]["height"]}.${obj.format}`
}
}
singleTest = video[0]["hasVideo"] && video[0]["hasAudio"];
return {
type: singleTest ? "bridge" : "render",
urls: singleTest ? video[0]["url"] : [video[0]["url"], audio[0]["url"]],
time: video[0]["approxDurationMs"],
filename: `youtube_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.${obj.format}`
}
}

7
src/modules/processing/servicesConfig.json
View file

@ -8,17 +8,18 @@
"enabled": true
},
"reddit": {
"alias": "reddit videos & gifs",
"patterns": ["r/:sub/comments/:id/:title"],
"enabled": true
},
"twitter": {
"alias": "twitter posts & spaces",
"alias": "twitter posts & spaces & voice",
"patterns": [":user/status/:id", ":user/status/:id/video/:v", "i/spaces/:spaceId"],
"enabled": true
},
"vk": {
"alias": "vk video & clips",
"patterns": ["video-:userId_:videoId", "clip-:userId_:videoId", "clips-:userId?z=clip-:userId_:videoId"],
"patterns": ["video-:userId_:videoId", "clip-:userId_:videoId", "clips-:duplicate?z=clip-:userId_:videoId"],
"quality_match": {
"2160": 7,
"1440": 6,
@ -69,7 +70,7 @@
"enabled": true
},
"douyin": {
"alias": "douyin videos & photos & audio",
"alias": "douyin videos & audio",
"patterns": ["video/:postId", ":id"],
"enabled": true
},

29
src/modules/processing/servicesPatternTesters.js
View file

@ -1,27 +1,28 @@
export const testers = {
"twitter": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length < 20) || (patternMatch["spaceId"] && patternMatch["spaceId"].length === 13),
"twitter": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length < 20)
|| (patternMatch["spaceId"] && patternMatch["spaceId"].length === 13),
"vk": (patternMatch) => (patternMatch["userId"] && patternMatch["videoId"] &&
patternMatch["userId"].length <= 10 && patternMatch["videoId"].length === 9),
"vk": (patternMatch) => (patternMatch["userId"] && patternMatch["videoId"]
&& patternMatch["userId"].length <= 10 && patternMatch["videoId"].length === 9),
"bilibili": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length >= 12),
"youtube": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length >= 11),
"reddit": (patternMatch) => (patternMatch["sub"] && patternMatch["id"] && patternMatch["title"] &&
patternMatch["sub"].length <= 22 && patternMatch["id"].length <= 10 && patternMatch["title"].length <= 96),
"reddit": (patternMatch) => (patternMatch["sub"] && patternMatch["id"] && patternMatch["title"]
&& patternMatch["sub"].length <= 22 && patternMatch["id"].length <= 10 && patternMatch["title"].length <= 96),
"tiktok": (patternMatch) => ((patternMatch["user"] && patternMatch["postId"] && patternMatch["postId"].length <= 21) ||
(patternMatch["id"] && patternMatch["id"].length <= 13)),
"tiktok": (patternMatch) => ((patternMatch["user"] && patternMatch["postId"] && patternMatch["postId"].length <= 21)
|| (patternMatch["id"] && patternMatch["id"].length <= 13)),
"douyin": (patternMatch) => ((patternMatch["postId"] && patternMatch["postId"].length <= 21) ||
(patternMatch["id"] && patternMatch["id"].length <= 13)),
"douyin": (patternMatch) => ((patternMatch["postId"] && patternMatch["postId"].length <= 21)
|| (patternMatch["id"] && patternMatch["id"].length <= 13)),
"tumblr": (patternMatch) => ((patternMatch["id"] && patternMatch["id"].length < 21) ||
(patternMatch["id"] && patternMatch["id"].length < 21 && patternMatch["user"] && patternMatch["user"].length <= 32)),
"tumblr": (patternMatch) => ((patternMatch["id"] && patternMatch["id"].length < 21)
|| (patternMatch["id"] && patternMatch["id"].length < 21 && patternMatch["user"] && patternMatch["user"].length <= 32)),
"vimeo": (patternMatch) => ((patternMatch["id"] && patternMatch["id"].length <= 11)),
"soundcloud": (patternMatch) => ((patternMatch["author"] && patternMatch["song"] && (patternMatch["author"].length + patternMatch["song"].length) <= 96) ||
(patternMatch["shortLink"] && patternMatch["shortLink"].length <= 32))
};
"soundcloud": (patternMatch) => ((patternMatch["author"] && patternMatch["song"]
&& (patternMatch["author"].length + patternMatch["song"].length) <= 96) || (patternMatch["shortLink"] && patternMatch["shortLink"].length <= 32))
}

29
src/modules/services/bilibili.js
View file

@ -1,29 +0,0 @@
import { genericUserAgent, maxVideoDuration } from "../config.js";
export default async function(obj) {
try {
let html = await fetch(`https://bilibili.com/video/${obj.id}`, {
headers: {"user-agent": genericUserAgent}
}).then((r) => {return r.text()}).catch(() => {return false});
if (!html) return { error: 'ErrorCouldntFetch' };
if (html.includes('<script>window.__playinfo__=') && html.includes('"video_codecid"')) {
let streamData = JSON.parse(html.split('<script>window.__playinfo__=')[1].split('</script>')[0]);
if (streamData.data.timelength <= maxVideoDuration) {
let video = streamData["data"]["dash"]["video"].filter((v) => {
if (!v["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/")) return true;
}).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth));
let audio = streamData["data"]["dash"]["audio"].filter((a) => {
if (!a["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/")) return true;
}).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth));
return { urls: [video[0]["baseUrl"], audio[0]["baseUrl"]], time: streamData.data.timelength, audioFilename: `bilibili_${obj.id}_audio`, filename: `bilibili_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.mp4` };
} else {
return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
}
} else {
return { error: 'ErrorEmptyDownload' };
}
} catch (e) {
return { error: 'ErrorBadFetch' };
}
}

27
src/modules/services/reddit.js
View file

@ -1,27 +0,0 @@
import { maxVideoDuration } from "../config.js";
export default async function(obj) {
try {
let data = await fetch(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}/${obj.name}.json`).then((r) => {return r.json()}).catch(() => {return false});
if (!data) return { error: 'ErrorCouldntFetch' };
data = data[0]["data"]["children"][0]["data"];
if ("reddit_video" in data["secure_media"] && data["secure_media"]["reddit_video"]["duration"] * 1000 < maxVideoDuration) {
let video = data["secure_media"]["reddit_video"]["fallback_url"].split('?')[0],
audio = video.match('.mp4') ? `${video.split('_')[0]}_audio.mp4` : `${data["secure_media"]["reddit_video"]["fallback_url"].split('DASH')[0]}audio`;
await fetch(audio, {method: "HEAD"}).then((r) => {if (r.status != 200) audio = ''}).catch(() => {audio = ''});
let id = data["secure_media"]["reddit_video"]["fallback_url"].split('/')[3]
if (audio.length > 0) {
return { typeId: 2, type: "render", urls: [video, audio], audioFilename: `reddit_${id}_audio`, filename: `reddit_${id}.mp4` };
} else {
return { typeId: 1, urls: video };
}
} else {
return { error: 'ErrorEmptyDownload' };
}
} catch (err) {
return { error: 'ErrorBadFetch' };
}
}

79
src/modules/services/soundcloud.js
View file

@ -1,79 +0,0 @@
import { genericUserAgent, maxAudioDuration } from "../config.js";
let cachedID = {}
async function findClientID() {
try {
let sc = await fetch('https://soundcloud.com/').then((r) => {return r.text()}).catch(() => {return false});
let sc_version = String(sc.match(/<script>window\.__sc_version="[0-9]{10}"<\/script>/)[0].match(/[0-9]{10}/));
if (cachedID.version == sc_version) {
return cachedID.id
} else {
let scripts = sc.matchAll(/<script.+src="(.+)">/g);
let clientid;
for (let script of scripts) {
let url = script[1];
if (url && !url.startsWith('https://a-v2.sndcdn.com')) return;
let scrf = await fetch(url).then((r) => {return r.text()}).catch(() => {return false});
let id = scrf.match(/\("client_id=[A-Za-z0-9]{32}"\)/);
if (id && typeof id[0] === 'string') {
clientid = id[0].match(/[A-Za-z0-9]{32}/)[0];
break;
}
}
cachedID.version = sc_version;
cachedID.id = clientid;
return clientid;
}
} catch (e) {
return false;
}
}
export default async function(obj) {
try {
let html;
if (!obj.author && !obj.song && obj.shortLink) {
html = await fetch(`https://soundcloud.app.goo.gl/${obj.shortLink}/`, {
headers: {"user-agent": genericUserAgent}
}).then((r) => {return r.text()}).catch(() => {return false});
}
if (obj.author && obj.song) {
html = await fetch(`https://soundcloud.com/${obj.author}/${obj.song}${obj.accessKey ? `/s-${obj.accessKey}` : ''}`, {
headers: {"user-agent": genericUserAgent}
}).then((r) => {return r.text()}).catch(() => {return false});
}
if (!html) return { error: 'ErrorCouldntFetch'};
if (html.includes('<script>window.__sc_hydration = ') && html.includes('"format":{"protocol":"progressive","mime_type":"audio/mpeg"},') && html.includes('{"hydratable":"sound","data":')) {
let json = JSON.parse(html.split('{"hydratable":"sound","data":')[1].split('}];</script>')[0])
if (json["media"]["transcodings"]) {
let clientId = await findClientID();
if (clientId) {
let fileUrlBase = json.media.transcodings[0]["url"].replace("/hls", "/progressive")
let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`;
if (fileUrl.substring(0, 54) === "https://api-v2.soundcloud.com/media/soundcloud:tracks:") {
if (json.duration < maxAudioDuration) {
let file = await fetch(fileUrl).then(async (r) => {return (await r.json()).url}).catch(() => {return false});
if (!file) return { error: 'ErrorCouldntFetch' };
return {
urls: file,
audioFilename: `soundcloud_${json.id}`,
fileMetadata: {
title: json.title,
artist: json.user.username,
}
}
} else return { error: ['ErrorLengthAudioConvert', maxAudioDuration / 60000] }
}
} else return { error: 'ErrorSoundCloudNoClientId' }
} else return { error: 'ErrorEmptyDownload' }
} else return { error: ['ErrorBrokenLink', 'soundcloud'] }
} catch (e) {
return { error: 'ErrorBadFetch' };
}
}

116
src/modules/services/tiktok.js
View file

@ -1,116 +0,0 @@
import { genericUserAgent } from "../config.js";
let userAgent = genericUserAgent.split(' Chrome/1')[0]
let config = {
tiktok: {
short: "https://vt.tiktok.com/",
api: "https://api2.musical.ly/aweme/v1/feed/?aweme_id={postId}&version_code=262&app_name=musical_ly&channel=App&device_id=null&os_version=14.4.2&device_platform=iphone&device_type=iPhone9&region=US&carrier_region=US",
},
douyin: {
short: "https://v.douyin.com/",
api: "https://www.iesdouyin.com/web/api/v2/aweme/iteminfo/?item_ids={postId}",
}
}
function selector(j, h, id) {
if (j) {
let t;
switch (h) {
case "tiktok":
t = j["aweme_list"].filter((v) => { if (v["aweme_id"] == id) return true })
break;
case "douyin":
t = j['item_list'].filter((v) => { if (v["aweme_id"] == id) return true })
break;
}
if (t.length > 0) { return t[0] } else return false
} else return false
}
export default async function(obj) {
try {
if (!obj.postId) {
let html = await fetch(`${config[obj.host]["short"]}${obj.id}`, {
redirect: "manual",
headers: { "user-agent": userAgent }
}).then((r) => {return r.text()}).catch(() => {return false});
if (!html) return { error: 'ErrorCouldntFetch' };
if (html.slice(0, 17) === '<a href="https://' && html.includes('/video/')) {
obj.postId = html.split('/video/')[1].split('?')[0].replace("/", '')
} else if (html.slice(0, 32) === '<a href="https://m.tiktok.com/v/' && html.includes('/v/')) {
obj.postId = html.split('/v/')[1].split('.html')[0].replace("/", '')
}
}
if (!obj.postId) return { error: 'ErrorCantGetID' };
let detail;
detail = await fetch(config[obj.host]["api"].replace("{postId}", obj.postId), {
headers: {"user-agent": "TikTok 26.2.0 rv:262018 (iPhone; iOS 14.4.2; en_US) Cronet"}
}).then((r) => {return r.json()}).catch(() => {return false});
detail = selector(detail, obj.host, obj.postId);
if (!detail) return { error: 'ErrorCouldntFetch' }
let video, videoFilename, audioFilename, isMp3, audio, images,
filenameBase = `${obj.host}_${obj.postId}`;
if (obj.host == "tiktok") {
images = detail["image_post_info"] ? detail["image_post_info"]["images"] : false
} else {
images = detail["images"] ? detail["images"] : false
}
if (!obj.isAudioOnly && !images) {
video = obj.host === "tiktok" ? detail["video"]["play_addr"]["url_list"][0] : detail["video"]["play_addr"]["url_list"][0].replace("playwm", "play");
videoFilename = `${filenameBase}_video_nw.mp4` // nw - no watermark
if (!obj.noWatermark) {
video = obj.host === "tiktok" ? detail["video"]["download_addr"]["url_list"][0] : detail['video']['play_addr']['url_list'][0]
videoFilename = `${filenameBase}_video.mp4`
}
} else {
let fallback = obj.host === "douyin" ? detail["video"]["play_addr"]["url_list"][0].replace("playwm", "play") : detail["video"]["play_addr"]["url_list"][0];
if (obj.fullAudio || fallback.includes("music")) {
audio = detail["music"]["play_url"]["url_list"][0]
audioFilename = `${filenameBase}_audio`
} else {
audio = fallback
audioFilename = `${filenameBase}_audio_fv` // fv - from video
}
if (audio.slice(-4) === ".mp3") isMp3 = true;
}
if (video) return {
urls: video,
filename: videoFilename
}
if (images && obj.isAudioOnly) {
return {
urls: audio,
audioFilename: audioFilename,
isAudioOnly: true,
isMp3: isMp3,
}
}
if (images) {
let imageLinks = [];
for (let i in images) {
let sel = obj.host == "tiktok" ? images[i]["display_image"]["url_list"] : images[i]["url_list"];
sel = sel.filter((p) => { if (p.includes(".jpeg?")) return true; })
imageLinks.push({url: sel[0]})
}
return {
picker: imageLinks,
urls: audio,
audioFilename: audioFilename,
isAudioOnly: true,
isMp3: isMp3,
}
}
if (audio) return {
urls: audio,
audioFilename: audioFilename,
isAudioOnly: true,
isMp3: isMp3,
}
} catch (e) {
return { error: 'ErrorBadFetch' };
}
}

16
src/modules/services/tumblr.js
View file

@ -1,16 +0,0 @@
import { genericUserAgent } from "../config.js";
export default async function(obj) {
try {
let user = obj.user ? obj.user : obj.url.split('.')[0].replace('https://', '');
let html = await fetch(`https://${user}.tumblr.com/post/${obj.id}`, {
headers: {"user-agent": genericUserAgent}
}).then((r) => {return r.text()}).catch(() => {return false});
if (!html) return { error: 'ErrorCouldntFetch' };
if (html.includes('property="og:video" content="https://va.media.tumblr.com/')) {
return { urls: `https://va.media.tumblr.com/${html.split('property="og:video" content="https://va.media.tumblr.com/')[1].split('"')[0]}`, audioFilename: `tumblr_${obj.id}_audio` }
} else return { error: 'ErrorEmptyDownload' }
} catch (e) {
return { error: 'ErrorBadFetch' };
}
}

102
src/modules/services/twitter.js
View file

@ -1,102 +0,0 @@
import { genericUserAgent } from "../config.js";
function bestQuality(arr) {
return arr.filter((v) => { if (v["content_type"] === "video/mp4") return true; }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate))[0]["url"].split("?")[0]
}
const apiURL = "https://api.twitter.com/1.1"
export default async function(obj) {
try {
let _headers = {
"user-agent": genericUserAgent,
"authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
"host": "api.twitter.com"
};
let req_act = await fetch(`${apiURL}/guest/activate.json`, {
method: "POST",
headers: _headers
}).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false});
if (!req_act) return { error: 'ErrorCouldntFetch' };
_headers["x-guest-token"] = req_act["guest_token"];
let showURL = `${apiURL}/statuses/show/${obj.id}.json?tweet_mode=extended&include_user_entities=0&trim_user=1&include_entities=0&cards_platform=Web-12&include_cards=1`
if (!obj.spaceId) {
let req_status = await fetch(showURL, { headers: _headers }).then((r) => { return r.status == 200 ? r.json() : false;}).catch((e) => { return false});
if (!req_status) {
_headers.authorization = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw";
delete _headers["x-guest-token"]
req_act = await fetch(`${apiURL}/guest/activate.json`, {
method: "POST",
headers: _headers
}).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false});
if (!req_act) return { error: 'ErrorCouldntFetch' };
_headers["x-guest-token"] = req_act["guest_token"];
req_status = await fetch(showURL, { headers: _headers }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false});
}
if (!req_status) return { error: 'ErrorCouldntFetch' }
if (req_status["extended_entities"] && req_status["extended_entities"]["media"]) {
let single, multiple = [], media = req_status["extended_entities"]["media"];
media = media.filter((i) => { if (i["type"] === "video" || i["type"] === "animated_gif") return true })
if (media.length > 1) {
for (let i in media) { multiple.push({type: "video", thumb: media[i]["media_url_https"], url: bestQuality(media[i]["video_info"]["variants"])}) }
} else if (media.length > 0) {
single = bestQuality(media[0]["video_info"]["variants"])
} else {
return { error: 'ErrorNoVideosInTweet' }
}
if (single) {
return { urls: single, filename: `twitter_${obj.id}.mp4`, audioFilename: `twitter_${obj.id}_audio` }
} else if (multiple) {
return { picker: multiple }
} else {
return { error: 'ErrorNoVideosInTweet' }
}
} else {
return { error: 'ErrorNoVideosInTweet' }
}
} else {
_headers["host"] = "twitter.com"
_headers["content-type"] = "application/json"
let query = {
variables: {"id": obj.spaceId,"isMetatagsQuery":true,"withSuperFollowsUserFields":true,"withDownvotePerspective":false,"withReactionsMetadata":false,"withReactionsPerspective":false,"withSuperFollowsTweetFields":true,"withReplays":true}, features: {"spaces_2022_h2_clipping":true,"spaces_2022_h2_spaces_communities":true,"verified_phone_label_enabled":false,"tweetypie_unmention_optimization_enabled":true,"responsive_web_uc_gql_enabled":true,"vibe_api_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":false,"interactive_text_enabled":true,"responsive_web_text_conversations_enabled":false,"responsive_web_enhance_cards_enabled":true}
}
let AudioSpaceById = await fetch(`https://twitter.com/i/api/graphql/wJ5g4zf7v8qPHSQbaozYuw/AudioSpaceById?variables=${new URLSearchParams(JSON.stringify(query.variables)).toString().slice(0, -1)}&features=${new URLSearchParams(JSON.stringify(query.features)).toString().slice(0, -1)}`, { headers: _headers }).then((r) => {
return r.status == 200 ? r.json() : false;
}).catch((e) => {return false});
if (AudioSpaceById) {
if (AudioSpaceById.data.audioSpace.metadata.is_space_available_for_replay === true) {
let streamStatus = await fetch(`https://twitter.com/i/api/1.1/live_video_stream/status/${AudioSpaceById.data.audioSpace.metadata.media_key}`, { headers: _headers }).then((r) => {return r.status == 200 ? r.json() : false;}).catch(() => {return false;});
if (!streamStatus) return { error: 'ErrorCouldntFetch' };
let participants = AudioSpaceById.data.audioSpace.participants.speakers
let listOfParticipants = `Twitter Space speakers: `
for (let i in participants) {
listOfParticipants += `@${participants[i]["twitter_screen_name"]}, `
}
listOfParticipants = listOfParticipants.slice(0, -2);
return {
urls: streamStatus.source.noRedirectPlaybackUrl,
audioFilename: `twitterspaces_${obj.spaceId}`,
isAudioOnly: true,
fileMetadata: {
title: AudioSpaceById.data.audioSpace.metadata.title,
artist: `Twitter Space by @${AudioSpaceById.data.audioSpace.metadata.creator_results.result.legacy.screen_name}`,
comment: listOfParticipants,
// cover: AudioSpaceById.data.audioSpace.metadata.creator_results.result.legacy.profile_image_url_https.replace("_normal", "")
}
}
} else {
return { error: 'TwitterSpaceWasntRecorded' };
}
} else {
return { error: 'ErrorEmptyDownload' }
}
}
} catch (err) {
return { error: 'ErrorBadFetch' };
}
}

80
src/modules/services/vimeo.js
View file

@ -1,80 +0,0 @@
import { quality, services } from "../config.js";
export default async function(obj) {
try {
let api = await fetch(`https://player.vimeo.com/video/${obj.id}/config`).then((r) => {return r.json()}).catch(() => {return false});
if (!api) return { error: 'ErrorCouldntFetch' };
let downloadType = "";
if (JSON.stringify(api).includes('"progressive":[{')) {
downloadType = "progressive";
} else if (JSON.stringify(api).includes('"files":{"dash":{"')) downloadType = "dash";
switch(downloadType) {
case "progressive":
let all = api["request"]["files"]["progressive"].sort((a, b) => Number(b.width) - Number(a.width));
let best = all[0]
try {
if (obj.quality != "max") {
let pref = parseInt(quality[obj.quality], 10)
for (let i in all) {
let currQuality = parseInt(all[i]["quality"].replace('p', ''), 10)
if (currQuality < pref) {
break;
} else if (currQuality == pref) {
best = all[i]
}
}
}
} catch (e) {
best = all[0]
}
return { urls: best["url"], filename: `tumblr_${obj.id}.mp4` };
case "dash":
let masterJSONURL = api["request"]["files"]["dash"]["cdns"]["akfire_interconnect_quic"]["url"];
let masterJSON = await fetch(masterJSONURL).then((r) => {return r.json()}).catch(() => {return false});
if (!masterJSON) return { error: 'ErrorCouldntFetch' };
if (masterJSON.video) {
let type = "";
if (masterJSON.base_url.includes("parcel")) {
type = "parcel"
} else if (masterJSON.base_url == "../") {
type = "chop"
}
let masterJSON_Video = masterJSON.video.sort((a, b) => Number(b.width) - Number(a.width));
let masterJSON_Audio = masterJSON.audio.sort((a, b) => Number(b.bitrate) - Number(a.bitrate)).filter((a)=> {if (a['mime_type'] === "audio/mp4") return true;});
let bestVideo = masterJSON_Video[0]
let bestAudio = masterJSON_Audio[0]
switch (type) {
case "parcel":
if (obj.quality != "max") {
let pref = parseInt(quality[obj.quality], 10)
for (let i in masterJSON_Video) {
let currQuality = parseInt(services.vimeo.resolutionMatch[masterJSON_Video[i]["width"]], 10)
if (currQuality < pref) {
break;
} else if (currQuality == pref) {
bestVideo = masterJSON_Video[i]
}
}
}
let baseUrl = masterJSONURL.split("/sep/")[0]
let videoUrl = `${baseUrl}/parcel/video/${bestVideo.index_segment.split('?')[0]}`;
let audioUrl = `${baseUrl}/parcel/audio/${bestAudio.index_segment.split('?')[0]}`;
return { urls: [videoUrl, audioUrl], audioFilename: `vimeo_${obj.id}_audio`, filename: `vimeo_${obj.id}_${bestVideo["width"]}x${bestVideo["height"]}.mp4` }
case "chop": // TO-DO: support chop type of streams
default:
return { error: 'ErrorEmptyDownload' }
}
} else {
return { error: 'ErrorEmptyDownload' }
}
default:
return { error: 'ErrorEmptyDownload' }
}
} catch (e) {
return { error: 'ErrorBadFetch' };
}
}

58
src/modules/services/vk.js
View file

@ -1,58 +0,0 @@
import { xml2json } from "xml-js";
import { genericUserAgent, maxVideoDuration, services } from "../config.js";
import selectQuality from "../stream/selectQuality.js";
export default async function(obj) {
try {
let html;
html = await fetch(`https://vk.com/video-${obj.userId}_${obj.videoId}`, {
headers: {"user-agent": genericUserAgent}
}).then((r) => {return r.text()}).catch(() => {return false});
if (!html) return { error: 'ErrorCouldntFetch' };
if (html.includes(`{"lang":`)) {
let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]);
if (js["mvData"]["is_active_live"] == '0') {
if (js["mvData"]["duration"] <= maxVideoDuration / 1000) {
let mpd = JSON.parse(xml2json(js["player"]["params"][0]["manifest"], { compact: true, spaces: 4 }));
let repr = mpd["MPD"]["Period"]["AdaptationSet"]["Representation"];
if (!mpd["MPD"]["Period"]["AdaptationSet"]["Representation"]) {
repr = mpd["MPD"]["Period"]["AdaptationSet"][0]["Representation"];
}
let attr = repr[repr.length - 1]["_attributes"];
let selectedQuality;
let qualities = Object.keys(services.vk.quality_match);
for (let i in qualities) {
if (qualities[i] == attr["height"]) {
selectedQuality = `url${attr["height"]}`;
break;
}
if (qualities[i] == attr["width"]) {
selectedQuality = `url${attr["width"]}`;
break;
}
}
let maxQuality = js["player"]["params"][0][selectedQuality].split('type=')[1].slice(0, 1)
let userQuality = selectQuality('vk', obj.quality, Object.entries(services.vk.quality_match).reduce((r, [k, v]) => { r[v] = k; return r; })[maxQuality]);
let userRepr = repr[services.vk.representation_match[userQuality]]["_attributes"];
if (selectedQuality in js["player"]["params"][0]) {
return {
urls: js["player"]["params"][0][`url${userQuality}`],
filename: `vk_${obj.userId}_${obj.videoId}_${userRepr["width"]}x${userRepr['height']}.mp4`
};
} else {
return { error: 'ErrorEmptyDownload' };
}
} else {
return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
}
} else {
return { error: 'ErrorLiveVideo' };
}
} else {
return { error: 'ErrorEmptyDownload' };
}
} catch (err) {
return { error: 'ErrorBadFetch' };
}
}

98
src/modules/services/youtube.js
View file

@ -1,98 +0,0 @@
import ytdl from "better-ytdl-core";
import { maxVideoDuration, quality as mq } from "../config.js";
import selectQuality from "../stream/selectQuality.js";
export default async function(obj) {
try {
let infoInitial = await ytdl.getInfo(obj.id);
if (infoInitial) {
let info = infoInitial.formats;
if (!info[0]["isLive"]) {
let videoMatch = [], fullVideoMatch = [], video = [], audio = info.filter((a) => {
if (!a["isHLS"] && !a["isDashMPD"] && a["hasAudio"] && !a["hasVideo"] && a["container"] == obj.format) return true;
}).sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
if (!obj.isAudioOnly) {
video = info.filter((a) => {
if (!a["isHLS"] && !a["isDashMPD"] && a["hasVideo"] && a["container"] == obj.format) {
if (obj.quality != "max") {
if (a["hasAudio"] && mq[obj.quality] == a["height"]) {
fullVideoMatch.push(a)
} else if (!a["hasAudio"] && mq[obj.quality] == a["height"]) {
videoMatch.push(a);
}
}
return true
}
}).sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
if (obj.quality != "max") {
if (videoMatch.length == 0) {
let ss = selectQuality("youtube", obj.quality, video[0]["qualityLabel"].slice(0, 5).replace('p', '').trim())
videoMatch = video.filter((a) => {
if (a["qualityLabel"].slice(0, 5).replace('p', '').trim() == ss) return true;
})
} else if (fullVideoMatch.length > 0) {
videoMatch = [fullVideoMatch[0]]
}
} else videoMatch = [video[0]];
if (obj.quality == "los") videoMatch = [video[video.length - 1]];
}
let generalMeta = {
title: infoInitial.videoDetails.title,
artist: infoInitial.videoDetails.ownerChannelName.replace("- Topic", "").trim(),
}
if (audio[0]["approxDurationMs"] <= maxVideoDuration) {
if (!obj.isAudioOnly && videoMatch.length > 0) {
if (video.length > 0 && audio.length > 0) {
if (videoMatch[0]["hasVideo"] && videoMatch[0]["hasAudio"]) {
return {
type: "bridge", urls: videoMatch[0]["url"], time: videoMatch[0]["approxDurationMs"],
filename: `youtube_${obj.id}_${videoMatch[0]["width"]}x${videoMatch[0]["height"]}.${obj.format}`
};
} else {
return {
type: "render", urls: [videoMatch[0]["url"], audio[0]["url"]], time: videoMatch[0]["approxDurationMs"],
filename: `youtube_${obj.id}_${videoMatch[0]["width"]}x${videoMatch[0]["height"]}.${obj.format}`
};
}
} else {
return { error: 'ErrorBadFetch' };
}
} else if (!obj.isAudioOnly) {
return {
type: "render", urls: [video[0]["url"], audio[0]["url"]], time: video[0]["approxDurationMs"],
filename: `youtube_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.${video[0]["container"]}`
};
} else if (audio.length > 0) {
let r = {
type: "render",
isAudioOnly: true,
urls: audio[0]["url"],
audioFilename: `youtube_${obj.id}_audio`,
fileMetadata: generalMeta
};
if (infoInitial.videoDetails.description) {
let isAutoGenAudio = infoInitial.videoDetails.description.startsWith("Provided to YouTube by");
if (isAutoGenAudio) {
let descItems = infoInitial.videoDetails.description.split("\n\n")
r.fileMetadata.album = descItems[2]
r.fileMetadata.copyright = descItems[3]
if (descItems[4].startsWith("Released on:")) r.fileMetadata.date = descItems[4].replace("Released on: ", '').trim();
}
}
return r
} else {
return { error: 'ErrorBadFetch' };
}
} else {
return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
}
} else {
return { error: 'ErrorLiveVideo' };
}
} else {
return { error: 'ErrorCantConnectToServiceAPI' };
}
} catch (e) {
return { error: 'ErrorBadFetch' };
}
}

21
src/modules/setup.js
View file

@ -33,22 +33,15 @@ console.log(
)
rl.question(q, r1 => {
if (r1) {
ob['selfURL'] = `https://${r1}/`
} else {
ob['selfURL'] = `http://localhost`
}
console.log(Bright("\nGreat! Now, what's the port it'll be running on? (9000)"))
rl.question(q, r2 => {
if (!r1 && !r2) {
ob['selfURL'] = `http://localhost:9000/`
ob['port'] = 9000
} else if (!r1 && r2) {
ob['selfURL'] = `http://localhost:${r2}/`
ob['port'] = r2
} else {
ob['port'] = r2
}
if (r1) ob['selfURL'] = `https://${r1}/`
console.log(Bright("\nGreat! Now, what's the port it'll be running on? (9000)"))
rl.question(q, r2 => {
if (r2) ob['port'] = r2
if (!r1 && r2) ob['selfURL'] = `http://localhost:${r2}/`
final()
});
})

15
src/modules/stream/manage.js
View file

@ -43,16 +43,15 @@ export function createStream(obj) {
export function verifyStream(ip, id, hmac, exp) {
try {
let streamInfo = streamCache.get(id);
if (streamInfo) {
let ghmac = sha256(`${id},${streamInfo.service},${ip},${exp}`, salt);
if (hmac == ghmac && exp.toString() == streamInfo.exp && ghmac == streamInfo.hmac && ip == streamInfo.ip && exp > Math.floor(new Date().getTime())) {
return streamInfo;
} else {
return { error: 'Unauthorized', status: 401 };
}
} else {
if (!streamInfo) {
return { error: 'this stream token does not exist', status: 400 };
}
let ghmac = sha256(`${id},${streamInfo.service},${ip},${exp}`, salt);
if (String(hmac) === ghmac && String(exp) === String(streamInfo.exp) && ghmac === String(streamInfo.hmac)
&& String(ip) === streamInfo.ip && Number(exp) > Math.floor(new Date().getTime())) {
return streamInfo;
}
return { error: 'Unauthorized', status: 401 };
} catch (e) {
return { status: 500, body: { status: "error", text: "Internal Server Error" } };
}

10
src/modules/stream/selectQuality.js
View file

@ -1,5 +1,6 @@
import { services, quality as mq } from "../config.js";
// TO-DO: remake entirety of this module to be more of how quality picking is done in vimeo module
function closest(goal, array) {
return array.sort().reduce(function (prev, curr) {
return (Math.abs(curr - goal) < Math.abs(prev - goal) ? curr : prev);
@ -7,17 +8,15 @@ function closest(goal, array) {
}
export default function(service, quality, maxQuality) {
if (quality == "max") return maxQuality;
if (quality === "max") return maxQuality;
quality = parseInt(mq[quality], 10)
maxQuality = parseInt(maxQuality, 10)
if (quality >= maxQuality || quality == maxQuality) return maxQuality;
if (quality >= maxQuality || quality === maxQuality) return maxQuality;
if (quality < maxQuality) {
if (services[service]["quality"][quality]) {
return quality
} else {
if (!services[service]["quality"][quality]) {
let s = Object.keys(services[service]["quality_match"]).filter((q) => {
if (q <= quality) {
return true
@ -25,5 +24,6 @@ export default function(service, quality, maxQuality) {
})
return closest(quality, s)
}
return quality
}
}

12
src/modules/stream/stream.js
View file

@ -5,10 +5,14 @@ import { streamAudioOnly, streamDefault, streamLiveRender, streamVideoOnly } fro
export default function(res, ip, id, hmac, exp) {
try {
let streamInfo = verifyStream(ip, id, hmac, exp);
if (!streamInfo.error) {
if (streamInfo.error) {
res.status(streamInfo.status).json(apiJSON(0, { t: streamInfo.error }).body);
return;
}
if (streamInfo.isAudioOnly && streamInfo.type !== "bridge") {
streamAudioOnly(streamInfo, res);
} else {
return;
}
switch (streamInfo.type) {
case "render":
streamLiveRender(streamInfo, res);
@ -20,10 +24,6 @@ export default function(res, ip, id, hmac, exp) {
streamDefault(streamInfo, res);
break;
}
}
} else {
res.status(streamInfo.status).json(apiJSON(0, { t: streamInfo.error }).body);
}
} catch (e) {
res.status(500).json({ status: "error", text: "Internal Server Error" });
}

24
src/modules/stream/types.js
View file

@ -15,10 +15,10 @@ export function streamDefault(streamInfo, res) {
},
isStream: true
});
stream.pipe(res).on('error', (err) => {
stream.pipe(res).on('error', () => {
res.end();
});
stream.on('error', (err) => {
stream.on('error', () => {
res.end();
});
} catch (e) {
@ -27,7 +27,10 @@ export function streamDefault(streamInfo, res) {
}
export function streamLiveRender(streamInfo, res) {
try {
if (streamInfo.urls.length === 2) {
if (streamInfo.urls.length !== 2) {
res.end();
return;
}
let format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1], args = [
'-loglevel', '-8',
'-i', streamInfo.urls[0],
@ -48,18 +51,17 @@ export function streamLiveRender(streamInfo, res) {
res.setHeader('Connection', 'keep-alive');
res.setHeader('Content-Disposition', `attachment; filename="${streamInfo.filename}"`);
ffmpegProcess.stdio[3].pipe(res);
ffmpegProcess.on('disconnect', () => ffmpegProcess.kill());
ffmpegProcess.on('close', () => ffmpegProcess.kill());
ffmpegProcess.on('exit', () => ffmpegProcess.kill());
res.on('finish', () => ffmpegProcess.kill());
res.on('close', () => ffmpegProcess.kill());
ffmpegProcess.on('error', (err) => {
ffmpegProcess.on('error', () => {
ffmpegProcess.kill();
res.end();
});
} else {
res.end();
}
} catch (e) {
res.end();
}
@ -93,12 +95,13 @@ export function streamAudioOnly(streamInfo, res) {
res.setHeader('Connection', 'keep-alive');
res.setHeader('Content-Disposition', `attachment; filename="${streamInfo.filename}.${streamInfo.audioFormat}"`);
ffmpegProcess.stdio[3].pipe(res);
ffmpegProcess.on('disconnect', () => ffmpegProcess.kill());
ffmpegProcess.on('close', () => ffmpegProcess.kill());
ffmpegProcess.on('exit', () => ffmpegProcess.kill());
res.on('finish', () => ffmpegProcess.kill());
res.on('close', () => ffmpegProcess.kill());
ffmpegProcess.on('error', (err) => {
ffmpegProcess.on('error', () => {
ffmpegProcess.kill();
res.end();
});
@ -113,7 +116,7 @@ export function streamVideoOnly(streamInfo, res) {
'-i', streamInfo.urls,
'-c', 'copy', '-an'
]
if (format == "mp4") args.push('-movflags', 'faststart+frag_keyframe+empty_moov')
if (format === "mp4") args.push('-movflags', 'faststart+frag_keyframe+empty_moov')
args.push('-f', format, 'pipe:3');
const ffmpegProcess = spawn(ffmpeg, args, {
windowsHide: true,
@ -125,12 +128,13 @@ export function streamVideoOnly(streamInfo, res) {
res.setHeader('Connection', 'keep-alive');
res.setHeader('Content-Disposition', `attachment; filename="${streamInfo.filename.split('.')[0]}_mute.${format}"`);
ffmpegProcess.stdio[3].pipe(res);
ffmpegProcess.on('disconnect', () => ffmpegProcess.kill());
ffmpegProcess.on('close', () => ffmpegProcess.kill());
ffmpegProcess.on('exit', () => ffmpegProcess.kill());
res.on('finish', () => ffmpegProcess.kill());
res.on('close', () => ffmpegProcess.kill());
ffmpegProcess.on('error', (err) => {
ffmpegProcess.on('error', () => {
ffmpegProcess.kill();
res.end();
});

19
src/modules/sub/currentCommit.js
View file

@ -1,10 +1,23 @@
import { execSync } from "child_process";
let commit, commitInfo, branch;
export function shortCommit() {
return execSync('git rev-parse --short HEAD').toString().trim()
if (commit) return commit;
let c = execSync('git rev-parse --short HEAD').toString().trim();
commit = c;
return c
}
export function getCommitInfo() {
let d = execSync(`git show -s --format='%s;;;%B'`).toString().trim().replace(/[\r\n]/gm, '\n').split(';;;')
d[1] = d[1].replace(d[0], '').trim().toString().replace(/[\r\n]/gm, '<br>')
if (commitInfo) return commitInfo;
let d = execSync(`git show -s --format='%s;;;%B'`).toString().trim().replace(/[\r\n]/gm, '\n').split(';;;');
d[1] = d[1].replace(d[0], '').trim().toString().replace(/[\r\n]/gm, '<br>');
commitInfo = d;
return d
}
export function getCurrentBranch() {
if (branch) return branch;
let b = execSync('git branch --show-current').toString().trim();
branch = b;
return b
}

5
src/modules/sub/errors.js
View file

@ -3,6 +3,9 @@ import loc from "../../localization/manager.js";
export function errorUnsupported(lang) {
return loc(lang, 'ErrorUnsupported');
}
export function genericError(lang, host) {
export function brokenLink(lang, host) {
return loc(lang, 'ErrorBrokenLink', host);
}
export function genericError(lang, host) {
return loc(lang, 'ErrorBadFetch', host);
}

17
src/modules/sub/utils.js
View file

@ -64,6 +64,7 @@ export function msToTime(d) {
export function cleanURL(url, host) {
let forbiddenChars = ['}', '{', '(', ')', '\\', '%', '>', '<', '^', '*', '!', '~', ';', ':', ',', '`', '[', ']', '#', '$', '"', "'", "@"]
switch(host) {
case "vk":
case "youtube":
url = url.split('&')[0];
break;
@ -103,8 +104,9 @@ export function checkJSONPost(obj) {
}
try {
let objKeys = Object.keys(obj);
if (objKeys.length < 8 && obj.url) {
if (!(objKeys.length <= 8 && obj.url)) return false;
let defKeys = Object.keys(def);
for (let i in objKeys) {
if (String(objKeys[i]) !== "url" && defKeys.includes(objKeys[i])) {
if (apiVar.booleanOnly.includes(objKeys[i])) {
@ -114,15 +116,14 @@ export function checkJSONPost(obj) {
}
}
}
obj["url"] = decodeURIComponent(String(obj["url"]))
obj["url"] = decodeURIComponent(String(obj["url"]));
let hostname = obj["url"].replace("https://", "").replace(' ', '').split('&')[0].split("/")[0].split("."),
host = hostname[hostname.length - 2]
def["url"] = encodeURIComponent(cleanURL(obj["url"], host))
host = hostname[hostname.length - 2];
def["url"] = encodeURIComponent(cleanURL(obj["url"], host));
return def
} else {
} catch (e) {
return false
}
} catch (e) {
return false;
}
}

71
src/test/test.js Normal file
View file

@ -0,0 +1,71 @@
import "dotenv/config";
import { getJSON } from "../modules/api.js";
import { services } from "../modules/config.js";
import loadJSON from "../modules/sub/loadJSON.js";
import { checkJSONPost } from "../modules/sub/utils.js";
let tests = loadJSON('./src/test/tests.json');
let noTest = [];
let failed = [];
let success = 0;
function addToFail(service, testName, url, status, response) {
failed.push({
service: service,
name: testName,
url: url,
status: status,
response: response
})
}
for (let i in services) {
if (tests[i]) {
console.log(`\nRunning tests for ${i}...\n`)
for (let k = 0; k < tests[i].length; k++) {
let test = tests[i][k];
console.log(`Running test ${k+1}: ${test.name}`);
console.log('params:');
let params = {...{url: test.url}, ...test.params};
console.log(params);
let chck = checkJSONPost(params);
if (chck) {
chck["ip"] = "d21ec524bc2ade41bef569c0361ac57728c69e2764b5cb3cb310fe36568ca53f"; // random sha256
let j = await getJSON(chck["url"], "en", chck);
console.log('\nReceived:');
console.log(j)
if (j.status === test.expected.code && j.body.status === test.expected.status) {
console.log("\n✅ Success.\n");
success++
} else {
console.log(`\n❌ Fail. Expected: ${test.expected.code} & ${test.expected.status}, received: ${j.status} & ${j.body.status}\n`);
addToFail(i, test.name, test.url, j.body.status, j)
}
} else {
console.log("\n❌ couldn't validate the request JSON.\n");
addToFail(i, test.name, test.url, "unknown", {})
}
}
console.log("\n\n")
} else {
console.warn(`No tests found for ${i}.`);
noTest.push(i)
}
}
console.log(`✅ ${success} tests succeeded.`);
console.log(`❌ ${failed.length} tests failed.`);
console.log(`❔ ${noTest.length} services weren't tested.`);
if (failed.length > 0) {
console.log(`\nFailed tests:`);
console.log(failed)
}
if (noTest.length > 0) {
console.log(`\nMissing tests:`);
console.log(noTest)
}

742
src/test/tests.json Normal file
View file

@ -0,0 +1,742 @@
{
"twitter": [{
"name": "regular video",
"url": "https://twitter.com/TwitterSpaces/status/1526955853743546372?s=20",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "redirect"
}
}, {
"name": "embedded twitter video",
"url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "redirect"
}
}, {
"name": "mixed media (image + gif)",
"url": "https://twitter.com/Twitter/status/1580661436132757506?s=20",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "redirect"
}
}, {
"name": "picker: mixed media (3 gifs + image)",
"url": "https://twitter.com/emerald_pedrod/status/1582418163521581063?s=20",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "picker"
}
}, {
"name": "audio from embedded twitter video (mp3, isAudioOnly)",
"url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
"params": {
"aFormat": "mp3",
"isAudioOnly": true,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "audio from embedded twitter video (best, isAudioOnly)",
"url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
"params": {
"aFormat": "best",
"isAudioOnly": true,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "audio from embedded twitter video (ogg, isAudioOnly, isAudioMuted)",
"url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
"params": {
"aFormat": "best",
"isAudioOnly": true,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "muted embedded twitter video",
"url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "inexistent post",
"url": "https://twitter.com/test/status/9487653",
"params": {
"aFormat": "best",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 400,
"status": "error"
}
}, {
"name": "post with no media content",
"url": "https://twitter.com/elonmusk/status/1604617643973124097?s=20",
"params": {
"aFormat": "best",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 400,
"status": "error"
}
}, {
"name": "recorded space by nyc (best)",
"url": "https://twitter.com/i/spaces/1gqxvyLoYQkJB",
"params": {
"aFormat": "best",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "recorded space by nyc (mp3)",
"url": "https://twitter.com/i/spaces/1gqxvyLoYQkJB",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "recorded space by nyc (wav, isAudioMuted)",
"url": "https://twitter.com/i/spaces/1gqxvyLoYQkJB",
"params": {
"aFormat": "wav",
"isAudioOnly": false,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "recorded space by service95 & dualipa (mp3, isAudioMuted, isAudioOnly)",
"url": "https://twitter.com/i/spaces/1nAJErvvVXgxL",
"params": {
"aFormat": "mp3",
"isAudioOnly": true,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "unavailable space",
"url": "https://twitter.com/i/spaces/1OwGWwjRjVVGQ?s=20",
"params": {},
"expected": {
"code": 400,
"status": "error"
}
}, {
"name": "inexistent space",
"url": "https://twitter.com/i/spaces/10Wkie2j29iiI",
"params": {},
"expected": {
"code": 400,
"status": "error"
}
}],
"soundcloud": [{
"name": "public song (best)",
"url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing",
"params": {
"aFormat": "best",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "public song (mp3, isAudioMuted)",
"url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "private song",
"url": "https://soundcloud.com/4kayy/unhappy-new-year-prod4kay/s-9bKbvwLdRWG",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "private song (wav, isAudioMuted)",
"url": "https://soundcloud.com/4kayy/unhappy-new-year-prod4kay/s-9bKbvwLdRWG",
"params": {
"aFormat": "wav",
"isAudioOnly": false,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "private song (ogg, isAudioMuted, isAudioOnly)",
"url": "https://soundcloud.com/4kayy/unhappy-new-year-prod4kay/s-9bKbvwLdRWG",
"params": {
"aFormat": "ogg",
"isAudioOnly": true,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}],
"youtube": [{
"name": "4k video (mp4, hig)",
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
"params": {
"vFormat": "mp4",
"vQuality": "hig",
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "4k video (webm, mid)",
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
"params": {
"vFormat": "webm",
"vQuality": "mid",
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "4k video (mp4, max)",
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
"params": {
"vFormat": "mp4",
"vQuality": "max",
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "4k video (webm, max)",
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
"params": {
"vFormat": "webm",
"vQuality": "max",
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "4k video (webm, max, isAudioMuted)",
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
"params": {
"vFormat": "webm",
"vQuality": "max",
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "4k video (mp4, max, isAudioMuted)",
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
"params": {
"vFormat": "webm",
"vQuality": "max",
"aFormat": "mp3",
"isAudioOnly": true,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "4k video (mp4, max, isAudioMuted, isAudioOnly, mp3)",
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
"params": {
"vFormat": "webm",
"vQuality": "max",
"aFormat": "mp3",
"isAudioOnly": true,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "4k video (mp4, max, isAudioMuted, isAudioOnly, best)",
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
"params": {
"vFormat": "webm",
"vQuality": "max",
"aFormat": "best",
"isAudioOnly": true,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "music (mp3, isAudioOnly, isAudioMuted)",
"url": "https://music.youtube.com/watch?v=5rGTsvZCEdk&feature=share",
"params": {
"aFormat": "mp3",
"isAudioOnly": true,
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "music (mp3)",
"url": "https://music.youtube.com/watch?v=5rGTsvZCEdk&feature=share",
"params": {
"aFormat": "mp3",
"isAudioOnly": false,
"isAudioMuted": false
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "short, defaults",
"url": "https://www.youtube.com/shorts/r5FpeOJItbw",
"params": {},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "inexistent video",
"url": "https://youtube.com/watch?v=gnjuHYWGEW",
"params": {},
"expected": {
"code": 400,
"status": "error"
}
}],
"vk": [{
"name": "clip, defaults",
"url": "https://vk.com/clip-57274055_456239788",
"params": {},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "clip, low",
"url": "https://vk.com/clip-57274055_456239788",
"params": {
"vQuality": "low"
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "clip different link, max",
"url": "https://vk.com/clips-57274055?z=clip-57274055_456239788",
"params": {
"vQuality": "max"
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "video, defaults",
"url": "https://vk.com/video-57274055_456239399",
"params": {},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "inexistent video",
"url": "https://vk.com/video-53333333_456233333",
"params": {},
"expected": {
"code": 400,
"status": "error"
}
}],
"douyin": [{
"name": "short link video, with watermark",
"url": "https://v.douyin.com/2p4Aya7/",
"params": {},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "short link video (isNoTTWatermark)",
"url": "https://v.douyin.com/2p4Aya7/",
"params": {
"isNoTTWatermark": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "short link video (isAudioOnly)",
"url": "https://v.douyin.com/2p4Aya7/",
"params": {
"isAudioOnly": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "short link video (isAudioOnly, isTTFullAudio)",
"url": "https://v.douyin.com/2p4Aya7/",
"params": {
"isAudioOnly": true,
"isTTFullAudio": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "long link video (isNoTTWatermark)",
"url": "https://www.douyin.com/video/7120601033314716968",
"params": {
"isNoTTWatermark": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "images",
"url": "https://v.douyin.com/MdVwo31/",
"params": {},
"expected": {
"code": 200,
"status": "picker"
}
}, {
"name": "long link inexistent",
"url": "https://www.douyin.com/video/7120851458451417478",
"params": {},
"expected": {
"code": 400,
"status": "error"
}
}, {
"name": "short link inexistent",
"url": "https://v.douyin.com/2p4ewa7/",
"params": {},
"expected": {
"code": 400,
"status": "error"
}
}],
"tiktok": [{
"name": "short link (vt) video, with watermark",
"url": "https://vt.tiktok.com/ZS85U86aa/",
"params": {},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "short link (vt) video (isNoTTWatermark)",
"url": "https://vt.tiktok.com/ZS85U86aa/",
"params": {
"isNoTTWatermark": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "short link (vm) video (isAudioOnly)",
"url": "https://vm.tiktok.com/ZMYrYAf34/",
"params": {
"isAudioOnly": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "short link (vm) video (isAudioOnly, isTTFullAudio)",
"url": "https://vm.tiktok.com/ZMYrYAf34/",
"params": {
"isAudioOnly": true,
"isTTFullAudio": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "long link video (isNoTTWatermark)",
"url": "https://www.tiktok.com/@fatfatmillycat/video/7195741644585454894",
"params": {
"isNoTTWatermark": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "images",
"url": "https://vt.tiktok.com/ZS8JP89eB/",
"params": {},
"expected": {
"code": 200,
"status": "picker"
}
}, {
"name": "long link inexistent",
"url": "https://www.tiktok.com/@blablabla/video/7120851458451417478",
"params": {},
"expected": {
"code": 400,
"status": "error"
}
}, {
"name": "short link inexistent",
"url": "https://vt.tiktok.com/2p4ewa7/",
"params": {},
"expected": {
"code": 400,
"status": "error"
}
}],
"bilibili": [{
"name": "1080p video",
"url": "https://www.bilibili.com/video/BV18i4y1m7xV/",
"params": {},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "1080p video muted",
"url": "https://www.bilibili.com/video/BV18i4y1m7xV/",
"params": {
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "1080p vertical video",
"url": "https://www.bilibili.com/video/BV1uu411z7VV/",
"params": {},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "1080p vertical video muted",
"url": "https://www.bilibili.com/video/BV1uu411z7VV/",
"params": {
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}],
"tumblr": [{
"name": "at.tumblr link",
"url": "https://at.tumblr.com/music/704177038274281472/n7x7pr7x4w2b",
"params": {},
"expected": {
"code": 200,
"status": "redirect"
}
}, {
"name": "user subdomain link",
"url": "https://garfield-69.tumblr.com/post/696499862852780032",
"params": {},
"expected": {
"code": 200,
"status": "redirect"
}
}, {
"name": "web app link",
"url": "https://www.tumblr.com/rongzhi/707729381162958848/english-added-by-me?source=share",
"params": {},
"expected": {
"code": 200,
"status": "redirect"
}
}],
"vimeo": [{
"name": "4k progressive",
"url": "https://vimeo.com/288386543",
"params": {
"vQuality": "max"
},
"expected": {
"code": 200,
"status": "redirect"
}
}, {
"name": "720p progressive",
"url": "https://vimeo.com/288386543",
"params": {
"vQuality": "mid"
},
"expected": {
"code": 200,
"status": "redirect"
}
}, {
"name": "1080p dash parcel",
"url": "https://vimeo.com/774694040",
"params": {
"vQuality": "hig"
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "720p dash parcel",
"url": "https://vimeo.com/774694040",
"params": {
"vQuality": "mid"
},
"expected": {
"code": 200,
"status": "stream"
}
}],
"reddit": [{
"name": "video with audio",
"url": "https://www.reddit.com/r/catvideos/comments/b2rygq/my_new_kittens_1st_day_checking_out_his_new_home/?utm_source=share&utm_medium=web2x&context=3",
"params": {},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "video with audio (isAudioOnly)",
"url": "https://www.reddit.com/r/catvideos/comments/b2rygq/my_new_kittens_1st_day_checking_out_his_new_home/?utm_source=share&utm_medium=web2x&context=3",
"params": {
"isAudioOnly": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "video with audio (isAudioMuted)",
"url": "https://www.reddit.com/r/catvideos/comments/b2rygq/my_new_kittens_1st_day_checking_out_his_new_home/?utm_source=share&utm_medium=web2x&context=3",
"params": {
"isAudioMuted": true
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "video without audio",
"url": "https://www.reddit.com/r/catvideos/comments/ftoeo7/luna_doesnt_want_to_be_bothered_while_shes_napping/?utm_source=share&utm_medium=web2x&context=3",
"params": {},
"expected": {
"code": 200,
"status": "redirect"
}
}, {
"name": "actual gif, not looping video",
"url": "https://www.reddit.com/r/whenthe/comments/109wqy1/god_really_did_some_trolling/?utm_source=share&utm_medium=web2x&context=3",
"params": {},
"expected": {
"code": 200,
"status": "redirect"
}
}]
}