tumblr audio, youtube vr, updated setup script, further mitosis accommodations

This commit is contained in:
wukko 2023-06-05 12:43:04 +06:00
parent 1014ee3413
commit 55f1e4b704
10 changed files with 173 additions and 51 deletions

View file

@ -1,6 +1,6 @@
# cobalt
Best way to save what you love.
Main instance: [co.wukko.me](https://co.wukko.me/)
Live web app: [co.wukko.me](https://co.wukko.me/)
![cobalt logo with repeated logo pattern background](https://raw.githubusercontent.com/wukko/cobalt/current/src/front/icons/pattern.png "cobalt logo with repeated logo pattern background")
@ -21,14 +21,14 @@ Paste the link, get the video, move on. It's that simple. Just how it should be.
| Reddit | ✅ | ✅ | ✅ | Support for GIFs and videos. |
| SoundCloud | | ✅ | | Audio metadata, downloads from private links. |
| TikTok | ✅ | ✅ | ✅ | Supports downloads of: videos with or without watermark, images from slideshow without watermark, full (original) audios. |
| Tumblr | ✅ | ✅ | ✅ | |
| Tumblr | ✅ | ✅ | ✅ | Support for audio file downloads. |
| Twitter | ✅ | ✅ | ✅ | Ability to pick what to save from multi-media tweets. |
| Twitter Spaces | | ✅ | | Audio metadata with all participants and other info. |
| Vimeo | ✅ | ✅ | ✅ | Audio downloads are only available for dash files. |
| Vine Archive | ✅ | ✅ | ✅ | |
| VK Videos | ✅ | ❌ | ❌ | |
| VK Clips | ✅ | ❌ | ❌ | |
| YouTube Videos & Shorts | ✅ | ✅ | ✅ | Support for 8K, 4K, HDR, and high FPS videos. Audio metadata & dubs. h264/av1/vp9 codecs. |
| YouTube Videos & Shorts | ✅ | ✅ | ✅ | Support for 8K, 4K, HDR, VR, and high FPS videos. Audio metadata & dubs. h264/av1/vp9 codecs. |
| YouTube Music | | ✅ | | Audio metadata. |
This list is not final and keeps expanding over time, make sure to check it once in a while!
@ -36,6 +36,7 @@ This list is not final and keeps expanding over time, make sure to check it once
## cobalt API
cobalt has an open API that you can use in your projects for **free**.
It's easy and straightforward to use, [check out the docs](https://github.com/wukko/cobalt/blob/current/docs/API.md) and see for yourself.
Feel free to use the main API instance ([co.wuk.sh](https://co.wuk.sh/)) in your projects.
## How to contribute translations
You can translate cobalt to any language you want on [cobalt's Crowdin](https://crowdin-co.wukko.me/). Feel free to ignore QA errors if you think you know better. If you don't see a language you want to translate cobalt to, open an issue, and I'll add it to Crowdin.
@ -62,6 +63,8 @@ Setup script installs all needed `npm` dependencies, but you have to install `No
3. Run cobalt via `npm start`
4. Done.
You need to host API and web app separately ever since v.6.0. Setup script will help you with that!
### Ubuntu 22.04+ workaround
`nscd` needs to be installed and running so that the `ffmpeg-static` binary can resolve DNS ([#101](https://github.com/wukko/cobalt/issues/101#issuecomment-1494822258)):
@ -71,13 +74,8 @@ sudo service nscd start
```
### Docker
It's also possible to run cobalt via Docker, but you **need** to set all environment variables yourself:
| Variable | Description | Example |
| -------- | :--- | :--- |
| `selfURL` | Instance URL | `http://localhost:9000/` or `https://co.wukko.me/` or etc |
| `port` | Instance port | `9000` |
| `cors` | CORS toggle | `0` |
It's also possible to run cobalt via Docker. I *highly* recommend using Docker compose.
Check out the [example compose file](https://github.com/wukko/cobalt/blob/current/docker-compose.yml.example) and alter it for your needs.
## Disclaimer
cobalt is my passion project, so update schedule depends solely on my free time, motivation, and mood.

View file

@ -11,7 +11,6 @@ services:
- apiPort=9000
- apiURL=https://co.wuk.sh/
- apiName=eu-nl
- cors=1
cobalt-web:
build: .
restart: unless-stopped
@ -21,9 +20,7 @@ services:
environment:
- webPort=9000
- webURL=https://co.wukko.me/
- apiPort=9000
- apiURL=https://co.wuk.sh/
- cors=1
cobalt-both:
build: .
restart: unless-stopped
@ -33,4 +30,3 @@ services:
environment:
- port=9000
- selfURL=https://co.wukko.me/
- cors=1

View file

@ -1,5 +1,13 @@
# cobalt API Documentation
This document provides info about methods and acceptable variables for all cobalt API requests.<br>
```
⚠️ Main API instance has moved to https://co.wuk.sh/
Previous API domain will stop redirecting users to correct API instance after July 25th.
Make sure to update your projects in time.
```
## POST: ``/api/json``
Main processing endpoint.<br>

View file

@ -4,9 +4,18 @@ import { Bright, Cyan } from "../modules/sub/consoleText.js";
import { buildFront } from "../modules/build.js";
import findRendered from "../modules/pageRender/findRendered.js";
// * will be removed in the future
import cors from "cors";
// *
export async function runWeb(express, app, gitCommit, gitBranch, __dirname) {
await buildFront(gitCommit, gitBranch);
// * will be removed in the future
const corsConfig = process.env.cors === '0' ? { origin: process.env.webURL, optionsSuccessStatus: 200 } : {};
app.use('/api/:type', cors(corsConfig));
// *
app.use('/', express.static('./build/min'));
app.use('/', express.static('./src/front'));
@ -23,6 +32,14 @@ export async function runWeb(express, app, gitCommit, gitBranch, __dirname) {
app.get("/favicon.ico", (req, res) => {
res.sendFile(`${__dirname}/src/front/icons/favicon.ico`)
});
// * will be removed in the future
app.get("/api/*", (req, res) => {
res.redirect(308, process.env.apiURL.slice(0, -1) + req.url)
});
app.post("/api/*", (req, res) => {
res.redirect(308, process.env.apiURL.slice(0, -1) + req.url)
});
// *
app.get("/*", (req, res) => {
res.redirect('/')
});

View file

@ -34,8 +34,9 @@ export async function getJSON(originalURL, lang, obj) {
}
if (!(host && host.length < 20 && host in patterns && patterns[host]["enabled"])) return apiJSON(0, { t: errorUnsupported(lang) });
let pathToMatch = cleanURL(url, host).split(`.${patterns[host]['tld'] ? patterns[host]['tld'] : "com"}/`)[1].replace('.', '');
for (let i in patterns[host]["patterns"]) {
patternMatch = new UrlPattern(patterns[host]["patterns"][i]).match(cleanURL(url, host).split(`.${patterns[host]['tld'] ? patterns[host]['tld'] : "com"}/`)[1].replace('.', ''));
patternMatch = new UrlPattern(patterns[host]["patterns"][i]).match(pathToMatch);
if (patternMatch) break
}
if (!patternMatch) return apiJSON(0, { t: errorUnsupported(lang) });

View file

@ -113,9 +113,11 @@ export default function(r, host, ip, audioFormat, isAudioOnly, lang, isAudioMute
processType = "bridge"
}
}
if ((audioFormat === "best" && services[host]["bestAudio"])
|| services[host]["bestAudio"] && (audioFormat === services[host]["bestAudio"])) {
if (host === "tumblr" && !r.filename && (audioFormat === "best" || audioFormat === "mp3")) {
audioFormat = "mp3";
processType = "bridge"
}
if ((audioFormat === "best" && services[host]["bestAudio"]) || (services[host]["bestAudio"] && (audioFormat === services[host]["bestAudio"]))) {
audioFormat = services[host]["bestAudio"];
processType = "bridge"
} else if (audioFormat === "best") {

View file

@ -8,7 +8,21 @@ export default async function(obj) {
}).then((r) => { return r.text() }).catch(() => { return false });
if (!html) return { error: 'ErrorCouldntFetch' };
if (!html.includes('property="og:video" content="https://va.media.tumblr.com/')) return { error: 'ErrorEmptyDownload' };
return { urls: `https://va.media.tumblr.com/${html.split('property="og:video" content="https://va.media.tumblr.com/')[1].split('"')[0]}`, filename: `tumblr_${obj.id}.mp4`, audioFilename: `tumblr_${obj.id}_audio` }
let r;
if (html.includes('property="og:video" content="https://va.media.tumblr.com/')) {
r = {
urls: `https://va.media.tumblr.com/${html.split('property="og:video" content="https://va.media.tumblr.com/')[1].split('"')[0]}`,
filename: `tumblr_${obj.id}.mp4`,
audioFilename: `tumblr_${obj.id}_audio`
}
} else if (html.includes('property="og:audio" content="https://a.tumblr.com/')) {
r = {
urls: `https://a.tumblr.com/${html.split('property="og:audio" content="https://a.tumblr.com/')[1].split('"')[0]}`,
audioFilename: `tumblr_${obj.id}`,
isAudioOnly: true
}
} else r = { error: 'ErrorEmptyDownload' };
return r;
}

View file

@ -23,6 +23,10 @@ const c = {
export default async function(o) {
let info, isDubbed, quality = o.quality === "max" ? "9000" : o.quality; //set quality 9000(p) to be interpreted as max
function qual(i) {
return i['quality_label'].split('p')[0].split('s')[0]
}
try {
info = await yt.getBasicInfo(o.id, 'ANDROID');
} catch (e) {
@ -30,6 +34,7 @@ export default async function(o) {
}
if (!info) return { error: 'ErrorCantConnectToServiceAPI' };
if (info.playability_status.status !== 'OK') return { error: 'ErrorYTUnavailable' };
if (info.basic_info.is_live) return { error: 'ErrorLiveVideo' };
@ -40,7 +45,7 @@ export default async function(o) {
bestQuality = adaptive_formats.find(i => i["has_video"]);
hasAudio = adaptive_formats.find(i => i["has_audio"]);
if (bestQuality) bestQuality = bestQuality['quality_label'].split('p')[0];
if (bestQuality) bestQuality = qual(bestQuality);
if (!bestQuality && !o.isAudioOnly || !hasAudio) return { error: 'ErrorYTTryOtherCodec' };
if (info.basic_info.duration > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
@ -73,9 +78,9 @@ export default async function(o) {
};
return r
}
let checkSingle = (i) => ((i['quality_label'].split('p')[0] === quality || i['quality_label'].split('p')[0] === bestQuality) && i["mime_type"].includes(c[o.format].codec)),
checkBestVideo = (i) => (i["has_video"] && !i["has_audio"] && i['quality_label'].split('p')[0] === bestQuality),
checkRightVideo = (i) => (i["has_video"] && !i["has_audio"] && i['quality_label'].split('p')[0] === quality);
let checkSingle = (i) => ((qual(i) === quality || qual(i) === bestQuality) && i["mime_type"].includes(c[o.format].codec)),
checkBestVideo = (i) => (i["has_video"] && !i["has_audio"] && qual(i) === bestQuality),
checkRightVideo = (i) => (i["has_video"] && !i["has_audio"] && qual(i) === quality);
if (!o.isAudioOnly && !o.isAudioMuted && o.format === 'h264') {
let single = info.streaming_data.formats.find(i => checkSingle(i));

View file

@ -5,48 +5,100 @@ import { execSync } from "child_process";
let envPath = './.env';
let q = `${Cyan('?')} \x1b[1m`;
let ob = {}
let ob = {};
let rl = createInterface({ input: process.stdin, output: process.stdout });
let final = () => {
if (existsSync(envPath)) {
unlinkSync(envPath)
}
if (existsSync(envPath)) unlinkSync(envPath);
for (let i in ob) {
appendFileSync(envPath, `${i}=${ob[i]}\n`)
}
console.log(Bright("\nAwesome! I've created a fresh .env file for you."))
console.log(`${Bright("Now I'll run")} ${Cyan("npm install")} ${Bright("to install all dependencies. It shouldn't take long.\n\n")}`)
console.log(Bright("\nAwesome! I've created a fresh .env file for you."));
console.log(`${Bright("Now I'll run")} ${Cyan("npm install")} ${Bright("to install all dependencies. It shouldn't take long.\n\n")}`);
execSync('npm install', { stdio: [0, 1, 2] });
console.log(`\n\n${Cyan("All done!\n")}`)
console.log(Bright("You can re-run this script at any time to update the configuration."))
console.log(Bright("\nYou're now ready to start cobalt. Simply run ") + Cyan("npm start") + Bright('!\nHave fun :)'))
console.log(`\n\n${Cyan("All done!\n")}`);
console.log(Bright("You can re-run this script at any time to update the configuration."));
console.log(Bright("\nYou're now ready to start cobalt. Simply run ") + Cyan("npm start") + Bright('!\nHave fun :)'));
rl.close()
}
console.log(
`${Cyan("Welcome to cobalt!")}\n${Bright("Let's start by creating a new ")}${Cyan(".env")}${Bright(" file. You can always change it later.")}`
`${Cyan("Hey, this is cobalt.")}\n${Bright("Let's start by creating a new ")}${Cyan(".env")}${Bright(" file. You can always change it later.")}`
)
console.log(
Bright("\nWhat's the domain this instance will be running on? (localhost)\nExample: co.wukko.me")
`\n${Bright("⚠️ Please notice that since v.6.0 cobalt is hosted in two parts. API and web app are now separate.\nMerged hosting is deprecated and will be removed in the future.")}`
)
function setup() {
console.log(Bright("\nWhat kind of server will this instance be?\nOptions: api, web."));
rl.question(q, r1 => {
ob['selfURL'] = `http://localhost:9000/`
ob['port'] = 9000
if (r1) ob['selfURL'] = `https://${r1}/`
rl.question(q, r1 => {
switch (r1.toLowerCase()) {
case 'api':
console.log(Bright("\nCool! What's the domain this API instance will be running on? (localhost)\nExample: co.wuk.sh"));
console.log(Bright("\nGreat! Now, what's the port it'll be running on? (9000)"))
rl.question(q, apiURL => {
ob['apiURL'] = `http://localhost:9000/`;
ob['apiPort'] = 9000;
if (apiURL && apiURL !== "localhost") ob['apiURL'] = `https://${apiURL.toLowerCase()}/`;
rl.question(q, r2 => {
if (r2) ob['port'] = r2
if (!r1 && r2) ob['selfURL'] = `http://localhost:${r2}/`
console.log(Bright("\nGreat! Now, what port will it be running on? (9000)"));
console.log(Bright("\nWould you like to enable CORS? It allows other websites and extensions to use your instance's API.\ny/n (n)"))
rl.question(q, apiPort => {
if (apiPort) ob['apiPort'] = apiPort;
if (apiPort && (apiURL === "localhost" || !apiURL)) ob['apiURL'] = `http://localhost:${apiPort}/`;
rl.question(q, r3 => {
if (r3.toLowerCase() !== 'y') ob['cors'] = '0'
console.log(Bright("\nWhat will your instance's name be? Usually it's something like eu-nl aka region-country. (local)"));
rl.question(q, apiName => {
ob['apiName'] = apiName.toLowerCase();
if (!apiName || apiName === "local") ob['apiName'] = "local";
console.log(Bright("\nOne last thing: would you like to enable CORS? It allows other websites and extensions to use your instance's API.\ny/n (n)"));
rl.question(q, apiCors => {
if (apiCors.toLowerCase() !== 'y') ob['cors'] = '0'
final()
})
})
});
})
break;
case 'web':
console.log(Bright("\nAwesome! What's the domain this web app instance will be running on? (localhost)\nExample: co.wukko.me"));
rl.question(q, webURL => {
ob['webURL'] = `http://localhost:9001/`;
ob['webPort'] = 9001;
if (webURL && webURL !== "localhost") ob['webURL'] = `https://${webURL.toLowerCase()}/`;
console.log(
Bright("\nGreat! Now, what port will it be running on? (9001)")
)
rl.question(q, webPort => {
if (webPort) ob['webPort'] = webPort;
if (webPort && (webURL === "localhost" || !webURL)) ob['webURL'] = `http://localhost:${webPort}/`;
console.log(
Bright("\nOne last thing: what default API domain should be used? (co.wuk.sh)\nIf it's hosted locally, make sure to include the port:") + Cyan(" localhost:9000")
);
rl.question(q, apiURL => {
ob['apiURL'] = `https://${apiURL.toLowerCase()}/`;
if (apiURL.includes(':')) ob['apiURL'] = `http://${apiURL.toLowerCase()}/`;
if (!apiURL) ob['apiURL'] = "https://co.wuk.sh/";
final()
})
});
})
});
break;
default:
console.log(Bright("\nThis is not an option. Try again."));
setup()
}
})
}
setup()

View file

@ -446,6 +446,17 @@
"code": 200,
"status": "stream"
}
}, {
"name": "vr 360, av1, max",
"url": "https://www.youtube.com/watch?v=hEdzv7D4CbQ",
"params": {
"vCodec": "vp9",
"vQuality": "max"
},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "inexistent video",
"url": "https://youtube.com/watch?v=gnjuHYWGEW",
@ -717,6 +728,24 @@
"code": 200,
"status": "redirect"
}
}, {
"name": "tumblr audio",
"url": "https://rf9weu8hjf789234hf9.tumblr.com/post/172006661342/everyone-thats-made-a-video-out-of-this-without",
"params": {},
"expected": {
"code": 200,
"status": "stream"
}
}, {
"name": "tumblr video converted to audio",
"url": "https://garfield-69.tumblr.com/post/696499862852780032",
"params": {
"isAudioOnly": true
},
"expected": {
"code": 200,
"status": "stream"
}
}],
"vimeo": [{
"name": "4k progressive",