From 55f1e4b704cafcca64bc9bc4fe14c0eaa0cd907c Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 5 Jun 2023 12:43:04 +0600 Subject: [PATCH] tumblr audio, youtube vr, updated setup script, further mitosis accommodations --- README.md | 20 ++-- docker-compose.yml.example | 4 - docs/API.md | 8 ++ src/core/web.js | 17 +++ src/modules/api.js | 3 +- src/modules/processing/matchActionDecider.js | 8 +- src/modules/processing/services/tumblr.js | 18 +++- src/modules/processing/services/youtube.js | 13 ++- src/modules/setup.js | 104 ++++++++++++++----- src/test/tests.json | 29 ++++++ 10 files changed, 173 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 881a634..e5355d1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # cobalt Best way to save what you love. -Main instance: [co.wukko.me](https://co.wukko.me/) +Live web app: [co.wukko.me](https://co.wukko.me/) ![cobalt logo with repeated logo pattern background](https://raw.githubusercontent.com/wukko/cobalt/current/src/front/icons/pattern.png "cobalt logo with repeated logo pattern background") @@ -21,21 +21,22 @@ Paste the link, get the video, move on. It's that simple. Just how it should be. | Reddit | ✅ | ✅ | ✅ | Support for GIFs and videos. | | SoundCloud | ➖ | ✅ | ➖ | Audio metadata, downloads from private links. | | TikTok | ✅ | ✅ | ✅ | Supports downloads of: videos with or without watermark, images from slideshow without watermark, full (original) audios. | -| Tumblr | ✅ | ✅ | ✅ | | +| Tumblr | ✅ | ✅ | ✅ | Support for audio file downloads. | | Twitter | ✅ | ✅ | ✅ | Ability to pick what to save from multi-media tweets. | | Twitter Spaces | ➖ | ✅ | ➖ | Audio metadata with all participants and other info. | | Vimeo | ✅ | ✅ | ✅ | Audio downloads are only available for dash files. | | Vine Archive | ✅ | ✅ | ✅ | | | VK Videos | ✅ | ❌ | ❌ | | | VK Clips | ✅ | ❌ | ❌ | | -| YouTube Videos & Shorts | ✅ | ✅ | ✅ | Support for 8K, 4K, HDR, and high FPS videos. Audio metadata & dubs. h264/av1/vp9 codecs. | +| YouTube Videos & Shorts | ✅ | ✅ | ✅ | Support for 8K, 4K, HDR, VR, and high FPS videos. Audio metadata & dubs. h264/av1/vp9 codecs. | | YouTube Music | ➖ | ✅ | ➖ | Audio metadata. | This list is not final and keeps expanding over time, make sure to check it once in a while! ## cobalt API cobalt has an open API that you can use in your projects for **free**. -It's easy and straightforward to use, [check out the docs](https://github.com/wukko/cobalt/blob/current/docs/API.md) and see for yourself. +It's easy and straightforward to use, [check out the docs](https://github.com/wukko/cobalt/blob/current/docs/API.md) and see for yourself. +Feel free to use the main API instance ([co.wuk.sh](https://co.wuk.sh/)) in your projects. ## How to contribute translations You can translate cobalt to any language you want on [cobalt's Crowdin](https://crowdin-co.wukko.me/). Feel free to ignore QA errors if you think you know better. If you don't see a language you want to translate cobalt to, open an issue, and I'll add it to Crowdin. @@ -62,6 +63,8 @@ Setup script installs all needed `npm` dependencies, but you have to install `No 3. Run cobalt via `npm start` 4. Done. +You need to host API and web app separately ever since v.6.0. Setup script will help you with that! + ### Ubuntu 22.04+ workaround `nscd` needs to be installed and running so that the `ffmpeg-static` binary can resolve DNS ([#101](https://github.com/wukko/cobalt/issues/101#issuecomment-1494822258)): @@ -71,13 +74,8 @@ sudo service nscd start ``` ### Docker -It's also possible to run cobalt via Docker, but you **need** to set all environment variables yourself: - -| Variable | Description | Example | -| -------- | :--- | :--- | -| `selfURL` | Instance URL | `http://localhost:9000/` or `https://co.wukko.me/` or etc | -| `port` | Instance port | `9000` | -| `cors` | CORS toggle | `0` | +It's also possible to run cobalt via Docker. I *highly* recommend using Docker compose. +Check out the [example compose file](https://github.com/wukko/cobalt/blob/current/docker-compose.yml.example) and alter it for your needs. ## Disclaimer cobalt is my passion project, so update schedule depends solely on my free time, motivation, and mood. diff --git a/docker-compose.yml.example b/docker-compose.yml.example index cb50b9f..8ba8c87 100644 --- a/docker-compose.yml.example +++ b/docker-compose.yml.example @@ -11,7 +11,6 @@ services: - apiPort=9000 - apiURL=https://co.wuk.sh/ - apiName=eu-nl - - cors=1 cobalt-web: build: . restart: unless-stopped @@ -21,9 +20,7 @@ services: environment: - webPort=9000 - webURL=https://co.wukko.me/ - - apiPort=9000 - apiURL=https://co.wuk.sh/ - - cors=1 cobalt-both: build: . restart: unless-stopped @@ -33,4 +30,3 @@ services: environment: - port=9000 - selfURL=https://co.wukko.me/ - - cors=1 diff --git a/docs/API.md b/docs/API.md index f45e2c0..38038bd 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1,5 +1,13 @@ # cobalt API Documentation This document provides info about methods and acceptable variables for all cobalt API requests.
+ +``` +⚠️ Main API instance has moved to https://co.wuk.sh/ + +Previous API domain will stop redirecting users to correct API instance after July 25th. +Make sure to update your projects in time. +``` + ## POST: ``/api/json`` Main processing endpoint.
diff --git a/src/core/web.js b/src/core/web.js index 95f1fd4..afde9fe 100644 --- a/src/core/web.js +++ b/src/core/web.js @@ -4,9 +4,18 @@ import { Bright, Cyan } from "../modules/sub/consoleText.js"; import { buildFront } from "../modules/build.js"; import findRendered from "../modules/pageRender/findRendered.js"; +// * will be removed in the future +import cors from "cors"; +// * + export async function runWeb(express, app, gitCommit, gitBranch, __dirname) { await buildFront(gitCommit, gitBranch); + // * will be removed in the future + const corsConfig = process.env.cors === '0' ? { origin: process.env.webURL, optionsSuccessStatus: 200 } : {}; + app.use('/api/:type', cors(corsConfig)); + // * + app.use('/', express.static('./build/min')); app.use('/', express.static('./src/front')); @@ -23,6 +32,14 @@ export async function runWeb(express, app, gitCommit, gitBranch, __dirname) { app.get("/favicon.ico", (req, res) => { res.sendFile(`${__dirname}/src/front/icons/favicon.ico`) }); + // * will be removed in the future + app.get("/api/*", (req, res) => { + res.redirect(308, process.env.apiURL.slice(0, -1) + req.url) + }); + app.post("/api/*", (req, res) => { + res.redirect(308, process.env.apiURL.slice(0, -1) + req.url) + }); + // * app.get("/*", (req, res) => { res.redirect('/') }); diff --git a/src/modules/api.js b/src/modules/api.js index d544681..94ed504 100644 --- a/src/modules/api.js +++ b/src/modules/api.js @@ -34,8 +34,9 @@ export async function getJSON(originalURL, lang, obj) { } if (!(host && host.length < 20 && host in patterns && patterns[host]["enabled"])) return apiJSON(0, { t: errorUnsupported(lang) }); + let pathToMatch = cleanURL(url, host).split(`.${patterns[host]['tld'] ? patterns[host]['tld'] : "com"}/`)[1].replace('.', ''); for (let i in patterns[host]["patterns"]) { - patternMatch = new UrlPattern(patterns[host]["patterns"][i]).match(cleanURL(url, host).split(`.${patterns[host]['tld'] ? patterns[host]['tld'] : "com"}/`)[1].replace('.', '')); + patternMatch = new UrlPattern(patterns[host]["patterns"][i]).match(pathToMatch); if (patternMatch) break } if (!patternMatch) return apiJSON(0, { t: errorUnsupported(lang) }); diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index ba10af2..e000504 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -113,9 +113,11 @@ export default function(r, host, ip, audioFormat, isAudioOnly, lang, isAudioMute processType = "bridge" } } - - if ((audioFormat === "best" && services[host]["bestAudio"]) - || services[host]["bestAudio"] && (audioFormat === services[host]["bestAudio"])) { + if (host === "tumblr" && !r.filename && (audioFormat === "best" || audioFormat === "mp3")) { + audioFormat = "mp3"; + processType = "bridge" + } + if ((audioFormat === "best" && services[host]["bestAudio"]) || (services[host]["bestAudio"] && (audioFormat === services[host]["bestAudio"]))) { audioFormat = services[host]["bestAudio"]; processType = "bridge" } else if (audioFormat === "best") { diff --git a/src/modules/processing/services/tumblr.js b/src/modules/processing/services/tumblr.js index 95ba7f4..7ae7336 100644 --- a/src/modules/processing/services/tumblr.js +++ b/src/modules/processing/services/tumblr.js @@ -8,7 +8,21 @@ export default async function(obj) { }).then((r) => { return r.text() }).catch(() => { return false }); if (!html) return { error: 'ErrorCouldntFetch' }; - if (!html.includes('property="og:video" content="https://va.media.tumblr.com/')) return { error: 'ErrorEmptyDownload' }; - return { urls: `https://va.media.tumblr.com/${html.split('property="og:video" content="https://va.media.tumblr.com/')[1].split('"')[0]}`, filename: `tumblr_${obj.id}.mp4`, audioFilename: `tumblr_${obj.id}_audio` } + let r; + if (html.includes('property="og:video" content="https://va.media.tumblr.com/')) { + r = { + urls: `https://va.media.tumblr.com/${html.split('property="og:video" content="https://va.media.tumblr.com/')[1].split('"')[0]}`, + filename: `tumblr_${obj.id}.mp4`, + audioFilename: `tumblr_${obj.id}_audio` + } + } else if (html.includes('property="og:audio" content="https://a.tumblr.com/')) { + r = { + urls: `https://a.tumblr.com/${html.split('property="og:audio" content="https://a.tumblr.com/')[1].split('"')[0]}`, + audioFilename: `tumblr_${obj.id}`, + isAudioOnly: true + } + } else r = { error: 'ErrorEmptyDownload' }; + + return r; } diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 2526654..e686b4c 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -23,6 +23,10 @@ const c = { export default async function(o) { let info, isDubbed, quality = o.quality === "max" ? "9000" : o.quality; //set quality 9000(p) to be interpreted as max + function qual(i) { + return i['quality_label'].split('p')[0].split('s')[0] + } + try { info = await yt.getBasicInfo(o.id, 'ANDROID'); } catch (e) { @@ -30,6 +34,7 @@ export default async function(o) { } if (!info) return { error: 'ErrorCantConnectToServiceAPI' }; + if (info.playability_status.status !== 'OK') return { error: 'ErrorYTUnavailable' }; if (info.basic_info.is_live) return { error: 'ErrorLiveVideo' }; @@ -40,7 +45,7 @@ export default async function(o) { bestQuality = adaptive_formats.find(i => i["has_video"]); hasAudio = adaptive_formats.find(i => i["has_audio"]); - if (bestQuality) bestQuality = bestQuality['quality_label'].split('p')[0]; + if (bestQuality) bestQuality = qual(bestQuality); if (!bestQuality && !o.isAudioOnly || !hasAudio) return { error: 'ErrorYTTryOtherCodec' }; if (info.basic_info.duration > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; @@ -73,9 +78,9 @@ export default async function(o) { }; return r } - let checkSingle = (i) => ((i['quality_label'].split('p')[0] === quality || i['quality_label'].split('p')[0] === bestQuality) && i["mime_type"].includes(c[o.format].codec)), - checkBestVideo = (i) => (i["has_video"] && !i["has_audio"] && i['quality_label'].split('p')[0] === bestQuality), - checkRightVideo = (i) => (i["has_video"] && !i["has_audio"] && i['quality_label'].split('p')[0] === quality); + let checkSingle = (i) => ((qual(i) === quality || qual(i) === bestQuality) && i["mime_type"].includes(c[o.format].codec)), + checkBestVideo = (i) => (i["has_video"] && !i["has_audio"] && qual(i) === bestQuality), + checkRightVideo = (i) => (i["has_video"] && !i["has_audio"] && qual(i) === quality); if (!o.isAudioOnly && !o.isAudioMuted && o.format === 'h264') { let single = info.streaming_data.formats.find(i => checkSingle(i)); diff --git a/src/modules/setup.js b/src/modules/setup.js index 2740a77..3aae054 100644 --- a/src/modules/setup.js +++ b/src/modules/setup.js @@ -5,48 +5,100 @@ import { execSync } from "child_process"; let envPath = './.env'; let q = `${Cyan('?')} \x1b[1m`; -let ob = {} +let ob = {}; let rl = createInterface({ input: process.stdin, output: process.stdout }); let final = () => { - if (existsSync(envPath)) { - unlinkSync(envPath) - } + if (existsSync(envPath)) unlinkSync(envPath); + for (let i in ob) { appendFileSync(envPath, `${i}=${ob[i]}\n`) } - console.log(Bright("\nAwesome! I've created a fresh .env file for you.")) - console.log(`${Bright("Now I'll run")} ${Cyan("npm install")} ${Bright("to install all dependencies. It shouldn't take long.\n\n")}`) + console.log(Bright("\nAwesome! I've created a fresh .env file for you.")); + console.log(`${Bright("Now I'll run")} ${Cyan("npm install")} ${Bright("to install all dependencies. It shouldn't take long.\n\n")}`); execSync('npm install', { stdio: [0, 1, 2] }); - console.log(`\n\n${Cyan("All done!\n")}`) - console.log(Bright("You can re-run this script at any time to update the configuration.")) - console.log(Bright("\nYou're now ready to start cobalt. Simply run ") + Cyan("npm start") + Bright('!\nHave fun :)')) + console.log(`\n\n${Cyan("All done!\n")}`); + console.log(Bright("You can re-run this script at any time to update the configuration.")); + console.log(Bright("\nYou're now ready to start cobalt. Simply run ") + Cyan("npm start") + Bright('!\nHave fun :)')); rl.close() } console.log( - `${Cyan("Welcome to cobalt!")}\n${Bright("Let's start by creating a new ")}${Cyan(".env")}${Bright(" file. You can always change it later.")}` + `${Cyan("Hey, this is cobalt.")}\n${Bright("Let's start by creating a new ")}${Cyan(".env")}${Bright(" file. You can always change it later.")}` ) + console.log( - Bright("\nWhat's the domain this instance will be running on? (localhost)\nExample: co.wukko.me") + `\n${Bright("⚠️ Please notice that since v.6.0 cobalt is hosted in two parts. API and web app are now separate.\nMerged hosting is deprecated and will be removed in the future.")}` ) +function setup() { + console.log(Bright("\nWhat kind of server will this instance be?\nOptions: api, web.")); -rl.question(q, r1 => { - ob['selfURL'] = `http://localhost:9000/` - ob['port'] = 9000 - if (r1) ob['selfURL'] = `https://${r1}/` + rl.question(q, r1 => { + switch (r1.toLowerCase()) { + case 'api': + console.log(Bright("\nCool! What's the domain this API instance will be running on? (localhost)\nExample: co.wuk.sh")); - console.log(Bright("\nGreat! Now, what's the port it'll be running on? (9000)")) + rl.question(q, apiURL => { + ob['apiURL'] = `http://localhost:9000/`; + ob['apiPort'] = 9000; + if (apiURL && apiURL !== "localhost") ob['apiURL'] = `https://${apiURL.toLowerCase()}/`; - rl.question(q, r2 => { - if (r2) ob['port'] = r2 - if (!r1 && r2) ob['selfURL'] = `http://localhost:${r2}/` + console.log(Bright("\nGreat! Now, what port will it be running on? (9000)")); - console.log(Bright("\nWould you like to enable CORS? It allows other websites and extensions to use your instance's API.\ny/n (n)")) + rl.question(q, apiPort => { + if (apiPort) ob['apiPort'] = apiPort; + if (apiPort && (apiURL === "localhost" || !apiURL)) ob['apiURL'] = `http://localhost:${apiPort}/`; - rl.question(q, r3 => { - if (r3.toLowerCase() !== 'y') ob['cors'] = '0' - final() - }) - }); -}) + console.log(Bright("\nWhat will your instance's name be? Usually it's something like eu-nl aka region-country. (local)")); + + rl.question(q, apiName => { + ob['apiName'] = apiName.toLowerCase(); + if (!apiName || apiName === "local") ob['apiName'] = "local"; + + console.log(Bright("\nOne last thing: would you like to enable CORS? It allows other websites and extensions to use your instance's API.\ny/n (n)")); + + rl.question(q, apiCors => { + if (apiCors.toLowerCase() !== 'y') ob['cors'] = '0' + final() + }) + }) + }); + + }) + break; + case 'web': + console.log(Bright("\nAwesome! What's the domain this web app instance will be running on? (localhost)\nExample: co.wukko.me")); + + rl.question(q, webURL => { + ob['webURL'] = `http://localhost:9001/`; + ob['webPort'] = 9001; + if (webURL && webURL !== "localhost") ob['webURL'] = `https://${webURL.toLowerCase()}/`; + + console.log( + Bright("\nGreat! Now, what port will it be running on? (9001)") + ) + rl.question(q, webPort => { + if (webPort) ob['webPort'] = webPort; + if (webPort && (webURL === "localhost" || !webURL)) ob['webURL'] = `http://localhost:${webPort}/`; + + console.log( + Bright("\nOne last thing: what default API domain should be used? (co.wuk.sh)\nIf it's hosted locally, make sure to include the port:") + Cyan(" localhost:9000") + ); + + rl.question(q, apiURL => { + ob['apiURL'] = `https://${apiURL.toLowerCase()}/`; + if (apiURL.includes(':')) ob['apiURL'] = `http://${apiURL.toLowerCase()}/`; + if (!apiURL) ob['apiURL'] = "https://co.wuk.sh/"; + final() + }) + }); + + }); + break; + default: + console.log(Bright("\nThis is not an option. Try again.")); + setup() + } + }) +} +setup() diff --git a/src/test/tests.json b/src/test/tests.json index 17ed2c4..cda2653 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -446,6 +446,17 @@ "code": 200, "status": "stream" } + }, { + "name": "vr 360, av1, max", + "url": "https://www.youtube.com/watch?v=hEdzv7D4CbQ", + "params": { + "vCodec": "vp9", + "vQuality": "max" + }, + "expected": { + "code": 200, + "status": "stream" + } }, { "name": "inexistent video", "url": "https://youtube.com/watch?v=gnjuHYWGEW", @@ -717,6 +728,24 @@ "code": 200, "status": "redirect" } + }, { + "name": "tumblr audio", + "url": "https://rf9weu8hjf789234hf9.tumblr.com/post/172006661342/everyone-thats-made-a-video-out-of-this-without", + "params": {}, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "tumblr video converted to audio", + "url": "https://garfield-69.tumblr.com/post/696499862852780032", + "params": { + "isAudioOnly": true + }, + "expected": { + "code": 200, + "status": "stream" + } }], "vimeo": [{ "name": "4k progressive",