From 4d369170ff80be1e4af991611097eaf6909fcb1d Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 19 May 2023 16:13:38 +0600 Subject: [PATCH 1/8] separated web and api, build improvements wip: - separate web and api servers. - script for building static pages. - building improvements. - async localisation preloading. --- .gitignore | 3 + package.json | 7 +- src/api.js | 196 +++++++++++++++++++++++++++++ src/cobalt.js | 28 +++-- src/front/cobalt.css | 5 +- src/localization/languages/en.json | 2 +- src/localization/languages/ru.json | 2 +- src/localization/manager.js | 15 +-- src/modules/build.js | 12 +- src/modules/buildStatic.js | 7 ++ src/modules/pageRender/elements.js | 3 +- src/modules/stream/manage.js | 2 +- src/web.js | 59 +++++++++ 13 files changed, 315 insertions(+), 26 deletions(-) create mode 100644 src/api.js create mode 100644 src/modules/buildStatic.js create mode 100644 src/web.js diff --git a/.gitignore b/.gitignore index 305746b2..8af52058 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ build # stuff i already made but delayed future + +# docker +docker-compose.yml diff --git a/package.json b/package.json index 1382b1f5..2c0f7258 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "5.7", + "version": "6.0-dev", "author": "wukko", "exports": "./src/cobalt.js", "type": "module", @@ -11,7 +11,10 @@ "scripts": { "start": "node src/cobalt", "setup": "node src/modules/setup", - "test": "node src/test/test" + "test": "node src/test/test", + "build": "node src/modules/buildStatic", + "api": "node src/api", + "web": "node src/web" }, "repository": { "type": "git", diff --git a/src/api.js b/src/api.js new file mode 100644 index 00000000..74c33e1f --- /dev/null +++ b/src/api.js @@ -0,0 +1,196 @@ +import "dotenv/config"; + +import express from "express"; +import cors from "cors"; +import rateLimit from "express-rate-limit"; +import { randomBytes } from "crypto"; + +const ipSalt = randomBytes(64).toString('hex'); + +import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js"; +import { appName, version } from "./modules/config.js"; +import { getJSON } from "./modules/api.js"; +import { apiJSON, checkJSONPost, getIP, languageCode } from "./modules/sub/utils.js"; +import { Bright, Cyan, Green, Red } from "./modules/sub/consoleText.js"; +import stream from "./modules/stream/stream.js"; +import loc, { loadLoc } from "./localization/manager.js"; +import { changelogHistory } from "./modules/pageRender/onDemand.js"; +import { sha256 } from "./modules/sub/crypto.js"; +import { celebrationsEmoji } from "./modules/pageRender/elements.js"; + +if (process.env.apiURL && process.env.apiPort) { + const commitHash = shortCommit(); + const branch = getCurrentBranch(); + const app = express(); + + app.disable('x-powered-by'); + + const corsConfig = process.env.cors === '0' ? { origin: process.env.webURL, optionsSuccessStatus: 200 } : {}; + + const apiLimiter = rateLimit({ + windowMs: 60000, + max: 25, + standardHeaders: false, + legacyHeaders: false, + keyGenerator: (req, res) => sha256(getIP(req), ipSalt), + handler: (req, res, next, opt) => { + res.status(429).json({ "status": "error", "text": loc(languageCode(req), 'ErrorRateLimit') }); + return; + } + }); + const apiLimiterStream = rateLimit({ + windowMs: 60000, + max: 28, + standardHeaders: false, + legacyHeaders: false, + keyGenerator: (req, res) => sha256(getIP(req), ipSalt), + handler: (req, res, next, opt) => { + res.status(429).json({ "status": "error", "text": loc(languageCode(req), 'ErrorRateLimit') }); + return; + } + }); + + const startTime = new Date(); + const startTimestamp = Math.floor(startTime.getTime()); + + // preload localization files + await loadLoc(); + + app.use('/api/:type', cors(corsConfig)); + app.use('/api/json', apiLimiter); + app.use('/api/stream', apiLimiterStream); + app.use('/api/onDemand', apiLimiter); + + app.use((req, res, next) => { + try { decodeURIComponent(req.path) } catch (e) { return res.redirect('/') } + next(); + }); + app.use((req, res, next) => { + if (req.header("user-agent") && req.header("user-agent").includes("Trident")) res.destroy(); + next(); + }); + app.use('/api/json', express.json({ + verify: (req, res, buf) => { + try { + JSON.parse(buf); + if (buf.length > 720) throw new Error(); + if (String(req.header('Content-Type')) !== "application/json") { + res.status(400).json({ 'status': 'error', 'text': 'invalid content type header' }); + return; + } + if (String(req.header('Accept')) !== "application/json") { + res.status(400).json({ 'status': 'error', 'text': 'invalid accept header' }); + return; + } + } catch(e) { + res.status(400).json({ 'status': 'error', 'text': 'invalid json body.' }); + return; + } + } + })); + + app.post('/api/json', async (req, res) => { + try { + let ip = sha256(getIP(req), ipSalt); + let lang = languageCode(req); + let j = apiJSON(0, { t: "Bad request" }); + try { + let request = req.body; + if (request.url) { + request.dubLang = request.dubLang ? lang : false; + let chck = checkJSONPost(request); + if (chck) chck["ip"] = ip; + j = chck ? await getJSON(chck["url"], lang, chck) : apiJSON(0, { t: loc(lang, 'ErrorCouldntFetch') }); + } else { + j = apiJSON(0, { t: loc(lang, 'ErrorNoLink') }); + } + } catch (e) { + j = apiJSON(0, { t: loc(lang, 'ErrorCantProcess') }); + } + res.status(j.status).json(j.body); + return; + } catch (e) { + res.destroy(); + return + } + }); + + app.get('/api/:type', (req, res) => { + try { + let ip = sha256(getIP(req), ipSalt); + switch (req.params.type) { + case 'stream': + if (req.query.p) { + res.status(200).json({ "status": "continue" }); + return; + } else if (req.query.t && req.query.h && req.query.e) { + stream(res, ip, req.query.t, req.query.h, req.query.e); + } else { + let j = apiJSON(0, { t: "no stream id" }) + res.status(j.status).json(j.body); + return; + } + break; + case 'onDemand': + if (req.query.blockId) { + let blockId = req.query.blockId.slice(0, 3); + let r, j; + switch(blockId) { + case "0": // changelog history + r = changelogHistory(); + j = r ? apiJSON(3, { t: r }) : apiJSON(0, { t: "couldn't render this block" }) + break; + case "1": // celebrations emoji + r = celebrationsEmoji(); + j = r ? apiJSON(3, { t: r }) : false + break; + default: + j = apiJSON(0, { t: "couldn't find a block with this id" }) + break; + } + if (j.body) { + res.status(j.status).json(j.body) + } else { + res.status(204).end() + } + } else { + let j = apiJSON(0, { t: "no block id" }); + res.status(j.status).json(j.body) + } + break; + case 'serverInfo': + res.status(200).json({ + version: version, + commit: commitHash, + branch: branch, + url: process.env.apiURL, + cors: process.env.cors, + startTime: `${startTimestamp}` + }); + break; + default: + let j = apiJSON(0, { t: "unknown response type" }) + res.status(j.status).json(j.body); + break; + } + } catch (e) { + res.status(500).json({ 'status': 'error', 'text': loc(languageCode(req), 'ErrorCantProcess') }); + return; + } + }); + app.get("/api/status", (req, res) => { + res.status(200).end() + }); + app.get("/favicon.ico", (req, res) => { + res.redirect('/icons/favicon.ico'); + }); + app.get("/*", (req, res) => { + res.redirect('/api/json') + }); + + app.listen(process.env.apiPort, () => { + console.log(`\n${Cyan(appName)} API ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${startTimestamp})`)}\n\nURL: ${Cyan(`${process.env.apiURL}`)}\nPort: ${process.env.apiPort}\n`) + }) +} else { + console.log(Red(`cobalt api hasn't been configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`)); +} diff --git a/src/cobalt.js b/src/cobalt.js index 990f5d31..066385a7 100644 --- a/src/cobalt.js +++ b/src/cobalt.js @@ -18,7 +18,7 @@ import { getJSON } from "./modules/api.js"; import { apiJSON, checkJSONPost, getIP, languageCode } from "./modules/sub/utils.js"; import { Bright, Cyan, Green, Red } from "./modules/sub/consoleText.js"; import stream from "./modules/stream/stream.js"; -import loc from "./localization/manager.js"; +import loc, { loadLoc } from "./localization/manager.js"; import { buildFront } from "./modules/build.js"; import { changelogHistory } from "./modules/pageRender/onDemand.js"; import { sha256 } from "./modules/sub/crypto.js"; @@ -56,7 +56,12 @@ if (process.env.selfURL && process.env.port) { return; } }); + + const startTime = new Date(); + const startTimestamp = Math.floor(startTime.getTime()); + // preload localization files and build static pages + await loadLoc(); await buildFront(commitHash, branch); app.use('/api/:type', cors(corsConfig)); @@ -64,7 +69,7 @@ if (process.env.selfURL && process.env.port) { app.use('/api/stream', apiLimiterStream); app.use('/api/onDemand', apiLimiter); - app.use('/', express.static('./min')); + app.use('/', express.static('./build/min')); app.use('/', express.static('./src/front')); app.use((req, res, next) => { @@ -164,6 +169,16 @@ if (process.env.selfURL && process.env.port) { res.status(j.status).json(j.body) } break; + case 'serverInfo': + res.status(200).json({ + version: version, + commit: commitHash, + branch: branch, + url: process.env.apiURL, + cors: process.env.cors, + startTime: `${startTimestamp}` + }); + break; default: let j = apiJSON(0, { t: "unknown response type" }) res.status(j.status).json(j.body); @@ -174,12 +189,12 @@ if (process.env.selfURL && process.env.port) { return; } }); + app.get("/api/status", (req, res) => { + res.status(200).end() + }); app.get("/api", (req, res) => { res.redirect('/api/json') }); - app.get("/status", (req, res) => { - res.status(200).end() - }); app.get("/", (req, res) => { res.sendFile(`${__dirname}/${findRendered(languageCode(req), req.header('user-agent') ? req.header('user-agent') : genericUserAgent)}`); }); @@ -191,8 +206,7 @@ if (process.env.selfURL && process.env.port) { }); app.listen(process.env.port, () => { - let startTime = new Date(); - console.log(`\n${Cyan(appName)} ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(new Date().getTime())})`)}\n\nURL: ${Cyan(`${process.env.selfURL}`)}\nPort: ${process.env.port}\n`) + console.log(`\n${Cyan(appName)} ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(startTimestamp)})`)}\n\nURL: ${Cyan(`${process.env.selfURL}`)}\nPort: ${process.env.port}\n`) }) } else { console.log(Red(`cobalt hasn't been configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`)); diff --git a/src/front/cobalt.css b/src/front/cobalt.css index 9f9c8a9c..338f0dc1 100644 --- a/src/front/cobalt.css +++ b/src/front/cobalt.css @@ -156,7 +156,7 @@ button:active, .text-to-copy:active { background: var(--accent-press); cursor: pointer; - transform: scale(0.95) + transform: scale(0.95); } .collapse-header:active { background: var(--accent-press); @@ -681,6 +681,9 @@ button:active, #about-donate-footer:active::before { opacity: 0; } +.popup-tabs-child { + width: 100%; +} /* adapt the page according to screen size */ @media screen and (min-width: 2300px) { html { diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json index 4f052578..4fe42dbd 100644 --- a/src/localization/languages/en.json +++ b/src/localization/languages/en.json @@ -47,7 +47,7 @@ "SettingsQualityDescription": "if selected quality isn't available, closest one is used instead.", "LinkGitHubChanges": ">> see previous commits and contribute on github", "NoScriptMessage": "{appName} uses javascript for api requests and interactive interface. you have to allow javascript to use this site. there are no pesty scripts, pinky promise.", - "DownloadPopupDescriptionIOS": "easiest way to save videos on ios:\n1. add this siri shortcut.\n2. press \"share\" above and select \"save to photos\" in appeared share sheet.\nif asked, review the permission request popup on top, and press \"always allow\".\n\nalternative method: press and hold the download button, hide the video preview, and select \"download linked file\" to download.\nthen, open safari downloads, select the file you downloaded, open share menu, and finally press \"save video\".", + "DownloadPopupDescriptionIOS": "easiest way to save videos on ios:\n1. add this siri shortcut.\n2. press \"share\" above and select \"save to photos\" in appeared share sheet.\nif asked, review the permission request, and press \"always allow\".\n\nalternative method:\npress and hold the download button, hide the video preview, and select \"download linked file\" to download.\nthen, open safari downloads, select the file you downloaded, open share menu, and finally press \"save video\".", "DownloadPopupDescription": "download button opens a new tab with requested file. you can disable this popup in settings.", "DownloadPopupWayToSave": "pick a way to save", "ClickToCopy": "press to copy", diff --git a/src/localization/languages/ru.json b/src/localization/languages/ru.json index 7f59f356..8a0845f8 100644 --- a/src/localization/languages/ru.json +++ b/src/localization/languages/ru.json @@ -47,7 +47,7 @@ "SettingsQualityDescription": "если выбранное качество недоступно, то выбирается ближайшее к нему.", "LinkGitHubChanges": ">> смотри предыдущие изменения на github", "NoScriptMessage": "{appName} использует javascript для обработки ссылок и интерактивного интерфейса. ты должен разрешить использование javascript, чтобы пользоваться сайтом. тут нет никаких зловредных скриптов, обещаю.", - "DownloadPopupDescriptionIOS": "наиболее простой метод скачивания видео на ios:\n1. добавь этот сценарий siri.\n2. нажми \"поделиться\" выше и выбери \"save to photos\" в открывшемся окне.\nесли появляется окно с запросом разрешения, то прочитай его, потом нажми \"всегда разрешать\".\n\nальтернативный метод: зажми кнопку \"скачать\", затем скрой превью и выбери \"загрузить файл по ссылке\" в появившемся окне.\nпотом открой загрузки в safari, выбери скачанный файл, нажми иконку \"поделиться\", и, наконец, нажми \"сохранить видео\".", + "DownloadPopupDescriptionIOS": "наиболее простой метод скачивания видео на ios:\n1. добавь этот сценарий siri.\n2. нажми \"поделиться\" выше и выбери \"save to photos\" в открывшемся окне.\nесли появляется окно с запросом разрешения, то прочитай его, потом нажми \"всегда разрешать\".\n\nальтернативный метод:\nзажми кнопку \"скачать\", затем скрой превью и выбери \"загрузить файл по ссылке\" в появившемся окне.\nпотом открой загрузки в safari, выбери скачанный файл, нажми иконку \"поделиться\", и, наконец, нажми \"сохранить видео\".", "DownloadPopupDescription": "кнопка скачивания открывает новое окно с файлом. ты можешь отключить выбор метода скачивания файла в настройках.", "DownloadPopupWayToSave": "выбери, как сохранить", "ClickToCopy": "нажми, чтобы скопировать", diff --git a/src/localization/manager.js b/src/localization/manager.js index 2f5a334a..e5eec9bb 100644 --- a/src/localization/manager.js +++ b/src/localization/manager.js @@ -7,16 +7,13 @@ const locPath = './src/localization/languages'; let loc = {} let languages = []; -export function loadLoc() { - fs.readdir(locPath, (err, files) => { - if (err) return false; - files.forEach(file => { - loc[file.split('.')[0]] = loadJson(`${locPath}/${file}`); - languages.push(file.split('.')[0]) - }); - }) +export async function loadLoc() { + const files = await fs.promises.readdir(locPath).catch((e) => { return [] }); + files.forEach(file => { + loc[file.split('.')[0]] = loadJson(`${locPath}/${file}`); + languages.push(file.split('.')[0]) + }); } -loadLoc(); export function replaceBase(s) { return s.replace(/\n/g, '
').replace(/{saveToGalleryShortcut}/g, links.saveToGalleryShortcut).replace(/{appName}/g, appName).replace(/{repo}/g, repo).replace(/\*;/g, "•"); diff --git a/src/modules/build.js b/src/modules/build.js index 6c4bee85..a2515b0e 100644 --- a/src/modules/build.js +++ b/src/modules/build.js @@ -1,6 +1,7 @@ import * as esbuild from "esbuild"; import * as fs from "fs"; -import { languageList } from "../localization/manager.js"; +import { loadLoc, languageList } from "../localization/manager.js"; + import page from "./pageRender/page.js"; function cleanHTML(html) { @@ -10,6 +11,9 @@ function cleanHTML(html) { } export async function buildFront(commitHash, branch) { try { + // preload localization files + await loadLoc(); + // build html if (!fs.existsSync('./build/')){ fs.mkdirSync('./build/'); @@ -17,6 +21,10 @@ export async function buildFront(commitHash, branch) { fs.mkdirSync('./build/pc/'); fs.mkdirSync('./build/mob/'); } + // get rid of old build path + if (fs.existsSync('./min')) { + fs.rmSync('./min', { recursive: true, force: true }); + } for (let i in languageList) { i = languageList[i]; let params = { @@ -36,7 +44,7 @@ export async function buildFront(commitHash, branch) { // build js & css await esbuild.build({ entryPoints: ['src/front/cobalt.js', 'src/front/cobalt.css'], - outdir: 'min/', + outdir: 'build/min/', minify: true, loader: { '.js': 'js', '.css': 'css', }, charset: 'utf8' diff --git a/src/modules/buildStatic.js b/src/modules/buildStatic.js new file mode 100644 index 00000000..d3ed909f --- /dev/null +++ b/src/modules/buildStatic.js @@ -0,0 +1,7 @@ +import { buildFront } from "./build.js"; +import { getCurrentBranch, shortCommit } from "./sub/currentCommit.js"; + +const commitHash = shortCommit(); +const branch = getCurrentBranch(); + +await buildFront(commitHash, branch); diff --git a/src/modules/pageRender/elements.js b/src/modules/pageRender/elements.js index cdf155ed..dd9861a0 100644 --- a/src/modules/pageRender/elements.js +++ b/src/modules/pageRender/elements.js @@ -89,14 +89,13 @@ export function multiPagePopup(obj) { tabs += `` tabContent += `
${obj.tabs[i]["content"]}
` } - tabs += `` return ` ` } export function collapsibleList(arr) { diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js index e01c2cb0..26dfc08c 100644 --- a/src/modules/stream/manage.js +++ b/src/modules/stream/manage.js @@ -39,7 +39,7 @@ export function createStream(obj) { exp = streamInfo.exp; ghmac = streamInfo.hmac; } - return `${process.env.selfURL}api/stream?t=${streamID}&e=${exp}&h=${ghmac}`; + return `${process.env.apiURL || process.env.selfURL}api/stream?t=${streamID}&e=${exp}&h=${ghmac}`; } export function verifyStream(ip, id, hmac, exp) { diff --git a/src/web.js b/src/web.js new file mode 100644 index 00000000..4f298975 --- /dev/null +++ b/src/web.js @@ -0,0 +1,59 @@ +import "dotenv/config"; + +import express from "express"; + +import path from 'path'; +import { fileURLToPath } from 'url'; +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename).slice(0, -4); // go up another level (get rid of src/) + +import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js"; +import { appName, genericUserAgent, version } from "./modules/config.js"; +import { languageCode } from "./modules/sub/utils.js"; +import { Bright, Cyan, Green, Red } from "./modules/sub/consoleText.js"; +import { loadLoc } from "./localization/manager.js"; +import { buildFront } from "./modules/build.js"; +import findRendered from "./modules/pageRender/findRendered.js"; + +if (process.env.webURL && process.env.webPort) { + const commitHash = shortCommit(); + const branch = getCurrentBranch(); + const app = express(); + + app.disable('x-powered-by'); + + // preload localization files and build static pages + await loadLoc(); + await buildFront(commitHash, branch); + + app.use('/', express.static('./build/min')); + app.use('/', express.static('./src/front')); + + app.use((req, res, next) => { + try { decodeURIComponent(req.path) } catch (e) { return res.redirect('/') } + next(); + }); + app.use((req, res, next) => { + if (req.header("user-agent") && req.header("user-agent").includes("Trident")) res.destroy(); + next(); + }); + app.get("/status", (req, res) => { + res.status(200).end() + }); + app.get("/", (req, res) => { + res.sendFile(`${__dirname}/${findRendered(languageCode(req), req.header('user-agent') ? req.header('user-agent') : genericUserAgent)}`); + }); + app.get("/favicon.ico", (req, res) => { + res.redirect('/icons/favicon.ico'); + }); + app.get("/*", (req, res) => { + res.redirect('/') + }); + + app.listen(process.env.webPort, () => { + let startTime = new Date(); + console.log(`\n${Cyan(appName)} WEB ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(new Date().getTime())})`)}\n\nURL: ${Cyan(`${process.env.webURL}`)}\nPort: ${process.env.webPort}\n`) + }) +} else { + console.log(Red(`cobalt web hasn't been configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`)); +} From 5636a27ff041c18924a66ff33f1195c9bbdacced Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 19 May 2023 16:23:10 +0600 Subject: [PATCH 2/8] experimenting with docker --- dockerfile_api | 15 +++++++++++++++ dockerfile_web | 15 +++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 dockerfile_api create mode 100644 dockerfile_web diff --git a/dockerfile_api b/dockerfile_api new file mode 100644 index 00000000..60fdbec8 --- /dev/null +++ b/dockerfile_api @@ -0,0 +1,15 @@ +FROM node:18-bullseye-slim +WORKDIR /app + +RUN apt-get update +RUN apt-get install -y git +RUN rm -rf /var/lib/apt/lists/* + +COPY package*.json ./ +RUN npm install + +RUN git clone -n https://github.com/wukko/cobalt.git --depth 1 && mv cobalt/.git ./ && rm -rf cobalt + +COPY . . +EXPOSE 9000 +CMD [ "node", "src/api" ] diff --git a/dockerfile_web b/dockerfile_web new file mode 100644 index 00000000..e20460c3 --- /dev/null +++ b/dockerfile_web @@ -0,0 +1,15 @@ +FROM node:18-bullseye-slim +WORKDIR /app + +RUN apt-get update +RUN apt-get install -y git +RUN rm -rf /var/lib/apt/lists/* + +COPY package*.json ./ +RUN npm install + +RUN git clone -n https://github.com/wukko/cobalt.git --depth 1 && mv cobalt/.git ./ && rm -rf cobalt + +COPY . . +EXPOSE 9000 +CMD [ "node", "src/web" ] From 19bc17b1dd72c52886415fd25534227de02830eb Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 19 May 2023 19:13:48 +0600 Subject: [PATCH 3/8] updated dockerfiles this is a mess, i have to come up with something better --- docker/docker-compose.yml.example | 39 +++++++++++++++++++++++++ dockerfile_api => docker/dockerfile_api | 0 dockerfile_web => docker/dockerfile_web | 0 src/front/cobalt.js | 6 +++- 4 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 docker/docker-compose.yml.example rename dockerfile_api => docker/dockerfile_api (100%) rename dockerfile_web => docker/dockerfile_web (100%) diff --git a/docker/docker-compose.yml.example b/docker/docker-compose.yml.example new file mode 100644 index 00000000..9284e1d3 --- /dev/null +++ b/docker/docker-compose.yml.example @@ -0,0 +1,39 @@ +version: '3.5' + +services: + cobalt-api: + build: + context: . + dockerfile: ./docker/dockerfile_api + restart: unless-stopped + container_name: cobalt-api + ports: + - 9000:9000/tcp + environment: + - apiPort=9000 + - apiURL='https://co.wuk.sh/' + - cors=1 + cobalt-web: + build: + context: . + dockerfile: ./docker/dockerfile_web + restart: unless-stopped + container_name: cobalt-web + ports: + - 9000:9000/tcp + environment: + - apiPort=9000 + - apiURL='https://co.wuk.sh/' + - cors=1 + cobalt-full: + build: + context: . + dockerfile: ./dockerfile + restart: unless-stopped + container_name: cobalt-full + ports: + - 9000:9000/tcp + environment: + - apiPort=9000 + - apiURL='https://co.wuk.sh/' + - cors=1 diff --git a/dockerfile_api b/docker/dockerfile_api similarity index 100% rename from dockerfile_api rename to docker/dockerfile_api diff --git a/dockerfile_web b/docker/dockerfile_web similarity index 100% rename from dockerfile_web rename to docker/dockerfile_web diff --git a/src/front/cobalt.js b/src/front/cobalt.js index cd77b2d2..59ec949a 100644 --- a/src/front/cobalt.js +++ b/src/front/cobalt.js @@ -19,10 +19,14 @@ const exceptions = { // used for mobile devices "vQuality": "720" }; -const apiURL = ''; +let apiURL = ''; let store = {}; +function changeAPI(url) { + apiURL = url; + return true +} function eid(id) { return document.getElementById(id) } From 9edc4bd61b685800bb6291374d1948ec26ae48f9 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 22 May 2023 01:13:05 +0600 Subject: [PATCH 4/8] new inner layout - one main controlling script (cobalt.js). - added api server name to serverInfo endpoint. - one dockerfile. - less mess. --- .gitignore | 7 +- ....yml.example => docker-compose.yml.example | 27 +-- docker/dockerfile_api | 15 -- docker/dockerfile_web | 15 -- package.json | 4 +- src/cobalt.js | 219 ++---------------- src/{ => core}/api.js | 63 +++-- src/core/both.js | 197 ++++++++++++++++ src/core/web.js | 34 +++ src/front/cobalt.css | 4 - src/front/cobalt.js | 2 - src/modules/changelog/changelog.json | 4 +- src/modules/pageRender/page.js | 12 +- src/web.js | 59 ----- 14 files changed, 304 insertions(+), 358 deletions(-) rename docker/docker-compose.yml.example => docker-compose.yml.example (54%) delete mode 100644 docker/dockerfile_api delete mode 100644 docker/dockerfile_web rename src/{ => core}/api.js (78%) create mode 100644 src/core/both.js create mode 100644 src/core/web.js delete mode 100644 src/web.js diff --git a/.gitignore b/.gitignore index 8af52058..7d12aa29 100644 --- a/.gitignore +++ b/.gitignore @@ -5,10 +5,8 @@ package-lock.json # secrets .env -# esbuild -min - # page build +min build # stuff i already made but delayed @@ -16,3 +14,6 @@ future # docker docker-compose.yml + +# vscode +.vscode diff --git a/docker/docker-compose.yml.example b/docker-compose.yml.example similarity index 54% rename from docker/docker-compose.yml.example rename to docker-compose.yml.example index 9284e1d3..cb50b9f7 100644 --- a/docker/docker-compose.yml.example +++ b/docker-compose.yml.example @@ -2,38 +2,35 @@ version: '3.5' services: cobalt-api: - build: - context: . - dockerfile: ./docker/dockerfile_api + build: . restart: unless-stopped container_name: cobalt-api ports: - 9000:9000/tcp environment: - apiPort=9000 - - apiURL='https://co.wuk.sh/' + - apiURL=https://co.wuk.sh/ + - apiName=eu-nl - cors=1 cobalt-web: - build: - context: . - dockerfile: ./docker/dockerfile_web + build: . restart: unless-stopped container_name: cobalt-web ports: - 9000:9000/tcp environment: + - webPort=9000 + - webURL=https://co.wukko.me/ - apiPort=9000 - - apiURL='https://co.wuk.sh/' + - apiURL=https://co.wuk.sh/ - cors=1 - cobalt-full: - build: - context: . - dockerfile: ./dockerfile + cobalt-both: + build: . restart: unless-stopped - container_name: cobalt-full + container_name: cobalt-both ports: - 9000:9000/tcp environment: - - apiPort=9000 - - apiURL='https://co.wuk.sh/' + - port=9000 + - selfURL=https://co.wukko.me/ - cors=1 diff --git a/docker/dockerfile_api b/docker/dockerfile_api deleted file mode 100644 index 60fdbec8..00000000 --- a/docker/dockerfile_api +++ /dev/null @@ -1,15 +0,0 @@ -FROM node:18-bullseye-slim -WORKDIR /app - -RUN apt-get update -RUN apt-get install -y git -RUN rm -rf /var/lib/apt/lists/* - -COPY package*.json ./ -RUN npm install - -RUN git clone -n https://github.com/wukko/cobalt.git --depth 1 && mv cobalt/.git ./ && rm -rf cobalt - -COPY . . -EXPOSE 9000 -CMD [ "node", "src/api" ] diff --git a/docker/dockerfile_web b/docker/dockerfile_web deleted file mode 100644 index e20460c3..00000000 --- a/docker/dockerfile_web +++ /dev/null @@ -1,15 +0,0 @@ -FROM node:18-bullseye-slim -WORKDIR /app - -RUN apt-get update -RUN apt-get install -y git -RUN rm -rf /var/lib/apt/lists/* - -COPY package*.json ./ -RUN npm install - -RUN git clone -n https://github.com/wukko/cobalt.git --depth 1 && mv cobalt/.git ./ && rm -rf cobalt - -COPY . . -EXPOSE 9000 -CMD [ "node", "src/web" ] diff --git a/package.json b/package.json index 2c0f7258..9db76a44 100644 --- a/package.json +++ b/package.json @@ -12,9 +12,7 @@ "start": "node src/cobalt", "setup": "node src/modules/setup", "test": "node src/test/test", - "build": "node src/modules/buildStatic", - "api": "node src/api", - "web": "node src/web" + "build": "node src/modules/buildStatic" }, "repository": { "type": "git", diff --git a/src/cobalt.js b/src/cobalt.js index 066385a7..6f3dd795 100644 --- a/src/cobalt.js +++ b/src/cobalt.js @@ -1,213 +1,36 @@ import "dotenv/config"; import express from "express"; -import cors from "cors"; -import rateLimit from "express-rate-limit"; -import { randomBytes } from "crypto"; -const ipSalt = randomBytes(64).toString('hex'); +import { Bright, Green, Red } from "./modules/sub/consoleText.js"; +import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js"; +import { loadLoc } from "./localization/manager.js"; import path from 'path'; import { fileURLToPath } from 'url'; + +import { runWeb } from "./core/web.js"; +import { runAPI } from "./core/api.js"; +import { runBoth } from "./core/both.js"; + +const app = express(); + +const gitCommit = shortCommit(); +const gitBranch = getCurrentBranch(); + const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename).slice(0, -4); // go up another level (get rid of src/) -import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js"; -import { appName, genericUserAgent, version } from "./modules/config.js"; -import { getJSON } from "./modules/api.js"; -import { apiJSON, checkJSONPost, getIP, languageCode } from "./modules/sub/utils.js"; -import { Bright, Cyan, Green, Red } from "./modules/sub/consoleText.js"; -import stream from "./modules/stream/stream.js"; -import loc, { loadLoc } from "./localization/manager.js"; -import { buildFront } from "./modules/build.js"; -import { changelogHistory } from "./modules/pageRender/onDemand.js"; -import { sha256 } from "./modules/sub/crypto.js"; -import findRendered from "./modules/pageRender/findRendered.js"; -import { celebrationsEmoji } from "./modules/pageRender/elements.js"; +app.disable('x-powered-by'); -if (process.env.selfURL && process.env.port) { - const commitHash = shortCommit(); - const branch = getCurrentBranch(); - const app = express(); +await loadLoc(); // preload localization - app.disable('x-powered-by'); - - const corsConfig = process.env.cors === '0' ? { origin: process.env.selfURL, optionsSuccessStatus: 200 } : {}; - - const apiLimiter = rateLimit({ - windowMs: 60000, - max: 25, - standardHeaders: false, - legacyHeaders: false, - keyGenerator: (req, res) => sha256(getIP(req), ipSalt), - handler: (req, res, next, opt) => { - res.status(429).json({ "status": "error", "text": loc(languageCode(req), 'ErrorRateLimit') }); - return; - } - }); - const apiLimiterStream = rateLimit({ - windowMs: 60000, - max: 28, - standardHeaders: false, - legacyHeaders: false, - keyGenerator: (req, res) => sha256(getIP(req), ipSalt), - handler: (req, res, next, opt) => { - res.status(429).json({ "status": "error", "text": loc(languageCode(req), 'ErrorRateLimit') }); - return; - } - }); - - const startTime = new Date(); - const startTimestamp = Math.floor(startTime.getTime()); - - // preload localization files and build static pages - await loadLoc(); - await buildFront(commitHash, branch); - - app.use('/api/:type', cors(corsConfig)); - app.use('/api/json', apiLimiter); - app.use('/api/stream', apiLimiterStream); - app.use('/api/onDemand', apiLimiter); - - app.use('/', express.static('./build/min')); - app.use('/', express.static('./src/front')); - - app.use((req, res, next) => { - try { decodeURIComponent(req.path) } catch (e) { return res.redirect('/') } - next(); - }); - app.use((req, res, next) => { - if (req.header("user-agent") && req.header("user-agent").includes("Trident")) res.destroy(); - next(); - }); - app.use('/api/json', express.json({ - verify: (req, res, buf) => { - try { - JSON.parse(buf); - if (buf.length > 720) throw new Error(); - if (String(req.header('Content-Type')) !== "application/json") { - res.status(400).json({ 'status': 'error', 'text': 'invalid content type header' }); - return; - } - if (String(req.header('Accept')) !== "application/json") { - res.status(400).json({ 'status': 'error', 'text': 'invalid accept header' }); - return; - } - } catch(e) { - res.status(400).json({ 'status': 'error', 'text': 'invalid json body.' }); - return; - } - } - })); - - app.post('/api/json', async (req, res) => { - try { - let ip = sha256(getIP(req), ipSalt); - let lang = languageCode(req); - let j = apiJSON(0, { t: "Bad request" }); - try { - let request = req.body; - if (request.url) { - request.dubLang = request.dubLang ? lang : false; - let chck = checkJSONPost(request); - if (chck) chck["ip"] = ip; - j = chck ? await getJSON(chck["url"], lang, chck) : apiJSON(0, { t: loc(lang, 'ErrorCouldntFetch') }); - } else { - j = apiJSON(0, { t: loc(lang, 'ErrorNoLink') }); - } - } catch (e) { - j = apiJSON(0, { t: loc(lang, 'ErrorCantProcess') }); - } - res.status(j.status).json(j.body); - return; - } catch (e) { - res.destroy(); - return - } - }); - - app.get('/api/:type', (req, res) => { - try { - let ip = sha256(getIP(req), ipSalt); - switch (req.params.type) { - case 'stream': - if (req.query.p) { - res.status(200).json({ "status": "continue" }); - return; - } else if (req.query.t && req.query.h && req.query.e) { - stream(res, ip, req.query.t, req.query.h, req.query.e); - } else { - let j = apiJSON(0, { t: "no stream id" }) - res.status(j.status).json(j.body); - return; - } - break; - case 'onDemand': - if (req.query.blockId) { - let blockId = req.query.blockId.slice(0, 3); - let r, j; - switch(blockId) { - case "0": // changelog history - r = changelogHistory(); - j = r ? apiJSON(3, { t: r }) : apiJSON(0, { t: "couldn't render this block" }) - break; - case "1": // celebrations emoji - r = celebrationsEmoji(); - j = r ? apiJSON(3, { t: r }) : false - break; - default: - j = apiJSON(0, { t: "couldn't find a block with this id" }) - break; - } - if (j.body) { - res.status(j.status).json(j.body) - } else { - res.status(204).end() - } - } else { - let j = apiJSON(0, { t: "no block id" }); - res.status(j.status).json(j.body) - } - break; - case 'serverInfo': - res.status(200).json({ - version: version, - commit: commitHash, - branch: branch, - url: process.env.apiURL, - cors: process.env.cors, - startTime: `${startTimestamp}` - }); - break; - default: - let j = apiJSON(0, { t: "unknown response type" }) - res.status(j.status).json(j.body); - break; - } - } catch (e) { - res.status(500).json({ 'status': 'error', 'text': loc(languageCode(req), 'ErrorCantProcess') }); - return; - } - }); - app.get("/api/status", (req, res) => { - res.status(200).end() - }); - app.get("/api", (req, res) => { - res.redirect('/api/json') - }); - app.get("/", (req, res) => { - res.sendFile(`${__dirname}/${findRendered(languageCode(req), req.header('user-agent') ? req.header('user-agent') : genericUserAgent)}`); - }); - app.get("/favicon.ico", (req, res) => { - res.redirect('/icons/favicon.ico'); - }); - app.get("/*", (req, res) => { - res.redirect('/') - }); - - app.listen(process.env.port, () => { - console.log(`\n${Cyan(appName)} ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(startTimestamp)})`)}\n\nURL: ${Cyan(`${process.env.selfURL}`)}\nPort: ${process.env.port}\n`) - }) +if (process.env.apiURL && process.env.apiPort && !((process.env.webURL && process.env.webPort) || (process.env.selfURL && process.env.port))) { + await runAPI(express, app, gitCommit, gitBranch, __dirname); +} else if (process.env.webURL && process.env.webPort && !((process.env.apiURL && process.env.apiPort) || (process.env.selfURL && process.env.port))) { + await runWeb(express, app, gitCommit, gitBranch, __dirname); +} else if (process.env.selfURL && process.env.port && !((process.env.apiURL && process.env.apiPort) || (process.env.webURL && process.env.webPort))) { + await runBoth(express, app, gitCommit, gitBranch, __dirname) } else { console.log(Red(`cobalt hasn't been configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`)); } diff --git a/src/api.js b/src/core/api.js similarity index 78% rename from src/api.js rename to src/core/api.js index 74c33e1f..0c9bfdbb 100644 --- a/src/api.js +++ b/src/core/api.js @@ -1,30 +1,21 @@ -import "dotenv/config"; - -import express from "express"; import cors from "cors"; import rateLimit from "express-rate-limit"; import { randomBytes } from "crypto"; const ipSalt = randomBytes(64).toString('hex'); -import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js"; -import { appName, version } from "./modules/config.js"; -import { getJSON } from "./modules/api.js"; -import { apiJSON, checkJSONPost, getIP, languageCode } from "./modules/sub/utils.js"; -import { Bright, Cyan, Green, Red } from "./modules/sub/consoleText.js"; -import stream from "./modules/stream/stream.js"; -import loc, { loadLoc } from "./localization/manager.js"; -import { changelogHistory } from "./modules/pageRender/onDemand.js"; -import { sha256 } from "./modules/sub/crypto.js"; -import { celebrationsEmoji } from "./modules/pageRender/elements.js"; - -if (process.env.apiURL && process.env.apiPort) { - const commitHash = shortCommit(); - const branch = getCurrentBranch(); - const app = express(); - - app.disable('x-powered-by'); +import { appName, version } from "../modules/config.js"; +import { getJSON } from "../modules/api.js"; +import { apiJSON, checkJSONPost, getIP, languageCode } from "../modules/sub/utils.js"; +import { Bright, Cyan } from "../modules/sub/consoleText.js"; +import stream from "../modules/stream/stream.js"; +import loc from "../localization/manager.js"; +import { changelogHistory } from "../modules/pageRender/onDemand.js"; +import { sha256 } from "../modules/sub/crypto.js"; +import { celebrationsEmoji } from "../modules/pageRender/elements.js"; +import { verifyStream } from "../modules/stream/manage.js"; +export async function runAPI(express, app, gitCommit, gitBranch, __dirname) { const corsConfig = process.env.cors === '0' ? { origin: process.env.webURL, optionsSuccessStatus: 200 } : {}; const apiLimiter = rateLimit({ @@ -53,9 +44,6 @@ if (process.env.apiURL && process.env.apiPort) { const startTime = new Date(); const startTimestamp = Math.floor(startTime.getTime()); - // preload localization files - await loadLoc(); - app.use('/api/:type', cors(corsConfig)); app.use('/api/json', apiLimiter); app.use('/api/stream', apiLimiterStream); @@ -65,10 +53,6 @@ if (process.env.apiURL && process.env.apiPort) { try { decodeURIComponent(req.path) } catch (e) { return res.redirect('/') } next(); }); - app.use((req, res, next) => { - if (req.header("user-agent") && req.header("user-agent").includes("Trident")) res.destroy(); - next(); - }); app.use('/api/json', express.json({ verify: (req, res, buf) => { try { @@ -120,6 +104,12 @@ if (process.env.apiURL && process.env.apiPort) { let ip = sha256(getIP(req), ipSalt); switch (req.params.type) { case 'stream': + let streamInfo = verifyStream(ip, req.query.t, req.query.h, req.query.e); + if (streamInfo.error) { + res.status(streamInfo.status).json(apiJSON(0, { t: streamInfo.error }).body); + return; + } + if (req.query.p) { res.status(200).json({ "status": "continue" }); return; @@ -161,8 +151,9 @@ if (process.env.apiURL && process.env.apiPort) { case 'serverInfo': res.status(200).json({ version: version, - commit: commitHash, - branch: branch, + commit: gitCommit, + branch: gitBranch, + name: process.env.apiName ? process.env.apiName : "unknown", url: process.env.apiURL, cors: process.env.cors, startTime: `${startTimestamp}` @@ -178,19 +169,17 @@ if (process.env.apiURL && process.env.apiPort) { return; } }); - app.get("/api/status", (req, res) => { + app.get('/api/status', (req, res) => { res.status(200).end() }); - app.get("/favicon.ico", (req, res) => { - res.redirect('/icons/favicon.ico'); + app.get('/favicon.ico', (req, res) => { + res.sendFile(`${__dirname}/src/front/icons/favicon.ico`) }); - app.get("/*", (req, res) => { + app.get('/*', (req, res) => { res.redirect('/api/json') }); app.listen(process.env.apiPort, () => { - console.log(`\n${Cyan(appName)} API ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${startTimestamp})`)}\n\nURL: ${Cyan(`${process.env.apiURL}`)}\nPort: ${process.env.apiPort}\n`) - }) -} else { - console.log(Red(`cobalt api hasn't been configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`)); + console.log(`\n${Cyan(appName)} API ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${startTimestamp})`)}\n\nURL: ${Cyan(`${process.env.apiURL}`)}\nPort: ${process.env.apiPort}\n`) + }); } diff --git a/src/core/both.js b/src/core/both.js new file mode 100644 index 00000000..4b14592f --- /dev/null +++ b/src/core/both.js @@ -0,0 +1,197 @@ +import cors from "cors"; +import rateLimit from "express-rate-limit"; +import { randomBytes } from "crypto"; + +const ipSalt = randomBytes(64).toString('hex'); + +import { appName, genericUserAgent, version } from "../modules/config.js"; +import { getJSON } from "../modules/api.js"; +import { apiJSON, checkJSONPost, getIP, languageCode } from "../modules/sub/utils.js"; +import { Bright, Cyan, Green, Red } from "../modules/sub/consoleText.js"; +import stream from "../modules/stream/stream.js"; +import loc from "../localization/manager.js"; +import { buildFront } from "../modules/build.js"; +import { changelogHistory } from "../modules/pageRender/onDemand.js"; +import { sha256 } from "../modules/sub/crypto.js"; +import findRendered from "../modules/pageRender/findRendered.js"; +import { celebrationsEmoji } from "../modules/pageRender/elements.js"; + +export async function runBoth(express, app, gitCommit, gitBranch, __dirname) { + const corsConfig = process.env.cors === '0' ? { origin: process.env.selfURL, optionsSuccessStatus: 200 } : {}; + + const apiLimiter = rateLimit({ + windowMs: 60000, + max: 25, + standardHeaders: false, + legacyHeaders: false, + keyGenerator: (req, res) => sha256(getIP(req), ipSalt), + handler: (req, res, next, opt) => { + res.status(429).json({ "status": "error", "text": loc(languageCode(req), 'ErrorRateLimit') }); + return; + } + }); + const apiLimiterStream = rateLimit({ + windowMs: 60000, + max: 28, + standardHeaders: false, + legacyHeaders: false, + keyGenerator: (req, res) => sha256(getIP(req), ipSalt), + handler: (req, res, next, opt) => { + res.status(429).json({ "status": "error", "text": loc(languageCode(req), 'ErrorRateLimit') }); + return; + } + }); + + const startTime = new Date(); + const startTimestamp = Math.floor(startTime.getTime()); + + // preload localization files and build static pages + await buildFront(gitCommit, gitBranch); + + app.use('/api/:type', cors(corsConfig)); + app.use('/api/json', apiLimiter); + app.use('/api/stream', apiLimiterStream); + app.use('/api/onDemand', apiLimiter); + + app.use('/', express.static('./build/min')); + app.use('/', express.static('./src/front')); + + app.use((req, res, next) => { + try { decodeURIComponent(req.path) } catch (e) { return res.redirect('/') } + next(); + }); + app.use((req, res, next) => { + if (req.header("user-agent") && req.header("user-agent").includes("Trident")) res.destroy(); + next(); + }); + app.use('/api/json', express.json({ + verify: (req, res, buf) => { + try { + JSON.parse(buf); + if (buf.length > 720) throw new Error(); + if (String(req.header('Content-Type')) !== "application/json") { + res.status(400).json({ 'status': 'error', 'text': 'invalid content type header' }); + return; + } + if (String(req.header('Accept')) !== "application/json") { + res.status(400).json({ 'status': 'error', 'text': 'invalid accept header' }); + return; + } + } catch(e) { + res.status(400).json({ 'status': 'error', 'text': 'invalid json body.' }); + return; + } + } + })); + + app.post('/api/json', async (req, res) => { + try { + let ip = sha256(getIP(req), ipSalt); + let lang = languageCode(req); + let j = apiJSON(0, { t: "Bad request" }); + try { + let request = req.body; + if (request.url) { + request.dubLang = request.dubLang ? lang : false; + let chck = checkJSONPost(request); + if (chck) chck["ip"] = ip; + j = chck ? await getJSON(chck["url"], lang, chck) : apiJSON(0, { t: loc(lang, 'ErrorCouldntFetch') }); + } else { + j = apiJSON(0, { t: loc(lang, 'ErrorNoLink') }); + } + } catch (e) { + j = apiJSON(0, { t: loc(lang, 'ErrorCantProcess') }); + } + res.status(j.status).json(j.body); + return; + } catch (e) { + res.destroy(); + return + } + }); + + app.get('/api/:type', (req, res) => { + try { + let ip = sha256(getIP(req), ipSalt); + switch (req.params.type) { + case 'stream': + if (req.query.p) { + res.status(200).json({ "status": "continue" }); + return; + } else if (req.query.t && req.query.h && req.query.e) { + stream(res, ip, req.query.t, req.query.h, req.query.e); + } else { + let j = apiJSON(0, { t: "no stream id" }) + res.status(j.status).json(j.body); + return; + } + break; + case 'onDemand': + if (req.query.blockId) { + let blockId = req.query.blockId.slice(0, 3); + let r, j; + switch(blockId) { + case "0": // changelog history + r = changelogHistory(); + j = r ? apiJSON(3, { t: r }) : apiJSON(0, { t: "couldn't render this block" }) + break; + case "1": // celebrations emoji + r = celebrationsEmoji(); + j = r ? apiJSON(3, { t: r }) : false + break; + default: + j = apiJSON(0, { t: "couldn't find a block with this id" }) + break; + } + if (j.body) { + res.status(j.status).json(j.body) + } else { + res.status(204).end() + } + } else { + let j = apiJSON(0, { t: "no block id" }); + res.status(j.status).json(j.body) + } + break; + case 'serverInfo': + res.status(200).json({ + version: version, + commit: gitCommit, + branch: gitBranch, + name: process.env.apiName ? process.env.apiName : "unknown", + url: process.env.apiURL, + cors: process.env.cors, + startTime: `${startTimestamp}` + }); + break; + default: + let j = apiJSON(0, { t: "unknown response type" }) + res.status(j.status).json(j.body); + break; + } + } catch (e) { + res.status(500).json({ 'status': 'error', 'text': loc(languageCode(req), 'ErrorCantProcess') }); + return; + } + }); + app.get("/api/status", (req, res) => { + res.status(200).end() + }); + app.get("/api", (req, res) => { + res.redirect('/api/json') + }); + app.get("/", (req, res) => { + res.sendFile(`${__dirname}/${findRendered(languageCode(req), req.header('user-agent') ? req.header('user-agent') : genericUserAgent)}`); + }); + app.get("/favicon.ico", (req, res) => { + res.redirect('/icons/favicon.ico'); + }); + app.get("/*", (req, res) => { + res.redirect('/') + }); + + app.listen(process.env.port, () => { + console.log(`${Red("⚠️ This way of running cobalt has been deprecated and will be removed soon.\nCheck the docs and get ready: ")}${Green("WIP")}`) + console.log(`\n${Cyan(appName)} ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(startTimestamp)})`)}\n\nURL: ${Cyan(`${process.env.selfURL}`)}\nPort: ${process.env.port}\n`) + }) +} diff --git a/src/core/web.js b/src/core/web.js new file mode 100644 index 00000000..95f1fd4a --- /dev/null +++ b/src/core/web.js @@ -0,0 +1,34 @@ +import { appName, genericUserAgent, version } from "../modules/config.js"; +import { languageCode } from "../modules/sub/utils.js"; +import { Bright, Cyan } from "../modules/sub/consoleText.js"; +import { buildFront } from "../modules/build.js"; +import findRendered from "../modules/pageRender/findRendered.js"; + +export async function runWeb(express, app, gitCommit, gitBranch, __dirname) { + await buildFront(gitCommit, gitBranch); + + app.use('/', express.static('./build/min')); + app.use('/', express.static('./src/front')); + + app.use((req, res, next) => { + try { decodeURIComponent(req.path) } catch (e) { return res.redirect('/') } + next(); + }); + app.get("/status", (req, res) => { + res.status(200).end() + }); + app.get("/", (req, res) => { + res.sendFile(`${__dirname}/${findRendered(languageCode(req), req.header('user-agent') ? req.header('user-agent') : genericUserAgent)}`) + }); + app.get("/favicon.ico", (req, res) => { + res.sendFile(`${__dirname}/src/front/icons/favicon.ico`) + }); + app.get("/*", (req, res) => { + res.redirect('/') + }); + + app.listen(process.env.webPort, () => { + let startTime = new Date(); + console.log(`\n${Cyan(appName)} WEB ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(new Date().getTime())})`)}\n\nURL: ${Cyan(`${process.env.webURL}`)}\nPort: ${process.env.webPort}\n`) + }) +} diff --git a/src/front/cobalt.css b/src/front/cobalt.css index 338f0dc1..749ff065 100644 --- a/src/front/cobalt.css +++ b/src/front/cobalt.css @@ -664,10 +664,6 @@ button:active, #pd-share { display: none; } -#hop-attribution { - display: block; - text-align: right; -} #about-donate-footer::before { content: ""; position: absolute; diff --git a/src/front/cobalt.js b/src/front/cobalt.js index 59ec949a..f801a704 100644 --- a/src/front/cobalt.js +++ b/src/front/cobalt.js @@ -19,8 +19,6 @@ const exceptions = { // used for mobile devices "vQuality": "720" }; -let apiURL = ''; - let store = {}; function changeAPI(url) { diff --git a/src/modules/changelog/changelog.json b/src/modules/changelog/changelog.json index 0636d947..aa573457 100644 --- a/src/modules/changelog/changelog.json +++ b/src/modules/changelog/changelog.json @@ -1,9 +1,9 @@ { "current": { "version": "5.4", - "title": "instagram support, hop, docker, and more!", + "title": "instagram support, docker, and more!", "banner": "catphonestand.webp", - "content": "something many of you've been waiting for is finally here! try it out and let me know what you think :)\n\ntl;dr:\n*; added experimental instagram support! download any reels or videos you like, and make sure to report any issues you encounter. yes, you can convert either to audio.\n*; fixed support for on.soundcloud links.\n*; added share button to \"how to save?\" popup.\n*; added docker support.\n*; main instance is now powered by hop.io.\n\nservice improvements:\n*; added experimental support for videos from instagram. currently only reels and post videos are downloadable, but i'm looking into ways to save high resolution photos too. if you experience any issues, please report them on either of support platforms.\n*; fixed support for on.soundcloud share links. should work just as well as other versions!\n*; fixed an issue that made some youtube videos impossible to download.\n\ninterface improvements:\n*; new css-only checkmark! yes, i can't stop tinkering with it because slight flashing on svg load annoyed me. now it loads instantly (and also looks slightly better).\n*; fixed copy animation.\n*; minor localization improvements.\n*; fixed the embed logo that i broke somewhere in between 5.3 and 5.4.\n\ninternal improvements:\n*; now using nanoid for live render stream ids.\n*; added support for docker. it's kind of clumsy because of how i get .git folder inside the container, but if you know how to do it better, feel free to make a pr.\n*; cobalt now checks only for existence of environment variables, not exactly the .env file.\n*; changed the way user ip address is retrieved for instances using cloudflare.\n*; added ability to disable cors, both to setup script and environment variables.\n*; moved main instance to hop.io infra. there should no longer be random downtimes. huge shout out to the hop team for being so nice and helping me out :D\n\ni can't believe how diverse and widespread cobalt has become. it's used in all fields: music production, education, content creation, and even game development. thank you. this is absolutely nuts.\nif you don't mind sharing, please tell me about your use case. i'd really love to hear how you use cobalt and how i could make it even more useful for you." + "content": "something many of you've been waiting for is finally here! try it out and let me know what you think :)\n\ntl;dr:\n*; added experimental instagram support! download any reels or videos you like, and make sure to report any issues you encounter. yes, you can convert either to audio.\n*; fixed support for on.soundcloud links.\n*; added share button to \"how to save?\" popup.\n*; added docker support.\n\nservice improvements:\n*; added experimental support for videos from instagram. currently only reels and post videos are downloadable, but i'm looking into ways to save high resolution photos too. if you experience any issues, please report them on either of support platforms.\n*; fixed support for on.soundcloud share links. should work just as well as other versions!\n*; fixed an issue that made some youtube videos impossible to download.\n\ninterface improvements:\n*; new css-only checkmark! yes, i can't stop tinkering with it because slight flashing on svg load annoyed me. now it loads instantly (and also looks slightly better).\n*; fixed copy animation.\n*; minor localization improvements.\n*; fixed the embed logo that i broke somewhere in between 5.3 and 5.4.\n\ninternal improvements:\n*; now using nanoid for live render stream ids.\n*; added support for docker. it's kind of clumsy because of how i get .git folder inside the container, but if you know how to do it better, feel free to make a pr.\n*; cobalt now checks only for existence of environment variables, not exactly the .env file.\n*; changed the way user ip address is retrieved for instances using cloudflare.\n*; added ability to disable cors, both to setup script and environment variables.\n\ni can't believe how diverse and widespread cobalt has become. it's used in all fields: music production, education, content creation, and even game development. thank you. this is absolutely nuts.\nif you don't mind sharing, please tell me about your use case. i'd really love to hear how you use cobalt and how i could make it even more useful for you." }, "history": [{ "version": "5.3", diff --git a/src/modules/pageRender/page.js b/src/modules/pageRender/page.js index d506890a..f4046e80 100644 --- a/src/modules/pageRender/page.js +++ b/src/modules/pageRender/page.js @@ -48,10 +48,10 @@ export default function(obj) { ${appName} - + - + @@ -106,7 +106,6 @@ export default function(obj) { "title": t("CollapsePrivacy"), "body": t("PrivacyPolicy") }]) - + `${process.env.DEPLOYMENT_ID && process.env.INTERNAL_IP ? 'powered by hop.io' : ''}` }] }) }, { @@ -408,7 +407,8 @@ export default function(obj) { }])} - + }; + let apiURL = '${process.env.apiURL ? process.env.apiURL.slice(0, -1) : ''}'; + `; } catch (err) { diff --git a/src/web.js b/src/web.js deleted file mode 100644 index 4f298975..00000000 --- a/src/web.js +++ /dev/null @@ -1,59 +0,0 @@ -import "dotenv/config"; - -import express from "express"; - -import path from 'path'; -import { fileURLToPath } from 'url'; -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename).slice(0, -4); // go up another level (get rid of src/) - -import { getCurrentBranch, shortCommit } from "./modules/sub/currentCommit.js"; -import { appName, genericUserAgent, version } from "./modules/config.js"; -import { languageCode } from "./modules/sub/utils.js"; -import { Bright, Cyan, Green, Red } from "./modules/sub/consoleText.js"; -import { loadLoc } from "./localization/manager.js"; -import { buildFront } from "./modules/build.js"; -import findRendered from "./modules/pageRender/findRendered.js"; - -if (process.env.webURL && process.env.webPort) { - const commitHash = shortCommit(); - const branch = getCurrentBranch(); - const app = express(); - - app.disable('x-powered-by'); - - // preload localization files and build static pages - await loadLoc(); - await buildFront(commitHash, branch); - - app.use('/', express.static('./build/min')); - app.use('/', express.static('./src/front')); - - app.use((req, res, next) => { - try { decodeURIComponent(req.path) } catch (e) { return res.redirect('/') } - next(); - }); - app.use((req, res, next) => { - if (req.header("user-agent") && req.header("user-agent").includes("Trident")) res.destroy(); - next(); - }); - app.get("/status", (req, res) => { - res.status(200).end() - }); - app.get("/", (req, res) => { - res.sendFile(`${__dirname}/${findRendered(languageCode(req), req.header('user-agent') ? req.header('user-agent') : genericUserAgent)}`); - }); - app.get("/favicon.ico", (req, res) => { - res.redirect('/icons/favicon.ico'); - }); - app.get("/*", (req, res) => { - res.redirect('/') - }); - - app.listen(process.env.webPort, () => { - let startTime = new Date(); - console.log(`\n${Cyan(appName)} WEB ${Bright(`v.${version}-${commitHash} (${branch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(new Date().getTime())})`)}\n\nURL: ${Cyan(`${process.env.webURL}`)}\nPort: ${process.env.webPort}\n`) - }) -} else { - console.log(Red(`cobalt web hasn't been configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`)); -} From 5940bf0b18ca7975bbb80ac7592a09021b1a1612 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 22 May 2023 01:14:59 +0600 Subject: [PATCH 5/8] comment --- src/cobalt.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cobalt.js b/src/cobalt.js index 6f3dd795..d349a79e 100644 --- a/src/cobalt.js +++ b/src/cobalt.js @@ -25,6 +25,7 @@ app.disable('x-powered-by'); await loadLoc(); // preload localization +// i don't like this at all if (process.env.apiURL && process.env.apiPort && !((process.env.webURL && process.env.webPort) || (process.env.selfURL && process.env.port))) { await runAPI(express, app, gitCommit, gitBranch, __dirname); } else if (process.env.webURL && process.env.webPort && !((process.env.apiURL && process.env.apiPort) || (process.env.selfURL && process.env.port))) { From 1014ee3413f7a1ffd341d19c73d80ca772348d11 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 23 May 2023 22:52:17 +0600 Subject: [PATCH 6/8] added discord server to about --- src/config.json | 4 ++++ src/front/emoji/alien_monster.svg | 9 +++++++++ src/modules/emoji.js | 3 ++- src/modules/pageRender/page.js | 5 +++-- 4 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 src/front/emoji/alien_monster.svg diff --git a/src/config.json b/src/config.json index 2871c3ea..91915bcf 100644 --- a/src/config.json +++ b/src/config.json @@ -14,6 +14,10 @@ "mastodon": { "url": "https://wetdry.world/@cobalt", "handle": "@cobalt@wetdry.world" + }, + "discord": { + "url": "https://discord.gg/pQPt8HBUPu", + "handle": "cobalt community server" } } }, diff --git a/src/front/emoji/alien_monster.svg b/src/front/emoji/alien_monster.svg new file mode 100644 index 00000000..66be00bd --- /dev/null +++ b/src/front/emoji/alien_monster.svg @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/src/modules/emoji.js b/src/modules/emoji.js index 284404d4..3565a452 100644 --- a/src/modules/emoji.js +++ b/src/modules/emoji.js @@ -23,7 +23,8 @@ const names = { "🐙": "octopus", "🔮": "crystal_ball", "💪": "biceps", - "💖": "sparkling_heart" + "💖": "sparkling_heart", + "👾": "alien_monster" } let sizing = { 22: 0.4, diff --git a/src/modules/pageRender/page.js b/src/modules/pageRender/page.js index f4046e80..9e6bf957 100644 --- a/src/modules/pageRender/page.js +++ b/src/modules/pageRender/page.js @@ -96,8 +96,9 @@ export default function(obj) { "name": "support", "title": t("CollapseSupport"), "body": `${t("FollowSupport")}
- ${socialLink(emoji("🐘"), "mastodon", authorInfo.support.mastodon.handle, authorInfo.support.mastodon.url)} - ${socialLink(emoji("🐦"), "twitter", authorInfo.support.twitter.handle, authorInfo.support.twitter.url)}
+ ${socialLink(emoji("🐦"), "twitter", authorInfo.support.twitter.handle, authorInfo.support.twitter.url)} + ${socialLink(emoji("👾"), "discord", authorInfo.support.discord.handle, authorInfo.support.discord.url)} + ${socialLink(emoji("🐘"), "mastodon", authorInfo.support.mastodon.handle, authorInfo.support.mastodon.url)}
${t("SourceCode")}
${socialLink(emoji("🐙"), "github", repo.replace("https://github.com/", ''), repo)}
${t("SupportNote")}` From 55f1e4b704cafcca64bc9bc4fe14c0eaa0cd907c Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 5 Jun 2023 12:43:04 +0600 Subject: [PATCH 7/8] tumblr audio, youtube vr, updated setup script, further mitosis accommodations --- README.md | 20 ++-- docker-compose.yml.example | 4 - docs/API.md | 8 ++ src/core/web.js | 17 +++ src/modules/api.js | 3 +- src/modules/processing/matchActionDecider.js | 8 +- src/modules/processing/services/tumblr.js | 18 +++- src/modules/processing/services/youtube.js | 13 ++- src/modules/setup.js | 104 ++++++++++++++----- src/test/tests.json | 29 ++++++ 10 files changed, 173 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 881a6347..e5355d1d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # cobalt Best way to save what you love. -Main instance: [co.wukko.me](https://co.wukko.me/) +Live web app: [co.wukko.me](https://co.wukko.me/) ![cobalt logo with repeated logo pattern background](https://raw.githubusercontent.com/wukko/cobalt/current/src/front/icons/pattern.png "cobalt logo with repeated logo pattern background") @@ -21,21 +21,22 @@ Paste the link, get the video, move on. It's that simple. Just how it should be. | Reddit | ✅ | ✅ | ✅ | Support for GIFs and videos. | | SoundCloud | ➖ | ✅ | ➖ | Audio metadata, downloads from private links. | | TikTok | ✅ | ✅ | ✅ | Supports downloads of: videos with or without watermark, images from slideshow without watermark, full (original) audios. | -| Tumblr | ✅ | ✅ | ✅ | | +| Tumblr | ✅ | ✅ | ✅ | Support for audio file downloads. | | Twitter | ✅ | ✅ | ✅ | Ability to pick what to save from multi-media tweets. | | Twitter Spaces | ➖ | ✅ | ➖ | Audio metadata with all participants and other info. | | Vimeo | ✅ | ✅ | ✅ | Audio downloads are only available for dash files. | | Vine Archive | ✅ | ✅ | ✅ | | | VK Videos | ✅ | ❌ | ❌ | | | VK Clips | ✅ | ❌ | ❌ | | -| YouTube Videos & Shorts | ✅ | ✅ | ✅ | Support for 8K, 4K, HDR, and high FPS videos. Audio metadata & dubs. h264/av1/vp9 codecs. | +| YouTube Videos & Shorts | ✅ | ✅ | ✅ | Support for 8K, 4K, HDR, VR, and high FPS videos. Audio metadata & dubs. h264/av1/vp9 codecs. | | YouTube Music | ➖ | ✅ | ➖ | Audio metadata. | This list is not final and keeps expanding over time, make sure to check it once in a while! ## cobalt API cobalt has an open API that you can use in your projects for **free**. -It's easy and straightforward to use, [check out the docs](https://github.com/wukko/cobalt/blob/current/docs/API.md) and see for yourself. +It's easy and straightforward to use, [check out the docs](https://github.com/wukko/cobalt/blob/current/docs/API.md) and see for yourself. +Feel free to use the main API instance ([co.wuk.sh](https://co.wuk.sh/)) in your projects. ## How to contribute translations You can translate cobalt to any language you want on [cobalt's Crowdin](https://crowdin-co.wukko.me/). Feel free to ignore QA errors if you think you know better. If you don't see a language you want to translate cobalt to, open an issue, and I'll add it to Crowdin. @@ -62,6 +63,8 @@ Setup script installs all needed `npm` dependencies, but you have to install `No 3. Run cobalt via `npm start` 4. Done. +You need to host API and web app separately ever since v.6.0. Setup script will help you with that! + ### Ubuntu 22.04+ workaround `nscd` needs to be installed and running so that the `ffmpeg-static` binary can resolve DNS ([#101](https://github.com/wukko/cobalt/issues/101#issuecomment-1494822258)): @@ -71,13 +74,8 @@ sudo service nscd start ``` ### Docker -It's also possible to run cobalt via Docker, but you **need** to set all environment variables yourself: - -| Variable | Description | Example | -| -------- | :--- | :--- | -| `selfURL` | Instance URL | `http://localhost:9000/` or `https://co.wukko.me/` or etc | -| `port` | Instance port | `9000` | -| `cors` | CORS toggle | `0` | +It's also possible to run cobalt via Docker. I *highly* recommend using Docker compose. +Check out the [example compose file](https://github.com/wukko/cobalt/blob/current/docker-compose.yml.example) and alter it for your needs. ## Disclaimer cobalt is my passion project, so update schedule depends solely on my free time, motivation, and mood. diff --git a/docker-compose.yml.example b/docker-compose.yml.example index cb50b9f7..8ba8c870 100644 --- a/docker-compose.yml.example +++ b/docker-compose.yml.example @@ -11,7 +11,6 @@ services: - apiPort=9000 - apiURL=https://co.wuk.sh/ - apiName=eu-nl - - cors=1 cobalt-web: build: . restart: unless-stopped @@ -21,9 +20,7 @@ services: environment: - webPort=9000 - webURL=https://co.wukko.me/ - - apiPort=9000 - apiURL=https://co.wuk.sh/ - - cors=1 cobalt-both: build: . restart: unless-stopped @@ -33,4 +30,3 @@ services: environment: - port=9000 - selfURL=https://co.wukko.me/ - - cors=1 diff --git a/docs/API.md b/docs/API.md index f45e2c0e..38038bd0 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1,5 +1,13 @@ # cobalt API Documentation This document provides info about methods and acceptable variables for all cobalt API requests.
+ +``` +⚠️ Main API instance has moved to https://co.wuk.sh/ + +Previous API domain will stop redirecting users to correct API instance after July 25th. +Make sure to update your projects in time. +``` + ## POST: ``/api/json`` Main processing endpoint.
diff --git a/src/core/web.js b/src/core/web.js index 95f1fd4a..afde9fe7 100644 --- a/src/core/web.js +++ b/src/core/web.js @@ -4,9 +4,18 @@ import { Bright, Cyan } from "../modules/sub/consoleText.js"; import { buildFront } from "../modules/build.js"; import findRendered from "../modules/pageRender/findRendered.js"; +// * will be removed in the future +import cors from "cors"; +// * + export async function runWeb(express, app, gitCommit, gitBranch, __dirname) { await buildFront(gitCommit, gitBranch); + // * will be removed in the future + const corsConfig = process.env.cors === '0' ? { origin: process.env.webURL, optionsSuccessStatus: 200 } : {}; + app.use('/api/:type', cors(corsConfig)); + // * + app.use('/', express.static('./build/min')); app.use('/', express.static('./src/front')); @@ -23,6 +32,14 @@ export async function runWeb(express, app, gitCommit, gitBranch, __dirname) { app.get("/favicon.ico", (req, res) => { res.sendFile(`${__dirname}/src/front/icons/favicon.ico`) }); + // * will be removed in the future + app.get("/api/*", (req, res) => { + res.redirect(308, process.env.apiURL.slice(0, -1) + req.url) + }); + app.post("/api/*", (req, res) => { + res.redirect(308, process.env.apiURL.slice(0, -1) + req.url) + }); + // * app.get("/*", (req, res) => { res.redirect('/') }); diff --git a/src/modules/api.js b/src/modules/api.js index d5446810..94ed5040 100644 --- a/src/modules/api.js +++ b/src/modules/api.js @@ -34,8 +34,9 @@ export async function getJSON(originalURL, lang, obj) { } if (!(host && host.length < 20 && host in patterns && patterns[host]["enabled"])) return apiJSON(0, { t: errorUnsupported(lang) }); + let pathToMatch = cleanURL(url, host).split(`.${patterns[host]['tld'] ? patterns[host]['tld'] : "com"}/`)[1].replace('.', ''); for (let i in patterns[host]["patterns"]) { - patternMatch = new UrlPattern(patterns[host]["patterns"][i]).match(cleanURL(url, host).split(`.${patterns[host]['tld'] ? patterns[host]['tld'] : "com"}/`)[1].replace('.', '')); + patternMatch = new UrlPattern(patterns[host]["patterns"][i]).match(pathToMatch); if (patternMatch) break } if (!patternMatch) return apiJSON(0, { t: errorUnsupported(lang) }); diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index ba10af27..e0005044 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -113,9 +113,11 @@ export default function(r, host, ip, audioFormat, isAudioOnly, lang, isAudioMute processType = "bridge" } } - - if ((audioFormat === "best" && services[host]["bestAudio"]) - || services[host]["bestAudio"] && (audioFormat === services[host]["bestAudio"])) { + if (host === "tumblr" && !r.filename && (audioFormat === "best" || audioFormat === "mp3")) { + audioFormat = "mp3"; + processType = "bridge" + } + if ((audioFormat === "best" && services[host]["bestAudio"]) || (services[host]["bestAudio"] && (audioFormat === services[host]["bestAudio"]))) { audioFormat = services[host]["bestAudio"]; processType = "bridge" } else if (audioFormat === "best") { diff --git a/src/modules/processing/services/tumblr.js b/src/modules/processing/services/tumblr.js index 95ba7f4e..7ae7336c 100644 --- a/src/modules/processing/services/tumblr.js +++ b/src/modules/processing/services/tumblr.js @@ -8,7 +8,21 @@ export default async function(obj) { }).then((r) => { return r.text() }).catch(() => { return false }); if (!html) return { error: 'ErrorCouldntFetch' }; - if (!html.includes('property="og:video" content="https://va.media.tumblr.com/')) return { error: 'ErrorEmptyDownload' }; - return { urls: `https://va.media.tumblr.com/${html.split('property="og:video" content="https://va.media.tumblr.com/')[1].split('"')[0]}`, filename: `tumblr_${obj.id}.mp4`, audioFilename: `tumblr_${obj.id}_audio` } + let r; + if (html.includes('property="og:video" content="https://va.media.tumblr.com/')) { + r = { + urls: `https://va.media.tumblr.com/${html.split('property="og:video" content="https://va.media.tumblr.com/')[1].split('"')[0]}`, + filename: `tumblr_${obj.id}.mp4`, + audioFilename: `tumblr_${obj.id}_audio` + } + } else if (html.includes('property="og:audio" content="https://a.tumblr.com/')) { + r = { + urls: `https://a.tumblr.com/${html.split('property="og:audio" content="https://a.tumblr.com/')[1].split('"')[0]}`, + audioFilename: `tumblr_${obj.id}`, + isAudioOnly: true + } + } else r = { error: 'ErrorEmptyDownload' }; + + return r; } diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 25266547..e686b4c7 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -23,6 +23,10 @@ const c = { export default async function(o) { let info, isDubbed, quality = o.quality === "max" ? "9000" : o.quality; //set quality 9000(p) to be interpreted as max + function qual(i) { + return i['quality_label'].split('p')[0].split('s')[0] + } + try { info = await yt.getBasicInfo(o.id, 'ANDROID'); } catch (e) { @@ -30,6 +34,7 @@ export default async function(o) { } if (!info) return { error: 'ErrorCantConnectToServiceAPI' }; + if (info.playability_status.status !== 'OK') return { error: 'ErrorYTUnavailable' }; if (info.basic_info.is_live) return { error: 'ErrorLiveVideo' }; @@ -40,7 +45,7 @@ export default async function(o) { bestQuality = adaptive_formats.find(i => i["has_video"]); hasAudio = adaptive_formats.find(i => i["has_audio"]); - if (bestQuality) bestQuality = bestQuality['quality_label'].split('p')[0]; + if (bestQuality) bestQuality = qual(bestQuality); if (!bestQuality && !o.isAudioOnly || !hasAudio) return { error: 'ErrorYTTryOtherCodec' }; if (info.basic_info.duration > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; @@ -73,9 +78,9 @@ export default async function(o) { }; return r } - let checkSingle = (i) => ((i['quality_label'].split('p')[0] === quality || i['quality_label'].split('p')[0] === bestQuality) && i["mime_type"].includes(c[o.format].codec)), - checkBestVideo = (i) => (i["has_video"] && !i["has_audio"] && i['quality_label'].split('p')[0] === bestQuality), - checkRightVideo = (i) => (i["has_video"] && !i["has_audio"] && i['quality_label'].split('p')[0] === quality); + let checkSingle = (i) => ((qual(i) === quality || qual(i) === bestQuality) && i["mime_type"].includes(c[o.format].codec)), + checkBestVideo = (i) => (i["has_video"] && !i["has_audio"] && qual(i) === bestQuality), + checkRightVideo = (i) => (i["has_video"] && !i["has_audio"] && qual(i) === quality); if (!o.isAudioOnly && !o.isAudioMuted && o.format === 'h264') { let single = info.streaming_data.formats.find(i => checkSingle(i)); diff --git a/src/modules/setup.js b/src/modules/setup.js index 2740a770..3aae0543 100644 --- a/src/modules/setup.js +++ b/src/modules/setup.js @@ -5,48 +5,100 @@ import { execSync } from "child_process"; let envPath = './.env'; let q = `${Cyan('?')} \x1b[1m`; -let ob = {} +let ob = {}; let rl = createInterface({ input: process.stdin, output: process.stdout }); let final = () => { - if (existsSync(envPath)) { - unlinkSync(envPath) - } + if (existsSync(envPath)) unlinkSync(envPath); + for (let i in ob) { appendFileSync(envPath, `${i}=${ob[i]}\n`) } - console.log(Bright("\nAwesome! I've created a fresh .env file for you.")) - console.log(`${Bright("Now I'll run")} ${Cyan("npm install")} ${Bright("to install all dependencies. It shouldn't take long.\n\n")}`) + console.log(Bright("\nAwesome! I've created a fresh .env file for you.")); + console.log(`${Bright("Now I'll run")} ${Cyan("npm install")} ${Bright("to install all dependencies. It shouldn't take long.\n\n")}`); execSync('npm install', { stdio: [0, 1, 2] }); - console.log(`\n\n${Cyan("All done!\n")}`) - console.log(Bright("You can re-run this script at any time to update the configuration.")) - console.log(Bright("\nYou're now ready to start cobalt. Simply run ") + Cyan("npm start") + Bright('!\nHave fun :)')) + console.log(`\n\n${Cyan("All done!\n")}`); + console.log(Bright("You can re-run this script at any time to update the configuration.")); + console.log(Bright("\nYou're now ready to start cobalt. Simply run ") + Cyan("npm start") + Bright('!\nHave fun :)')); rl.close() } console.log( - `${Cyan("Welcome to cobalt!")}\n${Bright("Let's start by creating a new ")}${Cyan(".env")}${Bright(" file. You can always change it later.")}` + `${Cyan("Hey, this is cobalt.")}\n${Bright("Let's start by creating a new ")}${Cyan(".env")}${Bright(" file. You can always change it later.")}` ) + console.log( - Bright("\nWhat's the domain this instance will be running on? (localhost)\nExample: co.wukko.me") + `\n${Bright("⚠️ Please notice that since v.6.0 cobalt is hosted in two parts. API and web app are now separate.\nMerged hosting is deprecated and will be removed in the future.")}` ) +function setup() { + console.log(Bright("\nWhat kind of server will this instance be?\nOptions: api, web.")); -rl.question(q, r1 => { - ob['selfURL'] = `http://localhost:9000/` - ob['port'] = 9000 - if (r1) ob['selfURL'] = `https://${r1}/` + rl.question(q, r1 => { + switch (r1.toLowerCase()) { + case 'api': + console.log(Bright("\nCool! What's the domain this API instance will be running on? (localhost)\nExample: co.wuk.sh")); - console.log(Bright("\nGreat! Now, what's the port it'll be running on? (9000)")) + rl.question(q, apiURL => { + ob['apiURL'] = `http://localhost:9000/`; + ob['apiPort'] = 9000; + if (apiURL && apiURL !== "localhost") ob['apiURL'] = `https://${apiURL.toLowerCase()}/`; - rl.question(q, r2 => { - if (r2) ob['port'] = r2 - if (!r1 && r2) ob['selfURL'] = `http://localhost:${r2}/` + console.log(Bright("\nGreat! Now, what port will it be running on? (9000)")); - console.log(Bright("\nWould you like to enable CORS? It allows other websites and extensions to use your instance's API.\ny/n (n)")) + rl.question(q, apiPort => { + if (apiPort) ob['apiPort'] = apiPort; + if (apiPort && (apiURL === "localhost" || !apiURL)) ob['apiURL'] = `http://localhost:${apiPort}/`; - rl.question(q, r3 => { - if (r3.toLowerCase() !== 'y') ob['cors'] = '0' - final() - }) - }); -}) + console.log(Bright("\nWhat will your instance's name be? Usually it's something like eu-nl aka region-country. (local)")); + + rl.question(q, apiName => { + ob['apiName'] = apiName.toLowerCase(); + if (!apiName || apiName === "local") ob['apiName'] = "local"; + + console.log(Bright("\nOne last thing: would you like to enable CORS? It allows other websites and extensions to use your instance's API.\ny/n (n)")); + + rl.question(q, apiCors => { + if (apiCors.toLowerCase() !== 'y') ob['cors'] = '0' + final() + }) + }) + }); + + }) + break; + case 'web': + console.log(Bright("\nAwesome! What's the domain this web app instance will be running on? (localhost)\nExample: co.wukko.me")); + + rl.question(q, webURL => { + ob['webURL'] = `http://localhost:9001/`; + ob['webPort'] = 9001; + if (webURL && webURL !== "localhost") ob['webURL'] = `https://${webURL.toLowerCase()}/`; + + console.log( + Bright("\nGreat! Now, what port will it be running on? (9001)") + ) + rl.question(q, webPort => { + if (webPort) ob['webPort'] = webPort; + if (webPort && (webURL === "localhost" || !webURL)) ob['webURL'] = `http://localhost:${webPort}/`; + + console.log( + Bright("\nOne last thing: what default API domain should be used? (co.wuk.sh)\nIf it's hosted locally, make sure to include the port:") + Cyan(" localhost:9000") + ); + + rl.question(q, apiURL => { + ob['apiURL'] = `https://${apiURL.toLowerCase()}/`; + if (apiURL.includes(':')) ob['apiURL'] = `http://${apiURL.toLowerCase()}/`; + if (!apiURL) ob['apiURL'] = "https://co.wuk.sh/"; + final() + }) + }); + + }); + break; + default: + console.log(Bright("\nThis is not an option. Try again.")); + setup() + } + }) +} +setup() diff --git a/src/test/tests.json b/src/test/tests.json index 17ed2c41..cda2653d 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -446,6 +446,17 @@ "code": 200, "status": "stream" } + }, { + "name": "vr 360, av1, max", + "url": "https://www.youtube.com/watch?v=hEdzv7D4CbQ", + "params": { + "vCodec": "vp9", + "vQuality": "max" + }, + "expected": { + "code": 200, + "status": "stream" + } }, { "name": "inexistent video", "url": "https://youtube.com/watch?v=gnjuHYWGEW", @@ -717,6 +728,24 @@ "code": 200, "status": "redirect" } + }, { + "name": "tumblr audio", + "url": "https://rf9weu8hjf789234hf9.tumblr.com/post/172006661342/everyone-thats-made-a-video-out-of-this-without", + "params": {}, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "tumblr video converted to audio", + "url": "https://garfield-69.tumblr.com/post/696499862852780032", + "params": { + "isAudioOnly": true + }, + "expected": { + "code": 200, + "status": "stream" + } }], "vimeo": [{ "name": "4k progressive", From 66c797e2520baa7e096ee7e8d2efdb472a6fb578 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 5 Jun 2023 12:47:03 +0600 Subject: [PATCH 8/8] oops --- src/core/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/api.js b/src/core/api.js index 0c9bfdbb..6826efd2 100644 --- a/src/core/api.js +++ b/src/core/api.js @@ -15,7 +15,7 @@ import { sha256 } from "../modules/sub/crypto.js"; import { celebrationsEmoji } from "../modules/pageRender/elements.js"; import { verifyStream } from "../modules/stream/manage.js"; -export async function runAPI(express, app, gitCommit, gitBranch, __dirname) { +export function runAPI(express, app, gitCommit, gitBranch, __dirname) { const corsConfig = process.env.cors === '0' ? { origin: process.env.webURL, optionsSuccessStatus: 200 } : {}; const apiLimiter = rateLimit({