6.2: no more ip verification

- removed ip verification and updated privacy policy to reflect this change.
- streamable links now last for 20 seconds instead of 2 minutes.
- cleaned up stream verification algorithm. now the same function isn't run 4 times in a row.
- removed deprecated way of hosting a cobalt instance.
This commit is contained in:
wukko 2023-06-27 19:56:15 +06:00
parent 0e1c885266
commit 65161107fa
16 changed files with 42 additions and 267 deletions

View file

@ -64,7 +64,7 @@ Setup script installs all needed `npm` dependencies, but you have to install `No
3. Run cobalt via `npm start`
4. Done.
You need to host API and web app separately ever since v.6.0. Setup script will help you with that!
You need to host API and web app separately since v.6.0. Setup script will help you with that!
### Ubuntu 22.04+ workaround
`nscd` needs to be installed and running so that the `ffmpeg-static` binary can resolve DNS ([#101](https://github.com/wukko/cobalt/issues/101#issuecomment-1494822258)):

View file

@ -21,12 +21,3 @@ services:
- webPort=9000
- webURL=https://co.wukko.me/
- apiURL=https://co.wuk.sh/
cobalt-both:
build: .
restart: unless-stopped
container_name: cobalt-both
ports:
- 9000:9000/tcp
environment:
- port=9000
- selfURL=https://co.wukko.me/

View file

@ -1,7 +1,7 @@
{
"name": "cobalt",
"description": "save what you love",
"version": "6.1",
"version": "6.2",
"author": "wukko",
"exports": "./src/cobalt.js",
"type": "module",

View file

@ -11,7 +11,6 @@ import { fileURLToPath } from 'url';
import { runWeb } from "./core/web.js";
import { runAPI } from "./core/api.js";
import { runBoth } from "./core/both.js";
const app = express();
@ -19,19 +18,16 @@ const gitCommit = shortCommit();
const gitBranch = getCurrentBranch();
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename).slice(0, -4); // go up another level (get rid of src/)
const __dirname = path.dirname(__filename).slice(0, -4);
app.disable('x-powered-by');
await loadLoc(); // preload localization
await loadLoc();
// i don't like this at all
if (process.env.apiURL && process.env.apiPort && !((process.env.webURL && process.env.webPort) || (process.env.selfURL && process.env.port))) {
await runAPI(express, app, gitCommit, gitBranch, __dirname);
runAPI(express, app, gitCommit, gitBranch, __dirname);
} else if (process.env.webURL && process.env.webPort && !((process.env.apiURL && process.env.apiPort) || (process.env.selfURL && process.env.port))) {
await runWeb(express, app, gitCommit, gitBranch, __dirname);
} else if (process.env.selfURL && process.env.port && !((process.env.apiURL && process.env.apiPort) || (process.env.webURL && process.env.webPort))) {
await runBoth(express, app, gitCommit, gitBranch, __dirname)
} else {
console.log(Red(`cobalt hasn't been configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`));
console.log(Red(`cobalt wasn't configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`));
}

View file

@ -1,7 +1,7 @@
{
"streamLifespan": 120000,
"streamLifespan": 20000,
"maxVideoDuration": 10800000,
"genericUserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36",
"genericUserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
"authorInfo": {
"name": "wukko",
"link": "https://wukko.me/",

View file

@ -20,7 +20,7 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
const apiLimiter = rateLimit({
windowMs: 60000,
max: 25,
max: 20,
standardHeaders: false,
legacyHeaders: false,
keyGenerator: (req, res) => sha256(getIP(req), ipSalt),
@ -31,7 +31,7 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
});
const apiLimiterStream = rateLimit({
windowMs: 60000,
max: 28,
max: 25,
standardHeaders: false,
legacyHeaders: false,
keyGenerator: (req, res) => sha256(getIP(req), ipSalt),
@ -75,7 +75,6 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
app.post('/api/json', async (req, res) => {
try {
let ip = sha256(getIP(req), ipSalt);
let lang = languageCode(req);
let j = apiJSON(0, { t: "Bad request" });
try {
@ -83,7 +82,6 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
if (request.url) {
request.dubLang = request.dubLang ? lang : false;
let chck = checkJSONPost(request);
if (chck) chck["ip"] = ip;
j = chck ? await getJSON(chck["url"], lang, chck) : apiJSON(0, { t: loc(lang, 'ErrorCouldntFetch') });
} else {
j = apiJSON(0, { t: loc(lang, 'ErrorNoLink') });
@ -101,22 +99,22 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
app.get('/api/:type', (req, res) => {
try {
let ip = sha256(getIP(req), ipSalt);
switch (req.params.type) {
case 'stream':
let streamInfo = verifyStream(ip, req.query.t, req.query.h, req.query.e);
if (req.query.t && req.query.h && req.query.e && req.query.t.toString().length === 21
&& req.query.h.toString().length === 64 && req.query.e.toString().length === 13) {
let streamInfo = verifyStream(req.query.t, req.query.h, req.query.e);
if (streamInfo.error) {
res.status(streamInfo.status).json(apiJSON(0, { t: streamInfo.error }).body);
return;
}
if (req.query.p) {
res.status(200).json({ "status": "continue" });
return;
} else if (req.query.t && req.query.h && req.query.e) {
stream(res, ip, req.query.t, req.query.h, req.query.e);
}
stream(res, streamInfo);
} else {
let j = apiJSON(0, { t: "no stream id" })
let j = apiJSON(0, { t: "stream token, hmac, or expiry timestamp is missing." })
res.status(j.status).json(j.body);
return;
}

View file

@ -1,197 +0,0 @@
import cors from "cors";
import rateLimit from "express-rate-limit";
import { randomBytes } from "crypto";
const ipSalt = randomBytes(64).toString('hex');
import { appName, genericUserAgent, version } from "../modules/config.js";
import { getJSON } from "../modules/api.js";
import { apiJSON, checkJSONPost, getIP, languageCode } from "../modules/sub/utils.js";
import { Bright, Cyan, Green, Red } from "../modules/sub/consoleText.js";
import stream from "../modules/stream/stream.js";
import loc from "../localization/manager.js";
import { buildFront } from "../modules/build.js";
import { changelogHistory } from "../modules/pageRender/onDemand.js";
import { sha256 } from "../modules/sub/crypto.js";
import findRendered from "../modules/pageRender/findRendered.js";
import { celebrationsEmoji } from "../modules/pageRender/elements.js";
export async function runBoth(express, app, gitCommit, gitBranch, __dirname) {
const corsConfig = process.env.cors === '0' ? { origin: process.env.selfURL, optionsSuccessStatus: 200 } : {};
const apiLimiter = rateLimit({
windowMs: 60000,
max: 25,
standardHeaders: false,
legacyHeaders: false,
keyGenerator: (req, res) => sha256(getIP(req), ipSalt),
handler: (req, res, next, opt) => {
res.status(429).json({ "status": "error", "text": loc(languageCode(req), 'ErrorRateLimit') });
return;
}
});
const apiLimiterStream = rateLimit({
windowMs: 60000,
max: 28,
standardHeaders: false,
legacyHeaders: false,
keyGenerator: (req, res) => sha256(getIP(req), ipSalt),
handler: (req, res, next, opt) => {
res.status(429).json({ "status": "error", "text": loc(languageCode(req), 'ErrorRateLimit') });
return;
}
});
const startTime = new Date();
const startTimestamp = Math.floor(startTime.getTime());
// preload localization files and build static pages
await buildFront(gitCommit, gitBranch);
app.use('/api/:type', cors(corsConfig));
app.use('/api/json', apiLimiter);
app.use('/api/stream', apiLimiterStream);
app.use('/api/onDemand', apiLimiter);
app.use('/', express.static('./build/min'));
app.use('/', express.static('./src/front'));
app.use((req, res, next) => {
try { decodeURIComponent(req.path) } catch (e) { return res.redirect('/') }
next();
});
app.use((req, res, next) => {
if (req.header("user-agent") && req.header("user-agent").includes("Trident")) res.destroy();
next();
});
app.use('/api/json', express.json({
verify: (req, res, buf) => {
try {
JSON.parse(buf);
if (buf.length > 720) throw new Error();
if (String(req.header('Content-Type')) !== "application/json") {
res.status(400).json({ 'status': 'error', 'text': 'invalid content type header' });
return;
}
if (String(req.header('Accept')) !== "application/json") {
res.status(400).json({ 'status': 'error', 'text': 'invalid accept header' });
return;
}
} catch(e) {
res.status(400).json({ 'status': 'error', 'text': 'invalid json body.' });
return;
}
}
}));
app.post('/api/json', async (req, res) => {
try {
let ip = sha256(getIP(req), ipSalt);
let lang = languageCode(req);
let j = apiJSON(0, { t: "Bad request" });
try {
let request = req.body;
if (request.url) {
request.dubLang = request.dubLang ? lang : false;
let chck = checkJSONPost(request);
if (chck) chck["ip"] = ip;
j = chck ? await getJSON(chck["url"], lang, chck) : apiJSON(0, { t: loc(lang, 'ErrorCouldntFetch') });
} else {
j = apiJSON(0, { t: loc(lang, 'ErrorNoLink') });
}
} catch (e) {
j = apiJSON(0, { t: loc(lang, 'ErrorCantProcess') });
}
res.status(j.status).json(j.body);
return;
} catch (e) {
res.destroy();
return
}
});
app.get('/api/:type', (req, res) => {
try {
let ip = sha256(getIP(req), ipSalt);
switch (req.params.type) {
case 'stream':
if (req.query.p) {
res.status(200).json({ "status": "continue" });
return;
} else if (req.query.t && req.query.h && req.query.e) {
stream(res, ip, req.query.t, req.query.h, req.query.e);
} else {
let j = apiJSON(0, { t: "no stream id" })
res.status(j.status).json(j.body);
return;
}
break;
case 'onDemand':
if (req.query.blockId) {
let blockId = req.query.blockId.slice(0, 3);
let r, j;
switch(blockId) {
case "0": // changelog history
r = changelogHistory();
j = r ? apiJSON(3, { t: r }) : apiJSON(0, { t: "couldn't render this block" })
break;
case "1": // celebrations emoji
r = celebrationsEmoji();
j = r ? apiJSON(3, { t: r }) : false
break;
default:
j = apiJSON(0, { t: "couldn't find a block with this id" })
break;
}
if (j.body) {
res.status(j.status).json(j.body)
} else {
res.status(204).end()
}
} else {
let j = apiJSON(0, { t: "no block id" });
res.status(j.status).json(j.body)
}
break;
case 'serverInfo':
res.status(200).json({
version: version,
commit: gitCommit,
branch: gitBranch,
name: process.env.apiName ? process.env.apiName : "unknown",
url: process.env.apiURL,
cors: process.env.cors && process.env.cors === "0" ? 0 : 1,
startTime: `${startTimestamp}`
});
break;
default:
let j = apiJSON(0, { t: "unknown response type" })
res.status(j.status).json(j.body);
break;
}
} catch (e) {
res.status(500).json({ 'status': 'error', 'text': loc(languageCode(req), 'ErrorCantProcess') });
return;
}
});
app.get("/api/status", (req, res) => {
res.status(200).end()
});
app.get("/api", (req, res) => {
res.redirect('/api/json')
});
app.get("/", (req, res) => {
res.sendFile(`${__dirname}/${findRendered(languageCode(req), req.header('user-agent') ? req.header('user-agent') : genericUserAgent)}`);
});
app.get("/favicon.ico", (req, res) => {
res.redirect('/icons/favicon.ico');
});
app.get("/*", (req, res) => {
res.redirect('/')
});
app.listen(process.env.port, () => {
console.log(`${Red("⚠️ This way of running cobalt has been deprecated and will be removed soon.\nCheck the docs and get ready: ")}${Green("WIP")}`)
console.log(`\n${Cyan(appName)} ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\nStart time: ${Bright(`${startTime.toUTCString()} (${Math.floor(startTimestamp)})`)}\n\nURL: ${Cyan(`${process.env.selfURL}`)}\nPort: ${process.env.port}\n`)
})
}

View file

@ -107,7 +107,7 @@
"FollowSupport": "keep in touch with {appName} for support, polls, news, and more:",
"SupportNote": "please note that questions and issues may take a while to respond to, there's only one person managing everything.",
"SourceCode": "report issues, explore source code, star or fork the repo:",
"PrivacyPolicy": "{appName}'s privacy policy is simple: no data about you is ever collected or stored. zero, zilch, nada, nothing.\nwhat you download is your business, not mine.\n\nsome non-backtraceable data does get temporarily stored when requested download requires live render. it's necessary for that feature to function.\n\nin that case, <span class=\"text-backdrop\">salted sha256 hash of your ip address</span> and information about requested stream are temporarily stored in server's RAM for <span class=\"text-backdrop\">2 minutes</span>. after 2 minutes all previously stored information is permanently removed. hash of your ip address is <span class=\"text-backdrop\">used for limiting stream access only to you</span>.\nno one (even me) has access to this data, because official {appName} codebase doesn't provide a way to read it outside of processing functions in the first place.\n\nyou can check {appName}'s <a class=\"text-backdrop italic\" href=\"{repo}\" target=\"_blank\">github repo</a> yourself and see that everything is as stated.",
"PrivacyPolicy": "{appName}'s privacy policy is simple: no data about you is ever collected or stored. zero, zilch, nada, nothing.\nwhat you download is your business, not mine.\n\nsome non-backtraceable data does get temporarily stored when requested download requires live render. it's necessary for that feature to function.\n\nin that case, information about requested stream is temporarily stored in server's RAM for <span class=\"text-backdrop\">20 seconds</span>. as 20 seconds have passed, all previously stored information is permanently removed.\nno one (even me) has access to this data, because official {appName} codebase doesn't provide a way to read it outside of processing functions.\n\nyou can check {appName}'s <a class=\"text-backdrop italic\" href=\"{repo}\" target=\"_blank\">github repo</a> yourself and see that everything is as stated.",
"ErrorYTUnavailable": "this youtube video is unavailable or age restricted. i am currently unable to download videos with sensitive content. try another one!",
"ErrorYTTryOtherCodec": "i couldn't find anything to download with your settings. try another codec or quality!\n\nnote: youtube api sometimes acts unexpectedly. blame google for this, not me.",
"SettingsCodecSubtitle": "youtube codec",
@ -120,7 +120,6 @@
"SettingsVimeoPreferDescription": "progressive: direct file link to vimeo's cdn. max quality is 1080p.\ndash: video and audio are merged by {appName} into one file. max quality is 4k.\n\npick \"progressive\" if you want best editor/player/social media compatibility. if progressive download isn't available, dash is used instead.",
"ShareURL": "share",
"ErrorTweetUnavailable": "couldn't find anything about this tweet. this could be because its visibility is limited. try another one!",
"UrgentUpdate6": "all network issues have been fixed!",
"ErrorReload": "i couldn't verify whether you have access to this stream. try again or refresh the page!"
"UrgentUpdate6": "all network issues have been fixed!"
}
}

View file

@ -107,7 +107,7 @@
"FollowSupport": "оставайтесь на связи с {appName} для новостей, поддержки, участия в опросах, и многого другого:",
"SupportNote": "так как я один занимаюсь разработкой и поддержкой в одиночку, время ожидания ответа может достигать нескольких часов. но я отвечаю всем, так что не стесняйся.",
"SourceCode": "пиши о проблемах, шарься в исходнике, или же форкай репозиторий:",
"PrivacyPolicy": "политика конфиденциальности {appName} довольно проста: ничего не хранится об истории твоих действий или загрузок. совсем. даже ошибки.\nто, что ты скачиваешь - только твоё личное дело.\n\nв случаях, когда твоей загрузке требуется лайв-рендер, временно хранится неотслеживаемая информация. это необходимо для работы такого типа загрузок.\n\nв этом случае, <span class=\"text-backdrop\">sha256 хэш (с солью) твоего ip адреса</span> и данные о запрошенном стриме хранятся в ОЗУ сервера в течение <span class=\"text-backdrop\">двух минут</span>. по истечении этого периода всё стирается. хэш твоего ip адреса используется для предоставления доступа к стриму только тебе. ни у кого (даже у меня) нет доступа к временно хранящимся данным, так как оригинальный код {appName} не предоставляет такой возможности.\n\nты всегда можешь посмотреть <a class=\"text-backdrop italic\" href=\"{repo}\" target=\"_blank\">исходный код {appName}</a> и убедиться, что всё так, как описано.",
"PrivacyPolicy": "политика конфиденциальности {appName} довольно проста: ничего не хранится об истории твоих действий или загрузок. совсем. даже ошибки.\nто, что ты скачиваешь - только твоё личное дело.\n\nв случаях, когда твоей загрузке требуется лайв-рендер, временно хранится неотслеживаемая информация. это необходимо для работы такого типа загрузок.\n\nв этом случае данные о запрошенном стриме хранятся в ОЗУ сервера в течение <span class=\"text-backdrop\">20 секунд</span>. по истечении этого периода всё стирается. ни у кого (даже у меня) нет доступа к временно хранящимся данным, так как официальный код {appName} не предоставляет такой возможности.\n\nты всегда можешь посмотреть <a class=\"text-backdrop italic\" href=\"{repo}\" target=\"_blank\">исходный код {appName}</a> и убедиться, что всё так, как описано.",
"ErrorYTUnavailable": "это видео недоступно или же ограничено по возрасту на youtube. пока что я не умею скачивать подобные видео. попробуй другое!",
"ErrorYTTryOtherCodec": "я не нашёл того, что мог бы скачать с твоими настройками. попробуй другой кодек или качество!",
"SettingsCodecSubtitle": "кодек для видео с youtube",
@ -120,7 +120,6 @@
"SettingsVimeoPreferDescription": "progressive: прямая ссылка на файл с сервера vimeo. максимальное качество: 1080p.\ndash: {appName} совмещает видео и аудио в один файл. максимальное качество: 4k.\n\nвыбирай \"progressive\", если тебе нужна наилучшая совместимость с плеерами/редакторами/соцсетями. если \"progressive\" файл недоступен, {appName} скачает \"dash\".",
"ShareURL": "поделиться",
"ErrorTweetUnavailable": "не смог найти что-либо об этом твите. возможно его видимость была ограничена. попробуй другой!",
"UrgentUpdate6": "теперь всё работает!",
"ErrorReload": "я не смог удостовериться, что у тебя есть доступ к этому стриму. попробуй ещё раз или перезагрузи страницу!"
"UrgentUpdate6": "на этот раз точно: всё работает!"
}
}

View file

@ -364,6 +364,7 @@ export default function(obj) {
body: `<div id="desc-error" class="desc-padding subtext"></div>`
})}
<div id="popup-backdrop" style="visibility: hidden;" onclick="hideAllPopups()"></div>
<div id="urgent-notice" class="urgent-notice explanation center" onclick="popup('about', 1, 'changelog')" style="visibility: hidden;">${emoji("🎉", 18)} ${t("UrgentUpdate6")}</div>
<div id="cobalt-main-box" class="center" style="visibility: hidden;">
<div id="logo">${appName}</div>
<div id="download-area">

View file

@ -123,7 +123,7 @@ export default async function (host, patternMatch, url, lang, obj) {
if (r.error) return apiJSON(0, { t: Array.isArray(r.error) ? loc(lang, r.error[0], r.error[1]) : loc(lang, r.error) });
return matchActionDecider(r, host, obj.ip, obj.aFormat, isAudioOnly, lang, isAudioMuted);
return matchActionDecider(r, host, obj.aFormat, isAudioOnly, lang, isAudioMuted);
} catch (e) {
return apiJSON(0, { t: genericError(lang, host) })
}

View file

@ -2,13 +2,12 @@ import { audioIgnore, services, supportedAudio } from "../config.js";
import { apiJSON } from "../sub/utils.js";
import loc from "../../localization/manager.js";
export default function(r, host, ip, audioFormat, isAudioOnly, lang, isAudioMuted) {
export default function(r, host, audioFormat, isAudioOnly, lang, isAudioMuted) {
let action,
responseType = 2,
defaultParams = {
u: r.urls,
service: host,
ip: ip,
filename: r.filename,
},
params = {}

View file

@ -28,7 +28,7 @@ console.log(
)
console.log(
`\n${Bright("⚠️ Please notice that since v.6.0 cobalt is hosted in two parts. API and web app are now separate.\nMerged hosting is deprecated and will be removed in the future.")}`
`\n${Bright("⚠️ Please notice that since v.6.0 cobalt is hosted in two parts. API and web app are now separate.\nMerged hosting is no longer available.")}`
)
function setup() {
console.log(Bright("\nWhat kind of server will this instance be?\nOptions: api, web."));

View file

@ -15,7 +15,7 @@ streamCache.on("expired", (key) => {
export function createStream(obj) {
let streamID = nanoid(),
exp = Math.floor(new Date().getTime()) + streamLifespan,
ghmac = sha256(`${streamID},${obj.ip},${obj.service},${exp}`, streamSalt);
ghmac = sha256(`${streamID},${obj.service},${exp}`, streamSalt);
if (!streamCache.has(streamID)) {
streamCache.set(streamID, {
@ -25,7 +25,6 @@ export function createStream(obj) {
urls: obj.u,
filename: obj.filename,
hmac: ghmac,
ip: obj.ip,
exp: exp,
isAudioOnly: !!obj.isAudioOnly,
audioFormat: obj.audioFormat,
@ -42,19 +41,17 @@ export function createStream(obj) {
return `${process.env.apiURL || process.env.selfURL}api/stream?t=${streamID}&e=${exp}&h=${ghmac}`;
}
export function verifyStream(ip, id, hmac, exp) {
export function verifyStream(id, hmac, exp) {
try {
if (id.toString().length === 21) {
let streamInfo = streamCache.get(id.toString());
if (!streamInfo) return { error: "requested stream does not exist", status: 400 };
if (!streamInfo) return { error: "this download link has expired or doesn't exist. go back and try again!", status: 400 };
let ghmac = sha256(`${id},${ip},${streamInfo.service},${exp}`, streamSalt);
let ghmac = sha256(`${id},${streamInfo.service},${exp}`, streamSalt);
if (String(hmac) === ghmac && String(exp) === String(streamInfo.exp) && ghmac === String(streamInfo.hmac)
&& String(ip) === streamInfo.ip && Number(exp) > Math.floor(new Date().getTime())) {
&& Number(exp) > Math.floor(new Date().getTime())) {
return streamInfo;
}
}
return { error: "i couldn't verify whether you have access to this download. try again or refresh the page!", status: 401 };
return { error: "i couldn't verify if you have access to this download. go back and try again!", status: 401 };
} catch (e) {
return { status: 500, body: { status: "error", text: "Internal Server Error" } };
}

View file

@ -1,14 +1,7 @@
import { apiJSON } from "../sub/utils.js";
import { verifyStream } from "./manage.js";
import { streamAudioOnly, streamDefault, streamLiveRender, streamVideoOnly } from "./types.js";
export default function(res, ip, id, hmac, exp) {
export default async function(res, streamInfo) {
try {
let streamInfo = verifyStream(ip, id, hmac, exp);
if (streamInfo.error) {
res.status(streamInfo.status).json(apiJSON(0, { t: streamInfo.error }).body);
return;
}
if (streamInfo.isAudioOnly && streamInfo.type !== "bridge") {
streamAudioOnly(streamInfo, res);
return;

View file

@ -73,7 +73,6 @@ export function cleanURL(url, host) {
case "tiktok":
url = url.replace(/@([a-zA-Z]+(\.[a-zA-Z]+)+)/, "@a")
case "pinterest":
// Redirect all TLDs back to .com
url = url.replace(/:\/\/(?:www.)pinterest(?:\.[a-z.]+)/, "://pinterest.com")
default:
url = url.split('?')[0];