From 741dfd40f5d6a076c3598ffcdd52e7071faac3d5 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Fri, 4 Oct 2024 14:58:56 +0000 Subject: [PATCH] api/security: implement api keys as method of authentication --- .gitignore | 1 + api/package.json | 2 +- api/src/config.js | 5 + api/src/misc/console-text.js | 7 ++ api/src/security/api-keys.js | 201 +++++++++++++++++++++++++++++++++++ docs/run-an-instance.md | 48 +++++++++ pnpm-lock.yaml | 13 ++- 7 files changed, 273 insertions(+), 4 deletions(-) create mode 100644 api/src/security/api-keys.js diff --git a/.gitignore b/.gitignore index b408b65c..ae56830a 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ build .env.* !.env.example cookies.json +keys.json # docker docker-compose.yml diff --git a/api/package.json b/api/package.json index e37c839d..89fab0be 100644 --- a/api/package.json +++ b/api/package.json @@ -33,7 +33,7 @@ "express-rate-limit": "^6.3.0", "ffmpeg-static": "^5.1.0", "hls-parser": "^0.10.7", - "ipaddr.js": "2.1.0", + "ipaddr.js": "2.2.0", "nanoid": "^4.0.2", "node-cache": "^5.1.2", "psl": "1.9.0", diff --git a/api/src/config.js b/api/src/config.js index 1f00231e..3a28d7ce 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -43,6 +43,11 @@ const env = { && process.env.TURNSTILE_SECRET && process.env.JWT_SECRET, + apiKeyURL: process.env.API_KEY_URL && new URL(process.env.API_KEY_URL), + authRequired: process.env.API_AUTH_REQUIRED === '1', + + keyReloadInterval: 900, + enabledServices, } diff --git a/api/src/misc/console-text.js b/api/src/misc/console-text.js index 014584ae..6ce747d7 100644 --- a/api/src/misc/console-text.js +++ b/api/src/misc/console-text.js @@ -5,12 +5,19 @@ function t(color, tt) { export function Bright(tt) { return t("\x1b[1m", tt) } + export function Red(tt) { return t("\x1b[31m", tt) } + export function Green(tt) { return t("\x1b[32m", tt) } + export function Cyan(tt) { return t("\x1b[36m", tt) } + +export function Yellow(tt) { + return t("\x1b[93m", tt) +} diff --git a/api/src/security/api-keys.js b/api/src/security/api-keys.js new file mode 100644 index 00000000..6ab11e78 --- /dev/null +++ b/api/src/security/api-keys.js @@ -0,0 +1,201 @@ +import { env } from "../config.js"; +import { readFile } from "node:fs/promises"; +import { Yellow } from "../misc/console-text.js"; +import ip from "ipaddr.js"; + +// this function is a modified variation of code +// from https://stackoverflow.com/a/32402438/14855621 +const generateWildcardRegex = rule => { + var escapeRegex = (str) => str.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1"); + return new RegExp("^" + rule.split("*").map(escapeRegex).join(".*") + "$"); +} + +const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +let keys = {}; + +const ALLOWED_KEYS = new Set(['name', 'ips', 'userAgents', 'limit']); + +/* Expected format pseudotype: +** type KeyFileContents = Record< +** UUIDv4String, +** { +** name?: string, +** limit?: number | "unlimited", +** ips?: CIDRString[], +** userAgents?: string[] +** } +** >; +*/ + +const validateKeys = (input) => { + if (typeof input !== 'object' || input === null) { + throw "input is not an object"; + } + + if (Object.keys(input).some(x => !UUID_REGEX.test(x))) { + throw "key file contains invalid key(s)"; + } + + Object.values(input).forEach(details => { + if (typeof details !== 'object' || details === null) { + throw "some key(s) are incorrectly configured"; + } + + const unexpected_key = Object.keys(details).find(k => !ALLOWED_KEYS.has(k)); + if (unexpected_key) { + throw "detail object contains unexpected key: " + unexpected_key; + } + + if (details.limit && details.limit !== 'unlimited') { + if (typeof details.limit !== 'number') + throw "detail object contains invalid limit (not a number)"; + else if (details.limit < 1) + throw "detail object contains invalid limit (not a number)"; + } + + if (details.ips) { + if (!Array.isArray(details.ips)) + throw "details object contains value for `ips` which is not an array"; + + const invalid_ip = details.ips.find( + addr => typeof addr !== 'string' || (!ip.isValidCIDR(addr) && !ip.isValid(addr)) + ); + + if (invalid_ip) { + throw "`ips` in details contains an invalid IP or CIDR range: " + invalid_ip; + } + } + + if (details.userAgents) { + if (!Array.isArray(details.userAgents)) + throw "details object contains value for `userAgents` which is not an array"; + + const invalid_ua = details.userAgents.find(ua => typeof ua !== 'string'); + if (invalid_ua) { + throw "`userAgents` in details contains an invalid user agent: " + invalid_ua; + } + } + }); +} + +const formatKeys = (keyData) => { + const formatted = {}; + + for (let key in keyData) { + const data = keyData[key]; + key = key.toLowerCase(); + + formatted[key] = {}; + + if (data.limit) { + formatted[key].limit = data.limit; + } + + if (data.ips) { + formatted[key].ips = data.ips.map(addr => { + if (ip.isValid(addr)) { + return [ ip.parse(addr), 32 ]; + } + + return ip.parseCIDR(addr); + }); + } + + if (data.userAgents) { + formatted[key].userAgents = data.userAgents.map(generateWildcardRegex); + } + } + + return formatted; +} + +const loadKeys = async (source) => { + let updated; + if (source.protocol === 'file:') { + const pathname = source.pathname === '/' ? '' : source.pathname; + updated = JSON.parse( + await readFile( + decodeURIComponent(source.host + pathname), + 'utf8' + ) + ); + } else { + updated = await fetch(source).then(a => a.json()); + } + + validateKeys(updated); + keys = formatKeys(updated); +} + +const wrapLoad = (url) => { + loadKeys(url) + .then(() => {}) + .catch((e) => { + console.error(`${Yellow('[!]')} Failed loading API keys at ${new Date().toISOString()}.`); + console.error('Error:', e); + }) +} + +const err = (reason) => ({ success: false, error: reason }); + +export const validateAuthorization = (req) => { + const authHeader = req.get('Authorization'); + + if (typeof authHeader !== 'string') { + return err("missing"); + } + + const [ authType, keyString ] = authHeader.split(' ', 2); + if (authType.toLowerCase() !== 'api-key') { + return err("not_api_key"); + } + + if (!UUID_REGEX.test(keyString) || `${authType} ${keyString}` !== authHeader) { + return err("invalid"); + } + + const matchingKey = keys[keyString.toLowerCase()]; + if (!matchingKey) { + return err("not_found"); + } + + if (matchingKey.ips) { + let addr; + try { + addr = ip.parse(req.ip); + } catch { + return err("invalid_ip"); + } + + const ip_allowed = matchingKey.ips.some( + ([ allowed, size ]) => { + return addr.kind() === allowed.kind() + && addr.match(allowed, size); + } + ); + + if (!ip_allowed) { + return err("ip_not_allowed"); + } + } + + if (matchingKey.userAgents) { + const userAgent = req.get('User-Agent'); + if (!matchingKey.userAgents.some(regex => regex.test(userAgent))) { + return err("ua_not_allowed"); + } + } + + req.rateLimitKey = keyString.toLowerCase(); + req.rateLimitMax = matchingKey.limit; + + return { success: true }; +} + +export const setup = (url) => { + wrapLoad(url); + if (env.keyReloadInterval > 0) { + setInterval(() => wrapLoad(url), env.keyReloadInterval * 1000); + } +} diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md index 4e41c73b..32a928a6 100644 --- a/docs/run-an-instance.md +++ b/docs/run-an-instance.md @@ -72,6 +72,8 @@ sudo service nscd start | `RATELIMIT_MAX` | `20` | `30` | max requests per time window. requests above this amount will be blocked for the rate limit window duration. | | `DURATION_LIMIT` | `10800` | `18000` | max allowed video duration in **seconds**. | | `TUNNEL_LIFESPAN` | `90` | `120` | the duration for which tunnel info is stored in ram, **in seconds**. | +| `API_KEY_URL` | ➖ | `file://keys.json` | the location of the api key database. for loading API keys, cobalt supports HTTP(S) urls, or local files by specifying a local path using the `file://` protocol. see the "api key file format" below for more details. | +| `API_AUTH_REQUIRED` | ➖ | `1` | when set to `1`, the user always needs to be authenticated in some way before they can access the API (either via an api key or via turnstile, if enabled). | \* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)). @@ -80,3 +82,49 @@ setting a `FREEBIND_CIDR` allows cobalt to pick a random IP for every download a requests it makes for that particular download. to use freebind in cobalt, you need to follow its [setup instructions](https://github.com/imputnet/freebind.js?tab=readme-ov-file#setup) first. if you configure this option while running cobalt in a docker container, you also need to set the `API_LISTEN_ADDRESS` env to `127.0.0.1`, and set `network_mode` for the container to `host`. + +#### api key file format +the file is a JSON-serialized object with the following structure: +```typescript + +type KeyFileContents = Record< + UUIDv4String, + { + name?: string, + limit?: number | "unlimited", + ips?: (CIDRString | IPString)[], + userAgents?: string[] + } +>; +``` + +where *`UUIDv4String`* is a stringified version of a UUIDv4 identifier. +- **name** is a field for your own reference, it is not used by cobalt anywhere. + +- **`limit`** specifies how many requests the API key can make during the window specified in the `RATELIMIT_WINDOW` env. + - when omitted, the limit specified in `RATELIMIT_MAX` will be used. + +- **`ips`** contains an array of allowlisted IP ranges, which can be specified both as individual ips or CIDR ranges (e.g. *`["192.168.42.69", "2001:db8::48", "10.0.0.0/8", "fe80::/10"]`*). + - when specified, only requests from these ip ranges can use the specified api key. + - when omitted, any IP can be used to make requests with that API key. + +- **`userAgents`** contains an array of allowed user agents, with support for wildcards (e.g. *`["cobaltbot/1.0", "Mozilla/5.0 * Chrome/*"]`*). + - when specified, requests with a `user-agent` that does not appear in this array will be rejected. + - when omitted, any user agent can be specified to make requests with that API key. + +- if both `ips` and `userAgents` are set, the tokens will be limited by both parameters. +- if cobalt detects any problem with your key file, it will be ignored and a warning will be printed to the console. + +an example key file could look like this: +```json +{ + "b5c7160a-b655-4c7a-b500-de839f094550": { + "limit": 10, + "ips": ["10.0.0.0/8", "192.168.42.42"], + "userAgents": ["*Chrome*"] + } +} +``` + +if you are configuring a key file, **do not use the UUID from the example** but instead generate your own. you can do this by running the following command if you have node.js installed: +`node -e "console.log(crypto.randomUUID())"` diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 220a2cf3..7ce32e54 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -38,8 +38,8 @@ importers: specifier: ^0.10.7 version: 0.10.9 ipaddr.js: - specifier: 2.1.0 - version: 2.1.0 + specifier: 2.2.0 + version: 2.2.0 nanoid: specifier: ^4.0.2 version: 4.0.2 @@ -1448,6 +1448,10 @@ packages: resolution: {integrity: sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==} engines: {node: '>= 10'} + ipaddr.js@2.2.0: + resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==} + engines: {node: '>= 10'} + is-binary-path@2.1.0: resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} engines: {node: '>=8'} @@ -3471,7 +3475,10 @@ snapshots: ipaddr.js@1.9.1: {} - ipaddr.js@2.1.0: {} + ipaddr.js@2.1.0: + optional: true + + ipaddr.js@2.2.0: {} is-binary-path@2.1.0: dependencies: