From 80a01494c7eb0c0369f93d6617cd668d8e97851d Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 12 Sep 2024 14:30:21 +0600 Subject: [PATCH 01/13] api/match-action: add missing twitch case to redirect group closes #741 --- api/src/processing/match-action.js | 1 + 1 file changed, 1 insertion(+) diff --git a/api/src/processing/match-action.js b/api/src/processing/match-action.js index 2e7b7f1b..6b5395d4 100644 --- a/api/src/processing/match-action.js +++ b/api/src/processing/match-action.js @@ -148,6 +148,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab case "streamable": case "snapchat": case "loom": + case "twitch": responseType = "redirect"; break; } From e768e7f6faeea1c3638b3af0b83dc01b076432f4 Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 12 Sep 2024 14:35:42 +0600 Subject: [PATCH 02/13] api/create-filename: don't assign any of potentially blank tags --- api/src/processing/create-filename.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/api/src/processing/create-filename.js b/api/src/processing/create-filename.js index 4f248377..216b15a4 100644 --- a/api/src/processing/create-filename.js +++ b/api/src/processing/create-filename.js @@ -2,11 +2,19 @@ export default (f, style, isAudioOnly, isAudioMuted) => { let filename = ''; let infoBase = [f.service, f.id]; - let classicTags = [...infoBase, f.resolution]; - let basicTags = [f.qualityLabel]; + let classicTags = [...infoBase]; + let basicTags = []; const title = `${f.title} - ${f.author}`; + if (f.resolution) { + classicTags.push(f.resolution); + } + + if (f.qualityLabel) { + basicTags.push(f.qualityLabel); + } + if (f.youtubeFormat) { classicTags.push(f.youtubeFormat); basicTags.push(f.youtubeFormat); From b90a58f4f0d61add36a93c7820e4b1c2e5f1878a Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 12 Sep 2024 14:38:36 +0600 Subject: [PATCH 03/13] api/tests/twitch: fix expected twitch status --- api/src/util/tests.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/util/tests.json b/api/src/util/tests.json index 86658d88..bc7adae2 100644 --- a/api/src/util/tests.json +++ b/api/src/util/tests.json @@ -1113,7 +1113,7 @@ "params": {}, "expected": { "code": 200, - "status": "tunnel" + "status": "redirect" } }, { From acb4e49e18a4ed9b2b49284b77d8ae477eca0ace Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Thu, 12 Sep 2024 08:40:37 +0000 Subject: [PATCH 04/13] ci: increase timeout on api sanity check --- .github/test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/test.sh b/.github/test.sh index e46f4ad6..3d175da9 100755 --- a/.github/test.sh +++ b/.github/test.sh @@ -14,7 +14,7 @@ waitport() { test_api() { waitport 3000 curl -m 3 http://localhost:3000/ - API_RESPONSE=$(curl -m 3 http://localhost:3000/ \ + API_RESPONSE=$(curl -m 10 http://localhost:3000/ \ -X POST \ -H "Accept: application/json" \ -H "Content-Type: application/json" \ @@ -24,7 +24,7 @@ test_api() { STATUS=$(echo "$API_RESPONSE" | jq -r .status) STREAM_URL=$(echo "$API_RESPONSE" | jq -r .url) [ "$STATUS" = tunnel ] || exit 1; - S=$(curl -I -m 3 "$STREAM_URL") + S=$(curl -I -m 10 "$STREAM_URL") CONTENT_LENGTH=$(echo "$S" \ | grep -i content-length \ @@ -64,4 +64,4 @@ else exit 1 fi -wait || exit $? \ No newline at end of file +wait || exit $? From ca538a2e6ca453ebe52ec4c33198c666b56194ce Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 12 Sep 2024 20:07:56 +0600 Subject: [PATCH 05/13] api/youtube: use webm container for av1 and opus --- api/src/processing/services/youtube.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index 7551765e..cf83fbd0 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -18,8 +18,8 @@ const codecMatch = { }, av1: { videoCodec: "av01", - audioCodec: "mp4a", - container: "mp4" + audioCodec: "opus", + container: "webm" }, vp9: { videoCodec: "vp9", From 474c8e284fff2b4a2bbf88ca87f4185ca4f98981 Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 12 Sep 2024 20:08:20 +0600 Subject: [PATCH 06/13] web/i18n/settings: update av1 codec string --- web/i18n/en/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/i18n/en/settings.json b/web/i18n/en/settings.json index 01f56081..3cf181a4 100644 --- a/web/i18n/en/settings.json +++ b/web/i18n/en/settings.json @@ -31,7 +31,7 @@ "video.youtube.codec": "youtube video codec and container", "video.youtube.codec.h264": "h264 (mp4)", - "video.youtube.codec.av1": "av1 (mp4)", + "video.youtube.codec.av1": "av1 (webm)", "video.youtube.codec.vp9": "vp9 (webm)", "video.youtube.codec.description": "h264: best compatibility, average bitrate. max quality is 1080p. \nav1: best quality, efficiency, and bitrate. supports 8k & HDR. \nvp9: same quality & bitrate as av1, but file is approximately two times bigger. supports 4k & HDR.\n\nav1 and vp9 aren't as widely supported as h264.", From a1feadb9176db41b61d8c717498d18fe6f1cdeb0 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 13 Sep 2024 09:54:05 +0600 Subject: [PATCH 07/13] api/bluesky: add support for recordWithMedia embed type & catch various api errors --- api/src/processing/services/bluesky.js | 31 ++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/api/src/processing/services/bluesky.js b/api/src/processing/services/bluesky.js index 84571bcd..5f5cbcec 100644 --- a/api/src/processing/services/bluesky.js +++ b/api/src/processing/services/bluesky.js @@ -2,8 +2,8 @@ import HLS from "hls-parser"; import { cobaltUserAgent } from "../../config.js"; import { createStream } from "../../stream/manage.js"; -const extractVideo = async ({ getPost, filename }) => { - const urlMasterHLS = getPost?.thread?.post?.embed?.playlist; +const extractVideo = async ({ media, filename }) => { + const urlMasterHLS = media?.playlist; if (!urlMasterHLS) return { error: "fetch.empty" }; if (!urlMasterHLS.startsWith("https://video.bsky.app/")) return { error: "fetch.empty" }; @@ -77,14 +77,37 @@ export default async function ({ user, post, alwaysProxy }) { } }).then(r => r.json()).catch(() => {}); - if (!getPost || getPost?.error) return { error: "fetch.empty" }; + if (!getPost) return { error: "fetch.empty" }; + + if (getPost.error) { + switch (getPost.error) { + case "NotFound": + case "InternalServerError": + return { error: "content.post.unavailable" }; + case "InvalidRequest": + return { error: "link.unsupported" }; + default: + return { error: "fetch.empty" }; + } + } const embedType = getPost?.thread?.post?.embed?.$type; const filename = `bluesky_${user}_${post}`; if (embedType === "app.bsky.embed.video#view") { - return extractVideo({ getPost, filename }); + return extractVideo({ + media: getPost.thread?.post?.embed, + filename, + }) } + + if (embedType === "app.bsky.embed.recordWithMedia#view") { + return extractVideo({ + media: getPost.thread?.post?.embed?.media, + filename, + }) + } + if (embedType === "app.bsky.embed.images#view") { return extractImages({ getPost, filename, alwaysProxy }); } From a2414682c7d7ede8aa7a2fff8f1b622f08539e95 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 13 Sep 2024 09:55:06 +0600 Subject: [PATCH 08/13] api/tests: update bluesky tests --- api/src/util/tests.json | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/api/src/util/tests.json b/api/src/util/tests.json index bc7adae2..dee2f291 100644 --- a/api/src/util/tests.json +++ b/api/src/util/tests.json @@ -1401,7 +1401,16 @@ "bsky": [ { "name": "horizontal video", - "url": "https://bsky.app/profile/samuel.bsky.team/post/3l2udah76ch2c", + "url": "https://bsky.app/profile/haileyok.com/post/3l3giwtwp222m", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "horizontal video, recordWithMedia", + "url": "https://bsky.app/profile/juicysteak117.gay/post/3l3wonhk23g2i", "params": {}, "expected": { "code": 200, @@ -1410,7 +1419,7 @@ }, { "name": "vertical video", - "url": "https://bsky.app/profile/samuel.bsky.team/post/3l2uftgmitr2p", + "url": "https://bsky.app/profile/haileyok.com/post/3l3jhpomhjk2m", "params": {}, "expected": { "code": 200, @@ -1419,7 +1428,7 @@ }, { "name": "vertical video (muted)", - "url": "https://bsky.app/profile/samuel.bsky.team/post/3l2uftgmitr2p", + "url": "https://bsky.app/profile/haileyok.com/post/3l3jhpomhjk2m", "params": { "downloadMode": "mute" }, @@ -1430,7 +1439,7 @@ }, { "name": "vertical video (audio)", - "url": "https://bsky.app/profile/samuel.bsky.team/post/3l2uftgmitr2p", + "url": "https://bsky.app/profile/haileyok.com/post/3l3jhpomhjk2m", "params": { "downloadMode": "audio" }, @@ -1456,6 +1465,15 @@ "code": 200, "status": "picker" } + }, + { + "name": "deleted post", + "url": "https://bsky.app/profile/samuel.bsky.team/post/3l2udah76ch2c", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } } ] } \ No newline at end of file From f830a1219d2d96b18350fa50c0c54fd67a77a8dd Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 13 Sep 2024 12:35:58 +0600 Subject: [PATCH 09/13] docs/run-an-instance: update tutorial for running the api locally --- docs/run-an-instance.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md index 8e3afd48..fb339c4c 100644 --- a/docs/run-an-instance.md +++ b/docs/run-an-instance.md @@ -32,13 +32,19 @@ cobalt package will update automatically thanks to watchtower. it's highly recommended to use a reverse proxy (such as nginx) if you want your instance to face the public internet. look up tutorials online. -## using regular node.js (useful for local development) -setup script installs all needed `npm` dependencies, but you have to install `node.js` *(version 18 or above)* and `git` yourself. +## run cobalt api outside of docker (useful for local development) +requirements: +- node.js >= 18 +- git +- pnpm 1. clone the repo: `git clone https://github.com/imputnet/cobalt`. -2. run setup script and follow instructions: `npm run setup`. you need to host api and web instances separately, so pick whichever applies. -3. run cobalt via `npm start`. -4. done. +2. go to api/src directory: `cd cobalt/api/src`. +4. install dependencies: `pnpm install`. +5. create `.env` file in the same directory. +6. add needed environment variables to `.env` file. only `API_URL` is required to run cobalt. + - if you don't know what api url to use for local development, use `http://localhost:9000/`. +8. run cobalt: `pnpm start`. ### ubuntu 22.04 workaround `nscd` needs to be installed and running so that the `ffmpeg-static` binary can resolve DNS ([#101](https://github.com/imputnet/cobalt/issues/101#issuecomment-1494822258)): @@ -72,4 +78,4 @@ sudo service nscd start setting a `FREEBIND_CIDR` allows cobalt to pick a random IP for every download and use it for all requests it makes for that particular download. to use freebind in cobalt, you need to follow its [setup instructions](https://github.com/imputnet/freebind.js?tab=readme-ov-file#setup) first. if you configure this option while running cobalt in a docker container, you also need to set the `API_LISTEN_ADDRESS` env to `127.0.0.1`, and set -`network_mode` for the container to `host`. \ No newline at end of file +`network_mode` for the container to `host`. From 9c2babfc1bfafcd1a703233ab37cfdffc393c1a2 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 13 Sep 2024 12:39:10 +0600 Subject: [PATCH 10/13] docs/run-an-instance: teaching myself how to count to 6 sorry guys, it takes a ton of practice :( --- docs/run-an-instance.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md index fb339c4c..8144c037 100644 --- a/docs/run-an-instance.md +++ b/docs/run-an-instance.md @@ -40,11 +40,11 @@ requirements: 1. clone the repo: `git clone https://github.com/imputnet/cobalt`. 2. go to api/src directory: `cd cobalt/api/src`. -4. install dependencies: `pnpm install`. -5. create `.env` file in the same directory. -6. add needed environment variables to `.env` file. only `API_URL` is required to run cobalt. +3. install dependencies: `pnpm install`. +4. create `.env` file in the same directory. +5. add needed environment variables to `.env` file. only `API_URL` is required to run cobalt. - if you don't know what api url to use for local development, use `http://localhost:9000/`. -8. run cobalt: `pnpm start`. +6. run cobalt: `pnpm start`. ### ubuntu 22.04 workaround `nscd` needs to be installed and running so that the `ffmpeg-static` binary can resolve DNS ([#101](https://github.com/imputnet/cobalt/issues/101#issuecomment-1494822258)): From 47625490ce37d2da3e0bbe2b0d6b5b28aa067d24 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 13 Sep 2024 21:25:27 +0600 Subject: [PATCH 11/13] web/settings/video: move codec names away from i18n --- web/i18n/en/settings.json | 3 --- web/src/routes/settings/video/+page.svelte | 8 +++++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/web/i18n/en/settings.json b/web/i18n/en/settings.json index 3cf181a4..85c3d58b 100644 --- a/web/i18n/en/settings.json +++ b/web/i18n/en/settings.json @@ -30,9 +30,6 @@ "video.quality.description": "if preferred video quality isn't available, next best is picked instead.", "video.youtube.codec": "youtube video codec and container", - "video.youtube.codec.h264": "h264 (mp4)", - "video.youtube.codec.av1": "av1 (webm)", - "video.youtube.codec.vp9": "vp9 (webm)", "video.youtube.codec.description": "h264: best compatibility, average bitrate. max quality is 1080p. \nav1: best quality, efficiency, and bitrate. supports 8k & HDR. \nvp9: same quality & bitrate as av1, but file is approximately two times bigger. supports 4k & HDR.\n\nav1 and vp9 aren't as widely supported as h264.", "video.twitter.gif": "twitter/x", diff --git a/web/src/routes/settings/video/+page.svelte b/web/src/routes/settings/video/+page.svelte index 2cdaf976..38ce0d8d 100644 --- a/web/src/routes/settings/video/+page.svelte +++ b/web/src/routes/settings/video/+page.svelte @@ -8,6 +8,12 @@ import Switcher from "$components/buttons/Switcher.svelte"; import SettingsButton from "$components/buttons/SettingsButton.svelte"; import SettingsToggle from "$components/buttons/SettingsToggle.svelte"; + + const codecTitles = { + h264: "h264 (mp4)", + av1: "av1 (webm)", + vp9: "vp9 (webm)", + } - {$t(`settings.video.youtube.codec.${value}`)} + {codecTitles[value]} {/each} From 5facbc96572fa87fbe1533579511c6e59a154e72 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 13 Sep 2024 23:28:14 +0600 Subject: [PATCH 12/13] api/tests/bluesky: update deleted post test --- api/src/util/tests.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/util/tests.json b/api/src/util/tests.json index dee2f291..78340691 100644 --- a/api/src/util/tests.json +++ b/api/src/util/tests.json @@ -1467,8 +1467,8 @@ } }, { - "name": "deleted post", - "url": "https://bsky.app/profile/samuel.bsky.team/post/3l2udah76ch2c", + "name": "deleted post/invalid user", + "url": "https://bsky.app/profile/notreal.bsky.team/post/3l2udah76ch2c", "params": {}, "expected": { "code": 400, From 0ccd08470b80915b4c1bd5bd7ad2ed06acfe213e Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 13 Sep 2024 23:57:18 +0600 Subject: [PATCH 13/13] web/about/general: more clarity in privacy section --- web/src/routes/about/general/+page.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/routes/about/general/+page.svelte b/web/src/routes/about/general/+page.svelte index 3236515c..a7ecacdf 100644 --- a/web/src/routes/about/general/+page.svelte +++ b/web/src/routes/about/general/+page.svelte @@ -18,7 +18,7 @@

leading privacy

all requests to backend are anonymous and all tunnels are encrypted. - we have a strict zero log policy and don't track anything at all. + we have a strict zero log policy and don't track anything about individual people.

to avoid caching or storing downloaded files, cobalt processes them on-fly, sending processed pieces directly to client.