From 31a2136c90d950a7cf35b31d5dbec1a123c20a82 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 28 Oct 2023 18:51:20 +0600 Subject: [PATCH 01/19] updated docs --- .gitignore | 4 + README.md | 137 +++++++++--------- crowdin.yml | 3 - docs/API.md | 128 ++++++++-------- .../examples/cookies.example.json | 0 .../examples/docker-compose.example.yml | 2 +- docs/run-an-instance.md | 44 ++++++ docs/troubleshooting.md | 33 +++++ jsconfig.json | 13 -- package.json | 2 +- src/front/cobalt.css | 8 +- src/localization/languages/en.json | 8 +- src/localization/languages/ru.json | 8 +- src/modules/pageRender/elements.js | 4 + src/modules/pageRender/page.js | 4 +- 15 files changed, 241 insertions(+), 157 deletions(-) delete mode 100644 crowdin.yml rename src/modules/processing/cookie/cookies_example.json => docs/examples/cookies.example.json (100%) rename docker-compose.example.yml => docs/examples/docker-compose.example.yml (96%) create mode 100644 docs/run-an-instance.md create mode 100644 docs/troubleshooting.md delete mode 100644 jsconfig.json diff --git a/.gitignore b/.gitignore index 887344cc..a21273d6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# os stuff +.DS_Store +desktop.ini + # npm node_modules package-lock.json diff --git a/README.md b/README.md index f62ec3a1..bbfd42a0 100644 --- a/README.md +++ b/README.md @@ -1,78 +1,81 @@ # cobalt -Best way to save what you love. -Live web app: [cobalt.tools](https://cobalt.tools/) +best way to save what you love: [cobalt.tools](https://cobalt.tools/) -![cobalt logo with repeated logo pattern background](https://raw.githubusercontent.com/wukko/cobalt/current/src/front/icons/pattern.png "cobalt logo with repeated logo pattern background") +![cobalt logo with repeated logo (double arrow) pattern background](https://raw.githubusercontent.com/wukko/cobalt/current/src/front/icons/pattern.png "cobalt logo with repeated logo (double arrow) pattern background") -[![DeepSource](https://deepsource.io/gh/wukko/cobalt.svg/?label=active+issues&token=MsmsJ9zUOKwcQor0yaiFot84)](https://deepsource.io/gh/wukko/cobalt/?ref=repository-badge) +## what's cobalt? +cobalt is a media downloader that doesn't piss you off. it's fast, friendly, and doesn't have any bullshit that modern web is filled with: ***no ads, trackers, or analytics***. -## What's cobalt? -cobalt is social and media platform downloader that doesn't piss you off. +paste the link, get the file, move on. it's that simple. just how it should be. -It's fast, friendly, and doesn't have any bullshit that modern web is filled with: no ads, trackers, or analytics. -Paste the link, get the video, move on. It's that simple. Just how it should be. +## supported services +this list is not final and keeps expanding over time. if support for a service you want is missing, create an issue (or a pull request 👀). -## Supported services -| Service | Video + Audio | Only audio | Only video | Additional notes or features | -| -------- | :---: | :---: | :---: | :----- | -| bilibili.com | ✅ | ✅ | ✅ | | -| Instagram | ✅ | ✅ | ✅ | Supports photos, videos, and stories. Lets you pick what to save from multi-media posts. | -| Instagram Reels | ✅ | ✅ | ✅ | | -| Pinterest | ✅ | ✅ | ✅ | Support for videos and stories. | -| Reddit | ✅ | ✅ | ✅ | Support for GIFs and videos. | -| Rutube | ✅ | ✅ | ✅ | | -| SoundCloud | ➖ | ✅ | ➖ | Audio metadata, downloads from private links. | -| Streamable | ✅ | ✅ | ✅ | | -| TikTok | ✅ | ✅ | ✅ | Supports downloads of: videos with or without watermark, images from slideshow without watermark, full (original) audios. | -| Tumblr | ✅ | ✅ | ✅ | Support for audio file downloads. | -| Twitch Clips | ✅ | ✅ | ✅ | | -| Twitter/X * | ✅ | ✅ | ✅ | Ability to pick what to save from multi-media tweets. | -| Vimeo | ✅ | ✅ | ✅ | Audio downloads are only available for dash files. | -| Vine Archive | ✅ | ✅ | ✅ | | -| VK Videos | ✅ | ❌ | ❌ | | -| VK Clips | ✅ | ❌ | ❌ | | -| YouTube Videos & Shorts | ✅ | ✅ | ✅ | Support for 8K, 4K, HDR, VR, and high FPS videos. Audio metadata & dubs. h264/av1/vp9 codecs. | -| YouTube Music | ➖ | ✅ | ➖ | Audio metadata. | +| service | video + audio | only audio | only video | metadata | rich file names | +| -------- | :-----------: | :--------: | :--------: | :------: | :-------------: | +| bilibili.com | ✅ | ✅ | ✅ | ➖ | ➖ | +| instagram posts & stories | ✅ | ✅ | ✅ | ➖ | ➖ | +| instagram reels | ✅ | ✅ | ✅ | ➖ | ➖ | +| pinterest | ✅ | ✅ | ✅ | ➖ | ➖ | +| reddit | ✅ | ✅ | ✅ | ❌ | ❌ | +| rutube | ✅ | ✅ | ✅ | ✅ | ✅ | +| soundcloud | ➖ | ✅ | ➖ | ✅ | ✅ | +| streamable | ✅ | ✅ | ✅ | ➖ | ➖ | +| tiktok | ✅ | ✅ | ✅ | ❌ | ❌ | +| tumblr | ✅ | ✅ | ✅ | ➖ | ➖ | +| twitch clips | ✅ | ✅ | ✅ | ✅ | ❌ | +| twitter/x | ✅ | ✅ | ✅ | ➖ | ➖ | +| vimeo | ✅ | ✅ | ✅ | ✅ | ✅ | +| vine archive | ✅ | ✅ | ✅ | ➖ | ➖ | +| vk videos & clips | ✅ | ❌ | ❌ | ✅ | ✅ | +| youtube videos, shorts & music | ✅ | ✅ | ✅ | ✅ | ✅ | -This list is not final and keeps expanding over time, make sure to check it once in a while! - -*Reliability of downloads from Twitter is questionable due to its current management. - -## cobalt API -cobalt has an open API that you can use in your projects for **free**. -It's easy and straightforward to use, [check out the docs](https://github.com/wukko/cobalt/blob/current/docs/API.md) and see for yourself. -Feel free to use the main API instance ([co.wuk.sh](https://co.wuk.sh/)) in your projects. - -## Host an instance yourself -### Requirements -- Node.js 18 or above -- git - -Setup script installs all needed `npm` dependencies, but you have to install `Node.js` and `git` yourself. - -1. Clone the repo: `git clone https://github.com/wukko/cobalt` -2. Run setup script and follow instructions: `npm run setup` -3. Run cobalt via `npm start` -4. Done. - -You need to host API and web app separately since v.6.0. Setup script will help you with that! - -### Ubuntu 22.04+ workaround -`nscd` needs to be installed and running so that the `ffmpeg-static` binary can resolve DNS ([#101](https://github.com/wukko/cobalt/issues/101#issuecomment-1494822258)): - -```bash -sudo apt install nscd -sudo service nscd start +``` +✅ : supported +➖ : impossible/unreasonable +❌ : not supported ``` -### Docker -It's also possible to run cobalt via Docker. I *highly* recommend using Docker compose. -Check out the [example compose file](https://github.com/wukko/cobalt/blob/current/docker-compose.example.yml) and alter it for your needs. -## Disclaimer -cobalt is my passion project, so update schedule depends solely on my free time, motivation, and mood. -Don't expect any consistency in that. +### additional notes or features (per service) +| service | notes or features | +| -------- | :----- | +| instagram | supports photos, videos, and stories. lets you pick what to save from multi-media posts. | +| pinterest | supports videos and stories. | +| reddit | supports gifs and videos. | +| soundcloud | supports private links. | +| tiktok | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios. | +| twitter/x | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management. | +| vimeo | audio downloads are only available for dash. | +| youtube | supports videos, music, and shorts. 8K, 4K, HDR, VR, and high FPS videos. rich metadata & dubs. h264/av1/vp9 codecs. | -## License -cobalt is under [AGPL-3.0](https://github.com/wukko/cobalt/blob/current/LICENSE) license. -[Fluent Emoji](https://github.com/microsoft/fluentui-emoji) used in the project is under [MIT](https://github.com/microsoft/fluentui-emoji/blob/main/LICENSE) license. +## cobalt api +cobalt has an open api that you can use in projects *for completely free~*. it's easy and straightforward to use, [check out the docs](https://github.com/wukko/cobalt/blob/current/docs/api.md) to learn how to use it. + +you can use the main api instance ([co.wuk.sh](https://co.wuk.sh/)) in your projects. + +## how to run your own instance +if you want to run your own instance for whatever purpose, [follow this guide](https://github.com/wukko/cobalt/blob/current/docs/run-an-instance.md). +it's *highly* recommended to use a docker compose method unless you run for developing/debugging purposes. + +## sponsors +cobalt is sponsored by [royalehosting.net](https://royalehosting.net/), all main instances are currently hosted on their network :) + +## ethics and disclaimer +cobalt is a tool for easing content downloads from internet and takes ***zero liability***. you are responsible for what you download, how you use and distribute that content. please be mindful when using content of others and always credit original creators. fair use and credits benefit everyone. + +cobalt is ***NOT*** a piracy tool and cannot be used as such. it can only download free, publicly accessible content. such content can be easily downloaded through any browser's dev tools. pressing one button is easier, so i made a convenient, ad-less tool for such repeated actions. + +cobalt is my passion project, update schedule depends solely on my free time, motivation, and mood. don't expect any consistency in update releases. + +## cobalt licenses +cobalt code is licensed under [AGPL-3.0](https://github.com/wukko/cobalt/blob/current/LICENSE). + +update banners and various assets of cobalt branding included within the repo are *not* covered by the AGPL-3.0 license and cannot be used using same terms. + +## 3rd party licenses +[Fluent Emoji by Microsoft](https://github.com/microsoft/fluentui-emoji) (used in cobalt) is under [MIT](https://github.com/microsoft/fluentui-emoji/blob/main/LICENSE) license. + +[Noto Sans Mono](https://fonts.google.com/noto/specimen/Noto+Sans+Mono/) fonts (used in cobalt) are licensed under the [OFL](https://fonts.google.com/noto/specimen/Noto+Sans+Mono/about) license. + +many update banners were taken from [tenor.com](https://tenor.com/). \ No newline at end of file diff --git a/crowdin.yml b/crowdin.yml deleted file mode 100644 index ba9e2f42..00000000 --- a/crowdin.yml +++ /dev/null @@ -1,3 +0,0 @@ -files: - - source: /src/localization/languages/en.json - translation: /src/localization/languages/%two_letters_code%.json diff --git a/docs/API.md b/docs/API.md index 6b3c105e..d7b20295 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1,73 +1,79 @@ -# cobalt API Documentation -This document provides info about methods and acceptable variables for all cobalt API requests.
+# cobalt api documentation +this document provides info about methods and acceptable variables for all cobalt api requests. ``` -⚠️ Main API instance has moved to https://co.wuk.sh/ - -Make sure your projects use the correct API domain. +👍 you can use co.wuk.sh instance in your projects for free, just don't be an asshole. ``` -## POST: ``/api/json`` -Main processing endpoint.
+## POST: `/api/json` +cobalt's main processing endpoint. -Request Body Type: ``application/json``
-Response Body Type: ``application/json`` +request body type: `application/json` +response body type: `application/json` -### Request Body Variables -| key | type | variables | default | description | -|:--------------------|:------------|:-------------------------------------|:------------|:-------------------------------------------------------------------------------| -| ``url`` | ``string`` | Sharable URL encoded as URI | ``null`` | **Must** be included in every request. | -| ``vCodec`` | ``string`` | ``h264 / av1 / vp9`` | ``h264`` | Applies only to YouTube downloads. ``h264`` is recommended for phones. | -| ``vQuality`` | ``string`` | ``144 / ... / 2160 / max`` | ``720`` | ``720`` quality is recommended for phones. | -| ``aFormat`` | ``string`` | ``best / mp3 / ogg / wav / opus`` | ``mp3`` | | -| ``filenamePattern`` | ``boolean`` | ``classic / pretty / basic / nerdy`` | ``classic`` | Changes the way files are named. Previews can be seen in the web app. | -| ``isAudioOnly`` | ``boolean`` | ``true / false`` | ``false`` | | -| ``isNoTTWatermark`` | ``boolean`` | ``true / false`` | ``false`` | Changes whether downloaded TikTok videos have watermarks. | -| ``isTTFullAudio`` | ``boolean`` | ``true / false`` | ``false`` | Enables download of original sound used in a TikTok video. | -| ``isAudioMuted`` | ``boolean`` | ``true / false`` | ``false`` | Disables audio track in video downloads. | -| ``dubLang`` | ``boolean`` | ``true / false`` | ``false`` | Backend uses Accept-Language for YouTube video audio tracks when ``true``. | -| ``disableMetadata`` | ``boolean`` | ``true / false`` | ``false`` | Disables file metadata when set to ``true``. | +``` +⚠️ you must include Accept and Content-Type headers with every POST /api/json request. -### Response Body Variables -| key | type | variables | -|:---------------|:-----------|:--------------------------------------------------------------| -| ``status`` | ``string`` | ``error / redirect / stream / success / rate-limit / picker`` | -| ``text`` | ``string`` | Text | -| ``url`` | ``string`` | Direct link to a file / link to cobalt's live render | -| ``pickerType`` | ``string`` | ``various / images`` | -| ``picker`` | ``array`` | Array of picker items | -| ``audio`` | ``string`` | Direct link to a file / link to cobalt's live render | +Accept: application/json +Content-Type: application/json +``` -### Picker Item Variables -Item type: ``object`` -| key | type | variables | description | -|:---------------|:-----------|:------------------------------------------------|:--------------------------------------------| -| ``type`` | ``string`` | ``video`` | Used only if ``pickerType`` is ``various``. | -| ``url`` | ``string`` | Direct link to a file / link to cobalt's live render | | -| ``thumb`` | ``string`` | Item thumbnail that's displayed in the picker | Used only for ``video`` type. | +### request body variables +| key | type | variables | default | description | +|:------------------|:----------|:-----------------------------------|:----------|:-------------------------------------------------------------------------------| +| `url` | `string` | URL encoded as URI | `null` | **must** be included in every request. | +| `vCodec` | `string` | `h264 / av1 / vp9` | `h264` | applies only to youtube downloads. `h264` is recommended for phones. | +| `vQuality` | `string` | `144 / ... / 2160 / max` | `720` | `720` quality is recommended for phones. | +| `aFormat` | `string` | `best / mp3 / ogg / wav / opus` | `mp3` | | +| `filenamePattern` | `boolean` | `classic / pretty / basic / nerdy` | `classic` | changes the way files are named. previews can be seen in the web app. | +| `isAudioOnly` | `boolean` | `true / false` | `false` | | +| `isNoTTWatermark` | `boolean` | `true / false` | `false` | changes whether downloaded tiktok videos have watermarks. | +| `isTTFullAudio` | `boolean` | `true / false` | `false` | enables download of original sound used in a tiktok video. | +| `isAudioMuted` | `boolean` | `true / false` | `false` | disables audio track in video downloads. | +| `dubLang` | `boolean` | `true / false` | `false` | backend uses Accept-Language hader for youtube video audio tracks when `true`. | +| `disableMetadata` | `boolean` | `true / false` | `false` | disables file metadata when set to `true`. | -## GET: ``/api/stream`` -Content live render streaming endpoint.
+### response body variables +| key | type | variables | +|:-------------|:---------|:------------------------------------------------------------| +| `status` | `string` | `error / redirect / stream / success / rate-limit / picker` | +| `text` | `string` | various text, mostly used for errors | +| `url` | `string` | direct link to a file or a link to cobalt's live render | +| `pickerType` | `string` | `various / images` | +| `picker` | `array` | array of picker items | +| `audio` | `string` | direct link to a file or a link to cobalt's live render | -### Request Query Variables -| key | variables | description | -|:--------|:-----------------|:-------------------------------------------------------------------------------------------------------------------------------| -| ``p`` | ``1`` | Used for probing whether user is rate limited. | -| ``t`` | Stream token | Unique stream ID. Used for retrieving cached stream info data. | -| ``h`` | HMAC | Hashed combination of: (hashed) ip address, stream token, expiry timestamp, and service name. Used for verification of stream. | -| ``e`` | Expiry timestamp | | +### picker item variables +item type: `object` -## GET: ``/api/serverInfo`` -Returns current basic server info.
-Response Body Type: ``application/json`` +| key | type | variables | description | +|:--------|:---------|:--------------------------------------------------------|:---------------------------------------| +| `type` | `string` | `video` | used only if `pickerType`is `various`. | +| `url` | `string` | direct link to a file or a link to cobalt's live render | | +| `thumb` | `string` | item thumbnail that's displayed in the picker | used only for `video` type. | -### Response Body Variables -| key | type | variables | -|:--------------|:-----------|:------------------| -| ``version`` | ``string`` | cobalt version | -| ``commit`` | ``string`` | Git commit | -| ``branch`` | ``string`` | Git branch | -| ``name`` | ``string`` | Server name | -| ``url`` | ``string`` | Server url | -| ``cors`` | ``int`` | CORS status | -| ``startTime`` | ``string`` | Server start time | +## GET: `/api/stream` +cobalt's live render (or stream) endpoint. used for sending various media content over to the user. + +### request query variables +| key | variables | description | +|:-----|:-----------------|:-------------------------------------------------------------------------------------------------------------------------------| +| `p` | `1` | used for probing whether user is rate limited. | +| `t` | stream token | unique stream id. used for retrieving cached stream info data. | +| `h` | hmac | hashed combination of: (hashed) ip address, stream token, expiry timestamp, and service name. used for verification of stream. | +| `e` | expiry timestamp | | + +## GET: `/api/serverInfo` +returns current basic server info. +response body type: `application/json` + +### response body variables +| key | type | variables | +|:------------|:---------|:------------------| +| `version` | `string` | cobalt version | +| `commit` | `string` | git commit | +| `branch` | `string` | git branch | +| `name` | `string` | server name | +| `url` | `string` | server url | +| `cors` | `int` | cors status | +| `startTime` | `string` | server start time | diff --git a/src/modules/processing/cookie/cookies_example.json b/docs/examples/cookies.example.json similarity index 100% rename from src/modules/processing/cookie/cookies_example.json rename to docs/examples/cookies.example.json diff --git a/docker-compose.example.yml b/docs/examples/docker-compose.example.yml similarity index 96% rename from docker-compose.example.yml rename to docs/examples/docker-compose.example.yml index a74a89af..8a5f9d67 100644 --- a/docker-compose.example.yml +++ b/docs/examples/docker-compose.example.yml @@ -24,7 +24,7 @@ services: - apiName=eu-nl # if you want to use cookies when fetching data from services, uncomment the next line #- cookiePath=/cookies.json - # see src/modules/processing/cookie/cookies_example.json for example file. + # see cookies_example.json for example file. labels: - com.centurylinklabs.watchtower.scope=cobalt diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md new file mode 100644 index 00000000..54a7d3b8 --- /dev/null +++ b/docs/run-an-instance.md @@ -0,0 +1,44 @@ +# how to host a cobalt instance yourself +## using docker compose and package from github (recommended) +to run the cobalt docker package, you need to have `docker` and `docker-compose` installed and configured. + +if you need help with installing docker, follow *only the first step* of these tutorials by digitalocean: +- [how to install docker](https://www.digitalocean.com/community/tutorial-collections/how-to-install-and-use-docker) +- [how to install docker compose](https://www.digitalocean.com/community/tutorial-collections/how-to-install-docker-compose) + +## how to run a cobalt docker package: +1. create a folder for cobalt config file, something like this: + ```sh + mkdir cobalt + ``` + +2. go to cobalt folder, and create a docker compose config file: + ```sh + cd cobalt && nano docker-compose.yml + ``` + i'm using `nano` in this example, it may not be available in your distro. you can use any other text editor. + +3. copy and paste the [sample config from here](https://github.com/wukko/cobalt/blob/current/docs/examples/docker-compose.example.json) for either web or api instance (or both, if you wish) and edit it to your needs. + make sure to replace default URLs with yours or cobalt won't work correctly. + +if you want your instance to support services that require authentication to view public content, create `cookies.json` file in the same directory as `docker-compose.yml`. example file for cookies file [can be found here](https://github.com/wukko/cobalt/blob/current/docs/examples/cookies.example.json). + +cobalt package will automatically update itself thanks to watchtower. + +it's highly recommended to use a reverse proxy (such as nginx) if you want your instance to face the public internet. look up tutorials for that online. + +## using regular node.js (useful for local development) +setup script installs all needed `npm` dependencies, but you have to install `node.js` *(version 18 or above)* and `git` yourself. + +1. clone the repo: `git clone https://github.com/wukko/cobalt`. +2. run setup script and follow instructions: `npm run setup`. you need to host api and web instances separately, so pick whichever applies. +3. run cobalt via `npm start`. +4. done. + +### ubuntu 22.04 workaround +`nscd` needs to be installed and running so that the `ffmpeg-static` binary can resolve DNS ([#101](https://github.com/wukko/cobalt/issues/101#issuecomment-1494822258)): + +```bash +sudo apt install nscd +sudo service nscd start +``` diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 00000000..6ad6335b --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,33 @@ +# self-troubleshooting cobalt +``` +🚧 this page is work-in-progress. expect more guides to be added in the future! +``` +if any issues occur while using cobalt, you can fix many of them yourself. this document aims to provide guides on how to fix most complicated of them. +use wiki navigation on right to jump between solutions. + +## how to fix clipboard pasting in firefox +you can fix this issue by changing a single preference in `about:config`. + +### steps to enable clipboard functionality +1. go to `about:config`: + + ![screenshot showing about:config entered into address bar](https://github.com/wukko/cobalt/assets/71202418/9ad78612-a372-4949-aeac-99dfc41e273c) + +2. if asked, read what firefox has to say and press "accept the risk and continue". + ⚠ tinkering with other preferences may break your browser. **do not** edit them unless you know what you're doing. + + ![screenshot showing about:config security warning that reads: "proceed with caution. changing advanced configuration preferences can impact firefox performance or security." lower there's a pre-checked checkbox that says: "warn me when i attempt to access these preferences". lowest element is a blue button that says "accept the risk and continue"](https://github.com/wukko/cobalt/assets/71202418/02328729-dbfe-4ea4-b2ca-7bcf1998c2ca) + +3. search for `dom.events.asyncclipboard.readtext` + + ![screenshot showing "dom.events.asyncclipboard.readtext" entered into search on about:config page](https://github.com/wukko/cobalt/assets/71202418/7c7f7e3c-6a6a-40df-8436-277489e72e0b) + +4. press the toggle button on very right. + + ![screenshot showing "dom.events.asyncclipboard.readtext" preference on about:config page with highlighted toggle button on very right](https://github.com/wukko/cobalt/assets/71202418/b45db18e-f4bf-4f1c-9a8c-f13a63a21335) + +5. "false" should change to "true". + + ![screenshot showing "dom.events.asyncclipboard.readtext" preference on about:config page, this one with "true" text highlighted](https://github.com/wukko/cobalt/assets/71202418/4869b4ff-8385-4cd3-ae59-aa2e03a58b5f) + +6. go back to cobalt, reload the page, press `paste and download` button again. this time it works! enjoy simpler downloading experience :) diff --git a/jsconfig.json b/jsconfig.json deleted file mode 100644 index 3a840ef5..00000000 --- a/jsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "compilerOptions": { - "module": "ESNext", - "moduleResolution": "Node", - "target": "ES2020", - "strictNullChecks": true, - "strictFunctionTypes": true - }, - "exclude": [ - "node_modules", - "**/node_modules/*" - ] -} diff --git a/package.json b/package.json index 946476c2..81ba899d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "7.6.2", + "version": "7.6.3", "author": "wukko", "exports": "./src/cobalt.js", "type": "module", diff --git a/src/front/cobalt.css b/src/front/cobalt.css index d90b0618..46b28ea7 100644 --- a/src/front/cobalt.css +++ b/src/front/cobalt.css @@ -9,6 +9,7 @@ --padding-1: 0.75rem; --line-height: 1.65rem; --red: rgb(249, 47, 96); + --blue: rgb(47, 138, 249); --gap: 0.5rem; --gap-no-icon: 0.6rem; } @@ -266,6 +267,11 @@ button:active, height: 2.5rem; align-items: center; display: flex; + gap: 0.3rem; +} +.logo-sub { + color: var(--blue); + font-size: 0.8rem; } #download-area { display: flex; @@ -1179,4 +1185,4 @@ button:active, .popup-title { line-height: inherit; } -} \ No newline at end of file +} diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json index 0e4a9c58..4e9132f3 100644 --- a/src/localization/languages/en.json +++ b/src/localization/languages/en.json @@ -125,17 +125,17 @@ "SettingsReduceTransparency": "reduce transparency", "SettingsDisableAnimations": "disable animations", "FeatureErrorGeneric": "your browser doesn't allow or support this feature. check if there are any updates available and try again!", - "ClipboardErrorFirefox": "you're using firefox where all clipboard reading functionality is disabled.\n\nyou can fix this by following steps listed here!\n\n...or you can paste the link manually instead.", + "ClipboardErrorFirefox": "you're using firefox where all clipboard reading functionality is disabled.\n\nyou can fix this by following steps listed here!\n\n...or you can paste the link manually instead.", "ClipboardErrorNoPermission": "cobalt can't access the most recent item in your clipboard without your permission.\n\nif you don't want to give access, just paste the link manually instead.\n\nif you do, go to site settings and enable the clipboard permission.", - "SupportSelfTroubleshooting": "experiencing issues? try self-troubleshooting guide first!", + "SupportSelfTroubleshooting": "experiencing issues? try self-troubleshooting guide first!", "AccessibilityGoBack": "go back and close the popup", "CollapseKeyboard": "keyboard shortcuts", "KeyboardShortcutsIntro": "use cobalt even faster with keyboard shortcuts:", "KeyboardShortcutQuickPaste": "paste the link", "KeyboardShortcutClear": "clear link input area", "KeyboardShortcutClosePopup": "close all popups", - "CollapseLegal": "legal stuff", - "FairUse": "cobalt is a tool for easing content downloads from internet and takes zero liability. you are responsible for what you download, how you use and distribute that content.\n\ncobalt does not log any info about you, it's impossible for me to snitch on you, but please be mindful when using content of others and always credit original creators!\n\nwhen used in education purposes (lecture, homework, etc) please attach the source link.\n\nfair use and credits benefit everyone.", + "CollapseLegal": "terms and ethics", + "FairUse": "cobalt is a tool for easing content downloads from internet and takes zero liability. you are responsible for what you download, how you use and distribute that content. please be mindful when using content of others and always credit original creators. fair use and credits benefit everyone.\n\nwhen used in education purposes (lecture, homework, etc) please attach the source link.\n\nfair use and credits benefit everyone.", "UrgentFeatureUpdate71": "more supported services!", "UrgentThanks": "thank you for support!", "SettingsDisableMetadata": "don't add metadata", diff --git a/src/localization/languages/ru.json b/src/localization/languages/ru.json index ad45062c..ec2e60d7 100644 --- a/src/localization/languages/ru.json +++ b/src/localization/languages/ru.json @@ -126,17 +126,17 @@ "SettingsReduceTransparency": "уменьшить прозрачность", "SettingsDisableAnimations": "убрать анимации", "FeatureErrorGeneric": "твой браузер не разрешает или не поддерживает эту функцию. проверь наличие обновлений и попробуй ещё раз!", - "ClipboardErrorFirefox": "ты используешь firefox в котором все функции чтения из буфера обмена отключены по умолчанию.\n\nно это можно исправить следуя шагам, описанным здесь\n\n...или же ты можешь просто вставить ссылку вручную.", + "ClipboardErrorFirefox": "ты используешь firefox в котором все функции чтения из буфера обмена отключены по умолчанию.\n\nно это можно исправить следуя шагам, описанным здесь\n\n...или же ты можешь просто вставить ссылку вручную.", "ClipboardErrorNoPermission": "кобальт не может прочитать последний элемент в буфере обмена без твоего разрешения.\n\nесли ты не хочешь давать доступ, просто вставь ссылку вручную.\n\nну а если хочешь, то открой настройки сайта и разреши доступ на чтение буфера обмена.", - "SupportSelfTroubleshooting": "возникли проблемы? попробуй сначала исправить всё сам по этому гиду!", + "SupportSelfTroubleshooting": "возникли проблемы? попробуй сначала исправить всё сам по этому гиду!", "AccessibilityGoBack": "вернуться назад и закрыть окно", "CollapseKeyboard": "горячие клавиши", "KeyboardShortcutsIntro": "пользуйся кобальтом ещё быстрее с горячими клавишами:", "KeyboardShortcutQuickPaste": "вставить ссылку", "KeyboardShortcutClear": "очистить зону вставки ссылки", "KeyboardShortcutClosePopup": "закрыть все окна", - "CollapseLegal": "правовые штучки", - "FairUse": "кобальт - это инструмент для облегчения скачивания контента из интернета, и он не несёт никакой ответственности. ты несёшь ответственность за то, что скачиваешь, как используешь и распространяешь скачанный контент.\n\nкобальт не собирает никакой информации о тебе, и не может донести на тебя, но, пожалуйста, будь сознателен при использовании чужого контента и всегда указывай авторов!\n\nпри использовании в образовательных целях (лекции, домашние задания и т.д.), пожалуйста, прикладывай ссылку на источник.\n\nчестное использование и указание авторства выгодно всем.", + "CollapseLegal": "принципы и этика", + "FairUse": "кобальт - это инструмент для облегчения скачивания контента из интернета, и он не несёт никакой ответственности. ты несёшь ответственность за то, что скачиваешь, как используешь и распространяешь скачанный контент. пожалуйста, будь сознателен при использовании чужого контента и всегда указывай авторов!\n\nпри использовании в образовательных целях (лекции, домашние задания и т.д.), пожалуйста, прикладывай ссылку на источник.\n\nчестное использование и указание авторства выгодно всем.", "UrgentFeatureUpdate71": "расширение поддержки сервисов!", "UrgentThanks": "спасибо за поддержку!", "SettingsDisableMetadata": "не добавлять метаданные", diff --git a/src/modules/pageRender/elements.js b/src/modules/pageRender/elements.js index c0c4ed44..8415fb25 100644 --- a/src/modules/pageRender/elements.js +++ b/src/modules/pageRender/elements.js @@ -274,3 +274,7 @@ export function sponsoredList() { } return `` } + +export function betaTag() { + return process.env.isBeta ? 'β' : '' +} diff --git a/src/modules/pageRender/page.js b/src/modules/pageRender/page.js index 8381d424..b572a8b7 100644 --- a/src/modules/pageRender/page.js +++ b/src/modules/pageRender/page.js @@ -1,4 +1,4 @@ -import { checkbox, collapsibleList, explanation, footerButtons, multiPagePopup, popup, popupWithBottomButtons, sep, settingsCategory, switcher, socialLink, socialLinks, urgentNotice, keyboardShortcuts, webLoc, sponsoredList } from "./elements.js"; +import { checkbox, collapsibleList, explanation, footerButtons, multiPagePopup, popup, popupWithBottomButtons, sep, settingsCategory, switcher, socialLink, socialLinks, urgentNotice, keyboardShortcuts, webLoc, sponsoredList, betaTag } from "./elements.js"; import { services as s, authorInfo, version, repo, donations, supportedAudio } from "../config.js"; import { getCommitInfo } from "../sub/currentCommit.js"; import loc from "../../localization/manager.js"; @@ -568,7 +568,7 @@ export default function(obj) { action: "popup('about', 1, 'changelog')" })}
- +
From 3e6fdb74038fba605eb41b806ae7e0a0c7920500 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 28 Oct 2023 18:55:47 +0600 Subject: [PATCH 02/19] update emoji meaning in readme --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index bbfd42a0..517b26ee 100644 --- a/README.md +++ b/README.md @@ -30,12 +30,11 @@ this list is not final and keeps expanding over time. if support for a service y | vk videos & clips | ✅ | ❌ | ❌ | ✅ | ✅ | | youtube videos, shorts & music | ✅ | ✅ | ✅ | ✅ | ✅ | -``` -✅ : supported -➖ : impossible/unreasonable -❌ : not supported -``` - +| emoji | meaning | +| :-----: | :---------------------- | +| ✅ | supported | +| ➖ | impossible/unreasonable | +| ❌ | not supported | ### additional notes or features (per service) | service | notes or features | From c7dc868498d6d92399bdd591cb8a7d626621913c Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 28 Oct 2023 18:58:01 +0600 Subject: [PATCH 03/19] fix alignment --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 517b26ee..820758a7 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ paste the link, get the file, move on. it's that simple. just how it should be. this list is not final and keeps expanding over time. if support for a service you want is missing, create an issue (or a pull request 👀). | service | video + audio | only audio | only video | metadata | rich file names | -| -------- | :-----------: | :--------: | :--------: | :------: | :-------------: | +| :-------- | :-----------: | :--------: | :--------: | :------: | :-------------: | | bilibili.com | ✅ | ✅ | ✅ | ➖ | ➖ | | instagram posts & stories | ✅ | ✅ | ✅ | ➖ | ➖ | | instagram reels | ✅ | ✅ | ✅ | ➖ | ➖ | @@ -38,7 +38,7 @@ this list is not final and keeps expanding over time. if support for a service y ### additional notes or features (per service) | service | notes or features | -| -------- | :----- | +| :-------- | :----- | | instagram | supports photos, videos, and stories. lets you pick what to save from multi-media posts. | | pinterest | supports videos and stories. | | reddit | supports gifs and videos. | From e72b412fedc887da41cbc472ecab5c4156c59d78 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 28 Oct 2023 19:01:52 +0600 Subject: [PATCH 04/19] finish instance selfhosting guide --- docs/run-an-instance.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md index 54a7d3b8..775f7f64 100644 --- a/docs/run-an-instance.md +++ b/docs/run-an-instance.md @@ -19,13 +19,18 @@ if you need help with installing docker, follow *only the first step* of these t i'm using `nano` in this example, it may not be available in your distro. you can use any other text editor. 3. copy and paste the [sample config from here](https://github.com/wukko/cobalt/blob/current/docs/examples/docker-compose.example.json) for either web or api instance (or both, if you wish) and edit it to your needs. - make sure to replace default URLs with yours or cobalt won't work correctly. + make sure to replace default URLs with your own or cobalt won't work correctly. -if you want your instance to support services that require authentication to view public content, create `cookies.json` file in the same directory as `docker-compose.yml`. example file for cookies file [can be found here](https://github.com/wukko/cobalt/blob/current/docs/examples/cookies.example.json). +4. finally, start the cobalt container (from cobalt directory): + ```sh + docker compose up -d + ``` -cobalt package will automatically update itself thanks to watchtower. +if you want your instance to support services that require authentication to view public content, create `cookies.json` file in the same directory as `docker-compose.yml`. example cookies file [can be found here](https://github.com/wukko/cobalt/blob/current/docs/examples/cookies.example.json). -it's highly recommended to use a reverse proxy (such as nginx) if you want your instance to face the public internet. look up tutorials for that online. +cobalt package will update automatically thanks to watchtower. + +it's highly recommended to use a reverse proxy (such as nginx) if you want your instance to face the public internet. look up tutorials online. ## using regular node.js (useful for local development) setup script installs all needed `npm` dependencies, but you have to install `node.js` *(version 18 or above)* and `git` yourself. From e4c105f2a2c83e41dd27f5904ec33b785bf9285b Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 28 Oct 2023 19:03:49 +0600 Subject: [PATCH 05/19] fix line duplication (due to skill issue) --- src/localization/languages/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json index 4e9132f3..62ef6a90 100644 --- a/src/localization/languages/en.json +++ b/src/localization/languages/en.json @@ -135,7 +135,7 @@ "KeyboardShortcutClear": "clear link input area", "KeyboardShortcutClosePopup": "close all popups", "CollapseLegal": "terms and ethics", - "FairUse": "cobalt is a tool for easing content downloads from internet and takes zero liability. you are responsible for what you download, how you use and distribute that content. please be mindful when using content of others and always credit original creators. fair use and credits benefit everyone.\n\nwhen used in education purposes (lecture, homework, etc) please attach the source link.\n\nfair use and credits benefit everyone.", + "FairUse": "cobalt is a tool for easing content downloads from internet and takes zero liability. you are responsible for what you download, how you use and distribute that content. please be mindful when using content of others and always credit original creators.\n\nwhen used in education purposes (lecture, homework, etc) please attach the source link.\n\nfair use and credits benefit everyone.", "UrgentFeatureUpdate71": "more supported services!", "UrgentThanks": "thank you for support!", "SettingsDisableMetadata": "don't add metadata", From 7d603f395e531b0308bc332dd0fe6d0e5136190f Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 28 Oct 2023 19:05:21 +0600 Subject: [PATCH 06/19] rename api doc to lowercase --- docs/{API.md => api.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{API.md => api.md} (100%) diff --git a/docs/API.md b/docs/api.md similarity index 100% rename from docs/API.md rename to docs/api.md From 575c4789f6417722773c7cf868f0804b2218de84 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 28 Oct 2023 19:11:35 +0600 Subject: [PATCH 07/19] docker compose file isnt a json you silly goose --- docs/run-an-instance.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md index 775f7f64..801895dc 100644 --- a/docs/run-an-instance.md +++ b/docs/run-an-instance.md @@ -18,7 +18,7 @@ if you need help with installing docker, follow *only the first step* of these t ``` i'm using `nano` in this example, it may not be available in your distro. you can use any other text editor. -3. copy and paste the [sample config from here](https://github.com/wukko/cobalt/blob/current/docs/examples/docker-compose.example.json) for either web or api instance (or both, if you wish) and edit it to your needs. +3. copy and paste the [sample config from here](https://github.com/wukko/cobalt/blob/current/docs/examples/docker-compose.example.yml) for either web or api instance (or both, if you wish) and edit it to your needs. make sure to replace default URLs with your own or cobalt won't work correctly. 4. finally, start the cobalt container (from cobalt directory): From 3c1fedc4ef41df1b2bfdee291faae62cdfb8b216 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 28 Oct 2023 20:22:42 +0600 Subject: [PATCH 08/19] troubleshooting: fixed a typo --- docs/troubleshooting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 6ad6335b..8241ef98 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -18,7 +18,7 @@ you can fix this issue by changing a single preference in `about:config`. ![screenshot showing about:config security warning that reads: "proceed with caution. changing advanced configuration preferences can impact firefox performance or security." lower there's a pre-checked checkbox that says: "warn me when i attempt to access these preferences". lowest element is a blue button that says "accept the risk and continue"](https://github.com/wukko/cobalt/assets/71202418/02328729-dbfe-4ea4-b2ca-7bcf1998c2ca) -3. search for `dom.events.asyncclipboard.readtext` +3. search for `dom.events.asyncClipboard.readText` ![screenshot showing "dom.events.asyncclipboard.readtext" entered into search on about:config page](https://github.com/wukko/cobalt/assets/71202418/7c7f7e3c-6a6a-40df-8436-277489e72e0b) From 9001d401dae64579c647cf0f107685b7a8186446 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Thu, 19 Oct 2023 20:36:05 +0000 Subject: [PATCH 09/19] stream: improve shutdown handling, minor clean up - try to close as many things as possible when shutting down - remove redundant (e.g. `exit` on process when listening for `close`) and straight up useless (`disconnect`) event listeners --- package.json | 1 + src/modules/stream/types.js | 130 ++++++++++++++++-------------------- 2 files changed, 58 insertions(+), 73 deletions(-) diff --git a/package.json b/package.json index 946476c2..6f376579 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ }, "homepage": "https://github.com/wukko/cobalt#readme", "dependencies": { + "abort-controller": "3.0.0", "content-disposition-header": "0.6.0", "cors": "^2.8.5", "dotenv": "^16.0.1", diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index e979e347..65df4200 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -2,40 +2,48 @@ import { spawn } from "child_process"; import ffmpeg from "ffmpeg-static"; import { ffmpegArgs, genericUserAgent } from "../config.js"; import { getThreads, metadataManager } from "../sub/utils.js"; -import { request } from 'undici'; +import { request } from "undici"; import { create as contentDisposition } from "content-disposition-header"; +import { AbortController } from "abort-controller" -function fail(res) { +function closeResponse(res) { if (!res.headersSent) res.sendStatus(500); return res.destroy(); } export async function streamDefault(streamInfo, res) { + const abortController = new AbortController(); + const shutdown = () => (abortController.abort(), closeResponse(res)); + try { - let format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1]; - res.setHeader('Content-disposition', contentDisposition(streamInfo.isAudioOnly ? `${streamInfo.filename}.${streamInfo.audioFormat}` : streamInfo.filename)); + const filename = streamInfo.isAudioOnly ? `${streamInfo.filename}.${streamInfo.audioFormat}` : streamInfo.filename; + res.setHeader('Content-disposition', contentDisposition(filename)); const { body: stream, headers } = await request(streamInfo.urls, { headers: { 'user-agent': genericUserAgent }, + signal: abortController.signal, maxRedirections: 16 }); res.setHeader('content-type', headers['content-type']); res.setHeader('content-length', headers['content-length']); - stream.pipe(res).on('error', () => fail(res)); - stream.on('error', () => fail(res)); - stream.on('aborted', () => fail(res)); - } catch (e) { - fail(res); + stream.on('error', shutdown) + .pipe(res).on('error', shutdown); + } catch { + shutdown(); } } -export async function streamLiveRender(streamInfo, res) { - try { - if (streamInfo.urls.length !== 2) return fail(res); - let { body: audio } = await request(streamInfo.urls[1], { - maxRedirections: 16 +export async function streamLiveRender(streamInfo, res) { + let abortController = new AbortController(), process; + const shutdown = () => (abortController.abort(), process?.kill(), closeResponse(res)); + + try { + if (streamInfo.urls.length !== 2) return shutdown(); + + const { body: audio } = await request(streamInfo.urls[1], { + maxRedirections: 16, signal: abortController.signal }); let format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1], @@ -51,58 +59,41 @@ export async function streamLiveRender(streamInfo, res) { args = args.concat(ffmpegArgs[format]); if (streamInfo.metadata) args = args.concat(metadataManager(streamInfo.metadata)); args.push('-f', format, 'pipe:4'); - let ffmpegProcess = spawn(ffmpeg, args, { + + process = spawn(ffmpeg, args, { windowsHide: true, stdio: [ 'inherit', 'inherit', 'inherit', 'pipe', 'pipe' ], }); + res.setHeader('Connection', 'keep-alive'); res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename)); - res.on('error', () => { - ffmpegProcess.kill(); - fail(res); - }); - ffmpegProcess.stdio[4].pipe(res).on('error', () => { - ffmpegProcess.kill(); - fail(res); - }); - audio.pipe(ffmpegProcess.stdio[3]).on('error', () => { - ffmpegProcess.kill(); - fail(res); - }); - - audio.on('error', () => { - ffmpegProcess.kill(); - fail(res); - }); - audio.on('aborted', () => { - ffmpegProcess.kill(); - fail(res); - }); - ffmpegProcess.on('disconnect', () => ffmpegProcess.kill()); - ffmpegProcess.on('close', () => ffmpegProcess.kill()); - ffmpegProcess.on('exit', () => ffmpegProcess.kill()); - res.on('finish', () => ffmpegProcess.kill()); - res.on('close', () => ffmpegProcess.kill()); - ffmpegProcess.on('error', () => { - ffmpegProcess.kill(); - fail(res); - }); + audio.on('error', shutdown) + .pipe(process.stdio[3]).on('error', shutdown); - } catch (e) { - fail(res); + process.stdio[4].pipe(res).on('error', shutdown); + process.on('close', shutdown); + res.on('finish', shutdown); + res.on('close', shutdown); + } catch { + shutdown(); } } + export function streamAudioOnly(streamInfo, res) { + let process; + const shutdown = () => (process?.kill(), closeResponse(res)); + try { let args = [ '-loglevel', '-8', '-threads', `${getThreads()}`, '-i', streamInfo.urls ] + if (streamInfo.metadata) { if (streamInfo.metadata.cover) { // currently corrupts the audio args.push('-i', streamInfo.metadata.cover, '-map', '0:a', '-map', '1:0') @@ -113,13 +104,14 @@ export function streamAudioOnly(streamInfo, res) { } else { args.push('-vn') } + let arg = streamInfo.copy ? ffmpegArgs["copy"] : ffmpegArgs["audio"]; args = args.concat(arg); if (ffmpegArgs[streamInfo.audioFormat]) args = args.concat(ffmpegArgs[streamInfo.audioFormat]); args.push('-f', streamInfo.audioFormat === "m4a" ? "ipod" : streamInfo.audioFormat, 'pipe:3'); - const ffmpegProcess = spawn(ffmpeg, args, { + process = spawn(ffmpeg, args, { windowsHide: true, stdio: [ 'inherit', 'inherit', 'inherit', @@ -128,22 +120,20 @@ export function streamAudioOnly(streamInfo, res) { }); res.setHeader('Connection', 'keep-alive'); res.setHeader('Content-Disposition', contentDisposition(`${streamInfo.filename}.${streamInfo.audioFormat}`)); - ffmpegProcess.stdio[3].pipe(res); - ffmpegProcess.on('disconnect', () => ffmpegProcess.kill()); - ffmpegProcess.on('close', () => ffmpegProcess.kill()); - ffmpegProcess.on('exit', () => ffmpegProcess.kill()); - res.on('finish', () => ffmpegProcess.kill()); - res.on('close', () => ffmpegProcess.kill()); - ffmpegProcess.on('error', () => { - ffmpegProcess.kill(); - fail(res); - }); - } catch (e) { - fail(res); + process.stdio[3].pipe(res); + process.on('close', shutdown); + res.on('finish', shutdown); + res.on('close', shutdown); + } catch { + shutdown(); } } + export function streamVideoOnly(streamInfo, res) { + let process; + const shutdown = () => (process?.kill(), closeResponse(res)); + try { let format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1], args = [ '-loglevel', '-8', @@ -155,7 +145,7 @@ export function streamVideoOnly(streamInfo, res) { if (streamInfo.service === "vimeo" || streamInfo.service === "rutube") args.push('-bsf:a', 'aac_adtstoasc'); if (format === "mp4") args.push('-movflags', 'faststart+frag_keyframe+empty_moov'); args.push('-f', format, 'pipe:3'); - const ffmpegProcess = spawn(ffmpeg, args, { + process = spawn(ffmpeg, args, { windowsHide: true, stdio: [ 'inherit', 'inherit', 'inherit', @@ -164,18 +154,12 @@ export function streamVideoOnly(streamInfo, res) { }); res.setHeader('Connection', 'keep-alive'); res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename)); - ffmpegProcess.stdio[3].pipe(res); - ffmpegProcess.on('disconnect', () => ffmpegProcess.kill()); - ffmpegProcess.on('close', () => ffmpegProcess.kill()); - ffmpegProcess.on('exit', () => ffmpegProcess.kill()); - res.on('finish', () => ffmpegProcess.kill()); - res.on('close', () => ffmpegProcess.kill()); - ffmpegProcess.on('error', () => { - ffmpegProcess.kill(); - fail(res); - }); - } catch (e) { - fail(res); + process.stdio[3].pipe(res); + process.on('close', shutdown); + res.on('finish', shutdown); + res.on('close', shutdown); + } catch { + shutdown(); } } From 1508a0bff48049b92bdb8f1262fefd87c6f2b435 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sun, 5 Nov 2023 22:07:34 +0000 Subject: [PATCH 10/19] stream: send SIGKILL after timeout in case the ffmpeg process decides to hang when SIGTERM'd --- src/modules/stream/types.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index 65df4200..5c7fbdb8 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -11,6 +11,14 @@ function closeResponse(res) { return res.destroy(); } +function killProcess(p) { + p?.kill(); + setTimeout(() => { + if (p?.exitCode === null) + p?.kill(9); + }, 5000); +} + export async function streamDefault(streamInfo, res) { const abortController = new AbortController(); const shutdown = () => (abortController.abort(), closeResponse(res)); @@ -37,7 +45,7 @@ export async function streamDefault(streamInfo, res) { export async function streamLiveRender(streamInfo, res) { let abortController = new AbortController(), process; - const shutdown = () => (abortController.abort(), process?.kill(), closeResponse(res)); + const shutdown = () => (abortController.abort(), killProcess(process), closeResponse(res)); try { if (streamInfo.urls.length !== 2) return shutdown(); @@ -85,7 +93,7 @@ export async function streamLiveRender(streamInfo, res) { export function streamAudioOnly(streamInfo, res) { let process; - const shutdown = () => (process?.kill(), closeResponse(res)); + const shutdown = () => (killProcess(process), closeResponse(res)); try { let args = [ @@ -132,7 +140,7 @@ export function streamAudioOnly(streamInfo, res) { export function streamVideoOnly(streamInfo, res) { let process; - const shutdown = () => (process?.kill(), closeResponse(res)); + const shutdown = () => (killProcess(process), closeResponse(res)); try { let format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1], args = [ From ed646b826db54ad3afd532eec51cdea29b090a08 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sun, 5 Nov 2023 22:09:54 +0000 Subject: [PATCH 11/19] stream: wrap abort controller in try-catch --- src/modules/stream/types.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index 5c7fbdb8..2c1becee 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -6,6 +6,10 @@ import { request } from "undici"; import { create as contentDisposition } from "content-disposition-header"; import { AbortController } from "abort-controller" +function closeRequest(controller) { + try { controller.abort() } catch {} +} + function closeResponse(res) { if (!res.headersSent) res.sendStatus(500); return res.destroy(); @@ -21,7 +25,7 @@ function killProcess(p) { export async function streamDefault(streamInfo, res) { const abortController = new AbortController(); - const shutdown = () => (abortController.abort(), closeResponse(res)); + const shutdown = () => (closeRequest(abortController), closeResponse(res)); try { const filename = streamInfo.isAudioOnly ? `${streamInfo.filename}.${streamInfo.audioFormat}` : streamInfo.filename; @@ -45,7 +49,7 @@ export async function streamDefault(streamInfo, res) { export async function streamLiveRender(streamInfo, res) { let abortController = new AbortController(), process; - const shutdown = () => (abortController.abort(), killProcess(process), closeResponse(res)); + const shutdown = () => (closeRequest(abortController), killProcess(process), closeResponse(res)); try { if (streamInfo.urls.length !== 2) return shutdown(); From aabde229ed39bab6701e39d545490fe9f20b0533 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sun, 5 Nov 2023 22:12:17 +0000 Subject: [PATCH 12/19] stream: generalize pipe event handling --- src/modules/stream/types.js | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index 2c1becee..8a1afd73 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -23,6 +23,16 @@ function killProcess(p) { }, 5000); } +function pipe(from, to, done) { + from.on('error', done) + .on('close', done); + + to.on('error', done) + .on('close', done); + + from.pipe(to); +} + export async function streamDefault(streamInfo, res) { const abortController = new AbortController(); const shutdown = () => (closeRequest(abortController), closeResponse(res)); @@ -40,8 +50,7 @@ export async function streamDefault(streamInfo, res) { res.setHeader('content-type', headers['content-type']); res.setHeader('content-length', headers['content-length']); - stream.on('error', shutdown) - .pipe(res).on('error', shutdown); + pipe(stream, res, shutdown); } catch { shutdown(); } @@ -83,13 +92,11 @@ export async function streamLiveRender(streamInfo, res) { res.setHeader('Connection', 'keep-alive'); res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename)); - audio.on('error', shutdown) - .pipe(process.stdio[3]).on('error', shutdown); + pipe(audio, process.stdio[3], shutdown); + pipe(process.stdio[4], res, shutdown); - process.stdio[4].pipe(res).on('error', shutdown); process.on('close', shutdown); res.on('finish', shutdown); - res.on('close', shutdown); } catch { shutdown(); } @@ -133,10 +140,8 @@ export function streamAudioOnly(streamInfo, res) { res.setHeader('Connection', 'keep-alive'); res.setHeader('Content-Disposition', contentDisposition(`${streamInfo.filename}.${streamInfo.audioFormat}`)); - process.stdio[3].pipe(res); - process.on('close', shutdown); + pipe(process.stdio[3], res, shutdown); res.on('finish', shutdown); - res.on('close', shutdown); } catch { shutdown(); } @@ -167,10 +172,10 @@ export function streamVideoOnly(streamInfo, res) { res.setHeader('Connection', 'keep-alive'); res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename)); - process.stdio[3].pipe(res); + pipe(process.stdio[3], res, shutdown); + process.on('close', shutdown); res.on('finish', shutdown); - res.on('close', shutdown); } catch { shutdown(); } From 58f7ed7827131aed59e4039e77606d095a5bf9bf Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sun, 5 Nov 2023 22:16:49 +0000 Subject: [PATCH 13/19] stream: use descriptive variables for i/o for better readability --- src/modules/stream/types.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index 8a1afd73..a9260986 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -88,12 +88,13 @@ export async function streamLiveRender(streamInfo, res) { 'pipe', 'pipe' ], }); + const [,,, audioInput, muxOutput] = process.stdio; res.setHeader('Connection', 'keep-alive'); res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename)); - pipe(audio, process.stdio[3], shutdown); - pipe(process.stdio[4], res, shutdown); + pipe(audio, audioInput, shutdown); + pipe(muxOutput, res, shutdown); process.on('close', shutdown); res.on('finish', shutdown); @@ -137,10 +138,11 @@ export function streamAudioOnly(streamInfo, res) { 'pipe' ], }); + const [,,, muxOutput] = process.stdio; res.setHeader('Connection', 'keep-alive'); res.setHeader('Content-Disposition', contentDisposition(`${streamInfo.filename}.${streamInfo.audioFormat}`)); - pipe(process.stdio[3], res, shutdown); + pipe(muxOutput, res, shutdown); res.on('finish', shutdown); } catch { shutdown(); @@ -169,10 +171,11 @@ export function streamVideoOnly(streamInfo, res) { 'pipe' ], }); + const [,,, muxOutput] = process.stdio; res.setHeader('Connection', 'keep-alive'); res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename)); - pipe(process.stdio[3], res, shutdown); + pipe(muxOutput, res, shutdown); process.on('close', shutdown); res.on('finish', shutdown); From 33072003bcb83a8f25ebb62143282817948ce7f2 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Mon, 6 Nov 2023 00:31:44 +0000 Subject: [PATCH 14/19] stream: use strings for signals instead of number hopefully a little more explanatory than "9" --- src/modules/stream/types.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index a9260986..38ebc684 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -16,10 +16,10 @@ function closeResponse(res) { } function killProcess(p) { - p?.kill(); + p?.kill('SIGTERM'); setTimeout(() => { if (p?.exitCode === null) - p?.kill(9); + p?.kill('SIGKILL'); }, 5000); } From 758bb8fef7f6322aee60edb9804164e0c3447bb4 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 6 Nov 2023 06:44:34 +0600 Subject: [PATCH 15/19] types: added comments --- src/modules/stream/types.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index 38ebc684..dc9579a9 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -16,9 +16,11 @@ function closeResponse(res) { } function killProcess(p) { + // ask the process to terminate itself gracefully p?.kill('SIGTERM'); setTimeout(() => { if (p?.exitCode === null) + // brutally murder the process if it didn't quit p?.kill('SIGKILL'); }, 5000); } From b01c9f3e54789de25166115b92cec4f66683f22d Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 6 Nov 2023 06:53:54 +0600 Subject: [PATCH 16/19] types: make streamVideoOnly more readable --- src/modules/stream/types.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index dc9579a9..bf6d7594 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -90,6 +90,7 @@ export async function streamLiveRender(streamInfo, res) { 'pipe', 'pipe' ], }); + const [,,, audioInput, muxOutput] = process.stdio; res.setHeader('Connection', 'keep-alive'); @@ -140,7 +141,9 @@ export function streamAudioOnly(streamInfo, res) { 'pipe' ], }); + const [,,, muxOutput] = process.stdio; + res.setHeader('Connection', 'keep-alive'); res.setHeader('Content-Disposition', contentDisposition(`${streamInfo.filename}.${streamInfo.audioFormat}`)); @@ -156,7 +159,7 @@ export function streamVideoOnly(streamInfo, res) { const shutdown = () => (killProcess(process), closeResponse(res)); try { - let format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1], args = [ + let args = [ '-loglevel', '-8', '-threads', `${getThreads()}`, '-i', streamInfo.urls, @@ -164,8 +167,11 @@ export function streamVideoOnly(streamInfo, res) { ] if (streamInfo.mute) args.push('-an'); if (streamInfo.service === "vimeo" || streamInfo.service === "rutube") args.push('-bsf:a', 'aac_adtstoasc'); + + let format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1]; if (format === "mp4") args.push('-movflags', 'faststart+frag_keyframe+empty_moov'); args.push('-f', format, 'pipe:3'); + process = spawn(ffmpeg, args, { windowsHide: true, stdio: [ @@ -173,7 +179,9 @@ export function streamVideoOnly(streamInfo, res) { 'pipe' ], }); + const [,,, muxOutput] = process.stdio; + res.setHeader('Connection', 'keep-alive'); res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename)); From ab0889ce4c74277e1524270909e595ad0f168bc2 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 6 Nov 2023 06:57:06 +0600 Subject: [PATCH 17/19] package: bump the version to 7.6.4 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2defa111..104bbe0d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "7.6.3", + "version": "7.6.4", "author": "wukko", "exports": "./src/cobalt.js", "type": "module", From 408f5e99f0e2e8141919591416d8cfb916be870c Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 6 Nov 2023 07:17:44 +0600 Subject: [PATCH 18/19] Revert "improve stream shutdown handling" --- package.json | 1 - src/modules/stream/types.js | 158 ++++++++++++++++-------------------- 2 files changed, 72 insertions(+), 87 deletions(-) diff --git a/package.json b/package.json index 104bbe0d..ff28a75d 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,6 @@ }, "homepage": "https://github.com/wukko/cobalt#readme", "dependencies": { - "abort-controller": "3.0.0", "content-disposition-header": "0.6.0", "cors": "^2.8.5", "dotenv": "^16.0.1", diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index bf6d7594..e979e347 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -2,71 +2,40 @@ import { spawn } from "child_process"; import ffmpeg from "ffmpeg-static"; import { ffmpegArgs, genericUserAgent } from "../config.js"; import { getThreads, metadataManager } from "../sub/utils.js"; -import { request } from "undici"; +import { request } from 'undici'; import { create as contentDisposition } from "content-disposition-header"; -import { AbortController } from "abort-controller" -function closeRequest(controller) { - try { controller.abort() } catch {} -} - -function closeResponse(res) { +function fail(res) { if (!res.headersSent) res.sendStatus(500); return res.destroy(); } -function killProcess(p) { - // ask the process to terminate itself gracefully - p?.kill('SIGTERM'); - setTimeout(() => { - if (p?.exitCode === null) - // brutally murder the process if it didn't quit - p?.kill('SIGKILL'); - }, 5000); -} - -function pipe(from, to, done) { - from.on('error', done) - .on('close', done); - - to.on('error', done) - .on('close', done); - - from.pipe(to); -} - export async function streamDefault(streamInfo, res) { - const abortController = new AbortController(); - const shutdown = () => (closeRequest(abortController), closeResponse(res)); - try { - const filename = streamInfo.isAudioOnly ? `${streamInfo.filename}.${streamInfo.audioFormat}` : streamInfo.filename; - res.setHeader('Content-disposition', contentDisposition(filename)); + let format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1]; + res.setHeader('Content-disposition', contentDisposition(streamInfo.isAudioOnly ? `${streamInfo.filename}.${streamInfo.audioFormat}` : streamInfo.filename)); const { body: stream, headers } = await request(streamInfo.urls, { headers: { 'user-agent': genericUserAgent }, - signal: abortController.signal, maxRedirections: 16 }); res.setHeader('content-type', headers['content-type']); res.setHeader('content-length', headers['content-length']); - pipe(stream, res, shutdown); - } catch { - shutdown(); + stream.pipe(res).on('error', () => fail(res)); + stream.on('error', () => fail(res)); + stream.on('aborted', () => fail(res)); + } catch (e) { + fail(res); } } - export async function streamLiveRender(streamInfo, res) { - let abortController = new AbortController(), process; - const shutdown = () => (closeRequest(abortController), killProcess(process), closeResponse(res)); - try { - if (streamInfo.urls.length !== 2) return shutdown(); + if (streamInfo.urls.length !== 2) return fail(res); - const { body: audio } = await request(streamInfo.urls[1], { - maxRedirections: 16, signal: abortController.signal + let { body: audio } = await request(streamInfo.urls[1], { + maxRedirections: 16 }); let format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1], @@ -82,41 +51,58 @@ export async function streamLiveRender(streamInfo, res) { args = args.concat(ffmpegArgs[format]); if (streamInfo.metadata) args = args.concat(metadataManager(streamInfo.metadata)); args.push('-f', format, 'pipe:4'); - - process = spawn(ffmpeg, args, { + let ffmpegProcess = spawn(ffmpeg, args, { windowsHide: true, stdio: [ 'inherit', 'inherit', 'inherit', 'pipe', 'pipe' ], }); - - const [,,, audioInput, muxOutput] = process.stdio; - res.setHeader('Connection', 'keep-alive'); res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename)); + res.on('error', () => { + ffmpegProcess.kill(); + fail(res); + }); + ffmpegProcess.stdio[4].pipe(res).on('error', () => { + ffmpegProcess.kill(); + fail(res); + }); + audio.pipe(ffmpegProcess.stdio[3]).on('error', () => { + ffmpegProcess.kill(); + fail(res); + }); + + audio.on('error', () => { + ffmpegProcess.kill(); + fail(res); + }); + audio.on('aborted', () => { + ffmpegProcess.kill(); + fail(res); + }); - pipe(audio, audioInput, shutdown); - pipe(muxOutput, res, shutdown); + ffmpegProcess.on('disconnect', () => ffmpegProcess.kill()); + ffmpegProcess.on('close', () => ffmpegProcess.kill()); + ffmpegProcess.on('exit', () => ffmpegProcess.kill()); + res.on('finish', () => ffmpegProcess.kill()); + res.on('close', () => ffmpegProcess.kill()); + ffmpegProcess.on('error', () => { + ffmpegProcess.kill(); + fail(res); + }); - process.on('close', shutdown); - res.on('finish', shutdown); - } catch { - shutdown(); + } catch (e) { + fail(res); } } - export function streamAudioOnly(streamInfo, res) { - let process; - const shutdown = () => (killProcess(process), closeResponse(res)); - try { let args = [ '-loglevel', '-8', '-threads', `${getThreads()}`, '-i', streamInfo.urls ] - if (streamInfo.metadata) { if (streamInfo.metadata.cover) { // currently corrupts the audio args.push('-i', streamInfo.metadata.cover, '-map', '0:a', '-map', '1:0') @@ -127,39 +113,39 @@ export function streamAudioOnly(streamInfo, res) { } else { args.push('-vn') } - let arg = streamInfo.copy ? ffmpegArgs["copy"] : ffmpegArgs["audio"]; args = args.concat(arg); if (ffmpegArgs[streamInfo.audioFormat]) args = args.concat(ffmpegArgs[streamInfo.audioFormat]); args.push('-f', streamInfo.audioFormat === "m4a" ? "ipod" : streamInfo.audioFormat, 'pipe:3'); - process = spawn(ffmpeg, args, { + const ffmpegProcess = spawn(ffmpeg, args, { windowsHide: true, stdio: [ 'inherit', 'inherit', 'inherit', 'pipe' ], }); - - const [,,, muxOutput] = process.stdio; - res.setHeader('Connection', 'keep-alive'); res.setHeader('Content-Disposition', contentDisposition(`${streamInfo.filename}.${streamInfo.audioFormat}`)); + ffmpegProcess.stdio[3].pipe(res); - pipe(muxOutput, res, shutdown); - res.on('finish', shutdown); - } catch { - shutdown(); + ffmpegProcess.on('disconnect', () => ffmpegProcess.kill()); + ffmpegProcess.on('close', () => ffmpegProcess.kill()); + ffmpegProcess.on('exit', () => ffmpegProcess.kill()); + res.on('finish', () => ffmpegProcess.kill()); + res.on('close', () => ffmpegProcess.kill()); + ffmpegProcess.on('error', () => { + ffmpegProcess.kill(); + fail(res); + }); + } catch (e) { + fail(res); } } - export function streamVideoOnly(streamInfo, res) { - let process; - const shutdown = () => (killProcess(process), closeResponse(res)); - try { - let args = [ + let format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1], args = [ '-loglevel', '-8', '-threads', `${getThreads()}`, '-i', streamInfo.urls, @@ -167,29 +153,29 @@ export function streamVideoOnly(streamInfo, res) { ] if (streamInfo.mute) args.push('-an'); if (streamInfo.service === "vimeo" || streamInfo.service === "rutube") args.push('-bsf:a', 'aac_adtstoasc'); - - let format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1]; if (format === "mp4") args.push('-movflags', 'faststart+frag_keyframe+empty_moov'); args.push('-f', format, 'pipe:3'); - - process = spawn(ffmpeg, args, { + const ffmpegProcess = spawn(ffmpeg, args, { windowsHide: true, stdio: [ 'inherit', 'inherit', 'inherit', 'pipe' ], }); - - const [,,, muxOutput] = process.stdio; - res.setHeader('Connection', 'keep-alive'); res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename)); + ffmpegProcess.stdio[3].pipe(res); - pipe(muxOutput, res, shutdown); - - process.on('close', shutdown); - res.on('finish', shutdown); - } catch { - shutdown(); + ffmpegProcess.on('disconnect', () => ffmpegProcess.kill()); + ffmpegProcess.on('close', () => ffmpegProcess.kill()); + ffmpegProcess.on('exit', () => ffmpegProcess.kill()); + res.on('finish', () => ffmpegProcess.kill()); + res.on('close', () => ffmpegProcess.kill()); + ffmpegProcess.on('error', () => { + ffmpegProcess.kill(); + fail(res); + }); + } catch (e) { + fail(res); } } From 7de8d723d2c113eb7c1b6144bb1e5abf3d0d2c74 Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 6 Nov 2023 07:19:28 +0600 Subject: [PATCH 19/19] go back to 7.6.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ff28a75d..81ba899d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "7.6.4", + "version": "7.6.3", "author": "wukko", "exports": "./src/cobalt.js", "type": "module",