diff --git a/.github/test.sh b/.github/test.sh index 3d175da9..90f7f19c 100755 --- a/.github/test.sh +++ b/.github/test.sh @@ -18,7 +18,7 @@ test_api() { -X POST \ -H "Accept: application/json" \ -H "Content-Type: application/json" \ - -d '{"url":"https://www.tiktok.com/@fatfatmillycat/video/7195741644585454894"}') + -d '{"url":"https://garfield-69.tumblr.com/post/696499862852780032","alwaysProxy":true}') echo "API_RESPONSE=$API_RESPONSE" STATUS=$(echo "$API_RESPONSE" | jq -r .status) @@ -46,6 +46,7 @@ setup_api() { } setup_web() { + pnpm run --prefix web check pnpm run --prefix web build } diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..47ea374e --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,93 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: + - '**' + pull_request: + branches: [ "main", "7" ] + schedule: + - cron: '33 7 * * 5' + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + permissions: + # required for all workflows + security-events: write + + # required to fetch internal or private CodeQL packs + packages: read + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + include: + - language: javascript-typescript + build-mode: none + # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - if: matrix.build-mode == 'manual' + shell: bash + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + echo ' make bootstrap' + echo ' make release' + exit 1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8dadd1d3..e25378b3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -51,7 +51,7 @@ jobs: uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64,linux/arm64,linux/arm/v7 + platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/test-services.yml b/.github/workflows/test-services.yml index 6291c736..77242cb8 100644 --- a/.github/workflows/test-services.yml +++ b/.github/workflows/test-services.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 - id: checkServices - run: pnpm i --frozen-lockfile && echo "service_list=$(node api/src/util/test-ci get-services)" >> "$GITHUB_OUTPUT" + run: pnpm i --frozen-lockfile && echo "service_list=$(node api/src/util/test get-services)" >> "$GITHUB_OUTPUT" test-services: needs: check-services @@ -30,4 +30,4 @@ jobs: steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 - - run: pnpm i --frozen-lockfile && node api/src/util/test-ci run-tests-for ${{ matrix.service }} \ No newline at end of file + - run: pnpm i --frozen-lockfile && node api/src/util/test run-tests-for ${{ matrix.service }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2668242b..379dae5a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,23 @@ if you're reading this, you are probably interested in contributing to cobalt, w this document serves as a guide to help you make contributions that we can merge into the cobalt codebase. ## translations -currently, we are **not accepting** translations of cobalt. this is because we are making significant changes to the frontend, and the currently used localization structure is being completely reworked. if this changes, this document will be updated. +we are currently accepting translations via the [i18n platform](https://i18n.imput.net). + +thank you for showing interest in making cobalt more accessible around the world, we really appreciate it! here are some guidelines for how a cobalt translation should look: + +- cobalt's writing style is informal. please do not use formal language, unless there is no other way to express the same idea of the original text in your language. +- all cobalt text is written in lowercase. this is a stylistic choice, please do not capitalize translated sentences. +- do not translate the name "cobalt", or "imput" +- you can translate "meowbalt" into whatever your language's equivalent of _meow_ is (e.g. _miaubalt_ in German) +- **please don't translate cobalt into languages which you are not experienced in.** we can use google translate ourselves, but we would prefer cobalt to be translated by humans, not computers. + +if your language does not exist on the translation platform yet, you can request to add it by adding it to any of cobalt's components (e.g. [here](https://i18n.imput.net/projects/cobalt/about/)). + +before translating a piece of text, check that no one has made a translation yet. pending translations are displayed in the **Suggestions** tab on the translate page. if someone already made a suggestion, and you think it's correct, you can upvote it! this helps us distinguish that a translation is correct. + +if no one has submitted a translation, or the submitted translation seems wrong to you, you can submit your translation by clicking the **Suggest** button for each individual string, which sends it off for human review. we will then check it to to ensure no malicious translations are submitted, and add it to cobalt. + +if any translation string's meaning seems unclear to you, please leave a comment on the *Comments* tab, and we will either add an explanation or a screenshot. ## adding features or support for services before putting in the effort to implement a feature, it's worth considering whether it would be appropriate to add it to cobalt. the cobalt api is built to assist people **only with downloading freely accessible content**. other functionality, such as: @@ -22,9 +38,9 @@ when contributing code to cobalt, there are a few guidelines in place to ensure ### clean commit messages internally, we use a format similar to [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) - the first part signifies which part of the code you are changing (the *scope*), and the second part explains the change. for inspiration on how to write appropriate commit titles, you can take a look at the [commit history](https://github.com/imputnet/cobalt/commits/). -the scope is not strictly defined, you can write whatever you find most fitting for the particular change. suppose you are changing a small part of a more significant part of the codebase. in that case, you can specify both the larger and smaller scopes in the commit message for clarity (e.g., if you were changing something in internal streams, the commit could be something like `stream/internal: fix object not being handled properly`). +the scope is not strictly defined, you can write whatever you find most fitting for the particular change. suppose you are changing a small part of a more significant part of the codebase. in that case, you can specify both the larger and smaller scopes in the commit message for clarity (e.g., if you were changing something in internal streams, the commit could be something like `api/stream: fix object not being handled properly`). -if you think a change deserves further explanation, we encourage you to write a short explanation in the commit message ([example](https://github.com/imputnet/cobalt/commit/d2e5b6542f71f3809ba94d56c26f382b5cb62762)), which will save both you and us time having to enquire about the change, and you explaining the reason behind it. +if you think a change deserves further explanation, we encourage you to write a short explanation in the commit message ([example](https://github.com/imputnet/cobalt/commit/31be60484de8eaf63bba8a4f508e16438aa7ba6e)), which will save both you and us time having to enquire about the change, and you explaining the reason behind it. if your contribution has uninformative commit titles, you may be asked to interactively rebase your branch and amend each commit to include a meaningful title. diff --git a/Dockerfile b/Dockerfile index e933c468..7bfc3dac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20-bullseye-slim AS base +FROM node:23-alpine AS base ENV PNPM_HOME="/pnpm" ENV PATH="$PNPM_HOME:$PATH" @@ -7,8 +7,7 @@ WORKDIR /app COPY . /app RUN corepack enable -RUN apt-get update && \ - apt-get install -y python3 build-essential +RUN apk add --no-cache python3 alpine-sdk RUN --mount=type=cache,id=pnpm,target=/pnpm/store \ pnpm install --prod --frozen-lockfile @@ -18,8 +17,10 @@ RUN pnpm deploy --filter=@imput/cobalt-api --prod /prod/api FROM base AS api WORKDIR /app -COPY --from=build /prod/api /app -COPY --from=build /app/.git /app/.git +COPY --from=build --chown=node:node /prod/api /app +COPY --from=build --chown=node:node /app/.git /app/.git + +USER node EXPOSE 9000 -CMD [ "node", "src/cobalt" ] \ No newline at end of file +CMD [ "node", "src/cobalt" ] diff --git a/README.md b/README.md index 54eacafe..9a5a05e7 100644 --- a/README.md +++ b/README.md @@ -14,111 +14,47 @@ đŸ’Ŧ community discord server +
- đŸĻ twitter/x + đŸĻ twitter + + + đŸĻ‹ bluesky


-cobalt is a media downloader that doesn't piss you off. it's fast, friendly, and doesn't have any bullshit that modern web is filled with: ***no ads, trackers, or paywalls***. +cobalt is a media downloader that doesn't piss you off. it's friendly, efficient, and doesn't have ads, trackers, paywalls or other nonsense. -paste the link, get the file, move on. it's that simple. just how it should be. +paste the link, get the file, move on. that simple, just how it should be. -### supported services -this list is not final and keeps expanding over time. if support for a service you want is missing, create an issue (or a pull request 👀). +### cobalt monorepo +this monorepo includes source code for api, frontend, and related packages: +- [api tree & readme](/api/) +- [web tree & readme](/web/) +- [packages tree](/packages/) -| service | video + audio | only audio | only video | metadata | rich file names | -| :-------- | :-----------: | :--------: | :--------: | :------: | :-------------: | -| bilibili | ✅ | ✅ | ✅ | ➖ | ➖ | -| bluesky | ✅ | ✅ | ✅ | ➖ | ➖ | -| dailymotion | ✅ | ✅ | ✅ | ✅ | ✅ | -| instagram | ✅ | ✅ | ✅ | ➖ | ➖ | -| facebook | ✅ | ❌ | ✅ | ➖ | ➖ | -| loom | ✅ | ❌ | ✅ | ✅ | ➖ | -| ok.ru | ✅ | ❌ | ✅ | ✅ | ✅ | -| pinterest | ✅ | ✅ | ✅ | ➖ | ➖ | -| reddit | ✅ | ✅ | ✅ | ❌ | ❌ | -| rutube | ✅ | ✅ | ✅ | ✅ | ✅ | -| snapchat | ✅ | ✅ | ✅ | ➖ | ➖ | -| soundcloud | ➖ | ✅ | ➖ | ✅ | ✅ | -| streamable | ✅ | ✅ | ✅ | ➖ | ➖ | -| threads posts | ✅ | ✅ | ✅ | ➖ | ➖ | -| tiktok | ✅ | ✅ | ✅ | ❌ | ❌ | -| tumblr | ✅ | ✅ | ✅ | ➖ | ➖ | -| twitch clips | ✅ | ✅ | ✅ | ✅ | ✅ | -| twitter/x | ✅ | ✅ | ✅ | ➖ | ➖ | -| vimeo | ✅ | ✅ | ✅ | ✅ | ✅ | -| vine | ✅ | ✅ | ✅ | ➖ | ➖ | -| vk videos & clips | ✅ | ❌ | ✅ | ✅ | ✅ | -| youtube | ✅ | ✅ | ✅ | ✅ | ✅ | +it also includes documentation in the [docs tree](/docs/): +- [cobalt api documentation](/docs/api.md) +- [how to run a cobalt instance](/docs/run-an-instance.md) +- [how to protect a cobalt instance](/docs/protect-an-instance.md) (recommended if you host a public instance) -| emoji | meaning | -| :-----: | :---------------------- | -| ✅ | supported | -| ➖ | impossible/unreasonable | -| ❌ | not supported | +### thank you +cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt). a part of our infrastructure is hosted on their network. we really appreciate their kindness and support! -### additional notes or features (per service) -| service | notes or features | -| :-------- | :----- | -| instagram | supports reels, photos, and videos. lets you pick what to save from multi-media posts. | -| facebook | supports public accessible videos content only. | -| pinterest | supports photos, gifs, videos and stories. | -| reddit | supports gifs and videos. | -| snapchat | supports spotlights and stories. lets you pick what to save from stories. | -| rutube | supports yappy & private links. | -| soundcloud | supports private links. | -| threads | supports photos and videos. lets you pick what to save from multi-media posts. | -| tiktok | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios. | -| twitter/x | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management. | -| vimeo | audio downloads are only available for dash. | -| youtube | supports videos, music, and shorts. 8K, 4K, HDR, VR, and high FPS videos. rich metadata & dubs. h264/av1/vp9 codecs. | +### ethics +cobalt is a tool that makes downloading public content easier. it takes **zero liability**. +the end user is responsible for what they download, how they use and distribute that content. +cobalt never caches any content, it [works like a fancy proxy](/api/src/stream/). -### partners -cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt), all main instances are currently hosted on their network :) +cobalt is in no way a piracy tool and cannot be used as such. +it can only download free & publicly accessible content. +same content can be downloaded via dev tools of any modern web browser. -### ethics and disclaimer -cobalt is a tool for easing content downloads from internet and takes ***zero liability***. you are responsible for what you download, how you use and distribute that content. please be mindful when using content of others and always credit original creators. fair use and credits benefit everyone. +### contributing +if you're considering contributing to cobalt, first of all, thank you! check the [contribution guidelines here](/CONTRIBUTING.md) before getting started, they'll help you do your best right away. -cobalt is ***NOT*** a piracy tool and cannot be used as such. it can only download free, publicly accessible content. such content can be easily downloaded through any browser's dev tools. pressing one button is easier, so i made a convenient, ad-less tool for such repeated actions. - -### cobalt license +### licenses for relevant licensing information, see the [api](api/README.md) and [web](web/README.md) READMEs. unless specified otherwise, the remainder of this repository is licensed under [AGPL-3.0](LICENSE). - -## acknowledgements -### ffmpeg -cobalt heavily relies on ffmpeg for converting and merging media files. it's an absolutely amazing piece of software offered for anyone for free, yet doesn't receive as much credit as it should. - -you can [support ffmpeg here](https://ffmpeg.org/donations.html)! - -#### ffmpeg-static -we use [ffmpeg-static](https://github.com/eugeneware/ffmpeg-static) to get binaries for ffmpeg depending on the platform. - -you can support the developer via various methods listed on their github page! (linked above) - -### youtube.js -cobalt relies on [youtube.js](https://github.com/LuanRT/YouTube.js) for interacting with the innertube api, it wouldn't have been possible without it. - -you can support the developer via various methods listed on their github page! (linked above) - -### many others -cobalt also depends on: - -- [content-disposition-header](https://www.npmjs.com/package/content-disposition-header) to simplify the provision of `content-disposition` headers. -- [cors](https://www.npmjs.com/package/cors) to manage cross-origin resource sharing within expressjs. -- [dotenv](https://www.npmjs.com/package/dotenv) to load environment variables from the `.env` file. -- [esbuild](https://www.npmjs.com/package/esbuild) to minify the frontend files. -- [express](https://www.npmjs.com/package/express) as the backbone of cobalt servers. -- [express-rate-limit](https://www.npmjs.com/package/express-rate-limit) to rate limit api endpoints. -- [hls-parser](https://www.npmjs.com/package/hls-parser) to parse `m3u8` playlists for certain services. -- [ipaddr.js](https://www.npmjs.com/package/ipaddr.js) to parse ip addresses (for rate limiting). -- [nanoid](https://www.npmjs.com/package/nanoid) to generate unique (temporary) identifiers for each requested stream. -- [node-cache](https://www.npmjs.com/package/node-cache) to cache stream info in server ram for a limited amount of time. -- [psl](https://www.npmjs.com/package/psl) as the domain name parser. -- [set-cookie-parser](https://www.npmjs.com/package/set-cookie-parser) to parse cookies that cobalt receives from certain services. -- [undici](https://www.npmjs.com/package/undici) for making http requests. -- [url-pattern](https://www.npmjs.com/package/url-pattern) to match provided links with supported patterns. - -...and many other packages that these packages rely on. diff --git a/api/README.md b/api/README.md index 5c281246..70d85de6 100644 --- a/api/README.md +++ b/api/README.md @@ -1,4 +1,64 @@ # cobalt api +this directory includes the source code for cobalt api. it's made with [express.js](https://www.npmjs.com/package/express) and love! + +## running your own instance +if you want to run your own instance for whatever purpose, [follow this guide](/docs/run-an-instance.md). +we recommend to use docker compose unless you intend to run cobalt for developing/debugging purposes. + +## accessing the api +there is currently no publicly available pre-hosted api. +we recommend [deploying your own instance](/docs/run-an-instance.md) if you wish to use the cobalt api. + +you can read [the api documentation here](/docs/api.md). + +## supported services +this list is not final and keeps expanding over time! +if the desired service isn't supported yet, feel free to create an appropriate issue (or a pull request 👀). + +| service | video + audio | only audio | only video | metadata | rich file names | +| :-------- | :-----------: | :--------: | :--------: | :------: | :-------------: | +| bilibili | ✅ | ✅ | ✅ | ➖ | ➖ | +| bluesky | ✅ | ✅ | ✅ | ➖ | ➖ | +| dailymotion | ✅ | ✅ | ✅ | ✅ | ✅ | +| instagram | ✅ | ✅ | ✅ | ➖ | ➖ | +| facebook | ✅ | ❌ | ✅ | ➖ | ➖ | +| loom | ✅ | ❌ | ✅ | ✅ | ➖ | +| ok.ru | ✅ | ❌ | ✅ | ✅ | ✅ | +| pinterest | ✅ | ✅ | ✅ | ➖ | ➖ | +| reddit | ✅ | ✅ | ✅ | ❌ | ❌ | +| rutube | ✅ | ✅ | ✅ | ✅ | ✅ | +| snapchat | ✅ | ✅ | ✅ | ➖ | ➖ | +| soundcloud | ➖ | ✅ | ➖ | ✅ | ✅ | +| streamable | ✅ | ✅ | ✅ | ➖ | ➖ | +| tiktok | ✅ | ✅ | ✅ | ❌ | ❌ | +| tumblr | ✅ | ✅ | ✅ | ➖ | ➖ | +| twitch clips | ✅ | ✅ | ✅ | ✅ | ✅ | +| twitter/x | ✅ | ✅ | ✅ | ➖ | ➖ | +| vimeo | ✅ | ✅ | ✅ | ✅ | ✅ | +| vk videos & clips | ✅ | ❌ | ✅ | ✅ | ✅ | +| xiaohongshu | ✅ | ✅ | ✅ | ➖ | ➖ | +| youtube | ✅ | ✅ | ✅ | ✅ | ✅ | + +| emoji | meaning | +| :-----: | :---------------------- | +| ✅ | supported | +| ➖ | unreasonable/impossible | +| ❌ | not supported | + +### additional notes or features (per service) +| service | notes or features | +| :-------- | :----- | +| instagram | supports reels, photos, and videos. lets you pick what to save from multi-media posts. | +| facebook | supports public accessible videos content only. | +| pinterest | supports photos, gifs, videos and stories. | +| reddit | supports gifs and videos. | +| snapchat | supports spotlights and stories. lets you pick what to save from stories. | +| rutube | supports yappy & private links. | +| soundcloud | supports private links. | +| tiktok | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios. | +| twitter/x | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management. | +| vimeo | audio downloads are only available for dash. | +| youtube | supports videos, music, and shorts. 8K, 4K, HDR, VR, and high FPS videos. rich metadata & dubs. h264/av1/vp9 codecs. | ## license cobalt api code is licensed under [AGPL-3.0](LICENSE). @@ -9,14 +69,35 @@ as long as you: - provide a link to the license and indicate if changes to the code were made, and - release the code under the **same license** -## running your own instance -if you want to run your own instance for whatever purpose, [follow this guide](/docs/run-an-instance.md). -it's *highly* recommended to use a docker compose method unless you run for developing/debugging purposes. +## open source acknowledgements +### ffmpeg +cobalt relies on ffmpeg for muxing and encoding media files. ffmpeg is absolutely spectacular and we're privileged to have an ability to use it for free, just like anyone else. we believe it should be way more recognized. -## accessing the api -currently, there is no publicly accessible main api. we plan on providing a public api for -cobalt 10 in some form in the future. we recommend deploying your own instance if you wish -to use the latest api. you can access [the documentation](/docs/api.md) for it here. +you can [support ffmpeg here](https://ffmpeg.org/donations.html)! -if you are looking for the documentation for the old (7.x) api, you can find -it [here](https://github.com/imputnet/cobalt/blob/7/docs/api.md) \ No newline at end of file +### youtube.js +cobalt relies on **[youtube.js](https://github.com/LuanRT/YouTube.js)** for interacting with youtube's innertube api, it wouldn't have been possible without this package. + +you can support the developer via various methods listed on their github page! +(linked above) + +### many others +cobalt-api also depends on: + +- **[content-disposition-header](https://www.npmjs.com/package/content-disposition-header)** to simplify the provision of `content-disposition` headers. +- **[cors](https://www.npmjs.com/package/cors)** to manage cross-origin resource sharing within expressjs. +- **[dotenv](https://www.npmjs.com/package/dotenv)** to load environment variables from the `.env` file. +- **[express](https://www.npmjs.com/package/express)** as the backbone of cobalt servers. +- **[express-rate-limit](https://www.npmjs.com/package/express-rate-limit)** to rate limit api endpoints. +- **[ffmpeg-static](https://www.npmjs.com/package/ffmpeg-static)** to get binaries for ffmpeg depending on the platform. +- **[hls-parser](https://www.npmjs.com/package/hls-parser)** to parse HLS playlists according to spec (very impressive stuff). +- **[ipaddr.js](https://www.npmjs.com/package/ipaddr.js)** to parse ip addresses (used for rate limiting). +- **[nanoid](https://www.npmjs.com/package/nanoid)** to generate unique identifiers for each requested tunnel. +- **[set-cookie-parser](https://www.npmjs.com/package/set-cookie-parser)** to parse cookies that cobalt receives from certain services. +- **[undici](https://www.npmjs.com/package/undici)** for making http requests. +- **[url-pattern](https://www.npmjs.com/package/url-pattern)** to match provided links with supported patterns. +- **[zod](https://www.npmjs.com/package/zod)** to lock down the api request schema. +- **[@datastructures-js/priority-queue](https://www.npmjs.com/package/@datastructures-js/priority-queue)** for sorting stream caches for future clean up (without redis). +- **[@imput/psl](https://www.npmjs.com/package/@imput/psl)** as the domain name parser, our fork of [psl](https://www.npmjs.com/package/psl). + +...and many other packages that these packages rely on. diff --git a/api/package.json b/api/package.json index 339d383f..829106c3 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "10.1.0", + "version": "10.6", "author": "imput", "exports": "./src/cobalt.js", "type": "module", @@ -10,9 +10,9 @@ }, "scripts": { "start": "node src/cobalt", - "setup": "node src/util/setup", "test": "node src/util/test", - "token:youtube": "node src/util/generate-youtube-tokens" + "token:youtube": "node src/util/generate-youtube-tokens", + "token:jwt": "node src/util/generate-jwt-secret" }, "repository": { "type": "git", @@ -24,26 +24,27 @@ }, "homepage": "https://github.com/imputnet/cobalt#readme", "dependencies": { + "@datastructures-js/priority-queue": "^6.3.1", + "@imput/psl": "^2.0.4", "@imput/version-info": "workspace:^", "content-disposition-header": "0.6.0", "cors": "^2.8.5", "dotenv": "^16.0.1", - "esbuild": "^0.14.51", - "express": "^4.21.0", - "express-rate-limit": "^6.3.0", + "express": "^4.21.2", + "express-rate-limit": "^7.4.1", "ffmpeg-static": "^5.1.0", "hls-parser": "^0.10.7", "ipaddr.js": "2.2.0", - "nanoid": "^4.0.2", - "node-cache": "^5.1.2", - "psl": "1.9.0", + "nanoid": "^5.0.9", "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", - "youtubei.js": "^10.5.0", + "youtubei.js": "^13.0.0", "zod": "^3.23.8" }, "optionalDependencies": { - "freebind": "^0.2.2" + "freebind": "^0.2.2", + "rate-limit-redis": "^4.2.0", + "redis": "^4.7.0" } } diff --git a/api/src/cobalt.js b/api/src/cobalt.js index c548e792..5cac208d 100644 --- a/api/src/cobalt.js +++ b/api/src/cobalt.js @@ -1,27 +1,32 @@ import "dotenv/config"; import express from "express"; +import cluster from "node:cluster"; -import path from 'path'; -import { fileURLToPath } from 'url'; +import path from "path"; +import { fileURLToPath } from "url"; -import { env } from "./config.js" -import { Bright, Green, Red } from "./misc/console-text.js"; +import { env, isCluster } from "./config.js" +import { Red } from "./misc/console-text.js"; +import { initCluster } from "./misc/cluster.js"; const app = express(); const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename).slice(0, -4); -app.disable('x-powered-by'); +app.disable("x-powered-by"); if (env.apiURL) { - const { runAPI } = await import('./core/api.js'); - runAPI(express, app, __dirname) + const { runAPI } = await import("./core/api.js"); + + if (isCluster) { + await initCluster(); + } + + runAPI(express, app, __dirname, cluster.isPrimary); } else { console.log( - Red(`cobalt wasn't configured yet or configuration is invalid.\n`) - + Bright(`please run the setup script to fix this: `) - + Green(`npm run setup`) + Red("API_URL env variable is missing, cobalt api can't start.") ) } diff --git a/api/src/config.js b/api/src/config.js index 3a28d7ce..191e8441 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -1,5 +1,6 @@ import { getVersion } from "@imput/version-info"; import { services } from "./processing/service-config.js"; +import { supportsReusePort } from "./misc/cluster.js"; const version = await getVersion(); @@ -13,6 +14,7 @@ const enabledServices = new Set(Object.keys(services).filter(e => { const env = { apiURL: process.env.API_URL || '', apiPort: process.env.API_PORT || 9000, + tunnelPort: process.env.API_PORT || 9000, listenAddress: process.env.API_LISTEN_ADDRESS, freebindCIDR: process.platform === 'linux' && process.env.FREEBIND_CIDR, @@ -45,7 +47,8 @@ const env = { apiKeyURL: process.env.API_KEY_URL && new URL(process.env.API_KEY_URL), authRequired: process.env.API_AUTH_REQUIRED === '1', - + redisURL: process.env.API_REDIS_URL, + instanceCount: (process.env.API_INSTANCE_COUNT && parseInt(process.env.API_INSTANCE_COUNT)) || 1, keyReloadInterval: 900, enabledServices, @@ -54,6 +57,23 @@ const env = { const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"; const cobaltUserAgent = `cobalt/${version} (+https://github.com/imputnet/cobalt)`; +export const setTunnelPort = (port) => env.tunnelPort = port; +export const isCluster = env.instanceCount > 1; + +if (env.sessionEnabled && env.jwtSecret.length < 16) { + throw new Error("JWT_SECRET env is too short (must be at least 16 characters long)"); +} + +if (env.instanceCount > 1 && !env.redisURL) { + throw new Error("API_REDIS_URL is required when API_INSTANCE_COUNT is >= 2"); +} else if (env.instanceCount > 1 && !await supportsReusePort()) { + console.error('API_INSTANCE_COUNT is not supported in your environment. to use this env, your node.js'); + console.error('version must be >= 23.1.0, and you must be running a recent enough version of linux'); + console.error('(or other OS that supports it). for more info, see `reusePort` option on'); + console.error('https://nodejs.org/api/net.html#serverlistenoptions-callback'); + throw new Error('SO_REUSEPORT is not supported'); +} + export { env, genericUserAgent, diff --git a/api/src/core/api.js b/api/src/core/api.js index b11d689a..e4d3dfcf 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -1,4 +1,5 @@ import cors from "cors"; +import http from "node:http"; import rateLimit from "express-rate-limit"; import { setGlobalDispatcher, ProxyAgent } from "undici"; import { getCommit, getBranch, getRemote, getVersion } from "@imput/version-info"; @@ -7,17 +8,18 @@ import jwt from "../security/jwt.js"; import stream from "../stream/stream.js"; import match from "../processing/match.js"; -import { env } from "../config.js"; +import { env, isCluster, setTunnelPort } from "../config.js"; import { extract } from "../processing/url.js"; -import { languageCode } from "../misc/utils.js"; -import { Bright, Cyan } from "../misc/console-text.js"; -import { generateHmac, generateSalt } from "../misc/crypto.js"; +import { Green, Bright, Cyan } from "../misc/console-text.js"; +import { hashHmac } from "../security/secrets.js"; +import { createStore } from "../store/redis-ratelimit.js"; import { randomizeCiphers } from "../misc/randomize-ciphers.js"; import { verifyTurnstileToken } from "../security/turnstile.js"; import { friendlyServiceName } from "../processing/service-alias.js"; import { verifyStream, getInternalStream } from "../stream/manage.js"; import { createResponse, normalizeRequest, getIP } from "../processing/request.js"; import * as APIKeys from "../security/api-keys.js"; +import * as Cookies from "../processing/cookie/manager.js"; const git = { branch: await getBranch(), @@ -29,7 +31,6 @@ const version = await getVersion(); const acceptRegex = /^application\/json(; charset=utf-8)?$/; -const ipSalt = generateSalt(); const corsConfig = env.corsWildcard ? {} : { origin: env.corsURL, optionsSuccessStatus: 200 @@ -40,7 +41,7 @@ const fail = (res, code, context) => { res.status(status).json(body); } -export const runAPI = (express, app, __dirname) => { +export const runAPI = async (express, app, __dirname, isPrimary = true) => { const startTime = new Date(); const startTimestamp = startTime.getTime(); @@ -68,31 +69,36 @@ export const runAPI = (express, app, __dirname) => { return res.status(status).json(body); }; + const keyGenerator = (req) => hashHmac(getIP(req), 'rate').toString('base64url'); + const sessionLimiter = rateLimit({ windowMs: 60000, - max: 10, - standardHeaders: true, + limit: 10, + standardHeaders: 'draft-6', legacyHeaders: false, - keyGenerator: req => generateHmac(getIP(req), ipSalt), + keyGenerator, + store: await createStore('session'), handler: handleRateExceeded }); const apiLimiter = rateLimit({ windowMs: env.rateLimitWindow * 1000, - max: (req) => req.rateLimitMax || env.rateLimitMax, - standardHeaders: true, + limit: (req) => req.rateLimitMax || env.rateLimitMax, + standardHeaders: 'draft-6', legacyHeaders: false, - keyGenerator: req => req.rateLimitKey || generateHmac(getIP(req), ipSalt), + keyGenerator: req => req.rateLimitKey || keyGenerator(req), + store: await createStore('api'), handler: handleRateExceeded }) const apiTunnelLimiter = rateLimit({ windowMs: env.rateLimitWindow * 1000, - max: (req) => req.rateLimitMax || env.rateLimitMax, - standardHeaders: true, + limit: (req) => req.rateLimitMax || env.rateLimitMax, + standardHeaders: 'draft-6', legacyHeaders: false, - keyGenerator: req => req.rateLimitKey || generateHmac(getIP(req), ipSalt), - handler: (req, res) => { + keyGenerator: req => req.rateLimitKey || keyGenerator(req), + store: await createStore('tunnel'), + handler: (_, res) => { return res.sendStatus(429) } }) @@ -158,19 +164,20 @@ export const runAPI = (express, app, __dirname) => { return fail(res, "error.api.auth.jwt.missing"); } - if (!authorization.startsWith("Bearer ") || authorization.length > 256) { + if (authorization.length >= 256) { return fail(res, "error.api.auth.jwt.invalid"); } - const verifyJwt = jwt.verify( - authorization.split("Bearer ", 2)[1] - ); - - if (!verifyJwt) { + const [ type, token, ...rest ] = authorization.split(" "); + if (!token || type.toLowerCase() !== 'bearer' || rest.length) { return fail(res, "error.api.auth.jwt.invalid"); } - req.rateLimitKey = generateHmac(req.header("Authorization"), ipSalt); + if (!jwt.verify(token)) { + return fail(res, "error.api.auth.jwt.invalid"); + } + + req.rateLimitKey = hashHmac(token, 'rate'); } catch { return fail(res, "error.api.generic"); } @@ -220,16 +227,11 @@ export const runAPI = (express, app, __dirname) => { app.post('/', async (req, res) => { const request = req.body; - const lang = languageCode(req); if (!request.url) { return fail(res, "error.api.link.missing"); } - if (request.youtubeDubBrowserLang) { - request.youtubeDubLang = lang; - } - const { success, data: normalizedRequest } = await normalizeRequest(request); if (!success) { return fail(res, "error.api.invalid_body"); @@ -261,7 +263,7 @@ export const runAPI = (express, app, __dirname) => { } }) - app.get('/tunnel', apiTunnelLimiter, (req, res) => { + app.get('/tunnel', apiTunnelLimiter, async (req, res) => { const id = String(req.query.id); const exp = String(req.query.exp); const sig = String(req.query.sig); @@ -280,7 +282,7 @@ export const runAPI = (express, app, __dirname) => { return res.status(200).end(); } - const streamInfo = verifyStream(id, sig, exp, sec, iv); + const streamInfo = await verifyStream(id, sig, exp, sec, iv); if (!streamInfo?.service) { return res.status(streamInfo.status).end(); } @@ -292,7 +294,7 @@ export const runAPI = (express, app, __dirname) => { return stream(res, streamInfo); }) - app.get('/itunnel', (req, res) => { + const itunnelHandler = (req, res) => { if (!req.ip.endsWith('127.0.0.1')) { return res.sendStatus(403); } @@ -311,8 +313,10 @@ export const runAPI = (express, app, __dirname) => { ...Object.entries(req.headers) ]); - return stream(res, { type: 'internal', ...streamInfo }); - }) + return stream(res, { type: 'internal', data: streamInfo }); + }; + + app.get('/itunnel', itunnelHandler); app.get('/', (_, res) => { res.type('json'); @@ -343,24 +347,48 @@ export const runAPI = (express, app, __dirname) => { setGlobalDispatcher(new ProxyAgent(env.externalProxy)) } - if (env.apiKeyURL) { - APIKeys.setup(env.apiKeyURL); + http.createServer(app).listen({ + port: env.apiPort, + host: env.listenAddress, + reusePort: env.instanceCount > 1 || undefined + }, () => { + if (isPrimary) { + console.log(`\n` + + Bright(Cyan("cobalt ")) + Bright("API ^Ī‰â ^") + "\n" + + + "~~~~~~\n" + + Bright("version: ") + version + "\n" + + Bright("commit: ") + git.commit + "\n" + + Bright("branch: ") + git.branch + "\n" + + Bright("remote: ") + git.remote + "\n" + + Bright("start time: ") + startTime.toUTCString() + "\n" + + "~~~~~~\n" + + + Bright("url: ") + Bright(Cyan(env.apiURL)) + "\n" + + Bright("port: ") + env.apiPort + "\n" + ); + } + + if (env.apiKeyURL) { + APIKeys.setup(env.apiKeyURL); + } + + if (env.cookiePath) { + Cookies.setup(env.cookiePath); + } + }); + + if (isCluster) { + const istreamer = express(); + istreamer.get('/itunnel', itunnelHandler); + const server = istreamer.listen({ + port: 0, + host: '127.0.0.1', + exclusive: true + }, () => { + const { port } = server.address(); + console.log(`${Green('[✓]')} cobalt sub-instance running on 127.0.0.1:${port}`); + setTunnelPort(port); + }); } - - app.listen(env.apiPort, env.listenAddress, () => { - console.log(`\n` + - Bright(Cyan("cobalt ")) + Bright("API ^Ī‰â ^") + "\n" + - - "~~~~~~\n" + - Bright("version: ") + version + "\n" + - Bright("commit: ") + git.commit + "\n" + - Bright("branch: ") + git.branch + "\n" + - Bright("remote: ") + git.remote + "\n" + - Bright("start time: ") + startTime.toUTCString() + "\n" + - "~~~~~~\n" + - - Bright("url: ") + Bright(Cyan(env.apiURL)) + "\n" + - Bright("port: ") + env.apiPort + "\n" - ) - }) } diff --git a/api/src/misc/cluster.js b/api/src/misc/cluster.js new file mode 100644 index 00000000..56664d15 --- /dev/null +++ b/api/src/misc/cluster.js @@ -0,0 +1,71 @@ +import cluster from "node:cluster"; +import net from "node:net"; +import { syncSecrets } from "../security/secrets.js"; +import { env, isCluster } from "../config.js"; + +export { isPrimary, isWorker } from "node:cluster"; + +export const supportsReusePort = async () => { + try { + await new Promise((resolve, reject) => { + const server = net.createServer().listen({ port: 0, reusePort: true }); + server.on('listening', () => server.close(resolve)); + server.on('error', (err) => (server.close(), reject(err))); + }); + + return true; + } catch { + return false; + } +} + +export const initCluster = async () => { + if (cluster.isPrimary) { + for (let i = 1; i < env.instanceCount; ++i) { + cluster.fork(); + } + } + + await syncSecrets(); +} + +export const broadcast = (message) => { + if (!isCluster || !cluster.isPrimary || !cluster.workers) { + return; + } + + for (const worker of Object.values(cluster.workers)) { + worker.send(message); + } +} + +export const send = (message) => { + if (!isCluster) { + return; + } + + if (cluster.isPrimary) { + return broadcast(message); + } else { + return process.send(message); + } +} + +export const waitFor = (key) => { + return new Promise(resolve => { + const listener = (message) => { + if (key in message) { + process.off('message', listener); + return resolve(message); + } + } + + process.on('message', listener); + }); +} + +export const mainOnMessage = (cb) => { + for (const worker of Object.values(cluster.workers)) { + worker.on('message', cb); + } +} diff --git a/api/src/misc/console-text.js b/api/src/misc/console-text.js index 6ce747d7..8df8fcc6 100644 --- a/api/src/misc/console-text.js +++ b/api/src/misc/console-text.js @@ -1,23 +1,36 @@ -function t(color, tt) { - return color + tt + "\x1b[0m" +const ANSI = { + RESET: "\x1b[0m", + BRIGHT: "\x1b[1m", + RED: "\x1b[31m", + GREEN: "\x1b[32m", + CYAN: "\x1b[36m", + YELLOW: "\x1b[93m" } -export function Bright(tt) { - return t("\x1b[1m", tt) +function wrap(color, text) { + if (!ANSI[color.toUpperCase()]) { + throw "invalid color"; + } + + return ANSI[color.toUpperCase()] + text + ANSI.RESET; } -export function Red(tt) { - return t("\x1b[31m", tt) +export function Bright(text) { + return wrap('bright', text); } -export function Green(tt) { - return t("\x1b[32m", tt) +export function Red(text) { + return wrap('red', text); } -export function Cyan(tt) { - return t("\x1b[36m", tt) +export function Green(text) { + return wrap('green', text); } -export function Yellow(tt) { - return t("\x1b[93m", tt) +export function Cyan(text) { + return wrap('cyan', text); +} + +export function Yellow(text) { + return wrap('yellow', text); } diff --git a/api/src/misc/crypto.js b/api/src/misc/crypto.js index 3a520156..e0f8858b 100644 --- a/api/src/misc/crypto.js +++ b/api/src/misc/crypto.js @@ -1,15 +1,7 @@ -import { createHmac, createCipheriv, createDecipheriv, randomBytes } from "crypto"; +import { createCipheriv, createDecipheriv } from "crypto"; const algorithm = "aes256"; -export function generateSalt() { - return randomBytes(64).toString('hex'); -} - -export function generateHmac(str, salt) { - return createHmac("sha256", salt).update(str).digest("base64url"); -} - export function encryptStream(plaintext, iv, secret) { const buff = Buffer.from(JSON.stringify(plaintext)); const key = Buffer.from(secret, "base64url"); diff --git a/api/src/misc/run-test.js b/api/src/misc/run-test.js index 10d19aef..21d97d04 100644 --- a/api/src/misc/run-test.js +++ b/api/src/misc/run-test.js @@ -41,4 +41,4 @@ export async function runTest(url, params, expect) { if (result.body.status === 'tunnel') { // TODO: stream testing } -} \ No newline at end of file +} diff --git a/api/src/misc/utils.js b/api/src/misc/utils.js index 34666d1c..331528d4 100644 --- a/api/src/misc/utils.js +++ b/api/src/misc/utils.js @@ -1,55 +1,16 @@ -const forbiddenCharsString = ['}', '{', '%', '>', '<', '^', ';', '`', '$', '"', "@", '=']; +const redirectStatuses = new Set([301, 302, 303, 307, 308]); -export function metadataManager(obj) { - const keys = Object.keys(obj); - const tags = [ - "album", - "copyright", - "title", - "artist", - "track", - "date" - ] - let commands = [] - - for (const i in keys) { - if (tags.includes(keys[i])) - commands.push('-metadata', `${keys[i]}=${obj[keys[i]]}`) - } - return commands; -} - -export function cleanString(string) { - for (const i in forbiddenCharsString) { - string = string.replaceAll("/", "_") - .replaceAll(forbiddenCharsString[i], '') - } - return string; -} -export function verifyLanguageCode(code) { - const langCode = String(code.slice(0, 2).toLowerCase()); - if (RegExp(/[a-z]{2}/).test(code)) { - return langCode - } - return "en" -} -export function languageCode(req) { - if (req.header('Accept-Language')) { - return verifyLanguageCode(req.header('Accept-Language')) - } - return "en" -} -export function cleanHTML(html) { - let clean = html.replace(/ {4}/g, ''); - clean = clean.replace(/\n/g, ''); - return clean -} - -export function getRedirectingURL(url) { - return fetch(url, { redirect: 'manual' }).then((r) => { - if ([301, 302, 303].includes(r.status) && r.headers.has('location')) +export async function getRedirectingURL(url, dispatcher) { + const location = await fetch(url, { + redirect: 'manual', + dispatcher, + }).then((r) => { + if (redirectStatuses.has(r.status) && r.headers.has('location')) { return r.headers.get('location'); + } }).catch(() => null); + + return location; } export function merge(a, b) { @@ -76,3 +37,7 @@ export function splitFilenameExtension(filename) { return [ parts.join('.'), ext ] } } + +export function zip(a, b) { + return a.map((value, i) => [ value, b[i] ]); +} diff --git a/api/src/processing/cookie/cookie.js b/api/src/processing/cookie/cookie.js index 6dd95fc3..1d9636d5 100644 --- a/api/src/processing/cookie/cookie.js +++ b/api/src/processing/cookie/cookie.js @@ -4,16 +4,24 @@ export default class Cookie { constructor(input) { assert(typeof input === 'object'); this._values = {}; - this.set(input) + + for (const [ k, v ] of Object.entries(input)) + this.set(k, v); } - set(values) { - Object.entries(values).forEach( - ([ key, value ]) => this._values[key] = value - ) + + set(key, value) { + const old = this._values[key]; + if (old === value) + return false; + + this._values[key] = value; + return true; } + unset(keys) { for (const key of keys) delete this._values[key] } + static fromString(str) { const obj = {}; @@ -25,12 +33,15 @@ export default class Cookie { return new Cookie(obj) } + toString() { return Object.entries(this._values).map(([ name, value ]) => `${name}=${value}`).join('; ') } + toJSON() { return this.toString() } + values() { return Object.freeze({ ...this._values }) } diff --git a/api/src/processing/cookie/manager.js b/api/src/processing/cookie/manager.js index 25bf9c90..25f41c2c 100644 --- a/api/src/processing/cookie/manager.js +++ b/api/src/processing/cookie/manager.js @@ -1,50 +1,145 @@ import Cookie from './cookie.js'; + import { readFile, writeFile } from 'fs/promises'; +import { Red, Green, Yellow } from '../../misc/console-text.js'; import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser'; -import { env } from '../../config.js'; +import * as cluster from '../../misc/cluster.js'; +import { isCluster } from '../../config.js'; -const WRITE_INTERVAL = 60000, - cookiePath = env.cookiePath, - COUNTER = Symbol('counter'); +const WRITE_INTERVAL = 60000; +const VALID_SERVICES = new Set([ + 'instagram', + 'instagram_bearer', + 'reddit', + 'twitter', + 'youtube', + 'youtube_oauth' +]); +const invalidCookies = {}; let cookies = {}, dirty = false, intervalId; -const setup = async () => { - try { - if (!cookiePath) return; - - cookies = await readFile(cookiePath, 'utf8'); - cookies = JSON.parse(cookies); - intervalId = setInterval(writeChanges, WRITE_INTERVAL) - } catch { /* no cookies for you */ } -} - -setup(); - -function writeChanges() { +function writeChanges(cookiePath) { if (!dirty) return; dirty = false; - writeFile(cookiePath, JSON.stringify(cookies, null, 4)).catch(() => { - clearInterval(intervalId) + const cookieData = JSON.stringify({ ...cookies, ...invalidCookies }, null, 4); + writeFile(cookiePath, cookieData).catch((e) => { + console.warn(`${Yellow('[!]')} failed writing updated cookies to storage`); + console.warn(e); + clearInterval(intervalId); + intervalId = null; }) } -export function getCookie(service) { - if (!cookies[service] || !cookies[service].length) return; +const setupMain = async (cookiePath) => { + try { + cookies = await readFile(cookiePath, 'utf8'); + cookies = JSON.parse(cookies); + for (const serviceName in cookies) { + if (!VALID_SERVICES.has(serviceName)) { + console.warn(`${Yellow('[!]')} ignoring unknown service in cookie file: ${serviceName}`); + } else if (!Array.isArray(cookies[serviceName])) { + console.warn(`${Yellow('[!]')} ${serviceName} in cookies file is not an array, ignoring it`); + } else if (cookies[serviceName].some(c => typeof c !== 'string')) { + console.warn(`${Yellow('[!]')} some cookie for ${serviceName} contains non-string value in cookies file`); + } else continue; - let n; - if (cookies[service][COUNTER] === undefined) { - n = cookies[service][COUNTER] = 0 - } else { - ++cookies[service][COUNTER] - n = (cookies[service][COUNTER] %= cookies[service].length) + invalidCookies[serviceName] = cookies[serviceName]; + delete cookies[serviceName]; + } + + if (!intervalId) { + intervalId = setInterval(() => writeChanges(cookiePath), WRITE_INTERVAL); + } + + cluster.broadcast({ cookies }); + + console.log(`${Green('[✓]')} cookies loaded successfully!`); + } catch (e) { + console.error(`${Yellow('[!]')} failed to load cookies.`); + console.error('error:', e); + } +} + +const setupWorker = async () => { + cookies = (await cluster.waitFor('cookies')).cookies; +} + +export const loadFromFile = async (path) => { + if (cluster.isPrimary) { + await setupMain(path); + } else if (cluster.isWorker) { + await setupWorker(); } - const cookie = cookies[service][n]; - if (typeof cookie === 'string') cookies[service][n] = Cookie.fromString(cookie); + dirty = false; +} - return cookies[service][n] +export const setup = async (path) => { + await loadFromFile(path); + + if (isCluster) { + const messageHandler = (message) => { + if ('cookieUpdate' in message) { + const { cookieUpdate } = message; + + if (cluster.isPrimary) { + dirty = true; + cluster.broadcast({ cookieUpdate }); + } + + const { service, idx, cookie } = cookieUpdate; + cookies[service][idx] = cookie; + } + } + + if (cluster.isPrimary) { + cluster.mainOnMessage(messageHandler); + } else { + process.on('message', messageHandler); + } + } +} + +export function getCookie(service) { + if (!VALID_SERVICES.has(service)) { + console.error( + `${Red('[!]')} ${service} not in allowed services list for cookies.` + + ' if adding a new cookie type, include it there.' + ); + return; + } + + if (!cookies[service] || !cookies[service].length) return; + + const idx = Math.floor(Math.random() * cookies[service].length); + + const cookie = cookies[service][idx]; + if (typeof cookie === 'string') { + cookies[service][idx] = Cookie.fromString(cookie); + } + + cookies[service][idx].meta = { service, idx }; + return cookies[service][idx]; +} + +export function updateCookieValues(cookie, values) { + let changed = false; + + for (const [ key, value ] of Object.entries(values)) { + changed = cookie.set(key, value) || changed; + } + + if (changed && cookie.meta) { + dirty = true; + if (isCluster) { + const message = { cookieUpdate: { ...cookie.meta, cookie } }; + cluster.send(message); + } + } + + return changed; } export function updateCookie(cookie, headers) { @@ -57,10 +152,6 @@ export function updateCookie(cookie, headers) { cookie.unset(parsed.filter(c => c.expires < new Date()).map(c => c.name)); parsed.filter(c => !c.expires || c.expires > new Date()).forEach(c => values[c.name] = c.value); + updateCookieValues(cookie, values); } - -export function updateCookieValues(cookie, values) { - cookie.set(values); - if (Object.keys(values).length) dirty = true -} diff --git a/api/src/processing/create-filename.js b/api/src/processing/create-filename.js index 216b15a4..911b5603 100644 --- a/api/src/processing/create-filename.js +++ b/api/src/processing/create-filename.js @@ -1,3 +1,13 @@ +const illegalCharacters = ['}', '{', '%', '>', '<', '^', ';', ':', '`', '$', '"', "@", '=', '?', '|', '*']; + +const sanitizeString = (string) => { + for (const i in illegalCharacters) { + string = string.replaceAll("/", "_").replaceAll("\\", "_") + .replaceAll(illegalCharacters[i], '') + } + return string; +} + export default (f, style, isAudioOnly, isAudioMuted) => { let filename = ''; @@ -5,7 +15,11 @@ export default (f, style, isAudioOnly, isAudioMuted) => { let classicTags = [...infoBase]; let basicTags = []; - const title = `${f.title} - ${f.author}`; + let title = sanitizeString(f.title); + + if (f.author) { + title += ` - ${sanitizeString(f.author)}`; + } if (f.resolution) { classicTags.push(f.resolution); diff --git a/api/src/processing/match-action.js b/api/src/processing/match-action.js index 28e6c62a..33d1f65a 100644 --- a/api/src/processing/match-action.js +++ b/api/src/processing/match-action.js @@ -9,13 +9,14 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab let action, responseType = "tunnel", defaultParams = { - u: r.urls, + url: r.urls, headers: r.headers, service: host, filename: r.filenameAttributes ? createFilename(r.filenameAttributes, filenameStyle, isAudioOnly, isAudioMuted) : r.filename, fileMetadata: !disableMetadata ? r.fileMetadata : false, - requestIP + requestIP, + originalRequest: r.originalRequest }, params = {}; @@ -24,7 +25,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab else if (r.isGif && twitterGif) action = "gif"; else if (isAudioOnly) action = "audio"; else if (isAudioMuted) action = "muteVideo"; - else if (r.isM3U8) action = "m3u8"; + else if (r.isHLS) action = "hls"; else action = "video"; if (action === "picker" || action === "audio") { @@ -47,27 +48,29 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab }); case "photo": - responseType = "redirect"; + params = { type: "proxy" }; break; case "gif": params = { type: "gif" }; break; - case "m3u8": + case "hls": params = { - type: Array.isArray(r.urls) ? "merge" : "remux" + type: Array.isArray(r.urls) ? "merge" : "remux", + isHLS: true, } break; case "muteVideo": let muteType = "mute"; - if (Array.isArray(r.urls) && !r.isM3U8) { + if (Array.isArray(r.urls) && !r.isHLS) { muteType = "proxy"; } params = { type: muteType, - u: Array.isArray(r.urls) ? r.urls[0] : r.urls + url: Array.isArray(r.urls) ? r.urls[0] : r.urls, + isHLS: r.isHLS } if (host === "reddit" && r.typeId === "redirect") { responseType = "redirect"; @@ -82,6 +85,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab case "threads": case "snapchat": case "bsky": + case "xiaohongshu": params = { picker: r.picker }; break; @@ -93,14 +97,15 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab } params = { picker: r.picker, - u: createStream({ + url: createStream({ service: "tiktok", type: audioStreamType, - u: r.urls, + url: r.urls, headers: r.headers, - filename: r.audioFilename, + filename: `${r.audioFilename}.${audioFormat}`, isAudioOnly: true, audioFormat, + audioBitrate }) } break; @@ -141,11 +146,11 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab case "ok": case "vk": case "tiktok": + case "xiaohongshu": params = { type: "proxy" }; break; case "facebook": - case "vine": case "instagram": case "tumblr": case "pinterest": @@ -162,7 +167,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab case "audio": if (audioIgnore.includes(host) || (host === "reddit" && r.typeId === "redirect")) { return createResponse("error", { - code: "error.api.fetch.empty" + code: "error.api.service.audio_not_supported" }) } @@ -186,18 +191,20 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab } } - if (r.isM3U8 || host === "vimeo") { + if (r.isHLS || host === "vimeo") { copy = false; processType = "audio"; } params = { type: processType, - u: Array.isArray(r.urls) ? r.urls[1] : r.urls, + url: Array.isArray(r.urls) ? r.urls[1] : r.urls, audioBitrate, audioCopy: copy, audioFormat, + + isHLS: r.isHLS, } break; } diff --git a/api/src/processing/match.js b/api/src/processing/match.js index 63c2df98..eb581984 100644 --- a/api/src/processing/match.js +++ b/api/src/processing/match.js @@ -19,7 +19,6 @@ import tumblr from "./services/tumblr.js"; import vimeo from "./services/vimeo.js"; import soundcloud from "./services/soundcloud.js"; import instagram from "./services/instagram.js"; -import vine from "./services/vine.js"; import pinterest from "./services/pinterest.js"; import streamable from "./services/streamable.js"; import twitch from "./services/twitch.js"; @@ -30,6 +29,7 @@ import loom from "./services/loom.js"; import threads from "./services/threads.js"; import facebook from "./services/facebook.js"; import bluesky from "./services/bluesky.js"; +import xiaohongshu from "./services/xiaohongshu.js"; let freebind; @@ -79,8 +79,9 @@ export default async function({ host, patternMatch, params }) { case "vk": r = await vk({ - userId: patternMatch.userId, + ownerId: patternMatch.ownerId, videoId: patternMatch.videoId, + accessKey: patternMatch.accessKey, quality: params.videoQuality }); break; @@ -98,13 +99,14 @@ export default async function({ host, patternMatch, params }) { case "youtube": let fetchInfo = { + dispatcher, id: patternMatch.id.slice(0, 11), quality: params.videoQuality, format: params.youtubeVideoCodec, isAudioOnly, isAudioMuted, dubLang: params.youtubeDubLang, - dispatcher + youtubeHLS: params.youtubeHLS, } if (url.hostname === "music.youtube.com" || isAudioOnly) { @@ -128,7 +130,7 @@ export default async function({ host, patternMatch, params }) { case "tiktok": r = await tiktok({ postId: patternMatch.postId, - id: patternMatch.id, + shortLink: patternMatch.shortLink, fullAudio: params.tiktokFullAudio, isAudioOnly, h265: params.tiktokH265, @@ -175,12 +177,6 @@ export default async function({ host, patternMatch, params }) { }) break; - case "vine": - r = await vine({ - id: patternMatch.id - }); - break; - case "pinterest": r = await pinterest({ id: patternMatch.id, @@ -249,7 +245,17 @@ export default async function({ host, patternMatch, params }) { case "bsky": r = await bluesky({ ...patternMatch, - alwaysProxy: params.alwaysProxy + alwaysProxy: params.alwaysProxy, + dispatcher + }); + break; + + case "xiaohongshu": + r = await xiaohongshu({ + ...patternMatch, + h265: params.tiktokH265, + isAudioOnly, + dispatcher, }); break; diff --git a/api/src/processing/request.js b/api/src/processing/request.js index 4287267c..d512bfe5 100644 --- a/api/src/processing/request.js +++ b/api/src/processing/request.js @@ -37,7 +37,7 @@ export function createResponse(responseType, responseData) { case "redirect": response = { - url: responseData?.u, + url: responseData?.url, filename: responseData?.filename } break; @@ -52,7 +52,7 @@ export function createResponse(responseType, responseData) { case "picker": response = { picker: responseData?.picker, - audio: responseData?.u, + audio: responseData?.url, audioFilename: responseData?.filename } break; diff --git a/api/src/processing/schema.js b/api/src/processing/schema.js index 172d480c..48d8b058 100644 --- a/api/src/processing/schema.js +++ b/api/src/processing/schema.js @@ -1,7 +1,5 @@ import { z } from "zod"; - import { normalizeURL } from "./url.js"; -import { verifyLanguageCode } from "../misc/utils.js"; export const apiSchema = z.object({ url: z.string() @@ -33,15 +31,21 @@ export const apiSchema = z.object({ ).default("1080"), youtubeDubLang: z.string() - .length(2) - .transform(verifyLanguageCode) + .min(2) + .max(8) + .regex(/^[0-9a-zA-Z\-]+$/) .optional(), + // TODO: remove this variable as it's no longer used + // and is kept for schema compatibility reasons + youtubeDubBrowserLang: z.boolean().default(false), + alwaysProxy: z.boolean().default(false), disableMetadata: z.boolean().default(false), tiktokFullAudio: z.boolean().default(false), tiktokH265: z.boolean().default(false), twitterGif: z.boolean().default(true), - youtubeDubBrowserLang: z.boolean().default(false), + + youtubeHLS: z.boolean().default(false), }) .strict(); diff --git a/api/src/processing/service-config.js b/api/src/processing/service-config.js index c48000d3..295d041d 100644 --- a/api/src/processing/service-config.js +++ b/api/src/processing/service-config.js @@ -1,7 +1,7 @@ import UrlPattern from "url-pattern"; export const audioIgnore = ["vk", "ok", "loom"]; -export const hlsExceptions = ["dailymotion", "vimeo", "rutube", "bsky"]; +export const hlsExceptions = ["dailymotion", "vimeo", "rutube", "bsky", "youtube"]; export const services = { bilibili: { @@ -30,7 +30,7 @@ export const services = { "reel/:id", "share/:shareType/:id" ], - subdomains: ["web"], + subdomains: ["web", "m"], altDomains: ["fb.watch"], }, instagram: { @@ -46,7 +46,7 @@ export const services = { altDomains: ["ddinstagram.com"], }, loom: { - patterns: ["share/:id"], + patterns: ["share/:id", "embed/:id"], }, ok: { patterns: [ @@ -115,10 +115,10 @@ export const services = { tiktok: { patterns: [ ":user/video/:postId", - ":id", - "t/:id", + ":shortLink", + "t/:shortLink", ":user/photo/:postId", - "v/:id.html" + "v/:postId.html" ], subdomains: ["vt", "vm", "m"], }, @@ -147,10 +147,6 @@ export const services = { subdomains: ["mobile"], altDomains: ["x.com", "vxtwitter.com", "fixvx.com"], }, - vine: { - patterns: ["v/:id"], - tld: "co", - }, vimeo: { patterns: [ ":id", @@ -162,11 +158,25 @@ export const services = { }, vk: { patterns: [ - "video:userId_:videoId", - "clip:userId_:videoId", - "clips:duplicate?z=clip:userId_:videoId" + "video:ownerId_:videoId", + "clip:ownerId_:videoId", + "clips:duplicate?z=clip:ownerId_:videoId", + "videos:duplicate?z=video:ownerId_:videoId", + "video:ownerId_:videoId_:accessKey", + "clip:ownerId_:videoId_:accessKey", + "clips:duplicate?z=clip:ownerId_:videoId_:accessKey", + "videos:duplicate?z=video:ownerId_:videoId_:accessKey" ], subdomains: ["m"], + altDomains: ["vkvideo.ru", "vk.ru"], + }, + xiaohongshu: { + patterns: [ + "explore/:id?xsec_token=:token", + "discovery/item/:id?xsec_token=:token", + "a/:shareId" + ], + altDomains: ["xhslink.com"], }, youtube: { patterns: [ diff --git a/api/src/processing/service-patterns.js b/api/src/processing/service-patterns.js index 723ab8a1..0d800e44 100644 --- a/api/src/processing/service-patterns.js +++ b/api/src/processing/service-patterns.js @@ -36,13 +36,13 @@ export const testers = { || pattern.shortLink?.length <= 16, "streamable": pattern => - pattern.id?.length === 6, + pattern.id?.length <= 6, "threads": pattern => pattern.user?.length <= 33 && pattern.id?.length <= 32, "tiktok": pattern => - pattern.postId?.length <= 21 || pattern.id?.length <= 13, + pattern.postId?.length <= 21 || pattern.shortLink?.length <= 13, "tumblr": pattern => pattern.id?.length < 21 @@ -58,11 +58,9 @@ export const testers = { pattern.id?.length <= 11 && (!pattern.password || pattern.password.length < 16), - "vine": pattern => - pattern.id?.length <= 12, - "vk": pattern => - pattern.userId?.length <= 10 && pattern.videoId?.length <= 10, + (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10) || + (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10 && pattern.videoId?.accessKey <= 18), "youtube": pattern => pattern.id?.length <= 11, @@ -76,4 +74,8 @@ export const testers = { "bsky": pattern => pattern.user?.length <= 128 && pattern.post?.length <= 128, + + "xiaohongshu": pattern => + pattern.id?.length <= 24 && pattern.token?.length <= 64 + || pattern.shareId?.length <= 12, } diff --git a/api/src/processing/services/bluesky.js b/api/src/processing/services/bluesky.js index 5f5cbcec..598e9739 100644 --- a/api/src/processing/services/bluesky.js +++ b/api/src/processing/services/bluesky.js @@ -2,12 +2,19 @@ import HLS from "hls-parser"; import { cobaltUserAgent } from "../../config.js"; import { createStream } from "../../stream/manage.js"; -const extractVideo = async ({ media, filename }) => { - const urlMasterHLS = media?.playlist; - if (!urlMasterHLS) return { error: "fetch.empty" }; - if (!urlMasterHLS.startsWith("https://video.bsky.app/")) return { error: "fetch.empty" }; +const extractVideo = async ({ media, filename, dispatcher }) => { + let urlMasterHLS = media?.playlist; - const masterHLS = await fetch(urlMasterHLS) + if (!urlMasterHLS || !urlMasterHLS.startsWith("https://video.bsky.app/")) { + return { error: "fetch.empty" }; + } + + urlMasterHLS = urlMasterHLS.replace( + "video.bsky.app/watch/", + "video.cdn.bsky.app/hls/" + ); + + const masterHLS = await fetch(urlMasterHLS, { dispatcher }) .then(r => { if (r.status !== 200) return; return r.text(); @@ -26,7 +33,7 @@ const extractVideo = async ({ media, filename }) => { urls: videoURL, filename: `${filename}.mp4`, audioFilename: `${filename}_audio`, - isM3U8: true, + isHLS: true, } } @@ -48,7 +55,7 @@ const extractImages = ({ getPost, filename, alwaysProxy }) => { let proxiedImage = createStream({ service: "bluesky", type: "proxy", - u: url, + url, filename: `${filename}_${i + 1}.jpg`, }); @@ -64,7 +71,25 @@ const extractImages = ({ getPost, filename, alwaysProxy }) => { return { picker }; } -export default async function ({ user, post, alwaysProxy }) { +const extractGif = ({ url, filename }) => { + const gifUrl = new URL(url); + + if (!gifUrl || gifUrl.hostname !== "media.tenor.com") { + return { error: "fetch.empty" }; + } + + // remove downscaling params from gif url + // such as "?hh=498&ww=498" + gifUrl.search = ""; + + return { + urls: gifUrl, + isPhoto: true, + filename: `${filename}.gif`, + } +} + +export default async function ({ user, post, alwaysProxy, dispatcher }) { const apiEndpoint = new URL("https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread?depth=0&parentHeight=0"); apiEndpoint.searchParams.set( "uri", @@ -73,8 +98,9 @@ export default async function ({ user, post, alwaysProxy }) { const getPost = await fetch(apiEndpoint, { headers: { - "user-agent": cobaltUserAgent - } + "user-agent": cobaltUserAgent, + }, + dispatcher }).then(r => r.json()).catch(() => {}); if (!getPost) return { error: "fetch.empty" }; @@ -87,29 +113,44 @@ export default async function ({ user, post, alwaysProxy }) { case "InvalidRequest": return { error: "link.unsupported" }; default: - return { error: "fetch.empty" }; + return { error: "content.post.unavailable" }; } } const embedType = getPost?.thread?.post?.embed?.$type; const filename = `bluesky_${user}_${post}`; - if (embedType === "app.bsky.embed.video#view") { - return extractVideo({ - media: getPost.thread?.post?.embed, - filename, - }) - } + switch (embedType) { + case "app.bsky.embed.video#view": + return extractVideo({ + media: getPost.thread?.post?.embed, + filename, + }); - if (embedType === "app.bsky.embed.recordWithMedia#view") { - return extractVideo({ - media: getPost.thread?.post?.embed?.media, - filename, - }) - } + case "app.bsky.embed.images#view": + return extractImages({ + getPost, + filename, + alwaysProxy + }); - if (embedType === "app.bsky.embed.images#view") { - return extractImages({ getPost, filename, alwaysProxy }); + case "app.bsky.embed.external#view": + return extractGif({ + url: getPost?.thread?.post?.embed?.external?.uri, + filename, + }); + + case "app.bsky.embed.recordWithMedia#view": + if (getPost?.thread?.post?.embed?.media?.$type === "app.bsky.embed.external#view") { + return extractGif({ + url: getPost?.thread?.post?.embed?.media?.external?.uri, + filename, + }); + } + return extractVideo({ + media: getPost.thread?.post?.embed?.media, + filename, + }); } return { error: "fetch.empty" }; diff --git a/api/src/processing/services/dailymotion.js b/api/src/processing/services/dailymotion.js index a403a16b..a30a8bc7 100644 --- a/api/src/processing/services/dailymotion.js +++ b/api/src/processing/services/dailymotion.js @@ -92,7 +92,7 @@ export default async function({ id }) { return { urls: bestQuality.uri, - isM3U8: true, + isHLS: true, filenameAttributes: { service: 'dailymotion', id: media.xid, diff --git a/api/src/processing/services/instagram.js b/api/src/processing/services/instagram.js index 17e78ec4..d9a646aa 100644 --- a/api/src/processing/services/instagram.js +++ b/api/src/processing/services/instagram.js @@ -177,7 +177,7 @@ export default function(obj) { if (alwaysProxy) proxyFile = createStream({ service: "instagram", type: "proxy", - u: url, + url, filename: `instagram_${id}_${i + 1}.${itemExt}` }); @@ -189,7 +189,7 @@ export default function(obj) { thumb: createStream({ service: "instagram", type: "proxy", - u: e.node?.display_url, + url: e.node?.display_url, filename: `instagram_${id}_${i + 1}.jpg` }) } @@ -230,7 +230,7 @@ export default function(obj) { if (alwaysProxy) proxyFile = createStream({ service: "instagram", type: "proxy", - u: url, + url, filename: `instagram_${id}_${i + 1}.${itemExt}` }); @@ -242,7 +242,7 @@ export default function(obj) { thumb: createStream({ service: "instagram", type: "proxy", - u: imageUrl, + url: imageUrl, filename: `instagram_${id}_${i + 1}.jpg` }) } @@ -266,6 +266,7 @@ export default function(obj) { } async function getPost(id, alwaysProxy) { + const hasData = (data) => data && data.gql_data !== null; let data, result; try { const cookie = getCookie('instagram'); @@ -282,16 +283,16 @@ export default function(obj) { if (media_id && token) data = await requestMobileApi(media_id, { token }); // mobile api (no cookie, cookie) - if (media_id && !data) data = await requestMobileApi(media_id); - if (media_id && cookie && !data) data = await requestMobileApi(media_id, { cookie }); + if (media_id && !hasData(data)) data = await requestMobileApi(media_id); + if (media_id && cookie && !hasData(data)) data = await requestMobileApi(media_id, { cookie }); // html embed (no cookie, cookie) - if (!data) data = await requestHTML(id); - if (!data && cookie) data = await requestHTML(id, cookie); + if (!hasData(data)) data = await requestHTML(id); + if (!hasData(data) && cookie) data = await requestHTML(id, cookie); // web app graphql api (no cookie, cookie) - if (!data) data = await requestGQL(id); - if (!data && cookie) data = await requestGQL(id, cookie); + if (!hasData(data)) data = await requestGQL(id); + if (!hasData(data) && cookie) data = await requestGQL(id, cookie); } catch {} if (!data) return { error: "fetch.fail" }; diff --git a/api/src/processing/services/ok.js b/api/src/processing/services/ok.js index 2fb6082d..10fb785b 100644 --- a/api/src/processing/services/ok.js +++ b/api/src/processing/services/ok.js @@ -1,5 +1,4 @@ import { genericUserAgent, env } from "../../config.js"; -import { cleanString } from "../../misc/utils.js"; const resolutions = { "ultra": "2160", @@ -44,8 +43,8 @@ export default async function(o) { let bestVideo = videos.find(v => resolutions[v.name] === quality) || videos[videos.length - 1]; let fileMetadata = { - title: cleanString(videoData.movie.title.trim()), - author: cleanString((videoData.author?.name || videoData.compilationTitle).trim()), + title: videoData.movie.title.trim(), + author: (videoData.author?.name || videoData.compilationTitle).trim(), } if (bestVideo) return { diff --git a/api/src/processing/services/rutube.js b/api/src/processing/services/rutube.js index 4305241a..5b502452 100644 --- a/api/src/processing/services/rutube.js +++ b/api/src/processing/services/rutube.js @@ -1,7 +1,5 @@ import HLS from "hls-parser"; - import { env } from "../../config.js"; -import { cleanString } from "../../misc/utils.js"; async function requestJSON(url) { try { @@ -35,6 +33,10 @@ export default async function(obj) { const play = await requestJSON(requestURL); if (!play) return { error: "fetch.fail" }; + if (play.detail?.type === "blocking_rule") { + return { error: "content.video.region" }; + } + if (play.detail || !play.video_balancer) return { error: "fetch.empty" }; if (play.live_streams?.hls) return { error: "content.video.live" }; @@ -59,13 +61,13 @@ export default async function(obj) { }); const fileMetadata = { - title: cleanString(play.title.trim()), - artist: cleanString(play.author.name.trim()), + title: play.title.trim(), + artist: play.author.name.trim(), } return { urls: matchingQuality.uri, - isM3U8: true, + isHLS: true, filenameAttributes: { service: "rutube", id: obj.id, diff --git a/api/src/processing/services/snapchat.js b/api/src/processing/services/snapchat.js index acb6813a..4c62a5ff 100644 --- a/api/src/processing/services/snapchat.js +++ b/api/src/processing/services/snapchat.js @@ -73,7 +73,7 @@ async function getStory(username, storyId, alwaysProxy) { const proxy = createStream({ service: "snapchat", type: "proxy", - u: snapUrl, + url: snapUrl, filename: `snapchat_${username}_${snap.timestampInSec.value}.${snapExt}`, }); @@ -81,7 +81,7 @@ async function getStory(username, storyId, alwaysProxy) { if (snapType === "video") thumbProxy = createStream({ service: "snapchat", type: "proxy", - u: snap.snapUrls.mediaPreviewUrl.value, + url: snap.snapUrls.mediaPreviewUrl.value, }); if (alwaysProxy) snapUrl = proxy; diff --git a/api/src/processing/services/soundcloud.js b/api/src/processing/services/soundcloud.js index 394f7dfe..ad535479 100644 --- a/api/src/processing/services/soundcloud.js +++ b/api/src/processing/services/soundcloud.js @@ -1,5 +1,4 @@ import { env } from "../../config.js"; -import { cleanString } from "../../misc/utils.js"; const cachedID = { version: '', @@ -63,7 +62,17 @@ export default async function(obj) { if (!json) return { error: "fetch.fail" }; - if (!json.media.transcodings) return { error: "fetch.empty" }; + if (json?.policy === "BLOCK") { + return { error: "content.region" }; + } + + if (json?.policy === "SNIP") { + return { error: "content.paid" }; + } + + if (!json?.media?.transcodings || !json?.media?.transcodings.length === 0) { + return { error: "fetch.empty" }; + } let bestAudio = "opus", selectedStream = json.media.transcodings.find(v => v.preset === "opus_0_0"), @@ -75,6 +84,10 @@ export default async function(obj) { bestAudio = "mp3" } + if (!selectedStream) { + return { error: "fetch.empty" }; + } + let fileUrlBase = selectedStream.url; let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`; @@ -91,8 +104,8 @@ export default async function(obj) { if (!file) return { error: "fetch.empty" }; let fileMetadata = { - title: cleanString(json.title.trim()), - artist: cleanString(json.user.username.trim()), + title: json.title.trim(), + artist: json.user.username.trim(), } return { diff --git a/api/src/processing/services/tiktok.js b/api/src/processing/services/tiktok.js index 3c70e033..6fec01d8 100644 --- a/api/src/processing/services/tiktok.js +++ b/api/src/processing/services/tiktok.js @@ -12,7 +12,7 @@ export default async function(obj) { let postId = obj.postId; if (!postId) { - let html = await fetch(`${shortDomain}${obj.id}`, { + let html = await fetch(`${shortDomain}${obj.shortLink}`, { redirect: "manual", headers: { "user-agent": genericUserAgent.split(' Chrome/1')[0] @@ -24,13 +24,13 @@ export default async function(obj) { if (html.startsWith('')[1] - .split('')[0] - const data = JSON.parse(json) - detail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"] + .split('')[0]; + + const data = JSON.parse(json); + const videoDetail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]; + + if (!videoDetail) throw "no video detail found"; + + // status_deleted or etc + if (videoDetail.statusMsg) { + return { error: "content.post.unavailable"}; + } + + detail = videoDetail?.itemInfo?.itemStruct; } catch { return { error: "fetch.fail" }; } + if (detail.isContentClassified) { + return { error: "content.post.age" }; + } + + if (!detail.author) { + return { error: "fetch.empty" }; + } + let video, videoFilename, audioFilename, audio, images, - filenameBase = `tiktok_${detail.author.uniqueId}_${postId}`, + filenameBase = `tiktok_${detail.author?.uniqueId}_${postId}`, bestAudio; // will get defaulted to m4a later on in match-action images = detail.imagePost?.images; - let playAddr = detail.video.playAddr; + let playAddr = detail.video?.playAddr; + if (obj.h265) { const h265PlayAddr = detail?.video?.bitrateInfo?.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0] playAddr = h265PlayAddr || playAddr @@ -102,7 +121,7 @@ export default async function(obj) { if (obj.alwaysProxy) url = createStream({ service: "tiktok", type: "proxy", - u: url, + url, filename: `${filenameBase}_photo_${i + 1}.jpg` }) diff --git a/api/src/processing/services/tumblr.js b/api/src/processing/services/tumblr.js index b361b98c..2b8aa4ce 100644 --- a/api/src/processing/services/tumblr.js +++ b/api/src/processing/services/tumblr.js @@ -1,4 +1,4 @@ -import psl from "psl"; +import psl from "@imput/psl"; const API_KEY = 'jrsCWX1XDuVxAFO4GkK147syAoN8BJZ5voz8tS80bPcj26Vc5Z'; const API_BASE = 'https://api-http2.tumblr.com'; diff --git a/api/src/processing/services/twitch.js b/api/src/processing/services/twitch.js index ac85fbcf..4b9d4551 100644 --- a/api/src/processing/services/twitch.js +++ b/api/src/processing/services/twitch.js @@ -1,5 +1,4 @@ import { env } from "../../config.js"; -import { cleanString } from '../../misc/utils.js'; const gqlURL = "https://gql.twitch.tv/gql"; const clientIdHead = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" }; @@ -73,13 +72,13 @@ export default async function (obj) { token: req_token[0].data.clip.playbackAccessToken.value })}`, fileMetadata: { - title: cleanString(clipMetadata.title.trim()), + title: clipMetadata.title.trim(), artist: `Twitch Clip by @${clipMetadata.broadcaster.login}, clipped by @${clipMetadata.curator.login}`, }, filenameAttributes: { service: "twitch", id: clipMetadata.id, - title: cleanString(clipMetadata.title.trim()), + title: clipMetadata.title.trim(), author: `${clipMetadata.broadcaster.login}, clipped by ${clipMetadata.curator.login}`, qualityLabel: `${format.quality}p`, extension: 'mp4' diff --git a/api/src/processing/services/twitter.js b/api/src/processing/services/twitter.js index 18866b49..b4a1d557 100644 --- a/api/src/processing/services/twitter.js +++ b/api/src/processing/services/twitter.js @@ -159,10 +159,10 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { const getFileExt = (url) => new URL(url).pathname.split(".", 2)[1]; - const proxyMedia = (u, filename) => createStream({ + const proxyMedia = (url, filename) => createStream({ service: "twitter", type: "proxy", - u, filename, + url, filename, }) switch (media?.length) { @@ -208,7 +208,7 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { let url = bestQuality(content.video_info.variants); const shouldRenderGif = content.type === "animated_gif" && toGif; - const videoFilename = `twitter_${id}_${i + 1}.mp4`; + const videoFilename = `twitter_${id}_${i + 1}.${shouldRenderGif ? "gif" : "mp4"}`; let type = "video"; if (shouldRenderGif) type = "gif"; @@ -217,7 +217,7 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { url = createStream({ service: "twitter", type: shouldRenderGif ? "gif" : "remux", - u: url, + url, filename: videoFilename, }) } else if (alwaysProxy) { diff --git a/api/src/processing/services/vimeo.js b/api/src/processing/services/vimeo.js index 23e84191..8d704771 100644 --- a/api/src/processing/services/vimeo.js +++ b/api/src/processing/services/vimeo.js @@ -1,7 +1,6 @@ import HLS from "hls-parser"; - import { env } from "../../config.js"; -import { cleanString, merge } from '../../misc/utils.js'; +import { merge } from '../../misc/utils.js'; const resolutionMatch = { "3840": 2160, @@ -122,7 +121,7 @@ const getHLS = async (configURL, obj) => { return { urls, - isM3U8: true, + isHLS: true, filenameAttributes: { resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`, qualityLabel: `${resolutionMatch[bestQuality.resolution.width]}p`, @@ -152,8 +151,8 @@ export default async function(obj) { } const fileMetadata = { - title: cleanString(info.name), - artist: cleanString(info.user.name), + title: info.name, + artist: info.user.name, }; return merge( diff --git a/api/src/processing/services/vine.js b/api/src/processing/services/vine.js deleted file mode 100644 index e0826720..00000000 --- a/api/src/processing/services/vine.js +++ /dev/null @@ -1,15 +0,0 @@ -export default async function(obj) { - let post = await fetch(`https://archive.vine.co/posts/${obj.id}.json`) - .then(r => r.json()) - .catch(() => {}); - - if (!post) return { error: "fetch.empty" }; - - if (post.videoUrl) return { - urls: post.videoUrl.replace("http://", "https://"), - filename: `vine_${obj.id}.mp4`, - audioFilename: `vine_${obj.id}_audio` - } - - return { error: "fetch.empty" } -} diff --git a/api/src/processing/services/vk.js b/api/src/processing/services/vk.js index e3c18e47..33224d69 100644 --- a/api/src/processing/services/vk.js +++ b/api/src/processing/services/vk.js @@ -1,63 +1,140 @@ -import { cleanString } from "../../misc/utils.js"; -import { genericUserAgent, env } from "../../config.js"; +import { env } from "../../config.js"; -const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240"]; +const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240", "144"]; -export default async function(o) { - let html, url, quality = o.quality === "max" ? 2160 : o.quality; +const oauthUrl = "https://oauth.vk.com/oauth/get_anonym_token"; +const apiUrl = "https://api.vk.com/method"; - html = await fetch(`https://vk.com/video${o.userId}_${o.videoId}`, { - headers: { - "user-agent": genericUserAgent - } - }) - .then(r => r.arrayBuffer()) - .catch(() => {}); +const clientId = "51552953"; +const clientSecret = "qgr0yWwXCrsxA1jnRtRX"; - if (!html) return { error: "fetch.fail" }; +// used in stream/shared.js for accessing media files +export const vkClientAgent = "com.vk.vkvideo.prod/822 (iPhone, iOS 16.7.7, iPhone10,4, Scale/2.0) SAK/1.119"; - // decode cyrillic from windows-1251 because vk still uses apis from prehistoric times - let decoder = new TextDecoder('windows-1251'); - html = decoder.decode(html); +const cachedToken = { + token: "", + expiry: 0, + device_id: "", +}; - if (!html.includes(`{"lang":`)) return { error: "fetch.empty" }; - - let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]); - - if (Number(js.mvData.is_active_live) !== 0) { - return { error: "content.video.live" }; +const getToken = async () => { + if (cachedToken.expiry - 10 > Math.floor(new Date().getTime() / 1000)) { + return cachedToken.token; } - if (js.mvData.duration > env.durationLimit) { + const randomDeviceId = crypto.randomUUID().toUpperCase(); + + const anonymOauth = new URL(oauthUrl); + anonymOauth.searchParams.set("client_id", clientId); + anonymOauth.searchParams.set("client_secret", clientSecret); + anonymOauth.searchParams.set("device_id", randomDeviceId); + + const oauthResponse = await fetch(anonymOauth.toString(), { + headers: { + "user-agent": vkClientAgent, + } + }).then(r => { + if (r.status === 200) { + return r.json(); + } + }); + + if (!oauthResponse) return; + + if (oauthResponse?.token && oauthResponse?.expired_at && typeof oauthResponse?.expired_at === "number") { + cachedToken.token = oauthResponse.token; + cachedToken.expiry = oauthResponse.expired_at; + cachedToken.device_id = randomDeviceId; + } + + if (!cachedToken.token) return; + + return cachedToken.token; +} + +const getVideo = async (ownerId, videoId, accessKey) => { + const video = await fetch(`${apiUrl}/video.get`, { + method: "POST", + headers: { + "content-type": "application/x-www-form-urlencoded; charset=utf-8", + "user-agent": vkClientAgent, + }, + body: new URLSearchParams({ + anonymous_token: cachedToken.token, + device_id: cachedToken.device_id, + lang: "en", + v: "5.244", + videos: `${ownerId}_${videoId}${accessKey ? `_${accessKey}` : ''}` + }).toString() + }) + .then(r => { + if (r.status === 200) { + return r.json(); + } + }); + + return video; +} + +export default async function ({ ownerId, videoId, accessKey, quality }) { + const token = await getToken(); + if (!token) return { error: "fetch.fail" }; + + const videoGet = await getVideo(ownerId, videoId, accessKey); + + if (!videoGet || !videoGet.response || videoGet.response.items.length !== 1) { + return { error: "fetch.empty" }; + } + + const video = videoGet.response.items[0]; + + if (video.restriction) { + const title = video.restriction.title; + if (title.endsWith("country") || title.endsWith("region.")) { + return { error: "content.video.region" }; + } + if (title === "Processing video") { + return { error: "fetch.empty" }; + } + return { error: "content.video.unavailable" }; + } + + if (!video.files || !video.duration) { + return { error: "fetch.fail" }; + } + + if (video.duration > env.durationLimit) { return { error: "content.too_long" }; } - for (let i in resolutions) { - if (js.player.params[0][`url${resolutions[i]}`]) { - quality = resolutions[i]; + const userQuality = quality === "max" ? resolutions[0] : quality; + let pickedQuality; + + for (const resolution of resolutions) { + if (video.files[`mp4_${resolution}`] && +resolution <= +userQuality) { + pickedQuality = resolution; break } } - if (Number(quality) > Number(o.quality)) quality = o.quality; - url = js.player.params[0][`url${quality}`]; + const url = video.files[`mp4_${pickedQuality}`]; - let fileMetadata = { - title: cleanString(js.player.params[0].md_title.trim()), - author: cleanString(js.player.params[0].md_author.trim()), + if (!url) return { error: "fetch.fail" }; + + const fileMetadata = { + title: video.title.trim(), } - if (url) return { + return { urls: url, + fileMetadata, filenameAttributes: { service: "vk", - id: `${o.userId}_${o.videoId}`, + id: `${ownerId}_${videoId}${accessKey ? `_${accessKey}` : ''}`, title: fileMetadata.title, - author: fileMetadata.author, - resolution: `${quality}p`, - qualityLabel: `${quality}p`, + resolution: `${pickedQuality}p`, + qualityLabel: `${pickedQuality}p`, extension: "mp4" } } - return { error: "fetch.empty" } } diff --git a/api/src/processing/services/xiaohongshu.js b/api/src/processing/services/xiaohongshu.js new file mode 100644 index 00000000..bbb53ab1 --- /dev/null +++ b/api/src/processing/services/xiaohongshu.js @@ -0,0 +1,116 @@ +import { extract, normalizeURL } from "../url.js"; +import { genericUserAgent } from "../../config.js"; +import { createStream } from "../../stream/manage.js"; +import { getRedirectingURL } from "../../misc/utils.js"; + +const https = (url) => { + return url.replace(/^http:/i, 'https:'); +} + +export default async function ({ id, token, shareId, h265, isAudioOnly, dispatcher }) { + let noteId = id; + let xsecToken = token; + + if (!noteId) { + const extractedURL = await getRedirectingURL( + `https://xhslink.com/a/${shareId}`, + dispatcher + ); + + if (extractedURL) { + const { patternMatch } = extract(normalizeURL(extractedURL)); + + if (patternMatch) { + noteId = patternMatch.id; + xsecToken = patternMatch.token; + } + } + } + + if (!noteId || !xsecToken) return { error: "fetch.short_link" }; + + const res = await fetch(`https://www.xiaohongshu.com/explore/${noteId}?xsec_token=${xsecToken}`, { + headers: { + "user-agent": genericUserAgent, + }, + dispatcher, + }); + + const html = await res.text(); + + let note; + try { + const initialState = html + .split('')[0] + .replace(/:\s*undefined/g, ":null"); + + const data = JSON.parse(initialState); + + const noteInfo = data?.note?.noteDetailMap; + if (!noteInfo) throw "no note detail map"; + + const currentNote = noteInfo[noteId]; + if (!currentNote) throw "no current note in detail map"; + + note = currentNote.note; + } catch {} + + if (!note) return { error: "fetch.empty" }; + + const video = note.video; + const images = note.imageList; + + const filenameBase = `xiaohongshu_${noteId}`; + + if (video) { + const videoFilename = `${filenameBase}.mp4`; + const audioFilename = `${filenameBase}_audio`; + + let videoURL; + + if (h265 && !isAudioOnly && video.consumer?.originVideoKey) { + videoURL = `https://sns-video-bd.xhscdn.com/${video.consumer.originVideoKey}`; + } else { + const h264Streams = video.media?.stream?.h264; + + if (h264Streams?.length) { + videoURL = h264Streams.reduce((a, b) => Number(a?.videoBitrate) > Number(b?.videoBitrate) ? a : b).masterUrl; + } + } + + if (!videoURL) return { error: "fetch.empty" }; + + return { + urls: https(videoURL), + filename: videoFilename, + audioFilename: audioFilename, + } + } + + if (!images || images.length === 0) { + return { error: "fetch.empty" }; + } + + if (images.length === 1) { + return { + isPhoto: true, + urls: https(images[0].urlDefault), + filename: `${filenameBase}.jpg`, + } + } + + const picker = images.map((image, i) => { + return { + type: "photo", + url: createStream({ + service: "xiaohongshu", + type: "proxy", + url: https(image.urlDefault), + filename: `${filenameBase}_${i + 1}.jpg`, + }) + } + }); + + return { picker }; +} diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index 46f72a5b..f0766b3f 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -1,16 +1,16 @@ -import { fetch } from "undici"; +import HLS from "hls-parser"; +import { fetch } from "undici"; import { Innertube, Session } from "youtubei.js"; import { env } from "../../config.js"; -import { cleanString } from "../../misc/utils.js"; import { getCookie, updateCookieValues } from "../cookie/manager.js"; const PLAYER_REFRESH_PERIOD = 1000 * 60 * 15; // ms let innertube, lastRefreshedAt; -const codecMatch = { +const codecList = { h264: { videoCodec: "avc1", audioCodec: "mp4a", @@ -28,12 +28,27 @@ const codecMatch = { } } +const hlsCodecList = { + h264: { + videoCodec: "avc1", + audioCodec: "mp4a", + container: "mp4" + }, + vp9: { + videoCodec: "vp09", + audioCodec: "mp4a", + container: "webm" + } +} + +const videoQualities = [144, 240, 360, 480, 720, 1080, 1440, 2160, 4320]; + const transformSessionData = (cookie) => { if (!cookie) return; const values = { ...cookie.values() }; - const REQUIRED_VALUES = [ 'access_token', 'refresh_token' ]; + const REQUIRED_VALUES = ['access_token', 'refresh_token']; if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) { return; @@ -51,9 +66,18 @@ const transformSessionData = (cookie) => { const cloneInnertube = async (customFetch) => { const shouldRefreshPlayer = lastRefreshedAt + PLAYER_REFRESH_PERIOD < new Date(); + + const rawCookie = getCookie('youtube'); + const rawCookieValues = rawCookie?.values(); + const cookie = rawCookie?.toString(); + if (!innertube || shouldRefreshPlayer) { innertube = await Innertube.create({ - fetch: customFetch + fetch: customFetch, + retrieve_player: !!cookie, + cookie, + po_token: rawCookieValues?.po_token, + visitor_data: rawCookieValues?.visitor_data, }); lastRefreshedAt = +new Date(); } @@ -64,30 +88,30 @@ const cloneInnertube = async (customFetch) => { innertube.session.api_version, innertube.session.account_index, innertube.session.player, - undefined, + cookie, customFetch ?? innertube.session.http.fetch, innertube.session.cache ); - const cookie = getCookie('youtube_oauth'); - const oauthData = transformSessionData(cookie); + const oauthCookie = getCookie('youtube_oauth'); + const oauthData = transformSessionData(oauthCookie); if (!session.logged_in && oauthData) { await session.oauth.init(oauthData); session.logged_in = true; } - if (session.logged_in) { + if (session.logged_in && oauthData) { if (session.oauth.shouldRefreshToken()) { await session.oauth.refreshAccessToken(); } - const cookieValues = cookie.values(); + const cookieValues = oauthCookie.values(); const oldExpiry = new Date(cookieValues.expiry_date); const newExpiry = new Date(session.oauth.oauth2_tokens.expiry_date); if (oldExpiry.getTime() !== newExpiry.getTime()) { - updateCookieValues(cookie, { + updateCookieValues(oauthCookie, { ...session.oauth.client_id, ...session.oauth.oauth2_tokens, expiry_date: newExpiry.toISOString() @@ -99,7 +123,7 @@ const cloneInnertube = async (customFetch) => { return yt; } -export default async function(o) { +export default async function (o) { let yt; try { yt = await cloneInnertube( @@ -108,7 +132,7 @@ export default async function(o) { dispatcher: o.dispatcher }) ); - } catch(e) { + } catch (e) { if (e.message?.endsWith("decipher algorithm")) { return { error: "youtube.decipher" } } else if (e.message?.includes("refresh access token")) { @@ -116,29 +140,46 @@ export default async function(o) { } else throw e; } - const quality = o.quality === "max" ? "9000" : o.quality; + const cookie = getCookie('youtube')?.toString(); - let info, isDubbed, - format = o.format || "h264"; + let useHLS = o.youtubeHLS; - function qual(i) { - if (!i.quality_label) { - return; - } - - return i.quality_label.split('p')[0].split('s')[0] + // HLS playlists don't contain the av1 video format, at least with the iOS client + if (useHLS && o.format === "av1") { + useHLS = false; } + let innertubeClient = o.innertubeClient || "ANDROID"; + + if (cookie) { + useHLS = false; + innertubeClient = "WEB"; + } + + if (useHLS) { + innertubeClient = "IOS"; + } + + let info; try { - info = await yt.getBasicInfo(o.id, yt.session.logged_in ? 'ANDROID' : 'IOS'); - } catch(e) { - if (e?.info?.reason === "This video is private") { - return { error: "content.video.private" }; - } else if (e?.message === "This video is unavailable") { - return { error: "content.video.unavailable" }; - } else { - return { error: "fetch.fail" }; + info = await yt.getBasicInfo(o.id, innertubeClient); + } catch (e) { + if (e?.info) { + const errorInfo = JSON.parse(e?.info); + + if (errorInfo?.reason === "This video is private") { + return { error: "content.video.private" }; + } + if (["INVALID_ARGUMENT", "UNAUTHENTICATED"].includes(errorInfo?.error?.status)) { + return { error: "youtube.api_error" }; + } } + + if (e?.message === "This video is unavailable") { + return { error: "content.video.unavailable" }; + } + + return { error: "fetch.fail" }; } if (!info) return { error: "fetch.fail" }; @@ -146,37 +187,47 @@ export default async function(o) { const playability = info.playability_status; const basicInfo = info.basic_info; - if (playability.status === "LOGIN_REQUIRED") { - if (playability.reason.endsWith("bot")) { - return { error: "youtube.login" } - } - if (playability.reason.endsWith("age")) { - return { error: "content.video.age" } - } - if (playability?.error_screen?.reason?.text === "Private video") { - return { error: "content.video.private" } - } - } + switch (playability.status) { + case "LOGIN_REQUIRED": + if (playability.reason.endsWith("bot")) { + return { error: "youtube.login" } + } + if (playability.reason.endsWith("age")) { + return { error: "content.video.age" } + } + if (playability?.error_screen?.reason?.text === "Private video") { + return { error: "content.video.private" } + } + break; - if (playability.status === "UNPLAYABLE") { - if (playability?.reason?.endsWith("request limit.")) { - return { error: "fetch.rate" } - } - if (playability?.error_screen?.subreason?.text?.endsWith("in your country")) { - return { error: "content.video.region" } - } - if (playability?.error_screen?.reason?.text === "Private video") { - return { error: "content.video.private" } - } + case "UNPLAYABLE": + if (playability?.reason?.endsWith("request limit.")) { + return { error: "fetch.rate" } + } + if (playability?.error_screen?.subreason?.text?.endsWith("in your country")) { + return { error: "content.video.region" } + } + if (playability?.error_screen?.reason?.text === "Private video") { + return { error: "content.video.private" } + } + break; + + case "AGE_VERIFICATION_REQUIRED": + return { error: "content.video.age" }; } if (playability.status !== "OK") { return { error: "content.video.unavailable" }; } + if (basicInfo.is_live) { return { error: "content.video.live" }; } + if (basicInfo.duration > env.durationLimit) { + return { error: "content.too_long" }; + } + // return a critical error if returned video is "Video Not Available" // or a similar stub by youtube if (basicInfo.id !== o.id) { @@ -186,64 +237,204 @@ export default async function(o) { } } - const filterByCodec = (formats) => - formats - .filter(e => - e.mime_type.includes(codecMatch[format].videoCodec) - || e.mime_type.includes(codecMatch[format].audioCodec) - ) - .sort((a, b) => Number(b.bitrate) - Number(a.bitrate)); + const quality = o.quality === "max" ? 9000 : Number(o.quality); - let adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats); - - if (adaptive_formats.length === 0 && format === "vp9") { - format = "h264" - adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats) + const normalizeQuality = res => { + const shortestSide = Math.min(res.height, res.width); + return videoQualities.find(qual => qual >= shortestSide); } - let bestQuality; + let video, audio, dubbedLanguage, + codec = o.format || "h264", itag = o.itag; - const bestVideo = adaptive_formats.find(i => i.has_video && i.content_length); - const hasAudio = adaptive_formats.find(i => i.has_audio && i.content_length); + if (useHLS) { + const hlsManifest = info.streaming_data.hls_manifest_url; - if (bestVideo) bestQuality = qual(bestVideo); + if (!hlsManifest) { + return { error: "youtube.no_hls_streams" }; + } - if ((!bestQuality && !o.isAudioOnly) || !hasAudio) - return { error: "youtube.codec" }; + const fetchedHlsManifest = await fetch(hlsManifest, { + dispatcher: o.dispatcher, + }).then(r => { + if (r.status === 200) { + return r.text(); + } else { + throw new Error("couldn't fetch the HLS playlist"); + } + }).catch(() => { }); - if (basicInfo.duration > env.durationLimit) - return { error: "content.too_long" }; + if (!fetchedHlsManifest) { + return { error: "youtube.no_hls_streams" }; + } - const checkBestAudio = (i) => (i.has_audio && !i.has_video); + const variants = HLS.parse(fetchedHlsManifest).variants.sort( + (a, b) => Number(b.bandwidth) - Number(a.bandwidth) + ); - let audio = adaptive_formats.find(i => - checkBestAudio(i) && i.is_original - ); + if (!variants || variants.length === 0) { + return { error: "youtube.no_hls_streams" }; + } - if (o.dubLang) { - let dubbedAudio = adaptive_formats.find(i => - checkBestAudio(i) - && i.language === o.dubLang - && i.audio_track - ) + const matchHlsCodec = codecs => ( + codecs.includes(hlsCodecList[codec].videoCodec) + ); - if (dubbedAudio && !dubbedAudio?.audio_track?.audio_is_default) { - audio = dubbedAudio; - isDubbed = true; + const best = variants.find(i => matchHlsCodec(i.codecs)); + + const preferred = variants.find(i => + matchHlsCodec(i.codecs) && normalizeQuality(i.resolution) === quality + ); + + let selected = preferred || best; + + if (!selected) { + codec = "h264"; + selected = variants.find(i => matchHlsCodec(i.codecs)); + } + + if (!selected) { + return { error: "youtube.no_matching_format" }; + } + + audio = selected.audio.find(i => i.isDefault); + + // some videos (mainly those with AI dubs) don't have any tracks marked as default + // why? god knows, but we assume that a default track is marked as such in the title + if (!audio) { + audio = selected.audio.find(i => i.name.endsWith("- original")); + } + + if (o.dubLang) { + const dubbedAudio = selected.audio.find(i => + i.language?.startsWith(o.dubLang) + ); + + if (dubbedAudio && !dubbedAudio.isDefault) { + dubbedLanguage = dubbedAudio.language; + audio = dubbedAudio; + } + } + + selected.audio = []; + selected.subtitles = []; + video = selected; + } else { + // i miss typescript so bad + const sorted_formats = { + h264: { + video: [], + audio: [], + bestVideo: undefined, + bestAudio: undefined, + }, + vp9: { + video: [], + audio: [], + bestVideo: undefined, + bestAudio: undefined, + }, + av1: { + video: [], + audio: [], + bestVideo: undefined, + bestAudio: undefined, + }, + } + + const checkFormat = (format, pCodec) => format.content_length && + (format.mime_type.includes(codecList[pCodec].videoCodec) + || format.mime_type.includes(codecList[pCodec].audioCodec)); + + // sort formats & weed out bad ones + info.streaming_data.adaptive_formats.sort((a, b) => + Number(b.bitrate) - Number(a.bitrate) + ).forEach(format => { + Object.keys(codecList).forEach(yCodec => { + const matchingItag = slot => !itag?.[slot] || itag[slot] === format.itag; + const sorted = sorted_formats[yCodec]; + const goodFormat = checkFormat(format, yCodec); + if (!goodFormat) return; + + if (format.has_video && matchingItag('video')) { + sorted.video.push(format); + if (!sorted.bestVideo) + sorted.bestVideo = format; + } + + if (format.has_audio && matchingItag('audio')) { + sorted.audio.push(format); + if (!sorted.bestAudio) + sorted.bestAudio = format; + } + }) + }); + + const noBestMedia = () => { + const vid = sorted_formats[codec]?.bestVideo; + const aud = sorted_formats[codec]?.bestAudio; + return (!vid && !o.isAudioOnly) || (!aud && o.isAudioOnly) + }; + + if (noBestMedia()) { + if (codec === "av1") codec = "vp9"; + else if (codec === "vp9") codec = "av1"; + + // if there's no higher quality fallback, then use h264 + if (noBestMedia()) codec = "h264"; + } + + // if there's no proper combo of av1, vp9, or h264, then give up + if (noBestMedia()) { + return { error: "youtube.no_matching_format" }; + } + + audio = sorted_formats[codec].bestAudio; + + if (audio?.audio_track && !audio?.audio_track?.audio_is_default) { + audio = sorted_formats[codec].audio.find(i => + i?.audio_track?.audio_is_default + ); + } + + if (o.dubLang) { + const dubbedAudio = sorted_formats[codec].audio.find(i => + i.language?.startsWith(o.dubLang) && i.audio_track + ); + + if (dubbedAudio && !dubbedAudio?.audio_track?.audio_is_default) { + audio = dubbedAudio; + dubbedLanguage = dubbedAudio.language; + } + } + + if (!o.isAudioOnly) { + const qual = (i) => { + return normalizeQuality({ + width: i.width, + height: i.height, + }) + } + + const bestQuality = qual(sorted_formats[codec].bestVideo); + const useBestQuality = quality >= bestQuality; + + video = useBestQuality + ? sorted_formats[codec].bestVideo + : sorted_formats[codec].video.find(i => qual(i) === quality); + + if (!video) video = sorted_formats[codec].bestVideo; } } - if (!audio) { - audio = adaptive_formats.find(i => checkBestAudio(i)); - } - - let fileMetadata = { - title: cleanString(basicInfo.title.trim()), - artist: cleanString(basicInfo.author.replace("- Topic", "").trim()), + const fileMetadata = { + title: basicInfo.title.trim(), + artist: basicInfo.author.replace("- Topic", "").trim() } if (basicInfo?.short_description?.startsWith("Provided to YouTube by")) { - let descItems = basicInfo.short_description.split("\n\n", 5); + const descItems = basicInfo.short_description.split("\n\n", 5); + if (descItems.length === 5) { fileMetadata.album = descItems[2]; fileMetadata.copyright = descItems[3]; @@ -253,61 +444,94 @@ export default async function(o) { } } - let filenameAttributes = { + const filenameAttributes = { service: "youtube", id: o.id, title: fileMetadata.title, author: fileMetadata.artist, - youtubeDubName: isDubbed ? o.dubLang : false + youtubeDubName: dubbedLanguage || false, } - if (audio && o.isAudioOnly) return { - type: "audio", - isAudioOnly: true, - urls: audio.decipher(yt.session.player), - filenameAttributes: filenameAttributes, - fileMetadata: fileMetadata, - bestAudio: format === "h264" ? "m4a" : "opus" - } + itag = { + video: video?.itag, + audio: audio?.itag + }; - const matchingQuality = Number(quality) > Number(bestQuality) ? bestQuality : quality, - checkSingle = i => - qual(i) === matchingQuality && i.mime_type.includes(codecMatch[format].videoCodec), - checkRender = i => - qual(i) === matchingQuality && i.has_video && !i.has_audio; + const originalRequest = { + ...o, + dispatcher: undefined, + itag, + innertubeClient + }; - let match, type, urls; + if (audio && o.isAudioOnly) { + let bestAudio = codec === "h264" ? "m4a" : "opus"; + let urls = audio.url; - // prefer good premuxed videos if available - if (!o.isAudioOnly && !o.isAudioMuted && format === "h264" && bestVideo.fps <= 30) { - match = info.streaming_data.formats.find(checkSingle); - type = "proxy"; - urls = match?.decipher(yt.session.player); - } + if (useHLS) { + bestAudio = "mp3"; + urls = audio.uri; + } - const video = adaptive_formats.find(checkRender); + if (innertubeClient === "WEB" && innertube) { + urls = audio.decipher(innertube.session.player); + } - if (!match && video && audio) { - match = video; - type = "merge"; - urls = [ - video.decipher(yt.session.player), - audio.decipher(yt.session.player) - ] - } - - if (match) { - filenameAttributes.qualityLabel = match.quality_label; - filenameAttributes.resolution = `${match.width}x${match.height}`; - filenameAttributes.extension = codecMatch[format].container; - filenameAttributes.youtubeFormat = format; return { - type, + type: "audio", + isAudioOnly: true, urls, filenameAttributes, - fileMetadata + fileMetadata, + bestAudio, + isHLS: useHLS, + originalRequest } } - return { error: "fetch.fail" } + if (video && audio) { + let resolution; + + if (useHLS) { + resolution = normalizeQuality(video.resolution); + filenameAttributes.resolution = `${video.resolution.width}x${video.resolution.height}`; + filenameAttributes.extension = hlsCodecList[codec].container; + + video = video.uri; + audio = audio.uri; + } else { + resolution = normalizeQuality({ + width: video.width, + height: video.height, + }); + + filenameAttributes.resolution = `${video.width}x${video.height}`; + filenameAttributes.extension = codecList[codec].container; + + if (innertubeClient === "WEB" && innertube) { + video = video.decipher(innertube.session.player); + audio = audio.decipher(innertube.session.player); + } else { + video = video.url; + audio = audio.url; + } + } + + filenameAttributes.qualityLabel = `${resolution}p`; + filenameAttributes.youtubeFormat = codec; + + return { + type: "merge", + urls: [ + video, + audio, + ], + filenameAttributes, + fileMetadata, + isHLS: useHLS, + originalRequest + } + } + + return { error: "youtube.no_matching_format" }; } diff --git a/api/src/processing/url.js b/api/src/processing/url.js index 034a5d73..cfbbecc0 100644 --- a/api/src/processing/url.js +++ b/api/src/processing/url.js @@ -1,4 +1,4 @@ -import psl from "psl"; +import psl from "@imput/psl"; import { strict as assert } from "node:assert"; import { env } from "../config.js"; @@ -42,7 +42,7 @@ function aliasURL(url) { case "fixvx": case "x": if (services.twitter.altDomains.includes(url.hostname)) { - url.hostname = 'twitter.com' + url.hostname = 'twitter.com'; } break; @@ -85,9 +85,21 @@ function aliasURL(url) { url.hostname = 'instagram.com'; } break; + + case "vk": + case "vkvideo": + if (services.vk.altDomains.includes(url.hostname)) { + url.hostname = 'vk.com'; + } + break; + + case "xhslink": + if (url.hostname === 'xhslink.com' && parts.length === 3) { + url = new URL(`https://www.xiaohongshu.com/a/${parts[2]}`); + } } - return url + return url; } function cleanURL(url) { @@ -107,36 +119,41 @@ function cleanURL(url) { break; case "vk": if (url.pathname.includes('/clip') && url.searchParams.get('z')) { - limitQuery('z') + limitQuery('z'); } break; case "youtube": if (url.searchParams.get('v')) { - limitQuery('v') + limitQuery('v'); } break; case "rutube": if (url.searchParams.get('p')) { - limitQuery('p') + limitQuery('p'); } break; case "twitter": if (url.searchParams.get('post_id')) { - limitQuery('post_id') + limitQuery('post_id'); + } + break; + case "xiaohongshu": + if (url.searchParams.get('xsec_token')) { + limitQuery('xsec_token'); } break; } if (stripQuery) { - url.search = '' + url.search = ''; } - url.username = url.password = url.port = url.hash = '' + url.username = url.password = url.port = url.hash = ''; if (url.pathname.endsWith('/')) url.pathname = url.pathname.slice(0, -1); - return url + return url; } function getHostIfValid(url) { @@ -174,6 +191,11 @@ export function extract(url) { } if (!env.enabledServices.has(host)) { + // show a different message when youtube is disabled on official instances + // as it only happens when shit hits the fan + if (new URL(env.apiURL).hostname.endsWith(".imput.net") && host === "youtube") { + return { error: "youtube.temporary_disabled" }; + } return { error: "service.disabled" }; } diff --git a/api/src/security/api-keys.js b/api/src/security/api-keys.js index eee48da3..d534999c 100644 --- a/api/src/security/api-keys.js +++ b/api/src/security/api-keys.js @@ -1,7 +1,8 @@ import { env } from "../config.js"; import { readFile } from "node:fs/promises"; -import { Yellow } from "../misc/console-text.js"; +import { Green, Yellow } from "../misc/console-text.js"; import ip from "ipaddr.js"; +import * as cluster from "../misc/cluster.js"; // this function is a modified variation of code // from https://stackoverflow.com/a/32402438/14855621 @@ -99,7 +100,9 @@ const formatKeys = (keyData) => { if (data.ips) { formatted[key].ips = data.ips.map(addr => { if (ip.isValid(addr)) { - return [ ip.parse(addr), 32 ]; + const parsed = ip.parse(addr); + const range = parsed.kind() === 'ipv6' ? 128 : 32; + return [ parsed, range ]; } return ip.parseCIDR(addr); @@ -114,6 +117,10 @@ const formatKeys = (keyData) => { return formatted; } +const updateKeys = (newKeys) => { + keys = formatKeys(newKeys); +} + const loadKeys = async (source) => { let updated; if (source.protocol === 'file:') { @@ -129,12 +136,19 @@ const loadKeys = async (source) => { } validateKeys(updated); - keys = formatKeys(updated); + + cluster.broadcast({ api_keys: updated }); + + updateKeys(updated); } -const wrapLoad = (url) => { +const wrapLoad = (url, initial = false) => { loadKeys(url) - .then(() => {}) + .then(() => { + if (initial) { + console.log(`${Green('[✓]')} api keys loaded successfully!`) + } + }) .catch((e) => { console.error(`${Yellow('[!]')} Failed loading API keys at ${new Date().toISOString()}.`); console.error('Error:', e); @@ -198,8 +212,16 @@ export const validateAuthorization = (req) => { } export const setup = (url) => { - wrapLoad(url); - if (env.keyReloadInterval > 0) { - setInterval(() => wrapLoad(url), env.keyReloadInterval * 1000); + if (cluster.isPrimary) { + wrapLoad(url, true); + if (env.keyReloadInterval > 0) { + setInterval(() => wrapLoad(url), env.keyReloadInterval * 1000); + } + } else if (cluster.isWorker) { + process.on('message', (message) => { + if ('api_keys' in message) { + updateKeys(message.api_keys); + } + }); } } diff --git a/api/src/security/secrets.js b/api/src/security/secrets.js new file mode 100644 index 00000000..fff24f84 --- /dev/null +++ b/api/src/security/secrets.js @@ -0,0 +1,62 @@ +import cluster from "node:cluster"; +import { createHmac, randomBytes } from "node:crypto"; + +const generateSalt = () => { + if (cluster.isPrimary) + return randomBytes(64); + + return null; +} + +let rateSalt = generateSalt(); +let streamSalt = generateSalt(); + +export const syncSecrets = () => { + return new Promise((resolve, reject) => { + if (cluster.isPrimary) { + let remaining = Object.values(cluster.workers).length; + const handleReady = (worker, m) => { + if (m.ready) + worker.send({ rateSalt, streamSalt }); + + if (!--remaining) + resolve(); + } + + for (const worker of Object.values(cluster.workers)) { + worker.once( + 'message', + (m) => handleReady(worker, m) + ); + } + } else if (cluster.isWorker) { + if (rateSalt || streamSalt) + return reject(); + + process.send({ ready: true }); + process.once('message', (message) => { + if (rateSalt || streamSalt) + return reject(); + + if (message.rateSalt && message.streamSalt) { + streamSalt = Buffer.from(message.streamSalt); + rateSalt = Buffer.from(message.rateSalt); + resolve(); + } + }); + } else reject(); + }); +} + + +export const hashHmac = (value, type) => { + let salt; + if (type === 'rate') + salt = rateSalt; + else if (type === 'stream') + salt = streamSalt; + else + throw "unknown salt"; + + return createHmac("sha256", salt).update(value).digest(); +} diff --git a/api/src/store/base-store.js b/api/src/store/base-store.js new file mode 100644 index 00000000..c2a59ff8 --- /dev/null +++ b/api/src/store/base-store.js @@ -0,0 +1,48 @@ +const _stores = new Set(); + +export class Store { + id; + + constructor(name) { + name = name.toUpperCase(); + + if (_stores.has(name)) + throw `${name} store already exists`; + _stores.add(name); + + this.id = name; + } + + async _has(_key) { await Promise.reject("needs implementation"); } + has(key) { + if (typeof key !== 'string') { + key = key.toString(); + } + + return this._has(key); + } + + async _get(_key) { await Promise.reject("needs implementation"); } + async get(key) { + if (typeof key !== 'string') { + key = key.toString(); + } + + const val = await this._get(key); + if (val === null) + return null; + + return val; + } + + async _set(_key, _val, _exp_sec = -1) { await Promise.reject("needs implementation") } + set(key, val, exp_sec = -1) { + if (typeof key !== 'string') { + key = key.toString(); + } + + exp_sec = Math.round(exp_sec); + + return this._set(key, val, exp_sec); + } +}; diff --git a/api/src/store/memory-store.js b/api/src/store/memory-store.js new file mode 100644 index 00000000..100a0e09 --- /dev/null +++ b/api/src/store/memory-store.js @@ -0,0 +1,77 @@ +import { MinPriorityQueue } from '@datastructures-js/priority-queue'; +import { Store } from './base-store.js'; + +// minimum delay between sweeps to avoid repeatedly +// sweeping entries close in proximity one by one. +const MIN_THRESHOLD_MS = 2500; + +export default class MemoryStore extends Store { + #store = new Map(); + #timeouts = new MinPriorityQueue/*<{ t: number, k: unknown }>*/((obj) => obj.t); + #nextSweep = { id: null, t: null }; + + constructor(name) { + super(name); + } + + _has(key) { + return this.#store.has(key); + } + + _get(key) { + const val = this.#store.get(key); + + return val === undefined ? null : val; + } + + _set(key, val, exp_sec = -1) { + if (this.#store.has(key)) { + this.#timeouts.remove(o => o.k === key); + } + + if (exp_sec > 0) { + const exp = 1000 * exp_sec; + const timeout_at = +new Date() + exp; + + this.#timeouts.enqueue({ k: key, t: timeout_at }); + } + + this.#store.set(key, val); + this.#reschedule(); + } + + #reschedule() { + const current_time = new Date().getTime(); + const time = this.#timeouts.front()?.t; + if (!time) { + return; + } else if (time < current_time) { + return this.#sweepNow(); + } + + const sweep = this.#nextSweep; + if (sweep.id === null || sweep.t > time) { + if (sweep.id) { + clearTimeout(sweep.id); + } + + sweep.t = time; + sweep.id = setTimeout( + () => this.#sweepNow(), + Math.max(MIN_THRESHOLD_MS, time - current_time) + ); + sweep.id.unref(); + } + } + + #sweepNow() { + while (this.#timeouts.front()?.t < new Date().getTime()) { + const item = this.#timeouts.dequeue(); + this.#store.delete(item.k); + } + + this.#nextSweep.id = null; + this.#nextSweep.t = null; + this.#reschedule(); + } +} diff --git a/api/src/store/redis-ratelimit.js b/api/src/store/redis-ratelimit.js new file mode 100644 index 00000000..64d11e5e --- /dev/null +++ b/api/src/store/redis-ratelimit.js @@ -0,0 +1,19 @@ +import { env } from "../config.js"; + +let client, redis, redisLimiter; + +export const createStore = async (name) => { + if (!env.redisURL) return; + + if (!client) { + redis = await import('redis'); + redisLimiter = await import('rate-limit-redis'); + client = redis.createClient({ url: env.redisURL }); + await client.connect(); + } + + return new redisLimiter.default({ + prefix: `RL${name}_`, + sendCommand: (...args) => client.sendCommand(args), + }); +} diff --git a/api/src/store/redis-store.js b/api/src/store/redis-store.js new file mode 100644 index 00000000..0b359526 --- /dev/null +++ b/api/src/store/redis-store.js @@ -0,0 +1,64 @@ +import { commandOptions, createClient } from "redis"; +import { env } from "../config.js"; +import { Store } from "./base-store.js"; + +export default class RedisStore extends Store { + #client = createClient({ + url: env.redisURL, + }); + #connected; + + constructor(name) { + super(name); + this.#connected = this.#client.connect(); + } + + #keyOf(key) { + return this.id + '_' + key; + } + + async _has(key) { + await this.#connected; + + return this.#client.hExists(key); + } + + async _get(key) { + await this.#connected; + + const valueType = await this.#client.get(this.#keyOf(key) + '_t'); + const value = await this.#client.get( + commandOptions({ returnBuffers: true }), + this.#keyOf(key) + ); + + if (!value) { + return null; + } + + if (valueType === 'b') + return value; + else + return JSON.parse(value); + } + + async _set(key, val, exp_sec = -1) { + await this.#connected; + + const options = exp_sec > 0 ? { EX: exp_sec } : undefined; + + if (val instanceof Buffer) { + await this.#client.set( + this.#keyOf(key) + '_t', + 'b', + options + ); + } + + await this.#client.set( + this.#keyOf(key), + val, + options + ); + } +} diff --git a/api/src/store/store.js b/api/src/store/store.js new file mode 100644 index 00000000..e268d88d --- /dev/null +++ b/api/src/store/store.js @@ -0,0 +1,10 @@ +import { env } from '../config.js'; + +let _export; +if (env.redisURL) { + _export = await import('./redis-store.js'); +} else { + _export = await import('./memory-store.js'); +} + +export default _export.default; diff --git a/api/src/stream/internal-hls.js b/api/src/stream/internal-hls.js index 07fcebde..55634c71 100644 --- a/api/src/stream/internal-hls.js +++ b/api/src/stream/internal-hls.js @@ -16,15 +16,17 @@ function transformObject(streamInfo, hlsObject) { let fullUrl; if (getURL(hlsObject.uri)) { - fullUrl = hlsObject.uri; + fullUrl = new URL(hlsObject.uri); } else { fullUrl = new URL(hlsObject.uri, streamInfo.url); } - hlsObject.uri = createInternalStream(fullUrl.toString(), streamInfo); + if (fullUrl.hostname !== '127.0.0.1') { + hlsObject.uri = createInternalStream(fullUrl.toString(), streamInfo); - if (hlsObject.map) { - hlsObject.map = transformObject(streamInfo, hlsObject.map); + if (hlsObject.map) { + hlsObject.map = transformObject(streamInfo, hlsObject.map); + } } return hlsObject; @@ -53,7 +55,7 @@ function transformMediaPlaylist(streamInfo, hlsPlaylist) { const HLS_MIME_TYPES = ["application/vnd.apple.mpegurl", "audio/mpegurl", "application/x-mpegURL"]; -export function isHlsRequest (req) { +export function isHlsResponse (req) { return HLS_MIME_TYPES.includes(req.headers['content-type']); } diff --git a/api/src/stream/internal.js b/api/src/stream/internal.js index 51552d4c..8c94c485 100644 --- a/api/src/stream/internal.js +++ b/api/src/stream/internal.js @@ -1,13 +1,13 @@ import { request } from "undici"; import { Readable } from "node:stream"; import { closeRequest, getHeaders, pipe } from "./shared.js"; -import { handleHlsPlaylist, isHlsRequest } from "./internal-hls.js"; +import { handleHlsPlaylist, isHlsResponse } from "./internal-hls.js"; const CHUNK_SIZE = BigInt(8e6); // 8 MB const min = (a, b) => a < b ? a : b; async function* readChunks(streamInfo, size) { - let read = 0n; + let read = 0n, chunksSinceTransplant = 0; while (read < size) { if (streamInfo.controller.signal.aborted) { throw new Error("controller aborted"); @@ -22,6 +22,16 @@ async function* readChunks(streamInfo, size) { signal: streamInfo.controller.signal }); + if (chunk.statusCode === 403 && chunksSinceTransplant >= 3 && streamInfo.transplant) { + chunksSinceTransplant = 0; + try { + await streamInfo.transplant(streamInfo.dispatcher); + continue; + } catch {} + } + + chunksSinceTransplant++; + const expected = min(CHUNK_SIZE, size - read); const received = BigInt(chunk.headers['content-length']); @@ -83,7 +93,7 @@ async function handleGenericStream(streamInfo, res) { const cleanup = () => res.end(); try { - const req = await request(streamInfo.url, { + const fileResponse = await request(streamInfo.url, { headers: { ...Object.fromEntries(streamInfo.headers), host: undefined @@ -93,19 +103,28 @@ async function handleGenericStream(streamInfo, res) { maxRedirections: 16 }); - res.status(req.statusCode); - req.body.on('error', () => {}); + res.status(fileResponse.statusCode); + fileResponse.body.on('error', () => {}); - for (const [ name, value ] of Object.entries(req.headers)) - res.setHeader(name, value) + // bluesky's cdn responds with wrong content-type for the hls playlist, + // so we enforce it here until they fix it + const isHls = isHlsResponse(fileResponse) + || (streamInfo.service === "bsky" && streamInfo.url.endsWith('.m3u8')); - if (req.statusCode < 200 || req.statusCode > 299) + for (const [ name, value ] of Object.entries(fileResponse.headers)) { + if (!isHls || name.toLowerCase() !== 'content-length') { + res.setHeader(name, value); + } + } + + if (fileResponse.statusCode < 200 || fileResponse.statusCode > 299) { return cleanup(); + } - if (isHlsRequest(req)) { - await handleHlsPlaylist(streamInfo, req, res); + if (isHls) { + await handleHlsPlaylist(streamInfo, fileResponse, res); } else { - pipe(req.body, res, cleanup); + pipe(fileResponse.body, res, cleanup); } } catch { closeRequest(streamInfo.controller); @@ -114,7 +133,11 @@ async function handleGenericStream(streamInfo, res) { } export function internalStream(streamInfo, res) { - if (streamInfo.service === 'youtube') { + if (streamInfo.headers) { + streamInfo.headers.delete('icy-metadata'); + } + + if (streamInfo.service === 'youtube' && !streamInfo.isHLS) { return handleYoutubeStream(streamInfo, res); } diff --git a/api/src/stream/manage.js b/api/src/stream/manage.js index e25f4434..ebb5c6c7 100644 --- a/api/src/stream/manage.js +++ b/api/src/stream/manage.js @@ -1,4 +1,4 @@ -import NodeCache from "node-cache"; +import Store from "../store/store.js"; import { nanoid } from "nanoid"; import { randomBytes } from "crypto"; @@ -7,34 +7,27 @@ import { setMaxListeners } from "node:events"; import { env } from "../config.js"; import { closeRequest } from "./shared.js"; -import { decryptStream, encryptStream, generateHmac } from "../misc/crypto.js"; +import { decryptStream, encryptStream } from "../misc/crypto.js"; +import { hashHmac } from "../security/secrets.js"; +import { zip } from "../misc/utils.js"; // optional dependency const freebind = env.freebindCIDR && await import('freebind').catch(() => {}); -const streamCache = new NodeCache({ - stdTTL: env.streamLifespan, - checkperiod: 10, - deleteOnExpire: true -}) - -streamCache.on("expired", (key) => { - streamCache.del(key); -}) +const streamCache = new Store('streams'); const internalStreamCache = new Map(); -const hmacSalt = randomBytes(64).toString('hex'); export function createStream(obj) { const streamID = nanoid(), iv = randomBytes(16).toString('base64url'), secret = randomBytes(32).toString('base64url'), exp = new Date().getTime() + env.streamLifespan * 1000, - hmac = generateHmac(`${streamID},${exp},${iv},${secret}`, hmacSalt), + hmac = hashHmac(`${streamID},${exp},${iv},${secret}`, 'stream').toString('base64url'), streamData = { exp: exp, type: obj.type, - urls: obj.u, + urls: obj.url, service: obj.service, filename: obj.filename, @@ -46,12 +39,19 @@ export function createStream(obj) { audioBitrate: obj.audioBitrate, audioCopy: !!obj.audioCopy, audioFormat: obj.audioFormat, + + isHLS: obj.isHLS || false, + originalRequest: obj.originalRequest }; + // FIXME: this is now a Promise, but it is not awaited + // here. it may happen that the stream is not + // stored in the Store before it is requested. streamCache.set( streamID, - encryptStream(streamData, iv, secret) - ) + encryptStream(streamData, iv, secret), + env.streamLifespan + ); let streamLink = new URL('/tunnel', env.apiURL); @@ -77,7 +77,7 @@ export function getInternalStream(id) { export function createInternalStream(url, obj = {}) { assert(typeof url === 'string'); - let dispatcher; + let dispatcher = obj.dispatcher; if (obj.requestIP) { dispatcher = freebind?.dispatcherFromIP(obj.requestIP, { strict: false }) } @@ -100,10 +100,12 @@ export function createInternalStream(url, obj = {}) { service: obj.service, headers, controller, - dispatcher + dispatcher, + isHLS: obj.isHLS, + transplant: obj.transplant }); - let streamLink = new URL('/itunnel', `http://127.0.0.1:${env.apiPort}`); + let streamLink = new URL('/itunnel', `http://127.0.0.1:${env.tunnelPort}`); streamLink.searchParams.set('id', streamID); const cleanup = () => { @@ -116,13 +118,17 @@ export function createInternalStream(url, obj = {}) { return streamLink.toString(); } -export function destroyInternalStream(url) { +function getInternalTunnelId(url) { url = new URL(url); if (url.hostname !== '127.0.0.1') { return; } - const id = url.searchParams.get('id'); + return url.searchParams.get('id'); +} + +export function destroyInternalStream(url) { + const id = getInternalTunnelId(url); if (internalStreamCache.has(id)) { closeRequest(getInternalStream(id)?.controller); @@ -130,9 +136,68 @@ export function destroyInternalStream(url) { } } +const transplantInternalTunnels = function(tunnelUrls, transplantUrls) { + if (tunnelUrls.length !== transplantUrls.length) { + return; + } + + for (const [ tun, url ] of zip(tunnelUrls, transplantUrls)) { + const id = getInternalTunnelId(tun); + const itunnel = getInternalStream(id); + + if (!itunnel) continue; + itunnel.url = url; + } +} + +const transplantTunnel = async function (dispatcher) { + if (this.pendingTransplant) { + await this.pendingTransplant; + return; + } + + let finished; + this.pendingTransplant = new Promise(r => finished = r); + + try { + const handler = await import(`../processing/services/${this.service}.js`); + const response = await handler.default({ + ...this.originalRequest, + dispatcher + }); + + if (!response.urls) { + return; + } + + response.urls = [response.urls].flat(); + if (this.originalRequest.isAudioOnly && response.urls.length > 1) { + response.urls = [response.urls[1]]; + } else if (this.originalRequest.isAudioMuted) { + response.urls = [response.urls[0]]; + } + + const tunnels = [this.urls].flat(); + if (tunnels.length !== response.urls.length) { + return; + } + + transplantInternalTunnels(tunnels, response.urls); + } + catch {} + finally { + finished(); + delete this.pendingTransplant; + } +} + function wrapStream(streamInfo) { const url = streamInfo.urls; + if (streamInfo.originalRequest) { + streamInfo.transplant = transplantTunnel.bind(streamInfo); + } + if (typeof url === 'string') { streamInfo.urls = createInternalStream(url, streamInfo); } else if (Array.isArray(url)) { @@ -146,10 +211,10 @@ function wrapStream(streamInfo) { return streamInfo; } -export function verifyStream(id, hmac, exp, secret, iv) { +export async function verifyStream(id, hmac, exp, secret, iv) { try { - const ghmac = generateHmac(`${id},${exp},${iv},${secret}`, hmacSalt); - const cache = streamCache.get(id.toString()); + const ghmac = hashHmac(`${id},${exp},${iv},${secret}`, 'stream').toString('base64url'); + const cache = await streamCache.get(id.toString()); if (ghmac !== String(hmac)) return { status: 401 }; if (!cache) return { status: 404 }; diff --git a/api/src/stream/shared.js b/api/src/stream/shared.js index 91e1ac2f..65af03f0 100644 --- a/api/src/stream/shared.js +++ b/api/src/stream/shared.js @@ -1,4 +1,5 @@ import { genericUserAgent } from "../config.js"; +import { vkClientAgent } from "../processing/services/vk.js"; const defaultHeaders = { 'user-agent': genericUserAgent @@ -13,6 +14,9 @@ const serviceHeaders = { origin: 'https://www.youtube.com', referer: 'https://www.youtube.com', DNT: '?1' + }, + vk: { + 'user-agent': vkClientAgent } } diff --git a/api/src/stream/stream.js b/api/src/stream/stream.js index a6d41200..c7cf7b56 100644 --- a/api/src/stream/stream.js +++ b/api/src/stream/stream.js @@ -10,7 +10,7 @@ export default async function(res, streamInfo) { return await stream.proxy(streamInfo, res); case "internal": - return internalStream(streamInfo, res); + return internalStream(streamInfo.data, res); case "merge": return stream.merge(streamInfo, res); diff --git a/api/src/stream/types.js b/api/src/stream/types.js index 184af873..0a4e2d47 100644 --- a/api/src/stream/types.js +++ b/api/src/stream/types.js @@ -1,10 +1,9 @@ -import { request } from "undici"; +import { Agent, request } from "undici"; import ffmpeg from "ffmpeg-static"; import { spawn } from "child_process"; import { create as contentDisposition } from "content-disposition-header"; import { env } from "../config.js"; -import { metadataManager } from "../misc/utils.js"; import { destroyInternalStream } from "./manage.js"; import { hlsExceptions } from "../processing/service-config.js"; import { getHeaders, closeRequest, closeResponse, pipe } from "./shared.js"; @@ -16,6 +15,29 @@ const ffmpegArgs = { gif: ["-vf", "scale=-1:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse", "-loop", "0"] } +const metadataTags = [ + "album", + "copyright", + "title", + "artist", + "track", + "date", +]; + +const convertMetadataToFFmpeg = (metadata) => { + let args = []; + + for (const [ name, value ] of Object.entries(metadata)) { + if (metadataTags.includes(name)) { + args.push('-metadata', `${name}=${value.replace(/[\u0000-\u0009]/g, "")}`); + } else { + throw `${name} metadata tag is not supported.`; + } + } + + return args; +} + const toRawHeaders = (headers) => { return Object.entries(headers) .map(([key, value]) => `${key}: ${value}\r\n`) @@ -38,6 +60,8 @@ const getCommand = (args) => { return [ffmpeg, args] } +const defaultAgent = new Agent(); + const proxy = async (streamInfo, res) => { const abortController = new AbortController(); const shutdown = () => ( @@ -56,7 +80,8 @@ const proxy = async (streamInfo, res) => { Range: streamInfo.range }, signal: abortController.signal, - maxRedirections: 16 + maxRedirections: 16, + dispatcher: defaultAgent, }); res.status(statusCode); @@ -101,12 +126,16 @@ const merge = (streamInfo, res) => { args = args.concat(ffmpegArgs[format]); - if (hlsExceptions.includes(streamInfo.service)) { - args.push('-bsf:a', 'aac_adtstoasc') + if (hlsExceptions.includes(streamInfo.service) && streamInfo.isHLS) { + if (streamInfo.service === "youtube" && format === "webm") { + args.push('-c:a', 'libopus'); + } else { + args.push('-c:a', 'aac', '-bsf:a', 'aac_adtstoasc'); + } } if (streamInfo.metadata) { - args = args.concat(metadataManager(streamInfo.metadata)) + args = args.concat(convertMetadataToFFmpeg(streamInfo.metadata)) } args.push('-f', format, 'pipe:3'); @@ -238,7 +267,7 @@ const convertAudio = (streamInfo, res) => { } if (streamInfo.metadata) { - args = args.concat(metadataManager(streamInfo.metadata)) + args = args.concat(convertMetadataToFFmpeg(streamInfo.metadata)) } args.push('-f', streamInfo.audioFormat === "m4a" ? "ipod" : streamInfo.audioFormat, 'pipe:3'); diff --git a/api/src/util/generate-jwt-secret.js b/api/src/util/generate-jwt-secret.js new file mode 100644 index 00000000..8db6e230 --- /dev/null +++ b/api/src/util/generate-jwt-secret.js @@ -0,0 +1,22 @@ +// run with `pnpm -r token:jwt` + +const makeSecureString = (length = 64) => { + const alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-'; + const out = []; + + while (out.length < length) { + for (const byte of crypto.getRandomValues(new Uint8Array(length))) { + if (byte < alphabet.length) { + out.push(alphabet[byte]); + } + + if (out.length === length) { + break; + } + } + } + + return out.join(''); +} + +console.log(`JWT_SECRET: ${JSON.stringify(makeSecureString(64))}`) diff --git a/api/src/util/setup.js b/api/src/util/setup.js deleted file mode 100644 index 34b870cb..00000000 --- a/api/src/util/setup.js +++ /dev/null @@ -1,105 +0,0 @@ -import { existsSync, unlinkSync, appendFileSync } from "fs"; -import { createInterface } from "readline"; -import { Cyan, Bright } from "../misc/console-text.js"; -import { loadJSON } from "../misc/load-from-fs.js"; -import { execSync } from "child_process"; - -const { version } = loadJSON("./package.json"); - -let envPath = './.env'; -let q = `${Cyan('?')} \x1b[1m`; -let ob = {}; -let rl = createInterface({ input: process.stdin, output: process.stdout }); - -let final = () => { - if (existsSync(envPath)) unlinkSync(envPath); - - for (let i in ob) { - appendFileSync(envPath, `${i}=${ob[i]}\n`) - } - console.log(Bright("\nAwesome! I've created a fresh .env file for you.")); - console.log(`${Bright("Now I'll run")} ${Cyan("npm install")} ${Bright("to install all dependencies. It shouldn't take long.\n\n")}`); - execSync('pnpm install', { stdio: [0, 1, 2] }); - console.log(`\n\n${Cyan("All done!\n")}`); - console.log(Bright("You can re-run this script at any time to update the configuration.")); - console.log(Bright("\nYou're now ready to start cobalt. Simply run ") + Cyan("npm start") + Bright('!\nHave fun :)')); - rl.close() -} - -console.log( - `${Cyan(`Hey, this is cobalt v.${version}!`)}\n${Bright("Let's start by creating a new ")}${Cyan(".env")}${Bright(" file. You can always change it later.")}` -) - -function setup() { - console.log(Bright("\nWhat kind of server will this instance be?\nOptions: api, web.")); - - rl.question(q, r1 => { - switch (r1.toLowerCase()) { - case 'api': - console.log(Bright("\nCool! What's the domain this API instance will be running on? (localhost)\nExample: api.cobalt.tools")); - - rl.question(q, apiURL => { - ob.API_URL = `http://localhost:9000/`; - ob.API_PORT = 9000; - if (apiURL && apiURL !== "localhost") ob.API_URL = `https://${apiURL.toLowerCase()}/`; - - console.log(Bright("\nGreat! Now, what port will it be running on? (9000)")); - - rl.question(q, apiPort => { - if (apiPort) ob.API_PORT = apiPort; - if (apiPort && (apiURL === "localhost" || !apiURL)) ob.API_URL = `http://localhost:${apiPort}/`; - - console.log(Bright("\nWhat will your instance's name be? Usually it's something like eu-nl aka region-country. (local)")); - - rl.question(q, apiName => { - ob.API_NAME = apiName.toLowerCase(); - if (!apiName || apiName === "local") ob.API_NAME = "local"; - - console.log(Bright("\nOne last thing: would you like to enable CORS? It allows other websites and extensions to use your instance's API.\ny/n (n)")); - - rl.question(q, apiCors => { - let answCors = apiCors.toLowerCase().trim(); - if (answCors !== "y" && answCors !== "yes") ob.CORS_WILDCARD = '0' - final() - }) - }) - }); - - }) - break; - case 'web': - console.log(Bright("\nAwesome! What's the domain this web app instance will be running on? (localhost)\nExample: cobalt.tools")); - - rl.question(q, webURL => { - ob.WEB_URL = `http://localhost:9001/`; - ob.WEB_PORT = 9001; - if (webURL && webURL !== "localhost") ob.WEB_URL = `https://${webURL.toLowerCase()}/`; - - console.log( - Bright("\nGreat! Now, what port will it be running on? (9001)") - ) - rl.question(q, webPort => { - if (webPort) ob.WEB_PORT = webPort; - if (webPort && (webURL === "localhost" || !webURL)) ob.WEB_URL = `http://localhost:${webPort}/`; - - console.log( - Bright("\nOne last thing: what default API domain should be used? (api.cobalt.tools)\nIf it's hosted locally, make sure to include the port:") + Cyan(" localhost:9000") - ); - - rl.question(q, apiURL => { - ob.API_URL = `https://${apiURL.toLowerCase()}/`; - if (apiURL.includes(':')) ob.API_URL = `http://${apiURL.toLowerCase()}/`; - if (!apiURL) ob.API_URL = "https://api.cobalt.tools/"; - final() - }) - }); - - }); - break; - default: - console.log(Bright("\nThis is not an option. Try again.")); - setup() - } - }) -} -setup() diff --git a/api/src/util/test-ci.js b/api/src/util/test-ci.js deleted file mode 100644 index 988fac7a..00000000 --- a/api/src/util/test-ci.js +++ /dev/null @@ -1,82 +0,0 @@ -import { env } from "../config.js"; -import { runTest } from "../misc/run-test.js"; -import { loadJSON } from "../misc/load-from-fs.js"; -import { Red, Bright } from "../misc/console-text.js"; -import { randomizeCiphers } from "../misc/randomize-ciphers.js"; - -import { services } from "../processing/service-config.js"; - -const tests = loadJSON('./src/util/tests.json'); - -// services that are known to frequently fail due to external -// factors (e.g. rate limiting) -const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube']) - -const action = process.argv[2]; -switch (action) { - case "get-services": - const fromConfig = Object.keys(services); - - const missingTests = fromConfig.filter( - service => !tests[service] || tests[service].length === 0 - ); - - if (missingTests.length) { - console.error('services have no tests:', missingTests); - console.log('[]'); - process.exitCode = 1; - break; - } - - console.log(JSON.stringify(fromConfig)); - break; - - case "run-tests-for": - const service = process.argv[3]; - let failed = false; - - if (!tests[service]) { - console.error('no such service:', service); - } - - env.streamLifespan = 10000; - env.apiURL = 'http://x'; - randomizeCiphers(); - - for (const test of tests[service]) { - const { name, url, params, expected } = test; - const canFail = test.canFail || finnicky.has(service); - - try { - await runTest(url, params, expected); - console.log(`${service}/${name}: ok`); - - } catch(e) { - failed = !canFail; - - let failText = canFail ? `${Red('FAIL')} (ignored)` : Bright(Red('FAIL')); - if (canFail && process.env.GITHUB_ACTION) { - console.log(`::warning title=${service}/${name.replace(/,/g, ';')}::failed and was ignored`); - } - - console.error(`${service}/${name}: ${failText}`); - const errorString = e.toString().split('\n'); - let c = '┃'; - errorString.forEach((line, index) => { - line = line.replace('!=', Red('!=')); - - if (index === errorString.length - 1) { - c = '┗'; - } - - console.error(` ${c}`, line); - }); - } - } - - process.exitCode = Number(failed); - break; - default: - console.error('invalid action:', action); - process.exitCode = 1; -} diff --git a/api/src/util/test.js b/api/src/util/test.js index 34afde7e..2ba555ed 100644 --- a/api/src/util/test.js +++ b/api/src/util/test.js @@ -1,84 +1,129 @@ -import "dotenv/config"; +import path from "node:path"; + +import { env } from "../config.js"; +import { runTest } from "../misc/run-test.js"; +import { loadJSON } from "../misc/load-from-fs.js"; +import { Red, Bright } from "../misc/console-text.js"; +import { randomizeCiphers } from "../misc/randomize-ciphers.js"; import { services } from "../processing/service-config.js"; -import { extract } from "../processing/url.js"; -import match from "../processing/match.js"; -import { loadJSON } from "../misc/load-from-fs.js"; -import { normalizeRequest } from "../processing/request.js"; -import { env } from "../config.js"; -env.apiURL = 'http://localhost:9000' -let tests = loadJSON('./src/util/tests.json'); +const getTestPath = service => path.join('./src/util/tests/', `./${service}.json`); +const getTests = (service) => loadJSON(getTestPath(service)); -let noTest = []; -let failed = []; -let success = 0; +// services that are known to frequently fail due to external +// factors (e.g. rate limiting) +const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube', 'vk', 'twitter']); -function addToFail(service, testName, url, status, response) { - failed.push({ - service: service, - name: testName, - url: url, - status: status, - response: response - }) -} -for (let i in services) { - if (tests[i]) { - console.log(`\nRunning tests for ${i}...\n`) - for (let k = 0; k < tests[i].length; k++) { - let test = tests[i][k]; +const runTestsFor = async (service) => { + const tests = getTests(service); + let softFails = 0, fails = 0; - console.log(`Running test ${k+1}: ${test.name}`); - console.log('params:'); - let params = {...{url: test.url}, ...test.params}; - console.log(params); - - let chck = await normalizeRequest(params); - if (chck.success) { - chck = chck.data; - - const parsed = extract(chck.url); - if (parsed === null) { - throw `Invalid URL: ${chck.url}` - } - - let j = await match({ - host: parsed.host, - patternMatch: parsed.patternMatch, - params: chck, - }); - console.log('\nReceived:'); - console.log(j) - if (j.status === test.expected.code && j.body.status === test.expected.status) { - console.log("\n✅ Success.\n"); - success++ - } else { - console.log(`\n❌ Fail. Expected: ${test.expected.code} & ${test.expected.status}, received: ${j.status} & ${j.body.status}\n`); - addToFail(i, test.name, test.url, j.body.status, j) - } - } else { - console.log("\n❌ couldn't validate the request JSON.\n"); - addToFail(i, test.name, test.url, "unknown", {}) - } - } - console.log("\n\n") - } else { - console.warn(`No tests found for ${i}.`); - noTest.push(i) + if (!tests) { + throw "no such service: " + service; } + + for (const test of tests) { + const { name, url, params, expected } = test; + const canFail = test.canFail || finnicky.has(service); + + try { + await runTest(url, params, expected); + console.log(`${service}/${name}: ok`); + + } catch (e) { + softFails += !canFail; + fails++; + + let failText = canFail ? `${Red('FAIL')} (ignored)` : Bright(Red('FAIL')); + if (canFail && process.env.GITHUB_ACTION) { + console.log(`::warning title=${service}/${name.replace(/,/g, ';')}::failed and was ignored`); + } + + console.error(`${service}/${name}: ${failText}`); + const errorString = e.toString().split('\n'); + let c = '┃'; + errorString.forEach((line, index) => { + line = line.replace('!=', Red('!=')); + + if (index === errorString.length - 1) { + c = '┗'; + } + + console.error(` ${c}`, line); + }); + } + } + + return { fails, softFails }; } -console.log(`✅ ${success} tests succeeded.`); -console.log(`❌ ${failed.length} tests failed.`); -console.log(`❔ ${noTest.length} services weren't tested.`); - -if (failed.length > 0) { - console.log(`\nFailed tests:`); - console.log(failed) +const printHeader = (service, padLen) => { + const padding = padLen - service.length; + service = service.padEnd(1 + service.length + padding, ' '); + console.log(service + '='.repeat(50)); } -if (noTest.length > 0) { - console.log(`\nMissing tests:`); - console.log(noTest) +const action = process.argv[2]; +switch (action) { + case "get-services": + const fromConfig = Object.keys(services); + + const missingTests = fromConfig.filter( + service => { + const tests = getTests(service); + return !tests || tests.length === 0 + } + ); + + if (missingTests.length) { + console.error('services have no tests:', missingTests); + process.exitCode = 1; + break; + } + + console.log(JSON.stringify(fromConfig)); + break; + + case "run-tests-for": + env.streamLifespan = 10000; + env.apiURL = 'http://x/'; + randomizeCiphers(); + + try { + const { softFails } = await runTestsFor(process.argv[3]); + process.exitCode = Number(!!softFails); + } catch (e) { + console.error(e); + process.exitCode = 1; + break; + } + + break; + default: + const maxHeaderLen = Object.keys(services).reduce((n, v) => v.length > n ? v.length : n, 0); + const failCounters = {}; + + env.streamLifespan = 10000; + env.apiURL = 'http://x/'; + randomizeCiphers(); + + for (const service in services) { + printHeader(service, maxHeaderLen); + const { fails, softFails } = await runTestsFor(service); + failCounters[service] = fails; + console.log(); + + if (!process.exitCode && softFails) + process.exitCode = 1; + } + + console.log('='.repeat(50 + maxHeaderLen)); + console.log( + Bright('total fails:'), + Object.values(failCounters).reduce((a, b) => a + b) + ); + for (const [ service, fails ] of Object.entries(failCounters)) { + if (fails) console.log(`${Bright(service)} fails: ${fails}`); + } } diff --git a/api/src/util/tests.json b/api/src/util/tests.json deleted file mode 100644 index ec1d1453..00000000 --- a/api/src/util/tests.json +++ /dev/null @@ -1,1526 +0,0 @@ -{ - "twitter": [ - { - "name": "regular video", - "url": "https://twitter.com/X/status/1697304622749086011", - "params": { - "audioFormat": "mp3" - }, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "video with mobile web mediaviewer", - "url": "https://twitter.com/X/status/1697304622749086011/mediaViewer?currentTweet=1697304622749086011¤tTweetUser=X¤tTweet=1697304622749086011¤tTweetUser=X", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "embedded twitter video", - "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", - "params": { - "audioFormat": "mp3" - }, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "mixed media (image + gif)", - "url": "https://twitter.com/sky_mj26/status/1807756010712428565", - "params": { - "audioFormat": "mp3" - }, - "expected": { - "code": 200, - "status": "picker" - } - }, - { - "name": "picker: mixed media (3 videos)", - "url": "https://twitter.com/DankGameAlert/status/1584726006094794774", - "params": { - "audioFormat": "mp3" - }, - "expected": { - "code": 200, - "status": "picker" - } - }, - { - "name": "audio from embedded twitter video (mp3, isAudioOnly)", - "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", - "params": { - "downloadMode": "audio", - "audioFormat": "mp3" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "audio from embedded twitter video (best, isAudioOnly)", - "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", - "params": { - "downloadMode": "audio", - "audioFormat": "best" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "audio from embedded twitter video (ogg, isAudioOnly, isAudioMuted)", - "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", - "params": { - "downloadMode": "audio", - "audioFormat": "best" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "muted embedded twitter video", - "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", - "params": { - "downloadMode": "mute", - "audioFormat": "mp3" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "retweeted video", - "url": "https://twitter.com/uwukko/status/1696901469633421344", - "params": {}, - "canFail": true, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "age restricted video", - "url": "https://x.com/XSpaces/status/1526955853743546372", - "params": {}, - "canFail": true, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "twitter voice + x.com link", - "url": "https://x.com/eggsaladscreams/status/1693089534886506756?s=46", - "params": {}, - "canFail": true, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "vxtwitter link", - "url": "https://vxtwitter.com/dustbin_nie/status/1624596567188717568?s=20", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "post with 1 image", - "url": "https://x.com/PopCrave/status/1815960083475423235", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "post with 4 images", - "url": "https://x.com/PopCrave/status/1816260887147114696", - "params": {}, - "expected": { - "code": 200, - "status": "picker" - } - }, - { - "name": "retweeted video, isAudioOnly", - "url": "https://twitter.com/hugekiwinuts/status/1618671150829309953?s=46&t=gItGzgwGQQJJaJrO6qc1Pg", - "params": { - "downloadMode": "mute", - "audioFormat": "mp3" - }, - "canFail": true, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "inexistent post", - "url": "https://twitter.com/test/status/9487653", - "params": { - "audioFormat": "best" - }, - "expected": { - "code": 400, - "status": "error" - } - }, - { - "name": "post with no media content", - "url": "https://twitter.com/elonmusk/status/1604617643973124097?s=20", - "params": { - "audioFormat": "best" - }, - "expected": { - "code": 400, - "status": "error" - } - }, - { - "name": "bookmarked video", - "url": "https://twitter.com/i/bookmarks?post_id=1828099210220294314", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "bookmarked photo", - "url": "https://twitter.com/i/bookmarks?post_id=1837430141179289876", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - } - ], - "soundcloud": [ - { - "name": "public song (best)", - "url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing", - "params": { - "audioFormat": "best" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "public song (mp3, isAudioMuted)", - "url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing", - "params": { - "downloadMode": "mute", - "audioFormat": "mp3" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "private song", - "url": "https://soundcloud.com/4kayy/unhappy-new-year-prod4kay/s-9bKbvwLdRWG", - "params": { - "audioFormat": "mp3" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "private song (wav, isAudioMuted)", - "url": "https://soundcloud.com/4kayy/unhappy-new-year-prod4kay/s-9bKbvwLdRWG", - "params": { - "downloadMode": "mute", - "audioFormat": "wav" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "private song (ogg, isAudioMuted, isAudioOnly)", - "url": "https://soundcloud.com/4kayy/unhappy-new-year-prod4kay/s-9bKbvwLdRWG", - "params": { - "downloadMode": "audio", - "audioFormat": "ogg" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "on.soundcloud link", - "url": "https://on.soundcloud.com/wLZre", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "on.soundcloud link, different stream type", - "url": "https://on.soundcloud.com/AG4c", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "no opus audio, fallback to mp3", - "url": "https://soundcloud.com/frums/credits", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - } - ], - "youtube": [ - { - "name": "4k video (h264, 1440)", - "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", - "params": { - "youtubeVideoCodec": "h264", - "videoQuality": "1440" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "4k video (vp9, 720)", - "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", - "params": { - "youtubeVideoCodec": "vp9", - "videoQuality": "720" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "4k video (av1, max)", - "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", - "params": { - "youtubeVideoCodec": "av1", - "videoQuality": "max" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "4k video (h264, 720)", - "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", - "params": { - "youtubeVideoCodec": "h264", - "videoQuality": "720" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "4k video (vp9, max, isAudioMuted)", - "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", - "params": { - "downloadMode": "mute", - "youtubeVideoCodec": "vp9", - "videoQuality": "max" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "4k video (h264, max, isAudioMuted)", - "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", - "params": { - "downloadMode": "mute", - "youtubeVideoCodec": "h264", - "videoQuality": "max" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "4k video (av1, max, isAudioMuted, isAudioOnly, mp3)", - "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", - "params": { - "downloadMode": "audio", - "audioFormat": "mp3", - "youtubeVideoCodec": "av1", - "videoQuality": "max" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "4k video (av1, max, isAudioMuted, isAudioOnly, best)", - "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", - "params": { - "downloadMode": "audio", - "audioFormat": "best", - "youtubeVideoCodec": "av1", - "videoQuality": "max" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "music (mp3, isAudioOnly, isAudioMuted)", - "url": "https://music.youtube.com/watch?v=5rGTsvZCEdk&feature=share", - "params": { - "downloadMode": "audio", - "audioFormat": "mp3" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "music (mp3)", - "url": "https://music.youtube.com/watch?v=5rGTsvZCEdk&feature=share", - "params": { - "audioFormat": "mp3" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "audio bitrate higher than video, no vp9 video in response (mp3, isAudioOnly)", - "url": "https://www.youtube.com/watch?v=t5nC_ucYBrc", - "params": { - "downloadMode": "audio", - "audioFormat": "mp3" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "audio bitrate higher than video, no vp9 video in response (vp9)", - "url": "https://www.youtube.com/watch?v=t5nC_ucYBrc", - "params": { - "youtubeVideoCodec": "vp9" - }, - "expected": { - "code": 400, - "status": "error" - } - }, - { - "name": "short, defaults", - "url": "https://www.youtube.com/shorts/r5FpeOJItbw", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "vr 360, av1, max", - "url": "https://www.youtube.com/watch?v=hEdzv7D4CbQ", - "params": { - "youtubeVideoCodec": "vp9", - "videoQuality": "max" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "live link, defaults", - "url": "https://www.youtube.com/live/ENxZS6PUDuI?feature=shared", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "inexistent video", - "url": "https://youtube.com/watch?v=gnjuHYWGEW", - "params": {}, - "expected": { - "code": 400, - "status": "error" - } - } - ], - "vk": [ - { - "name": "clip, defaults", - "url": "https://vk.com/clip-57274055_456239788", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "clip, 360", - "url": "https://vk.com/clip-57274055_456239788", - "params": { - "videoQuality": "360" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "clip different link, max", - "url": "https://vk.com/clips-57274055?z=clip-57274055_456239788", - "params": { - "videoQuality": "max" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "video, defaults", - "url": "https://vk.com/video-57274055_456239399", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "4k video", - "url": "https://vk.com/video-1112285_456248465", - "params": { - "videoQuality": "max" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "ancient video (fallback to 240p)", - "url": "https://vk.com/video-1959_28496479", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "inexistent video", - "url": "https://vk.com/video-53333333_456233333", - "params": {}, - "expected": { - "code": 400, - "status": "error" - } - } - ], - "tiktok": [ - { - "name": "long link video", - "url": "https://www.tiktok.com/@fatfatmillycat/video/7195741644585454894", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "images", - "url": "https://www.tiktok.com/@matryoshk4/video/7231234675476532526", - "params": {}, - "expected": { - "code": 200, - "status": "picker" - } - }, - { - "name": "long link inexistent", - "url": "https://www.tiktok.com/@blablabla/video/7120851458451417478", - "params": {}, - "expected": { - "code": 400, - "status": "error" - } - }, - { - "name": "short link inexistent", - "url": "https://vt.tiktok.com/2p4ewa7/", - "params": {}, - "expected": { - "code": 400, - "status": "error" - } - } - ], - "bilibili": [ - { - "name": "1080p video", - "url": "https://www.bilibili.com/video/BV18i4y1m7xV/", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "1080p video muted", - "url": "https://www.bilibili.com/video/BV18i4y1m7xV/", - "params": { - "downloadMode": "mute" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "1080p vertical video", - "url": "https://www.bilibili.com/video/BV1uu411z7VV/", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "1080p vertical video muted", - "url": "https://www.bilibili.com/video/BV1uu411z7VV/", - "params": { - "downloadMode": "mute" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "b23.tv shortlink", - "url": "https://b23.tv/lbMyOI9", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "bilibili.tv link", - "url": "https://www.bilibili.tv/en/video/4789599404426256", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - } - ], - "tumblr": [ - { - "name": "at.tumblr link", - "url": "https://at.tumblr.com/music/704177038274281472/n7x7pr7x4w2b", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "user subdomain link", - "url": "https://garfield-69.tumblr.com/post/696499862852780032", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "web app link", - "url": "https://www.tumblr.com/rongzhi/707729381162958848/english-added-by-me?source=share", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "tumblr audio", - "url": "https://www.tumblr.com/zedneon/737815079301562368/zedneon-ft-mr-sauceman-tech-n9ne-speed-of?source=share", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "tumblr video converted to audio", - "url": "https://garfield-69.tumblr.com/post/696499862852780032", - "params": { - "downloadMode": "audio" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - } - ], - "vimeo": [ - { - "name": "4k progressive", - "url": "https://vimeo.com/288386543", - "params": { - "videoQuality": "2160" - }, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "720p progressive", - "url": "https://vimeo.com/288386543", - "params": { - "videoQuality": "720" - }, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "1080p dash parcel", - "url": "https://vimeo.com/967252742", - "params": { - "videoQuality": "1440" - }, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "720p dash parcel", - "url": "https://vimeo.com/967252742", - "params": { - "videoQuality": "360" - }, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "private video", - "url": "https://vimeo.com/903115595/f14d06da38", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "mature video", - "url": "https://vimeo.com/973212054", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - } - ], - "reddit": [ - { - "name": "video with audio", - "url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "video with audio (isAudioOnly)", - "url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3", - "params": { - "downloadMode": "audio" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "video with audio (isAudioMuted)", - "url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3", - "params": { - "downloadMode": "mute" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "video without audio", - "url": "https://www.reddit.com/r/catvideos/comments/ftoeo7/luna_doesnt_want_to_be_bothered_while_shes_napping/?utm_source=share&utm_medium=web2x&context=3", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "actual gif, not looping video", - "url": "https://www.reddit.com/r/whenthe/comments/109wqy1/god_really_did_some_trolling/?utm_source=share&utm_medium=web2x&context=3", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "different audio link, live render", - "url": "https://www.reddit.com/r/TikTokCringe/comments/15hce91/asian_daddy_kink/", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - } - ], - "instagram": [ - { - "name": "single photo post", - "url": "https://www.instagram.com/p/CwIgW8Yu5-I/", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "various picker (photos + video)", - "url": "https://www.instagram.com/p/CvYrSgnsKjv/", - "params": {}, - "expected": { - "code": 200, - "status": "picker" - } - }, - { - "name": "reel", - "url": "https://www.instagram.com/reel/CoEBV3eM4QR/", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "regular video", - "url": "https://www.instagram.com/p/CmCVWoIr9OH/", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "reel (isAudioOnly)", - "url": "https://www.instagram.com/reel/CoEBV3eM4QR/", - "params": { - "downloadMode": "audio" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "reel (isAudioMuted)", - "url": "https://www.instagram.com/reel/CoEBV3eM4QR/", - "params": { - "downloadMode": "mute" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "inexistent reel", - "url": "https://www.instagram.com/reel/XXXXXXXXXX/", - "params": {}, - "expected": { - "code": 400, - "status": "error" - } - }, - { - "name": "inexistent post", - "url": "https://www.instagram.com/p/XXXXXXXXXX/", - "params": {}, - "expected": { - "code": 400, - "status": "error" - } - }, - { - "name": "post info in an array (for whatever reason??)", - "url": "https://www.instagram.com/reel/CrVB9tatUDv/?igshid=blaBlABALALbLABULLSHIT==", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "prone to get rate limited", - "url": "https://www.instagram.com/reel/CrO-T7Qo6rq/?igshid=fuckYouNoTrackingIdForYou==", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "ddinstagram link", - "url": "https://ddinstagram.com/p/CmCVWoIr9OH/", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "d.ddinstagram.com link", - "url": "https://d.ddinstagram.com/p/CmCVWoIr9OH/", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "g.ddinstagram.com link", - "url": "https://g.ddinstagram.com/p/CmCVWoIr9OH/", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - } - ], - "vine": [ - { - "name": "regular vine link (9+10=21)", - "url": "https://vine.co/v/huwVJIEJW50", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "regular vine link (isAudioOnly)", - "url": "https://vine.co/v/huwVJIEJW50", - "params": { - "downloadMode": "audio" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "regular vine link (isAudioMuted)", - "url": "https://vine.co/v/huwVJIEJW50", - "params": { - "downloadMode": "mute" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - } - ], - "pinterest": [ - { - "name": "regular video", - "url": "https://www.pinterest.com/pin/70437485604616/", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "regular video (isAudioOnly)", - "url": "https://www.pinterest.com/pin/70437485604616/", - "params": { - "downloadMode": "audio" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "regular video (isAudioMuted)", - "url": "https://www.pinterest.com/pin/70437485604616/", - "params": { - "downloadMode": "mute" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "regular video (.ca TLD)", - "url": "https://www.pinterest.ca/pin/70437485604616/", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "story", - "url": "https://www.pinterest.com/pin/gadget-cool-products-amazon-product-technology-kitchen-gadgets--1084663891475263837/", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "regular picture", - "url": "https://www.pinterest.com/pin/412994228343400946/", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "regular picture (.ca TLD)", - "url": "https://www.pinterest.ca/pin/412994228343400946/", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "regular gif", - "url": "https://www.pinterest.com/pin/814447913881127862/", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "regular gif (.ca TLD)", - "url": "https://www.pinterest.ca/pin/814447913881127862/", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - } - ], - "streamable": [ - { - "name": "regular video", - "url": "https://streamable.com/p9cln4", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "embedded link", - "url": "https://streamable.com/e/rsmo56", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "regular video (isAudioOnly)", - "url": "https://streamable.com/p9cln4", - "params": { - "downloadMode": "audio" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "regular video (isAudioMuted)", - "url": "https://streamable.com/p9cln4", - "params": { - "downloadMode": "mute" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "inexistent video", - "url": "https://streamable.com/XXXXXX", - "params": {}, - "expected": { - "code": 400, - "status": "error" - } - } - ], - "twitch": [ - { - "name": "clip", - "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "clip (isAudioOnly)", - "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", - "params": { - "downloadMode": "audio" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "clip (isAudioMuted)", - "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", - "params": { - "downloadMode": "mute" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - } - ], - "rutube": [ - { - "name": "regular video", - "url": "https://rutube.ru/video/b2f6c27649907c2fde0af411b03825eb/", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "vertical video (isAudioMuted)", - "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/", - "params": { - "downloadMode": "mute" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "russian region lock", - "url": "https://rutube.ru/video/b521653b4f71ece57b8ff54e57ca9b82/", - "params": {}, - "expected": { - "code": 400, - "status": "error" - } - }, - { - "name": "vertical video", - "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "yappy", - "url": "https://rutube.ru/yappy/c8c32bf7aee04412837656ea26c2b25b/", - "canFail": true, - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "shorts", - "url": "https://rutube.ru/shorts/935c1afafd0e7d52836d671967d53dac/", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "vertical video (isAudioOnly)", - "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/", - "params": { - "downloadMode": "audio" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "vertical video (isAudioMuted)", - "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/", - "params": { - "downloadMode": "mute" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "private video", - "url": "https://rutube.ru/video/private/1161415be0e686214bb2a498165cab3e/?p=_IL1G8RSnKutunnTYwhZ5A", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - } - ], - "ok": [ - { - "name": "regular video", - "url": "https://ok.ru/video/7204071410346", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - } - ], - "dailymotion": [ - { - "name": "regular video", - "url": "https://www.dailymotion.com/video/x8t1eho", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "private video", - "url": "https://www.dailymotion.com/video/k41fZWpx2TaAORA2nok", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "dai.ly shortened link", - "url": "https://dai.ly/k41fZWpx2TaAORA2nok", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - } - ], - "snapchat": [ - { - "name": "spotlight", - "url": "https://www.snapchat.com/spotlight/W7_EDlXWTBiXAEEniNoMPwAAYdWxucG9pZmNqAY46j0a5AY46j0YbAAAAAQ", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "shortlinked spotlight", - "url": "https://t.snapchat.com/4ZsiBLDi", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "story", - "url": "https://www.snapchat.com/add/bazerkmakane", - "params": {}, - "expected": { - "code": 200, - "status": "picker" - } - } - ], - "loom": [ - { - "name": "1080p video", - "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "1080p video (muted)", - "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", - "params": { - "downloadMode": "mute" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "1080p video (audio only)", - "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", - "params": { - "downloadMode": "audio" - }, - "expected": { - "code": 400, - "status": "error" - } - } - ], - "facebook": [ - { - "name": "direct video with username and id", - "url": "https://web.facebook.com/100048111287134/videos/1157798148685638/", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "direct video with id as query param", - "url": "https://web.facebook.com/watch/?v=883839773514682&ref=sharing", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "direct video with caption", - "url": "https://web.facebook.com/wood57/videos/𝐒𝐞𝐛𝐚đŦ𝐤𝐨đĻ-𝐟𝐮đĨđĨ/883839773514682", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "shortlink video", - "url": "https://fb.watch/r1K6XHMfGT/", - "canFail": true, - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "reel video", - "url": "https://web.facebook.com/reel/730293269054758", - "canFail": true, - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "shared video link", - "url": "https://www.facebook.com/share/v/NEf87jbPTvFE8LsL/", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "shared video link v2", - "url": "https://web.facebook.com/share/r/JFZfPVgLkiJQmWrr/", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - } - ], - "bsky": [ - { - "name": "horizontal video", - "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3giwtwp222m", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "horizontal video, recordWithMedia", - "url": "https://bsky.app/profile/did:plc:ywbm3iywnhzep3ckt6efhoh7/post/3l3wonhk23g2i", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "vertical video", - "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m", - "params": {}, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "vertical video (muted)", - "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m", - "params": { - "downloadMode": "mute" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "vertical video (audio)", - "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m", - "params": { - "downloadMode": "audio" - }, - "expected": { - "code": 200, - "status": "tunnel" - } - }, - { - "name": "single image", - "url": "https://bsky.app/profile/did:plc:k4a7d65fcyevbrnntjxh57go/post/3l33flpoygt26", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "several images", - "url": "https://bsky.app/profile/did:plc:rai7s6su2sy22ss7skouedl7/post/3kzxuxbiul626", - "params": {}, - "expected": { - "code": 200, - "status": "picker" - } - }, - { - "name": "deleted post/invalid user", - "url": "https://bsky.app/profile/notreal.bsky.team/post/3l2udah76ch2c", - "params": {}, - "expected": { - "code": 400, - "status": "error" - } - } - ], - "threads": [ - { - "name": "video", - "url": "https://www.threads.net/@zuck/post/CzecNnZPaxr", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "photo", - "url": "https://www.threads.net/@soren.iverson/post/C8PdJ59pMLr", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, - { - "name": "mixed media", - "url": "https://www.threads.net/@snazzahguy/post/C8Q7UZDseWz", - "params": {}, - "expected": { - "code": 200, - "status": "picker" - } - } - ] -} diff --git a/api/src/util/tests/bilibili.json b/api/src/util/tests/bilibili.json new file mode 100644 index 00000000..61d60134 --- /dev/null +++ b/api/src/util/tests/bilibili.json @@ -0,0 +1,60 @@ +[ + { + "name": "1080p video", + "url": "https://www.bilibili.com/video/BV18i4y1m7xV/", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "1080p video muted", + "url": "https://www.bilibili.com/video/BV18i4y1m7xV/", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "1080p vertical video", + "url": "https://www.bilibili.com/video/BV1uu411z7VV/", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "1080p vertical video muted", + "url": "https://www.bilibili.com/video/BV1uu411z7VV/", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "b23.tv shortlink", + "url": "https://b23.tv/lbMyOI9", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "bilibili.tv link", + "url": "https://www.bilibili.tv/en/video/4789599404426256", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/bsky.json b/api/src/util/tests/bsky.json new file mode 100644 index 00000000..6e1d6b2b --- /dev/null +++ b/api/src/util/tests/bsky.json @@ -0,0 +1,96 @@ +[ + { + "name": "horizontal video", + "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3giwtwp222m", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "horizontal video, recordWithMedia", + "url": "https://bsky.app/profile/did:plc:ywbm3iywnhzep3ckt6efhoh7/post/3l3wonhk23g2i", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "vertical video", + "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "vertical video (muted)", + "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "vertical video (audio)", + "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "single image", + "url": "https://bsky.app/profile/did:plc:k4a7d65fcyevbrnntjxh57go/post/3l33flpoygt26", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "gif with a quoted post", + "url": "https://bsky.app/profile/imlunahey.com/post/3lgajpn5dtk2t", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "gif alone in a post", + "url": "https://bsky.app/profile/imlunahey.com/post/3lgah3ovxnc2q", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "several images", + "url": "https://bsky.app/profile/did:plc:rai7s6su2sy22ss7skouedl7/post/3kzxuxbiul626", + "params": {}, + "expected": { + "code": 200, + "status": "picker" + } + }, + { + "name": "deleted post/invalid user", + "url": "https://bsky.app/profile/notreal.bsky.team/post/3l2udah76ch2c", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/dailymotion.json b/api/src/util/tests/dailymotion.json new file mode 100644 index 00000000..4de9302c --- /dev/null +++ b/api/src/util/tests/dailymotion.json @@ -0,0 +1,29 @@ +[ + { + "name": "regular video", + "url": "https://www.dailymotion.com/video/x8t1eho", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "private video", + "url": "https://www.dailymotion.com/video/k41fZWpx2TaAORA2nok", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "dai.ly shortened link", + "url": "https://dai.ly/k41fZWpx2TaAORA2nok", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/facebook.json b/api/src/util/tests/facebook.json new file mode 100644 index 00000000..876ac7fe --- /dev/null +++ b/api/src/util/tests/facebook.json @@ -0,0 +1,67 @@ +[ + { + "name": "direct video with username and id", + "url": "https://web.facebook.com/100048111287134/videos/1157798148685638/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "direct video with id as query param", + "url": "https://web.facebook.com/watch/?v=883839773514682&ref=sharing", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "direct video with caption", + "url": "https://web.facebook.com/wood57/videos/𝐒𝐞𝐛𝐚đŦ𝐤𝐨đĻ-𝐟𝐮đĨđĨ/883839773514682", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "shortlink video", + "url": "https://fb.watch/r1K6XHMfGT/", + "canFail": true, + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "reel video", + "url": "https://web.facebook.com/reel/730293269054758", + "canFail": true, + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "shared video link", + "url": "https://www.facebook.com/share/v/NEf87jbPTvFE8LsL/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "shared video link v2", + "url": "https://web.facebook.com/share/r/JFZfPVgLkiJQmWrr/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/instagram.json b/api/src/util/tests/instagram.json new file mode 100644 index 00000000..2ee42194 --- /dev/null +++ b/api/src/util/tests/instagram.json @@ -0,0 +1,123 @@ +[ + { + "name": "single photo post", + "url": "https://www.instagram.com/p/CwIgW8Yu5-I/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "various picker (photos + video)", + "url": "https://www.instagram.com/p/CvYrSgnsKjv/", + "params": {}, + "expected": { + "code": 200, + "status": "picker" + } + }, + { + "name": "reel", + "url": "https://www.instagram.com/reel/CoEBV3eM4QR/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "regular video", + "url": "https://www.instagram.com/p/CmCVWoIr9OH/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "reel (isAudioOnly)", + "url": "https://www.instagram.com/reel/CoEBV3eM4QR/", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "reel (isAudioMuted)", + "url": "https://www.instagram.com/reel/CoEBV3eM4QR/", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "inexistent reel", + "url": "https://www.instagram.com/reel/XXXXXXXXXX/", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "inexistent post", + "url": "https://www.instagram.com/p/XXXXXXXXXX/", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "post info in an array (for whatever reason??)", + "url": "https://www.instagram.com/reel/CrVB9tatUDv/?igshid=blaBlABALALbLABULLSHIT==", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "prone to get rate limited", + "url": "https://www.instagram.com/reel/CrO-T7Qo6rq/?igshid=fuckYouNoTrackingIdForYou==", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "ddinstagram link", + "url": "https://ddinstagram.com/p/CmCVWoIr9OH/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "d.ddinstagram.com link", + "url": "https://d.ddinstagram.com/p/CmCVWoIr9OH/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "g.ddinstagram.com link", + "url": "https://g.ddinstagram.com/p/CmCVWoIr9OH/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/loom.json b/api/src/util/tests/loom.json new file mode 100644 index 00000000..cc4273d3 --- /dev/null +++ b/api/src/util/tests/loom.json @@ -0,0 +1,33 @@ +[ + { + "name": "1080p video", + "url": "https://www.loom.com/share/d165fd054a294d8a8587807bcc50c885", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "1080p video (muted)", + "url": "https://www.loom.com/share/d165fd054a294d8a8587807bcc50c885", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "1080p video (audio only)", + "url": "https://www.loom.com/share/d165fd054a294d8a8587807bcc50c885", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 400, + "status": "error" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/ok.json b/api/src/util/tests/ok.json new file mode 100644 index 00000000..8eb103eb --- /dev/null +++ b/api/src/util/tests/ok.json @@ -0,0 +1,11 @@ +[ + { + "name": "regular video", + "url": "https://ok.ru/video/7204071410346", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/pinterest.json b/api/src/util/tests/pinterest.json new file mode 100644 index 00000000..4760dd36 --- /dev/null +++ b/api/src/util/tests/pinterest.json @@ -0,0 +1,87 @@ +[ + { + "name": "regular video", + "url": "https://www.pinterest.com/pin/70437485604616/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "regular video (isAudioOnly)", + "url": "https://www.pinterest.com/pin/70437485604616/", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "regular video (isAudioMuted)", + "url": "https://www.pinterest.com/pin/70437485604616/", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "regular video (.ca TLD)", + "url": "https://www.pinterest.ca/pin/70437485604616/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "story", + "url": "https://www.pinterest.com/pin/gadget-cool-products-amazon-product-technology-kitchen-gadgets--1084663891475263837/", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "regular picture", + "url": "https://www.pinterest.com/pin/412994228343400946/", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "regular picture (.ca TLD)", + "url": "https://www.pinterest.ca/pin/412994228343400946/", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "regular gif", + "url": "https://www.pinterest.com/pin/643170390530326178/", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "regular gif (.ca TLD)", + "url": "https://www.pinterest.ca/pin/643170390530326178/", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/reddit.json b/api/src/util/tests/reddit.json new file mode 100644 index 00000000..3afc6126 --- /dev/null +++ b/api/src/util/tests/reddit.json @@ -0,0 +1,60 @@ +[ + { + "name": "video with audio", + "url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "video with audio (isAudioOnly)", + "url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "video with audio (isAudioMuted)", + "url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "video without audio", + "url": "https://www.reddit.com/r/catvideos/comments/ftoeo7/luna_doesnt_want_to_be_bothered_while_shes_napping/?utm_source=share&utm_medium=web2x&context=3", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "actual gif, not looping video", + "url": "https://www.reddit.com/r/whenthe/comments/109wqy1/god_really_did_some_trolling/?utm_source=share&utm_medium=web2x&context=3", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "different audio link, live render", + "url": "https://www.reddit.com/r/TikTokCringe/comments/15hce91/asian_daddy_kink/", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/rutube.json b/api/src/util/tests/rutube.json new file mode 100644 index 00000000..2eaf69bf --- /dev/null +++ b/api/src/util/tests/rutube.json @@ -0,0 +1,100 @@ +[ + { + "name": "regular video", + "url": "https://rutube.ru/video/b2f6c27649907c2fde0af411b03825eb/", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "vertical video (isAudioMuted)", + "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "russian region lock", + "url": "https://rutube.ru/video/b521653b4f71ece57b8ff54e57ca9b82/", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "vertical video", + "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "yappy", + "url": "https://rutube.ru/yappy/c8c32bf7aee04412837656ea26c2b25b/", + "canFail": true, + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "shorts", + "url": "https://rutube.ru/shorts/935c1afafd0e7d52836d671967d53dac/", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "vertical video (isAudioOnly)", + "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "vertical video (isAudioMuted)", + "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "private video", + "url": "https://rutube.ru/video/private/1161415be0e686214bb2a498165cab3e/?p=_IL1G8RSnKutunnTYwhZ5A", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "region locked video, should fail", + "canFail": true, + "url": "https://rutube.ru/video/e7ac82708cc22bd068a3bf6a7004d1b1/", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/snapchat.json b/api/src/util/tests/snapchat.json new file mode 100644 index 00000000..44f764ce --- /dev/null +++ b/api/src/util/tests/snapchat.json @@ -0,0 +1,29 @@ +[ + { + "name": "spotlight", + "url": "https://www.snapchat.com/spotlight/W7_EDlXWTBiXAEEniNoMPwAAYdWxucG9pZmNqAY46j0a5AY46j0YbAAAAAQ", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "shortlinked spotlight", + "url": "https://t.snapchat.com/4ZsiBLDi", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "story", + "url": "https://www.snapchat.com/add/bazerkmakane", + "params": {}, + "expected": { + "code": 200, + "status": "picker" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/soundcloud.json b/api/src/util/tests/soundcloud.json new file mode 100644 index 00000000..04ed8632 --- /dev/null +++ b/api/src/util/tests/soundcloud.json @@ -0,0 +1,106 @@ +[ + { + "name": "public song (best)", + "url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing", + "params": { + "audioFormat": "best" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "public song (mp3, isAudioMuted)", + "url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing", + "params": { + "downloadMode": "mute", + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "private song", + "url": "https://soundcloud.com/user-798052861/asdasdasdsdsd/s-9TqZ7edLJ90", + "params": { + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "private song (wav, isAudioMuted)", + "url": "https://soundcloud.com/user-798052861/asdasdasdsdsd/s-9TqZ7edLJ90", + "params": { + "downloadMode": "mute", + "audioFormat": "wav" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "private song (ogg, isAudioMuted, isAudioOnly)", + "url": "https://soundcloud.com/user-798052861/asdasdasdsdsd/s-9TqZ7edLJ90", + "params": { + "downloadMode": "audio", + "audioFormat": "ogg" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "on.soundcloud link", + "url": "https://on.soundcloud.com/wLZre", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "on.soundcloud link, different stream type", + "url": "https://on.soundcloud.com/AG4c", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "no opus audio, fallback to mp3", + "url": "https://soundcloud.com/frums/credits", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "go+ song, should fail", + "url": "https://soundcloud.com/dualipa/illusion-1", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "region locked song, should fail", + "canFail": true, + "url": "https://soundcloud.com/gotye/somebody-2024-feat-kimbra", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/streamable.json b/api/src/util/tests/streamable.json new file mode 100644 index 00000000..bf03c228 --- /dev/null +++ b/api/src/util/tests/streamable.json @@ -0,0 +1,51 @@ +[ + { + "name": "regular video", + "url": "https://streamable.com/p9cln4", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "embedded link", + "url": "https://streamable.com/e/rsmo56", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "regular video (isAudioOnly)", + "url": "https://streamable.com/p9cln4", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "regular video (isAudioMuted)", + "url": "https://streamable.com/p9cln4", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "inexistent video", + "url": "https://streamable.com/XXXXXX", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/tiktok.json b/api/src/util/tests/tiktok.json new file mode 100644 index 00000000..c8dbce8c --- /dev/null +++ b/api/src/util/tests/tiktok.json @@ -0,0 +1,47 @@ +[ + { + "name": "long link video", + "url": "https://www.tiktok.com/@fatfatmillycat/video/7195741644585454894", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "images", + "url": "https://www.tiktok.com/@matryoshk4/video/7231234675476532526", + "params": {}, + "expected": { + "code": 200, + "status": "picker" + } + }, + { + "name": "long link inexistent", + "url": "https://www.tiktok.com/@blablabla/video/7120851458451417478", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "short link inexistent", + "url": "https://vt.tiktok.com/2p4ewa7/", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "age restricted video", + "url": "https://www.tiktok.com/@.kyle.films/video/7415757181145877793", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/tumblr.json b/api/src/util/tests/tumblr.json new file mode 100644 index 00000000..87352255 --- /dev/null +++ b/api/src/util/tests/tumblr.json @@ -0,0 +1,49 @@ +[ + { + "name": "at.tumblr link", + "url": "https://at.tumblr.com/music/704177038274281472/n7x7pr7x4w2b", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "user subdomain link", + "url": "https://garfield-69.tumblr.com/post/696499862852780032", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "web app link", + "url": "https://www.tumblr.com/rongzhi/707729381162958848/english-added-by-me?source=share", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "tumblr audio", + "url": "https://www.tumblr.com/zedneon/737815079301562368/zedneon-ft-mr-sauceman-tech-n9ne-speed-of?source=share", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "tumblr video converted to audio", + "url": "https://garfield-69.tumblr.com/post/696499862852780032", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/twitch.json b/api/src/util/tests/twitch.json new file mode 100644 index 00000000..fd6b84af --- /dev/null +++ b/api/src/util/tests/twitch.json @@ -0,0 +1,33 @@ +[ + { + "name": "clip", + "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "clip (isAudioOnly)", + "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "clip (isAudioMuted)", + "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/twitter.json b/api/src/util/tests/twitter.json new file mode 100644 index 00000000..0024d097 --- /dev/null +++ b/api/src/util/tests/twitter.json @@ -0,0 +1,213 @@ +[ + { + "name": "regular video", + "url": "https://twitter.com/X/status/1697304622749086011", + "params": { + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "video with mobile web mediaviewer", + "url": "https://twitter.com/X/status/1697304622749086011/mediaViewer?currentTweet=1697304622749086011¤tTweetUser=X¤tTweet=1697304622749086011¤tTweetUser=X", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "embedded twitter video", + "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", + "params": { + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "mixed media (image + gif)", + "url": "https://twitter.com/sky_mj26/status/1807756010712428565", + "params": { + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "picker" + } + }, + { + "name": "picker: mixed media (3 videos)", + "url": "https://twitter.com/DankGameAlert/status/1584726006094794774", + "params": { + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "picker" + } + }, + { + "name": "audio from embedded twitter video (mp3, isAudioOnly)", + "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", + "params": { + "downloadMode": "audio", + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "audio from embedded twitter video (best, isAudioOnly)", + "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", + "params": { + "downloadMode": "audio", + "audioFormat": "best" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "audio from embedded twitter video (ogg, isAudioOnly, isAudioMuted)", + "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", + "params": { + "downloadMode": "audio", + "audioFormat": "best" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "muted embedded twitter video", + "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20", + "params": { + "downloadMode": "mute", + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "retweeted video", + "url": "https://twitter.com/uwukko/status/1696901469633421344", + "params": {}, + "canFail": true, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "age restricted video", + "url": "https://x.com/XSpaces/status/1526955853743546372", + "params": {}, + "canFail": true, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "twitter voice + x.com link", + "url": "https://x.com/eggsaladscreams/status/1693089534886506756?s=46", + "params": {}, + "canFail": true, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "vxtwitter link", + "url": "https://vxtwitter.com/dustbin_nie/status/1624596567188717568?s=20", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "post with 1 image", + "url": "https://x.com/PopCrave/status/1815960083475423235", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "post with 4 images", + "url": "https://x.com/PopCrave/status/1816260887147114696", + "params": {}, + "expected": { + "code": 200, + "status": "picker" + } + }, + { + "name": "retweeted video, isAudioOnly", + "url": "https://twitter.com/hugekiwinuts/status/1618671150829309953?s=46&t=gItGzgwGQQJJaJrO6qc1Pg", + "params": { + "downloadMode": "mute", + "audioFormat": "mp3" + }, + "canFail": true, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "inexistent post", + "url": "https://twitter.com/test/status/9487653", + "params": { + "audioFormat": "best" + }, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "post with no media content", + "url": "https://twitter.com/elonmusk/status/1604617643973124097?s=20", + "params": { + "audioFormat": "best" + }, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "bookmarked video", + "url": "https://twitter.com/i/bookmarks?post_id=1828099210220294314", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "bookmarked photo", + "url": "https://twitter.com/i/bookmarks?post_id=1837430141179289876", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/vimeo.json b/api/src/util/tests/vimeo.json new file mode 100644 index 00000000..6c44a47d --- /dev/null +++ b/api/src/util/tests/vimeo.json @@ -0,0 +1,64 @@ +[ + { + "name": "4k progressive", + "url": "https://vimeo.com/288386543", + "params": { + "videoQuality": "2160" + }, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "720p progressive", + "url": "https://vimeo.com/288386543", + "params": { + "videoQuality": "720" + }, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "1080p dash parcel", + "url": "https://vimeo.com/967252742", + "params": { + "videoQuality": "1440" + }, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "720p dash parcel", + "url": "https://vimeo.com/967252742", + "params": { + "videoQuality": "360" + }, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "private video", + "url": "https://vimeo.com/903115595/f14d06da38", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "mature video", + "url": "https://vimeo.com/973212054", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + } +] \ No newline at end of file diff --git a/api/src/util/tests/vk.json b/api/src/util/tests/vk.json new file mode 100644 index 00000000..71720af5 --- /dev/null +++ b/api/src/util/tests/vk.json @@ -0,0 +1,82 @@ +[ + { + "name": "clip, defaults", + "url": "https://vk.com/clip-57274055_456239788", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "clip, 360", + "url": "https://vk.com/clip-57274055_456239788", + "params": { + "videoQuality": "360" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "clip different link, max", + "url": "https://vk.com/clips-57274055?z=clip-57274055_456239788", + "params": { + "videoQuality": "max" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "video, defaults", + "url": "https://vk.com/video-57274055_456239399", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "big 4k video", + "url": "https://vk.com/video-1112285_456248465", + "params": { + "videoQuality": "max" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "short 4k video, 480p, vkvideo.ru domain", + "url": "https://vkvideo.ru/video-26006257_456245538", + "params": { + "videoQuality": "480" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "ancient video (fallback to 240p)", + "url": "https://vk.com/video-1959_28496479", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "inexistent video", + "url": "https://vk.com/video-53333333_456233333", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + } +] diff --git a/api/src/util/tests/xiaohongshu.json b/api/src/util/tests/xiaohongshu.json new file mode 100644 index 00000000..de632a77 --- /dev/null +++ b/api/src/util/tests/xiaohongshu.json @@ -0,0 +1,58 @@ +[ + { + "name": "long link video", + "url": "https://www.xiaohongshu.com/discovery/item/6789065900000000210035fc?source=webshare&xhsshare=pc_web&xsec_token=CBustnz_Twf1BSybpe5-D-BzUb-Bx28DPLb418TN9S9Kk&xsec_source=pc_share", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "picker with multiple live photos", + "url": "https://www.xiaohongshu.com/explore/67847fa1000000000203e6ed?xsec_token=CBzyP7Y44PPpsM20lgxqrIIJMHqOLemusDsRcmsX0cTpk", + "params": {}, + "expected": { + "code": 200, + "status": "picker" + } + }, + { + "name": "one photo", + "url": "https://www.xiaohongshu.com/explore/676e132d000000000b016f68?xsec_token=ABRv6LKzizOFeSaf2HnnBkdBqniB5Ak1fI8tMAHzO31jA", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "short link, might expire eventually", + "url": "https://xhslink.com/a/czn4z6c1tic4", + "canFail": true, + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "wrong note id", + "url": "https://www.xiaohongshu.com/discovery/item/6789065911100000210035fc?source=webshare&xhsshare=pc_web&xsec_token=CBustnz_Twf1BSybpe5-D-BzUb-Bx28DPLb418TN9S9Kk&xsec_source=pc_share", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "short link, wrong id", + "url": "https://xhslink.com/a/aaaaaa", + "canFail": true, + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + } +] diff --git a/api/src/util/tests/youtube.json b/api/src/util/tests/youtube.json new file mode 100644 index 00000000..0655e683 --- /dev/null +++ b/api/src/util/tests/youtube.json @@ -0,0 +1,240 @@ +[ + { + "name": "4k video (h264, 1440)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "youtubeVideoCodec": "h264", + "videoQuality": "1440" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "4k video (vp9, 720)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "youtubeVideoCodec": "vp9", + "videoQuality": "720" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "4k video (av1, max)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "youtubeVideoCodec": "av1", + "videoQuality": "max" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "4k video (h264, 720)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "youtubeVideoCodec": "h264", + "videoQuality": "720" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "4k video (vp9, max, isAudioMuted)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "downloadMode": "mute", + "youtubeVideoCodec": "vp9", + "videoQuality": "max" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "4k video (h264, max, isAudioMuted)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "downloadMode": "mute", + "youtubeVideoCodec": "h264", + "videoQuality": "max" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "4k video (av1, max, isAudioMuted, isAudioOnly, mp3)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "downloadMode": "audio", + "audioFormat": "mp3", + "youtubeVideoCodec": "av1", + "videoQuality": "max" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "4k video (av1, max, isAudioMuted, isAudioOnly, best)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "downloadMode": "audio", + "audioFormat": "best", + "youtubeVideoCodec": "av1", + "videoQuality": "max" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "music (mp3, isAudioOnly, isAudioMuted)", + "url": "https://music.youtube.com/watch?v=5rGTsvZCEdk&feature=share", + "params": { + "downloadMode": "audio", + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "music (mp3)", + "url": "https://music.youtube.com/watch?v=5rGTsvZCEdk&feature=share", + "params": { + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "audio bitrate higher than video, no vp9 video in response (mp3, isAudioOnly)", + "url": "https://www.youtube.com/watch?v=t5nC_ucYBrc", + "params": { + "downloadMode": "audio", + "audioFormat": "mp3" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "short, defaults", + "url": "https://www.youtube.com/shorts/r5FpeOJItbw", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "vr 360, av1, max", + "url": "https://www.youtube.com/watch?v=hEdzv7D4CbQ", + "params": { + "youtubeVideoCodec": "vp9", + "videoQuality": "max" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "live link, defaults", + "url": "https://www.youtube.com/live/ENxZS6PUDuI?feature=shared", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "inexistent video", + "url": "https://youtube.com/watch?v=gnjuHYWGEW", + "params": {}, + "expected": { + "code": 400, + "status": "error" + } + }, + { + "name": "broken audioOnly download", + "url": "https://www.youtube.com/watch?v=ink80Al5nbw", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "hls video (h264, 1440p)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "youtubeVideoCodec": "h264", + "videoQuality": "1440", + "youtubeHLS": true + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "hls video (vp9, 360p)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "youtubeVideoCodec": "vp9", + "videoQuality": "360", + "youtubeHLS": true + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "hls video (audio mode)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "downloadMode": "audio", + "youtubeHLS": true + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "hls video (audio mode, best format)", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "params": { + "downloadMode": "audio", + "youtubeHLS": true, + "audioFormat": "best" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + } +] \ No newline at end of file diff --git a/docs/api.md b/docs/api.md index fc09a441..fb1a1450 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,9 +1,44 @@ # cobalt api documentation this document provides info about methods and acceptable variables for all cobalt api requests. -> if you are looking for the documentation for the old (7.x) api, you can find -> it [here](https://github.com/imputnet/cobalt/blob/7/docs/api.md) - +> [!IMPORTANT] +> hosted api instances (such as `api.cobalt.tools`) use bot protection and are **not** intended to be used in other projects without explicit permission. if you want to access the cobalt api reliably, you should [host your own instance](/docs/run-an-instance.md) or ask an instance owner for access. + +## authentication +an api instance may be configured to require you to authenticate yourself. +if this is the case, you will typically receive an [error response](#error-response) +with a **`api.auth..missing`** code, which tells you that a particular method +of authentication is required. + +authentication is done by passing the `Authorization` header, containing +the authentication scheme and the token: +``` +Authorization: +``` + +currently, cobalt supports two ways of authentication. an instance can +choose to configure both, or neither: +- [`Api-Key`](#api-key-authentication) +- [`Bearer`](#bearer-authentication) + +### api-key authentication +the api key authentication is the most straightforward. the instance owner +will assign you an api key which you can then use to authenticate like so: +``` +Authorization: Api-Key aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee +``` + +if you are an instance owner and wish to configure api key authentication, +see the [instance](run-an-instance.md#api-key-file-format) documentation! + +### bearer authentication +the cobalt server may be configured to issue JWT bearers, which are short-lived +tokens intended for use by regular users (e.g. after passing a challenge). +currently, cobalt can issue tokens for successfully solved [turnstile](run-an-instance.md#list-of-all-environment-variables) +challenge, if the instance has turnstile configured. the resulting token is passed like so: +``` +Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9... +``` ## POST: `/` cobalt's main processing endpoint. @@ -11,9 +46,10 @@ cobalt's main processing endpoint. request body type: `application/json` response body type: `application/json` -``` -⚠ī¸ you must include Accept and Content-Type headers with every `POST /` request. +> [!IMPORTANT] +> you must include `Accept` and `Content-Type` headers with every `POST /` request. +``` Accept: application/json Content-Type: application/json ``` @@ -28,13 +64,13 @@ Content-Type: application/json | `filenameStyle` | `string` | `classic / pretty / basic / nerdy` | `classic` | changes the way files are named. previews can be seen in the web app. | | `downloadMode` | `string` | `auto / audio / mute` | `auto` | `audio` downloads only the audio, `mute` skips the audio track in videos. | | `youtubeVideoCodec` | `string` | `h264 / av1 / vp9` | `h264` | `h264` is recommended for phones. | -| `youtubeDubLang` | `string` | `en / ru / cs / ja / ...` | -- | specifies the language of audio to download, when the youtube video is dubbed | -| `youtubeDubBrowserLang` | `boolean` | `true / false` | `false` | uses value from the Accept-Language header for `youtubeDubLang`. | +| `youtubeDubLang` | `string` | `en / ru / cs / ja / es-US / ...` | -- | specifies the language of audio to download when a youtube video is dubbed. | | `alwaysProxy` | `boolean` | `true / false` | `false` | tunnels all downloads through the processing server, even when not necessary. | | `disableMetadata` | `boolean` | `true / false` | `false` | disables file metadata when set to `true`. | | `tiktokFullAudio` | `boolean` | `true / false` | `false` | enables download of original sound used in a tiktok video. | -| `tiktokH265` | `boolean` | `true / false` | `false` | changes whether 1080p h265 videos are preferred or not. | +| `tiktokH265` | `boolean` | `true / false` | `false` | allows h265 videos when enabled. applies to tiktok & xiaohongshu. | | `twitterGif` | `boolean` | `true / false` | `true` | changes whether twitter gifs are converted to .gif | +| `youtubeHLS` | `boolean` | `true / false` | `false` | specifies whether to use HLS for downloading video or audio from youtube. | ### response the response will always be a JSON object containing the `status` key, which will be one of: @@ -108,3 +144,18 @@ response body type: `application/json` | `commit` | `string` | commit hash | | `branch` | `string` | git branch | | `remote` | `string` | git remote | + +## POST: `/session` + +used for generating JWT tokens, if enabled. currently, cobalt only supports +generating tokens when a [turnstile](run-an-instance.md#list-of-all-environment-variables) challenge solution +is submitted by the client. + +the turnstile challenge response is submitted via the `cf-turnstile-response` header. +### response body +| key | type | description | +|:----------------|:-----------|:-------------------------------------------------------| +| `token` | `string` | a `Bearer` token used for later request authentication | +| `exp` | `number` | number in seconds indicating the token lifetime | + +on failure, an [error response](#error-response) is returned. diff --git a/docs/configure-for-youtube.md b/docs/configure-for-youtube.md new file mode 100644 index 00000000..fe286d86 --- /dev/null +++ b/docs/configure-for-youtube.md @@ -0,0 +1,33 @@ +# how to configure a cobalt instance for youtube +if you get various errors when attempting to download videos that are: +publicly available, not region locked, and not age-restricted; +then your instance's ip address may have bad reputation. + +in this case you have to use disposable google accounts. +there's no other known workaround as of time of writing this document. + +> [!CAUTION] +> **NEVER** use your personal google account for downloading videos via any means. +> you can use any google accounts that you're willing to sacrifice, +> but be prepared to have them **permanently suspended**. +> +> we recommend that you use accounts that don't link back to your personal google account or identity, just in case. +> +> use incognito mode when signing in. +> we also recommend using vpn/proxy services (such as [mullvad](https://mullvad.net/)). + +1. if you haven't done it already, clone the cobalt repo, go to the cloned directory, and run `pnpm install` + +2. run `pnpm -C api token:youtube` + +3. follow instructions, use incognito mode in your browser when signing in. +i cannot stress this enough, but again, **DO NOT USE YOUR PERSONAL GOOGLE ACCOUNT**. + +4. once you have the oauth token, add it to `youtube_oauth` in your cookies file. +you can see an [example here](/docs/examples/cookies.example.json). +you can have several account tokens in this file, if you like. + +5. all done! enjoy freedom. + +### liability +you're responsible for any damage done to any of your google accounts or any other damages. you do this by yourself and at your own risk. diff --git a/docs/examples/cookies.example.json b/docs/examples/cookies.example.json index 73f3378d..7996adeb 100644 --- a/docs/examples/cookies.example.json +++ b/docs/examples/cookies.example.json @@ -10,5 +10,8 @@ ], "twitter": [ "auth_token=; ct0=" + ], + "youtube_oauth": [ + "" ] } diff --git a/docs/examples/docker-compose.example.yml b/docs/examples/docker-compose.example.yml index 7d3442de..e56c0a21 100644 --- a/docs/examples/docker-compose.example.yml +++ b/docs/examples/docker-compose.example.yml @@ -1,30 +1,40 @@ services: cobalt-api: image: ghcr.io/imputnet/cobalt:10 + + init: true + read_only: true restart: unless-stopped container_name: cobalt-api - init: true - ports: - 9000:9000/tcp - # if you're using a reverse proxy, uncomment the next line and remove the one above (9000:9000/tcp): - #- 127.0.0.1:9000:9000 + # if you use a reverse proxy (such as nginx), + # uncomment the next line and remove the one above (9000:9000/tcp): + # - 127.0.0.1:9000:9000 environment: - # replace https://api.cobalt.tools/ with your instance's target url in same format - API_URL: "https://api.cobalt.tools/" - # if you want to use cookies when fetching data from services, uncomment the next line and the lines under volume + # replace https://api.url.example/ with your instance's url + # or else tunneling functionality won't work properly + API_URL: "https://api.url.example/" + + # if you want to use cookies for fetching data from services, + # uncomment the next line & volumes section # COOKIE_PATH: "/cookies.json" - # see docs/run-an-instance.md for more information + + # it's recommended to configure bot protection or api keys if the instance is public, + # see /docs/protect-an-instance.md for more info + + # see /docs/run-an-instance.md for more variables that you can use here + labels: - com.centurylinklabs.watchtower.scope=cobalt - # if you want to use cookies when fetching data from services, uncomment volumes and next line - #volumes: - #- ./cookies.json:/cookies.json + # uncomment only if you use the COOKIE_PATH variable + # volumes: + # - ./cookies.json:/cookies.json - # update the cobalt image automatically with watchtower + # watchtower updates the cobalt image automatically watchtower: image: ghcr.io/containrrr/watchtower restart: unless-stopped diff --git a/docs/images/protect-an-instance/add.png b/docs/images/protect-an-instance/add.png new file mode 100644 index 00000000..e186a65c Binary files /dev/null and b/docs/images/protect-an-instance/add.png differ diff --git a/docs/images/protect-an-instance/created.png b/docs/images/protect-an-instance/created.png new file mode 100644 index 00000000..546a6897 Binary files /dev/null and b/docs/images/protect-an-instance/created.png differ diff --git a/docs/images/protect-an-instance/domain.png b/docs/images/protect-an-instance/domain.png new file mode 100644 index 00000000..249a8a92 Binary files /dev/null and b/docs/images/protect-an-instance/domain.png differ diff --git a/docs/images/protect-an-instance/mode.png b/docs/images/protect-an-instance/mode.png new file mode 100644 index 00000000..242b35a5 Binary files /dev/null and b/docs/images/protect-an-instance/mode.png differ diff --git a/docs/images/protect-an-instance/name.png b/docs/images/protect-an-instance/name.png new file mode 100644 index 00000000..fd39dc95 Binary files /dev/null and b/docs/images/protect-an-instance/name.png differ diff --git a/docs/images/protect-an-instance/sidebar.png b/docs/images/protect-an-instance/sidebar.png new file mode 100644 index 00000000..8294c4a0 Binary files /dev/null and b/docs/images/protect-an-instance/sidebar.png differ diff --git a/docs/images/troubleshooting/clipboard/config.png b/docs/images/troubleshooting/clipboard/config.png deleted file mode 100644 index b0c0a048..00000000 Binary files a/docs/images/troubleshooting/clipboard/config.png and /dev/null differ diff --git a/docs/images/troubleshooting/clipboard/risk.png b/docs/images/troubleshooting/clipboard/risk.png deleted file mode 100644 index 1948f0eb..00000000 Binary files a/docs/images/troubleshooting/clipboard/risk.png and /dev/null differ diff --git a/docs/images/troubleshooting/clipboard/search.png b/docs/images/troubleshooting/clipboard/search.png deleted file mode 100644 index 95684ff4..00000000 Binary files a/docs/images/troubleshooting/clipboard/search.png and /dev/null differ diff --git a/docs/images/troubleshooting/clipboard/toggle.png b/docs/images/troubleshooting/clipboard/toggle.png deleted file mode 100644 index 32060dc7..00000000 Binary files a/docs/images/troubleshooting/clipboard/toggle.png and /dev/null differ diff --git a/docs/images/troubleshooting/clipboard/toggled.png b/docs/images/troubleshooting/clipboard/toggled.png deleted file mode 100644 index 6afa0ace..00000000 Binary files a/docs/images/troubleshooting/clipboard/toggled.png and /dev/null differ diff --git a/docs/protect-an-instance.md b/docs/protect-an-instance.md new file mode 100644 index 00000000..9b4131c1 --- /dev/null +++ b/docs/protect-an-instance.md @@ -0,0 +1,150 @@ +# how to protect your cobalt instance +if you keep getting a ton of unknown traffic that hurts the performance of your instance, then it might be a good idea to enable bot protection. + +> [!NOTE] +> this tutorial will work reliably on the latest official version of cobalt 10. +we can't promise full compatibility with anything else. + +## configure cloudflare turnstile +turnstile is a free, safe, and privacy-respecting alternative to captcha. +cobalt uses it automatically to weed out bots and automated scripts. +your instance doesn't have to be proxied by cloudflare to use turnstile. +all you need is a free cloudflare account to get started. + +cloudflare dashboard interface might change over time, but basics should stay the same. + +> [!WARNING] +> never share the turnstile secret key, always keep it private. if accidentally exposed, rotate it in widget settings. + +1. open [the cloudflare dashboard](https://dash.cloudflare.com/) and log into your account + +2. once logged in, select `Turnstile` in the sidebar +
+

+ +

+
+ +3. press `Add widget` +
+

+ +

+
+ +4. enter the widget name (can be anything, such as "cobalt") +
+

+ +

+
+ +5. add cobalt frontend domains you want the widget to work with, you can change this list later at any time + - if you want to use your processing instance with [cobalt.tools](https://cobalt.tools/) frontend, then add `cobalt.tools` to the list +
+

+ +

+
+ +6. select `invisible` widget mode +
+

+ +

+
+ +7. press `create` + +8. keep the page with sitekey and secret key open, you'll need them later. +if you closed it, no worries! +just open the same turnstile page and press "settings" on your freshly made turnstile widget. + +
+

+ +

+
+ +you've successfully created a turnstile widget! +time to add it to your processing instance. + +### enable turnstile on your processing instance +this tutorial assumes that you only have `API_URL` in your `environment` variables list. +if you have other variables there, just add new ones after existing ones. + +> [!CAUTION] +> never use any values from the tutorial, especially `JWT_SECRET`! + +1. open your `docker-compose.yml` config file in any text editor of choice. +2. copy the turnstile sitekey & secret key and paste them to their respective variables. +`TURNSTILE_SITEKEY` for the sitekey and `TURNSTILE_SECRET` for the secret key: +```yml +environment: + API_URL: "https://your.instance.url.here.local/" + TURNSTILE_SITEKEY: "2x00000000000000000000BB" # use your key + TURNSTILE_SECRET: "2x0000000000000000000000000000000AA" # use your key +``` +3. generate a `JWT_SECRET`. we recommend using an alphanumeric collection with a length of at least 64 characters. +this string will be used as salt for all JWT keys. + + you can generate a random secret with `pnpm -r token:jwt` or use any other that you like. + +```yml +environment: + API_URL: "https://your.instance.url.here.local/" + TURNSTILE_SITEKEY: "2x00000000000000000000BB" # use your key + TURNSTILE_SECRET: "2x0000000000000000000000000000000AA" # use your key + JWT_SECRET: "bgBmF4efNCKPirD" # create a new secret, NEVER use this one +``` +4. restart the docker container. + +## configure api keys +if you want to use your instance outside of web interface, you'll need an api key! + +> [!NOTE] +> this tutorial assumes that you'll keep your keys file locally, on the instance server. +> if you wish to upload your file to a remote location, +> replace the value for `API_KEYS_URL` with a direct url to the file +> and skip the second step. + +> [!WARNING] +> when storing keys file remotely, make sure that it's not publicly accessible +> and that link to it is either authenticated (via query) or impossible to guess. +> +> if api keys leak, you'll have to update/remove all UUIDs to revoke them. + +1. create a `keys.json` file following [the schema and example here](/docs//run-an-instance.md#api-key-file-format). + +2. expose the `keys.json` to the docker container: +```yml +volumes: + - ./keys.json:/keys.json:ro # ro - read-only +``` + +3. add a path to the keys file to container environment: +```yml +environment: + # ... other variables here ... + API_KEY_URL: "file:///keys.json" +``` + +4. restart the docker container. + +## limit access to an instance with api keys but no turnstile +by default, api keys are additional, meaning that they're not *required*, +but work alongside with turnstile or no auth (regular ip hash rate limiting). + +to always require auth (via keys or turnstile, if configured), set `API_AUTH_REQUIRED` to 1: +```yml +environment: + # ... other variables here ... + API_AUTH_REQUIRED: 1 +``` + +- if both keys and turnstile are enabled, then nothing will change. +- if only keys are configured, then all requests without a valid api key will be refused. + +### why not make keys exclusive by default? +keys may be useful for going around rate limiting, +while keeping the rest of api rate limited, with no turnstile in place. diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md index 08654d9f..60a3c8aa 100644 --- a/docs/run-an-instance.md +++ b/docs/run-an-instance.md @@ -1,4 +1,4 @@ -# how to host a cobalt instance yourself +# how to run a cobalt instance ## using docker compose and package from github (recommended) to run the cobalt docker package, you need to have `docker` and `docker-compose` installed and configured. @@ -54,8 +54,7 @@ sudo apt install nscd sudo service nscd start ``` -## list of all environment variables -### variables for api +## list of environment variables for api | variable name | default | example | description | |:----------------------|:----------|:------------------------|:------------| | `API_PORT` | `9000` | `9000` | changes port from which api server is accessible. | @@ -72,18 +71,27 @@ sudo service nscd start | `RATELIMIT_MAX` | `20` | `30` | max requests per time window. requests above this amount will be blocked for the rate limit window duration. | | `DURATION_LIMIT` | `10800` | `18000` | max allowed video duration in **seconds**. | | `TUNNEL_LIFESPAN` | `90` | `120` | the duration for which tunnel info is stored in ram, **in seconds**. | +| `TURNSTILE_SITEKEY` | ➖ | `1x00000000000000000000BB` | [cloudflare turnstile](https://www.cloudflare.com/products/turnstile/) sitekey used by browser clients to request a challenge.\*\* | +| `TURNSTILE_SECRET` | ➖ | `1x0000000000000000000000000000000AA` | [cloudflare turnstile](https://www.cloudflare.com/products/turnstile/) secret used by cobalt to verify the client successfully solved the challenge.\*\* | +| `JWT_SECRET` | ➖ | ➖ | the secret used for issuing JWT tokens for request authentication. to choose a value, generate a random, secure, long string (ideally >=16 characters).\*\* | +| `JWT_EXPIRY` | `120` | `240` | the duration of how long a cobalt-issued JWT token will remain valid, in seconds. | | `API_KEY_URL` | ➖ | `file://keys.json` | the location of the api key database. for loading API keys, cobalt supports HTTP(S) urls, or local files by specifying a local path using the `file://` protocol. see the "api key file format" below for more details. | | `API_AUTH_REQUIRED` | ➖ | `1` | when set to `1`, the user always needs to be authenticated in some way before they can access the API (either via an api key or via turnstile, if enabled). | +| `API_REDIS_URL` | ➖ | `redis://localhost:6379` | when set, cobalt uses redis instead of internal memory for the tunnel cache. | +| `API_INSTANCE_COUNT` | ➖ | `2` | supported only on Linux and node.js `>=23.1.0`. when configured, cobalt will spawn multiple sub-instances amongst which requests will be balanced. | +| `DISABLED_SERVICES` | ➖ | `bilibili,youtube` | comma-separated list which disables certain services from being used. | \* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)). +\*\* in order to enable turnstile bot protection, all three **`TURNSTILE_SITEKEY`, `TURNSTILE_SECRET` and `JWT_SECRET`** need to be set. + #### FREEBIND_CIDR setting a `FREEBIND_CIDR` allows cobalt to pick a random IP for every download and use it for all requests it makes for that particular download. to use freebind in cobalt, you need to follow its [setup instructions](https://github.com/imputnet/freebind.js?tab=readme-ov-file#setup) first. if you configure this option while running cobalt in a docker container, you also need to set the `API_LISTEN_ADDRESS` env to `127.0.0.1`, and set `network_mode` for the container to `host`. -#### api key file format +## api key file format the file is a JSON-serialized object with the following structure: ```typescript diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md deleted file mode 100644 index 4c97511f..00000000 --- a/docs/troubleshooting.md +++ /dev/null @@ -1,37 +0,0 @@ -# self-troubleshooting cobalt -``` -🚧 this page is work-in-progress. expect more guides to be added in the future! -``` -if any issues occur while using cobalt, you can fix many of them yourself. this document aims to provide guides on how to fix most complicated of them. -use wiki navigation on right to jump between solutions. - -## how to fix clipboard pasting in older versions of firefox -``` -🎉 firefox finally supports pasting by default starting from version 125. - -👍 you don't need to follow this tutorial if you're on the latest version of firefox. -``` -you can fix this issue by changing a single preference in `about:config`. - -### steps to enable clipboard functionality -1. go to `about:config`: - ![screenshot showing about:config entered into address bar](images/troubleshooting/clipboard/config.png) - -2. if asked, read what firefox has to say and press "accept the risk and continue". - ⚠ tinkering with other preferences may break your browser. **do not** edit them unless you know what you're doing. - - ![screenshot showing about:config security warning that reads: "proceed with caution. changing advanced configuration preferences can impact firefox performance or security." lower there's a pre-checked checkbox that says: "warn me when i attempt to access these preferences". lowest element is a blue button that says "accept the risk and continue"](images/troubleshooting/clipboard/risk.png) - -3. search for `dom.events.asyncClipboard.readText` - - ![screenshot showing "dom.events.asyncclipboard.readtext" entered into search on about:config page](images/troubleshooting/clipboard/search.png) - -4. press the toggle button on very right. - - ![screenshot showing "dom.events.asyncclipboard.readtext" preference on about:config page with highlighted toggle button on very right](images/troubleshooting/clipboard/toggle.png) - -5. "false" should change to "true". - - ![screenshot showing "dom.events.asyncclipboard.readtext" preference on about:config page, this one with "true" text highlighted](images/troubleshooting/clipboard/toggled.png) - -6. go back to cobalt, reload the page, press `paste` button again. this time it works! enjoy simpler downloading experience :) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 71456e11..9e12913a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,6 +10,12 @@ importers: api: dependencies: + '@datastructures-js/priority-queue': + specifier: ^6.3.1 + version: 6.3.1 + '@imput/psl': + specifier: ^2.0.4 + version: 2.0.4 '@imput/version-info': specifier: workspace:^ version: link:../packages/version-info @@ -22,15 +28,12 @@ importers: dotenv: specifier: ^16.0.1 version: 16.4.5 - esbuild: - specifier: ^0.14.51 - version: 0.14.54 express: - specifier: ^4.21.0 - version: 4.21.0 + specifier: ^4.21.2 + version: 4.21.2 express-rate-limit: - specifier: ^6.3.0 - version: 6.11.2(express@4.21.0) + specifier: ^7.4.1 + version: 7.4.1(express@4.21.2) ffmpeg-static: specifier: ^5.1.0 version: 5.2.0 @@ -41,14 +44,8 @@ importers: specifier: 2.2.0 version: 2.2.0 nanoid: - specifier: ^4.0.2 - version: 4.0.2 - node-cache: - specifier: ^5.1.2 - version: 5.1.2 - psl: - specifier: 1.9.0 - version: 1.9.0 + specifier: ^5.0.9 + version: 5.0.9 set-cookie-parser: specifier: 2.6.0 version: 2.6.0 @@ -59,8 +56,8 @@ importers: specifier: 1.0.3 version: 1.0.3 youtubei.js: - specifier: ^10.5.0 - version: 10.5.0 + specifier: ^13.0.0 + version: 13.0.0 zod: specifier: ^3.23.8 version: 3.23.8 @@ -68,6 +65,12 @@ importers: freebind: specifier: ^0.2.2 version: 0.2.2 + rate-limit-redis: + specifier: ^4.2.0 + version: 4.2.0(express-rate-limit@7.4.1(express@4.21.2)) + redis: + specifier: ^4.7.0 + version: 4.7.0 packages/api-client: devDependencies: @@ -104,11 +107,11 @@ importers: specifier: workspace:^ version: link:../packages/version-info '@sveltejs/adapter-static': - specifier: ^3.0.2 - version: 3.0.2(@sveltejs/kit@2.5.19(@sveltejs/vite-plugin-svelte@3.1.1(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)))(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14))) + specifier: ^3.0.6 + version: 3.0.6(@sveltejs/kit@2.9.1(@sveltejs/vite-plugin-svelte@3.1.1(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)))(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14))) '@sveltejs/kit': - specifier: ^2.0.0 - version: 2.5.19(@sveltejs/vite-plugin-svelte@3.1.1(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)))(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)) + specifier: ^2.9.1 + version: 2.9.1(@sveltejs/vite-plugin-svelte@3.1.1(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)))(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)) '@sveltejs/vite-plugin-svelte': specifier: ^3.0.0 version: 3.1.1(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)) @@ -134,11 +137,11 @@ importers: specifier: ^16.0.1 version: 16.4.5 eslint: - specifier: ^8.57.0 - version: 8.57.0 + specifier: ^9.16.0 + version: 9.16.0 glob: - specifier: ^10.4.5 - version: 10.4.5 + specifier: ^11.0.0 + version: 11.0.0 mdsvex: specifier: ^0.11.2 version: 0.11.2(svelte@4.2.19) @@ -154,6 +157,9 @@ importers: svelte-preprocess: specifier: ^6.0.2 version: 6.0.2(postcss@8.4.47)(svelte@4.2.19)(typescript@5.5.4) + svelte-sitemap: + specifier: 2.6.0 + version: 2.6.0 sveltekit-i18n: specifier: ^2.4.2 version: 2.4.2(svelte@4.2.19) @@ -170,8 +176,8 @@ importers: specifier: ^5.4.5 version: 5.5.4 typescript-eslint: - specifier: ^8.8.0 - version: 8.8.0(eslint@8.57.0)(typescript@5.5.4) + specifier: ^8.18.0 + version: 8.18.0(eslint@9.16.0)(typescript@5.5.4) vite: specifier: ^5.3.6 version: 5.4.8(@types/node@20.14.14) @@ -185,6 +191,12 @@ packages: '@bufbuild/protobuf@2.1.0': resolution: {integrity: sha512-+2Mx67Y3skJ4NCD/qNSdBJNWtu6x6Qr53jeNg+QcwiL6mt0wK+3jwHH2x1p7xaYH6Ve2JKOVn0OxU35WsmqI9A==} + '@datastructures-js/heap@4.3.3': + resolution: {integrity: sha512-UcUu/DLh/aM4W3C8zZfwxxm6/6FIZUlm3mcAXuNOCa6Aj4iizNvNXQyb8DjZQH2jKSQbMRyNlngP6TPimuGjpQ==} + + '@datastructures-js/priority-queue@6.3.1': + resolution: {integrity: sha512-eoxkWql/j0VJ0UFMFTpnyJz4KbEEVQ6aZ/JuJUgenu0Im4tYKylAycNGsYCHGXiVNEd7OKGVwfx1Ac3oYkuu7A==} + '@derhuerst/http-basic@8.2.4': resolution: {integrity: sha512-F9rL9k9Xjf5blCz8HsJRO4diy111cayL2vkY2XE4r4t3n0yPXVYy3KD3nJ1qbrSn9743UWSXH4IwuCa/HWlGFw==} engines: {node: '>=6.0.0'} @@ -321,12 +333,6 @@ packages: cpu: [ia32] os: [linux] - '@esbuild/linux-loong64@0.14.54': - resolution: {integrity: sha512-bZBrLAIX1kpWelV0XemxBZllyRmM6vgFQQG2GdNb+r3Fkp0FOh1NJSvekXDs7jq70k4euu1cryLMfU+mTXlEpw==} - engines: {node: '>=12'} - cpu: [loong64] - os: [linux] - '@esbuild/linux-loong64@0.21.5': resolution: {integrity: sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==} engines: {node: '>=12'} @@ -483,22 +489,38 @@ packages: peerDependencies: eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 - '@eslint-community/regexpp@4.11.0': - resolution: {integrity: sha512-G/M/tIiMrTAxEWRfLfQJMmGNX28IxBg4PBz8XqQhqUHLFI6TL2htpIB1iQCj144V5ee/JaKyT9/WZ0MGZWfA7A==} + '@eslint-community/regexpp@4.12.1': + resolution: {integrity: sha512-CCZCDJuduB9OUkFkY2IgppNZMi2lBQgD2qzwXkEia16cge2pijY/aXi96CJMquDMn3nJdlPV1A5KrJEXwfLNzQ==} engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} - '@eslint/eslintrc@2.1.4': - resolution: {integrity: sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + '@eslint/config-array@0.19.1': + resolution: {integrity: sha512-fo6Mtm5mWyKjA/Chy1BYTdn5mGJoDNjC7C64ug20ADsRDGrA85bN3uK3MaKbeRkRuuIEAR5N33Jr1pbm411/PA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@eslint/js@8.57.0': - resolution: {integrity: sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + '@eslint/core@0.9.1': + resolution: {integrity: sha512-GuUdqkyyzQI5RMIWkHhvTWLCyLo1jNK3vzkSyaExH5kHPDHcuL2VOpHjmMY+y3+NC69qAKToBqldTBgYeLSr9Q==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/eslintrc@3.2.0': + resolution: {integrity: sha512-grOjVNN8P3hjJn/eIETF1wwd12DdnwFDoyceUJLYYdkpbwq3nLi+4fqrTAONx7XDALqlL220wC/RHSC/QTI/0w==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/js@9.16.0': + resolution: {integrity: sha512-tw2HxzQkrbeuvyj1tG2Yqq+0H9wGoI2IMk4EOsQeX+vmd75FtJAzf+gTA69WF+baUKRYQ3x2kbLE08js5OsTVg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} '@eslint/js@9.8.0': resolution: {integrity: sha512-MfluB7EUfxXtv3i/++oh89uzAr4PDI4nn201hsp+qaXqsjAWzinlZEHEfPgAX4doIlKvPG/i0A9dpKxOLII8yA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@eslint/object-schema@2.1.5': + resolution: {integrity: sha512-o0bhxnL89h5Bae5T318nFoFzGy+YE5i/gGkoPAgkmTVdRKTiv3p8JHevPiPaMwoloKfEiiaHlawCqaZMqRm+XQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/plugin-kit@0.2.4': + resolution: {integrity: sha512-zSkKow6H5Kdm0ZUQUB2kV5JIXqoG0+uH5YADhaEHswm664N9Db8dXSi0nMJpacpMf+MyyglF1vnZohpEg5yUtg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@fastify/busboy@2.1.1': resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==} engines: {node: '>=14'} @@ -512,22 +534,32 @@ packages: '@fontsource/redaction-10@5.0.2': resolution: {integrity: sha512-PODxYvb06YrNxdUBGcygiMibpgcZihzmvkmlX/TQAA2F7BUU/anfSKQi/VnLdJ/8LIK81/bUY+i7L/GP27FkVw==} - '@humanwhocodes/config-array@0.11.14': - resolution: {integrity: sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==} - engines: {node: '>=10.10.0'} - deprecated: Use @eslint/config-array instead + '@humanfs/core@0.19.1': + resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} + engines: {node: '>=18.18.0'} + + '@humanfs/node@0.16.6': + resolution: {integrity: sha512-YuI2ZHQL78Q5HbhDiBA1X4LmYdXCKCMQIfw0pw7piHJwyREFebJUvrQN4cMssyES6x+vfUbx1CIpaQUKYdQZOw==} + engines: {node: '>=18.18.0'} '@humanwhocodes/module-importer@1.0.1': resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==} engines: {node: '>=12.22'} - '@humanwhocodes/object-schema@2.0.3': - resolution: {integrity: sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==} - deprecated: Use @eslint/object-schema instead + '@humanwhocodes/retry@0.3.1': + resolution: {integrity: sha512-JBxkERygn7Bv/GbN5Rv8Ul6LVknS+5Bp6RgDC/O8gEBU/yeH5Ui5C/OlWrTb6qct7LjjfT6Re2NxB0ln0yYybA==} + engines: {node: '>=18.18'} + + '@humanwhocodes/retry@0.4.1': + resolution: {integrity: sha512-c7hNEllBlenFTHBky65mhq8WD2kbN9Q6gk0bTk8lSBvc554jpXSkST1iePudpt7+A/AQvuHs9EMqjHDXMY1lrA==} + engines: {node: '>=18.18'} '@imput/libav.js-remux-cli@5.5.6': resolution: {integrity: sha512-XdAab90EZKf6ULtD/x9Y2bnlmNJodXSO6w8aWrn97+N2IRuOS8zv3tAFPRC69SWKa8Utjeu5YTYuTolnX3QprQ==} + '@imput/psl@2.0.4': + resolution: {integrity: sha512-vuy76JX78/DnJegLuJoLpMmw11JTA/9HvlIADg/f8dDVXyxbh0jnObL0q13h+WvlBO4Gk26Pu8sUa7/h0JGQig==} + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -562,6 +594,22 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@oozcitak/dom@1.15.10': + resolution: {integrity: sha512-0JT29/LaxVgRcGKvHmSrUTEvZ8BXvZhGl2LASRUgHqDTC1M5g1pLmVv56IYNyt3bG2CUjDkc67wnyZC14pbQrQ==} + engines: {node: '>=8.0'} + + '@oozcitak/infra@1.0.8': + resolution: {integrity: sha512-JRAUc9VR6IGHOL7OGF+yrvs0LO8SlqGnPAMqyzOuFZPSZSXI7Xf2O9+awQPSMXgIWGtgUf/dA6Hs6X6ySEaWTg==} + engines: {node: '>=6.0'} + + '@oozcitak/url@1.0.4': + resolution: {integrity: sha512-kDcD8y+y3FCSOvnBI6HJgl00viO/nGbQoCINmQ0h98OhnGITrWR3bOGfwYCthgcrV8AnTJz8MzslTQbC3SOAmw==} + engines: {node: '>=8.0'} + + '@oozcitak/util@8.3.8': + resolution: {integrity: sha512-T8TbSnGsxo6TDBJx/Sgv/BlVJL3tshxZP7Aq5R1mSnM5OcHY2dQaxLMu2+E8u3gN0MLOzdjurqN4ZRVuzQycOQ==} + engines: {node: '>=8.0'} + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -569,6 +617,35 @@ packages: '@polka/url@1.0.0-next.25': resolution: {integrity: sha512-j7P6Rgr3mmtdkeDGTe0E/aYyWEWVtc5yFXtHCRHs28/jptDEWfaVOc5T7cblqy1XKPPfCxJc/8DwQ5YgLOZOVQ==} + '@redis/bloom@1.2.0': + resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/client@1.6.0': + resolution: {integrity: sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==} + engines: {node: '>=14'} + + '@redis/graph@1.1.1': + resolution: {integrity: sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/json@1.0.7': + resolution: {integrity: sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/search@1.2.0': + resolution: {integrity: sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/time-series@1.1.0': + resolution: {integrity: sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==} + peerDependencies: + '@redis/client': ^1.0.0 + '@rollup/rollup-android-arm-eabi@4.24.0': resolution: {integrity: sha512-Q6HJd7Y6xdB48x8ZNVDOqsbh2uByBhgK8PiQgPhwkIw/HC/YX5Ghq2mQY5sRMZWHb3VsFkWooUVOZHKr7DmDIA==} cpu: [arm] @@ -649,19 +726,19 @@ packages: cpu: [x64] os: [win32] - '@sveltejs/adapter-static@3.0.2': - resolution: {integrity: sha512-/EBFydZDwfwFfFEuF1vzUseBoRziwKP7AoHAwv+Ot3M084sE/HTVBHf9mCmXfdM9ijprY5YEugZjleflncX5fQ==} + '@sveltejs/adapter-static@3.0.6': + resolution: {integrity: sha512-MGJcesnJWj7FxDcB/GbrdYD3q24Uk0PIL4QIX149ku+hlJuj//nxUbb0HxUTpjkecWfHjVveSUnUaQWnPRXlpg==} peerDependencies: '@sveltejs/kit': ^2.0.0 - '@sveltejs/kit@2.5.19': - resolution: {integrity: sha512-r/lah3nnYEZX1btlvpSy+Exkt1aWhmOP5pnCt+BBro+tZrh2Zci+26Xnm1fCBLLMeM5q7gHvWiS8c/UtrWjdvQ==} + '@sveltejs/kit@2.9.1': + resolution: {integrity: sha512-D+yH3DTvvkjXdl3Xv7akKmolrArDZRtsFv3nlxJPjlIKsZEpkkInnomKJuAql2TrNGJ2dJMGBO1YYgVn2ILmag==} engines: {node: '>=18.13'} hasBin: true peerDependencies: - '@sveltejs/vite-plugin-svelte': ^3.0.0 + '@sveltejs/vite-plugin-svelte': ^3.0.0 || ^4.0.0-next.1 || ^5.0.0 svelte: ^4.0.0 || ^5.0.0-next.0 - vite: ^5.0.3 + vite: ^5.0.3 || ^6.0.0 '@sveltejs/vite-plugin-svelte-inspector@2.1.0': resolution: {integrity: sha512-9QX28IymvBlSCqsCll5t0kQVxipsfhFFL+L2t3nTWfXnddYwxBuAEtTtlaVQpRz9c37BhJjltSeY4AJSC03SSg==} @@ -727,65 +804,52 @@ packages: '@types/unist@2.0.10': resolution: {integrity: sha512-IfYcSBWE3hLpBg8+X2SEa8LVkJdJEkT2Ese2aaLs3ptGdVtABxndrMaxuFlQ1qdFf9Q5rDvDpxI3WwgvKFAsQA==} - '@typescript-eslint/eslint-plugin@8.8.0': - resolution: {integrity: sha512-wORFWjU30B2WJ/aXBfOm1LX9v9nyt9D3jsSOxC3cCaTQGCW5k4jNpmjFv3U7p/7s4yvdjHzwtv2Sd2dOyhjS0A==} + '@typescript-eslint/eslint-plugin@8.18.0': + resolution: {integrity: sha512-NR2yS7qUqCL7AIxdJUQf2MKKNDVNaig/dEB0GBLU7D+ZdHgK1NoH/3wsgO3OnPVipn51tG3MAwaODEGil70WEw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: '@typescript-eslint/parser': ^8.0.0 || ^8.0.0-alpha.0 eslint: ^8.57.0 || ^9.0.0 - typescript: '*' - peerDependenciesMeta: - typescript: - optional: true + typescript: '>=4.8.4 <5.8.0' - '@typescript-eslint/parser@8.8.0': - resolution: {integrity: sha512-uEFUsgR+tl8GmzmLjRqz+VrDv4eoaMqMXW7ruXfgThaAShO9JTciKpEsB+TvnfFfbg5IpujgMXVV36gOJRLtZg==} + '@typescript-eslint/parser@8.18.0': + resolution: {integrity: sha512-hgUZ3kTEpVzKaK3uNibExUYm6SKKOmTU2BOxBSvOYwtJEPdVQ70kZJpPjstlnhCHcuc2WGfSbpKlb/69ttyN5Q==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: eslint: ^8.57.0 || ^9.0.0 - typescript: '*' - peerDependenciesMeta: - typescript: - optional: true + typescript: '>=4.8.4 <5.8.0' - '@typescript-eslint/scope-manager@8.8.0': - resolution: {integrity: sha512-EL8eaGC6gx3jDd8GwEFEV091210U97J0jeEHrAYvIYosmEGet4wJ+g0SYmLu+oRiAwbSA5AVrt6DxLHfdd+bUg==} + '@typescript-eslint/scope-manager@8.18.0': + resolution: {integrity: sha512-PNGcHop0jkK2WVYGotk/hxj+UFLhXtGPiGtiaWgVBVP1jhMoMCHlTyJA+hEj4rszoSdLTK3fN4oOatrL0Cp+Xw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@typescript-eslint/type-utils@8.8.0': - resolution: {integrity: sha512-IKwJSS7bCqyCeG4NVGxnOP6lLT9Okc3Zj8hLO96bpMkJab+10HIfJbMouLrlpyOr3yrQ1cA413YPFiGd1mW9/Q==} - engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - peerDependencies: - typescript: '*' - peerDependenciesMeta: - typescript: - optional: true - - '@typescript-eslint/types@8.8.0': - resolution: {integrity: sha512-QJwc50hRCgBd/k12sTykOJbESe1RrzmX6COk8Y525C9l7oweZ+1lw9JiU56im7Amm8swlz00DRIlxMYLizr2Vw==} - engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - - '@typescript-eslint/typescript-estree@8.8.0': - resolution: {integrity: sha512-ZaMJwc/0ckLz5DaAZ+pNLmHv8AMVGtfWxZe/x2JVEkD5LnmhWiQMMcYT7IY7gkdJuzJ9P14fRy28lUrlDSWYdw==} - engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - peerDependencies: - typescript: '*' - peerDependenciesMeta: - typescript: - optional: true - - '@typescript-eslint/utils@8.8.0': - resolution: {integrity: sha512-QE2MgfOTem00qrlPgyByaCHay9yb1+9BjnMFnSFkUKQfu7adBXDTnCAivURnuPPAG/qiB+kzKkZKmKfaMT0zVg==} + '@typescript-eslint/type-utils@8.18.0': + resolution: {integrity: sha512-er224jRepVAVLnMF2Q7MZJCq5CsdH2oqjP4dT7K6ij09Kyd+R21r7UVJrF0buMVdZS5QRhDzpvzAxHxabQadow==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <5.8.0' - '@typescript-eslint/visitor-keys@8.8.0': - resolution: {integrity: sha512-8mq51Lx6Hpmd7HnA2fcHQo3YgfX1qbccxQOgZcb4tvasu//zXRaA1j5ZRFeCw/VRAdFi4mRM9DnZw0Nu0Q2d1g==} + '@typescript-eslint/types@8.18.0': + resolution: {integrity: sha512-FNYxgyTCAnFwTrzpBGq+zrnoTO4x0c1CKYY5MuUTzpScqmY5fmsh2o3+57lqdI3NZucBDCzDgdEbIaNfAjAHQA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@ungap/structured-clone@1.2.0': - resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==} + '@typescript-eslint/typescript-estree@8.18.0': + resolution: {integrity: sha512-rqQgFRu6yPkauz+ms3nQpohwejS8bvgbPyIDq13cgEDbkXt4LH4OkDMT0/fN1RUtzG8e8AKJyDBoocuQh8qNeg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <5.8.0' + + '@typescript-eslint/utils@8.18.0': + resolution: {integrity: sha512-p6GLdY383i7h5b0Qrfbix3Vc3+J2k6QWw6UMUeY5JGfm3C5LbZ4QIZzJNoNOfgyRe0uuYKjvVOsO/jD4SJO+xg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <5.8.0' + + '@typescript-eslint/visitor-keys@8.18.0': + resolution: {integrity: sha512-pCh/qEA8Lb1wVIqNvBke8UaRjJ6wrAWkJO5yyIbs8Yx6TNGYyfNjOo61tLv+WwLvoLPp4BQ8B7AHKijl8NGUfw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} '@vitejs/plugin-basic-ssl@1.1.0': resolution: {integrity: sha512-wO4Dk/rm8u7RNhOf95ZzcEmC9rYOncYgvq4z3duaJrCgjN8BxAnDVyndanfcJZ0O6XZzHz6Q0hTimxTg8Y9g/A==} @@ -807,6 +871,11 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + acorn@8.14.0: + resolution: {integrity: sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==} + engines: {node: '>=0.4.0'} + hasBin: true + agent-base@6.0.2: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} @@ -837,6 +906,9 @@ packages: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} + argparse@1.0.10: + resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -911,9 +983,9 @@ packages: resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==} engines: {node: '>= 8.10.0'} - clone@2.1.2: - resolution: {integrity: sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==} - engines: {node: '>=0.8'} + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} code-red@1.0.4: resolution: {integrity: sha512-7qJWqItLA8/VPVlKJlFXU+NBlo/qyfs39aJcuMT/2ere32ZqvF5OSxgdM5xOfJJ7O429gg2HM47y8v9P+9wrNw==} @@ -961,6 +1033,10 @@ packages: resolution: {integrity: sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==} engines: {node: '>= 0.6'} + cookie@0.7.1: + resolution: {integrity: sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==} + engines: {node: '>= 0.6'} + cors@2.8.5: resolution: {integrity: sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==} engines: {node: '>= 0.10'} @@ -969,6 +1045,10 @@ packages: resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} engines: {node: '>= 8'} + cross-spawn@7.0.6: + resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} + engines: {node: '>= 8'} + css-tree@2.3.1: resolution: {integrity: sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw==} engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0} @@ -1017,12 +1097,8 @@ packages: resolution: {integrity: sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==} engines: {node: '>=8'} - devalue@5.0.0: - resolution: {integrity: sha512-gO+/OMXF7488D+u3ue+G7Y4AA3ZmUnB3eHJXmBTgNHvr4ZNzl36A0ZtG+XCRNYCkYx/bFmw4qtkoFLa+wSrwAA==} - - doctrine@3.0.0: - resolution: {integrity: sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==} - engines: {node: '>=6.0.0'} + devalue@5.1.1: + resolution: {integrity: sha512-maua5KUiapvEwiEAe+XnlZ3Rh0GD+qI1J/nb9vrJc3muPXvcF/8gXYTWF76+5DAqHyDUtOIImEuo0YKE9mshVw==} dotenv@16.4.5: resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==} @@ -1063,131 +1139,6 @@ packages: es6-promise@3.3.1: resolution: {integrity: sha512-SOp9Phqvqn7jtEUxPWdWfWoLmyt2VaJ6MpvP9Comy1MceMXqE6bxvaTu4iaxpYYPzhny28Lc+M87/c2cPK6lDg==} - esbuild-android-64@0.14.54: - resolution: {integrity: sha512-Tz2++Aqqz0rJ7kYBfz+iqyE3QMycD4vk7LBRyWaAVFgFtQ/O8EJOnVmTOiDWYZ/uYzB4kvP+bqejYdVKzE5lAQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [android] - - esbuild-android-arm64@0.14.54: - resolution: {integrity: sha512-F9E+/QDi9sSkLaClO8SOV6etqPd+5DgJje1F9lOWoNncDdOBL2YF59IhsWATSt0TLZbYCf3pNlTHvVV5VfHdvg==} - engines: {node: '>=12'} - cpu: [arm64] - os: [android] - - esbuild-darwin-64@0.14.54: - resolution: {integrity: sha512-jtdKWV3nBviOd5v4hOpkVmpxsBy90CGzebpbO9beiqUYVMBtSc0AL9zGftFuBon7PNDcdvNCEuQqw2x0wP9yug==} - engines: {node: '>=12'} - cpu: [x64] - os: [darwin] - - esbuild-darwin-arm64@0.14.54: - resolution: {integrity: sha512-OPafJHD2oUPyvJMrsCvDGkRrVCar5aVyHfWGQzY1dWnzErjrDuSETxwA2HSsyg2jORLY8yBfzc1MIpUkXlctmw==} - engines: {node: '>=12'} - cpu: [arm64] - os: [darwin] - - esbuild-freebsd-64@0.14.54: - resolution: {integrity: sha512-OKwd4gmwHqOTp4mOGZKe/XUlbDJ4Q9TjX0hMPIDBUWWu/kwhBAudJdBoxnjNf9ocIB6GN6CPowYpR/hRCbSYAg==} - engines: {node: '>=12'} - cpu: [x64] - os: [freebsd] - - esbuild-freebsd-arm64@0.14.54: - resolution: {integrity: sha512-sFwueGr7OvIFiQT6WeG0jRLjkjdqWWSrfbVwZp8iMP+8UHEHRBvlaxL6IuKNDwAozNUmbb8nIMXa7oAOARGs1Q==} - engines: {node: '>=12'} - cpu: [arm64] - os: [freebsd] - - esbuild-linux-32@0.14.54: - resolution: {integrity: sha512-1ZuY+JDI//WmklKlBgJnglpUL1owm2OX+8E1syCD6UAxcMM/XoWd76OHSjl/0MR0LisSAXDqgjT3uJqT67O3qw==} - engines: {node: '>=12'} - cpu: [ia32] - os: [linux] - - esbuild-linux-64@0.14.54: - resolution: {integrity: sha512-EgjAgH5HwTbtNsTqQOXWApBaPVdDn7XcK+/PtJwZLT1UmpLoznPd8c5CxqsH2dQK3j05YsB3L17T8vE7cp4cCg==} - engines: {node: '>=12'} - cpu: [x64] - os: [linux] - - esbuild-linux-arm64@0.14.54: - resolution: {integrity: sha512-WL71L+0Rwv+Gv/HTmxTEmpv0UgmxYa5ftZILVi2QmZBgX3q7+tDeOQNqGtdXSdsL8TQi1vIaVFHUPDe0O0kdig==} - engines: {node: '>=12'} - cpu: [arm64] - os: [linux] - - esbuild-linux-arm@0.14.54: - resolution: {integrity: sha512-qqz/SjemQhVMTnvcLGoLOdFpCYbz4v4fUo+TfsWG+1aOu70/80RV6bgNpR2JCrppV2moUQkww+6bWxXRL9YMGw==} - engines: {node: '>=12'} - cpu: [arm] - os: [linux] - - esbuild-linux-mips64le@0.14.54: - resolution: {integrity: sha512-qTHGQB8D1etd0u1+sB6p0ikLKRVuCWhYQhAHRPkO+OF3I/iSlTKNNS0Lh2Oc0g0UFGguaFZZiPJdJey3AGpAlw==} - engines: {node: '>=12'} - cpu: [mips64el] - os: [linux] - - esbuild-linux-ppc64le@0.14.54: - resolution: {integrity: sha512-j3OMlzHiqwZBDPRCDFKcx595XVfOfOnv68Ax3U4UKZ3MTYQB5Yz3X1mn5GnodEVYzhtZgxEBidLWeIs8FDSfrQ==} - engines: {node: '>=12'} - cpu: [ppc64] - os: [linux] - - esbuild-linux-riscv64@0.14.54: - resolution: {integrity: sha512-y7Vt7Wl9dkOGZjxQZnDAqqn+XOqFD7IMWiewY5SPlNlzMX39ocPQlOaoxvT4FllA5viyV26/QzHtvTjVNOxHZg==} - engines: {node: '>=12'} - cpu: [riscv64] - os: [linux] - - esbuild-linux-s390x@0.14.54: - resolution: {integrity: sha512-zaHpW9dziAsi7lRcyV4r8dhfG1qBidQWUXweUjnw+lliChJqQr+6XD71K41oEIC3Mx1KStovEmlzm+MkGZHnHA==} - engines: {node: '>=12'} - cpu: [s390x] - os: [linux] - - esbuild-netbsd-64@0.14.54: - resolution: {integrity: sha512-PR01lmIMnfJTgeU9VJTDY9ZerDWVFIUzAtJuDHwwceppW7cQWjBBqP48NdeRtoP04/AtO9a7w3viI+PIDr6d+w==} - engines: {node: '>=12'} - cpu: [x64] - os: [netbsd] - - esbuild-openbsd-64@0.14.54: - resolution: {integrity: sha512-Qyk7ikT2o7Wu76UsvvDS5q0amJvmRzDyVlL0qf5VLsLchjCa1+IAvd8kTBgUxD7VBUUVgItLkk609ZHUc1oCaw==} - engines: {node: '>=12'} - cpu: [x64] - os: [openbsd] - - esbuild-sunos-64@0.14.54: - resolution: {integrity: sha512-28GZ24KmMSeKi5ueWzMcco6EBHStL3B6ubM7M51RmPwXQGLe0teBGJocmWhgwccA1GeFXqxzILIxXpHbl9Q/Kw==} - engines: {node: '>=12'} - cpu: [x64] - os: [sunos] - - esbuild-windows-32@0.14.54: - resolution: {integrity: sha512-T+rdZW19ql9MjS7pixmZYVObd9G7kcaZo+sETqNH4RCkuuYSuv9AGHUVnPoP9hhuE1WM1ZimHz1CIBHBboLU7w==} - engines: {node: '>=12'} - cpu: [ia32] - os: [win32] - - esbuild-windows-64@0.14.54: - resolution: {integrity: sha512-AoHTRBUuYwXtZhjXZbA1pGfTo8cJo3vZIcWGLiUcTNgHpJJMC1rVA44ZereBHMJtotyN71S8Qw0npiCIkW96cQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [win32] - - esbuild-windows-arm64@0.14.54: - resolution: {integrity: sha512-M0kuUvXhot1zOISQGXwWn6YtS+Y/1RT9WrVIOywZnJHo3jCDyewAc79aKNQWFCQm+xNHVTq9h8dZKvygoXQQRg==} - engines: {node: '>=12'} - cpu: [arm64] - os: [win32] - - esbuild@0.14.54: - resolution: {integrity: sha512-Cy9llcy8DvET5uznocPyqL3BFRrFXSVqbgpMJ9Wz8oVjZlh/zUSNbPRbov0VX7VxN2JH1Oa0uNxZ7eLRb62pJA==} - engines: {node: '>=12'} - hasBin: true - esbuild@0.21.5: resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==} engines: {node: '>=12'} @@ -1205,25 +1156,39 @@ packages: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} - eslint-scope@7.2.2: - resolution: {integrity: sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + eslint-scope@8.2.0: + resolution: {integrity: sha512-PHlWUfG6lvPc3yvP5A4PNyBL1W8fkDUccmI21JUu/+GKZBoH/W5u6usENXUrWFRsyoW5ACUjFGgAFQp5gUlb/A==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} eslint-visitor-keys@3.4.3: resolution: {integrity: sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} - eslint@8.57.0: - resolution: {integrity: sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + eslint-visitor-keys@4.2.0: + resolution: {integrity: sha512-UyLnSehNt62FFhSwjZlHmeokpRK59rcz29j+F1/aDgbkbRTk7wIc9XzdoasMUbRNKDM0qQt/+BJ4BrpFeABemw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + eslint@9.16.0: + resolution: {integrity: sha512-whp8mSQI4C8VXd+fLgSM0lh3UlmcFtVwUQjyKCFfsp+2ItAIYhlq/hqGahGqHE6cv9unM41VlqKk2VtKYR2TaA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} hasBin: true + peerDependencies: + jiti: '*' + peerDependenciesMeta: + jiti: + optional: true - esm-env@1.0.0: - resolution: {integrity: sha512-Cf6VksWPsTuW01vU9Mk/3vRue91Zevka5SjyNf3nEpokFRuqt/KjUQoGAwq9qMmhpLTHmXzSIrFRw8zxWzmFBA==} + esm-env@1.2.1: + resolution: {integrity: sha512-U9JedYYjCnadUlXk7e1Kr+aENQhtUaoaV9+gZm1T8LC/YBAPJx3NSPIAurFOC0U5vrdSevnUJS2/wUVxGwPhng==} - espree@9.6.1: - resolution: {integrity: sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + espree@10.3.0: + resolution: {integrity: sha512-0QYC8b24HWY8zjRnDTL6RiHfDbAWn63qb4LMj1Z4b076A4une81+z03Kg7l7mn/48PUTqoLptSXez8oknU8Clg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + esprima@4.0.1: + resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} + engines: {node: '>=4'} + hasBin: true esquery@1.6.0: resolution: {integrity: sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==} @@ -1252,14 +1217,14 @@ packages: resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} engines: {node: '>=10'} - express-rate-limit@6.11.2: - resolution: {integrity: sha512-a7uwwfNTh1U60ssiIkuLFWHt4hAC5yxlLGU2VP0X4YNlyEDZAqF4tK3GD3NSitVBrCQmQ0++0uOyFOgC2y4DDw==} - engines: {node: '>= 14'} + express-rate-limit@7.4.1: + resolution: {integrity: sha512-KS3efpnpIDVIXopMc65EMbWbUht7qvTCdtCR2dD/IZmi9MIkopYESwyRqLgv8Pfu589+KqDqOdzJWW7AHoACeg==} + engines: {node: '>= 16'} peerDependencies: - express: ^4 || ^5 + express: 4 || 5 || ^5.0.0-beta.1 - express@4.21.0: - resolution: {integrity: sha512-VqcNGcj/Id5ZT1LZ/cfihi3ttTn+NJmkli2eZADigjq29qTlWi/hAQ43t/VLPq8+UX06FCEx3ByOYet6ZFblng==} + express@4.21.2: + resolution: {integrity: sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==} engines: {node: '>= 0.10.0'} fast-deep-equal@3.1.3: @@ -1290,9 +1255,9 @@ packages: resolution: {integrity: sha512-WrM7kLW+do9HLr+H6tk7LzQ7kPqbAgLjdzNE32+u3Ff11gXt9Kkkd2nusGFrlWMIe+XaA97t+I8JS7sZIrvRgA==} engines: {node: '>=16'} - file-entry-cache@6.0.1: - resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} - engines: {node: ^10.12.0 || >=12.0.0} + file-entry-cache@8.0.0: + resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} + engines: {node: '>=16.0.0'} fill-range@7.1.1: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} @@ -1306,9 +1271,9 @@ packages: resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} engines: {node: '>=10'} - flat-cache@3.2.0: - resolution: {integrity: sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==} - engines: {node: ^10.12.0 || >=12.0.0} + flat-cache@4.0.1: + resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} + engines: {node: '>=16'} flatted@3.3.1: resolution: {integrity: sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==} @@ -1339,6 +1304,10 @@ packages: function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + generic-pool@3.9.0: + resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==} + engines: {node: '>= 4'} + get-intrinsic@1.2.4: resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==} engines: {node: '>= 0.4'} @@ -1359,13 +1328,18 @@ packages: resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} hasBin: true + glob@11.0.0: + resolution: {integrity: sha512-9UiX/Bl6J2yaBbxKoEBRm4Cipxgok8kQYcOPEhScPwebu2I0HoQOuYdIO6S3hLuWoZgpDpwQZMzTFxgpkyT76g==} + engines: {node: 20 || >=22} + hasBin: true + glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} deprecated: Glob versions prior to v9 are no longer supported - globals@13.24.0: - resolution: {integrity: sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==} - engines: {node: '>=8'} + globals@14.0.0: + resolution: {integrity: sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==} + engines: {node: '>=18'} globalyzer@0.1.0: resolution: {integrity: sha512-40oNTM9UfG6aBmuKxk/giHn5nQ8RVz/SS4Ir6zgzOv9/qC3kKZ9v4etGTcJbEl/NyVQH7FGU7d+X1egr57Md2Q==} @@ -1477,10 +1451,6 @@ packages: resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} engines: {node: '>=0.12.0'} - is-path-inside@3.0.3: - resolution: {integrity: sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==} - engines: {node: '>=8'} - is-reference@3.0.2: resolution: {integrity: sha512-v3rht/LgVcsdZa3O2Nqs+NMowLOxeOm7Ay9+/ARQ2F+qEoANRcqrjAZKGN0v8ymUetZGgkp26LTnGT7H0Qo9Pg==} @@ -1494,13 +1464,21 @@ packages: jackspeak@3.4.3: resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} - jintr@2.1.1: - resolution: {integrity: sha512-89cwX4ouogeDGOBsEVsVYsnWWvWjchmwXBB4kiBhmjOKw19FiOKhNhMhpxhTlK2ctl7DS+d/ethfmuBpzoNNgA==} + jackspeak@4.0.2: + resolution: {integrity: sha512-bZsjR/iRjl1Nk1UkjGpAzLNfQtzuijhn2g+pbZb98HQ1Gk8vM9hfbxeMBP+M2/UUdwj0RqGG3mlvk2MsAqwvEw==} + engines: {node: 20 || >=22} + + jintr@3.2.0: + resolution: {integrity: sha512-psD1yf05kMKDNsUdW1l5YhO59pHScQ6OIHHb8W5SKSM2dCOFPsqolmIuSHgVA8+3Dc47NJR181CXZ4alCAPTkA==} joycon@3.1.1: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} + js-yaml@3.14.1: + resolution: {integrity: sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==} + hasBin: true + js-yaml@4.1.0: resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} hasBin: true @@ -1552,6 +1530,10 @@ packages: lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + lru-cache@11.0.2: + resolution: {integrity: sha512-123qHRfJBmo2jXDbo/a5YOQrJoHF/GNQTLzQ5+IdK5pWpceK17yRc6ozlWd25FxvGKQbIUs91fDFkXmDHTKcyA==} + engines: {node: 20 || >=22} + magic-string@0.30.11: resolution: {integrity: sha512-+Wri9p0QHMy+545hKww7YAu5NyzF8iomPL/RQazugQ9+Ez4Ic3mERMd8ZTX5rfK944j+560ZJi8iAwgak1Ac7A==} @@ -1611,6 +1593,10 @@ packages: resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} engines: {node: '>=4'} + minimatch@10.0.1: + resolution: {integrity: sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==} + engines: {node: 20 || >=22} + minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} @@ -1654,9 +1640,9 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true - nanoid@4.0.2: - resolution: {integrity: sha512-7ZtY5KTCNheRGfEFxnedV5zFiORN1+Y1N6zvPTnHQd8ENUvfaDBeuJDZb2bN/oXwXxu3qkTXDzy57W5vAmDTBw==} - engines: {node: ^14 || ^16 || >=18} + nanoid@5.0.9: + resolution: {integrity: sha512-Aooyr6MXU6HpvvWXKoVoXwKMs/KyVakWwg7xQfv5/S/RIgJMy0Ifa45H9qqYy7pTCszrHzP21Uk4PZq2HpEM8Q==} + engines: {node: ^18 || >=20} hasBin: true natural-compare@1.4.0: @@ -1666,10 +1652,6 @@ packages: resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} engines: {node: '>= 0.6'} - node-cache@5.1.2: - resolution: {integrity: sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==} - engines: {node: '>= 8.0.0'} - normalize-path@3.0.0: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} engines: {node: '>=0.10.0'} @@ -1739,8 +1721,12 @@ packages: resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} engines: {node: '>=16 || 14 >=14.18'} - path-to-regexp@0.1.10: - resolution: {integrity: sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==} + path-scurry@2.0.0: + resolution: {integrity: sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==} + engines: {node: 20 || >=22} + + path-to-regexp@0.1.12: + resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==} periscopic@3.1.0: resolution: {integrity: sha512-vKiQ8RRtkl9P+r/+oefh25C3fhybptkHKCZSPlcXiJux2tJF55GnEj3BVn4A5gKfq9NWWXXrxkHBwVPUfH0opw==} @@ -1809,9 +1795,6 @@ packages: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} - psl@1.9.0: - resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} - punycode@2.3.1: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} @@ -1827,6 +1810,12 @@ packages: resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} engines: {node: '>= 0.6'} + rate-limit-redis@4.2.0: + resolution: {integrity: sha512-wV450NQyKC24NmPosJb2131RoczLdfIJdKCReNwtVpm5998U8SgKrAZrIHaN/NfQgqOHaan8Uq++B4sa5REwjA==} + engines: {node: '>= 16'} + peerDependencies: + express-rate-limit: '>= 6' + raw-body@2.5.2: resolution: {integrity: sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==} engines: {node: '>= 0.8'} @@ -1839,6 +1828,9 @@ packages: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} engines: {node: '>=8.10.0'} + redis@4.7.0: + resolution: {integrity: sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==} + resolve-from@4.0.0: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} @@ -1856,11 +1848,6 @@ packages: deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true - rimraf@3.0.2: - resolution: {integrity: sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==} - deprecated: Rimraf versions prior to v4 are no longer supported - hasBin: true - rollup@4.24.0: resolution: {integrity: sha512-DOmrlGSXNk1DM0ljiQA+i+o0rSLhtii1je5wgk60j49d1jHT5YYttBv1iWOnYSTG+fZZESUOSNiAl89SIet+Cg==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} @@ -1924,9 +1911,9 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} - sirv@2.0.4: - resolution: {integrity: sha512-94Bdh3cC2PKrbgSOUqTiGPWVZeSiXfKOVZNJniWoqrWrRkB1CJzBU3NEbiTsPcYy1lDsANA/THzS+9WBiy5nfQ==} - engines: {node: '>= 10'} + sirv@3.0.0: + resolution: {integrity: sha512-BPwJGUeDaDCHihkORDchNyyTvWFhcusy1XMmhEVTQTwGeybFbp8YEmB+njbPnth1FibULBSBVwCQni25XlCUDg==} + engines: {node: '>=18'} sorcery@0.11.1: resolution: {integrity: sha512-o7npfeJE6wi6J9l0/5LKshFzZ2rMatRiCDwYeDQaOzqdzRJwALhX7mk/A/ecg6wjMu7wdZbmXfD2S/vpOg0bdQ==} @@ -1944,6 +1931,9 @@ packages: resolution: {integrity: sha512-2ymg6oRBpebeZi9UUNsgQ89bhx01TcTkmNTGnNO88imTmbSgy4nfujrgVEFKWpMTEGA11EDkTt7mqObTPdigIA==} engines: {node: '>= 8'} + sprintf-js@1.0.3: + resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + statuses@2.0.1: resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} engines: {node: '>= 0.8'} @@ -2074,6 +2064,11 @@ packages: typescript: optional: true + svelte-sitemap@2.6.0: + resolution: {integrity: sha512-WcwsuIeo8iJFG9a5cgvXwXEGoyjk6Zowb6JmL5BbwfnFXMzakGa1+mQjthw5Ni3UV/gGbE0PgJvc7Ygir3LmFg==} + engines: {node: '>= 14.17.0'} + hasBin: true + svelte@4.2.19: resolution: {integrity: sha512-IY1rnGr6izd10B0A8LqsBfmlT5OILVuZ7XsI0vdGPEvuonFV7NYEUK4dAkm9Zg2q0Um92kYjTpS1CAP3Nh/KWw==} engines: {node: '>=16'} @@ -2086,9 +2081,6 @@ packages: syscall-napi@0.0.6: resolution: {integrity: sha512-qHbwjyFXAAekKUXxl70lhDiBYJ3e7XM7kQwu7LV3F0pHMenKox+VcZPZkRkhdmL/wNJD3NmrMGnL7161kdecUQ==} - text-table@0.2.0: - resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==} - thenify-all@1.6.0: resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==} engines: {node: '>=0.8'} @@ -2164,10 +2156,6 @@ packages: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} - type-fest@0.20.2: - resolution: {integrity: sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==} - engines: {node: '>=10'} - type-is@1.6.18: resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} engines: {node: '>= 0.6'} @@ -2175,14 +2163,12 @@ packages: typedarray@0.0.6: resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==} - typescript-eslint@8.8.0: - resolution: {integrity: sha512-BjIT/VwJ8+0rVO01ZQ2ZVnjE1svFBiRczcpr1t1Yxt7sT25VSbPfrJtDsQ8uQTy2pilX5nI9gwxhUyLULNentw==} + typescript-eslint@8.18.0: + resolution: {integrity: sha512-Xq2rRjn6tzVpAyHr3+nmSg1/9k9aIHnJ2iZeOH7cfGOWqTkXTm3kwpQglEuLGdNrYvPF+2gtAs+/KF5rjVo+WQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: - typescript: '*' - peerDependenciesMeta: - typescript: - optional: true + eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <5.8.0' typescript@5.5.4: resolution: {integrity: sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==} @@ -2289,12 +2275,19 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + xmlbuilder2@3.1.1: + resolution: {integrity: sha512-WCSfbfZnQDdLQLiMdGUQpMxxckeQ4oZNMNhLVkcekTu7xhD4tuUDyAPoY8CwXvBYE6LwBHd6QW2WZXlOWr1vCw==} + engines: {node: '>=12.0'} + + yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} - youtubei.js@10.5.0: - resolution: {integrity: sha512-iyA+VF28c15tCCKH9ExM2RKC3zYiHzA/eixGlJ3vERANkuI+xYKzAZ4vtOhmyqwrAddu88R/DkzEsmpph5NWjg==} + youtubei.js@13.0.0: + resolution: {integrity: sha512-b1QkN9bfgphK+5tI4qteSK54kNxmPhoedvMw0jl4uSn+L8gbDbJ4z52amNuYNcOdp4X/SI3JuUb+f5V0DPJ8Vw==} zod@3.23.8: resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} @@ -2308,6 +2301,12 @@ snapshots: '@bufbuild/protobuf@2.1.0': {} + '@datastructures-js/heap@4.3.3': {} + + '@datastructures-js/priority-queue@6.3.1': + dependencies: + '@datastructures-js/heap': 4.3.3 + '@derhuerst/http-basic@8.2.4': dependencies: caseless: 0.12.0 @@ -2381,9 +2380,6 @@ snapshots: '@esbuild/linux-ia32@0.23.0': optional: true - '@esbuild/linux-loong64@0.14.54': - optional: true - '@esbuild/linux-loong64@0.21.5': optional: true @@ -2459,19 +2455,31 @@ snapshots: '@esbuild/win32-x64@0.23.0': optional: true - '@eslint-community/eslint-utils@4.4.0(eslint@8.57.0)': + '@eslint-community/eslint-utils@4.4.0(eslint@9.16.0)': dependencies: - eslint: 8.57.0 + eslint: 9.16.0 eslint-visitor-keys: 3.4.3 - '@eslint-community/regexpp@4.11.0': {} + '@eslint-community/regexpp@4.12.1': {} - '@eslint/eslintrc@2.1.4': + '@eslint/config-array@0.19.1': + dependencies: + '@eslint/object-schema': 2.1.5 + debug: 4.3.6 + minimatch: 3.1.2 + transitivePeerDependencies: + - supports-color + + '@eslint/core@0.9.1': + dependencies: + '@types/json-schema': 7.0.15 + + '@eslint/eslintrc@3.2.0': dependencies: ajv: 6.12.6 debug: 4.3.6 - espree: 9.6.1 - globals: 13.24.0 + espree: 10.3.0 + globals: 14.0.0 ignore: 5.3.1 import-fresh: 3.3.0 js-yaml: 4.1.0 @@ -2480,10 +2488,16 @@ snapshots: transitivePeerDependencies: - supports-color - '@eslint/js@8.57.0': {} + '@eslint/js@9.16.0': {} '@eslint/js@9.8.0': {} + '@eslint/object-schema@2.1.5': {} + + '@eslint/plugin-kit@0.2.4': + dependencies: + levn: 0.4.1 + '@fastify/busboy@2.1.1': {} '@fontsource-variable/noto-sans-mono@5.0.20': {} @@ -2492,20 +2506,25 @@ snapshots: '@fontsource/redaction-10@5.0.2': {} - '@humanwhocodes/config-array@0.11.14': + '@humanfs/core@0.19.1': {} + + '@humanfs/node@0.16.6': dependencies: - '@humanwhocodes/object-schema': 2.0.3 - debug: 4.3.6 - minimatch: 3.1.2 - transitivePeerDependencies: - - supports-color + '@humanfs/core': 0.19.1 + '@humanwhocodes/retry': 0.3.1 '@humanwhocodes/module-importer@1.0.1': {} - '@humanwhocodes/object-schema@2.0.3': {} + '@humanwhocodes/retry@0.3.1': {} + + '@humanwhocodes/retry@0.4.1': {} '@imput/libav.js-remux-cli@5.5.6': {} + '@imput/psl@2.0.4': + dependencies: + punycode: 2.3.1 + '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -2544,11 +2563,60 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.17.1 + '@oozcitak/dom@1.15.10': + dependencies: + '@oozcitak/infra': 1.0.8 + '@oozcitak/url': 1.0.4 + '@oozcitak/util': 8.3.8 + + '@oozcitak/infra@1.0.8': + dependencies: + '@oozcitak/util': 8.3.8 + + '@oozcitak/url@1.0.4': + dependencies: + '@oozcitak/infra': 1.0.8 + '@oozcitak/util': 8.3.8 + + '@oozcitak/util@8.3.8': {} + '@pkgjs/parseargs@0.11.0': optional: true '@polka/url@1.0.0-next.25': {} + '@redis/bloom@1.2.0(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + optional: true + + '@redis/client@1.6.0': + dependencies: + cluster-key-slot: 1.1.2 + generic-pool: 3.9.0 + yallist: 4.0.0 + optional: true + + '@redis/graph@1.1.1(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + optional: true + + '@redis/json@1.0.7(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + optional: true + + '@redis/search@1.2.0(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + optional: true + + '@redis/time-series@1.1.0(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + optional: true + '@rollup/rollup-android-arm-eabi@4.24.0': optional: true @@ -2597,24 +2665,24 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.24.0': optional: true - '@sveltejs/adapter-static@3.0.2(@sveltejs/kit@2.5.19(@sveltejs/vite-plugin-svelte@3.1.1(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)))(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)))': + '@sveltejs/adapter-static@3.0.6(@sveltejs/kit@2.9.1(@sveltejs/vite-plugin-svelte@3.1.1(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)))(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)))': dependencies: - '@sveltejs/kit': 2.5.19(@sveltejs/vite-plugin-svelte@3.1.1(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)))(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)) + '@sveltejs/kit': 2.9.1(@sveltejs/vite-plugin-svelte@3.1.1(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)))(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)) - '@sveltejs/kit@2.5.19(@sveltejs/vite-plugin-svelte@3.1.1(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)))(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14))': + '@sveltejs/kit@2.9.1(@sveltejs/vite-plugin-svelte@3.1.1(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)))(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14))': dependencies: '@sveltejs/vite-plugin-svelte': 3.1.1(svelte@4.2.19)(vite@5.4.8(@types/node@20.14.14)) '@types/cookie': 0.6.0 cookie: 0.6.0 - devalue: 5.0.0 - esm-env: 1.0.0 + devalue: 5.1.1 + esm-env: 1.2.1 import-meta-resolve: 4.1.0 kleur: 4.1.5 magic-string: 0.30.11 mrmime: 2.0.0 sade: 1.8.1 set-cookie-parser: 2.6.0 - sirv: 2.0.4 + sirv: 3.0.0 svelte: 4.2.19 tiny-glob: 0.2.9 vite: 5.4.8(@types/node@20.14.14) @@ -2686,88 +2754,82 @@ snapshots: '@types/unist@2.0.10': {} - '@typescript-eslint/eslint-plugin@8.8.0(@typescript-eslint/parser@8.8.0(eslint@8.57.0)(typescript@5.5.4))(eslint@8.57.0)(typescript@5.5.4)': + '@typescript-eslint/eslint-plugin@8.18.0(@typescript-eslint/parser@8.18.0(eslint@9.16.0)(typescript@5.5.4))(eslint@9.16.0)(typescript@5.5.4)': dependencies: - '@eslint-community/regexpp': 4.11.0 - '@typescript-eslint/parser': 8.8.0(eslint@8.57.0)(typescript@5.5.4) - '@typescript-eslint/scope-manager': 8.8.0 - '@typescript-eslint/type-utils': 8.8.0(eslint@8.57.0)(typescript@5.5.4) - '@typescript-eslint/utils': 8.8.0(eslint@8.57.0)(typescript@5.5.4) - '@typescript-eslint/visitor-keys': 8.8.0 - eslint: 8.57.0 + '@eslint-community/regexpp': 4.12.1 + '@typescript-eslint/parser': 8.18.0(eslint@9.16.0)(typescript@5.5.4) + '@typescript-eslint/scope-manager': 8.18.0 + '@typescript-eslint/type-utils': 8.18.0(eslint@9.16.0)(typescript@5.5.4) + '@typescript-eslint/utils': 8.18.0(eslint@9.16.0)(typescript@5.5.4) + '@typescript-eslint/visitor-keys': 8.18.0 + eslint: 9.16.0 graphemer: 1.4.0 ignore: 5.3.1 natural-compare: 1.4.0 ts-api-utils: 1.3.0(typescript@5.5.4) - optionalDependencies: typescript: 5.5.4 transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@8.8.0(eslint@8.57.0)(typescript@5.5.4)': + '@typescript-eslint/parser@8.18.0(eslint@9.16.0)(typescript@5.5.4)': dependencies: - '@typescript-eslint/scope-manager': 8.8.0 - '@typescript-eslint/types': 8.8.0 - '@typescript-eslint/typescript-estree': 8.8.0(typescript@5.5.4) - '@typescript-eslint/visitor-keys': 8.8.0 + '@typescript-eslint/scope-manager': 8.18.0 + '@typescript-eslint/types': 8.18.0 + '@typescript-eslint/typescript-estree': 8.18.0(typescript@5.5.4) + '@typescript-eslint/visitor-keys': 8.18.0 debug: 4.3.6 - eslint: 8.57.0 - optionalDependencies: + eslint: 9.16.0 typescript: 5.5.4 transitivePeerDependencies: - supports-color - '@typescript-eslint/scope-manager@8.8.0': + '@typescript-eslint/scope-manager@8.18.0': dependencies: - '@typescript-eslint/types': 8.8.0 - '@typescript-eslint/visitor-keys': 8.8.0 + '@typescript-eslint/types': 8.18.0 + '@typescript-eslint/visitor-keys': 8.18.0 - '@typescript-eslint/type-utils@8.8.0(eslint@8.57.0)(typescript@5.5.4)': + '@typescript-eslint/type-utils@8.18.0(eslint@9.16.0)(typescript@5.5.4)': dependencies: - '@typescript-eslint/typescript-estree': 8.8.0(typescript@5.5.4) - '@typescript-eslint/utils': 8.8.0(eslint@8.57.0)(typescript@5.5.4) + '@typescript-eslint/typescript-estree': 8.18.0(typescript@5.5.4) + '@typescript-eslint/utils': 8.18.0(eslint@9.16.0)(typescript@5.5.4) debug: 4.3.6 + eslint: 9.16.0 ts-api-utils: 1.3.0(typescript@5.5.4) - optionalDependencies: typescript: 5.5.4 transitivePeerDependencies: - - eslint - supports-color - '@typescript-eslint/types@8.8.0': {} + '@typescript-eslint/types@8.18.0': {} - '@typescript-eslint/typescript-estree@8.8.0(typescript@5.5.4)': + '@typescript-eslint/typescript-estree@8.18.0(typescript@5.5.4)': dependencies: - '@typescript-eslint/types': 8.8.0 - '@typescript-eslint/visitor-keys': 8.8.0 + '@typescript-eslint/types': 8.18.0 + '@typescript-eslint/visitor-keys': 8.18.0 debug: 4.3.6 fast-glob: 3.3.2 is-glob: 4.0.3 minimatch: 9.0.5 semver: 7.6.3 ts-api-utils: 1.3.0(typescript@5.5.4) - optionalDependencies: typescript: 5.5.4 transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.8.0(eslint@8.57.0)(typescript@5.5.4)': + '@typescript-eslint/utils@8.18.0(eslint@9.16.0)(typescript@5.5.4)': dependencies: - '@eslint-community/eslint-utils': 4.4.0(eslint@8.57.0) - '@typescript-eslint/scope-manager': 8.8.0 - '@typescript-eslint/types': 8.8.0 - '@typescript-eslint/typescript-estree': 8.8.0(typescript@5.5.4) - eslint: 8.57.0 + '@eslint-community/eslint-utils': 4.4.0(eslint@9.16.0) + '@typescript-eslint/scope-manager': 8.18.0 + '@typescript-eslint/types': 8.18.0 + '@typescript-eslint/typescript-estree': 8.18.0(typescript@5.5.4) + eslint: 9.16.0 + typescript: 5.5.4 transitivePeerDependencies: - supports-color - - typescript - '@typescript-eslint/visitor-keys@8.8.0': + '@typescript-eslint/visitor-keys@8.18.0': dependencies: - '@typescript-eslint/types': 8.8.0 - eslint-visitor-keys: 3.4.3 - - '@ungap/structured-clone@1.2.0': {} + '@typescript-eslint/types': 8.18.0 + eslint-visitor-keys: 4.2.0 '@vitejs/plugin-basic-ssl@1.1.0(vite@5.4.8(@types/node@20.14.14))': dependencies: @@ -2778,12 +2840,14 @@ snapshots: mime-types: 2.1.35 negotiator: 0.6.3 - acorn-jsx@5.3.2(acorn@8.12.1): + acorn-jsx@5.3.2(acorn@8.14.0): dependencies: - acorn: 8.12.1 + acorn: 8.14.0 acorn@8.12.1: {} + acorn@8.14.0: {} + agent-base@6.0.2: dependencies: debug: 4.3.6 @@ -2814,6 +2878,10 @@ snapshots: normalize-path: 3.0.0 picomatch: 2.3.1 + argparse@1.0.10: + dependencies: + sprintf-js: 1.0.3 + argparse@2.0.1: {} aria-query@5.3.0: @@ -2900,7 +2968,8 @@ snapshots: optionalDependencies: fsevents: 2.3.3 - clone@2.1.2: {} + cluster-key-slot@1.1.2: + optional: true code-red@1.0.4: dependencies: @@ -2943,6 +3012,8 @@ snapshots: cookie@0.6.0: {} + cookie@0.7.1: {} + cors@2.8.5: dependencies: object-assign: 4.1.1 @@ -2954,6 +3025,12 @@ snapshots: shebang-command: 2.0.0 which: 2.0.2 + cross-spawn@7.0.6: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + css-tree@2.3.1: dependencies: mdn-data: 2.0.30 @@ -2985,11 +3062,7 @@ snapshots: detect-indent@6.1.0: {} - devalue@5.0.0: {} - - doctrine@3.0.0: - dependencies: - esutils: 2.0.3 + devalue@5.1.1: {} dotenv@16.4.5: {} @@ -3015,90 +3088,6 @@ snapshots: es6-promise@3.3.1: {} - esbuild-android-64@0.14.54: - optional: true - - esbuild-android-arm64@0.14.54: - optional: true - - esbuild-darwin-64@0.14.54: - optional: true - - esbuild-darwin-arm64@0.14.54: - optional: true - - esbuild-freebsd-64@0.14.54: - optional: true - - esbuild-freebsd-arm64@0.14.54: - optional: true - - esbuild-linux-32@0.14.54: - optional: true - - esbuild-linux-64@0.14.54: - optional: true - - esbuild-linux-arm64@0.14.54: - optional: true - - esbuild-linux-arm@0.14.54: - optional: true - - esbuild-linux-mips64le@0.14.54: - optional: true - - esbuild-linux-ppc64le@0.14.54: - optional: true - - esbuild-linux-riscv64@0.14.54: - optional: true - - esbuild-linux-s390x@0.14.54: - optional: true - - esbuild-netbsd-64@0.14.54: - optional: true - - esbuild-openbsd-64@0.14.54: - optional: true - - esbuild-sunos-64@0.14.54: - optional: true - - esbuild-windows-32@0.14.54: - optional: true - - esbuild-windows-64@0.14.54: - optional: true - - esbuild-windows-arm64@0.14.54: - optional: true - - esbuild@0.14.54: - optionalDependencies: - '@esbuild/linux-loong64': 0.14.54 - esbuild-android-64: 0.14.54 - esbuild-android-arm64: 0.14.54 - esbuild-darwin-64: 0.14.54 - esbuild-darwin-arm64: 0.14.54 - esbuild-freebsd-64: 0.14.54 - esbuild-freebsd-arm64: 0.14.54 - esbuild-linux-32: 0.14.54 - esbuild-linux-64: 0.14.54 - esbuild-linux-arm: 0.14.54 - esbuild-linux-arm64: 0.14.54 - esbuild-linux-mips64le: 0.14.54 - esbuild-linux-ppc64le: 0.14.54 - esbuild-linux-riscv64: 0.14.54 - esbuild-linux-s390x: 0.14.54 - esbuild-netbsd-64: 0.14.54 - esbuild-openbsd-64: 0.14.54 - esbuild-sunos-64: 0.14.54 - esbuild-windows-32: 0.14.54 - esbuild-windows-64: 0.14.54 - esbuild-windows-arm64: 0.14.54 - esbuild@0.21.5: optionalDependencies: '@esbuild/aix-ppc64': 0.21.5 @@ -3156,63 +3145,63 @@ snapshots: escape-string-regexp@4.0.0: {} - eslint-scope@7.2.2: + eslint-scope@8.2.0: dependencies: esrecurse: 4.3.0 estraverse: 5.3.0 eslint-visitor-keys@3.4.3: {} - eslint@8.57.0: + eslint-visitor-keys@4.2.0: {} + + eslint@9.16.0: dependencies: - '@eslint-community/eslint-utils': 4.4.0(eslint@8.57.0) - '@eslint-community/regexpp': 4.11.0 - '@eslint/eslintrc': 2.1.4 - '@eslint/js': 8.57.0 - '@humanwhocodes/config-array': 0.11.14 + '@eslint-community/eslint-utils': 4.4.0(eslint@9.16.0) + '@eslint-community/regexpp': 4.12.1 + '@eslint/config-array': 0.19.1 + '@eslint/core': 0.9.1 + '@eslint/eslintrc': 3.2.0 + '@eslint/js': 9.16.0 + '@eslint/plugin-kit': 0.2.4 + '@humanfs/node': 0.16.6 '@humanwhocodes/module-importer': 1.0.1 - '@nodelib/fs.walk': 1.2.8 - '@ungap/structured-clone': 1.2.0 + '@humanwhocodes/retry': 0.4.1 + '@types/estree': 1.0.6 + '@types/json-schema': 7.0.15 ajv: 6.12.6 chalk: 4.1.2 - cross-spawn: 7.0.3 + cross-spawn: 7.0.6 debug: 4.3.6 - doctrine: 3.0.0 escape-string-regexp: 4.0.0 - eslint-scope: 7.2.2 - eslint-visitor-keys: 3.4.3 - espree: 9.6.1 + eslint-scope: 8.2.0 + eslint-visitor-keys: 4.2.0 + espree: 10.3.0 esquery: 1.6.0 esutils: 2.0.3 fast-deep-equal: 3.1.3 - file-entry-cache: 6.0.1 + file-entry-cache: 8.0.0 find-up: 5.0.0 glob-parent: 6.0.2 - globals: 13.24.0 - graphemer: 1.4.0 ignore: 5.3.1 imurmurhash: 0.1.4 is-glob: 4.0.3 - is-path-inside: 3.0.3 - js-yaml: 4.1.0 json-stable-stringify-without-jsonify: 1.0.1 - levn: 0.4.1 lodash.merge: 4.6.2 minimatch: 3.1.2 natural-compare: 1.4.0 optionator: 0.9.4 - strip-ansi: 6.0.1 - text-table: 0.2.0 transitivePeerDependencies: - supports-color - esm-env@1.0.0: {} + esm-env@1.2.1: {} - espree@9.6.1: + espree@10.3.0: dependencies: - acorn: 8.12.1 - acorn-jsx: 5.3.2(acorn@8.12.1) - eslint-visitor-keys: 3.4.3 + acorn: 8.14.0 + acorn-jsx: 5.3.2(acorn@8.14.0) + eslint-visitor-keys: 4.2.0 + + esprima@4.0.1: {} esquery@1.6.0: dependencies: @@ -3244,18 +3233,18 @@ snapshots: signal-exit: 3.0.7 strip-final-newline: 2.0.0 - express-rate-limit@6.11.2(express@4.21.0): + express-rate-limit@7.4.1(express@4.21.2): dependencies: - express: 4.21.0 + express: 4.21.2 - express@4.21.0: + express@4.21.2: dependencies: accepts: 1.3.8 array-flatten: 1.1.1 body-parser: 1.20.3 content-disposition: 0.5.4 content-type: 1.0.5 - cookie: 0.6.0 + cookie: 0.7.1 cookie-signature: 1.0.6 debug: 2.6.9 depd: 2.0.0 @@ -3269,7 +3258,7 @@ snapshots: methods: 1.1.2 on-finished: 2.4.1 parseurl: 1.3.3 - path-to-regexp: 0.1.10 + path-to-regexp: 0.1.12 proxy-addr: 2.0.7 qs: 6.13.0 range-parser: 1.2.1 @@ -3315,9 +3304,9 @@ snapshots: transitivePeerDependencies: - supports-color - file-entry-cache@6.0.1: + file-entry-cache@8.0.0: dependencies: - flat-cache: 3.2.0 + flat-cache: 4.0.1 fill-range@7.1.1: dependencies: @@ -3340,11 +3329,10 @@ snapshots: locate-path: 6.0.0 path-exists: 4.0.0 - flat-cache@3.2.0: + flat-cache@4.0.1: dependencies: flatted: 3.3.1 keyv: 4.5.4 - rimraf: 3.0.2 flatted@3.3.1: {} @@ -3371,6 +3359,9 @@ snapshots: function-bind@1.1.2: {} + generic-pool@3.9.0: + optional: true + get-intrinsic@1.2.4: dependencies: es-errors: 1.3.0 @@ -3398,6 +3389,15 @@ snapshots: package-json-from-dist: 1.0.0 path-scurry: 1.11.1 + glob@11.0.0: + dependencies: + foreground-child: 3.3.0 + jackspeak: 4.0.2 + minimatch: 10.0.1 + minipass: 7.1.2 + package-json-from-dist: 1.0.0 + path-scurry: 2.0.0 + glob@7.2.3: dependencies: fs.realpath: 1.0.0 @@ -3407,9 +3407,7 @@ snapshots: once: 1.4.0 path-is-absolute: 1.0.1 - globals@13.24.0: - dependencies: - type-fest: 0.20.2 + globals@14.0.0: {} globalyzer@0.1.0: {} @@ -3503,8 +3501,6 @@ snapshots: is-number@7.0.0: {} - is-path-inside@3.0.3: {} - is-reference@3.0.2: dependencies: '@types/estree': 1.0.5 @@ -3519,12 +3515,21 @@ snapshots: optionalDependencies: '@pkgjs/parseargs': 0.11.0 - jintr@2.1.1: + jackspeak@4.0.2: dependencies: - acorn: 8.12.1 + '@isaacs/cliui': 8.0.2 + + jintr@3.2.0: + dependencies: + acorn: 8.14.0 joycon@3.1.1: {} + js-yaml@3.14.1: + dependencies: + argparse: 1.0.10 + esprima: 4.0.1 + js-yaml@4.1.0: dependencies: argparse: 2.0.1 @@ -3564,6 +3569,8 @@ snapshots: lru-cache@10.4.3: {} + lru-cache@11.0.2: {} + magic-string@0.30.11: dependencies: '@jridgewell/sourcemap-codec': 1.5.0 @@ -3607,6 +3614,10 @@ snapshots: min-indent@1.0.1: {} + minimatch@10.0.1: + dependencies: + brace-expansion: 2.0.1 + minimatch@3.1.2: dependencies: brace-expansion: 1.1.11 @@ -3641,16 +3652,12 @@ snapshots: nanoid@3.3.7: {} - nanoid@4.0.2: {} + nanoid@5.0.9: {} natural-compare@1.4.0: {} negotiator@0.6.3: {} - node-cache@5.1.2: - dependencies: - clone: 2.1.2 - normalize-path@3.0.0: {} npm-run-path@4.0.1: @@ -3711,7 +3718,12 @@ snapshots: lru-cache: 10.4.3 minipass: 7.1.2 - path-to-regexp@0.1.10: {} + path-scurry@2.0.0: + dependencies: + lru-cache: 11.0.2 + minipass: 7.1.2 + + path-to-regexp@0.1.12: {} periscopic@3.1.0: dependencies: @@ -3756,8 +3768,6 @@ snapshots: forwarded: 0.2.0 ipaddr.js: 1.9.1 - psl@1.9.0: {} - punycode@2.3.1: {} qs@6.13.0: @@ -3768,6 +3778,11 @@ snapshots: range-parser@1.2.1: {} + rate-limit-redis@4.2.0(express-rate-limit@7.4.1(express@4.21.2)): + dependencies: + express-rate-limit: 7.4.1(express@4.21.2) + optional: true + raw-body@2.5.2: dependencies: bytes: 3.1.2 @@ -3785,6 +3800,16 @@ snapshots: dependencies: picomatch: 2.3.1 + redis@4.7.0: + dependencies: + '@redis/bloom': 1.2.0(@redis/client@1.6.0) + '@redis/client': 1.6.0 + '@redis/graph': 1.1.1(@redis/client@1.6.0) + '@redis/json': 1.0.7(@redis/client@1.6.0) + '@redis/search': 1.2.0(@redis/client@1.6.0) + '@redis/time-series': 1.1.0(@redis/client@1.6.0) + optional: true + resolve-from@4.0.0: {} resolve-from@5.0.0: {} @@ -3795,10 +3820,6 @@ snapshots: dependencies: glob: 7.2.3 - rimraf@3.0.2: - dependencies: - glob: 7.2.3 - rollup@4.24.0: dependencies: '@types/estree': 1.0.6 @@ -3899,7 +3920,7 @@ snapshots: signal-exit@4.1.0: {} - sirv@2.0.4: + sirv@3.0.0: dependencies: '@polka/url': 1.0.0-next.25 mrmime: 2.0.0 @@ -3920,6 +3941,8 @@ snapshots: dependencies: whatwg-url: 7.1.0 + sprintf-js@1.0.3: {} + statuses@2.0.1: {} string-width@4.2.3: @@ -4011,6 +4034,12 @@ snapshots: postcss: 8.4.47 typescript: 5.5.4 + svelte-sitemap@2.6.0: + dependencies: + fast-glob: 3.3.2 + minimist: 1.2.8 + xmlbuilder2: 3.1.1 + svelte@4.2.19: dependencies: '@ampproject/remapping': 2.3.0 @@ -4037,8 +4066,6 @@ snapshots: syscall-napi@0.0.6: optional: true - text-table@0.2.0: {} - thenify-all@1.6.0: dependencies: thenify: 3.3.1 @@ -4114,8 +4141,6 @@ snapshots: dependencies: prelude-ls: 1.2.1 - type-fest@0.20.2: {} - type-is@1.6.18: dependencies: media-typer: 0.3.0 @@ -4123,15 +4148,14 @@ snapshots: typedarray@0.0.6: {} - typescript-eslint@8.8.0(eslint@8.57.0)(typescript@5.5.4): + typescript-eslint@8.18.0(eslint@9.16.0)(typescript@5.5.4): dependencies: - '@typescript-eslint/eslint-plugin': 8.8.0(@typescript-eslint/parser@8.8.0(eslint@8.57.0)(typescript@5.5.4))(eslint@8.57.0)(typescript@5.5.4) - '@typescript-eslint/parser': 8.8.0(eslint@8.57.0)(typescript@5.5.4) - '@typescript-eslint/utils': 8.8.0(eslint@8.57.0)(typescript@5.5.4) - optionalDependencies: + '@typescript-eslint/eslint-plugin': 8.18.0(@typescript-eslint/parser@8.18.0(eslint@9.16.0)(typescript@5.5.4))(eslint@9.16.0)(typescript@5.5.4) + '@typescript-eslint/parser': 8.18.0(eslint@9.16.0)(typescript@5.5.4) + '@typescript-eslint/utils': 8.18.0(eslint@9.16.0)(typescript@5.5.4) + eslint: 9.16.0 typescript: 5.5.4 transitivePeerDependencies: - - eslint - supports-color typescript@5.5.4: {} @@ -4206,12 +4230,22 @@ snapshots: wrappy@1.0.2: {} + xmlbuilder2@3.1.1: + dependencies: + '@oozcitak/dom': 1.15.10 + '@oozcitak/infra': 1.0.8 + '@oozcitak/util': 8.3.8 + js-yaml: 3.14.1 + + yallist@4.0.0: + optional: true + yocto-queue@0.1.0: {} - youtubei.js@10.5.0: + youtubei.js@13.0.0: dependencies: '@bufbuild/protobuf': 2.1.0 - jintr: 2.1.1 + jintr: 3.2.0 tslib: 2.6.3 undici: 5.28.4 diff --git a/web/changelogs/10.3.md b/web/changelogs/10.3.md new file mode 100644 index 00000000..9adfc6e3 --- /dev/null +++ b/web/changelogs/10.3.md @@ -0,0 +1,101 @@ +--- +title: "fastest cobalt yet, new youtube features, translation platform, and a lot more" +date: "4 Nov, 2024" +banner: + file: "meowbalt_very_fast.webp" + alt: "meowbalt absolutely zooming through space and time (only meowbalt and his speed trail are pictured)." +--- + +## oh-so-fast +starting from this update, cobalt can run several instances in parallel, reducing load on individual instances and making it much faster. +previously cobalt ran on *only one thread*, and it's honestly impressive that it lasted this long. + +we tested cobalt under peak traffic load & same network conditions: +- initial request processing is now **~14 times faster than before**. +- starting a tunnel is now **~32 times faster**. + +
+ +| | 10.2 | **10.3** | +|-------------------|---------|------------| +| processing | 14780ms | **1070ms** | +| starting a tunnel | 11660ms | **360ms** | + +
+ +these tests weren't really scientific as we based them on screen recordings, +but the point still stands: cobalt no longer slows down and runs as fast as it can. + +## youtube improvements +- added a [new hls option](/settings/video#youtube-hls) that allows for downloading *more formats* of youtube videos. +- fixed an issue that caused long youtube videos to get abruptly cut off. if you still experience this issue, try enabling the [new hls option](/settings/video#youtube-hls) in settings! +- added an option to [pick any audio track language](/settings/audio#youtube-dub) for youtube videos in settings. all languages that youtube supports are listed, cobalt will fall back to default if preferred language isn't available. +- if a [youtube codec](/settings/video#youtube-codec) isn't available, cobalt will now fall back to the next best one. + +## meet weblate, a place where you can translate cobalt +we're finally ready to invite you to translate cobalt to any language you like! your translation contributions are linked to your github account, so you'll show up in cobalt's contributors list. + +you can start translating cobalt at [i18n.imput.net](https://i18n.imput.net/) right now! + +thank you for showing such an overwhelming amount of interest in making cobalt more accessible around the world, we really appreciate it! + +## other service improvements +- added support for bookmark links from twitter. +- fixed parsing of some mobile tiktok links. +- fixed twitter gifs having an incorrect extension in the content picker. +- fixed a bug that broke downloading older (shorter) links from streamable. +- fixed video downloading from odnoklassniki (ok.ru). + +## ui/ux improvements +- [always-on file tunneling](/settings/privacy#tunnel) is out of beta! feel free to use it if your isp tracks or filters your internet traffic. +- redesigned the [community & support page](/about/community), added bluesky and removed support email. +- improved the debug page: added a button to copy data, added current states, fixed padding. if you're curious, it can be enabled in [advanced settings](/settings/advanced#debug). +- reduced timeouts on action buttons in security warning popups as they were very annoying before. +- added a message about cobalt not being fully usable without javascript when the page is loaded without it. +- improved contrast of all emoji icons on the home/save page. +- improved contrast of the toggle button. +- fixed the color of text selection, it's no longer hideous. +- audio bitrate section now gets greyed out when it's not applicable. +- fixed cursor state (pointer, arrow, etc) on various buttons. +- fixed a bug when iphone landscape mode optimizations were applied incorrectly (fix for a bug in ios firefox). +- various text/phrasing improvements across ui. +- small padding improvements across ui. +- other small improvements. + +## documentation improvements +- all [documentation on github](https://github.com/imputnet/cobalt) was majorly improved. all projects and docs are now listed in the main readme. all docs are now easier to read and follow. +- added a new document outlining all [instance protection methods](https://github.com/imputnet/cobalt/blob/main/docs/protect-an-instance.md) along with step-by-step tutorials on how to configure them. +- added a tutorial for [configuring a cobalt instance for youtube downloading](https://github.com/imputnet/cobalt/blob/main/docs/configure-for-youtube.md). +- updated [contribution guidelines](https://github.com/imputnet/cobalt/blob/main/CONTRIBUTING.md). +- updated [examples](https://github.com/imputnet/cobalt/tree/main/docs/examples) for cookie & docker compose files. we now recommend running cobalt api as **read only** image, as it ensures that it wasn't tampered with. we do it on our servers, too. + +## internal improvements for nerds +- added support for api keys, api instance hosters are now able to limit access to a set of people. you can see [how to configure them on github](https://github.com/imputnet/cobalt/blob/main/docs/protect-an-instance.md#configure-api-keys). +- cobalt api docker image is now running alpine & node 23. it's also much smaller than before. +- instances now log whether they were able to load cookies or api keys. no more guessing if your config works or not. +- updated the console error when cobalt api is configured incorrectly. +- majorly refactored the youtube module. +- lots of general api code refactoring. +- improved settings schema migration on frontend. +- removed outdated api functions, util scripts, and docs. + +## fixed a XSS vulnerability that wasn't exploited +a malicious cobalt instance could serve links with the javascript: protocol, resulting in XSS when the user tries to download an item from a picker. + +as far as we know, this vulnerability was never found and exploited in the wild, but we still urge all frontend instance hosters to **update their instances asap**. cobalt.tools and all other instances that configured CSP correctly weren't affected by this vulnerability. + +this issue was fully fixed in [c4be1d3](https://github.com/imputnet/cobalt/commit/c4be1d3a37b0deb6b6087ec7a815262ac942daf1) and [an advisory with CVE was posted on github](https://github.com/imputnet/cobalt/security/advisories/GHSA-cm4c-v4cm-3735). + +if you ever discover a security vulnerability in cobalt, please report it responsibly [on github](https://github.com/imputnet/cobalt/security/advisories/new). we'll make sure to fix it as soon as possible! + +## where's 10.2? +we were very excited to release the first part of changes, so we bumped the version early. then, we decided to make cobalt faster, so now we're at 10.3! + +*we also silently released changes in prod before the announcement, teehee :3c* + +## all changes are on github +as always, you can check [all commits since the 10.1 release on github](https://github.com/imputnet/cobalt/compare/f461b02...c021293) for even more details. + +we hope you enjoy this update as much as you enjoy fresh air, because it really feels like one! + +\~ your friends at imput ❤ī¸ \ No newline at end of file diff --git a/web/changelogs/10.5.md b/web/changelogs/10.5.md new file mode 100644 index 00000000..10776b58 --- /dev/null +++ b/web/changelogs/10.5.md @@ -0,0 +1,91 @@ +--- +title: "merry christmas and happy new year!" +date: "23 Dec, 2024" +banner: + file: "newyear2025.webp" + alt: "meowth plush in a christmas hat sitting in front of a shiny christmas tree." +--- + +## where the elves at? +we are back once again with another cobalt update, whether you like it or not! just like santa, we come when you least expect us. + +we're back to the battlefield against youtube's scraper flattener, but we're winning so far! we even managed to squeeze in a ton of improvements that range from performance bumps to ui overhauls to brand new features. make sure to read further or you might end up on the naughty list... + +## even more youtube improvements +- countless infrastructure improvements and developments that allowed us to keep youtube support available during the worst times. +- improved youtube codec fallback. now cobalt goes through all codecs to find you the best one! +- improved youtube video quality selection & fallback. + +## improvements for other services +- added support for loom's video embed links. +- added support for facebook's mobile subdomain links. +- fixed a bug in the instagram module where it wouldn't use the graphql api on failure, due to which cobalt was unable to load slightly more posts successfully. now the majority of posts are accessible! +- removed support for vine because 𝕏, "The Everything App", broke the vine archive. +- increased performance of downloads from bluesky by using the video cdn directly. +- error messages from bluesky module are now more descriptive. +- rewrote the vk video extraction module to use the general api as the web app extraction was broken by a vk update. +- added support for new vk video links. +- cobalt now shows an appropriate error if: + - soundcloud track is region locked or paywalled. + - tiktok post is age restricted or otherwise unavailable. + - rutube video is region locked. + - vk video is region locked. + +*~ still reading? that's impressive. ~* + +## web app (and ui/ux) improvements +- added support for [instance access keys](/settings/instances#access-key)! now you can access private cobalt instances with no turnstile, directly from the web app. +- redesigned the [remux page](/remux) to indicate better what remuxing does and what it's for. +- majorly improved the reliability of turnstile. it no longer gets stuck in the background, and cobalt always keeps track of its state and displays it in the omnibox. +- rewrote almost all error messages in an effort to make them easier to understand at a glance. +- added more error messages to describe processing issues even better whenever possible. +- added animations to omnibox icons that make them more lively and cute. +- improved the toggle animation, made it stretchy and jumpy just like the rest of the ui. +- made the cobalt web app fully compatible with RTL languages (such as arabic). +- added an automatically generated sitemap, making the web app easier to index by search engine crawlers. +- made it way easier to override the selfhosted processing instance in a selfhosted web app. +- removed an extra security warning in the selfhosted web app which appeared when the processing instance didn't match the default one. +- added the "community instance" label to the web app that appears on instances different from the official one, making it easier to differentiate them from one another. +- updated cobalt embed description to be less corny. +- fixed a bug that caused settings to be exported improperly on ios in PWA mode. now they're extracted via the share api, just like all other files! +- fixed the weird focus borders in chromium browsers that appeared after a recent browser update. +- optimized rendering of the _supported services_ popover & updated its animation. +- improved accessibility of the web app all around. +- other tiny but mighty changes. + +*~ đŸĻ†đŸ”œ ~* + +## processing instance improvements +*(mostly nerd talk)* +- added support for one more way of youtube authentication: web cookies with poToken & visitorData, so that everyone can access youtube on their instances again! +- significantly refactored the cookie system for better error resistance. +- added success and error console messages to indicate whether cookies/keys were loaded successfully. +- cobalt now warns if it was unable to save updated cookies back to the file. +- majorly refactored the youtube module and removed unnecessary extra loops. +- cobalt no longer loads unnecessary data from youtube when not needed. +- fixed a bug where cobalt tried to proxy URLs on local network when global proxy was configured. +- fixed a bug that caused some HLS videos to be impossible to download in the "mute" download mode. +- fixed a bug where cobalt stacked HLS streams several times within itself which caused heavily reduced performance. +- fixed a bug where cobalt did not use a dispatcher on a HLS stream's chunks, sometimes causing it to access content from an incorrect IP address. +- refactored automatic testing CI, made service tests easier to manage. +- reduced docker container privileges to a regular user. +- improved rich filename & metadata support. all metadata is now added to the file "as-is" with no modifications at all. filenames are now compatible with all operating systems and files should never appear as "tunnel", even in some rare cases. + +## more details +as always, you can check [all commits since the last release on github](https://github.com/imputnet/cobalt/compare/c021293...41430ff) for *literally all changes!* + +## thank you! +our [github repo](https://github.com/imputnet/cobalt) reached over 20k stars recently, and around the same time the cobalt web app reached over 150k daily visitors. both of these numbers are insane to think about, thank you so much for your support! + +this is the last big update of 2024, the most transformative and exciting year for cobalt yet. +we're already working on new cool features that'll come out next year :3 + +we hope you have amazing holidays and 2025! + +\~ your jolly friends at imput 🎄 + +## donate to imput +plz [donate](/donate), we as elves work all day and night + +![sad hampter in a christmas hat](data:image/webp;base64,UklGRn4BAABXRUJQVlA4IHIBAAAwDgCdASpAAEkAP1Waw1oxqqckKbqq2jAqiWIA0kkRgW9ViVdiWdXKQi6/gdi6yh7EP2hdKybn20T+5U2HORdT1INF/azUgAe83P37UIt7DaMNjNpN1q36xYwYmvqRvyHZCbmjuEi8jMI5QwpK+A6PL5WzAeMK1HHSwAD+6mC6qPoWsYNuVCVokfhT4iULSdrgIUxMVYuFmvaB6EO1tiQsDKgGz3TT/evh4KRuHM3hK23nOULaAYPQUKFqt6mmdlUXEnnkybyuQspqBd7vYu7KCfAgexNvxKgitS1o+4JfpkOuihhRfUFRqB2Z63FsbgywZxKR9zkIWWPVYn5XIBJX6LS+AU0fc7hHnV7I0boYFlIgvVJQX0k1Tcuvk4aS9UnxcZXhLIrob7G+vHgUt4z1jVbRN+cMa/ymg+mH2qtsTW1QyhZqaerV930ZZFSsPbSlxCabNU46cRYJ3EIYwxRS6n16lWtc1hKg3Tk23rG9AAAA) +![one more sad hampter in a christmas hat](data:image/webp;base64,UklGRn4BAABXRUJQVlA4IHIBAAAwDgCdASpAAEkAP1Waw1oxqqckKbqq2jAqiWIA0kkRgW9ViVdiWdXKQi6/gdi6yh7EP2hdKybn20T+5U2HORdT1INF/azUgAe83P37UIt7DaMNjNpN1q36xYwYmvqRvyHZCbmjuEi8jMI5QwpK+A6PL5WzAeMK1HHSwAD+6mC6qPoWsYNuVCVokfhT4iULSdrgIUxMVYuFmvaB6EO1tiQsDKgGz3TT/evh4KRuHM3hK23nOULaAYPQUKFqt6mmdlUXEnnkybyuQspqBd7vYu7KCfAgexNvxKgitS1o+4JfpkOuihhRfUFRqB2Z63FsbgywZxKR9zkIWWPVYn5XIBJX6LS+AU0fc7hHnV7I0boYFlIgvVJQX0k1Tcuvk4aS9UnxcZXhLIrob7G+vHgUt4z1jVbRN+cMa/ymg+mH2qtsTW1QyhZqaerV930ZZFSsPbSlxCabNU46cRYJ3EIYwxRS6n16lWtc1hKg3Tk23rG9AAAA) diff --git a/web/i18n/en/about.json b/web/i18n/en/about.json index cfe9129f..72b7fe40 100644 --- a/web/i18n/en/about.json +++ b/web/i18n/en/about.json @@ -8,12 +8,6 @@ "page.terms": "terms and ethics", "page.credits": "thanks & licenses", - "community.discord": "community discord server", - "community.twitter": "news account on twitter", - "community.github": "github repo", - "community.email": "support email", - "community.telegram": "news channel on telegram", - "heading.general": "general terms", "heading.licenses": "licenses", "heading.summary": "best way to save what you love", @@ -27,5 +21,15 @@ "heading.responsibility": "user responsibilities", "heading.abuse": "reporting abuse", "heading.motivation": "motivation", - "heading.testers": "beta testers" + "heading.testers": "beta testers", + + "support.github": "check out cobalt's source code, contribute changes, or report issues", + "support.discord": "chat with the community and developers about cobalt or ask for help", + "support.twitter": "follow cobalt's updates and development on your twitter timeline", + "support.telegram": "stay up to date with latest cobalt updates via a telegram channel", + "support.bluesky": "follow cobalt's updates and development on your bluesky feed", + + "support.description.issue": "if you want to report a bug or some other recurring issue, please do it on github.", + "support.description.help": "use discord for any other questions. describe the issue properly in #cobalt-support or else no one will be able help you.", + "support.description.best-effort": "all support is best effort and not guaranteed, a reply might take some time." } diff --git a/web/i18n/en/about/credits.md b/web/i18n/en/about/credits.md index 6c001f58..812f3394 100644 --- a/web/i18n/en/about/credits.md +++ b/web/i18n/en/about/credits.md @@ -6,6 +6,17 @@ import BetaTesters from "$components/misc/BetaTesters.svelte"; +
+ + +cobalt is made with love and care by the [imput](https://imput.net/) research and development team. + +you can support us on the [donate page](/donate)! +
+
diff --git a/web/i18n/en/about/privacy.md b/web/i18n/en/about/privacy.md index b19ca762..7291aff4 100644 --- a/web/i18n/en/about/privacy.md +++ b/web/i18n/en/about/privacy.md @@ -58,7 +58,7 @@ plausible doesn't use cookies and is fully compliant with GDPR, CCPA, and PECR. [learn more about plausible's dedication to privacy.](https://plausible.io/privacy-focused-web-analytics) -if you wish to opt out of anonymous analytics, you can do it in privacy settings. +if you wish to opt out of anonymous analytics, you can do it in [privacy settings](/settings/privacy#analytics).
{/if} diff --git a/web/i18n/en/about/terms.md b/web/i18n/en/about/terms.md index a134ab84..634e7502 100644 --- a/web/i18n/en/about/terms.md +++ b/web/i18n/en/about/terms.md @@ -48,9 +48,10 @@ fair use and credits benefit everyone. sectionId="abuse" /> -we have no way of detecting abusive behavior automatically, as cobalt is 100% anonymous. -however, you can report such activities to us and we will do our best to comply manually: [safety@imput.net](mailto:safety@imput.net) +we have no way of detecting abusive behavior automatically because cobalt is 100% anonymous. +however, you can report such activities to us via email and we'll do our best to comply manually: abuse[at]imput.net + +**this email is not intended for user support, you will not get a response if your concern is not related to abuse.** -please note that this email is not intended for user support. if you're experiencing issues, contact us via any preferred method on [the support page](/about/community).
diff --git a/web/i18n/en/dialog.json b/web/i18n/en/dialog.json index a2688f6d..3e6f5dec 100644 --- a/web/i18n/en/dialog.json +++ b/web/i18n/en/dialog.json @@ -1,6 +1,6 @@ { - "reset.title": "reset all settings?", - "reset.body": "are you sure you want to reset all settings? this action is immediate and irreversible.", + "reset.title": "reset all data?", + "reset.body": "are you sure you want to reset all data? this action is immediate and irreversible.", "picker.title": "select what to save", "picker.description.desktop": "click an item to save it. images can also be saved via the right click menu.", diff --git a/web/i18n/en/error.json b/web/i18n/en/error.json index 67dd911c..6b83ceb9 100644 --- a/web/i18n/en/error.json +++ b/web/i18n/en/error.json @@ -1,54 +1,70 @@ { - "import.no_data": "there's nothing to load from the file. are you sure it's the right one?", - "import.invalid": "your file doesn't have valid cobalt settings to import. are you sure it's the right one?", + "import.no_data": "there are no settings to load from this file. are you sure it's the right one?", + "import.invalid": "this file doesn't have valid cobalt settings to import. are you sure it's the right one?", "import.unknown": "couldn't load data from the file. it may be corrupted or of wrong format. here's the error i got:\n\n{{ value }}", "remux.corrupted": "couldn't read the metadata from this file, it may be corrupted.", - "remux.out_of_resources": "cobalt ran out of resources and can't continue with on-device processing. this is related to limitations on your browser's side. try refreshing or reopening the app and trying again. some devices can only process tiny files.", + "remux.out_of_resources": "cobalt ran out of resources and can't continue with on-device processing. this is caused by your browser's limitations. refresh or reopen the app and try again!", - "tunnel.probe": "couldn't verify whether you can download this file. try again in a few seconds!", + "tunnel.probe": "couldn't test this tunnel. your browser or network configuration may be blocking access to one of cobalt servers. are you sure you don't have any weird browser extensions?", - "captcha_ongoing": "still checking if you're not a bot. wait for the spinner to disappear and try again.\n\nif it takes too long, please let us know! we use cloudflare turnstile for bot protection and sometimes it blocks people for no reason.", + "captcha_ongoing": "cloudflare turnstile is still checking if you're not a bot. if it takes too long, you can try: disabling weird browser extensions, changing networks, using a different browser, or checking your device for malware.", - "api.auth.jwt.missing": "couldn't confirm whether you're not a robot because the processing server didn't receive the human access token. try again in a few seconds or reload the page!", - "api.auth.jwt.invalid": "couldn't confirm whether you're not a robot because your human access token expired and wasn't renewed. try again in a few seconds or reload the page!", - "api.auth.turnstile.missing": "couldn't confirm whether you're not a robot because the processing server didn't receive the human access token. try again in a few seconds or reload the page!", - "api.auth.turnstile.invalid": "couldn't confirm whether you're not a robot because your human access token expired and wasn't renewed. try again in a few seconds or reload the page!", + "api.auth.jwt.missing": "couldn't authenticate with the processing instance because the access token is missing. try again in a few seconds or reload the page!", + "api.auth.jwt.invalid": "couldn't authenticate with the processing instance because the access token is invalid. try again in a few seconds or reload the page!", + "api.auth.turnstile.missing": "couldn't authenticate with the processing instance because the captcha solution is missing. try again in a few seconds or reload the page!", + "api.auth.turnstile.invalid": "couldn't authenticate with the processing instance because the captcha solution is invalid. try again in a few seconds or reload the page!", - "api.unreachable": "couldn't connect to the processing server. check your internet connection and try again.", - "api.timed_out": "the processing server took way too long to respond. it may be overwhelmed at the moment, try again in a few seconds!", - "api.rate_exceeded": "you're making way too many requests. try again in {{ limit }} seconds!", - "api.capacity": "cobalt is at capacity and can't process your request at the moment. try again in a few seconds. if it still doesn't work, let us know and we'll try to help!", + "api.auth.key.missing": "an access key is required to use this processing instance but it's missing. add it in instance settings!", + "api.auth.key.not_api_key": "an access key is required to use this processing instance but it's missing. add it in instance settings!", - "api.generic": "something went wrong and i couldn't get anything for you. try again in a few seconds, but if issue sticks, let us know and we'll try to help!", - "api.unknown_response": "couldn't parse the response from the server. this could be caused by a version mismatch. are you sure you're on the latest version of cobalt?", + "api.auth.key.invalid": "the access key is invalid. reset it in instance settings and use a proper one!", + "api.auth.key.not_found": "the access key you used couldn't be found. are you sure this instance has your key?", + "api.auth.key.invalid_ip": "your ip address couldn't be parsed. something went very wrong. report this issue!", + "api.auth.key.ip_not_allowed": "your ip address is not allowed to use this access key. use a different instance or network!", + "api.auth.key.ua_not_allowed": "your user agent is not allowed to use this access key. use a different client or device!", + + "api.unreachable": "couldn't connect to the processing instance. check your internet connection and try again!", + "api.timed_out": "the processing instance took too long to respond. it may be overwhelmed at the moment, try again in a few seconds!", + "api.rate_exceeded": "you're making too many requests. try again in {{ limit }} seconds.", + "api.capacity": "cobalt is at capacity and can't process your request at the moment. try again in a few seconds!", + + "api.generic": "something went wrong and i couldn't get anything for you, try again in a few seconds. if the issue sticks, please report it!", + "api.unknown_response": "couldn't read the response from the processing instance. this could be caused by a version mismatch between cobalt instances.", "api.service.unsupported": "this service is not supported yet. have you pasted the right link?", - "api.service.disabled": "this service is supported by cobalt, but it's disabled on this instance. try a link from another service!", + "api.service.disabled": "this service is generally supported by cobalt, but it's disabled on this processing instance. try a link from another service!", + "api.service.audio_not_supported": "this service doesn't support audio extraction. try a link from another service!", "api.link.invalid": "your link is invalid or this service is not supported yet. have you pasted the right link?", "api.link.unsupported": "{{ service }} is supported, but i couldn't recognize your link. have you pasted the right one?", - "api.fetch.fail": "something went wrong when fetching info from {{ service }} and i couldn't find anything for you. are you sure your link works? if it does and you still see this error, let us know and we'll try to help!", - "api.fetch.critical": "the {{ service }} module returned an error that i don't recognize. try again in a few seconds, but if issue sticks, let us know!", + "api.fetch.fail": "something went wrong when fetching info from {{ service }} and i couldn't get anything for you. if this issue sticks, please report it!", + "api.fetch.critical": "the {{ service }} module returned an error that i don't recognize. try again in a few seconds, but if this issue sticks, please report it!", "api.fetch.empty": "couldn't find any media that i could download for you. are you sure you pasted the right link?", - "api.fetch.rate": "the cobalt processing server got rate limited by the {{ service }} api. try again in a few seconds!", - "api.fetch.short_link": "couldn't get link info from the short link. are you sure it works? if it does and you still get this error, let us know, and we'll try to help!", + "api.fetch.rate": "the processing instance got rate limited by {{ service }}. try again in a few seconds!", + "api.fetch.short_link": "couldn't get info from the short link. are you sure it works? if it does and you still get this error, please report the issue!", - "api.content.too_long": "the media you requested is too long. current duration limit is {{ limit }} minutes. try something shorter instead!", + "api.content.too_long": "media you requested is too long. the duration limit on this instance is {{ limit }} minutes. try something shorter instead!", - "api.content.video.unavailable": "i can't access this video. it may be restricted on {{ service }}'s side. have you pasted the right link?", - "api.content.video.live": "this video is currently live, so i can't download it yet. wait for the live stream to finish, and then try again!", - "api.content.video.private": "this video is private, so i cannot access it. change its visibility or try another one!", - "api.content.video.age": "this video is age-restricted, so i can't access it anonymously. try another one!", - "api.content.video.region": "this video is region locked, and the processing server is in a different location. try another one!", + "api.content.video.unavailable": "i can't access this video. it may be restricted on {{ service }}'s side. try a different link!", + "api.content.video.live": "this video is currently live, so i can't download it yet. wait for the live stream to finish and try again!", + "api.content.video.private": "this video is private, so i can't access it. change its visibility or try another one!", + "api.content.video.age": "this video is age-restricted, so i can't access it anonymously. try a different link!", + "api.content.video.region": "this video is region locked, and the processing instance is in a different location. try a different link!", - "api.content.post.unavailable": "couldn't find anything about this post. its visibility may be limited or it may not exist at all. make sure your link works and try again in a few seconds!", - "api.content.post.private": "this post is from a private account, so i can't access it. have you pasted the right link?", - "api.content.post.age": "this post is age-restricted, so i can't access it anonymously. have you pasted the right link?", + "api.content.region": "this content is region locked, and the processing instance is in a different location. try a different link!", + "api.content.paid": "this content requires purchase. cobalt can't download paid content. try a different link!", - "api.youtube.codec": "youtube didn't return anything with your preferred video codec. try another one in settings!", - "api.youtube.decipher": "youtube updated its decipher algorithm and i couldn't extract the info about the video.\n\ntry again in a few seconds, but if issue sticks, contact us for support.", - "api.youtube.login": "couldn't get this video because youtube labeled me as a bot. this is potentially caused by the processing instance not having any active account tokens. try again in a few seconds, but if it still doesn't work, tell the instance owner about this error!", - "api.youtube.token_expired": "couldn't get this video because the youtube token expired and i couldn't refresh it. try again in a few seconds, but if it still doesn't work, tell the instance owner about this error!" + "api.content.post.unavailable": "couldn't find anything about this post. its visibility may be limited or it may not exist. make sure your link works and try again in a few seconds!", + "api.content.post.private": "couldn't get anything about this post because it's from a private account. try a different link!", + "api.content.post.age": "this post is age-restricted and isn't available without logging in. try a different link!", + + "api.youtube.no_matching_format": "youtube didn't return a valid video + audio format combo, either video or audio is missing. formats for this video may be re-encoding on youtube's side or something went wrong when parsing them. try enabling the hls option in video settings!", + "api.youtube.decipher": "youtube updated its decipher algorithm and i couldn't extract the info about the video. try again in a few seconds, but if this issue sticks, please report it!", + "api.youtube.login": "couldn't get this video because youtube asked the instance to log in. this is potentially caused by the processing instance not having any active account tokens or youtube updating something about their api. try again in a few seconds, but if it still doesn't work, please report this issue!", + "api.youtube.token_expired": "couldn't get this video because the youtube token expired and i couldn't refresh it. try again in a few seconds, but if it still doesn't work, tell the instance owner about this error!", + "api.youtube.no_hls_streams": "couldn't find any matching HLS streams for this video. try downloading it without HLS!", + "api.youtube.api_error": "youtube updated something about its api and i couldn't get any info about this video. try again in a few seconds, but if this issue sticks, please report it!", + "api.youtube.temporary_disabled": "youtube downloading is temporarily disabled due to restrictions from youtube's side. we're already looking for ways to go around them.\n\nwe apologize for the inconvenience and are doing our best to restore this functionality. check cobalt's socials or github for timely updates!" } diff --git a/web/i18n/en/general.json b/web/i18n/en/general.json index c3667f26..a50db46b 100644 --- a/web/i18n/en/general.json +++ b/web/i18n/en/general.json @@ -3,5 +3,5 @@ "meowbalt": "meowbalt", "beta": "beta", - "embed.description": "save what you love without ads, tracking, paywalls or other nonsense. cobalt is a truly open web app, built with love and care by imput." + "embed.description": "cobalt lets you save what you love without ads, tracking, paywalls or other nonsense. just paste the link and you're ready to rock!" } diff --git a/web/i18n/en/remux.json b/web/i18n/en/remux.json index b7a7f34a..d8b031c3 100644 --- a/web/i18n/en/remux.json +++ b/web/i18n/en/remux.json @@ -1,3 +1,8 @@ { - "description": "remuxing often fixes compatibility issues with old software. it's fast, lossless, and everything is processed on-device." + "bullet.purpose.title": "what does remux do?", + "bullet.purpose.description": "remux fixes any issues with the file container, such as missing time info. it helps increase compatibility with old software, such as vegas pro and windows media player.", + "bullet.explainer.title": "how does it work?", + "bullet.explainer.description": "remuxing takes existing codec data and copies it over to a new media container. it's lossless, media data doesn't get re-encoded.", + "bullet.privacy.title": "on-device processing", + "bullet.privacy.description": "cobalt remuxes files locally. files never leave your device, so processing is nearly instant." } diff --git a/web/i18n/en/save.json b/web/i18n/en/save.json index 79a65dfb..e6edc0de 100644 --- a/web/i18n/en/save.json +++ b/web/i18n/en/save.json @@ -19,5 +19,7 @@ "tutorial.step.3": "select the respective shortcut in the share sheet.", "tutorial.outro": "these shortcuts will work only from the cobalt app, sharing links from other apps will not work.", "tutorial.shortcut.photos": "to photos", - "tutorial.shortcut.files": "to files" + "tutorial.shortcut.files": "to files", + + "label.community_instance": "community instance" } diff --git a/web/i18n/en/settings.json b/web/i18n/en/settings.json index 0537982f..418410bf 100644 --- a/web/i18n/en/settings.json +++ b/web/i18n/en/settings.json @@ -5,7 +5,7 @@ "page.audio": "audio", "page.download": "downloading", "page.advanced": "advanced", - "page.debug": "debug information", + "page.debug": "info for nerds", "page.instances": "instances", "section.general": "general", @@ -29,16 +29,20 @@ "video.quality.144": "144p", "video.quality.description": "if preferred video quality isn't available, next best is picked instead.", - "video.youtube.codec": "youtube video codec and container", - "video.youtube.codec.description": "h264: best compatibility, average bitrate. max quality is 1080p. \nav1: best quality, efficiency, and bitrate. supports 8k & HDR. \nvp9: same quality & bitrate as av1, but file is approximately two times bigger. supports 4k & HDR.\n\nav1 and vp9 aren't as widely supported as h264.", + "video.youtube.codec": "youtube codec and container", + "video.youtube.codec.description": "h264: best compatibility, average quality. max quality is 1080p. \nav1: best quality and efficiency. supports 8k & HDR. \nvp9: same quality as av1, but file is ~2x bigger. supports 4k & HDR.\n\nav1 and vp9 aren't as widely supported as h264. if av1 or vp9 isn't available, h264 is used instead.", + + "video.youtube.hls": "youtube hls formats", + "video.youtube.hls.title": "prefer hls for video & audio", + "video.youtube.hls.description": "files download faster and are less prone to errors or getting abruptly cut off. only h264 and vp9 codecs are available in this mode. original audio codec is aac, it's re-encoded for compatibility, audio quality may be slightly worse than the non-HLS counterpart.\n\nthis option is experimental, it may go away or change in the future.", "video.twitter.gif": "twitter/x", "video.twitter.gif.title": "convert looping videos to GIF", "video.twitter.gif.description": "GIF conversion is inefficient, converted file may be obnoxiously big and low quality.", - "video.tiktok.h265": "tiktok", - "video.tiktok.h265.title": "prefer HEVC/H265 format", - "video.tiktok.h265.description": "allows downloading videos in 1080p at cost of compatibility.", + "video.h265": "high efficiency video codec", + "video.h265.title": "allow h265 for videos", + "video.h265.description": "allows downloading videos from platforms like tiktok and xiaohongshu in higher quality at cost of compatibility.", "audio.format": "audio format", "audio.format.best": "best", @@ -46,15 +50,16 @@ "audio.format.ogg": "ogg", "audio.format.wav": "wav", "audio.format.opus": "opus", - "audio.format.description": "all formats but \"best\" are converted, meaning that there'll be some quality loss. audio is not reencoded only when \"best\" format is selected.", + "audio.format.description": "all formats but \"best\" are converted from the source format, there will be some quality loss. when \"best\" format is selected, the audio is kept in its original format whenever possible.", "audio.bitrate": "audio bitrate", "audio.bitrate.kbps": "kb/s", "audio.bitrate.description": "bitrate is applied only when converting audio to a lossy format. cobalt can't improve the source audio quality, so choosing a bitrate over 128kbps may inflate the file size with no audible difference. perceived quality may vary by format.", - "audio.youtube.dub": "youtube", - "audio.youtube.dub.title": "use browser language for dubbed videos", - "audio.youtube.dub.description": "works even if cobalt isn't translated to your language.", + "audio.youtube.dub": "youtube audio track", + "audio.youtube.dub.title": "preferred dub language", + "audio.youtube.dub.description": "cobalt will use a dubbed audio track for selected language if it's available. if not, original will be used instead.", + "youtube.dub.original": "original", "audio.tiktok.original": "tiktok", "audio.tiktok.original.title": "download original sound", @@ -83,7 +88,7 @@ "accessibility": "accessibility", "accessibility.transparency.title": "reduce visual transparency", - "accessibility.transparency.description": "reduces transparency of surfaces and disables blur effects.", + "accessibility.transparency.description": "reduces transparency of surfaces and disables blur effects. may also improve ui performance on low performance devices.", "accessibility.motion.title": "reduce motion", "accessibility.motion.description": "disables animations and transitions whenever possible.", @@ -91,7 +96,7 @@ "language.auto.title": "automatic selection", "language.auto.description": "cobalt will use your browser's default language if translation is available. if not, english will be used instead.", "language.preferred.title": "preferred language", - "language.preferred.description": "this language will be used when automatic selection is disabled. any text that isn't translated will be displayed in english.\n\nwe use community-sourced translations for languages other than english, russian, and czech. they may be inaccurate or incomplete.", + "language.preferred.description": "this language will be used when automatic selection is disabled. any text that isn't translated will be displayed in english.\n\nsome languages use community-sourced translations, they may be inaccurate or incomplete.", "privacy.analytics": "anonymous traffic analytics", "privacy.analytics.title": "don't contribute to analytics", @@ -103,19 +108,19 @@ "privacy.tunnel.description": "cobalt will hide your ip address, browser info, and bypass local network restrictions. when enabled, files will also have readable filenames that otherwise would be gibberish.", "advanced.debug": "debug", - "advanced.debug.title": "enable debug features", - "advanced.debug.description": "gives you access to a page with various info that can be useful for debugging.", + "advanced.debug.title": "enable features for nerds", + "advanced.debug.description": "gives you easy access to app info that can be useful for debugging. enabling this does not affect functionality of cobalt in any way.", - "advanced.data": "settings data", - - "processing.override": "default instance override", - "processing.override.title": "use the instance-provided processing server", - "processing.override.description": "if web instance provides its own default processing server, you can choose to use it over the main processing server. make sure it's a server by someone you trust.", + "advanced.data": "data management", "processing.community": "community instances", - "processing.enable_custom.title": "use a custom processing server", - "processing.enable_custom.description": "cobalt will use a custom processing server if you choose to. even though cobalt has some security measures in place, we are not responsible for any damages done via a community instance, as we have no control over them.\n\nplease be mindful of what instances you use and make sure they're hosted by people you trust.", + "processing.enable_custom.description": "cobalt will use a custom processing instance if you choose to. even though cobalt has some security measures in place, we are not responsible for any damages done via a community instance, as we have no control over them.\n\nplease be mindful of what instances you use and make sure they're hosted by people you trust.", - "processing.custom.placeholder": "custom instance domain" + "processing.access_key": "instance access key", + "processing.access_key.title": "use an instance access key", + "processing.access_key.description": "cobalt will use this key to make requests to the processing instance instead of other authentication methods. make sure the instance supports api keys!", + + "processing.custom_instance.input.alt_text": "custom instance domain", + "processing.access_key.input.alt_text": "u-u-i-d access key" } diff --git a/web/package.json b/web/package.json index bd2854b5..0c621f02 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "@imput/cobalt-web", - "version": "10.1.0", + "version": "10.6", "type": "module", "private": true, "scripts": { @@ -30,8 +30,8 @@ "@fontsource/redaction-10": "^5.0.2", "@imput/libav.js-remux-cli": "^5.5.6", "@imput/version-info": "workspace:^", - "@sveltejs/adapter-static": "^3.0.2", - "@sveltejs/kit": "^2.0.0", + "@sveltejs/adapter-static": "^3.0.6", + "@sveltejs/kit": "^2.9.1", "@sveltejs/vite-plugin-svelte": "^3.0.0", "@tabler/icons-svelte": "3.6.0", "@types/eslint__js": "^8.42.3", @@ -40,19 +40,20 @@ "@vitejs/plugin-basic-ssl": "^1.1.0", "compare-versions": "^6.1.0", "dotenv": "^16.0.1", - "eslint": "^8.57.0", - "glob": "^10.4.5", + "eslint": "^9.16.0", + "glob": "^11.0.0", "mdsvex": "^0.11.2", "mime": "^4.0.4", "svelte": "^4.2.19", "svelte-check": "^3.6.0", "svelte-preprocess": "^6.0.2", + "svelte-sitemap": "2.6.0", "sveltekit-i18n": "^2.4.2", "ts-deepmerge": "^7.0.1", "tslib": "^2.4.1", "turnstile-types": "^1.2.2", "typescript": "^5.4.5", - "typescript-eslint": "^8.8.0", + "typescript-eslint": "^8.18.0", "vite": "^5.3.6" } } diff --git a/web/src/components/about/AboutSupport.svelte b/web/src/components/about/AboutSupport.svelte new file mode 100644 index 00000000..713c003f --- /dev/null +++ b/web/src/components/about/AboutSupport.svelte @@ -0,0 +1,121 @@ + + + + + diff --git a/web/src/components/buttons/SettingsToggle.svelte b/web/src/components/buttons/SettingsToggle.svelte index 75a564e5..5d5941a2 100644 --- a/web/src/components/buttons/SettingsToggle.svelte +++ b/web/src/components/buttons/SettingsToggle.svelte @@ -39,16 +39,14 @@ updateSetting({ [settingContext]: { [settingId]: !isEnabled, - }, - })} + } + }) + } >

{title}

- + {#if description}
{description}
{/if} @@ -78,7 +76,7 @@ align-items: center; gap: var(--padding); justify-content: space-between; - text-align: left; + text-align: start; transform: none; padding: calc(var(--switcher-padding) * 2) 16px; border-radius: var(--border-radius); diff --git a/web/src/components/buttons/Switcher.svelte b/web/src/components/buttons/Switcher.svelte index a72c4e77..0792c9e4 100644 --- a/web/src/components/buttons/Switcher.svelte +++ b/web/src/components/buttons/Switcher.svelte @@ -43,10 +43,25 @@ border-bottom-left-radius: 0; } + .switcher:not(.big):dir(rtl) :global(.button:first-child) { + border-top-right-radius: inherit; + border-bottom-right-radius: inherit; + border-top-left-radius: 0; + border-bottom-left-radius: 0; + } + + .switcher:not(.big):dir(rtl) :global(.button:last-child) { + border-top-left-radius: inherit; + border-bottom-left-radius: inherit; + border-top-right-radius: 0; + border-bottom-right-radius: 0; + } + .switcher.big { background: var(--button); box-shadow: var(--button-box-shadow); padding: var(--switcher-padding); + gap: calc(var(--switcher-padding) - 1.5px); } .switcher :global(.button.active) { diff --git a/web/src/components/changelog/ChangelogEntry.svelte b/web/src/components/changelog/ChangelogEntry.svelte index 18f474ef..e6ba53d1 100644 --- a/web/src/components/changelog/ChangelogEntry.svelte +++ b/web/src/components/changelog/ChangelogEntry.svelte @@ -141,8 +141,8 @@ :global(.changelog-banner) { display: block; object-fit: cover; - max-height: 320pt; - min-height: 210pt; + max-height: 350pt; + min-height: 180pt; width: 100%; aspect-ratio: 16/9; border-radius: var(--padding); diff --git a/web/src/components/dialog/DialogButton.svelte b/web/src/components/dialog/DialogButton.svelte index f977cc6a..69aaf722 100644 --- a/web/src/components/dialog/DialogButton.svelte +++ b/web/src/components/dialog/DialogButton.svelte @@ -23,21 +23,36 @@ onDestroy(() => clearInterval(interval)); } - - - +{#if button.link} + + {button.text} + +{:else} + +{/if} diff --git a/web/src/components/dialog/SavingDialog.svelte b/web/src/components/dialog/SavingDialog.svelte index ec719aed..03127353 100644 --- a/web/src/components/dialog/SavingDialog.svelte +++ b/web/src/components/dialog/SavingDialog.svelte @@ -144,6 +144,10 @@ gap: var(--padding); } + .dialog-inner-container:focus-visible { + box-shadow: none; + } + .dialog-inner-container { overflow-y: scroll; gap: 8px; diff --git a/web/src/components/dialog/SmallDialog.svelte b/web/src/components/dialog/SmallDialog.svelte index f76c91e1..bce93be4 100644 --- a/web/src/components/dialog/SmallDialog.svelte +++ b/web/src/components/dialog/SmallDialog.svelte @@ -134,7 +134,7 @@ } .align-left .body-text { - text-align: left; + text-align: start; } .align-left .popup-header { diff --git a/web/src/components/donate/DonateAltItem.svelte b/web/src/components/donate/DonateAltItem.svelte index 0bc45789..3f28d736 100644 --- a/web/src/components/donate/DonateAltItem.svelte +++ b/web/src/components/donate/DonateAltItem.svelte @@ -75,7 +75,7 @@ overflow: clip; justify-content: flex-start; align-items: center; - text-align: left; + text-align: start; line-break: anywhere; padding: 0; gap: 10px; diff --git a/web/src/components/donate/DonateBanner.svelte b/web/src/components/donate/DonateBanner.svelte index 0b2f1fb1..27482e25 100644 --- a/web/src/components/donate/DonateBanner.svelte +++ b/web/src/components/donate/DonateBanner.svelte @@ -100,6 +100,10 @@ bottom: 0; } + #banner-right:dir(rtl) { + position: relative; + } + #imput-logo { display: flex; } @@ -121,6 +125,11 @@ max-width: 55%; } + #banner-left:dir(rtl) { + padding-right: 47px; + padding-left: 0px; + } + #banner-title { font-family: serif; font-size: 48px; @@ -202,11 +211,6 @@ display: none; } - #banner-left { - max-width: 100%; - padding: 55px; - } - #banner-background { mask-image: linear-gradient( 180deg, @@ -219,7 +223,9 @@ justify-content: center; } - #banner-left { + #banner-left, + #banner-left:dir(rtl) { + max-width: 100%; padding: 45px 12px; gap: 14px; align-items: center; @@ -238,7 +244,8 @@ } @media screen and (max-width: 550px) { - #banner-left { + #banner-left, + #banner-left:dir(rtl) { padding: 32px 12px; gap: 12px; } diff --git a/web/src/components/donate/DonateCardContainer.svelte b/web/src/components/donate/DonateCardContainer.svelte index 026c133b..74a00339 100644 --- a/web/src/components/donate/DonateCardContainer.svelte +++ b/web/src/components/donate/DonateCardContainer.svelte @@ -34,7 +34,7 @@ display: flex; flex-direction: column; align-items: flex-start; - text-align: left; + text-align: start; border-radius: var(--donate-card-padding); background: rgba(255, 255, 255, 0.05); padding: 12px 16px; diff --git a/web/src/components/donate/DonateOptionsCard.svelte b/web/src/components/donate/DonateOptionsCard.svelte index dcd1d03c..727d2495 100644 --- a/web/src/components/donate/DonateOptionsCard.svelte +++ b/web/src/components/donate/DonateOptionsCard.svelte @@ -138,13 +138,12 @@