diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 89951fe0..00000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1 +0,0 @@ -custom: https://boosty.to/wukko/donate diff --git a/.github/ISSUE_TEMPLATE/bug-main-instance.md b/.github/ISSUE_TEMPLATE/bug-main-instance.md new file mode 100644 index 00000000..088811b3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-main-instance.md @@ -0,0 +1,36 @@ +--- +name: main instance bug report +about: report an issue with cobalt.tools or api.cobalt.tools +title: '[short description of the bug]' +labels: main instance issue +assignees: '' + +--- + +### bug description +clear and concise description of what the issue is. + +### reproduction steps +steps to reproduce the described behavior. +here's an example of what it could look like: +1. go to '...' +2. click on '....' +3. download [media type] from [service] +4. see error + +### screenshots +if applicable, add screenshots or screen recordings to support your explanation. +if not, remove this section. + +### links +if applicable, add links that cause the issue. more = better. +if not, remove this section. + +### platform information +- OS [e.g. iOS, windows] +- browser [e.g. chrome, safari, firefox] +- version [e.g. 115] + +### additional context +add any other context about the problem here if applicable. +if not, remove this section. diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index e5429401..4370171c 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -1,32 +1,36 @@ --- name: bug report -about: report an issue with downloads or something else -title: '' +about: report a global issue with the cobalt codebase +title: '[short description of the bug]' labels: bug assignees: '' --- -**bug description** -a clear and concise description of what the bug is. +### bug description +clear and concise description of what the issue is. -**reproduction steps** -steps to reproduce the behavior: +### reproduction steps +steps to reproduce the described behavior. +here's an example of what it could look like: 1. go to '...' 2. click on '....' -3. download this video: **[link here]** +3. download [media type] from [service] 4. see error -**screenshots** -if applicable, add screenshots or screen recordings to help explain your problem. +### screenshots +if applicable, add screenshots or screen recordings to support your explanation. +if not, remove this section. -**links** -if applicable, add links that cause the issue. more = better. +### links +if applicable, add links that cause the issue. more = better. +if not, remove this section. -**platform** +### platform information - OS [e.g. iOS, windows] - browser [e.g. chrome, safari, firefox] - version [e.g. 115] -**additional context** -add any other context about the problem here. +### additional context +add any other context about the problem here if applicable. +if not, remove this section. diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md index 18307f4f..806b2bdf 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -1,17 +1,15 @@ --- name: feature request about: suggest a feature for cobalt -title: '' +title: '[short feature request description]' labels: feature request assignees: '' --- -**describe the feature you'd like to see** -a clear and concise description of what you want to happen. +### describe the feature you'd like to see +clear and concise description of the feature you want to see in cobalt. -**describe alternatives you've considered** -a clear and concise description of any alternative solutions or features you've considered. - -**additional context** -add any other context or screenshots about the feature request here. +### additional context +if applicable, add any other context or screenshots related to the feature request here. +if not, remove this section. diff --git a/.github/ISSUE_TEMPLATE/hosting-help.md b/.github/ISSUE_TEMPLATE/hosting-help.md new file mode 100644 index 00000000..d10a677b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/hosting-help.md @@ -0,0 +1,12 @@ +--- +name: instance hosting help +about: ask any question regarding cobalt instance hosting +title: '[short description of the problem]' +labels: instance hosting help +assignees: '' + +--- + +### problem description +describe what issue you're having, clearly and concisely. +support your description with screenshots/links/etc when needed. diff --git a/.github/ISSUE_TEMPLATE/service-request.md b/.github/ISSUE_TEMPLATE/service-request.md new file mode 100644 index 00000000..ca948656 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/service-request.md @@ -0,0 +1,18 @@ +--- +name: service request +about: request service support in cobalt +title: 'add support for [service name]' +labels: service request +assignees: '' + +--- + +### service name & description +provide the service name and brief description of what it is. + +### link samples for the service you'd like cobalt to support +list of links that cobalt should recognize. +could be regular video link, shared video link, mobile video link, shortened link, etc. + +### additional context +any additional context or screenshots should go here. if there aren't any, just remove this part. diff --git a/.github/test.sh b/.github/test.sh index 85683b91..80de1fcd 100755 --- a/.github/test.sh +++ b/.github/test.sh @@ -18,7 +18,7 @@ test_api() { -X POST \ -H "Accept: application/json" \ -H "Content-Type: application/json" \ - -d '{"url":"https://www.youtube.com/watch?v=jNQXAC9IVRw"}') + -d '{"url":"https://vine.co/v/huwVJIEJW50", "isAudioOnly": true}') echo "$API_RESPONSE" STATUS=$(echo "$API_RESPONSE" | jq -r .status) diff --git a/.github/workflows/fast-forward.yml b/.github/workflows/fast-forward.yml new file mode 100644 index 00000000..adda8f35 --- /dev/null +++ b/.github/workflows/fast-forward.yml @@ -0,0 +1,22 @@ +name: fast-forward +on: + issue_comment: + types: [created, edited] +jobs: + fast-forward: + # Only run if the comment contains the /fast-forward command. + if: ${{ contains(github.event.comment.body, '/fast-forward') + && github.event.issue.pull_request }} + runs-on: ubuntu-latest + + permissions: + contents: write + pull-requests: write + issues: write + + steps: + - name: Fast forwarding + uses: sequoia-pgp/fast-forward@v1 + with: + merge: true + comment: 'on-error' \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 193ddd01..4ac2daf3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,3 +36,27 @@ jobs: uses: actions/checkout@v4 - name: Run test script run: .github/test.sh api + + check-services: + name: test service functionality + runs-on: ubuntu-latest + outputs: + services: ${{ steps.checkServices.outputs.service_list }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - id: checkServices + run: npm ci && echo "service_list=$(node src/util/test-ci get-services)" >> "$GITHUB_OUTPUT" + + test-services: + needs: check-services + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + service: ${{ fromJson(needs.check-services.outputs.services) }} + name: "test service: ${{ matrix.service }}" + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - run: npm ci && node src/util/test-ci run-tests-for ${{ matrix.service }} \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..2668242b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,39 @@ +# contributing to cobalt +if you're reading this, you are probably interested in contributing to cobalt, which we are very thankful for :3 + +this document serves as a guide to help you make contributions that we can merge into the cobalt codebase. + +## translations +currently, we are **not accepting** translations of cobalt. this is because we are making significant changes to the frontend, and the currently used localization structure is being completely reworked. if this changes, this document will be updated. + +## adding features or support for services +before putting in the effort to implement a feature, it's worth considering whether it would be appropriate to add it to cobalt. the cobalt api is built to assist people **only with downloading freely accessible content**. other functionality, such as: +- downloading paid / not publicly accessible content +- downloading content protected by DRM +- scraping unrelated information & exposing it outside of file metadata + +will not be reviewed or merged. + +if you plan on adding a feature or support for a service, but are unsure whether it would be appropriate, it's best to open an issue and discuss it beforehand. + +## git +when contributing code to cobalt, there are a few guidelines in place to ensure that the code history is readable and comprehensible. + +### clean commit messages +internally, we use a format similar to [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) - the first part signifies which part of the code you are changing (the *scope*), and the second part explains the change. for inspiration on how to write appropriate commit titles, you can take a look at the [commit history](https://github.com/imputnet/cobalt/commits/). + +the scope is not strictly defined, you can write whatever you find most fitting for the particular change. suppose you are changing a small part of a more significant part of the codebase. in that case, you can specify both the larger and smaller scopes in the commit message for clarity (e.g., if you were changing something in internal streams, the commit could be something like `stream/internal: fix object not being handled properly`). + +if you think a change deserves further explanation, we encourage you to write a short explanation in the commit message ([example](https://github.com/imputnet/cobalt/commit/d2e5b6542f71f3809ba94d56c26f382b5cb62762)), which will save both you and us time having to enquire about the change, and you explaining the reason behind it. + +if your contribution has uninformative commit titles, you may be asked to interactively rebase your branch and amend each commit to include a meaningful title. + +### clean commit history +if your branch is out of date and/or has some merge conflicts with the `current` branch, you should **rebase** it instead of merging. this prevents meaningless merge commits from being included in your branch, which would then end up in the cobalt git history. + +if you find a mistake or bug in your code before it's merged or reviewed, instead of making a brand new commit to fix it, it would be preferable to amend that specific commit where the mistake was first introduced. this also helps us easily revert a commit if we discover that it introduced a bug or some unwanted behavior. +- if the commit you are fixing is the latest one, you can add your files to staging and then use `git commit --amend` to apply the change. +- if the commit is somewhere deeper in your branch, you can use `git commit --fixup=HASH`, where *`HASH`* is the commit you are fixing. + - afterward, you must interactively rebase your branch with `git rebase -i current --autosquash`. + this will open up an editor, but you don't need to do anything else except save the file and exit. +- once you do either of these things, you will need to do a **force push** to your branch with `git push --force-with-lease`. diff --git a/README.md b/README.md index 55c5ee3c..5267cd53 100644 --- a/README.md +++ b/README.md @@ -19,11 +19,13 @@ this list is not final and keeps expanding over time. if support for a service y | bilibili.com & bilibili.tv | ✅ | ✅ | ✅ | ➖ | ➖ | | dailymotion | ✅ | ✅ | ✅ | ✅ | ✅ | | instagram posts & reels | ✅ | ✅ | ✅ | ➖ | ➖ | +| facebook videos | ✅ | ❌ | ❌ | ➖ | ➖ | | loom | ✅ | ❌ | ✅ | ✅ | ➖ | | ok video | ✅ | ❌ | ✅ | ✅ | ✅ | | pinterest | ✅ | ✅ | ✅ | ➖ | ➖ | | reddit | ✅ | ✅ | ✅ | ❌ | ❌ | | rutube | ✅ | ✅ | ✅ | ✅ | ✅ | +| snapchat stories & spotlights | ✅ | ✅ | ✅ | ➖ | ➖ | | soundcloud | ➖ | ✅ | ➖ | ✅ | ✅ | | streamable | ✅ | ✅ | ✅ | ➖ | ➖ | | threads posts | ✅ | ✅ | ✅ | ➖ | ➖ | @@ -46,8 +48,10 @@ this list is not final and keeps expanding over time. if support for a service y | service | notes or features | | :-------- | :----- | | instagram | supports reels, photos, and videos. lets you pick what to save from multi-media posts. | +| facebook | supports public accessible videos content only. | | pinterest | supports photos, gifs, videos and stories. | | reddit | supports gifs and videos. | +| snapchat | supports spotlights and stories. lets you pick what to save from stories. | | rutube | supports yappy & private links. | | soundcloud | supports private links. | | threads | supports photos and videos. lets you pick what to save from multi-media posts. | diff --git a/docs/api.md b/docs/api.md index 87ef2489..6cd66bba 100644 --- a/docs/api.md +++ b/docs/api.md @@ -49,9 +49,9 @@ item type: `object` | key | type | variables | description | |:--------|:---------|:--------------------------------------------------------|:---------------------------------------| -| `type` | `string` | `video` | used only if `pickerType`is `various`. | +| `type` | `string` | `video / photo / gif` | used only if `pickerType` is `various` | | `url` | `string` | direct link to a file or a link to cobalt's live render | | -| `thumb` | `string` | item thumbnail that's displayed in the picker | used only for `video` type. | +| `thumb` | `string` | item thumbnail that's displayed in the picker | used for `video` and `gif` types | ## GET: `/api/stream` cobalt's live render (or stream) endpoint. usually, you will receive a url to this endpoint @@ -73,5 +73,5 @@ response body type: `application/json` | `branch` | `string` | git branch | | `name` | `string` | server name | | `url` | `string` | server url | -| `cors` | `int` | cors status | +| `cors` | `number` | cors status | | `startTime` | `string` | server start time | diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md index a440d11d..d31a67f5 100644 --- a/docs/run-an-instance.md +++ b/docs/run-an-instance.md @@ -56,8 +56,9 @@ sudo service nscd start | `API_LISTEN_ADDRESS` | `0.0.0.0` | `127.0.0.1` | changes address from which api server is accessible. **if you are using docker, you usually don't need to configure this.** | | `API_URL` | ➖ | `https://api.cobalt.tools/` | changes url from which api server is accessible.
***REQUIRED TO RUN THE API***. | | `API_NAME` | `unknown` | `ams-1` | api server name that is shown in `/api/serverInfo`. | +| `API_EXTERNAL_PROXY` | ➖ | `http://user:password@127.0.0.1:8080`| url of the proxy that will be passed to [`ProxyAgent`](https://undici.nodejs.org/#/docs/api/ProxyAgent) and used for all external requests. HTTP(S) only. | | `CORS_WILDCARD` | `1` | `0` | toggles cross-origin resource sharing.
`0`: disabled. `1`: enabled. | -| `CORS_URL` | not used | `https://cobalt.tools/` | cross-origin resource sharing url. api will be available only from this url if `CORS_WILDCARD` is set to `0`. | +| `CORS_URL` | not used | `https://cobalt.tools` | cross-origin resource sharing url. api will be available only from this url if `CORS_WILDCARD` is set to `0`. | | `COOKIE_PATH` | not used | `/cookies.json` | path for cookie file relative to main folder. | | `PROCESSING_PRIORITY` | not used | `10` | changes `nice` value* for ffmpeg subprocess. available only on unix systems. | | `FREEBIND_CIDR` | ➖ | `2001:db8::/32` | IPv6 prefix used for randomly assigning addresses to cobalt requests. only supported on linux systems. see below for more info. | diff --git a/package-lock.json b/package-lock.json index a61508b0..87e11661 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cobalt", - "version": "7.14.4", + "version": "7.15", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cobalt", - "version": "7.14.4", + "version": "7.15", "license": "AGPL-3.0", "dependencies": { "content-disposition-header": "0.6.0", @@ -24,7 +24,7 @@ "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", - "youtubei.js": "^9.3.0" + "youtubei.js": "^10.2.0" }, "engines": { "node": ">=18" @@ -73,9 +73,9 @@ } }, "node_modules/acorn": { - "version": "8.11.3", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", - "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", + "version": "8.12.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", + "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==", "bin": { "acorn": "bin/acorn" }, @@ -683,9 +683,9 @@ } }, "node_modules/jintr": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/jintr/-/jintr-1.1.0.tgz", - "integrity": "sha512-Tu9wk3BpN2v+kb8yT6YBtue+/nbjeLFv4vvVC4PJ7oCidHKbifWhvORrAbQfxVIQZG+67am/mDagpiGSVtvrZg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jintr/-/jintr-2.0.0.tgz", + "integrity": "sha512-RiVlevxttZ4eHEYB2dXKXDXluzHfRuw0DJQGsYuKCc5IvZj5/GbOakeqVX+Bar/G9kTty9xDJREcxukurkmYLA==", "funding": [ "https://github.com/sponsors/LuanRT" ], @@ -1123,14 +1123,15 @@ } }, "node_modules/youtubei.js": { - "version": "9.4.0", - "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-9.4.0.tgz", - "integrity": "sha512-8plCOZD2WabqWSEgZU3RjzigIIeR7sF028EERJENYrC9xO/6awpLMZfeoE1gNrNEbKcA+bzbMvonqlvBdxGdKg==", + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-10.2.0.tgz", + "integrity": "sha512-JLKW9AHQ1qrTwBbre1aDkH8UJFmNcc4+kOSaVou5jSY7AzfFPFJK0yvX6afnLst0UVC9wfXHrLiNx93sutVErA==", "funding": [ "https://github.com/sponsors/LuanRT" ], + "license": "MIT", "dependencies": { - "jintr": "^1.1.0", + "jintr": "^2.0.0", "tslib": "^2.5.0", "undici": "^5.19.1" } diff --git a/package.json b/package.json index 1d44f380..58fdec83 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "7.14.4", + "version": "7.15", "author": "imput", "exports": "./src/cobalt.js", "type": "module", @@ -40,7 +40,7 @@ "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", - "youtubei.js": "^9.3.0" + "youtubei.js": "^10.2.0" }, "optionalDependencies": { "freebind": "^0.2.2" diff --git a/src/core/api.js b/src/core/api.js index a366ad09..5f4ee804 100644 --- a/src/core/api.js +++ b/src/core/api.js @@ -1,5 +1,6 @@ import cors from "cors"; import rateLimit from "express-rate-limit"; +import { setGlobalDispatcher, ProxyAgent } from "undici"; import { env, version } from "../modules/config.js"; @@ -26,7 +27,7 @@ const corsConfig = env.corsWildcard ? {} : { export function runAPI(express, app, gitCommit, gitBranch, __dirname) { const startTime = new Date(); const startTimestamp = startTime.getTime(); - + const serverInfo = { version: version, commit: gitCommit, @@ -81,38 +82,23 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { app.use((req, res, next) => { try { decodeURIComponent(req.path) - } catch { + } catch { return res.redirect('/') } next(); }) - app.use('/api/json', express.json({ - verify: (req, res, buf) => { - if (String(req.header('Accept')) === "application/json") { - if (buf.length > 720) throw new Error(); - JSON.parse(buf); - } else { - throw new Error(); - } - } - })) - - // handle express.json errors properly (https://github.com/expressjs/express/issues/4065) - app.use('/api/json', (err, req, res, next) => { - let errorText = "invalid json body"; - const acceptHeader = String(req.header('Accept')) !== "application/json"; - - if (err || acceptHeader) { - if (acceptHeader) errorText = "invalid accept header"; + app.use('/api/json', express.json({ limit: 1024 })); + app.use('/api/json', (err, _, res, next) => { + if (err) { return res.status(400).json({ status: "error", - text: errorText + text: "invalid json body" }); - } else { - next(); } - }) + + next(); + }); app.post('/api/json', async (req, res) => { const request = req.body; @@ -123,6 +109,10 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { res.status(status).json(body); } + if (!acceptRegex.test(req.header('Accept'))) { + return fail('ErrorInvalidAcceptHeader'); + } + if (!acceptRegex.test(req.header('Content-Type'))) { return fail('ErrorInvalidContentType'); } @@ -196,10 +186,10 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { return res.sendStatus(404); } - streamInfo.headers = { - ...streamInfo.headers, - ...req.headers - }; + streamInfo.headers = new Map([ + ...(streamInfo.headers || []), + ...Object.entries(req.headers) + ]); return stream(res, { type: 'internal', ...streamInfo }); }) @@ -219,6 +209,14 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { randomizeCiphers(); setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes + if (env.externalProxy) { + if (env.freebindCIDR) { + throw new Error('Freebind is not available when external proxy is enabled') + } + + setGlobalDispatcher(new ProxyAgent(env.externalProxy)) + } + app.listen(env.apiPort, env.listenAddress, () => { console.log(`\n` + `${Cyan("cobalt")} API ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\n` + diff --git a/src/core/web.js b/src/core/web.js index 9a281c47..d892c56c 100644 --- a/src/core/web.js +++ b/src/core/web.js @@ -26,45 +26,46 @@ export async function runWeb(express, app, gitCommit, gitBranch, __dirname) { app.get('/onDemand', (req, res) => { try { - if (req.query.blockId) { - let blockId = req.query.blockId.slice(0, 3); - let blockData; - switch(blockId) { - // changelog history - case "0": - let history = changelogHistory(); - if (history) { - blockData = createResponse("success", { t: history }) - } else { - blockData = createResponse("error", { - t: "couldn't render this block, please try again!" - }) - } - break; - // celebrations emoji - case "1": - let celebration = celebrationsEmoji(); - if (celebration) { - blockData = createResponse("success", { t: celebration }) - } - break; - default: - blockData = createResponse("error", { - t: "couldn't find a block with this id" - }) - break; - } - if (blockData?.body) { - return res.status(blockData.status).json(blockData.body); - } else { - return res.status(204).end(); - } - } else { + if (typeof req.query.blockId !== 'string') { return res.status(400).json({ status: "error", text: "couldn't render this block, please try again!" }); } + + let blockId = req.query.blockId.slice(0, 3); + let blockData; + switch(blockId) { + // changelog history + case "0": + let history = changelogHistory(); + if (history) { + blockData = createResponse("success", { t: history }) + } else { + blockData = createResponse("error", { + t: "couldn't render this block, please try again!" + }) + } + break; + // celebrations emoji + case "1": + let celebration = celebrationsEmoji(); + if (celebration) { + blockData = createResponse("success", { t: celebration }) + } + break; + default: + blockData = createResponse("error", { + t: "couldn't find a block with this id" + }) + break; + } + + if (blockData?.body) { + return res.status(blockData.status).json(blockData.body); + } else { + return res.status(204).end(); + } } catch { return res.status(400).json({ status: "error", diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json index eecd9ac1..2b10f41d 100644 --- a/src/localization/languages/en.json +++ b/src/localization/languages/en.json @@ -159,6 +159,7 @@ "UpdateOneMillion": "1 million users and blazing speed", "ErrorYTAgeRestrict": "this youtube video is age-restricted, so i can't see it. try another one!", "ErrorYTLogin": "couldn't get this youtube video because it requires an account to view.\n\nthis limitation is done by google to seemingly stop scraping, affecting all 3rd party tools and even their own clients.\n\ntry again, but if issue persists, {ContactLink}.", - "ErrorYTRateLimit": "i got rate limited by youtube. try again in a few seconds, but if issue persists, {ContactLink}." + "ErrorYTRateLimit": "i got rate limited by youtube. try again in a few seconds, but if issue persists, {ContactLink}.", + "ErrorInvalidAcceptHeader": "invalid accept header" } } diff --git a/src/modules/config.js b/src/modules/config.js index 530c5f0b..662d8b05 100644 --- a/src/modules/config.js +++ b/src/modules/config.js @@ -48,7 +48,9 @@ const processingPriority: process.platform !== 'win32' && process.env.PROCESSING_PRIORITY - && parseInt(process.env.PROCESSING_PRIORITY) + && parseInt(process.env.PROCESSING_PRIORITY), + + externalProxy: process.env.API_EXTERNAL_PROXY, } export const diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index c3ceb587..c1160e7d 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -24,8 +24,10 @@ import streamable from "./services/streamable.js"; import twitch from "./services/twitch.js"; import rutube from "./services/rutube.js"; import dailymotion from "./services/dailymotion.js"; +import snapchat from "./services/snapchat.js"; import loom from "./services/loom.js"; import threads from "./services/threads.js"; +import facebook from "./services/facebook.js"; let freebind; @@ -189,6 +191,12 @@ export default async function(host, patternMatch, lang, obj) { case "dailymotion": r = await dailymotion(patternMatch); break; + case "snapchat": + r = await snapchat({ + hostname: url.hostname, + ...patternMatch + }); + break; case "loom": r = await loom({ id: patternMatch.id @@ -199,7 +207,12 @@ export default async function(host, patternMatch, lang, obj) { ...patternMatch, quality: obj.vQuality, dispatcher - }) + }); + break; + case "facebook": + r = await facebook({ + ...patternMatch + }); break; default: return createResponse("error", { diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index c19136fd..ecbe3834 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -24,7 +24,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di else if (r.isGif && toGif) action = "gif"; else if (isAudioMuted) action = "muteVideo"; else if (isAudioOnly) action = "audio"; - else if (r.isM3U8) action = "singleM3U8"; + else if (r.isM3U8) action = "m3u8"; else action = "video"; if (action === "picker" || action === "audio") { @@ -48,13 +48,19 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di params = { type: "gif" } break; - case "singleM3U8": - params = { type: "remux" } + case "m3u8": + params = { + type: Array.isArray(r.urls) ? "render" : "remux" + } break; case "muteVideo": + let muteType = "mute"; + if (Array.isArray(r.urls) && !r.isM3U8) { + muteType = "bridge"; + } params = { - type: Array.isArray(r.urls) ? "bridge" : "mute", + type: muteType, u: Array.isArray(r.urls) ? r.urls[0] : r.urls, mute: true } @@ -68,6 +74,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di case "instagram": case "twitter": case "threads": + case "snapchat": params = { picker: r.picker }; break; case "tiktok": @@ -125,11 +132,13 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di params = { type: "bridge" }; break; + case "facebook": case "vine": case "instagram": case "tumblr": case "pinterest": case "streamable": + case "snapchat": case "loom": case "threads": responseType = "redirect"; diff --git a/src/modules/processing/services/facebook.js b/src/modules/processing/services/facebook.js new file mode 100644 index 00000000..17dedab4 --- /dev/null +++ b/src/modules/processing/services/facebook.js @@ -0,0 +1,56 @@ +import { genericUserAgent } from "../../config.js"; + +const headers = { + 'User-Agent': genericUserAgent, + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', +} + +const resolveUrl = (url) => { + return fetch(url, { headers }) + .then(r => { + if (r.headers.get('location')) { + return decodeURIComponent(r.headers.get('location')); + } + if (r.headers.get('link')) { + const linkMatch = r.headers.get('link').match(/<(.*?)\/>/); + return decodeURIComponent(linkMatch[1]); + } + return false; + }) + .catch(() => false); +} + +export default async function({ id, shareType, shortLink }) { + let url = `https://web.facebook.com/i/videos/${id}`; + + if (shareType) url = `https://web.facebook.com/share/${shareType}/${id}`; + if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`); + + const html = await fetch(url, { headers }) + .then(r => r.text()) + .catch(() => false); + + if (!html) return { error: 'ErrorCouldntFetch' }; + + const urls = []; + const hd = html.match('"browser_native_hd_url":(".*?")'); + const sd = html.match('"browser_native_sd_url":(".*?")'); + + if (hd?.[1]) urls.push(JSON.parse(hd[1])); + if (sd?.[1]) urls.push(JSON.parse(sd[1])); + + if (!urls.length) { + return { error: 'ErrorEmptyDownload' }; + } + + const baseFilename = `facebook_${id || shortLink}`; + + return { + urls: urls[0], + filename: `${baseFilename}.mp4`, + audioFilename: `${baseFilename}_audio`, + }; +} diff --git a/src/modules/processing/services/instagram.js b/src/modules/processing/services/instagram.js index 2fc533d8..6ea05178 100644 --- a/src/modules/processing/services/instagram.js +++ b/src/modules/processing/services/instagram.js @@ -256,11 +256,11 @@ export default function(obj) { if (!media_id && cookie) media_id = await getMediaId(id, { cookie }); // mobile api (bearer) - if (media_id && token) data = await requestMobileApi(id, { token }); + if (media_id && token) data = await requestMobileApi(media_id, { token }); // mobile api (no cookie, cookie) - if (!data && media_id) data = await requestMobileApi(id); - if (!data && media_id && cookie) data = await requestMobileApi(id, { cookie }); + if (media_id && !data) data = await requestMobileApi(media_id); + if (media_id && cookie && !data) data = await requestMobileApi(media_id, { cookie }); // html embed (no cookie, cookie) if (!data) data = await requestHTML(id); diff --git a/src/modules/processing/services/ok.js b/src/modules/processing/services/ok.js index 295d5b81..33847cd8 100644 --- a/src/modules/processing/services/ok.js +++ b/src/modules/processing/services/ok.js @@ -20,14 +20,15 @@ export default async function(o) { }).then(r => r.text()).catch(() => {}); if (!html) return { error: 'ErrorCouldntFetch' }; - if (!html.includes(`
/) + ?.[1] + ?.replaceAll(""", '"'); + + if (!videoData) { return { error: 'ErrorEmptyDownload' }; } - let videoData = html.split(`
Math.abs(a - b); + export default async function(obj) { if (obj.yappyId) { const yappy = await requestJSON( @@ -25,7 +27,7 @@ export default async function(obj) { } } - const quality = obj.quality === "max" ? "9000" : obj.quality; + const quality = Number(obj.quality) || 9000; const requestURL = new URL(`https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2`); if (obj.key) requestURL.searchParams.set('p', obj.key); @@ -45,12 +47,16 @@ export default async function(obj) { if (!m3u8) return { error: 'ErrorCouldntFetch' }; - m3u8 = HLS.parse(m3u8).variants.sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth)); + m3u8 = HLS.parse(m3u8).variants; - let bestQuality = m3u8[0]; - if (Number(quality) < bestQuality.resolution.height) { - bestQuality = m3u8.find((i) => (Number(quality) === i.resolution.height)); - } + const matchingQuality = m3u8.reduce((prev, next) => { + const diff = { + prev: delta(quality, prev.resolution.height), + next: delta(quality, next.resolution.height) + }; + + return diff.prev < diff.next ? prev : next; + }); const fileMetadata = { title: cleanString(play.title.trim()), @@ -58,15 +64,15 @@ export default async function(obj) { } return { - urls: bestQuality.uri, + urls: matchingQuality.uri, isM3U8: true, filenameAttributes: { service: "rutube", id: obj.id, title: fileMetadata.title, author: fileMetadata.artist, - resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`, - qualityLabel: `${bestQuality.resolution.height}p`, + resolution: `${matchingQuality.resolution.width}x${matchingQuality.resolution.height}`, + qualityLabel: `${matchingQuality.resolution.height}p`, extension: "mp4" }, fileMetadata: fileMetadata diff --git a/src/modules/processing/services/snapchat.js b/src/modules/processing/services/snapchat.js new file mode 100644 index 00000000..a93f0933 --- /dev/null +++ b/src/modules/processing/services/snapchat.js @@ -0,0 +1,96 @@ +import { genericUserAgent } from "../../config.js"; +import { getRedirectingURL } from "../../sub/utils.js"; +import { extract, normalizeURL } from "../url.js"; + +const SPOTLIGHT_VIDEO_REGEX = //; +const NEXT_DATA_REGEX = /