From cc1e9dcff88990dde35ff81c256f5d8608fd301d Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Mon, 13 May 2024 18:20:10 +0000 Subject: [PATCH 01/10] api: add API_LISTEN_ADDRESS env for specifying bind address --- docs/run-an-instance.md | 1 + src/core/api.js | 2 +- src/modules/config.js | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md index 9e607942..d89d506c 100644 --- a/docs/run-an-instance.md +++ b/docs/run-an-instance.md @@ -53,6 +53,7 @@ sudo service nscd start | variable name | default | example | description | |:----------------------|:----------|:------------------------|:------------| | `API_PORT` | `9000` | `9000` | changes port from which api server is accessible. | +| `API_LISTEN_ADDRESS` | `0.0.0.0` | `127.0.0.1` | changes address from which api server is accessible. **if you are using docker, you usually don't need to configure this.** | | `API_URL` | ➖ | `https://co.wuk.sh/` | changes url from which api server is accessible.
***REQUIRED TO RUN API***. | | `API_NAME` | `unknown` | `ams-1` | api server name that is shown in `/api/serverInfo`. | | `CORS_WILDCARD` | `1` | `0` | toggles cross-origin resource sharing.
`0`: disabled. `1`: enabled. | diff --git a/src/core/api.js b/src/core/api.js index 440c25c2..31ed7dd5 100644 --- a/src/core/api.js +++ b/src/core/api.js @@ -194,7 +194,7 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) { res.redirect('/api/json') }); - app.listen(env.apiPort, () => { + app.listen(env.apiPort, env.listenAddress, () => { console.log(`\n` + `${Cyan("cobalt")} API ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\n` + `Start time: ${Bright(`${startTime.toUTCString()} (${startTimestamp})`)}\n\n` + diff --git a/src/modules/config.js b/src/modules/config.js index 93347ad4..89ac1b33 100644 --- a/src/modules/config.js +++ b/src/modules/config.js @@ -29,6 +29,7 @@ const apiEnvs = { apiPort: process.env.API_PORT || 9000, apiName: process.env.API_NAME || 'unknown', + listenAddress: process.env.API_LISTEN_ADDRESS, corsWildcard: process.env.CORS_WILDCARD !== '0', corsURL: process.env.CORS_URL, cookiePath: process.env.COOKIE_PATH, From 0114e686b8a8ef3de984d1e003c66aa375e2dc48 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sun, 12 May 2024 14:06:04 +0000 Subject: [PATCH 02/10] api: add FREEBIND_CIDR env variable --- docs/run-an-instance.md | 11 +++++++++-- src/modules/config.js | 2 ++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md index d89d506c..75b6ef7e 100644 --- a/docs/run-an-instance.md +++ b/docs/run-an-instance.md @@ -60,7 +60,8 @@ sudo service nscd start | `CORS_URL` | not used | `https://cobalt.tools/` | cross-origin resource sharing url. api will be available only from this url if `CORS_WILDCARD` is set to `0`. | | `COOKIE_PATH` | not used | `/cookies.json` | path for cookie file relative to main folder. | | `PROCESSING_PRIORITY` | not used | `10` | changes `nice` value* for ffmpeg subprocess. available only on unix systems. | -| `TIKTOK_DEVICE_INFO` | ➖ | *see below* | device info (including `iid` and `device_id`) for tiktok functionality. required for tiktok to work. | +| `TIKTOK_DEVICE_INFO` | ➖ | *see below* | device info (including `iid` and `device_id`) for tiktok functionality. required for tiktok to work. | +| `FREEBIND_CIDR` | ➖ | `2001:db8::/32` | IPv6 prefix used for randomly assigning addresses to cobalt requests. Only supported on Linux systems. For more info, see below. | \* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)). @@ -86,6 +87,12 @@ you can compress the json to save space. if you're using a `.env` file then the TIKTOK_DEVICE_INFO='{"iid":"","device_id":"","channel":"googleplay","app_name":"musical_ly","version_code":"310503","device_platform":"android","device_type":"Redmi+7","os_version":"13"}' ``` +#### FREEBIND_CIDR +setting a `FREEBIND_CIDR` allows cobalt to pick a random IP for every download and use it for all +requests it makes for that particular download. to use freebind in cobalt, you need to follow its [setup instructions](https://github.com/imputnet/freebind.js?tab=readme-ov-file#setup) first. if you configure this option while running cobalt +in a docker container, you also need to set the `API_LISTEN_ADDRESS` env to `127.0.0.1`, and set +`network_mode` for the container to `host`. + ### variables for web | variable name | default | example | description | |:---------------------|:---------------------|:------------------------|:--------------------------------------------------------------------------------------| @@ -96,4 +103,4 @@ TIKTOK_DEVICE_INFO='{"iid":"","device_id":"","c | `IS_BETA` | `0` | `1` | toggles beta tag next to cobalt logo.
`0`: disabled. `1`: enabled. | | `PLAUSIBLE_HOSTNAME` | ➖ | `plausible.io`* | enables plausible analytics with provided hostname as receiver backend. | -\* don't use plausible.io as receiver backend unless you paid for their cloud service. use your own domain when hosting community edition of plausible. refer to their [docs](https://plausible.io/docs) when needed. +\* don't use plausible.io as receiver backend unless you paid for their cloud service. use your own domain when hosting community edition of plausible. refer to their [docs](https://plausible.io/docs) when needed. \ No newline at end of file diff --git a/src/modules/config.js b/src/modules/config.js index 89ac1b33..1722b795 100644 --- a/src/modules/config.js +++ b/src/modules/config.js @@ -1,5 +1,6 @@ import UrlPattern from "url-pattern"; import { loadJSON } from "./sub/loadFromFs.js"; + const config = loadJSON("./src/config.json"); const packageJson = loadJSON("./package.json"); const servicesConfigJson = loadJSON("./src/modules/processing/servicesConfig.json"); @@ -37,6 +38,7 @@ const && process.env.PROCESSING_PRIORITY && parseInt(process.env.PROCESSING_PRIORITY), tiktokDeviceInfo: process.env.TIKTOK_DEVICE_INFO && JSON.parse(process.env.TIKTOK_DEVICE_INFO), + freebindCIDR: process.platform === 'linux' && process.env.FREEBIND_CIDR, apiURL } From c306a944d953e6f8ab47f7bfe3f81f7627880fe8 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sun, 12 May 2024 16:12:45 +0000 Subject: [PATCH 03/10] match: add freebind support for youtube and instagram --- package.json | 3 +++ src/modules/processing/match.js | 18 ++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index ffb78672..1b97a04d 100644 --- a/package.json +++ b/package.json @@ -41,5 +41,8 @@ "undici": "^6.7.0", "url-pattern": "1.0.3", "youtubei.js": "^9.3.0" + }, + "optionalDependencies": { + "freebind": "^0.2.2" } } diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index a6432e49..d89dc9d9 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -25,9 +25,21 @@ import streamable from "./services/streamable.js"; import twitch from "./services/twitch.js"; import rutube from "./services/rutube.js"; import dailymotion from "./services/dailymotion.js"; +import { env } from '../config.js'; +let freebind; export default async function(host, patternMatch, url, lang, obj) { assert(url instanceof URL); + let dispatcher, requestIP; + + if (env.freebindCIDR) { + if (!freebind) { + freebind = await import('freebind'); + } + + requestIP = freebind.ip.random(env.freebindCIDR); + dispatcher = freebind.dispatcherFromIP(requestIP, { strict: false }); + } try { let r, isAudioOnly = !!obj.isAudioOnly, disableMetadata = !!obj.disableMetadata; @@ -66,7 +78,8 @@ export default async function(host, patternMatch, url, lang, obj) { format: obj.vCodec, isAudioOnly: isAudioOnly, isAudioMuted: obj.isAudioMuted, - dubLang: obj.dubLang + dubLang: obj.dubLang, + dispatcher } if (url.hostname === 'music.youtube.com' || isAudioOnly === true) { @@ -122,7 +135,8 @@ export default async function(host, patternMatch, url, lang, obj) { case "instagram": r = await instagram({ ...patternMatch, - quality: obj.vQuality + quality: obj.vQuality, + dispatcher }) break; case "vine": From d8913472ad69d32e4148ef3d9c99fe57dca3f880 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sun, 12 May 2024 16:04:05 +0000 Subject: [PATCH 04/10] package.json: revert undici version to 5.x so that it matches with youtubei.js's version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1b97a04d..2daa403c 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "node-cache": "^5.1.2", "psl": "1.9.0", "set-cookie-parser": "2.6.0", - "undici": "^6.7.0", + "undici": "^5.19.1", "url-pattern": "1.0.3", "youtubei.js": "^9.3.0" }, From d5aa27f5f904f3c993691ca901546b29bc4b06e5 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sun, 12 May 2024 16:13:01 +0000 Subject: [PATCH 05/10] youtube: use the freebind dispatcher if available --- src/modules/processing/services/youtube.js | 25 ++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 10b04a5c..d11b2739 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -1,8 +1,9 @@ -import { Innertube } from 'youtubei.js'; +import { Innertube, Session } from 'youtubei.js'; import { maxVideoDuration } from '../../config.js'; import { cleanString } from '../../sub/utils.js'; +import { fetch } from 'undici' -const yt = await Innertube.create(); +const ytBase = await Innertube.create(); const codecMatch = { h264: { @@ -22,7 +23,27 @@ const codecMatch = { } } +const cloneInnertube = (customFetch) => { + const session = new Session( + ytBase.session.context, + ytBase.session.key, + ytBase.session.api_version, + ytBase.session.account_index, + ytBase.session.player, + undefined, + customFetch ?? ytBase.session.http.fetch, + ytBase.session.cache + ); + + const yt = new Innertube(session); + return yt; +} + export default async function(o) { + const yt = cloneInnertube( + (input, init) => fetch(input, { ...init, dispatcher: o.dispatcher }) + ); + let info, isDubbed, format = o.format || "h264"; let quality = o.quality === "max" ? "9000" : o.quality; // 9000(p) - max quality From 9419266cd7c5b21088aaf5e00f2c7f8a570fa567 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sun, 12 May 2024 16:35:52 +0000 Subject: [PATCH 06/10] stream: use freebind dispatcher in internal streams --- src/modules/processing/match.js | 3 ++- src/modules/processing/matchActionDecider.js | 5 +++-- src/modules/stream/internal.js | 3 +++ src/modules/stream/manage.js | 14 ++++++++++++-- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index d89dc9d9..6bbd0d2a 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -195,7 +195,8 @@ export default async function(host, patternMatch, url, lang, obj) { return matchActionDecider( r, host, obj.aFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, - obj.filenamePattern, obj.twitterGif + obj.filenamePattern, obj.twitterGif, + requestIP ) } catch (e) { return apiJSON(0, { t: genericError(lang, host) }) diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index 44a4a81b..d440cff6 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -3,7 +3,7 @@ import { apiJSON } from "../sub/utils.js"; import loc from "../../localization/manager.js"; import createFilename from "./createFilename.js"; -export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern, toGif) { +export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern, toGif, requestIP) { let action, responseType = 2, defaultParams = { @@ -11,7 +11,8 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di service: host, filename: r.filenameAttributes ? createFilename(r.filenameAttributes, filenamePattern, isAudioOnly, isAudioMuted) : r.filename, - fileMetadata: !disableMetadata ? r.fileMetadata : false + fileMetadata: !disableMetadata ? r.fileMetadata : false, + requestIP }, params = {}, audioFormat = String(userFormat); diff --git a/src/modules/stream/internal.js b/src/modules/stream/internal.js index 412ba546..9578da9b 100644 --- a/src/modules/stream/internal.js +++ b/src/modules/stream/internal.js @@ -18,6 +18,7 @@ async function* readChunks(streamInfo, size) { ...getHeaders('youtube'), Range: `bytes=${read}-${read + CHUNK_SIZE}` }, + dispatcher: streamInfo.dispatcher, signal: streamInfo.controller.signal }); @@ -47,6 +48,7 @@ async function handleYoutubeStream(streamInfo, res) { const req = await fetch(streamInfo.url, { headers: getHeaders('youtube'), method: 'HEAD', + dispatcher: streamInfo.dispatcher, signal: streamInfo.controller.signal }); @@ -81,6 +83,7 @@ export async function internalStream(streamInfo, res) { ...streamInfo.headers, host: undefined }, + dispatcher: streamInfo.dispatcher, signal: streamInfo.controller.signal, maxRedirections: 16 }); diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js index e8357103..7d19354f 100644 --- a/src/modules/stream/manage.js +++ b/src/modules/stream/manage.js @@ -6,6 +6,9 @@ import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js"; import { streamLifespan, env } from "../config.js"; import { strict as assert } from "assert"; +// optional dependency +const freebind = env.freebindCIDR && await import('freebind').catch(() => {}); + const M3U_SERVICES = ['dailymotion', 'vimeo', 'rutube']; const streamNoAccess = { @@ -46,7 +49,8 @@ export function createStream(obj) { isAudioOnly: !!obj.isAudioOnly, copy: !!obj.copy, mute: !!obj.mute, - metadata: obj.fileMetadata || false + metadata: obj.fileMetadata || false, + requestIP: obj.requestIP }; streamCache.set( @@ -78,11 +82,17 @@ export function getInternalStream(id) { export function createInternalStream(url, obj = {}) { assert(typeof url === 'string'); + let dispatcher; + if (obj.requestIP) { + dispatcher = freebind?.dispatcherFromIP(obj.requestIP, { strict: false }) + } + const streamID = nanoid(); internalStreamCache[streamID] = { url, service: obj.service, - controller: new AbortController() + controller: new AbortController(), + dispatcher }; let streamLink = new URL('/api/istream', `http://127.0.0.1:${env.apiPort}`); From 3fe60046008364d6425a95a2db02cf6cdad5a54b Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Mon, 13 May 2024 18:55:42 +0000 Subject: [PATCH 07/10] dockerfile: fix freebind building, set up everything in one step this also shrinks the image by around 40MB, since the apt/lists layer no longer sticks around --- Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4eee25b9..c98785d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,13 @@ FROM node:18-bullseye-slim WORKDIR /app -RUN apt-get update -RUN apt-get install -y git -RUN rm -rf /var/lib/apt/lists/* - COPY package*.json ./ -RUN npm install -RUN git clone -n https://github.com/imputnet/cobalt.git --depth 1 && mv cobalt/.git ./ && rm -rf cobalt +RUN apt-get update && \ + apt-get install -y git python3 build-essential && \ + npm install && \ + apt purge --autoremove -y python3 build-essential && \ + rm -rf ~/.cache/ /var/lib/apt/lists/* COPY . . EXPOSE 9000 From ce6bafadf9b43e04f02528cc309b58b2564a8853 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 14 May 2024 15:09:44 +0600 Subject: [PATCH 08/10] docs: fix capitalisation in run-an-instance.md Signed-off-by: wukko --- docs/run-an-instance.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md index 75b6ef7e..25145224 100644 --- a/docs/run-an-instance.md +++ b/docs/run-an-instance.md @@ -61,7 +61,7 @@ sudo service nscd start | `COOKIE_PATH` | not used | `/cookies.json` | path for cookie file relative to main folder. | | `PROCESSING_PRIORITY` | not used | `10` | changes `nice` value* for ffmpeg subprocess. available only on unix systems. | | `TIKTOK_DEVICE_INFO` | ➖ | *see below* | device info (including `iid` and `device_id`) for tiktok functionality. required for tiktok to work. | -| `FREEBIND_CIDR` | ➖ | `2001:db8::/32` | IPv6 prefix used for randomly assigning addresses to cobalt requests. Only supported on Linux systems. For more info, see below. | +| `FREEBIND_CIDR` | ➖ | `2001:db8::/32` | IPv6 prefix used for randomly assigning addresses to cobalt requests. only supported on linux systems. for more info, see below. | \* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)). @@ -103,4 +103,4 @@ in a docker container, you also need to set the `API_LISTEN_ADDRESS` env to `127 | `IS_BETA` | `0` | `1` | toggles beta tag next to cobalt logo.
`0`: disabled. `1`: enabled. | | `PLAUSIBLE_HOSTNAME` | ➖ | `plausible.io`* | enables plausible analytics with provided hostname as receiver backend. | -\* don't use plausible.io as receiver backend unless you paid for their cloud service. use your own domain when hosting community edition of plausible. refer to their [docs](https://plausible.io/docs) when needed. \ No newline at end of file +\* don't use plausible.io as receiver backend unless you paid for their cloud service. use your own domain when hosting community edition of plausible. refer to their [docs](https://plausible.io/docs) when needed. From 4b0814a2ecd8ca9b49cf3f110918a74d41c9dd6e Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 14 May 2024 15:37:41 +0600 Subject: [PATCH 09/10] config: clean up --- src/modules/config.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/config.js b/src/modules/config.js index 1722b795..c66521e1 100644 --- a/src/modules/config.js +++ b/src/modules/config.js @@ -34,7 +34,7 @@ const corsWildcard: process.env.CORS_WILDCARD !== '0', corsURL: process.env.CORS_URL, cookiePath: process.env.COOKIE_PATH, - processingPriority: process.platform !== "win32" + processingPriority: process.platform !== 'win32' && process.env.PROCESSING_PRIORITY && parseInt(process.env.PROCESSING_PRIORITY), tiktokDeviceInfo: process.env.TIKTOK_DEVICE_INFO && JSON.parse(process.env.TIKTOK_DEVICE_INFO), @@ -49,7 +49,7 @@ export const streamLifespan = config.streamLifespan, maxVideoDuration = config.maxVideoDuration, genericUserAgent = config.genericUserAgent, - repo = packageJson["bugs"]["url"].replace('/issues', ''), + repo = packageJson.bugs.url.replace('/issues', ''), authorInfo = config.authorInfo, donations = config.donations, ffmpegArgs = config.ffmpegArgs, From e44927e5adf7f9d8477362f5a7fba678661cb442 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 14 May 2024 22:08:32 +0600 Subject: [PATCH 10/10] instagram: add freebind dispatcher support --- src/modules/processing/services/instagram.js | 591 ++++++++++--------- 1 file changed, 299 insertions(+), 292 deletions(-) diff --git a/src/modules/processing/services/instagram.js b/src/modules/processing/services/instagram.js index 68420a5d..bcbf78bb 100644 --- a/src/modules/processing/services/instagram.js +++ b/src/modules/processing/services/instagram.js @@ -41,299 +41,306 @@ const cachedDtsg = { expiry: 0 } -async function findDtsgId(cookie) { - try { - if (cachedDtsg.expiry > Date.now()) return cachedDtsg.value; - - const data = await fetch('https://www.instagram.com/', { - headers: { - ...commonHeaders, - cookie - } - }).then(r => r.text()); - - const token = data.match(/"dtsg":{"token":"(.*?)"/)[1]; - - cachedDtsg.value = token; - cachedDtsg.expiry = Date.now() + 86390000; - - if (token) return token; - return false; - } - catch {} -} - -async function request(url, cookie, method = 'GET', requestData) { - let headers = { - ...commonHeaders, - 'x-ig-www-claim': cookie?._wwwClaim || '0', - 'x-csrftoken': cookie?.values()?.csrftoken, - cookie - } - if (method === 'POST') { - headers['content-type'] = 'application/x-www-form-urlencoded'; - } - - const data = await fetch(url, { - method, - headers, - body: requestData && new URLSearchParams(requestData), - }); - - if (data.headers.get('X-Ig-Set-Www-Claim') && cookie) - cookie._wwwClaim = data.headers.get('X-Ig-Set-Www-Claim'); - - updateCookie(cookie, data.headers); - return data.json(); -} -async function getMediaId(id, { cookie, token } = {}) { - const oembedURL = new URL('https://i.instagram.com/api/v1/oembed/'); - oembedURL.searchParams.set('url', `https://www.instagram.com/p/${id}/`); - - const oembed = await fetch(oembedURL, { - headers: { - ...mobileHeaders, - ...( token && { authorization: `Bearer ${token}` } ), - cookie - } - }).then(r => r.json()).catch(() => {}); - - return oembed?.media_id; -} - -async function requestMobileApi(mediaId, { cookie, token } = {}) { - const mediaInfo = await fetch(`https://i.instagram.com/api/v1/media/${mediaId}/info/`, { - headers: { - ...mobileHeaders, - ...( token && { authorization: `Bearer ${token}` } ), - cookie - } - }).then(r => r.json()).catch(() => {}); - - return mediaInfo?.items?.[0]; -} -async function requestHTML(id, cookie) { - const data = await fetch(`https://www.instagram.com/p/${id}/embed/captioned/`, { - headers: { - ...embedHeaders, - cookie - } - }).then(r => r.text()).catch(() => {}); - - let embedData = JSON.parse(data?.match(/"init",\[\],\[(.*?)\]\],/)[1]); - - if (!embedData || !embedData?.contextJSON) return false; - - embedData = JSON.parse(embedData.contextJSON); - - return embedData; -} -async function requestGQL(id, cookie) { - let dtsgId; - - if (cookie) { - dtsgId = await findDtsgId(cookie); - } - const url = new URL('https://www.instagram.com/api/graphql/'); - - const requestData = { - jazoest: '26406', - variables: JSON.stringify({ - shortcode: id, - __relay_internal__pv__PolarisShareMenurelayprovider: false - }), - doc_id: '7153618348081770' - }; - if (dtsgId) { - requestData.fb_dtsg = dtsgId; - } - - return (await request(url, cookie, 'POST', requestData)) - .data - ?.xdt_api__v1__media__shortcode__web_info - ?.items - ?.[0]; -} - -function extractOldPost(data, id) { - const sidecar = data?.gql_data?.shortcode_media?.edge_sidecar_to_children; - if (sidecar) { - const picker = sidecar.edges.filter(e => e.node?.display_url) - .map(e => { - const type = e.node?.is_video ? "video" : "photo"; - const url = type === "video" ? e.node?.video_url : e.node?.display_url; - - return { - type, url, - /* thumbnails have `Cross-Origin-Resource-Policy` - ** set to `same-origin`, so we need to proxy them */ - thumb: createStream({ - service: "instagram", - type: "default", - u: e.node?.display_url, - filename: "image.jpg" - }) - } - }); - - if (picker.length) return { picker } - } else if (data?.gql_data?.shortcode_media?.video_url) { - return { - urls: data.gql_data.shortcode_media.video_url, - filename: `instagram_${id}.mp4`, - audioFilename: `instagram_${id}_audio` - } - } else if (data?.gql_data?.shortcode_media?.display_url) { - return { - urls: data.gql_data?.shortcode_media.display_url, - isPhoto: true - } - } -} - -function extractNewPost(data, id) { - const carousel = data.carousel_media; - if (carousel) { - const picker = carousel.filter(e => e?.image_versions2) - .map(e => { - const type = e.video_versions ? "video" : "photo"; - const imageUrl = e.image_versions2.candidates[0].url; - - let url = imageUrl; - if (type === 'video') { - const video = e.video_versions.reduce((a, b) => a.width * a.height < b.width * b.height ? b : a); - url = video.url; - } - - return { - type, url, - /* thumbnails have `Cross-Origin-Resource-Policy` - ** set to `same-origin`, so we need to proxy them */ - thumb: createStream({ - service: "instagram", - type: "default", - u: imageUrl, - filename: "image.jpg" - }) - } - }); - - if (picker.length) return { picker } - } else if (data.video_versions) { - const video = data.video_versions.reduce((a, b) => a.width * a.height < b.width * b.height ? b : a) - return { - urls: video.url, - filename: `instagram_${id}.mp4`, - audioFilename: `instagram_${id}_audio` - } - } else if (data.image_versions2?.candidates) { - return { - urls: data.image_versions2.candidates[0].url, - isPhoto: true - } - } -} - -async function getPost(id) { - let data, result; - try { - const cookie = getCookie('instagram'); - - const bearer = getCookie('instagram_bearer'); - const token = bearer?.values()?.token; - - // get media_id for mobile api, three methods - let media_id = await getMediaId(id); - if (!media_id && token) media_id = await getMediaId(id, { token }); - if (!media_id && cookie) media_id = await getMediaId(id, { cookie }); - - // mobile api (bearer) - if (media_id && token) data = await requestMobileApi(id, { token }); - - // mobile api (no cookie, cookie) - if (!data && media_id) data = await requestMobileApi(id); - if (!data && media_id && cookie) data = await requestMobileApi(id, { cookie }); - - // html embed (no cookie, cookie) - if (!data) data = await requestHTML(id); - if (!data && cookie) data = await requestHTML(id, cookie); - - // web app graphql api (no cookie, cookie) - if (!data) data = await requestGQL(id); - if (!data && cookie) data = await requestGQL(id, cookie); - } catch {} - - if (!data) return { error: 'ErrorCouldntFetch' }; - - if (data?.gql_data) { - result = extractOldPost(data, id) - } else { - result = extractNewPost(data, id) - } - - if (result) return result; - return { error: 'ErrorEmptyDownload' } -} - -async function usernameToId(username, cookie) { - const url = new URL('https://www.instagram.com/api/v1/users/web_profile_info/'); - url.searchParams.set('username', username); - - try { - const data = await request(url, cookie); - return data?.data?.user?.id; - } catch {} -} - -async function getStory(username, id) { - const cookie = getCookie('instagram'); - if (!cookie) return { error: 'ErrorUnsupported' }; - - const userId = await usernameToId(username, cookie); - if (!userId) return { error: 'ErrorEmptyDownload' }; - - const dtsgId = await findDtsgId(cookie); - - const url = new URL('https://www.instagram.com/api/graphql/'); - const requestData = { - fb_dtsg: dtsgId, - jazoest: '26438', - variables: JSON.stringify({ - reel_ids_arr : [ userId ], - }), - server_timestamps: true, - doc_id: '25317500907894419' - }; - - let media; - try { - const data = (await request(url, cookie, 'POST', requestData)); - media = data?.data?.xdt_api__v1__feed__reels_media?.reels_media?.find(m => m.id === userId); - } catch {} - - const item = media.items.find(m => m.pk === id); - if (!item) return { error: 'ErrorEmptyDownload' }; - - if (item.video_versions) { - const video = item.video_versions.reduce((a, b) => a.width * a.height < b.width * b.height ? b : a) - return { - urls: video.url, - filename: `instagram_${id}.mp4`, - audioFilename: `instagram_${id}_audio` - } - } - - if (item.image_versions2?.candidates) { - return { - urls: item.image_versions2.candidates[0].url, - isPhoto: true - } - } - - return { error: 'ErrorCouldntFetch' }; -} - export default function(obj) { + const dispatcher = obj.dispatcher; + + async function findDtsgId(cookie) { + try { + if (cachedDtsg.expiry > Date.now()) return cachedDtsg.value; + + const data = await fetch('https://www.instagram.com/', { + headers: { + ...commonHeaders, + cookie + }, + dispatcher + }).then(r => r.text()); + + const token = data.match(/"dtsg":{"token":"(.*?)"/)[1]; + + cachedDtsg.value = token; + cachedDtsg.expiry = Date.now() + 86390000; + + if (token) return token; + return false; + } + catch {} + } + + async function request(url, cookie, method = 'GET', requestData) { + let headers = { + ...commonHeaders, + 'x-ig-www-claim': cookie?._wwwClaim || '0', + 'x-csrftoken': cookie?.values()?.csrftoken, + cookie + } + if (method === 'POST') { + headers['content-type'] = 'application/x-www-form-urlencoded'; + } + + const data = await fetch(url, { + method, + headers, + body: requestData && new URLSearchParams(requestData), + dispatcher + }); + + if (data.headers.get('X-Ig-Set-Www-Claim') && cookie) + cookie._wwwClaim = data.headers.get('X-Ig-Set-Www-Claim'); + + updateCookie(cookie, data.headers); + return data.json(); + } + async function getMediaId(id, { cookie, token } = {}) { + const oembedURL = new URL('https://i.instagram.com/api/v1/oembed/'); + oembedURL.searchParams.set('url', `https://www.instagram.com/p/${id}/`); + + const oembed = await fetch(oembedURL, { + headers: { + ...mobileHeaders, + ...( token && { authorization: `Bearer ${token}` } ), + cookie + }, + dispatcher + }).then(r => r.json()).catch(() => {}); + + return oembed?.media_id; + } + + async function requestMobileApi(mediaId, { cookie, token } = {}) { + const mediaInfo = await fetch(`https://i.instagram.com/api/v1/media/${mediaId}/info/`, { + headers: { + ...mobileHeaders, + ...( token && { authorization: `Bearer ${token}` } ), + cookie + }, + dispatcher + }).then(r => r.json()).catch(() => {}); + + return mediaInfo?.items?.[0]; + } + async function requestHTML(id, cookie) { + const data = await fetch(`https://www.instagram.com/p/${id}/embed/captioned/`, { + headers: { + ...embedHeaders, + cookie + }, + dispatcher + }).then(r => r.text()).catch(() => {}); + + let embedData = JSON.parse(data?.match(/"init",\[\],\[(.*?)\]\],/)[1]); + + if (!embedData || !embedData?.contextJSON) return false; + + embedData = JSON.parse(embedData.contextJSON); + + return embedData; + } + async function requestGQL(id, cookie) { + let dtsgId; + + if (cookie) { + dtsgId = await findDtsgId(cookie); + } + const url = new URL('https://www.instagram.com/api/graphql/'); + + const requestData = { + jazoest: '26406', + variables: JSON.stringify({ + shortcode: id, + __relay_internal__pv__PolarisShareMenurelayprovider: false + }), + doc_id: '7153618348081770' + }; + if (dtsgId) { + requestData.fb_dtsg = dtsgId; + } + + return (await request(url, cookie, 'POST', requestData)) + .data + ?.xdt_api__v1__media__shortcode__web_info + ?.items + ?.[0]; + } + + function extractOldPost(data, id) { + const sidecar = data?.gql_data?.shortcode_media?.edge_sidecar_to_children; + if (sidecar) { + const picker = sidecar.edges.filter(e => e.node?.display_url) + .map(e => { + const type = e.node?.is_video ? "video" : "photo"; + const url = type === "video" ? e.node?.video_url : e.node?.display_url; + + return { + type, url, + /* thumbnails have `Cross-Origin-Resource-Policy` + ** set to `same-origin`, so we need to proxy them */ + thumb: createStream({ + service: "instagram", + type: "default", + u: e.node?.display_url, + filename: "image.jpg" + }) + } + }); + + if (picker.length) return { picker } + } else if (data?.gql_data?.shortcode_media?.video_url) { + return { + urls: data.gql_data.shortcode_media.video_url, + filename: `instagram_${id}.mp4`, + audioFilename: `instagram_${id}_audio` + } + } else if (data?.gql_data?.shortcode_media?.display_url) { + return { + urls: data.gql_data?.shortcode_media.display_url, + isPhoto: true + } + } + } + + function extractNewPost(data, id) { + const carousel = data.carousel_media; + if (carousel) { + const picker = carousel.filter(e => e?.image_versions2) + .map(e => { + const type = e.video_versions ? "video" : "photo"; + const imageUrl = e.image_versions2.candidates[0].url; + + let url = imageUrl; + if (type === 'video') { + const video = e.video_versions.reduce((a, b) => a.width * a.height < b.width * b.height ? b : a); + url = video.url; + } + + return { + type, url, + /* thumbnails have `Cross-Origin-Resource-Policy` + ** set to `same-origin`, so we need to proxy them */ + thumb: createStream({ + service: "instagram", + type: "default", + u: imageUrl, + filename: "image.jpg" + }) + } + }); + + if (picker.length) return { picker } + } else if (data.video_versions) { + const video = data.video_versions.reduce((a, b) => a.width * a.height < b.width * b.height ? b : a) + return { + urls: video.url, + filename: `instagram_${id}.mp4`, + audioFilename: `instagram_${id}_audio` + } + } else if (data.image_versions2?.candidates) { + return { + urls: data.image_versions2.candidates[0].url, + isPhoto: true + } + } + } + + async function getPost(id) { + let data, result; + try { + const cookie = getCookie('instagram'); + + const bearer = getCookie('instagram_bearer'); + const token = bearer?.values()?.token; + + // get media_id for mobile api, three methods + let media_id = await getMediaId(id); + if (!media_id && token) media_id = await getMediaId(id, { token }); + if (!media_id && cookie) media_id = await getMediaId(id, { cookie }); + + // mobile api (bearer) + if (media_id && token) data = await requestMobileApi(id, { token }); + + // mobile api (no cookie, cookie) + if (!data && media_id) data = await requestMobileApi(id); + if (!data && media_id && cookie) data = await requestMobileApi(id, { cookie }); + + // html embed (no cookie, cookie) + if (!data) data = await requestHTML(id); + if (!data && cookie) data = await requestHTML(id, cookie); + + // web app graphql api (no cookie, cookie) + if (!data) data = await requestGQL(id); + if (!data && cookie) data = await requestGQL(id, cookie); + } catch {} + + if (!data) return { error: 'ErrorCouldntFetch' }; + + if (data?.gql_data) { + result = extractOldPost(data, id) + } else { + result = extractNewPost(data, id) + } + + if (result) return result; + return { error: 'ErrorEmptyDownload' } + } + + async function usernameToId(username, cookie) { + const url = new URL('https://www.instagram.com/api/v1/users/web_profile_info/'); + url.searchParams.set('username', username); + + try { + const data = await request(url, cookie); + return data?.data?.user?.id; + } catch {} + } + + async function getStory(username, id) { + const cookie = getCookie('instagram'); + if (!cookie) return { error: 'ErrorUnsupported' }; + + const userId = await usernameToId(username, cookie); + if (!userId) return { error: 'ErrorEmptyDownload' }; + + const dtsgId = await findDtsgId(cookie); + + const url = new URL('https://www.instagram.com/api/graphql/'); + const requestData = { + fb_dtsg: dtsgId, + jazoest: '26438', + variables: JSON.stringify({ + reel_ids_arr : [ userId ], + }), + server_timestamps: true, + doc_id: '25317500907894419' + }; + + let media; + try { + const data = (await request(url, cookie, 'POST', requestData)); + media = data?.data?.xdt_api__v1__feed__reels_media?.reels_media?.find(m => m.id === userId); + } catch {} + + const item = media.items.find(m => m.pk === id); + if (!item) return { error: 'ErrorEmptyDownload' }; + + if (item.video_versions) { + const video = item.video_versions.reduce((a, b) => a.width * a.height < b.width * b.height ? b : a) + return { + urls: video.url, + filename: `instagram_${id}.mp4`, + audioFilename: `instagram_${id}_audio` + } + } + + if (item.image_versions2?.candidates) { + return { + urls: item.image_versions2.candidates[0].url, + isPhoto: true + } + } + + return { error: 'ErrorCouldntFetch' }; + } + const { postId, storyId, username } = obj; if (postId) return getPost(postId); if (username && storyId) return getStory(username, storyId);