From 7799fe4661c246e273dc3c1a73318e451d340e86 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 16 Apr 2024 21:51:03 +0600 Subject: [PATCH 1/4] instagram: update api endpoint, add dtsg token parsing & caching --- src/modules/processing/services/instagram.js | 96 +++++++++++++------- 1 file changed, 65 insertions(+), 31 deletions(-) diff --git a/src/modules/processing/services/instagram.js b/src/modules/processing/services/instagram.js index 8d8a11d..41775c3 100644 --- a/src/modules/processing/services/instagram.js +++ b/src/modules/processing/services/instagram.js @@ -3,28 +3,54 @@ import { genericUserAgent } from "../../config.js"; import { getCookie, updateCookie } from "../cookie/manager.js"; const commonInstagramHeaders = { - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', 'User-Agent': genericUserAgent, - 'X-Ig-App-Id': '936619743392459', - 'X-Asbd-Id': '129477', - 'x-requested-with': 'XMLHttpRequest', - 'Sec-Fetch-Dest': 'empty', - 'Sec-Fetch-Mode': 'cors', + 'sec-gpc': '1', 'Sec-Fetch-Site': 'same-origin', - 'upgrade-insecure-requests': '1', - 'accept-encoding': 'gzip, deflate, br', - 'accept-language': 'en-US,en;q=0.9,en;q=0.8', } -async function request(url, cookie) { +const cachedDtsg = { + value: '', + expiry: 0 +} + +async function findDtsgId(cookie) { + try { + if (cachedDtsg.expiry > Date.now()) return cachedDtsg.value; + + const data = await fetch('https://www.instagram.com/', { + headers: { + ...commonInstagramHeaders, + cookie + } + }).then(r => r.text()); + + const token = data.match(/"dtsg":{"token":"(.*?)"/)[1]; + + cachedDtsg.value = token; + cachedDtsg.expiry = Date.now() + 86400000; + + if (token) return token; + return false; + } + catch {} +} + +async function request(url, cookie, method = 'GET', requestData) { + let headers = { + ...commonInstagramHeaders, + 'x-ig-www-claim': cookie?._wwwClaim || '0', + 'x-csrftoken': cookie?.values()?.csrftoken, + cookie + } + if (method === 'POST') { + headers['content-type'] = 'application/x-www-form-urlencoded'; + } + const data = await fetch(url, { - headers: { - ...commonInstagramHeaders, - 'x-ig-www-claim': cookie?._wwwClaim || '0', - 'x-csrftoken': cookie?.values()?.csrftoken, - cookie - } - }) + method, + headers, + body: requestData && new URLSearchParams(requestData), + }); if (data.headers.get('X-Ig-Set-Www-Claim') && cookie) cookie._wwwClaim = data.headers.get('X-Ig-Set-Www-Claim'); @@ -37,24 +63,32 @@ async function getPost(id) { let data; try { const cookie = getCookie('instagram'); + let dtsgId; + + if (cookie) { + dtsgId = await findDtsgId(cookie); + } const url = new URL('https://www.instagram.com/graphql/query/'); - url.searchParams.set('query_hash', 'b3055c01b4b222b8a47dc12b090e4e64') - url.searchParams.set('variables', JSON.stringify({ - child_comment_count: 3, - fetch_comment_count: 40, - has_threaded_comments: true, - parent_comment_count: 24, - shortcode: id - })) - data = (await request(url, cookie)).data; + const requestData = { + jazoest: '26297', + variables: JSON.stringify({ + shortcode: id, + __relay_internal__pv__PolarisShareMenurelayprovider: false + }), + doc_id: '24852649951017035' + }; + if (dtsgId) { + requestData.fb_dtsg = dtsgId; + } + data = (await request(url, cookie, 'POST', requestData)).data; } catch {} if (!data) return { error: 'ErrorCouldntFetch' }; - const sidecar = data?.shortcode_media?.edge_sidecar_to_children; + const sidecar = data?.xdt_shortcode_media?.edge_sidecar_to_children; if (sidecar) { const picker = sidecar.edges.filter(e => e.node?.display_url) .map(e => { @@ -75,15 +109,15 @@ async function getPost(id) { }); if (picker.length) return { picker } - } else if (data?.shortcode_media?.video_url) { + } else if (data?.xdt_shortcode_media?.video_url) { return { - urls: data.shortcode_media.video_url, + urls: data.xdt_shortcode_media.video_url, filename: `instagram_${id}.mp4`, audioFilename: `instagram_${id}_audio` } - } else if (data?.shortcode_media?.display_url) { + } else if (data?.xdt_shortcode_media?.display_url) { return { - urls: data.shortcode_media.display_url, + urls: data.xdt_shortcode_media.display_url, isPhoto: true } } From 8f5642763bf552b9ad94ef0567e0f9de7fb24b34 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 16 Apr 2024 21:51:27 +0600 Subject: [PATCH 2/4] soundcloud: let -> const cachedID --- src/modules/processing/services/soundcloud.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/modules/processing/services/soundcloud.js b/src/modules/processing/services/soundcloud.js index 46aae5d..37baee5 100644 --- a/src/modules/processing/services/soundcloud.js +++ b/src/modules/processing/services/soundcloud.js @@ -1,7 +1,10 @@ import { maxVideoDuration } from "../../config.js"; import { cleanString } from "../../sub/utils.js"; -let cachedID = {}; +const cachedID = { + version: '', + id: '' +}; async function findClientID() { try { From 9d239525e1efb56c70c3b201e27aeab34241cbb3 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 16 Apr 2024 22:04:38 +0600 Subject: [PATCH 3/4] instagram: lower expiry frame by 10 seconds --- src/modules/processing/services/instagram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/services/instagram.js b/src/modules/processing/services/instagram.js index 41775c3..ddb1c2a 100644 --- a/src/modules/processing/services/instagram.js +++ b/src/modules/processing/services/instagram.js @@ -27,7 +27,7 @@ async function findDtsgId(cookie) { const token = data.match(/"dtsg":{"token":"(.*?)"/)[1]; cachedDtsg.value = token; - cachedDtsg.expiry = Date.now() + 86400000; + cachedDtsg.expiry = Date.now() + 86390000; if (token) return token; return false; From 3389c595dfbfea4d0cf2a6d2f040894ef146f2ff Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 16 Apr 2024 23:51:35 +0600 Subject: [PATCH 4/4] instagram: update stories endpoint and parsing --- src/modules/processing/services/instagram.js | 30 +++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/modules/processing/services/instagram.js b/src/modules/processing/services/instagram.js index ddb1c2a..a5b20d7 100644 --- a/src/modules/processing/services/instagram.js +++ b/src/modules/processing/services/instagram.js @@ -3,9 +3,10 @@ import { genericUserAgent } from "../../config.js"; import { getCookie, updateCookie } from "../cookie/manager.js"; const commonInstagramHeaders = { - 'User-Agent': genericUserAgent, + 'user-agent': genericUserAgent, 'sec-gpc': '1', - 'Sec-Fetch-Site': 'same-origin', + 'sec-fetch-site': 'same-origin', + 'x-ig-app-id': '936619743392459' } const cachedDtsg = { @@ -137,22 +138,31 @@ async function usernameToId(username, cookie) { async function getStory(username, id) { const cookie = getCookie('instagram'); - if (!cookie) return { error: 'ErrorUnsupported' } + if (!cookie) return { error: 'ErrorUnsupported' }; const userId = await usernameToId(username, cookie); - if (!userId) return { error: 'ErrorEmptyDownload' } + if (!userId) return { error: 'ErrorEmptyDownload' }; + + const dtsgId = await findDtsgId(cookie); - const url = new URL('https://www.instagram.com/api/v1/feed/reels_media/'); - url.searchParams.set('reel_ids', userId); - url.searchParams.set('media_id', id); + const url = new URL('https://www.instagram.com/api/graphql/'); + const requestData = { + fb_dtsg: dtsgId, + jazoest: '26438', + variables: JSON.stringify({ + reel_ids_arr : [ userId ], + }), + server_timestamps: true, + doc_id: '25317500907894419' + }; let media; try { - const data = await request(url, cookie); - media = data?.reels_media?.find(m => m.id === userId); + const data = (await request(url, cookie, 'POST', requestData)); + media = data?.data?.xdt_api__v1__feed__reels_media?.reels_media?.find(m => m.id === userId); } catch {} - const item = media.items[media.media_ids.indexOf(id)]; + const item = media.items.filter(m => m.pk === id)[0]; if (!item) return { error: 'ErrorEmptyDownload' }; if (item.video_versions) {