instagram: update api endpoint method, add dtsg token parsing & caching (#430)

src/modules/processing/services/instagram.js

@@ -3,28 +3,55 @@ import { genericUserAgent } from "../../config.js";
 import { getCookie, updateCookie } from "../cookie/manager.js";
 
 const commonInstagramHeaders = {
-    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+    'user-agent': genericUserAgent,
-    'User-Agent': genericUserAgent,
+    'sec-gpc': '1',
-    'X-Ig-App-Id': '936619743392459',
+    'sec-fetch-site': 'same-origin',
-    'X-Asbd-Id': '129477',
+    'x-ig-app-id': '936619743392459'
-    'x-requested-with': 'XMLHttpRequest',
-    'Sec-Fetch-Dest': 'empty',
-    'Sec-Fetch-Mode': 'cors',
-    'Sec-Fetch-Site': 'same-origin',
-    'upgrade-insecure-requests': '1',
-    'accept-encoding': 'gzip, deflate, br',
-    'accept-language': 'en-US,en;q=0.9,en;q=0.8',
 }
 
-async function request(url, cookie) {
+const cachedDtsg = {
+    value: '',
+    expiry: 0
+}
+
+async function findDtsgId(cookie) {
+    try {
+        if (cachedDtsg.expiry > Date.now()) return cachedDtsg.value;
+
+        const data = await fetch('https://www.instagram.com/', {
+            headers: {
+                ...commonInstagramHeaders,
+                cookie
+            }
+        }).then(r => r.text());
+
+        const token = data.match(/"dtsg":{"token":"(.*?)"/)[1];
+
+        cachedDtsg.value = token;
+        cachedDtsg.expiry = Date.now() + 86390000;
+
+        if (token) return token;
+        return false;
+    }
+    catch {}
+}
+
+async function request(url, cookie, method = 'GET', requestData) {
+    let headers = {
+        ...commonInstagramHeaders,
+        'x-ig-www-claim': cookie?._wwwClaim || '0',
+        'x-csrftoken': cookie?.values()?.csrftoken,
+        cookie
+    }
+    if (method === 'POST') {
+        headers['content-type'] = 'application/x-www-form-urlencoded';
+    }
+
     const data = await fetch(url, {
-        headers: {
+        method,
-            ...commonInstagramHeaders,
+        headers,
-            'x-ig-www-claim': cookie?._wwwClaim || '0',
+        body: requestData && new URLSearchParams(requestData),
-            'x-csrftoken': cookie?.values()?.csrftoken,
+    });
-            cookie
-        }
-    })
 
     if (data.headers.get('X-Ig-Set-Www-Claim') && cookie)
         cookie._wwwClaim = data.headers.get('X-Ig-Set-Www-Claim');
@@ -37,24 +64,32 @@ async function getPost(id) {
     let data;
     try {
         const cookie = getCookie('instagram');
+        let dtsgId;
+
+        if (cookie) {
+            dtsgId = await findDtsgId(cookie);
+        }
 
         const url = new URL('https://www.instagram.com/graphql/query/');
-        url.searchParams.set('query_hash', 'b3055c01b4b222b8a47dc12b090e4e64')
-        url.searchParams.set('variables', JSON.stringify({
-            child_comment_count: 3,
-            fetch_comment_count: 40,
-            has_threaded_comments: true,
-            parent_comment_count: 24,
-            shortcode: id
-        }))
 
-        data = (await request(url, cookie)).data;
+        const requestData = {
+            jazoest: '26297',
+            variables: JSON.stringify({
+              shortcode: id,
+              __relay_internal__pv__PolarisShareMenurelayprovider: false
+            }),
+            doc_id: '24852649951017035'
+        };
+        if (dtsgId) {
+            requestData.fb_dtsg = dtsgId;
+        }
 
+        data = (await request(url, cookie, 'POST', requestData)).data;
     } catch {}
 
     if (!data) return { error: 'ErrorCouldntFetch' };
 
-    const sidecar = data?.shortcode_media?.edge_sidecar_to_children;
+    const sidecar = data?.xdt_shortcode_media?.edge_sidecar_to_children;
     if (sidecar) {
         const picker = sidecar.edges.filter(e => e.node?.display_url)
             .map(e => {
@@ -75,15 +110,15 @@ async function getPost(id) {
             });
 
         if (picker.length) return { picker }
-    } else if (data?.shortcode_media?.video_url) {
+    } else if (data?.xdt_shortcode_media?.video_url) {
         return {
-            urls: data.shortcode_media.video_url,
+            urls: data.xdt_shortcode_media.video_url,
             filename: `instagram_${id}.mp4`,
             audioFilename: `instagram_${id}_audio`
         }
-    } else if (data?.shortcode_media?.display_url) {
+    } else if (data?.xdt_shortcode_media?.display_url) {
         return {
-            urls: data.shortcode_media.display_url,
+            urls: data.xdt_shortcode_media.display_url,
             isPhoto: true
         }
     }
@@ -103,22 +138,31 @@ async function usernameToId(username, cookie) {
 
 async function getStory(username, id) {
     const cookie = getCookie('instagram');
-    if (!cookie) return { error: 'ErrorUnsupported' }
+    if (!cookie) return { error: 'ErrorUnsupported' };
 
     const userId = await usernameToId(username, cookie);
-    if (!userId) return { error: 'ErrorEmptyDownload' }
+    if (!userId) return { error: 'ErrorEmptyDownload' };
     
-    const url = new URL('https://www.instagram.com/api/v1/feed/reels_media/');
+    const dtsgId = await findDtsgId(cookie);
-          url.searchParams.set('reel_ids', userId);
+
-          url.searchParams.set('media_id', id);
+    const url = new URL('https://www.instagram.com/api/graphql/');
+    const requestData = {
+        fb_dtsg: dtsgId,
+        jazoest: '26438',
+        variables: JSON.stringify({
+            reel_ids_arr : [ userId ],
+        }),
+        server_timestamps: true,
+        doc_id: '25317500907894419'
+    };
 
     let media;
     try {
-        const data = await request(url, cookie);
+        const data = (await request(url, cookie, 'POST', requestData));
-        media = data?.reels_media?.find(m => m.id === userId);
+        media = data?.data?.xdt_api__v1__feed__reels_media?.reels_media?.find(m => m.id === userId);
     } catch {}
 
-    const item = media.items[media.media_ids.indexOf(id)];
+    const item = media.items.filter(m => m.pk === id)[0];
     if (!item) return { error: 'ErrorEmptyDownload' };
     
     if (item.video_versions) {

src/modules/processing/services/soundcloud.js

@@ -1,7 +1,10 @@
 import { maxVideoDuration } from "../../config.js";
 import { cleanString } from "../../sub/utils.js";
 
-let cachedID = {};
+const cachedID = {
+    version: '',
+    id: ''
+};
 
 async function findClientID() {
     try {