From b54efb968f530f5854749a20ac3d3e74d7b90a94 Mon Sep 17 00:00:00 2001 From: dumbmoron <136796770+dumbmoron@users.noreply.github.com> Date: Sat, 26 Aug 2023 06:35:13 +0000 Subject: [PATCH 1/4] clean up posts/reels code --- src/modules/processing/services/instagram.js | 106 ++++++++----------- 1 file changed, 46 insertions(+), 60 deletions(-) diff --git a/src/modules/processing/services/instagram.js b/src/modules/processing/services/instagram.js index 9614bd9..2b88896 100644 --- a/src/modules/processing/services/instagram.js +++ b/src/modules/processing/services/instagram.js @@ -1,6 +1,20 @@ import { createStream } from "../../stream/manage.js"; import { genericUserAgent } from "../../config.js"; -import { getCookie, updateCookie } from '../cookie/manager.js'; +import { getCookie, updateCookie } from "../cookie/manager.js"; + +const commonInstagramHeaders = { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', + 'User-Agent': genericUserAgent, + 'X-Ig-App-Id': '936619743392459', + 'X-Asbd-Id': '129477', + 'x-requested-with': 'XMLHttpRequest', + 'Sec-Fetch-Dest': 'empty', + 'Sec-Fetch-Mode': 'cors', + 'Sec-Fetch-Site': 'same-origin', + 'upgrade-insecure-requests': '1', + 'accept-encoding': 'gzip, deflate, br', + 'accept-language': 'en-US,en;q=0.9,en;q=0.8', +} export default async function(obj) { let data; @@ -19,84 +33,56 @@ export default async function(obj) { data = await fetch(url, { headers: { - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', - 'User-Agent': genericUserAgent, - 'X-Ig-App-Id': '936619743392459', - 'X-Asbd-Id': '129477', + ...commonInstagramHeaders, 'x-ig-www-claim': cookie?._wwwClaim || '0', 'x-csrftoken': cookie?.values()?.csrftoken, - 'x-requested-with': 'XMLHttpRequest', - 'Sec-Fetch-Dest': 'empty', - 'Sec-Fetch-Mode': 'cors', - 'Sec-Fetch-Site': 'same-origin', - 'upgrade-insecure-requests': '1', - 'accept-encoding': 'gzip, deflate, br', - 'accept-language': 'en-US,en;q=0.9,en;q=0.8', cookie } }) - if (data.headers.get('X-Ig-Set-Www-Claim') && cookie) { + if (data.headers.get('X-Ig-Set-Www-Claim') && cookie) cookie._wwwClaim = data.headers.get('X-Ig-Set-Www-Claim'); - } updateCookie(cookie, data.headers); data = (await data.json()).data; - } catch (e) { - data = false; - } + } catch {} if (!data) return { error: 'ErrorCouldntFetch' }; - let single, multiple = []; const sidecar = data?.shortcode_media?.edge_sidecar_to_children; if (sidecar) { - sidecar.edges.forEach(e => { - if (e.node?.is_video) { - multiple.push({ - type: "video", - // thumbnails have `Cross-Origin-Resource-Policy` set to `same-origin`, so we need to proxy them - thumb: createStream({ - service: "instagram", - type: "default", - u: e.node?.display_url, - filename: "image.jpg" - }), - url: e.node?.video_url - }) - } else { - multiple.push({ - type: "photo", - thumb: createStream({ - service: "instagram", - type: "default", - u: e.node?.display_url, - filename: "image.jpg" - }), - url: e.node?.display_url - }) - } - }) - } else if (data?.shortcode_media?.video_url) { - single = data.shortcode_media.video_url - } else if (data?.shortcode_media?.display_url) { - return { - urls: data?.shortcode_media?.display_url, - isPhoto: true - } - } else { - return { error: 'ErrorEmptyDownload' } - } + const picker = sidecar.edges + .filter(e => e.node?.display_url) + .map(e => { + const type = e.node?.is_video ? "video" : "photo"; + const url = type === "video" ? e.node?.video_url : e.node?.display_url; - if (single) { + return { + type, url, + /* thumbnails have `Cross-Origin-Resource-Policy` + ** set to `same-origin`, so we need to proxy them */ + thumb: createStream({ + service: "instagram", + type: "default", + u: e.node?.display_url, + filename: "image.jpg" + }) + } + }); + + if (picker.length) return { picker } + } else if (data?.shortcode_media?.video_url) { return { - urls: single, + urls: data.shortcode_media.video_url, filename: `instagram_${obj.id}.mp4`, audioFilename: `instagram_${obj.id}_audio` } - } else if (multiple.length) { - return { picker: multiple } - } else { - return { error: 'ErrorEmptyDownload' } + } else if (data?.shortcode_media?.display_url) { + return { + urls: data.shortcode_media.display_url, + isPhoto: true + } } + + return { error: 'ErrorEmptyDownload' } } From 395a59a8b1119f220818201f85f71721143fee2b Mon Sep 17 00:00:00 2001 From: dumbmoron <136796770+dumbmoron@users.noreply.github.com> Date: Sat, 26 Aug 2023 14:58:38 +0000 Subject: [PATCH 2/4] add instagram stories support + some code cleanup and deduplication --- src/modules/processing/match.js | 5 +- src/modules/processing/services/instagram.js | 104 ++++++++++++++---- src/modules/processing/servicesConfig.json | 7 +- .../processing/servicesPatternTesters.js | 3 +- 4 files changed, 96 insertions(+), 23 deletions(-) diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 0552f2d..d95a4d3 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -107,7 +107,10 @@ export default async function (host, patternMatch, url, lang, obj) { }); break; case "instagram": - r = await instagram({ id: patternMatch["id"] }); + r = await instagram({ + ...patternMatch, + quality: obj.vQuality + }) break; case "vine": r = await vine({ id: patternMatch["id"] }); diff --git a/src/modules/processing/services/instagram.js b/src/modules/processing/services/instagram.js index 2b88896..f707a64 100644 --- a/src/modules/processing/services/instagram.js +++ b/src/modules/processing/services/instagram.js @@ -16,9 +16,28 @@ const commonInstagramHeaders = { 'accept-language': 'en-US,en;q=0.9,en;q=0.8', } -export default async function(obj) { +async function request(url, cookie) { + const data = await fetch(url, { + headers: { + ...commonInstagramHeaders, + 'x-ig-www-claim': cookie?._wwwClaim || '0', + 'x-csrftoken': cookie?.values()?.csrftoken, + cookie + } + }) + + if (data.headers.get('X-Ig-Set-Www-Claim') && cookie) + cookie._wwwClaim = data.headers.get('X-Ig-Set-Www-Claim'); + + updateCookie(cookie, data.headers); + return data.json(); +} + +async function getPost(id) { let data; try { + const cookie = getCookie('instagram'); + const url = new URL('https://www.instagram.com/graphql/query/'); url.searchParams.set('query_hash', 'b3055c01b4b222b8a47dc12b090e4e64') url.searchParams.set('variables', JSON.stringify({ @@ -26,25 +45,11 @@ export default async function(obj) { fetch_comment_count: 40, has_threaded_comments: true, parent_comment_count: 24, - shortcode: obj.id + shortcode: id })) - const cookie = getCookie('instagram'); + data = (await request(url, cookie)).data; - data = await fetch(url, { - headers: { - ...commonInstagramHeaders, - 'x-ig-www-claim': cookie?._wwwClaim || '0', - 'x-csrftoken': cookie?.values()?.csrftoken, - cookie - } - }) - - if (data.headers.get('X-Ig-Set-Www-Claim') && cookie) - cookie._wwwClaim = data.headers.get('X-Ig-Set-Www-Claim'); - - updateCookie(cookie, data.headers); - data = (await data.json()).data; } catch {} if (!data) return { error: 'ErrorCouldntFetch' }; @@ -74,8 +79,8 @@ export default async function(obj) { } else if (data?.shortcode_media?.video_url) { return { urls: data.shortcode_media.video_url, - filename: `instagram_${obj.id}.mp4`, - audioFilename: `instagram_${obj.id}_audio` + filename: `instagram_${id}.mp4`, + audioFilename: `instagram_${id}_audio` } } else if (data?.shortcode_media?.display_url) { return { @@ -86,3 +91,64 @@ export default async function(obj) { return { error: 'ErrorEmptyDownload' } } + +async function usernameToId(username, cookie) { + const url = new URL('https://www.instagram.com/api/v1/users/web_profile_info/'); + url.searchParams.set('username', username); + + try { + const data = await request(url, cookie); + return data?.data?.user?.id; + } catch {} +} + +async function getStory(username, id) { + const cookie = getCookie('instagram'); + if (!cookie) return { error: 'ErrorUnsupported' } + + const userId = await usernameToId(username, cookie); + if (!userId) return { error: 'ErrorEmptyDownload' } + + const url = new URL('https://www.instagram.com/api/v1/feed/reels_media/'); + url.searchParams.set('reel_ids', userId); + url.searchParams.set('media_id', id); + + let media; + try { + const data = await request(url, cookie); + media = data?.reels_media?.find(m => m.id === userId); + } catch {} + + const item = media.items[media.media_ids.indexOf(id)]; + if (!item) return { error: 'ErrorEmptyDownload' }; + + if (item.video_versions) { + // todo: closest quality? + const video = item.video_versions.reduce( + (a, b) => a.width * a.height < b.width * b.height ? b : a + ) + + return { + urls: video.url, + filename: `instagram_${id}.mp4`, + audioFilename: `instagram_${id}_audio` + } + } + + if (item.image_versions2?.candidates) { + return { + urls: item.image_versions2.candidates[0].url, + isPhoto: true + } + } + + return { error: 'ErrorCouldntFetch' }; +} + +export default function(obj) { + const { postId, storyId, username } = obj; + if (postId) return getPost(postId); + if (username && storyId) return getStory(username, storyId); + + return { error: 'ErrorUnsupported' } +} \ No newline at end of file diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index c8d3714..2e47893 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -53,8 +53,11 @@ "enabled": true }, "instagram": { - "alias": "instagram reels & posts", - "patterns": ["reels/:id", "reel/:id", "p/:id"], + "alias": "instagram reels, posts & stories", + "patterns": [ + "reels/:postId", "reel/:postId", "p/:postId", + "stories/:username/:storyId" + ], "enabled": true }, "vine": { diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index cd4d4d6..cd57376 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -26,7 +26,8 @@ export const testers = { "soundcloud": (patternMatch) => (patternMatch["author"]?.length <= 25 && patternMatch["song"]?.length <= 255) || (patternMatch["shortLink"] && patternMatch["shortLink"].length <= 32), - "instagram": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length <= 12), + "instagram": (patternMatch) => (patternMatch.postId?.length <= 12) + || (patternMatch.username?.length <= 30 && patternMatch.storyId?.length <= 24), "vine": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length <= 12), From 1504a8bae992fd17f3c4060ae0622d46fbf0a60e Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 14 Oct 2023 23:24:19 +0600 Subject: [PATCH 3/4] Update instagram.js --- src/modules/processing/services/instagram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/services/instagram.js b/src/modules/processing/services/instagram.js index f707a64..10c488b 100644 --- a/src/modules/processing/services/instagram.js +++ b/src/modules/processing/services/instagram.js @@ -151,4 +151,4 @@ export default function(obj) { if (username && storyId) return getStory(username, storyId); return { error: 'ErrorUnsupported' } -} \ No newline at end of file +} From 866792c8d5e6150444c9c9ceddb8a4a1bab5b914 Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 14 Oct 2023 23:48:06 +0600 Subject: [PATCH 4/4] spacing no need for closest quality btw --- src/modules/processing/services/instagram.js | 41 +++++++++----------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/src/modules/processing/services/instagram.js b/src/modules/processing/services/instagram.js index 10c488b..8d8a11d 100644 --- a/src/modules/processing/services/instagram.js +++ b/src/modules/processing/services/instagram.js @@ -56,24 +56,23 @@ async function getPost(id) { const sidecar = data?.shortcode_media?.edge_sidecar_to_children; if (sidecar) { - const picker = sidecar.edges - .filter(e => e.node?.display_url) - .map(e => { - const type = e.node?.is_video ? "video" : "photo"; - const url = type === "video" ? e.node?.video_url : e.node?.display_url; + const picker = sidecar.edges.filter(e => e.node?.display_url) + .map(e => { + const type = e.node?.is_video ? "video" : "photo"; + const url = type === "video" ? e.node?.video_url : e.node?.display_url; - return { - type, url, - /* thumbnails have `Cross-Origin-Resource-Policy` - ** set to `same-origin`, so we need to proxy them */ - thumb: createStream({ - service: "instagram", - type: "default", - u: e.node?.display_url, - filename: "image.jpg" - }) - } - }); + return { + type, url, + /* thumbnails have `Cross-Origin-Resource-Policy` + ** set to `same-origin`, so we need to proxy them */ + thumb: createStream({ + service: "instagram", + type: "default", + u: e.node?.display_url, + filename: "image.jpg" + }) + } + }); if (picker.length) return { picker } } else if (data?.shortcode_media?.video_url) { @@ -94,7 +93,7 @@ async function getPost(id) { async function usernameToId(username, cookie) { const url = new URL('https://www.instagram.com/api/v1/users/web_profile_info/'); - url.searchParams.set('username', username); + url.searchParams.set('username', username); try { const data = await request(url, cookie); @@ -123,11 +122,7 @@ async function getStory(username, id) { if (!item) return { error: 'ErrorEmptyDownload' }; if (item.video_versions) { - // todo: closest quality? - const video = item.video_versions.reduce( - (a, b) => a.width * a.height < b.width * b.height ? b : a - ) - + const video = item.video_versions.reduce((a, b) => a.width * a.height < b.width * b.height ? b : a) return { urls: video.url, filename: `instagram_${id}.mp4`,