From 02b67a314575298fe7b9bb68501411bde46df883 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Thu, 4 Jan 2024 16:26:52 +0000 Subject: [PATCH 1/6] twitter: cleanup/decomposition --- src/modules/processing/services/twitter.js | 154 +++++++++++---------- 1 file changed, 78 insertions(+), 76 deletions(-) diff --git a/src/modules/processing/services/twitter.js b/src/modules/processing/services/twitter.js index ef08cf70..c9fd6d12 100644 --- a/src/modules/processing/services/twitter.js +++ b/src/modules/processing/services/twitter.js @@ -15,99 +15,101 @@ function needsFixing(media) { } function bestQuality(arr) { - return arr.filter(v => v["content_type"] === "video/mp4").sort((a, b) => Number(b.bitrate) - Number(a.bitrate))[0]["url"] + return arr + .filter(v => v.content_type === "video/mp4") + .reduce((a, b) => Number(a?.bitrate) > Number(b?.bitrate) ? a : b) + .url } -export default async function(obj) { - let _headers = { - "user-agent": genericUserAgent, - "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA", - "host": "api.twitter.com", - "x-twitter-client-language": "en", - "x-twitter-active-user": "yes", - "accept-language": "en" - }; +const tweetFeatures = JSON.stringify({ "creator_subscriptions_tweet_preview_api_enabled": true, "c9s_tweet_anatomy_moderator_badge_enabled": true, "tweetypie_unmention_optimization_enabled": true, "responsive_web_edit_tweet_api_enabled": true, "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, "view_counts_everywhere_api_enabled": true, "longform_notetweets_consumption_enabled": true, "responsive_web_twitter_article_tweet_consumption_enabled": false, "tweet_awards_web_tipping_enabled": false, "responsive_web_home_pinned_timelines_enabled": true, "freedom_of_speech_not_reach_fetch_enabled": true, "standardized_nudges_misinfo": true, "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, "longform_notetweets_rich_text_read_enabled": true, "longform_notetweets_inline_media_enabled": true, "responsive_web_graphql_exclude_directive_enabled": true, "verified_phone_label_enabled": false, "responsive_web_media_download_video_enabled": false, "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, "responsive_web_graphql_timeline_navigation_enabled": true, "responsive_web_enhance_cards_enabled": false }); - let activateURL = `https://api.twitter.com/1.1/guest/activate.json`; - let graphqlTweetURL = `https://twitter.com/i/api/graphql/5GOHgZe-8U2j5sVHQzEm9A/TweetResultByRestId`; +const commonHeaders = { + "user-agent": genericUserAgent, + "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA", + "x-twitter-client-language": "en", + "x-twitter-active-user": "yes", + "accept-language": "en" +} - let req_act = await fetch(activateURL, { - method: "POST", - headers: _headers - }).then((r) => { return r.status === 200 ? r.json() : false }).catch(() => { return false }); - if (!req_act) return { error: 'ErrorCouldntFetch' }; +const getGuestToken = async () => { + const tokenResponse = await fetch( + 'https://api.twitter.com/1.1/guest/activate.json', + { method: 'POST', headers: commonHeaders } + ).then(r => r.status === 200 && r.json()).catch(() => {}) - _headers["host"] = "twitter.com"; - _headers["content-type"] = "application/json"; + if (tokenResponse?.guest_token) + return tokenResponse.guest_token +} - _headers["x-guest-token"] = req_act["guest_token"]; - _headers["cookie"] = `guest_id=v1%3A${req_act["guest_token"]}`; +const requestTweet = (tweetId, token) => { + const graphqlTweetURL = new URL('https://twitter.com/i/api/graphql/5GOHgZe-8U2j5sVHQzEm9A/TweetResultByRestId'); + graphqlTweetURL.searchParams.set( + 'variables', + JSON.stringify({ + tweetId, + withCommunity: false, + includePromotedContent: false, + withVoice: false + }) + ); - let query = { - variables: { "tweetId": obj.id, "withCommunity": false, "includePromotedContent": false, "withVoice": false }, - features: { "creator_subscriptions_tweet_preview_api_enabled": true, "c9s_tweet_anatomy_moderator_badge_enabled": true, "tweetypie_unmention_optimization_enabled": true, "responsive_web_edit_tweet_api_enabled": true, "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, "view_counts_everywhere_api_enabled": true, "longform_notetweets_consumption_enabled": true, "responsive_web_twitter_article_tweet_consumption_enabled": false, "tweet_awards_web_tipping_enabled": false, "responsive_web_home_pinned_timelines_enabled": true, "freedom_of_speech_not_reach_fetch_enabled": true, "standardized_nudges_misinfo": true, "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, "longform_notetweets_rich_text_read_enabled": true, "longform_notetweets_inline_media_enabled": true, "responsive_web_graphql_exclude_directive_enabled": true, "verified_phone_label_enabled": false, "responsive_web_media_download_video_enabled": false, "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, "responsive_web_graphql_timeline_navigation_enabled": true, "responsive_web_enhance_cards_enabled": false } - } - query.variables = encodeURIComponent(JSON.stringify(query.variables)); - query.features = encodeURIComponent(JSON.stringify(query.features)); - query = `${graphqlTweetURL}?variables=${query.variables}&features=${query.features}`; + graphqlTweetURL.searchParams.set('features', tweetFeatures); - let tweet = await fetch(query, { headers: _headers }).then((r) => { - return r.status === 200 ? r.json() : false - }).catch(() => { return false }); + return fetch(graphqlTweetURL, { + headers: { + ...commonHeaders, + 'content-type': 'application/json', + 'x-guest-token': token, + cookie: `guest_id=${encodeURIComponent(`v1:${token}`)}` + } + }); +} +export default async function({ id }) { + let guestToken = await getGuestToken(); + if (!guestToken) return { error: 'ErrorCouldntFetch' }; + + const tweet = await requestTweet(id, guestToken).then(t => t.json()); + // {"data":{"tweetResult":{"result":{"__typename":"TweetUnavailable","reason":"Protected"}}}} if (tweet?.data?.tweetResult?.result?.__typename !== "Tweet") { return { error: 'ErrorTweetUnavailable' } } - let baseMedia, - baseTweet = tweet.data.tweetResult.result.legacy; + const baseTweet = tweet.data.tweetResult.result.legacy, + repostedTweet = baseTweet.retweeted_status_result?.result.legacy.extended_entities; - if (baseTweet.retweeted_status_result?.result.legacy.extended_entities.media) { - baseMedia = baseTweet.retweeted_status_result.result.legacy.extended_entities - } else if (baseTweet.extended_entities?.media) { - baseMedia = baseTweet.extended_entities - } - if (!baseMedia) return { error: 'ErrorNoVideosInTweet' }; + const media = ( + repostedTweet?.media || baseTweet.extended_entities.media + )?.filter(m => ['video', 'animated_gif'].includes(m.type)); - let single, multiple = [], media = baseMedia["media"]; - media = media.filter((i) => { if (i["type"] === "video" || i["type"] === "animated_gif") return true }); - - if (media.length === 0) { - return { error: 'ErrorNoVideosInTweet' } - } - - if (media.length > 1) { - for (let i in media) { - let downloadUrl = bestQuality(media[i]["video_info"]["variants"]); - if (needsFixing(media[i])) { - downloadUrl = createStream({ - service: "twitter", - type: "remux", - u: bestQuality(media[i]["video_info"]["variants"]), - filename: `twitter_${obj.id}_${Number(i) + 1}.mp4` - }) + switch (media?.length) { + case undefined: + case 0: + return { error: 'ErrorNoVideosInTweet' } + case 1: + return { + type: needsFixing(media[0]) ? "remux" : "normal", + urls: bestQuality(media[0].video_info.variants), + filename: `twitter_${id}.mp4`, + audioFilename: `twitter_${id}_audio` } - multiple.push({ - type: "video", - thumb: media[i]["media_url_https"], - url: downloadUrl - }) - } - } else { - single = bestQuality(media[0]["video_info"]["variants"]) - } + default: + const picker = media.map((video, i) => { + let url = bestQuality(video.video_info.variants); + if (needsFixing(video)) { + url = createStream({ + service: 'twitter', type: 'remux', + u: url, filename: `twitter_${id}_${i + 1}.mp4` + }) + } - if (single) { - return { - type: needsFixing(media[0]) ? "remux" : "normal", - urls: single, - filename: `twitter_${obj.id}.mp4`, - audioFilename: `twitter_${obj.id}_audio` - } - } else if (multiple) { - return { picker: multiple } - } else { - return { error: 'ErrorNoVideosInTweet' } + return { + type: 'video', url, + thumb: video.media_url_https, + } + }); + + return { picker } } } From 7b41dfef9e8cc3176a323475185ee26c8a277374 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Thu, 4 Jan 2024 16:35:58 +0000 Subject: [PATCH 2/6] twitter: cache token, get new one & retry on error --- src/modules/processing/services/twitter.js | 23 +++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/modules/processing/services/twitter.js b/src/modules/processing/services/twitter.js index c9fd6d12..a02b792c 100644 --- a/src/modules/processing/services/twitter.js +++ b/src/modules/processing/services/twitter.js @@ -31,14 +31,20 @@ const commonHeaders = { "accept-language": "en" } -const getGuestToken = async () => { +let _cachedToken +const getGuestToken = async (forceReload = false) => { + if (_cachedToken && !forceReload) { + return _cachedToken; + } + const tokenResponse = await fetch( 'https://api.twitter.com/1.1/guest/activate.json', { method: 'POST', headers: commonHeaders } ).then(r => r.status === 200 && r.json()).catch(() => {}) - if (tokenResponse?.guest_token) - return tokenResponse.guest_token + if (tokenResponse?.guest_token) { + return _cachedToken = tokenResponse.guest_token + } } const requestTweet = (tweetId, token) => { @@ -69,8 +75,15 @@ export default async function({ id }) { let guestToken = await getGuestToken(); if (!guestToken) return { error: 'ErrorCouldntFetch' }; - const tweet = await requestTweet(id, guestToken).then(t => t.json()); - + let tweet = await requestTweet(id, guestToken); + + if ([403, 429].includes(tweet.status)) { // get new token & retry + guestToken = await getGuestToken(true); + tweet = await requestTweet(id, guestToken); + } + + tweet = await tweet.json(); + // {"data":{"tweetResult":{"result":{"__typename":"TweetUnavailable","reason":"Protected"}}}} if (tweet?.data?.tweetResult?.result?.__typename !== "Tweet") { return { error: 'ErrorTweetUnavailable' } From ed2c63f9a498cf6213fb2de3b71e03d2ff7006ca Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Fri, 5 Jan 2024 12:21:59 +0000 Subject: [PATCH 3/6] twitter: downloading specific video if given index --- src/modules/processing/match.js | 3 ++- src/modules/processing/services/twitter.js | 8 ++++++-- src/modules/processing/servicesConfig.json | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 3317dc6c..12c0bc10 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -36,7 +36,8 @@ export default async function(host, patternMatch, url, lang, obj) { switch (host) { case "twitter": r = await twitter({ - id: patternMatch["id"] + id: patternMatch.id, + index: patternMatch.index - 1 }); break; case "vk": diff --git a/src/modules/processing/services/twitter.js b/src/modules/processing/services/twitter.js index a02b792c..ea5f4818 100644 --- a/src/modules/processing/services/twitter.js +++ b/src/modules/processing/services/twitter.js @@ -71,7 +71,7 @@ const requestTweet = (tweetId, token) => { }); } -export default async function({ id }) { +export default async function({ id, index }) { let guestToken = await getGuestToken(); if (!guestToken) return { error: 'ErrorCouldntFetch' }; @@ -92,10 +92,14 @@ export default async function({ id }) { const baseTweet = tweet.data.tweetResult.result.legacy, repostedTweet = baseTweet.retweeted_status_result?.result.legacy.extended_entities; - const media = ( + let media = ( repostedTweet?.media || baseTweet.extended_entities.media )?.filter(m => ['video', 'animated_gif'].includes(m.type)); + if (index < media?.length) { + media = [ media[index] ]; + } + switch (media?.length) { case undefined: case 0: diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index e85b9070..02bf393a 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -15,7 +15,7 @@ "alias": "twitter videos & voice", "altDomains": ["x.com", "vxtwitter.com", "fixvx.com"], "subdomains": ["mobile"], - "patterns": [":user/status/:id", ":user/status/:id/video/:v"], + "patterns": [":user/status/:id", ":user/status/:id/video/:index"], "enabled": true }, "vk": { From 678e00430b5d6abb25bfb6f12bf9da9328ecba88 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Fri, 5 Jan 2024 21:11:14 +0000 Subject: [PATCH 4/6] twitter: add support for media[vV]iewer links --- src/modules/processing/servicesConfig.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index 02bf393a..6ecd2745 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -15,7 +15,12 @@ "alias": "twitter videos & voice", "altDomains": ["x.com", "vxtwitter.com", "fixvx.com"], "subdomains": ["mobile"], - "patterns": [":user/status/:id", ":user/status/:id/video/:index"], + "patterns": [ + ":user/status/:id", + ":user/status/:id/video/:index", + ":user/status/:id/mediaviewer", + ":user/status/:id/mediaViewer" + ], "enabled": true }, "vk": { From 678d6a56cab6e3dfafc244393609331386a6e9ba Mon Sep 17 00:00:00 2001 From: wukko Date: Sun, 7 Jan 2024 13:48:30 +0600 Subject: [PATCH 5/6] twitter: clean up --- src/modules/processing/services/twitter.js | 80 +++++++++++----------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/src/modules/processing/services/twitter.js b/src/modules/processing/services/twitter.js index ea5f4818..0c254a77 100644 --- a/src/modules/processing/services/twitter.js +++ b/src/modules/processing/services/twitter.js @@ -1,25 +1,8 @@ import { genericUserAgent } from "../../config.js"; import { createStream } from "../../stream/manage.js"; -// fix all videos affected by the container bug in twitter muxer (took them over two weeks to fix it????) -const TWITTER_EPOCH = 1288834974657n; -const badContainerStart = new Date(1701446400000); -const badContainerEnd = new Date(1702605600000); - -function needsFixing(media) { - const representativeId = media.source_status_id_str ?? media.id_str; - const mediaTimestamp = new Date( - Number((BigInt(representativeId) >> 22n) + TWITTER_EPOCH) - ) - return mediaTimestamp > badContainerStart && mediaTimestamp < badContainerEnd; -} - -function bestQuality(arr) { - return arr - .filter(v => v.content_type === "video/mp4") - .reduce((a, b) => Number(a?.bitrate) > Number(b?.bitrate) ? a : b) - .url -} +const graphqlURL = 'https://twitter.com/i/api/graphql/5GOHgZe-8U2j5sVHQzEm9A/TweetResultByRestId'; +const tokenURL = 'https://api.twitter.com/1.1/guest/activate.json'; const tweetFeatures = JSON.stringify({ "creator_subscriptions_tweet_preview_api_enabled": true, "c9s_tweet_anatomy_moderator_badge_enabled": true, "tweetypie_unmention_optimization_enabled": true, "responsive_web_edit_tweet_api_enabled": true, "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, "view_counts_everywhere_api_enabled": true, "longform_notetweets_consumption_enabled": true, "responsive_web_twitter_article_tweet_consumption_enabled": false, "tweet_awards_web_tipping_enabled": false, "responsive_web_home_pinned_timelines_enabled": true, "freedom_of_speech_not_reach_fetch_enabled": true, "standardized_nudges_misinfo": true, "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, "longform_notetweets_rich_text_read_enabled": true, "longform_notetweets_inline_media_enabled": true, "responsive_web_graphql_exclude_directive_enabled": true, "verified_phone_label_enabled": false, "responsive_web_media_download_video_enabled": false, "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, "responsive_web_graphql_timeline_navigation_enabled": true, "responsive_web_enhance_cards_enabled": false }); @@ -31,16 +14,35 @@ const commonHeaders = { "accept-language": "en" } -let _cachedToken +// fix all videos affected by the container bug in twitter muxer (took them over two weeks to fix it????) +const TWITTER_EPOCH = 1288834974657n; +const badContainerStart = new Date(1701446400000); +const badContainerEnd = new Date(1702605600000); + +function needsFixing(media) { + const representativeId = media.source_status_id_str ?? media.id_str; + const mediaTimestamp = new Date( + Number((BigInt(representativeId) >> 22n) + TWITTER_EPOCH) + ); + return mediaTimestamp > badContainerStart && mediaTimestamp < badContainerEnd +} + +function bestQuality(arr) { + return arr.filter(v => v.content_type === "video/mp4") + .reduce((a, b) => Number(a?.bitrate) > Number(b?.bitrate) ? a : b) + .url +} + +let _cachedToken; const getGuestToken = async (forceReload = false) => { if (_cachedToken && !forceReload) { return _cachedToken; } - const tokenResponse = await fetch( - 'https://api.twitter.com/1.1/guest/activate.json', - { method: 'POST', headers: commonHeaders } - ).then(r => r.status === 200 && r.json()).catch(() => {}) + const tokenResponse = await fetch(tokenURL, { + method: 'POST', + headers: commonHeaders + }).then(r => r.status === 200 && r.json()).catch(() => {}) if (tokenResponse?.guest_token) { return _cachedToken = tokenResponse.guest_token @@ -48,9 +50,9 @@ const getGuestToken = async (forceReload = false) => { } const requestTweet = (tweetId, token) => { - const graphqlTweetURL = new URL('https://twitter.com/i/api/graphql/5GOHgZe-8U2j5sVHQzEm9A/TweetResultByRestId'); - graphqlTweetURL.searchParams.set( - 'variables', + const graphqlTweetURL = new URL(graphqlURL); + + graphqlTweetURL.searchParams.set('variables', JSON.stringify({ tweetId, withCommunity: false, @@ -58,7 +60,6 @@ const requestTweet = (tweetId, token) => { withVoice: false }) ); - graphqlTweetURL.searchParams.set('features', tweetFeatures); return fetch(graphqlTweetURL, { @@ -68,7 +69,7 @@ const requestTweet = (tweetId, token) => { 'x-guest-token': token, cookie: `guest_id=${encodeURIComponent(`v1:${token}`)}` } - }); + }) } export default async function({ id, index }) { @@ -79,7 +80,7 @@ export default async function({ id, index }) { if ([403, 429].includes(tweet.status)) { // get new token & retry guestToken = await getGuestToken(true); - tweet = await requestTweet(id, guestToken); + tweet = await requestTweet(id, guestToken) } tweet = await tweet.json(); @@ -92,14 +93,12 @@ export default async function({ id, index }) { const baseTweet = tweet.data.tweetResult.result.legacy, repostedTweet = baseTweet.retweeted_status_result?.result.legacy.extended_entities; - let media = ( - repostedTweet?.media || baseTweet.extended_entities.media - )?.filter(m => ['video', 'animated_gif'].includes(m.type)); + let media = (repostedTweet?.media || baseTweet.extended_entities.media); + media = media?.filter(m => m.video_info?.variants?.length); if (index < media?.length) { - media = [ media[index] ]; + media = [media[index]] } - switch (media?.length) { case undefined: case 0: @@ -116,17 +115,18 @@ export default async function({ id, index }) { let url = bestQuality(video.video_info.variants); if (needsFixing(video)) { url = createStream({ - service: 'twitter', type: 'remux', - u: url, filename: `twitter_${id}_${i + 1}.mp4` + service: 'twitter', + type: 'remux', + u: url, + filename: `twitter_${id}_${i + 1}.mp4` }) } - return { - type: 'video', url, + type: 'video', + url, thumb: video.media_url_https, } }); - return { picker } } } From e1edecb859533914fe40353c262152ba0a4282e5 Mon Sep 17 00:00:00 2001 From: wukko Date: Sun, 7 Jan 2024 14:22:59 +0600 Subject: [PATCH 6/6] twitter: fix parsing from /video/ links error is no longer thrown if index is an unexpected value (such as 0 or negative) --- src/modules/processing/services/twitter.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/modules/processing/services/twitter.js b/src/modules/processing/services/twitter.js index 0c254a77..31b26b48 100644 --- a/src/modules/processing/services/twitter.js +++ b/src/modules/processing/services/twitter.js @@ -96,20 +96,22 @@ export default async function({ id, index }) { let media = (repostedTweet?.media || baseTweet.extended_entities.media); media = media?.filter(m => m.video_info?.variants?.length); - if (index < media?.length) { + // check if there's a video at given index (/video/) + if ([0, 1, 2, 3].includes(index) && index < media?.length) { media = [media[index]] } + switch (media?.length) { case undefined: case 0: - return { error: 'ErrorNoVideosInTweet' } + return { error: 'ErrorNoVideosInTweet' }; case 1: return { type: needsFixing(media[0]) ? "remux" : "normal", urls: bestQuality(media[0].video_info.variants), filename: `twitter_${id}.mp4`, audioFilename: `twitter_${id}_audio` - } + }; default: const picker = media.map((video, i) => { let url = bestQuality(video.video_info.variants); @@ -127,6 +129,6 @@ export default async function({ id, index }) { thumb: video.media_url_https, } }); - return { picker } + return { picker }; } }