twitter: more supported links, token reuse, and clean up

merge pull request #303 from dumbmoron/twitter-reuse-token

src/modules/processing/match.js

@@ -36,7 +36,8 @@ export default async function(host, patternMatch, url, lang, obj) {
         switch (host) {
             case "twitter":
                 r = await twitter({
-                    id: patternMatch["id"]
+                    id: patternMatch.id,
+                    index: patternMatch.index - 1
                 });
                 break;
             case "vk":

src/modules/processing/services/twitter.js

@@ -1,6 +1,19 @@
 import { genericUserAgent } from "../../config.js";
 import { createStream } from "../../stream/manage.js";
 
+const graphqlURL = 'https://twitter.com/i/api/graphql/5GOHgZe-8U2j5sVHQzEm9A/TweetResultByRestId';
+const tokenURL = 'https://api.twitter.com/1.1/guest/activate.json';
+
+const tweetFeatures = JSON.stringify({ "creator_subscriptions_tweet_preview_api_enabled": true, "c9s_tweet_anatomy_moderator_badge_enabled": true, "tweetypie_unmention_optimization_enabled": true, "responsive_web_edit_tweet_api_enabled": true, "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, "view_counts_everywhere_api_enabled": true, "longform_notetweets_consumption_enabled": true, "responsive_web_twitter_article_tweet_consumption_enabled": false, "tweet_awards_web_tipping_enabled": false, "responsive_web_home_pinned_timelines_enabled": true, "freedom_of_speech_not_reach_fetch_enabled": true, "standardized_nudges_misinfo": true, "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, "longform_notetweets_rich_text_read_enabled": true, "longform_notetweets_inline_media_enabled": true, "responsive_web_graphql_exclude_directive_enabled": true, "verified_phone_label_enabled": false, "responsive_web_media_download_video_enabled": false, "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, "responsive_web_graphql_timeline_navigation_enabled": true, "responsive_web_enhance_cards_enabled": false });
+
+const commonHeaders = {
+    "user-agent": genericUserAgent,
+    "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
+    "x-twitter-client-language": "en",
+    "x-twitter-active-user": "yes",
+    "accept-language": "en"
+}
+
 // fix all videos affected by the container bug in twitter muxer (took them over two weeks to fix it????)
 const TWITTER_EPOCH = 1288834974657n;
 const badContainerStart = new Date(1701446400000);
@@ -10,104 +23,112 @@ function needsFixing(media) {
     const representativeId = media.source_status_id_str ?? media.id_str;
     const mediaTimestamp = new Date(
         Number((BigInt(representativeId) >> 22n) + TWITTER_EPOCH)
-    )
+    );
-    return mediaTimestamp > badContainerStart && mediaTimestamp < badContainerEnd;
+    return mediaTimestamp > badContainerStart && mediaTimestamp < badContainerEnd
 }
 
 function bestQuality(arr) {
-    return arr.filter(v => v["content_type"] === "video/mp4").sort((a, b) => Number(b.bitrate) - Number(a.bitrate))[0]["url"]
+    return arr.filter(v => v.content_type === "video/mp4")
+        .reduce((a, b) => Number(a?.bitrate) > Number(b?.bitrate) ? a : b)
+        .url
 }
 
-export default async function(obj) {
+let _cachedToken;
-    let _headers = {
+const getGuestToken = async (forceReload = false) => {
-        "user-agent": genericUserAgent,
+    if (_cachedToken && !forceReload) {
-        "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
+        return _cachedToken;
-        "host": "api.twitter.com",
-        "x-twitter-client-language": "en",
-        "x-twitter-active-user": "yes",
-        "accept-language": "en"
-    };
-
-    let activateURL = `https://api.twitter.com/1.1/guest/activate.json`;
-    let graphqlTweetURL = `https://twitter.com/i/api/graphql/5GOHgZe-8U2j5sVHQzEm9A/TweetResultByRestId`;
-
-    let req_act = await fetch(activateURL, {
-        method: "POST",
-        headers: _headers
-    }).then((r) => { return r.status === 200 ? r.json() : false }).catch(() => { return false });
-    if (!req_act) return { error: 'ErrorCouldntFetch' };
-
-    _headers["host"] = "twitter.com";
-    _headers["content-type"] = "application/json";
-
-    _headers["x-guest-token"] = req_act["guest_token"];
-    _headers["cookie"] = `guest_id=v1%3A${req_act["guest_token"]}`;
-
-    let query = {
-        variables: { "tweetId": obj.id, "withCommunity": false, "includePromotedContent": false, "withVoice": false },
-        features: { "creator_subscriptions_tweet_preview_api_enabled": true, "c9s_tweet_anatomy_moderator_badge_enabled": true, "tweetypie_unmention_optimization_enabled": true, "responsive_web_edit_tweet_api_enabled": true, "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, "view_counts_everywhere_api_enabled": true, "longform_notetweets_consumption_enabled": true, "responsive_web_twitter_article_tweet_consumption_enabled": false, "tweet_awards_web_tipping_enabled": false, "responsive_web_home_pinned_timelines_enabled": true, "freedom_of_speech_not_reach_fetch_enabled": true, "standardized_nudges_misinfo": true, "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, "longform_notetweets_rich_text_read_enabled": true, "longform_notetweets_inline_media_enabled": true, "responsive_web_graphql_exclude_directive_enabled": true, "verified_phone_label_enabled": false, "responsive_web_media_download_video_enabled": false, "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, "responsive_web_graphql_timeline_navigation_enabled": true, "responsive_web_enhance_cards_enabled": false }
     }
-    query.variables = encodeURIComponent(JSON.stringify(query.variables));
-    query.features = encodeURIComponent(JSON.stringify(query.features));
-    query = `${graphqlTweetURL}?variables=${query.variables}&features=${query.features}`;
 
-    let tweet = await fetch(query, { headers: _headers }).then((r) => {
+    const tokenResponse = await fetch(tokenURL, {
-        return r.status === 200 ? r.json() : false
+        method: 'POST',
-    }).catch(() => { return false });
+        headers: commonHeaders
+    }).then(r => r.status === 200 && r.json()).catch(() => {})
+
+    if (tokenResponse?.guest_token) {
+        return _cachedToken = tokenResponse.guest_token
+    }
+}
+
+const requestTweet = (tweetId, token) => {
+    const graphqlTweetURL = new URL(graphqlURL);
+
+    graphqlTweetURL.searchParams.set('variables',
+        JSON.stringify({
+            tweetId,
+            withCommunity: false,
+            includePromotedContent: false,
+            withVoice: false
+        })
+    );
+    graphqlTweetURL.searchParams.set('features', tweetFeatures);
+
+    return fetch(graphqlTweetURL, {
+        headers: {
+            ...commonHeaders,
+            'content-type': 'application/json',
+            'x-guest-token': token,
+            cookie: `guest_id=${encodeURIComponent(`v1:${token}`)}`
+        }
+    })
+}
+
+export default async function({ id, index }) {
+    let guestToken = await getGuestToken();
+    if (!guestToken) return { error: 'ErrorCouldntFetch' };
+
+    let tweet = await requestTweet(id, guestToken);
+
+    if ([403, 429].includes(tweet.status)) { // get new token & retry
+        guestToken = await getGuestToken(true);
+        tweet = await requestTweet(id, guestToken)
+    }
+
+    tweet = await tweet.json();
 
     // {"data":{"tweetResult":{"result":{"__typename":"TweetUnavailable","reason":"Protected"}}}}
     if (tweet?.data?.tweetResult?.result?.__typename !== "Tweet") {
         return { error: 'ErrorTweetUnavailable' }
     }
 
-    let baseMedia,
+    const baseTweet = tweet.data.tweetResult.result.legacy,
-        baseTweet = tweet.data.tweetResult.result.legacy;
+          repostedTweet = baseTweet.retweeted_status_result?.result.legacy.extended_entities;
 
-    if (baseTweet.retweeted_status_result?.result.legacy.extended_entities.media) {
+    let media = (repostedTweet?.media || baseTweet.extended_entities.media);
-        baseMedia = baseTweet.retweeted_status_result.result.legacy.extended_entities
+    media = media?.filter(m => m.video_info?.variants?.length);
-    } else if (baseTweet.extended_entities?.media) {
-        baseMedia = baseTweet.extended_entities
-    }
-    if (!baseMedia) return { error: 'ErrorNoVideosInTweet' };
 
-    let single, multiple = [], media = baseMedia["media"];
+    // check if there's a video at given index (/video/<index>)
-    media = media.filter((i) => { if (i["type"] === "video" || i["type"] === "animated_gif") return true });
+    if ([0, 1, 2, 3].includes(index) && index < media?.length) {
-
+        media = [media[index]]
-    if (media.length === 0) {
-        return { error: 'ErrorNoVideosInTweet' }
     }
 
-    if (media.length > 1) {
+    switch (media?.length) {
-        for (let i in media) {
+        case undefined:
-            let downloadUrl = bestQuality(media[i]["video_info"]["variants"]);
+        case 0:
-            if (needsFixing(media[i])) {
+            return { error: 'ErrorNoVideosInTweet' };
-                downloadUrl = createStream({
+        case 1:
-                    service: "twitter",
+            return {
-                    type: "remux",
+                type: needsFixing(media[0]) ? "remux" : "normal",
-                    u: bestQuality(media[i]["video_info"]["variants"]),
+                urls: bestQuality(media[0].video_info.variants),
-                    filename: `twitter_${obj.id}_${Number(i) + 1}.mp4`
+                filename: `twitter_${id}.mp4`,
-                })
+                audioFilename: `twitter_${id}_audio`
-            }
+            };
-            multiple.push({
+        default:
-                type: "video",
+            const picker = media.map((video, i) => {
-                thumb: media[i]["media_url_https"],
+                let url = bestQuality(video.video_info.variants);
-                url: downloadUrl
+                if (needsFixing(video)) {
-            })
+                    url = createStream({
-        }
+                        service: 'twitter',
-    } else {
+                        type: 'remux',
-        single = bestQuality(media[0]["video_info"]["variants"])
+                        u: url,
-    }
+                        filename: `twitter_${id}_${i + 1}.mp4`
-
+                    })
-    if (single) {
+                }
-        return {
+                return {
-            type: needsFixing(media[0]) ? "remux" : "normal",
+                    type: 'video',
-            urls: single,
+                    url,
-            filename: `twitter_${obj.id}.mp4`,
+                    thumb: video.media_url_https,
-            audioFilename: `twitter_${obj.id}_audio`
+                }
-        }
+            });
-    } else if (multiple) {
+            return { picker };
-        return { picker: multiple }
-    } else {
-        return { error: 'ErrorNoVideosInTweet' }
     }
 }

src/modules/processing/servicesConfig.json

@@ -15,7 +15,12 @@
             "alias": "twitter videos & voice",
             "altDomains": ["x.com", "vxtwitter.com", "fixvx.com"],
             "subdomains": ["mobile"],
-            "patterns": [":user/status/:id", ":user/status/:id/video/:v"],
+            "patterns": [
+                ":user/status/:id",
+                ":user/status/:id/video/:index",
+                ":user/status/:id/mediaviewer",
+                ":user/status/:id/mediaViewer"
+            ],
             "enabled": true
         },
         "vk": {