api/twitter: add fallback to syndication api

it's back yet again, now for good, i suppose

api/src/processing/services/twitter.js

@@ -24,6 +24,11 @@ const badContainerEnd = new Date(1702605600000);
 
 function needsFixing(media) {
     const representativeId = media.source_status_id_str ?? media.id_str;
+
+    // syndication api doesn't have media ids in its response,
+    // so we just assume it's all good
+    if (!representativeId) return false;
+
     const mediaTimestamp = new Date(
         Number((BigInt(representativeId) >> 22n) + TWITTER_EPOCH)
     );
@@ -53,6 +58,25 @@ const getGuestToken = async (dispatcher, forceReload = false) => {
     }
 }
 
+const requestSyndication = async(dispatcher, tweetId) => {
+    // thank you
+    // https://github.com/yt-dlp/yt-dlp/blob/05c8023a27dd37c49163c0498bf98e3e3c1cb4b9/yt_dlp/extractor/twitter.py#L1334
+    const token = (id) => ((Number(id) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, '');
+    const syndicationUrl = new URL("https://cdn.syndication.twimg.com/tweet-result");
+
+    syndicationUrl.searchParams.set("id", tweetId);
+    syndicationUrl.searchParams.set("token", token(tweetId));
+
+    const result = await fetch(syndicationUrl, {
+        headers: {
+            "user-agent": genericUserAgent
+        },
+        dispatcher
+    });
+
+    return result;
+}
+
 const requestTweet = async(dispatcher, tweetId, token, cookie) => {
     const graphqlTweetURL = new URL(graphqlURL);
 
@@ -87,7 +111,7 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => {
     let result = await fetch(graphqlTweetURL, { headers, dispatcher });
     updateCookie(cookie, result.headers);
 
-    // we might have been missing the `ct0` cookie, retry
+    // we might have been missing the ct0 cookie, retry
     if (result.status === 403 && result.headers.get('set-cookie')) {
         const cookieValues = cookie?.values();
         if (cookieValues?.ct0) {
@@ -104,12 +128,31 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => {
     return result
 }
 
+const testResponse = (result) => {
+    const contentLength = result.headers.get("content-length");
+
+    if (!contentLength || contentLength === '0') {
+        return false;
+    }
+
+    if (!result.headers.get("content-type").startsWith("application/json")) {
+        return false;
+    }
+
+    return true;
+}
+
 export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
     const cookie = await getCookie('twitter');
 
+    let syndication = false;
+
     let guestToken = await getGuestToken(dispatcher);
     if (!guestToken) return { error: "fetch.fail" };
 
+    // for now we assume that graphql api will come back after some time,
+    // so we try it first
+
     let tweet = await requestTweet(dispatcher, id, guestToken);
 
     // get new token & retry if old one expired
@@ -122,48 +165,65 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
         }
     }
 
-    const contentLength = tweet.headers.get("content-length");
+    const testGraphql = testResponse(tweet);
 
-    if (!contentLength || tweet.headers.get("content-length") === '0') {
+    // if graphql requests fail, then resort to tweet embed api
-        return { error: "content.post.unavailable" }
+    if (!testGraphql) {
+        syndication = true;
+        tweet = await requestSyndication(dispatcher, id);
+
+        const testSyndication = testResponse(tweet);
+
+        // if even syndication request failed, then cry out loud
+        if (!testSyndication) {
+            return { error: "fetch.fail" };
+        }
     }
 
     tweet = await tweet.json();
 
-    let tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
+    let media;
 
-    if (!tweetTypename) {
+    if (!syndication) {
-        return { error: "fetch.empty" }
+        let tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
-    }
 
-    if (tweetTypename === "TweetUnavailable") {
+        if (!tweetTypename) {
-        const reason = tweet?.data?.tweetResult?.result?.reason;
+            return { error: "fetch.empty" }
-        switch(reason) {
-            case "Protected":
-                return { error: "content.post.private" }
-            case "NsfwLoggedOut":
-                if (cookie) {
-                    tweet = await requestTweet(dispatcher, id, guestToken, cookie);
-                    tweet = await tweet.json();
-                    tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
-                } else return { error: "content.post.age" }
         }
+
+        if (tweetTypename === "TweetUnavailable") {
+            const reason = tweet?.data?.tweetResult?.result?.reason;
+            switch(reason) {
+                case "Protected":
+                    return { error: "content.post.private" }
+                case "NsfwLoggedOut":
+                    if (cookie) {
+                        tweet = await requestTweet(dispatcher, id, guestToken, cookie);
+                        tweet = await tweet.json();
+                        tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
+                    } else return { error: "content.post.age" }
+            }
+        }
+
+        if (!["Tweet", "TweetWithVisibilityResults"].includes(tweetTypename)) {
+            return { error: "content.post.unavailable" }
+        }
+
+        let tweetResult = tweet.data.tweetResult.result,
+            baseTweet = tweetResult.legacy,
+            repostedTweet = baseTweet?.retweeted_status_result?.result.legacy.extended_entities;
+
+        if (tweetTypename === "TweetWithVisibilityResults") {
+            baseTweet = tweetResult.tweet.legacy;
+            repostedTweet = baseTweet?.retweeted_status_result?.result.tweet.legacy.extended_entities;
+        }
+
+        media = (repostedTweet?.media || baseTweet?.extended_entities?.media);
+    } else {
+        media = tweet.mediaDetails;
     }
 
-    if (!["Tweet", "TweetWithVisibilityResults"].includes(tweetTypename)) {
+    if (!media) return { error: "fetch.empty" }
-        return { error: "content.post.unavailable" }
-    }
-
-    let tweetResult = tweet.data.tweetResult.result,
-        baseTweet = tweetResult.legacy,
-        repostedTweet = baseTweet?.retweeted_status_result?.result.legacy.extended_entities;
-
-    if (tweetTypename === "TweetWithVisibilityResults") {
-        baseTweet = tweetResult.tweet.legacy;
-        repostedTweet = baseTweet?.retweeted_status_result?.result.tweet.legacy.extended_entities;
-    }
-
-    let media = (repostedTweet?.media || baseTweet?.extended_entities?.media);
 
     // check if there's a video at given index (/video/<index>)
     if (index >= 0 && index < media?.length) {