diff --git a/api/src/processing/services/twitter.js b/api/src/processing/services/twitter.js index 8fd26ee3..36ab2470 100644 --- a/api/src/processing/services/twitter.js +++ b/api/src/processing/services/twitter.js @@ -24,6 +24,11 @@ const badContainerEnd = new Date(1702605600000); function needsFixing(media) { const representativeId = media.source_status_id_str ?? media.id_str; + + // syndication api doesn't have media ids in its response, + // so we just assume it's all good + if (!representativeId) return false; + const mediaTimestamp = new Date( Number((BigInt(representativeId) >> 22n) + TWITTER_EPOCH) ); @@ -53,6 +58,25 @@ const getGuestToken = async (dispatcher, forceReload = false) => { } } +const requestSyndication = async(dispatcher, tweetId) => { + // thank you + // https://github.com/yt-dlp/yt-dlp/blob/05c8023a27dd37c49163c0498bf98e3e3c1cb4b9/yt_dlp/extractor/twitter.py#L1334 + const token = (id) => ((Number(id) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, ''); + const syndicationUrl = new URL("https://cdn.syndication.twimg.com/tweet-result"); + + syndicationUrl.searchParams.set("id", tweetId); + syndicationUrl.searchParams.set("token", token(tweetId)); + + const result = await fetch(syndicationUrl, { + headers: { + "user-agent": genericUserAgent + }, + dispatcher + }); + + return result; +} + const requestTweet = async(dispatcher, tweetId, token, cookie) => { const graphqlTweetURL = new URL(graphqlURL); @@ -87,7 +111,7 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => { let result = await fetch(graphqlTweetURL, { headers, dispatcher }); updateCookie(cookie, result.headers); - // we might have been missing the `ct0` cookie, retry + // we might have been missing the ct0 cookie, retry if (result.status === 403 && result.headers.get('set-cookie')) { const cookieValues = cookie?.values(); if (cookieValues?.ct0) { @@ -104,12 +128,31 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => { return result } +const testResponse = (result) => { + const contentLength = result.headers.get("content-length"); + + if (!contentLength || contentLength === '0') { + return false; + } + + if (!result.headers.get("content-type").startsWith("application/json")) { + return false; + } + + return true; +} + export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { const cookie = await getCookie('twitter'); + let syndication = false; + let guestToken = await getGuestToken(dispatcher); if (!guestToken) return { error: "fetch.fail" }; + // for now we assume that graphql api will come back after some time, + // so we try it first + let tweet = await requestTweet(dispatcher, id, guestToken); // get new token & retry if old one expired @@ -122,48 +165,65 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { } } - const contentLength = tweet.headers.get("content-length"); + const testGraphql = testResponse(tweet); - if (!contentLength || tweet.headers.get("content-length") === '0') { - return { error: "content.post.unavailable" } + // if graphql requests fail, then resort to tweet embed api + if (!testGraphql) { + syndication = true; + tweet = await requestSyndication(dispatcher, id); + + const testSyndication = testResponse(tweet); + + // if even syndication request failed, then cry out loud + if (!testSyndication) { + return { error: "fetch.fail" }; + } } tweet = await tweet.json(); - let tweetTypename = tweet?.data?.tweetResult?.result?.__typename; + let media; - if (!tweetTypename) { - return { error: "fetch.empty" } - } + if (!syndication) { + let tweetTypename = tweet?.data?.tweetResult?.result?.__typename; - if (tweetTypename === "TweetUnavailable") { - const reason = tweet?.data?.tweetResult?.result?.reason; - switch(reason) { - case "Protected": - return { error: "content.post.private" } - case "NsfwLoggedOut": - if (cookie) { - tweet = await requestTweet(dispatcher, id, guestToken, cookie); - tweet = await tweet.json(); - tweetTypename = tweet?.data?.tweetResult?.result?.__typename; - } else return { error: "content.post.age" } + if (!tweetTypename) { + return { error: "fetch.empty" } } + + if (tweetTypename === "TweetUnavailable") { + const reason = tweet?.data?.tweetResult?.result?.reason; + switch(reason) { + case "Protected": + return { error: "content.post.private" } + case "NsfwLoggedOut": + if (cookie) { + tweet = await requestTweet(dispatcher, id, guestToken, cookie); + tweet = await tweet.json(); + tweetTypename = tweet?.data?.tweetResult?.result?.__typename; + } else return { error: "content.post.age" } + } + } + + if (!["Tweet", "TweetWithVisibilityResults"].includes(tweetTypename)) { + return { error: "content.post.unavailable" } + } + + let tweetResult = tweet.data.tweetResult.result, + baseTweet = tweetResult.legacy, + repostedTweet = baseTweet?.retweeted_status_result?.result.legacy.extended_entities; + + if (tweetTypename === "TweetWithVisibilityResults") { + baseTweet = tweetResult.tweet.legacy; + repostedTweet = baseTweet?.retweeted_status_result?.result.tweet.legacy.extended_entities; + } + + media = (repostedTweet?.media || baseTweet?.extended_entities?.media); + } else { + media = tweet.mediaDetails; } - if (!["Tweet", "TweetWithVisibilityResults"].includes(tweetTypename)) { - return { error: "content.post.unavailable" } - } - - let tweetResult = tweet.data.tweetResult.result, - baseTweet = tweetResult.legacy, - repostedTweet = baseTweet?.retweeted_status_result?.result.legacy.extended_entities; - - if (tweetTypename === "TweetWithVisibilityResults") { - baseTweet = tweetResult.tweet.legacy; - repostedTweet = baseTweet?.retweeted_status_result?.result.tweet.legacy.extended_entities; - } - - let media = (repostedTweet?.media || baseTweet?.extended_entities?.media); + if (!media) return { error: "fetch.empty" } // check if there's a video at given index (/video/) if (index >= 0 && index < media?.length) {