From 27d872363dc57d22a62f05e0f3c0448c0926db73 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 15 Aug 2023 14:37:59 +0600 Subject: [PATCH] graphql twitter api & soundcloud fix closes #127 --- src/modules/pageRender/page.js | 2 +- src/modules/processing/match.js | 2 +- src/modules/processing/services/soundcloud.js | 19 ++++--- src/modules/processing/services/twitter.js | 49 +++++++++++++------ .../processing/services/twitter_lite.js | 39 --------------- src/test/tests.json | 12 +---- 6 files changed, 50 insertions(+), 73 deletions(-) delete mode 100644 src/modules/processing/services/twitter_lite.js diff --git a/src/modules/pageRender/page.js b/src/modules/pageRender/page.js index 5e02280..640745a 100644 --- a/src/modules/pageRender/page.js +++ b/src/modules/pageRender/page.js @@ -502,7 +502,7 @@ export default function(obj) { ${urgentNotice({ emoji: "🐱", text: "report any issues!", - visible: true, + visible: false, action: "popup('about', 1, 'changelog')" })}
diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index a8732ab..70a31a2 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -8,7 +8,7 @@ import matchActionDecider from "./matchActionDecider.js"; import bilibili from "./services/bilibili.js"; import reddit from "./services/reddit.js"; -import twitter from "./services/twitter_lite.js"; +import twitter from "./services/twitter.js"; import youtube from "./services/youtube.js"; import vk from "./services/vk.js"; import tiktok from "./services/tiktok.js"; diff --git a/src/modules/processing/services/soundcloud.js b/src/modules/processing/services/soundcloud.js index f285f56..12c2678 100644 --- a/src/modules/processing/services/soundcloud.js +++ b/src/modules/processing/services/soundcloud.js @@ -36,26 +36,31 @@ async function findClientID() { export default async function(obj) { let html; if (!obj.author && !obj.song && obj.shortLink) { - html = await fetch(`https://on.soundcloud.com/${obj.shortLink}/`).then((r) => { return r.status === 404 ? false : r.text() }).catch(() => { return false }); + html = await fetch(`https://on.soundcloud.com/${obj.shortLink}/`).then((r) => { + return r.status === 404 ? false : r.text() + }).catch(() => { return false }); } if (obj.author && obj.song) { - html = await fetch(`https://soundcloud.com/${obj.author}/${obj.song}${obj.accessKey ? `/s-${obj.accessKey}` : ''}`).then((r) => { return r.text() }).catch(() => { return false }); + html = await fetch( + `https://soundcloud.com/${obj.author}/${obj.song}${obj.accessKey ? `/s-${obj.accessKey}` : ''}` + ).then((r) => { + return r.text() + }).catch(() => { return false }); } if (!html) return { error: 'ErrorCouldntFetch' }; - if (!(html.includes('')[0]) + let json = JSON.parse(html.split('{"hydratable":"sound","data":')[1].split('}];')[0]); if (!json["media"]["transcodings"]) return { error: 'ErrorEmptyDownload' }; let clientId = await findClientID(); if (!clientId) return { error: 'ErrorSoundCloudNoClientId' }; - let fileUrlBase = json.media.transcodings[0]["url"].replace("/hls", "/progressive"), + let fileUrlBase = json.media.transcodings.filter((v) => { if (v["format"]["mime_type"].startsWith("audio/ogg")) return true })[0]["url"], fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`; + if (fileUrl.substring(0, 54) !== "https://api-v2.soundcloud.com/media/soundcloud:tracks:") return { error: 'ErrorEmptyDownload' }; if (json.duration > maxVideoDuration) return { error: ['ErrorLengthAudioConvert', maxVideoDuration / 60000] }; diff --git a/src/modules/processing/services/twitter.js b/src/modules/processing/services/twitter.js index 3d53b82..189a0c8 100644 --- a/src/modules/processing/services/twitter.js +++ b/src/modules/processing/services/twitter.js @@ -1,9 +1,8 @@ import { genericUserAgent } from "../../config.js"; function bestQuality(arr) { - return arr.filter((v) => { if (v["content_type"] === "video/mp4") return true }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate))[0]["url"].split("?")[0] + return arr.filter((v) => { if (v["content_type"] === "video/mp4") return true }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate))[0]["url"] } -const apiURL = "https://api.twitter.com" export default async function(obj) { let _headers = { @@ -12,10 +11,12 @@ export default async function(obj) { "host": "api.twitter.com", "x-twitter-client-language": "en", "x-twitter-active-user": "yes", - "Accept-Language": "en" + "accept-language": "en" }; - let conversationURL = `${apiURL}/2/timeline/conversation/${obj.id}.json?cards_platform=Web-12&tweet_mode=extended&include_cards=1&include_ext_media_availability=true&include_ext_sensitive_media_warning=true&simple_quoted_tweet=true&trim_user=1`; - let activateURL = `${apiURL}/1.1/guest/activate.json`; + + let activateURL = `https://api.twitter.com/1.1/guest/activate.json`; + let graphqlTweetURL = `https://twitter.com/i/api/graphql/0hWvDhmW8YQ-S_ib3azIrw/TweetResultByRestId`; + let graphqlSpaceURL = `https://twitter.com/i/api/graphql/Gdz2uCtmIGMmhjhHG3V7nA/AudioSpaceById`; let req_act = await fetch(activateURL, { method: "POST", @@ -23,23 +24,39 @@ export default async function(obj) { }).then((r) => { return r.status === 200 ? r.json() : false }).catch(() => { return false }); if (!req_act) return { error: 'ErrorCouldntFetch' }; + _headers["host"] = "twitter.com"; + _headers["content-type"] = "application/json"; + _headers["x-guest-token"] = req_act["guest_token"]; - _headers["cookie"] = `guest_id=v1%3A${req_act["guest_token"]};`; + _headers["cookie"] = `guest_id=v1%3A${req_act["guest_token"]}`; - if (!obj.spaceId) { - let conversation = await fetch(conversationURL, { headers: _headers }).then((r) => { return r.status === 200 ? r.json() : false }).catch((e) => { return false }); - if (!conversation || !conversation.globalObjects.tweets[obj.id]) return { error: 'ErrorTweetUnavailable' }; + if (obj.id) { + let query = { + variables: {"tweetId": obj.id, "withCommunity": false, "includePromotedContent": false, "withVoice": false}, + features: {"creator_subscriptions_tweet_preview_api_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":false,"tweet_awards_web_tipping_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"responsive_web_media_download_video_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false} + } + query.variables = new URLSearchParams(JSON.stringify(query.variables)).toString().slice(0, -1); + query.features = new URLSearchParams(JSON.stringify(query.features)).toString().slice(0, -1); + query = `${graphqlTweetURL}?variables=${query.variables}&features=${query.features}`; - let baseMedia, baseTweet = conversation.globalObjects.tweets[obj.id]; - if (baseTweet.retweeted_status_id_str && conversation.globalObjects.tweets[baseTweet.retweeted_status_id_str].extended_entities) { - baseMedia = conversation.globalObjects.tweets[baseTweet.retweeted_status_id_str].extended_entities + let TweetResultByRestId = await fetch(query, { headers: _headers }).then((r) => { return r.status === 200 ? r.json() : false }).catch((e) => { return false }); + + // {"data":{"tweetResult":{"result":{"__typename":"TweetUnavailable","reason":"Protected"}}}} + if (!TweetResultByRestId || TweetResultByRestId.data.tweetResult.result.__typename !== "Tweet") return { error: 'ErrorTweetUnavailable' }; + + let baseMedia, + baseTweet = TweetResultByRestId.data.tweetResult.result.legacy; + + if (baseTweet.retweeted_status_result && baseTweet.retweeted_status_result.result.legacy.extended_entities.media) { + baseMedia = baseTweet.retweeted_status_result.result.legacy.extended_entities } else if (baseTweet.extended_entities && baseTweet.extended_entities.media) { baseMedia = baseTweet.extended_entities } if (!baseMedia) return { error: 'ErrorNoVideosInTweet' }; let single, multiple = [], media = baseMedia["media"]; - media = media.filter((i) => { if (i["type"] === "video" || i["type"] === "animated_gif") return true }) + media = media.filter((i) => { if (i["type"] === "video" || i["type"] === "animated_gif") return true }); + if (media.length > 1) { for (let i in media) { multiple.push({type: "video", thumb: media[i]["media_url_https"], url: bestQuality(media[i]["video_info"]["variants"])}) } } else if (media.length === 1) { @@ -55,7 +72,9 @@ export default async function(obj) { } else { return { error: 'ErrorNoVideosInTweet' } } - } else { + } + // spaces no longer work with guest authorization + if (obj.spaceId) { _headers["host"] = "twitter.com"; _headers["content-type"] = "application/json"; @@ -65,7 +84,7 @@ export default async function(obj) { } query.variables = new URLSearchParams(JSON.stringify(query.variables)).toString().slice(0, -1); query.features = new URLSearchParams(JSON.stringify(query.features)).toString().slice(0, -1); - query = `https://twitter.com/i/api/graphql/Gdz2uCtmIGMmhjhHG3V7nA/AudioSpaceById?variables=${query.variables}&features=${query.features}`; + query = `${graphqlSpaceURL}?variables=${query.variables}&features=${query.features}`; let AudioSpaceById = await fetch(query, { headers: _headers }).then((r) => {return r.status === 200 ? r.json() : false}).catch((e) => { return false }); if (!AudioSpaceById) return { error: 'ErrorEmptyDownload' }; diff --git a/src/modules/processing/services/twitter_lite.js b/src/modules/processing/services/twitter_lite.js deleted file mode 100644 index 1cebff1..0000000 --- a/src/modules/processing/services/twitter_lite.js +++ /dev/null @@ -1,39 +0,0 @@ -function bestQuality(arr) { - return arr.filter((v) => { if (v["content_type"] === "video/mp4") return true }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate))[0]["url"] -} - -export default async function(obj) { - if (!obj.spaceId) { - let synd = await fetch(`https://cdn.syndication.twimg.com/tweet-result?id=${obj.id}`, { - headers: { - "User-Agent": "Googlebot/2.1 (+http://www.google.com/bot.html)" - } - }).then((r) => { return r.status === 200 ? r.text() : false }).catch((e) => { return false }); - if (!synd) { - return { error: 'ErrorTweetUnavailable' } - } else { - synd = JSON.parse(synd); - } - if (!synd.mediaDetails) return { error: 'ErrorNoVideosInTweet' }; - - let single, multiple = [], media = synd.mediaDetails; - media = media.filter((i) => { if (i["type"] === "video" || i["type"] === "animated_gif") return true }) - if (media.length > 1) { - for (let i in media) { multiple.push({type: "video", thumb: media[i]["media_url_https"], url: bestQuality(media[i]["video_info"]["variants"])}) } - } else if (media.length === 1) { - single = bestQuality(media[0]["video_info"]["variants"]) - } else { - return { error: 'ErrorNoVideosInTweet' } - } - - if (single) { - return { urls: single, filename: `twitter_${obj.id}.mp4`, audioFilename: `twitter_${obj.id}_audio` } - } else if (multiple) { - return { picker: multiple } - } else { - return { error: 'ErrorNoVideosInTweet' } - } - } else { - return { error: 'ErrorTwitterRIP' } - } -} diff --git a/src/test/tests.json b/src/test/tests.json index 51e6b16..bfac3be 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -97,15 +97,7 @@ } }, { "name": "retweeted video", - "url": "https://twitter.com/winload_exe/status/1639005390854602758", - "params": {}, - "expected": { - "code": 200, - "status": "redirect" - } - }, { - "name": "retweeted video", - "url": "https://twitter.com/winload_exe/status/1639005390854602758", + "url": "https://twitter.com/hugekiwinuts/status/1618671150829309953?s=46&t=gItGzgwGQQJJaJrO6qc1Pg", "params": {}, "expected": { "code": 200, @@ -121,7 +113,7 @@ } }, { "name": "retweeted video, isAudioOnly", - "url": "https://twitter.com/winload_exe/status/1633091769482063874", + "url": "https://twitter.com/hugekiwinuts/status/1618671150829309953?s=46&t=gItGzgwGQQJJaJrO6qc1Pg", "params": { "aFormat": "mp3", "isAudioOnly": false,