From db5d62ae58e46a38cd86a3de6a2f88fef12fc137 Mon Sep 17 00:00:00 2001
From: wukko <me@wukko.me>
Date: Fri, 30 Jun 2023 16:29:21 +0600
Subject: [PATCH] 6.2.2: fixes related to twitter screwing everything up

(also fixes an issue with some instagram links)
---
 README.md                                     |  5 ++-
 package.json                                  |  2 +-
 src/localization/languages/en.json            |  3 +-
 src/localization/languages/ru.json            |  3 +-
 src/modules/pageRender/page.js                |  1 -
 src/modules/processing/match.js               |  2 +-
 src/modules/processing/services/twitter.js    | 18 ---------
 .../processing/services/twitter_lite.js       | 39 +++++++++++++++++++
 src/modules/processing/servicesConfig.json    |  2 +-
 src/modules/sub/utils.js                      |  2 +-
 10 files changed, 50 insertions(+), 27 deletions(-)
 create mode 100644 src/modules/processing/services/twitter_lite.js

diff --git a/README.md b/README.md
index 981b9acd..e077d345 100644
--- a/README.md
+++ b/README.md
@@ -23,8 +23,7 @@ Paste the link, get the video, move on. It's that simple. Just how it should be.
 | SoundCloud              | ➖           | ✅         | ➖         | Audio metadata, downloads from private links.                                                                             |
 | TikTok                  | ✅           | ✅         | ✅         | Supports downloads of: videos with or without watermark, images from slideshow without watermark, full (original) audios. |
 | Tumblr                  | ✅           | ✅         | ✅         | Support for audio file downloads.                                                                                         |
-| Twitter                 | ✅           | ✅         | ✅         | Ability to pick what to save from multi-media tweets.                                                                     |
-| Twitter Spaces          | ➖           | ✅         | ➖         | Audio metadata with all participants and other info.                                                                      |
+| Twitter*                | ✅           | ✅         | ✅         | Ability to pick what to save from multi-media tweets.                                                                     |
 | Vimeo                   | ✅           | ✅         | ✅         | Audio downloads are only available for dash files.                                                                        |
 | Vine Archive            | ✅           | ✅         | ✅         |                                                                                                                           |
 | VK Videos               | ✅           | ❌         | ❌         |                                                                                                                           |
@@ -33,6 +32,8 @@ Paste the link, get the video, move on. It's that simple. Just how it should be.
 | YouTube Music           | ➖           | ✅         | ➖         | Audio metadata.                                                                                                           |
 
 This list is not final and keeps expanding over time, make sure to check it once in a while!  
+  
+* Reliability of downloads from Twitter is questionable due its to current management.
 
 ## cobalt API
 cobalt has an open API that you can use in your projects for **free**.  
diff --git a/package.json b/package.json
index c84753af..12851718 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
 {
     "name": "cobalt",
     "description": "save what you love",
-    "version": "6.2",
+    "version": "6.2.2",
     "author": "wukko",
     "exports": "./src/cobalt.js",
     "type": "module",
diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json
index d39ddfbb..0945542d 100644
--- a/src/localization/languages/en.json
+++ b/src/localization/languages/en.json
@@ -120,6 +120,7 @@
         "SettingsVimeoPreferDescription": "progressive: direct file link to vimeo's cdn. max quality is 1080p.\ndash: video and audio are merged by {appName} into one file. max quality is 4k.\n\npick \"progressive\" if you want best editor/player/social media compatibility. if progressive download isn't available, dash is used instead.",
         "ShareURL": "share",
         "ErrorTweetUnavailable": "couldn't find anything about this tweet. this could be because its visibility is limited. try another one!",
-        "UrgentUpdate6": "all network issues have been fixed!"
+        "UrgentUpdate6": "all network issues have been fixed!",
+        "ErrorTwitterRIP": "twitter has restricted access to any content to unauthenticated users. while there's a way to get regular tweets, spaces are, unfortunately, impossible to get at this time. i am looking into possible solutions."
     }
 }
diff --git a/src/localization/languages/ru.json b/src/localization/languages/ru.json
index bfc4b5bd..5948535d 100644
--- a/src/localization/languages/ru.json
+++ b/src/localization/languages/ru.json
@@ -120,6 +120,7 @@
         "SettingsVimeoPreferDescription": "progressive: прямая ссылка на файл с сервера vimeo. максимальное качество: 1080p.\ndash: {appName} совмещает видео и аудио в один файл. максимальное качество: 4k.\n\nвыбирай \"progressive\", если тебе нужна наилучшая совместимость с плеерами/редакторами/соцсетями. если \"progressive\" файл недоступен, {appName} скачает \"dash\".",
         "ShareURL": "поделиться",
         "ErrorTweetUnavailable": "не смог найти что-либо об этом твите. возможно его видимость была ограничена. попробуй другой!",
-        "UrgentUpdate6": "на этот раз точно: всё работает!"
+        "UrgentUpdate6": "на этот раз точно: всё работает!",
+        "ErrorTwitterRIP": "твиттер ограничил доступ к любому контенту на сайте для пользователей без аккаунтов. я нашёл лазейку, чтобы доставать обычные твиты, но для spaces, к сожалению, нет. я ищу возможные варианты выхода из ситуации."
     }
 }
diff --git a/src/modules/pageRender/page.js b/src/modules/pageRender/page.js
index 48190c80..52082cef 100644
--- a/src/modules/pageRender/page.js
+++ b/src/modules/pageRender/page.js
@@ -364,7 +364,6 @@ export default function(obj) {
             body: `<div id="desc-error" class="desc-padding subtext"></div>`
         })}
         <div id="popup-backdrop" style="visibility: hidden;" onclick="hideAllPopups()"></div>
-        <div id="urgent-notice" class="urgent-notice explanation center" onclick="popup('about', 1, 'changelog')" style="visibility: hidden;">${emoji("🎉", 18)} ${t("UrgentUpdate6")}</div>
         <div id="cobalt-main-box" class="center" style="visibility: hidden;">
             <div id="logo">${appName}</div>
             <div id="download-area">
diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js
index 70a31a21..a8732abe 100644
--- a/src/modules/processing/match.js
+++ b/src/modules/processing/match.js
@@ -8,7 +8,7 @@ import matchActionDecider from "./matchActionDecider.js";
 
 import bilibili from "./services/bilibili.js";
 import reddit from "./services/reddit.js";
-import twitter from "./services/twitter.js";
+import twitter from "./services/twitter_lite.js";
 import youtube from "./services/youtube.js";
 import vk from "./services/vk.js";
 import tiktok from "./services/tiktok.js";
diff --git a/src/modules/processing/services/twitter.js b/src/modules/processing/services/twitter.js
index 6cd8b4fd..3d53b826 100644
--- a/src/modules/processing/services/twitter.js
+++ b/src/modules/processing/services/twitter.js
@@ -9,7 +9,6 @@ export default async function(obj) {
     let _headers = {
         "user-agent": genericUserAgent,
         "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
-        // ^ no explicit content, but with multi media support
         "host": "api.twitter.com",
         "x-twitter-client-language": "en",
         "x-twitter-active-user": "yes",
@@ -29,23 +28,6 @@ export default async function(obj) {
 
     if (!obj.spaceId) {
         let conversation = await fetch(conversationURL, { headers: _headers }).then((r) => { return r.status === 200 ? r.json() : false }).catch((e) => { return false });
-        if (!conversation || !conversation.globalObjects.tweets[obj.id]) {
-            _headers.authorization = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw";
-            // ^ explicit content, but no multi media support
-            delete _headers["x-guest-token"];
-            delete _headers["cookie"];
-
-            req_act = await fetch(activateURL, {
-                method: "POST",
-                headers: _headers
-            }).then((r) => { return r.status === 200 ? r.json() : false}).catch(() => { return false });
-            if (!req_act) return { error: 'ErrorCouldntFetch' };
-
-            _headers["x-guest-token"] = req_act["guest_token"];
-            _headers['cookie'] = `guest_id=v1%3A${req_act["guest_token"]};`;
-
-            conversation = await fetch(conversationURL, { headers: _headers }).then((r) => { return r.status === 200 ? r.json() : false }).catch(() => { return false });
-        }
         if (!conversation || !conversation.globalObjects.tweets[obj.id]) return { error: 'ErrorTweetUnavailable' };
 
         let baseMedia, baseTweet = conversation.globalObjects.tweets[obj.id];
diff --git a/src/modules/processing/services/twitter_lite.js b/src/modules/processing/services/twitter_lite.js
new file mode 100644
index 00000000..dbdf82c4
--- /dev/null
+++ b/src/modules/processing/services/twitter_lite.js
@@ -0,0 +1,39 @@
+function bestQuality(arr) {
+    return arr.filter((v) => { if (v["content_type"] === "video/mp4") return true }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate))[0]["url"].split("?")[0]
+}
+
+export default async function(obj) {
+    if (!obj.spaceId) {
+        let synd = await fetch(`https://cdn.syndication.twimg.com/tweet-result?id=${obj.id}`, {
+            headers: {
+                "User-Agent": "Googlebot/2.1 (+http://www.google.com/bot.html)"
+            }
+        }).then((r) => { return r.status === 200 ? r.text() : false }).catch((e) => { return false });
+        if (!synd) {
+            return { error: 'ErrorTweetUnavailable' }
+        } else {
+            synd = JSON.parse(synd);
+        }
+        if (!synd.mediaDetails) return { error: 'ErrorNoVideosInTweet' };
+
+        let single, multiple = [], media = synd.mediaDetails;
+        media = media.filter((i) => { if (i["type"] === "video" || i["type"] === "animated_gif") return true })
+        if (media.length > 1) {
+            for (let i in media) { multiple.push({type: "video", thumb: media[i]["media_url_https"], url: bestQuality(media[i]["video_info"]["variants"])}) }
+        } else if (media.length === 1) {
+            single = bestQuality(media[0]["video_info"]["variants"])
+        } else {
+            return { error: 'ErrorNoVideosInTweet' }
+        }
+
+        if (single) {
+            return { urls: single, filename: `twitter_${obj.id}.mp4`, audioFilename: `twitter_${obj.id}_audio` }
+        } else if (multiple) {
+            return { picker: multiple }
+        } else {
+            return { error: 'ErrorNoVideosInTweet' }
+        }
+    } else {
+        return { error: 'ErrorTwitterRIP' }
+    }
+}
diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json
index 1890edac..ddd798c5 100644
--- a/src/modules/processing/servicesConfig.json
+++ b/src/modules/processing/servicesConfig.json
@@ -12,7 +12,7 @@
             "enabled": true
         },
         "twitter": {
-            "alias": "twitter posts & spaces & voice",
+            "alias": "twitter posts & voice",
             "patterns": [":user/status/:id", ":user/status/:id/video/:v", "i/spaces/:spaceId"],
             "enabled": true
         },
diff --git a/src/modules/sub/utils.js b/src/modules/sub/utils.js
index d6fc80f4..1cb9a8de 100644
--- a/src/modules/sub/utils.js
+++ b/src/modules/sub/utils.js
@@ -62,7 +62,7 @@ export function msToTime(d) {
     return r;
 }
 export function cleanURL(url, host) {
-    let forbiddenChars = ['}', '{', '(', ')', '\\', '%', '>', '<', '^', '*', '!', '~', ';', ':', ',', '`', '[', ']', '#', '$', '"', "'", "@"]
+    let forbiddenChars = ['}', '{', '(', ')', '\\', '%', '>', '<', '^', '*', '!', '~', ';', ':', ',', '`', '[', ']', '#', '$', '"', "'", "@", '==']
     switch(host) {
         case "vk":
             url = url.includes('clip') ? url.split('&')[0] : url.split('?')[0];