From 20798685c20cf3a6aa5b1dac07e4f4eb4fe40aca Mon Sep 17 00:00:00 2001 From: wukko Date: Mon, 1 Aug 2022 23:53:44 +0600 Subject: [PATCH] added support for tumblr - support for tumblr is back! (i tried all types of links in clear sight and everything seems to work, lmk if something doesn't) - increased stream lifespan up to 1 hour - made youtube audios download without additional conversion, speed should be a bit better but not by much cause youtube limits the audio download speed :( --- README.md | 6 ++++-- src/config.json | 2 +- src/modules/api.js | 4 ++++ src/modules/match.js | 11 +++++++++++ src/modules/services/_config.json | 8 ++------ src/modules/services/tumblr.js | 24 ++++++++++++++++++++++++ src/modules/services/youtube.js | 2 +- src/modules/stream/stream.js | 2 +- src/modules/stream/types.js | 2 +- 9 files changed, 49 insertions(+), 12 deletions(-) create mode 100644 src/modules/services/tumblr.js diff --git a/README.md b/README.md index bdeb81a..94ed663 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ cobalt aims to be the ultimate social media downloader, that is efficient, prett - douyin - Reddit - TikTok +- Tumblr - Twitter - YouTube - YouTube Music @@ -45,16 +46,17 @@ Take English or Russian localization from [this directory](https://github.com/wu ## TO-DO ### Services -- [ ] Tumblr support +- [x] Tumblr support - [ ] niconico support - [ ] Instagram support -- [ ] SoundCloud support (?) +- [ ] SoundCloud support - [ ] Add an option to save Twitter GIFs as `.gif` instead of `.mp4` - [ ] Quality switching for bilibili ### Other - [ ] Language picker in settings - [ ] Make switch buttons in settings selectable with keyboard +- [ ] Option to save audios in formats other than original - [ ] Make cobalt fully PWA compatible (add a service worker) - [ ] Make page rendering module more versatile diff --git a/src/config.json b/src/config.json index 4b0885b..c6cab14 100644 --- a/src/config.json +++ b/src/config.json @@ -1,5 +1,5 @@ { - "streamLifespan": 1800000, + "streamLifespan": 3600000, "maxVideoDuration": 1920000, "genericUserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36", "authorInfo": { diff --git a/src/modules/api.js b/src/modules/api.js index 998d853..e35aa6d 100644 --- a/src/modules/api.js +++ b/src/modules/api.js @@ -18,6 +18,10 @@ export async function getJSON(originalURL, ip, lang, format, quality) { host = "youtube"; url = `https://youtube.com/watch?v=${url.replace("youtu.be/", "").replace("https://", "")}`; } + if (host == "tumblr" && !url.includes("blog/view")) { + if (url.slice(-1) == '/') url = url.slice(0, -1); + url = url.replace(url.split('/')[5], ''); + } if (host && host.length < 20 && host in patterns && patterns[host]["enabled"]) { for (let i in patterns[host]["patterns"]) { patternMatch = new UrlPattern(patterns[host]["patterns"][i]).match(cleanURL(url, host).split(".com/")[1]); diff --git a/src/modules/match.js b/src/modules/match.js index 31211d0..c6cae0a 100644 --- a/src/modules/match.js +++ b/src/modules/match.js @@ -8,6 +8,7 @@ import youtube from "./services/youtube.js"; import vk from "./services/vk.js"; import tiktok from "./services/tiktok.js"; import douyin from "./services/douyin.js"; +import tumblr from "./services/tumblr.js"; export default async function (host, patternMatch, url, ip, lang, format, quality) { try { @@ -113,6 +114,16 @@ export default async function (host, patternMatch, url, ip, lang, format, qualit filename: r.filename, salt: process.env.streamSalt }) : apiJSON(0, { t: r.error }); } else throw Error() + case "tumblr": + if ((patternMatch["id"] && patternMatch["id"].length < 21) || + (patternMatch["id"] && patternMatch["id"].length < 21 && + patternMatch["user"] && patternMatch["user"].length <= 32)) { + let r = await tumblr({ + id: patternMatch["id"], url: url, user: patternMatch["user"] ? patternMatch["user"] : false, + lang: lang + }); + return (!r.error) ? apiJSON(1, { u: r.split('?')[0] }) : apiJSON(0, { t: r.error }) + } else throw Error() default: return apiJSON(0, { t: errorUnsupported(lang) }) } diff --git a/src/modules/services/_config.json b/src/modules/services/_config.json index 9b443a4..e3345ef 100644 --- a/src/modules/services/_config.json +++ b/src/modules/services/_config.json @@ -54,12 +54,8 @@ "enabled": true }, "tumblr": { - "patterns": ["post/:id"], - "enabled": false - }, - "facebook": { - "patterns": [":pageid/:type/:postid"], - "enabled": false + "patterns": ["post/:id", "blog/view/:user/:id"], + "enabled": true }, "instagram": { "patterns": [":type/:id"], diff --git a/src/modules/services/tumblr.js b/src/modules/services/tumblr.js new file mode 100644 index 0000000..7940d19 --- /dev/null +++ b/src/modules/services/tumblr.js @@ -0,0 +1,24 @@ +import got from "got"; +import loc from "../../localization/manager.js"; +import { genericUserAgent } from "../config.js"; + +export default async function(obj) { + try { + let user = obj.user ? obj.user : obj.url.split('.')[0].replace('https://', ''); + if (user.length <= 32) { + let html = await got.get(`https://${user}.tumblr.com/post/${obj.id}`, { headers: { "user-agent": genericUserAgent } }); + html.on('error', (err) => { + return { error: loc(obj.lang, 'ErrorCouldntFetch', 'tumblr') }; + }); + html = html.body + if (html.includes('')[0]) + if (json["video"] && json["video"]["contentUrl"]) { + return json["video"]["contentUrl"] + } else return { error: loc(obj.lang, 'ErrorEmptyDownload') } + } else return { error: loc(obj.lang, 'ErrorBrokenLink', 'tumblr') } + } else return { error: loc(obj.lang, 'ErrorBrokenLink', 'tumblr') } + } catch (e) { + return { error: loc(obj.lang, 'ErrorBadFetch') }; + } +} diff --git a/src/modules/services/youtube.js b/src/modules/services/youtube.js index d2f3cd0..07edd41 100644 --- a/src/modules/services/youtube.js +++ b/src/modules/services/youtube.js @@ -54,7 +54,7 @@ export default async function (obj) { return { type: "render", urls: [video[0]["url"], audio[0]["url"]], time: video[0]["approxDurationMs"], filename: `youtube_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.${video[0]["container"]}` }; } else if (audio.length > 0) { - return { type: "render", isAudioOnly: true, urls: [audio[0]["url"]], filename: `youtube_${obj.id}_${audio[0]["audioBitrate"]}kbps.opus` }; + return { type: "bridge", isAudioOnly: true, urls: audio[0]["url"], filename: `youtube_${obj.id}_${audio[0]["audioBitrate"]}kbps.${audio[0]["container"] == "webm" ? "opus" : "m4a"}` }; } else { return { error: loc(obj.lang, 'ErrorBadFetch') }; } diff --git a/src/modules/stream/stream.js b/src/modules/stream/stream.js index dc1ffd2..188799b 100644 --- a/src/modules/stream/stream.js +++ b/src/modules/stream/stream.js @@ -6,7 +6,7 @@ export default function(res, ip, id, hmac, exp) { try { let streamInfo = verifyStream(ip, id, hmac, exp, process.env.streamSalt); if (!streamInfo.error) { - if (streamInfo.isAudioOnly) { + if (streamInfo.isAudioOnly && streamInfo.type == "render") { streamAudioOnly(streamInfo, res); } else { switch (streamInfo.type) { diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index edb8643..976e7e7 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -81,7 +81,7 @@ export async function streamAudioOnly(streamInfo, res) { if (streamInfo.service == "bilibili") { headers = { "user-agent": genericUserAgent }; } - const audio = got.get(streamInfo.urls[0], { isStream: true, headers: headers }); + const audio = got.get(streamInfo.urls, { isStream: true, headers: headers }); const ffmpegProcess = spawn(ffmpeg, [ '-loglevel', '-8', '-i', 'pipe:3',