From 1b4872c1debae156d796318dabb717d1f5211e41 Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 28 Jul 2022 22:03:17 +0600 Subject: [PATCH] tiktok is back! - added support for tiktok (images won't work, they're only accessible through the app) - hopefully main input bar is now not rounded on ios, i fucking hate apple - if service is not supported, a correlating error will appear, not generic one - removed duplicates from config that are present in package json already - tiny bit of clean up --- README.md | 26 ++++++++++---------------- package.json | 6 +++--- src/config.json | 3 --- src/modules/api.js | 5 +++-- src/modules/config.js | 7 ++++--- src/modules/match.js | 24 +++++++++++++++++++++--- src/modules/pageRender.js | 2 +- src/modules/services/_config.json | 4 ++-- src/modules/services/tiktok.js | 30 ++++++++++++++++++++++++++++++ src/modules/stream/types.js | 4 ++-- src/modules/sub/utils.js | 7 ++++++- src/static/cobalt.css | 9 ++++----- 12 files changed, 86 insertions(+), 41 deletions(-) create mode 100644 src/modules/services/tiktok.js diff --git a/README.md b/README.md index 7cb6fcc..281ee69 100644 --- a/README.md +++ b/README.md @@ -4,53 +4,47 @@ Sleek and easy to use social media downloader built on JavaScript. Try it out li ![cobalt logo](https://raw.githubusercontent.com/wukko/cobalt/current/src/static/icons/wide.png "cobalt logo") ## What is cobalt? -Everyone is annoyed by the mess video downloaders are on the web, and cobalt aims to be the ultimate social media downloader, that is efficient, pretty, and doesn't bother you with ads or privacy invasion agreement popups. +cobalt aims to be the ultimate social media downloader, that is efficient, pretty, and doesn't bother you with ads or privacy invasion agreement popups. cobalt doesn't remux any videos, so you get videos of max quality available (unless you change that in settings). ## What's supported? - Twitter +- TikTok - YouTube and YouTube Music - bilibili.com - Reddit - VK -## What still has to be done +## TO-DO +- [ ] Instagram support - [ ] Quality switching for bilibili and Twitter - [ ] Language picker in settings -- [x] Clean up the mess that localisation is right now - - [x] Sort contents of .json files - - [x] Rename each entry key to be less linked to specific service (entries like youtubeBroke are awful, I'm sorry) -- [x] Add support for more languages when localisation clean up is done - [ ] Use esmbuild to minify frontend css and js - [ ] Make switch buttons in settings selectable with keyboard -- [ ] Do something about changelog because the way it is right now is not really great - [ ] Remake page rendering module to be more versatile - [ ] Matching could be redone, I'll see what I can do -- [ ] Facebook and Instagram support -- [ ] TikTok support (?) -- [ ] Support for bilibili.tv (?) ## Disclaimer This is my passion project, so update scheduele depends on my motivation. Don't expect any consistency in that. ## Host an instance yourself -Code might be a little messy, but I promise to improve it over time. +Code might be a little messy, but I do my best to improve it with every commit. ### Requirements - Node.js 14.16 or above - git ### npm modules -- express - cors -- got -- url-pattern -- xml-js - dotenv +- express - express-rate-limit - ffmpeg-static +- got - node-cache +- url-pattern +- xml-js - ytdl-core Setup script installs all needed **npm** dependencies, but you have to install Node.js and git yourself, if you don't have those already. @@ -61,4 +55,4 @@ Setup script installs all needed **npm** dependencies, but you have to install N 4. Done. ## License -cobalt is under [GPL-3.0 license](https://github.com/wukko/cobalt/blob/current/LICENSE), please keep that in mind. +cobalt is under [AGPL-3.0 license](https://github.com/wukko/cobalt/blob/current/LICENSE), please keep that in mind. diff --git a/package.json b/package.json index dac8475..888d55e 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,9 @@ { "name": "cobalt", "description": "probably the friendliest social media downloader yet", - "version": "2.2.5", + "version": "2.2.6", "author": "wukko", - "exports": "./cobalt.js", + "exports": "./src/cobalt.js", "type": "module", "engines": { "node": ">=14.16" @@ -29,7 +29,7 @@ "ffmpeg-static": "^5.0.0", "got": "^12.1.0", "node-cache": "^5.1.2", - "url-pattern": "^1.0.3", + "url-pattern": "1.0.3", "xml-js": "^1.6.11", "ytdl-core": "4.11.0" } diff --git a/src/config.json b/src/config.json index 75120ce..4b0885b 100644 --- a/src/config.json +++ b/src/config.json @@ -1,10 +1,7 @@ { - "appName": "cobalt", - "version": "2.2.5", "streamLifespan": 1800000, "maxVideoDuration": 1920000, "genericUserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36", - "repo": "https://github.com/wukko/cobalt", "authorInfo": { "name": "wukko", "link": "https://wukko.me/", diff --git a/src/modules/api.js b/src/modules/api.js index 056d7f7..740857f 100644 --- a/src/modules/api.js +++ b/src/modules/api.js @@ -21,11 +21,12 @@ export async function getJSON(originalURL, ip, lang, format, quality) { if (host && host.length < 20 && host in patterns && patterns[host]["enabled"]) { for (let i in patterns[host]["patterns"]) { patternMatch = new UrlPattern(patterns[host]["patterns"][i]).match(cleanURL(url, host).split(".com/")[1]); + if (patternMatch) break; } if (patternMatch) { return await match(host, patternMatch, url, ip, lang, format, quality); - } else throw Error() - } else throw Error() + } return apiJSON(0, { t: errorUnsupported(lang) } ) + } return apiJSON(0, { t: errorUnsupported(lang) } ) } else { return apiJSON(0, { t: errorUnsupported(lang) } ) } diff --git a/src/modules/config.js b/src/modules/config.js index 0cf9b81..da7f708 100644 --- a/src/modules/config.js +++ b/src/modules/config.js @@ -1,14 +1,15 @@ import loadJson from "./sub/loadJSON.js"; const config = loadJson("./src/config.json"); +const packageJson = loadJson("./package.json"); export const services = loadJson("./src/modules/services/_config.json"), -appName = config.appName, -version = config.version, +appName = packageJson.name, +version = packageJson.version, streamLifespan = config.streamLifespan, maxVideoDuration = config.maxVideoDuration, genericUserAgent = config.genericUserAgent, -repo = config.repo, +repo = packageJson["bugs"]["url"].replace('/issues', ''), authorInfo = config.authorInfo, supportedLanguages = config.supportedLanguages, quality = config.quality, diff --git a/src/modules/match.js b/src/modules/match.js index 3df9149..f2da4f1 100644 --- a/src/modules/match.js +++ b/src/modules/match.js @@ -6,6 +6,7 @@ import reddit from "./services/reddit.js"; import twitter from "./services/twitter.js"; import youtube from "./services/youtube.js"; import vk from "./services/vk.js"; +import tiktok from "./services/tiktok.js"; export default async function (host, patternMatch, url, ip, lang, format, quality) { try { @@ -19,13 +20,16 @@ export default async function (host, patternMatch, url, ip, lang, format, qualit return (!r.error) ? apiJSON(1, { u: r.split('?')[0] }) : apiJSON(0, { t: r.error }) } else throw Error() case "vk": - if (patternMatch["userId"] && patternMatch["videoId"] && patternMatch["userId"].length <= 10 && patternMatch["videoId"].length == 9) { + if (patternMatch["userId"] && patternMatch["videoId"] && + patternMatch["userId"].length <= 10 && patternMatch["videoId"].length == 9) { let r = await vk({ userId: patternMatch["userId"], videoId: patternMatch["videoId"], lang: lang, quality: quality }); - return (!r.error) ? apiJSON(2, { type: "bridge", lang: lang, u: r.url, filename: r.filename, service: host, ip: ip, salt: process.env.streamSalt }) : apiJSON(0, { t: r.error }); + return (!r.error) ? apiJSON(2, + { type: "bridge", lang: lang, u: r.url, filename: + r.filename, service: host, ip: ip, salt: process.env.streamSalt }) : apiJSON(0, { t: r.error }); } else throw Error() case "bilibili": if (patternMatch["id"] && patternMatch["id"].length >= 12) { @@ -69,7 +73,8 @@ export default async function (host, patternMatch, url, ip, lang, format, qualit }) : apiJSON(0, { t: r.error }); } else throw Error() case "reddit": - if (patternMatch["sub"] && patternMatch["id"] && patternMatch["title"] && patternMatch["sub"].length <= 22 && patternMatch["id"].length <= 10 && patternMatch["title"].length <= 96) { + if (patternMatch["sub"] && patternMatch["id"] && patternMatch["title"] && + patternMatch["sub"].length <= 22 && patternMatch["id"].length <= 10 && patternMatch["title"].length <= 96) { let r = await reddit({ sub: patternMatch["sub"], id: patternMatch["id"], @@ -81,6 +86,19 @@ export default async function (host, patternMatch, url, ip, lang, format, qualit filename: r.filename, salt: process.env.streamSalt }) : apiJSON(0, { t: r.error }); } else throw Error() + case "tiktok": + if ((patternMatch["user"] && patternMatch["type"] == "video" && patternMatch["postId"] && patternMatch["postId"].length <= 21) || + (patternMatch["id"] && patternMatch["id"].length <= 13)) { + let r = await tiktok({ + postId: patternMatch["postId"], + id: patternMatch["id"], lang: lang, + }); + return (!r.error) ? apiJSON(2, { + type: "bridge", u: r.urls, lang: lang, + service: host, ip: ip, + filename: r.filename, salt: process.env.streamSalt + }) : apiJSON(0, { t: r.error }); + } else throw Error() default: return apiJSON(0, { t: errorUnsupported(lang) }) } diff --git a/src/modules/pageRender.js b/src/modules/pageRender.js index 342affe..274f8b9 100644 --- a/src/modules/pageRender.js +++ b/src/modules/pageRender.js @@ -163,7 +163,7 @@ export default function(obj) {
${loc(obj.lang, 'DownloadPopupDescription')}
- ${loc(obj.lang, 'Download')} + ${loc(obj.lang, 'Download')}
${loc(obj.lang, 'CopyURL')}
diff --git a/src/modules/services/_config.json b/src/modules/services/_config.json index 94a89e9..291008e 100644 --- a/src/modules/services/_config.json +++ b/src/modules/services/_config.json @@ -66,7 +66,7 @@ "enabled": false }, "tiktok": { - "patterns": [":pageid/:type/:postid", ":id"], - "enabled": false + "patterns": [":user/:type/:postId", ":id"], + "enabled": true } } \ No newline at end of file diff --git a/src/modules/services/tiktok.js b/src/modules/services/tiktok.js new file mode 100644 index 0000000..427b0f7 --- /dev/null +++ b/src/modules/services/tiktok.js @@ -0,0 +1,30 @@ +import got from "got"; +import loc from "../../localization/manager.js"; +import { genericUserAgent} from "../config.js"; +import { unicodeDecode } from "../sub/utils.js"; + +export default async function(obj) { + try { + if (!obj.postId) { + let html = await got.get(`https://vt.tiktok.com/${obj.id}`, { headers: { "user-agent": genericUserAgent } }); + html.on('error', (err) => { + return { error: loc(obj.lang, 'ErrorCantConnectToServiceAPI', 'tiktok') }; + }); + html = html.body + obj.postId = html.split('video/')[1].split('?')[0] + } + let url = `https://tiktok.com/@video/video/${obj.postId}` + let html = await got.get(url, { headers: { "user-agent": genericUserAgent } }); + html.on('error', (err) => { + return { error: loc(obj.lang, 'ErrorCantConnectToServiceAPI', 'tiktok') }; + }); + html = html.body; + if (html.includes(',"preloadList":[{"url":"')) { + return { urls: unicodeDecode(html.split(',"preloadList":[{"url":"')[1].split('","id":"')[0].trim()), filename: `tiktok_${obj.postId}.mp4` }; + } else { + return { error: loc(obj.lang, 'ErrorEmptyDownload') }; + } + } catch (e) { + return { error: loc(obj.lang, 'ErrorBadFetch') }; + } +} diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index fc68c55..d910e25 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -16,10 +16,10 @@ export async function streamDefault(streamInfo, res) { isStream: true }); stream.pipe(res).on('error', (err) => { - throw Error("File stream pipe error."); + internalError(res); }); stream.on('error', (err) => { - throw Error("File stream error.") + internalError(res); }); } catch (e) { internalError(res); diff --git a/src/modules/sub/utils.js b/src/modules/sub/utils.js index 2286291..d683860 100644 --- a/src/modules/sub/utils.js +++ b/src/modules/sub/utils.js @@ -35,7 +35,7 @@ export function msToTime(d) { return r; } export function cleanURL(url, host) { - url = url.replace('}', '').replace('{', '').replace(')', '').replace('(', '').replace(' ', ''); + url = url.replace('}', '').replace('{', '').replace(')', '').replace('(', '').replace(' ', '').replace('@', ''); if (url.includes('youtube.com/shorts/')) { url = url.split('?')[0].replace('shorts/', 'watch?v='); } @@ -51,4 +51,9 @@ export function cleanURL(url, host) { } export function languageCode(req) { return req.header('Accept-Language') ? req.header('Accept-Language').slice(0, 2) : "en" +} +export function unicodeDecode(str) { + return str.replace(/\\u[\dA-F]{4}/gi, (unicode) => { + return String.fromCharCode(parseInt(unicode.replace(/\\u/g, ""), 16)); + }); } \ No newline at end of file diff --git a/src/static/cobalt.css b/src/static/cobalt.css index b2513fe..b1bb3fe 100644 --- a/src/static/cobalt.css +++ b/src/static/cobalt.css @@ -93,8 +93,10 @@ button { color: var(--accent); font-size: 0.9rem; } -input { - border-radius: none; +input, +input[type="text"], +[type="text"] { + border-radius: 0; } button:hover, .switch:hover, @@ -232,9 +234,6 @@ input[type="checkbox"] { font-size: 0.9rem; max-height: 80%; } -.popup-big { - width: 55%; -} #popup-backdrop { opacity: 0.5; background-color: var(--background);