From fa7af1bf444e3a473118ca5243380f32863d24cb Mon Sep 17 00:00:00 2001 From: Snazzah Date: Sat, 29 Apr 2023 14:33:36 -0500 Subject: [PATCH 01/32] feat: add twitch vod/clip support --- src/modules/processing/match.js | 10 + src/modules/processing/services/twitch.js | 223 ++++++++++++++++++ src/modules/processing/servicesConfig.json | 6 + .../processing/servicesPatternTesters.js | 4 +- src/modules/stream/types.js | 2 +- src/modules/sub/utils.js | 3 + src/test/tests.json | 57 +++++ 7 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 src/modules/processing/services/twitch.js diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 3328a32..aef1c17 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -17,6 +17,7 @@ import vimeo from "./services/vimeo.js"; import soundcloud from "./services/soundcloud.js"; import instagram from "./services/instagram.js"; import vine from "./services/vine.js"; +import twitch from "./services/twitch.js"; export default async function (host, patternMatch, url, lang, obj) { try { @@ -110,6 +111,15 @@ export default async function (host, patternMatch, url, lang, obj) { case "vine": r = await vine({ id: patternMatch["id"] }); break; + case "twitch": + r = await twitch({ + vodId: patternMatch["video"] ? patternMatch["video"] : false, + clipId: patternMatch["clip"] ? patternMatch["clip"] : false, + lang: lang, quality: obj.vQuality, + isAudioOnly: obj.isAudioOnly, + format: obj.vFormat + }); + break; default: return apiJSON(0, { t: errorUnsupported(lang) }); } diff --git a/src/modules/processing/services/twitch.js b/src/modules/processing/services/twitch.js new file mode 100644 index 0000000..bd8a869 --- /dev/null +++ b/src/modules/processing/services/twitch.js @@ -0,0 +1,223 @@ +import { maxVideoDuration } from "../../config.js"; + +const gqlURL = "https://gql.twitch.tv/gql"; +const m3u8URL = "https://usher.ttvnw.net"; + +function parseM3U8Line(line) { + const result = {}; + + let str = '', inQuotes = false, keyName = null, escaping = false; + for (let i = 0; i < line.length; i++) { + const char = line[i]; + if (char === '"' && !escaping) { + inQuotes = !inQuotes; + continue; + } else if (char === ',' && !escaping && !inQuotes) { + if (!keyName) break; + result[keyName] = str; + keyName = null; + str = ''; + continue; + } else if (char === '\\' && !escaping) { + escaping = true; + continue; + } else if (char === '=' && !escaping && !inQuotes) { + keyName = str; + str = ''; + continue; + } + + str += char; + escaping = false; + } + + if (keyName) result[keyName] = str; + return result; +} + +function getM3U8Formats(m3u8body) { + let formats = []; + const formatLines = m3u8body.split('\n').slice(2); + + for (let i = 0; i < formatLines.length; i += 3) { + const mediaLine = parseM3U8Line(formatLines[i].split(':')[1]); + const streamLine = parseM3U8Line(formatLines[i + 1].split(':')[1]); + formats.push({ + id: mediaLine['GROUP-ID'], + name: mediaLine.NAME, + resolution: streamLine.RESOLUTION ? streamLine.RESOLUTION.split('x') : null, + url: formatLines[i + 2] + }); + } + return formats; +}; + +export default async function(obj) { + try { + let _headers = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" }; + + if (!obj.clipId && !obj.vodId) return { error: 'ErrorCantGetID' }; + + if (obj.vodId) { + const req_metadata = await fetch(gqlURL, { + method: "POST", + headers: _headers, + body: JSON.stringify([ + { + "operationName": "VideoMetadata", + "variables": { + "channelLogin": "", + "videoID": obj.vodId + }, + "extensions": { + "persistedQuery": { + "version": 1, + "sha256Hash": "226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687" + } + } + } + ]) + }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); + if (!req_metadata) return { error: 'ErrorCouldntFetch' }; + const vodMetadata = req_metadata[0].data.video; + + if (vodMetadata.previewThumbnailURL.endsWith('/404_processing_{width}x{height}.png')) return { error: 'ErrorLiveVideo' }; + if (vodMetadata.lengthSeconds > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + if (!vodMetadata.owner) return { error: 'ErrorEmptyDownload' }; // Streamer was banned... + + const req_token = await fetch(gqlURL, { + method: "POST", + headers: _headers, + body: JSON.stringify({ + query: `{ + videoPlaybackAccessToken( + id: "${obj.vodId}", + params: { + platform: "web", + playerBackend: "mediaplayer", + playerType: "site" + } + ) + { + value + signature + } + }` + }) + }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); + if (!req_token) return { error: 'ErrorCouldntFetch' }; + + const access_token = req_token.data.videoPlaybackAccessToken; + const req_m3u8 = await fetch(`${m3u8URL}/vod/${obj.vodId}.m3u8?${new URLSearchParams({ + allow_source: 'true', + allow_audio_only: 'true', + allow_spectre: 'true', + player: 'twitchweb', + playlist_include_framerate: 'true', + nauth: access_token.value, + nauthsig: access_token.signature + })}`, { + headers: _headers + }).then((r) => { return r.status == 200 ? r.text() : false;}).catch(() => {return false}); + if (!req_m3u8) return { error: 'ErrorCouldntFetch' }; + + const formats = getM3U8Formats(req_m3u8); + const generalMeta = { + title: vodMetadata.title, + artist: `Twitch Broadcast by @${vodMetadata.owner.login}`, + } + + if (!obj.isAudioOnly) { + const format = formats.find(f => f.resolution && f.resolution[1] == obj.quality) || formats[0]; + + return { + urls: format.url, + isM3U8: true, + time: vodMetadata.lengthSeconds * 1000, + filename: `twitchvod_${obj.vodId}_${format.resolution[0]}x${format.resolution[1]}.mp4` + }; + } else { + return { + type: "render", + isM3U8: true, + time: vodMetadata.lengthSeconds * 1000, + urls: formats.find(f => f.id === 'audio_only').url, + audioFilename: `twitchvod_${obj.vodId}_audio`, + fileMetadata: generalMeta + } + } + } else if (obj.clipId) { + const req_metadata = await fetch(gqlURL, { + method: "POST", + headers: _headers, + body: JSON.stringify({ + query: `{ + clip(slug: "${obj.clipId}") { + broadcaster { + login + } + createdAt + curator { + login + } + durationSeconds + id + medium: thumbnailURL(width: 480, height: 272) + title + videoQualities { + quality + sourceURL + } + } + }` + }) + }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); + if (!req_metadata) return { error: 'ErrorCouldntFetch' }; + const clipMetadata = req_metadata.data.clip; + if (clipMetadata.durationSeconds > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + if (!clipMetadata.videoQualities || !clipMetadata.broadcaster) return { error: 'ErrorEmptyDownload' }; // Streamer was banned... + + const req_token = await fetch(gqlURL, { + method: "POST", + headers: _headers, + body: JSON.stringify([ + { + "operationName": "VideoAccessToken_Clip", + "variables": { + "slug": obj.clipId + }, + "extensions": { + "persistedQuery": { + "version": 1, + "sha256Hash": "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11" + } + } + } + ]) + }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); + if (!req_token) return { error: 'ErrorCouldntFetch' }; + + const generalMeta = { + title: clipMetadata.title, + artist: `Twitch Clip by @${clipMetadata.broadcaster.login}, clipped by @${clipMetadata.curator.login}`, + } + + const access_token = req_token[0].data.clip.playbackAccessToken; + const formats = clipMetadata.videoQualities; + const format = formats.find(f => f.quality == obj.quality) || formats[0]; + + return { + type: "bridge", + urls: `${format.sourceURL}?${new URLSearchParams({ + sig: access_token.signature, + token: access_token.value + })}`, + filename: `twitchclip_${clipMetadata.id}_${format.quality}.mp4`, + audioFilename: `twitchclip_${clipMetadata.id}_audio`, + fileMetadata: generalMeta + }; + } + } catch (err) { + return { error: 'ErrorBadFetch' }; + } +} \ No newline at end of file diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index 2bb7f8f..0318d4e 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -62,6 +62,12 @@ "tld": "co", "patterns": ["v/:id"], "enabled": true + }, + "twitch": { + "alias": "twitch vods & videos & clips", + "tld": "tv", + "patterns": ["videos/:video", ":channel/clip/:clip"], + "enabled": true } } } diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index 8f70613..35eaa90 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -28,5 +28,7 @@ export const testers = { "instagram": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length <= 12), - "vine": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length <= 12) + "vine": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length <= 12), + + "twitch": (patternMatch) => ((patternMatch["channel"] && patternMatch["clip"] && patternMatch["clip"].length <= 100 || patternMatch["video"] && patternMatch["video"].length <= 10)), } diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index a4eb233..3aa259f 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -145,7 +145,7 @@ export function streamVideoOnly(streamInfo, res) { '-c', 'copy' ] if (streamInfo.mute) args.push('-an'); - if (streamInfo.service === "vimeo") args.push('-bsf:a', 'aac_adtstoasc'); + if (streamInfo.service === "vimeo" || streamInfo.service === "twitch") args.push('-bsf:a', 'aac_adtstoasc'); if (format === "mp4") args.push('-movflags', 'faststart+frag_keyframe+empty_moov'); args.push('-f', format, 'pipe:3'); const ffmpegProcess = spawn(ffmpeg, args, { diff --git a/src/modules/sub/utils.js b/src/modules/sub/utils.js index 27a17b8..4bcfc6d 100644 --- a/src/modules/sub/utils.js +++ b/src/modules/sub/utils.js @@ -84,6 +84,9 @@ export function cleanURL(url, host) { if (url.includes('youtube.com/shorts/')) { url = url.split('?')[0].replace('shorts/', 'watch?v='); } + if (url.includes('clips.twitch.tv')) { + url = url.split('?')[0].replace('clips.twitch.tv/', 'twitch.tv/_/clip/'); + } return url.slice(0, 128) } export function verifyLanguageCode(code) { diff --git a/src/test/tests.json b/src/test/tests.json index 158a0ea..57fbefd 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -869,5 +869,62 @@ "code": 200, "status": "stream" } + }], + "twitch": [{ + "name": "clip", + "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", + "params": {}, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "clip (isAudioOnly)", + "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", + "params": { + "isAudioOnly": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "clip (isAudioMuted)", + "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", + "params": { + "isAudioMuted": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "video", + "url": "https://twitch.tv/videos/1315890970", + "params": {}, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "video (isAudioOnly)", + "url": "https://twitch.tv/videos/1315890970", + "params": { + "isAudioOnly": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "video (isAudioMuted)", + "url": "https://twitch.tv/videos/1315890970", + "params": { + "isAudioMuted": true + }, + "expected": { + "code": 200, + "status": "stream" + } }] } \ No newline at end of file From c8d0eb862cb00b6255776ae5cd7054db9b4dd951 Mon Sep 17 00:00:00 2001 From: Snazzah Date: Sat, 29 Apr 2023 17:13:26 -0500 Subject: [PATCH 02/32] chore: update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b1e636a..234ff4b 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ It's fast, friendly, and doesn't have any bullshit that modern web is filled wit | YouTube Music | ❌ | ✅ | Audio metadata. | | Reddit | ✅ | ✅ | GIFs and videos. | | TikTok | ✅ | ✅ | Video downloads with or without watermark; image slideshow downloads without watermark. Full audio downloads. | +| Twitch | ✅ | ✅ | | | SoundCloud | ❌ | ✅ | Audio metadata, downloads from private links. | | bilibili.com | ✅ | ✅ | | | Tumblr | ✅ | ✅ | | From 7ebe28ce50d3448a0cbc09b6d1eaa21812ebeb5d Mon Sep 17 00:00:00 2001 From: dumbmoron <136796770+dumbmoron@users.noreply.github.com> Date: Sun, 2 Jul 2023 00:40:47 +0200 Subject: [PATCH 03/32] create docker build github action --- .github/workflows/docker.yml | 52 ++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..96225d0 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,52 @@ +name: Build Docker image + +on: + push: + branches: ['current'] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Get version from package.json + id: package-version + uses: martinbeentjes/npm-get-version-action@v1.3.1 + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + tags: | + type=raw,value=latest + type=raw,value=${{ steps.package-version.outputs.current-version }} + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From 239db34c6752de36de892c51e9b6a5e8a133bb7f Mon Sep 17 00:00:00 2001 From: dumbmoron <136796770+dumbmoron@users.noreply.github.com> Date: Sun, 2 Jul 2023 05:21:08 +0200 Subject: [PATCH 04/32] remove support for armv6 no armv6 build of node:18-bullseye-slim, can be readded once this is fixed --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 96225d0..edfe0b5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -46,7 +46,7 @@ jobs: uses: docker/build-push-action@v4 with: context: . - platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 + platforms: linux/amd64,linux/arm64,linux/arm/v7 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From d2e5b6f1fbfefddea8f940d474613eee50065344 Mon Sep 17 00:00:00 2001 From: dumbmoron <136796770+dumbmoron@users.noreply.github.com> Date: Sun, 2 Jul 2023 05:24:28 +0200 Subject: [PATCH 05/32] add per-commit image tag --- .github/workflows/docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index edfe0b5..86ad0a7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -33,6 +33,9 @@ jobs: - name: Get version from package.json id: package-version uses: martinbeentjes/npm-get-version-action@v1.3.1 + - name: Get short commit hash + id: commit-hash + run: echo "commit_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v4 @@ -40,6 +43,7 @@ jobs: tags: | type=raw,value=latest type=raw,value=${{ steps.package-version.outputs.current-version }} + type=raw,value=${{ steps.package-version.outputs.current-version }}-${{ steps.commit-hash.outputs.commit_short }} images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Build and push Docker image From 93aa1f4db47a4f77a15baeb5d34ea5d4c946ba76 Mon Sep 17 00:00:00 2001 From: dumbmoron <136796770+dumbmoron@users.noreply.github.com> Date: Thu, 24 Aug 2023 08:31:39 +0000 Subject: [PATCH 06/32] add option to disable file metadata closes #142 --- docs/API.md | 1 + src/front/cobalt.js | 4 +++- src/localization/languages/en.json | 1 + src/localization/languages/ru.json | 1 + src/modules/pageRender/page.js | 4 ++++ src/modules/processing/match.js | 4 ++-- src/modules/processing/matchActionDecider.js | 6 +++--- src/modules/sub/utils.js | 5 +++-- 8 files changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/API.md b/docs/API.md index 7c2a5b3..69c5fb3 100644 --- a/docs/API.md +++ b/docs/API.md @@ -26,6 +26,7 @@ Response Body Type: ``application/json`` | isTTFullAudio | boolean | ``true / false`` | ``false`` | Enables download of original sound used in a TikTok video. | | isAudioMuted | boolean | ``true / false`` | ``false`` | Disables audio track in video downloads. | | dubLang | boolean | ``true / false`` | ``false`` | Backend uses Accept-Language for YouTube video audio tracks when ``true``. | +| disableMetadata | boolean | ``true / false`` | ``false`` | Does not add metadata to the downloaded audio/video | ### Response Body Variables | key | type | variables | diff --git a/src/front/cobalt.js b/src/front/cobalt.js index ca30951..754ed34 100644 --- a/src/front/cobalt.js +++ b/src/front/cobalt.js @@ -18,7 +18,7 @@ const switchers = { "vimeoDash": ["false", "true"], "audioMode": ["false", "true"] }; -const checkboxes = ["disableTikTokWatermark", "fullTikTokAudio", "muteAudio", "reduceTransparency", "disableAnimations"]; +const checkboxes = ["disableTikTokWatermark", "fullTikTokAudio", "muteAudio", "reduceTransparency", "disableAnimations", "disableMetadata"]; const exceptions = { // used for mobile devices "vQuality": "720" }; @@ -377,6 +377,8 @@ async function download(url) { if ((url.includes("tiktok.com/") || url.includes("douyin.com/")) && sGet("disableTikTokWatermark") === "true") req.isNoTTWatermark = true; } + if (sGet("disableMetadata") === "true") req.disableMetadata = true; + let j = await fetch(`${apiURL}/api/json`, { method: "POST", body: JSON.stringify(req), diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json index b087f99..ceb63ac 100644 --- a/src/localization/languages/en.json +++ b/src/localization/languages/en.json @@ -43,6 +43,7 @@ "SettingsKeepDownloadButton": "keep >> visible", "AccessibilityKeepDownloadButton": "keep the download button always visible", "SettingsEnableDownloadPopup": "ask how to save", + "SettingsDisableMetadata": "disable file metadata", "AccessibilityEnableDownloadPopup": "ask what to do with downloads", "SettingsQualityDescription": "if selected quality isn't available, closest one is used instead.", "LinkGitHubChanges": ">> see previous commits and contribute on github", diff --git a/src/localization/languages/ru.json b/src/localization/languages/ru.json index d65353b..6cb5108 100644 --- a/src/localization/languages/ru.json +++ b/src/localization/languages/ru.json @@ -43,6 +43,7 @@ "SettingsKeepDownloadButton": "всегда показывать >>", "AccessibilityKeepDownloadButton": "всегда показывать кнопку скачивания на экране", "SettingsEnableDownloadPopup": "выбор метода скачивания", + "SettingsDisableMetadata": "disable file metadata", "AccessibilityEnableDownloadPopup": "спрашивать, что делать с загрузками", "SettingsQualityDescription": "если выбранное качество недоступно, то выбирается ближайшее к нему.", "LinkGitHubChanges": ">> смотри предыдущие изменения на github", diff --git a/src/modules/pageRender/page.js b/src/modules/pageRender/page.js index 1c8a573..6baea34 100644 --- a/src/modules/pageRender/page.js +++ b/src/modules/pageRender/page.js @@ -447,6 +447,10 @@ export default function(obj) { name: t("SettingsEnableDownloadPopup"), padding: "no-margin", aria: t("AccessibilityEnableDownloadPopup") + }, { + action: "disableMetadata", + name: t("SettingsDisableMetadata"), + padding: "no-margin" }]) }) }], diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 0552f2d..5133382 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -22,7 +22,7 @@ import streamable from "./services/streamable.js"; export default async function (host, patternMatch, url, lang, obj) { try { - let r, isAudioOnly = !!obj.isAudioOnly; + let r, isAudioOnly = !!obj.isAudioOnly, disableMetadata = !!obj.disableMetadata; if (!testers[host]) return apiJSON(0, { t: errorUnsupported(lang) }); if (!(testers[host](patternMatch))) return apiJSON(0, { t: brokenLink(lang, host) }); @@ -131,7 +131,7 @@ export default async function (host, patternMatch, url, lang, obj) { if (r.error) return apiJSON(0, { t: Array.isArray(r.error) ? loc(lang, r.error[0], r.error[1]) : loc(lang, r.error) }); - return matchActionDecider(r, host, obj.aFormat, isAudioOnly, lang, isAudioMuted); + return matchActionDecider(r, host, obj.aFormat, isAudioOnly, lang, isAudioMuted, disableMetadata); } catch (e) { return apiJSON(0, { t: genericError(lang, host) }) } diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index c2db917..258fd81 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -2,17 +2,17 @@ import { audioIgnore, services, supportedAudio } from "../config.js"; import { apiJSON } from "../sub/utils.js"; import loc from "../../localization/manager.js"; -export default function(r, host, audioFormat, isAudioOnly, lang, isAudioMuted) { +export default function(r, host, audioFormat, isAudioOnly, lang, isAudioMuted, disableMetadata) { let action, responseType = 2, defaultParams = { u: r.urls, service: host, filename: r.filename, - fileMetadata: r.fileMetadata ? r.fileMetadata : false + fileMetadata: !disableMetadata ? r.fileMetadata : false }, params = {} - + if (r.isPhoto) action = "photo"; else if (r.picker) action = "picker" else if (isAudioMuted) action = "muteVideo"; diff --git a/src/modules/sub/utils.js b/src/modules/sub/utils.js index e237e6a..0a90511 100644 --- a/src/modules/sub/utils.js +++ b/src/modules/sub/utils.js @@ -6,7 +6,7 @@ const apiVar = { vQuality: ["max", "4320", "2160", "1440", "1080", "720", "480", "360", "240", "144"], aFormat: ["best", "mp3", "ogg", "wav", "opus"] }, - booleanOnly: ["isAudioOnly", "isNoTTWatermark", "isTTFullAudio", "isAudioMuted", "dubLang", "vimeoDash"] + booleanOnly: ["isAudioOnly", "isNoTTWatermark", "isTTFullAudio", "isAudioMuted", "dubLang", "vimeoDash", "disableMetadata"] } const forbiddenChars = ['}', '{', '(', ')', '\\', '%', '>', '<', '^', '*', '!', '~', ';', ':', ',', '`', '[', ']', '#', '$', '"', "'", "@", '==']; const forbiddenCharsString = ['}', '{', '%', '>', '<', '^', ';', '`', '$', '"', "@", '=']; @@ -101,13 +101,14 @@ export function checkJSONPost(obj) { isNoTTWatermark: false, isTTFullAudio: false, isAudioMuted: false, + disableMetadata: false, dubLang: false, vimeoDash: false } try { let objKeys = Object.keys(obj); - if (!(objKeys.length <= 9 && obj.url)) return false; let defKeys = Object.keys(def); + if (objKeys.length > defKeys.length + 1 || !obj.url) return false; for (let i in objKeys) { if (String(objKeys[i]) !== "url" && defKeys.includes(objKeys[i])) { From 055eff92da42ddb299b12a3e7c0378c6fe8e5a54 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 29 Aug 2023 23:24:00 +0600 Subject: [PATCH 07/32] 7.2: small improvements - increased video length to 5 hours from 3 hours. - fixed clickable area for urgent notice. - possibly fixed random 0kb files. --- package.json | 2 +- src/config.json | 2 +- src/front/cobalt.css | 6 +++++- src/modules/pageRender/elements.js | 4 +++- src/modules/stream/types.js | 7 +++++-- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 4c075ad..ca34e23 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cobalt", "description": "save what you love", - "version": "7.1.3", + "version": "7.2", "author": "wukko", "exports": "./src/cobalt.js", "type": "module", diff --git a/src/config.json b/src/config.json index 6337654..1537223 100644 --- a/src/config.json +++ b/src/config.json @@ -1,6 +1,6 @@ { "streamLifespan": 20000, - "maxVideoDuration": 10800000, + "maxVideoDuration": 18000000, "genericUserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36", "authorInfo": { "name": "wukko", diff --git a/src/front/cobalt.css b/src/front/cobalt.css index 8e666dd..896df72 100644 --- a/src/front/cobalt.css +++ b/src/front/cobalt.css @@ -797,12 +797,16 @@ button:active, width: 100%; text-align: center; position: absolute; - cursor: pointer; display: flex; justify-content: center; align-items: center; padding-top: calc(env(safe-area-inset-top) + 1rem); } +.urgent-text { + display: flex; + align-items: center; + cursor: pointer; +} .no-transparency .glass-bkg, .no-transparency #popup-backdrop { backdrop-filter: none; diff --git a/src/modules/pageRender/elements.js b/src/modules/pageRender/elements.js index 66e6de1..7ba13bd 100644 --- a/src/modules/pageRender/elements.js +++ b/src/modules/pageRender/elements.js @@ -205,7 +205,9 @@ export function celebrationsEmoji() { } export function urgentNotice(obj) { if (obj.visible) { - return `
${emoji(obj.emoji, 18)} ${obj.text}
` + return `
` + + `${emoji(obj.emoji, 18)} ${obj.text}` + + `
` } return `` } diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index eeec5a5..50cd11c 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -16,7 +16,8 @@ export async function streamDefault(streamInfo, res) { res.setHeader('Content-disposition', `attachment; filename="${streamInfo.isAudioOnly ? `${streamInfo.filename}.${streamInfo.audioFormat}` : regFilename}"`); const { body: stream, headers } = await request(streamInfo.urls, { - headers: { 'user-agent': genericUserAgent } + headers: { 'user-agent': genericUserAgent }, + maxRedirections: 16 }); res.setHeader('content-type', headers['content-type']); @@ -33,7 +34,9 @@ export async function streamLiveRender(streamInfo, res) { try { if (streamInfo.urls.length !== 2) return fail(res); - let { body: audio } = await request(streamInfo.urls[1]); + let { body: audio } = await request(streamInfo.urls[1], { + maxRedirections: 16 + }); let format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1], args = [ From f5ad598d2f8d3d0c1b8b465bb2a2894d9181c362 Mon Sep 17 00:00:00 2001 From: dumbmoron <136796770+dumbmoron@users.noreply.github.com> Date: Tue, 29 Aug 2023 20:18:19 +0200 Subject: [PATCH 08/32] github action: manual trigger instead of per-commit --- .github/workflows/docker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 86ad0a7..d70b182 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,8 +1,7 @@ name: Build Docker image on: - push: - branches: ['current'] + workflow_dispatch: env: REGISTRY: ghcr.io From ac3998ddda555b8893dd55a3f018df87aaddf72e Mon Sep 17 00:00:00 2001 From: dumbmoron <136796770+dumbmoron@users.noreply.github.com> Date: Thu, 24 Aug 2023 06:36:38 +0000 Subject: [PATCH 09/32] import particular mode module only if needed mostly relevant because frontend does not need an innertube session --- src/cobalt.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/cobalt.js b/src/cobalt.js index d03bca2..949cccb 100644 --- a/src/cobalt.js +++ b/src/cobalt.js @@ -9,9 +9,6 @@ import { loadLoc } from "./localization/manager.js"; import path from 'path'; import { fileURLToPath } from 'url'; -import { runWeb } from "./core/web.js"; -import { runAPI } from "./core/api.js"; - const app = express(); const gitCommit = shortCommit(); @@ -28,8 +25,10 @@ const apiMode = process.env.apiURL && process.env.apiPort && !((process.env.webU const webMode = process.env.webURL && process.env.webPort && !((process.env.apiURL && process.env.apiPort) || (process.env.selfURL && process.env.port)); if (apiMode) { + const { runAPI } = await import('./core/api.js'); runAPI(express, app, gitCommit, gitBranch, __dirname) } else if (webMode) { + const { runWeb } = await import('./core/web.js'); await runWeb(express, app, gitCommit, gitBranch, __dirname) } else { console.log(Red(`cobalt wasn't configured yet or configuration is invalid.\n`) + Bright(`please run the setup script to fix this: `) + Green(`npm run setup`)) From be00900a499c6a47d57111a6c03c58497b70e1eb Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 6 Sep 2023 19:37:03 +0600 Subject: [PATCH 10/32] fixes for poor rendering in chrome - fixed blurry header - fixed blurry tab bar - fixed blurry switches - fixed weirdly rounded corners in popups - fixed 1px gap on edges of various elements in popup - fixed overscrolling in other settings tab - fixed unexpected button highlight effect in mobile version - removed outdated fixed for tiny screens --- src/front/cobalt.css | 146 ++++++++++------------------- src/localization/languages/en.json | 3 +- src/localization/languages/ru.json | 3 +- src/modules/pageRender/elements.js | 20 +++- src/modules/pageRender/page.js | 10 +- src/test/tests.json | 2 +- 6 files changed, 73 insertions(+), 111 deletions(-) diff --git a/src/front/cobalt.css b/src/front/cobalt.css index 896df72..0e11615 100644 --- a/src/front/cobalt.css +++ b/src/front/cobalt.css @@ -175,6 +175,24 @@ input[type="text"], backdrop-filter: blur(7px); -webkit-backdrop-filter: blur(7px); } +.glass-bkg.alone { + z-index: -1; + top: 0; + left: 0; + bottom: 0; + right: 0; + position: absolute; +} +.glass-bkg.small { + top: 0; + left: 0; + bottom: 0; + right: 0; + z-index: -1; + position: absolute; + border: var(--accent-highlight) solid 0.15rem; + border-radius: 8px/9px; +} .desktop button:hover, .desktop .switch:hover, .desktop .checkbox:hover, @@ -198,7 +216,7 @@ button:active, .popup.small .switch { background: var(--accent-button-elevated); } -.popup.small .switch:hover { +.desktop .popup.small .switch:hover { background: var(--accent-hover-elevated); } .switch.text-backdrop, @@ -267,7 +285,6 @@ button:active, } .box { background: var(--background); - border: var(--glass) solid .2rem; color: var(--accent); } #url-input-area { @@ -375,7 +392,8 @@ button:active, max-height: 95%; opacity: 0; transform: translate(-50%,-48%)scale(.95); - box-shadow: 0 0 20px 0 var(--accent-hover-transparent); + box-shadow: 0 0 0 0.2rem var(--glass) inset, + 0 0 20px 0 var(--accent-hover-transparent); } .popup.visible { visibility: visible; @@ -404,7 +422,6 @@ button:active, .popup.small { width: 20%; box-shadow: 0px 0px 60px 0px var(--accent-hover); - border: var(--accent-highlight) solid 0.15rem; padding: 1.7rem; transform: translate(-50%,-50%)scale(.95); pointer-events: all; @@ -530,7 +547,6 @@ button:active, z-index: 999; padding-top: calc(env(safe-area-inset-top)/2 + 1.7rem); width: 100%; - border-bottom: var(--accent-highlight) solid 0.1rem; } .settings-category { padding-bottom: 1rem; @@ -629,7 +645,6 @@ button:active, width: auto; flex-direction: row; flex-wrap: nowrap; - overflow-x: scroll; scrollbar-width: none; } .switches .switch { @@ -672,7 +687,6 @@ button:active, width: 100%; padding-top: 0.2rem; padding-bottom: 1.7rem; - border-top: var(--accent-highlight) solid 0.1rem; } .popup-tabs-child { width: 100%; @@ -907,13 +921,17 @@ button:active, .scrollable #popup-content { border-radius: 8px / 9px; } -#popup-header { - border-top-left-radius: 5px; - border-top-right-radius: 5px; +#popup-header .glass-bkg { + border-top-left-radius: 8px 9px; + border-top-right-radius: 8px 9px; + border-bottom: var(--accent-highlight) solid 0.1rem; + top: -1px; } -#popup-tabs { - border-bottom-left-radius: 5px; - border-bottom-right-radius: 5px; +#popup-tabs .glass-bkg { + border-bottom-left-radius: 8px 9px; + border-bottom-right-radius: 8px 9px; + border-top: var(--accent-highlight) solid 0.1rem; + bottom: -1px; } .switches .first { border-top-left-radius: 5px 6px; @@ -1009,87 +1027,6 @@ button:active, width: calc(100% - 1.3rem); } } -@media screen and (max-width: 320px) { - :root { - --gap: 0.38rem; - --gap-no-icon: 0.38rem; - --line-height: 1.2rem; - } - #popup-title { - font-size: 1.07rem; - line-height: 1.5rem; - } - .checkbox { - width: calc(100% - 1rem); - } - .footer-button, - #audioMode-false, - #audioMode-true, - #paste { - font-size: 0!important; - } - .footer-button .emoji, - #audioMode-false .emoji, - #audioMode-true .emoji, - #paste .emoji { - margin-right: 0; - } - .switch, - .checkbox, - .category-title, - .subtitle, - #popup-desc, - .collapse-title { - font-size: 0.7rem; - } - .collapse-header { - padding: 0.5rem; - } - #popup-above-title, - #url-input-area { - font-size: 0.6rem; - } - .explanation { - font-size: 0.6rem; - margin-top: 0.5rem; - line-height: 1rem!important; - } - #popup-desc { - line-height: 1.2rem; - font-size: 0.64rem; - } - .changelog-subtitle, #popup-subtitle { - font-size: 0.8rem!important; - } - .category-title { - margin-bottom: 0.8rem; - } - .emoji { - height: 18px; - width: 18px; - } - .desc-padding { - padding-bottom: 0.8rem; - } - #logo { - font-size: 0.8rem; - } - .popup, - .popup.scrollable, - .popup.small { - height: 98%; - } - [type=checkbox] { - width: 15px; - height: 15px; - border: 0.12rem solid var(--accent); - } - [type=checkbox]:before { - transform: scaleY(.8)scaleX(.7)rotate(45deg); - left: 3.4px; - top: -2px; - } -} @media screen and (max-width: 720px) { #cobalt-main-box { width: calc(100% - (0.7rem * 2)); @@ -1128,10 +1065,20 @@ button:active, padding-top: calc(env(safe-area-inset-bottom)/2 + 1rem); } .popup, - #popup-header, - #popup-tabs { + #popup-header .glass-bkg, + #popup-tabs .glass-bkg, + .glass-bkg.small { border-radius: 0; } + #popup-tabs .glass-bkg { + bottom: 0; + } + .switches { + overflow-x: scroll; + } + .checkbox { + margin-right: 0; + } .popup.center { top: unset; left: unset; @@ -1145,11 +1092,13 @@ button:active, left: 0; transform: none; position: absolute; - border: none; - border-top: var(--accent-highlight) solid 0.15rem; padding-bottom: calc(env(safe-area-inset-bottom)/2 + 1.7rem); transform: translateY(30rem); } + .glass-bkg.small { + border: none; + border-top: var(--accent-highlight) solid 0.15rem; + } .popup.small.visible { transform: none; transition: transform 200ms cubic-bezier(0.075, 0.82, 0.165, 1), opacity 130ms ease-in-out; @@ -1177,6 +1126,7 @@ button:active, width: 100%; height: 100%; max-height: 100%; + box-shadow: none; } #popup-tabs { padding-bottom: calc(env(safe-area-inset-bottom)/2 + 1.5rem); diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json index b087f99..4f4d0ac 100644 --- a/src/localization/languages/en.json +++ b/src/localization/languages/en.json @@ -136,6 +136,7 @@ "KeyboardShortcutClosePopup": "close all popups", "CollapseLegal": "legal stuff", "FairUse": "cobalt is a tool for easing content downloads from internet and takes zero liability. you are responsible for what you download, how you use and distribute that content.\n\ncobalt does not log any info about you, it's impossible for me to snitch on you, but please be mindful when using content of others and always credit original creators!\n\nwhen used in education purposes (lecture, homework, etc) please attach the source link.\n\nfair use and credits benefit everyone.", - "UrgentFeatureUpdate71": "more supported services!" + "UrgentFeatureUpdate71": "more supported services!", + "UrgentThanks": "thank you for support!" } } diff --git a/src/localization/languages/ru.json b/src/localization/languages/ru.json index d65353b..3330dcb 100644 --- a/src/localization/languages/ru.json +++ b/src/localization/languages/ru.json @@ -137,6 +137,7 @@ "KeyboardShortcutClosePopup": "закрыть все окна", "CollapseLegal": "правовые штучки", "FairUse": "кобальт - это инструмент для облегчения скачивания контента из интернета, и он не несёт никакой ответственности. ты несёшь ответственность за то, что скачиваешь, как используешь и распространяешь скачанный контент.\n\nкобальт не собирает никакой информации о тебе, и не может донести на тебя, но, пожалуйста, будь сознателен при использовании чужого контента и всегда указывай авторов!\n\nпри использовании в образовательных целях (лекции, домашние задания и т.д.), пожалуйста, прикладывай ссылку на источник.\n\nчестное использование и указание авторства выгодно всем.", - "UrgentFeatureUpdate71": "расширение поддержки сервисов!" + "UrgentFeatureUpdate71": "расширение поддержки сервисов!", + "UrgentThanks": "спасибо за поддержку!" } } diff --git a/src/modules/pageRender/elements.js b/src/modules/pageRender/elements.js index 7ba13bd..a1940d5 100644 --- a/src/modules/pageRender/elements.js +++ b/src/modules/pageRender/elements.js @@ -68,17 +68,19 @@ export function popup(obj) { } return ` ${obj.standalone ? `
0 ? ' ' + classes.join(' ') : ''}">` : ''} -
+
${body}${obj.buttonOnly ? `` : ''}
+ ${classes.includes("small") ? `
`: ''} ${obj.standalone ? `
` : ''}` } @@ -97,14 +99,18 @@ export function multiPagePopup(obj) { return `