From fa7af1bf444e3a473118ca5243380f32863d24cb Mon Sep 17 00:00:00 2001 From: Snazzah Date: Sat, 29 Apr 2023 14:33:36 -0500 Subject: [PATCH 1/6] feat: add twitch vod/clip support --- src/modules/processing/match.js | 10 + src/modules/processing/services/twitch.js | 223 ++++++++++++++++++ src/modules/processing/servicesConfig.json | 6 + .../processing/servicesPatternTesters.js | 4 +- src/modules/stream/types.js | 2 +- src/modules/sub/utils.js | 3 + src/test/tests.json | 57 +++++ 7 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 src/modules/processing/services/twitch.js diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 3328a32..aef1c17 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -17,6 +17,7 @@ import vimeo from "./services/vimeo.js"; import soundcloud from "./services/soundcloud.js"; import instagram from "./services/instagram.js"; import vine from "./services/vine.js"; +import twitch from "./services/twitch.js"; export default async function (host, patternMatch, url, lang, obj) { try { @@ -110,6 +111,15 @@ export default async function (host, patternMatch, url, lang, obj) { case "vine": r = await vine({ id: patternMatch["id"] }); break; + case "twitch": + r = await twitch({ + vodId: patternMatch["video"] ? patternMatch["video"] : false, + clipId: patternMatch["clip"] ? patternMatch["clip"] : false, + lang: lang, quality: obj.vQuality, + isAudioOnly: obj.isAudioOnly, + format: obj.vFormat + }); + break; default: return apiJSON(0, { t: errorUnsupported(lang) }); } diff --git a/src/modules/processing/services/twitch.js b/src/modules/processing/services/twitch.js new file mode 100644 index 0000000..bd8a869 --- /dev/null +++ b/src/modules/processing/services/twitch.js @@ -0,0 +1,223 @@ +import { maxVideoDuration } from "../../config.js"; + +const gqlURL = "https://gql.twitch.tv/gql"; +const m3u8URL = "https://usher.ttvnw.net"; + +function parseM3U8Line(line) { + const result = {}; + + let str = '', inQuotes = false, keyName = null, escaping = false; + for (let i = 0; i < line.length; i++) { + const char = line[i]; + if (char === '"' && !escaping) { + inQuotes = !inQuotes; + continue; + } else if (char === ',' && !escaping && !inQuotes) { + if (!keyName) break; + result[keyName] = str; + keyName = null; + str = ''; + continue; + } else if (char === '\\' && !escaping) { + escaping = true; + continue; + } else if (char === '=' && !escaping && !inQuotes) { + keyName = str; + str = ''; + continue; + } + + str += char; + escaping = false; + } + + if (keyName) result[keyName] = str; + return result; +} + +function getM3U8Formats(m3u8body) { + let formats = []; + const formatLines = m3u8body.split('\n').slice(2); + + for (let i = 0; i < formatLines.length; i += 3) { + const mediaLine = parseM3U8Line(formatLines[i].split(':')[1]); + const streamLine = parseM3U8Line(formatLines[i + 1].split(':')[1]); + formats.push({ + id: mediaLine['GROUP-ID'], + name: mediaLine.NAME, + resolution: streamLine.RESOLUTION ? streamLine.RESOLUTION.split('x') : null, + url: formatLines[i + 2] + }); + } + return formats; +}; + +export default async function(obj) { + try { + let _headers = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" }; + + if (!obj.clipId && !obj.vodId) return { error: 'ErrorCantGetID' }; + + if (obj.vodId) { + const req_metadata = await fetch(gqlURL, { + method: "POST", + headers: _headers, + body: JSON.stringify([ + { + "operationName": "VideoMetadata", + "variables": { + "channelLogin": "", + "videoID": obj.vodId + }, + "extensions": { + "persistedQuery": { + "version": 1, + "sha256Hash": "226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687" + } + } + } + ]) + }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); + if (!req_metadata) return { error: 'ErrorCouldntFetch' }; + const vodMetadata = req_metadata[0].data.video; + + if (vodMetadata.previewThumbnailURL.endsWith('/404_processing_{width}x{height}.png')) return { error: 'ErrorLiveVideo' }; + if (vodMetadata.lengthSeconds > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + if (!vodMetadata.owner) return { error: 'ErrorEmptyDownload' }; // Streamer was banned... + + const req_token = await fetch(gqlURL, { + method: "POST", + headers: _headers, + body: JSON.stringify({ + query: `{ + videoPlaybackAccessToken( + id: "${obj.vodId}", + params: { + platform: "web", + playerBackend: "mediaplayer", + playerType: "site" + } + ) + { + value + signature + } + }` + }) + }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); + if (!req_token) return { error: 'ErrorCouldntFetch' }; + + const access_token = req_token.data.videoPlaybackAccessToken; + const req_m3u8 = await fetch(`${m3u8URL}/vod/${obj.vodId}.m3u8?${new URLSearchParams({ + allow_source: 'true', + allow_audio_only: 'true', + allow_spectre: 'true', + player: 'twitchweb', + playlist_include_framerate: 'true', + nauth: access_token.value, + nauthsig: access_token.signature + })}`, { + headers: _headers + }).then((r) => { return r.status == 200 ? r.text() : false;}).catch(() => {return false}); + if (!req_m3u8) return { error: 'ErrorCouldntFetch' }; + + const formats = getM3U8Formats(req_m3u8); + const generalMeta = { + title: vodMetadata.title, + artist: `Twitch Broadcast by @${vodMetadata.owner.login}`, + } + + if (!obj.isAudioOnly) { + const format = formats.find(f => f.resolution && f.resolution[1] == obj.quality) || formats[0]; + + return { + urls: format.url, + isM3U8: true, + time: vodMetadata.lengthSeconds * 1000, + filename: `twitchvod_${obj.vodId}_${format.resolution[0]}x${format.resolution[1]}.mp4` + }; + } else { + return { + type: "render", + isM3U8: true, + time: vodMetadata.lengthSeconds * 1000, + urls: formats.find(f => f.id === 'audio_only').url, + audioFilename: `twitchvod_${obj.vodId}_audio`, + fileMetadata: generalMeta + } + } + } else if (obj.clipId) { + const req_metadata = await fetch(gqlURL, { + method: "POST", + headers: _headers, + body: JSON.stringify({ + query: `{ + clip(slug: "${obj.clipId}") { + broadcaster { + login + } + createdAt + curator { + login + } + durationSeconds + id + medium: thumbnailURL(width: 480, height: 272) + title + videoQualities { + quality + sourceURL + } + } + }` + }) + }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); + if (!req_metadata) return { error: 'ErrorCouldntFetch' }; + const clipMetadata = req_metadata.data.clip; + if (clipMetadata.durationSeconds > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + if (!clipMetadata.videoQualities || !clipMetadata.broadcaster) return { error: 'ErrorEmptyDownload' }; // Streamer was banned... + + const req_token = await fetch(gqlURL, { + method: "POST", + headers: _headers, + body: JSON.stringify([ + { + "operationName": "VideoAccessToken_Clip", + "variables": { + "slug": obj.clipId + }, + "extensions": { + "persistedQuery": { + "version": 1, + "sha256Hash": "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11" + } + } + } + ]) + }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); + if (!req_token) return { error: 'ErrorCouldntFetch' }; + + const generalMeta = { + title: clipMetadata.title, + artist: `Twitch Clip by @${clipMetadata.broadcaster.login}, clipped by @${clipMetadata.curator.login}`, + } + + const access_token = req_token[0].data.clip.playbackAccessToken; + const formats = clipMetadata.videoQualities; + const format = formats.find(f => f.quality == obj.quality) || formats[0]; + + return { + type: "bridge", + urls: `${format.sourceURL}?${new URLSearchParams({ + sig: access_token.signature, + token: access_token.value + })}`, + filename: `twitchclip_${clipMetadata.id}_${format.quality}.mp4`, + audioFilename: `twitchclip_${clipMetadata.id}_audio`, + fileMetadata: generalMeta + }; + } + } catch (err) { + return { error: 'ErrorBadFetch' }; + } +} \ No newline at end of file diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index 2bb7f8f..0318d4e 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -62,6 +62,12 @@ "tld": "co", "patterns": ["v/:id"], "enabled": true + }, + "twitch": { + "alias": "twitch vods & videos & clips", + "tld": "tv", + "patterns": ["videos/:video", ":channel/clip/:clip"], + "enabled": true } } } diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index 8f70613..35eaa90 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -28,5 +28,7 @@ export const testers = { "instagram": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length <= 12), - "vine": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length <= 12) + "vine": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length <= 12), + + "twitch": (patternMatch) => ((patternMatch["channel"] && patternMatch["clip"] && patternMatch["clip"].length <= 100 || patternMatch["video"] && patternMatch["video"].length <= 10)), } diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index a4eb233..3aa259f 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -145,7 +145,7 @@ export function streamVideoOnly(streamInfo, res) { '-c', 'copy' ] if (streamInfo.mute) args.push('-an'); - if (streamInfo.service === "vimeo") args.push('-bsf:a', 'aac_adtstoasc'); + if (streamInfo.service === "vimeo" || streamInfo.service === "twitch") args.push('-bsf:a', 'aac_adtstoasc'); if (format === "mp4") args.push('-movflags', 'faststart+frag_keyframe+empty_moov'); args.push('-f', format, 'pipe:3'); const ffmpegProcess = spawn(ffmpeg, args, { diff --git a/src/modules/sub/utils.js b/src/modules/sub/utils.js index 27a17b8..4bcfc6d 100644 --- a/src/modules/sub/utils.js +++ b/src/modules/sub/utils.js @@ -84,6 +84,9 @@ export function cleanURL(url, host) { if (url.includes('youtube.com/shorts/')) { url = url.split('?')[0].replace('shorts/', 'watch?v='); } + if (url.includes('clips.twitch.tv')) { + url = url.split('?')[0].replace('clips.twitch.tv/', 'twitch.tv/_/clip/'); + } return url.slice(0, 128) } export function verifyLanguageCode(code) { diff --git a/src/test/tests.json b/src/test/tests.json index 158a0ea..57fbefd 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -869,5 +869,62 @@ "code": 200, "status": "stream" } + }], + "twitch": [{ + "name": "clip", + "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", + "params": {}, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "clip (isAudioOnly)", + "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", + "params": { + "isAudioOnly": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "clip (isAudioMuted)", + "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", + "params": { + "isAudioMuted": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "video", + "url": "https://twitch.tv/videos/1315890970", + "params": {}, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "video (isAudioOnly)", + "url": "https://twitch.tv/videos/1315890970", + "params": { + "isAudioOnly": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "video (isAudioMuted)", + "url": "https://twitch.tv/videos/1315890970", + "params": { + "isAudioMuted": true + }, + "expected": { + "code": 200, + "status": "stream" + } }] } \ No newline at end of file From c8d0eb862cb00b6255776ae5cd7054db9b4dd951 Mon Sep 17 00:00:00 2001 From: Snazzah Date: Sat, 29 Apr 2023 17:13:26 -0500 Subject: [PATCH 2/6] chore: update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b1e636a..234ff4b 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ It's fast, friendly, and doesn't have any bullshit that modern web is filled wit | YouTube Music | ❌ | ✅ | Audio metadata. | | Reddit | ✅ | ✅ | GIFs and videos. | | TikTok | ✅ | ✅ | Video downloads with or without watermark; image slideshow downloads without watermark. Full audio downloads. | +| Twitch | ✅ | ✅ | | | SoundCloud | ❌ | ✅ | Audio metadata, downloads from private links. | | bilibili.com | ✅ | ✅ | | | Tumblr | ✅ | ✅ | | From ad9b6ebdd38f5a8711621f09dd6cb5c0b6d4e8ee Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 16 Sep 2023 16:27:53 +0600 Subject: [PATCH 3/6] twitch clean up --- src/modules/processing/hostOverrides.js | 8 + src/modules/processing/match.js | 6 +- src/modules/processing/matchActionDecider.js | 2 +- src/modules/processing/services/twitch.js | 377 ++++++++---------- src/modules/processing/servicesConfig.json | 6 +- .../processing/servicesPatternTesters.js | 4 +- src/modules/sub/utils.js | 55 ++- 7 files changed, 233 insertions(+), 225 deletions(-) diff --git a/src/modules/processing/hostOverrides.js b/src/modules/processing/hostOverrides.js index a9f2fc2..88553e3 100644 --- a/src/modules/processing/hostOverrides.js +++ b/src/modules/processing/hostOverrides.js @@ -8,6 +8,9 @@ export default function (inHost, inURL) { url = url.split("?")[0].replace("www.", ""); url = `https://youtube.com/watch?v=${url.replace("https://youtube.com/live/", "")}` } + if (url.includes('youtube.com/shorts/')) { + url = url.split('?')[0].replace('shorts/', 'watch?v='); + } break; case "youtu": if (url.startsWith("https://youtu.be/")) { @@ -32,6 +35,11 @@ export default function (inHost, inURL) { url = url.replace(url.split('/')[5], '') } break; + case "twitch": + if (url.includes('clips.twitch.tv')) { + url = url.split('?')[0].replace('clips.twitch.tv/', 'twitch.tv/_/clip/'); + } + break; } return { host: host, diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 6425f93..5c34b6d 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -127,10 +127,10 @@ export default async function (host, patternMatch, url, lang, obj) { r = await twitch({ vodId: patternMatch["video"] ? patternMatch["video"] : false, clipId: patternMatch["clip"] ? patternMatch["clip"] : false, - lang: lang, quality: obj.vQuality, - isAudioOnly: obj.isAudioOnly, - format: obj.vFormat + quality: obj.vQuality, + isAudioOnly: obj.isAudioOnly }); + break; default: return apiJSON(0, { t: errorUnsupported(lang) }); } diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index 258fd81..82eb341 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -55,7 +55,7 @@ export default function(r, host, audioFormat, isAudioOnly, lang, isAudioMuted, d case "tiktok": params = { type: "bridge" }; break; - + case "vine": case "instagram": case "tumblr": diff --git a/src/modules/processing/services/twitch.js b/src/modules/processing/services/twitch.js index bd8a869..31b239e 100644 --- a/src/modules/processing/services/twitch.js +++ b/src/modules/processing/services/twitch.js @@ -1,223 +1,180 @@ import { maxVideoDuration } from "../../config.js"; +import { getM3U8Formats } from "../../sub/utils.js"; const gqlURL = "https://gql.twitch.tv/gql"; const m3u8URL = "https://usher.ttvnw.net"; +const clientIdHead = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" }; -function parseM3U8Line(line) { - const result = {}; - - let str = '', inQuotes = false, keyName = null, escaping = false; - for (let i = 0; i < line.length; i++) { - const char = line[i]; - if (char === '"' && !escaping) { - inQuotes = !inQuotes; - continue; - } else if (char === ',' && !escaping && !inQuotes) { - if (!keyName) break; - result[keyName] = str; - keyName = null; - str = ''; - continue; - } else if (char === '\\' && !escaping) { - escaping = true; - continue; - } else if (char === '=' && !escaping && !inQuotes) { - keyName = str; - str = ''; - continue; - } - - str += char; - escaping = false; - } - - if (keyName) result[keyName] = str; - return result; -} - -function getM3U8Formats(m3u8body) { - let formats = []; - const formatLines = m3u8body.split('\n').slice(2); - - for (let i = 0; i < formatLines.length; i += 3) { - const mediaLine = parseM3U8Line(formatLines[i].split(':')[1]); - const streamLine = parseM3U8Line(formatLines[i + 1].split(':')[1]); - formats.push({ - id: mediaLine['GROUP-ID'], - name: mediaLine.NAME, - resolution: streamLine.RESOLUTION ? streamLine.RESOLUTION.split('x') : null, - url: formatLines[i + 2] - }); - } - return formats; -}; - -export default async function(obj) { - try { - let _headers = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" }; - - if (!obj.clipId && !obj.vodId) return { error: 'ErrorCantGetID' }; - - if (obj.vodId) { - const req_metadata = await fetch(gqlURL, { - method: "POST", - headers: _headers, - body: JSON.stringify([ - { - "operationName": "VideoMetadata", - "variables": { - "channelLogin": "", - "videoID": obj.vodId - }, - "extensions": { - "persistedQuery": { - "version": 1, - "sha256Hash": "226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687" - } - } - } - ]) - }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); - if (!req_metadata) return { error: 'ErrorCouldntFetch' }; - const vodMetadata = req_metadata[0].data.video; - - if (vodMetadata.previewThumbnailURL.endsWith('/404_processing_{width}x{height}.png')) return { error: 'ErrorLiveVideo' }; - if (vodMetadata.lengthSeconds > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; - if (!vodMetadata.owner) return { error: 'ErrorEmptyDownload' }; // Streamer was banned... - - const req_token = await fetch(gqlURL, { - method: "POST", - headers: _headers, - body: JSON.stringify({ - query: `{ - videoPlaybackAccessToken( - id: "${obj.vodId}", - params: { - platform: "web", - playerBackend: "mediaplayer", - playerType: "site" - } - ) - { - value - signature - } - }` - }) - }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); - if (!req_token) return { error: 'ErrorCouldntFetch' }; - - const access_token = req_token.data.videoPlaybackAccessToken; - const req_m3u8 = await fetch(`${m3u8URL}/vod/${obj.vodId}.m3u8?${new URLSearchParams({ - allow_source: 'true', - allow_audio_only: 'true', - allow_spectre: 'true', - player: 'twitchweb', - playlist_include_framerate: 'true', - nauth: access_token.value, - nauthsig: access_token.signature - })}`, { - headers: _headers - }).then((r) => { return r.status == 200 ? r.text() : false;}).catch(() => {return false}); - if (!req_m3u8) return { error: 'ErrorCouldntFetch' }; - - const formats = getM3U8Formats(req_m3u8); - const generalMeta = { - title: vodMetadata.title, - artist: `Twitch Broadcast by @${vodMetadata.owner.login}`, - } - - if (!obj.isAudioOnly) { - const format = formats.find(f => f.resolution && f.resolution[1] == obj.quality) || formats[0]; - - return { - urls: format.url, - isM3U8: true, - time: vodMetadata.lengthSeconds * 1000, - filename: `twitchvod_${obj.vodId}_${format.resolution[0]}x${format.resolution[1]}.mp4` - }; - } else { - return { - type: "render", - isM3U8: true, - time: vodMetadata.lengthSeconds * 1000, - urls: formats.find(f => f.id === 'audio_only').url, - audioFilename: `twitchvod_${obj.vodId}_audio`, - fileMetadata: generalMeta +async function getClip(obj) { + let req_metadata = await fetch(gqlURL, { + method: "POST", + headers: clientIdHead, + body: JSON.stringify({ + query: `{ + clip(slug: "${obj.clipId}") { + broadcaster { + login + } + createdAt + curator { + login + } + durationSeconds + id + medium: thumbnailURL(width: 480, height: 272) + title + videoQualities { + quality + sourceURL } } - } else if (obj.clipId) { - const req_metadata = await fetch(gqlURL, { - method: "POST", - headers: _headers, - body: JSON.stringify({ - query: `{ - clip(slug: "${obj.clipId}") { - broadcaster { - login - } - createdAt - curator { - login - } - durationSeconds - id - medium: thumbnailURL(width: 480, height: 272) - title - videoQualities { - quality - sourceURL - } - } - }` - }) - }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); - if (!req_metadata) return { error: 'ErrorCouldntFetch' }; - const clipMetadata = req_metadata.data.clip; - if (clipMetadata.durationSeconds > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; - if (!clipMetadata.videoQualities || !clipMetadata.broadcaster) return { error: 'ErrorEmptyDownload' }; // Streamer was banned... + }` + }) + }).then((r) => { return r.status === 200 ? r.json() : false; }).catch(() => { return false }); + if (!req_metadata) return { error: 'ErrorCouldntFetch' }; - const req_token = await fetch(gqlURL, { - method: "POST", - headers: _headers, - body: JSON.stringify([ - { - "operationName": "VideoAccessToken_Clip", - "variables": { - "slug": obj.clipId - }, - "extensions": { - "persistedQuery": { - "version": 1, - "sha256Hash": "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11" - } - } - } - ]) - }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); - if (!req_token) return { error: 'ErrorCouldntFetch' }; + let clipMetadata = req_metadata.data.clip; - const generalMeta = { - title: clipMetadata.title, - artist: `Twitch Clip by @${clipMetadata.broadcaster.login}, clipped by @${clipMetadata.curator.login}`, + if (clipMetadata.durationSeconds > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + if (!clipMetadata.videoQualities || !clipMetadata.broadcaster) return { error: 'ErrorEmptyDownload' }; + + let req_token = await fetch(gqlURL, { + method: "POST", + headers: clientIdHead, + body: JSON.stringify([ + { + "operationName": "VideoAccessToken_Clip", + "variables": { + "slug": obj.clipId + }, + "extensions": { + "persistedQuery": { + "version": 1, + "sha256Hash": "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11" + } + } } + ]) + }).then((r) => { return r.status === 200 ? r.json() : false; }).catch(() => { return false }); - const access_token = req_token[0].data.clip.playbackAccessToken; - const formats = clipMetadata.videoQualities; - const format = formats.find(f => f.quality == obj.quality) || formats[0]; + if (!req_token) return { error: 'ErrorCouldntFetch' }; - return { - type: "bridge", - urls: `${format.sourceURL}?${new URLSearchParams({ - sig: access_token.signature, - token: access_token.value - })}`, - filename: `twitchclip_${clipMetadata.id}_${format.quality}.mp4`, - audioFilename: `twitchclip_${clipMetadata.id}_audio`, - fileMetadata: generalMeta - }; - } - } catch (err) { - return { error: 'ErrorBadFetch' }; + let formats = clipMetadata.videoQualities; + let format = formats.find(f => f.quality === obj.quality) || formats[0]; + + return { + type: "bridge", + urls: `${format.sourceURL}?${new URLSearchParams({ + sig: req_token[0].data.clip.playbackAccessToken.signature, + token: req_token[0].data.clip.playbackAccessToken.value + })}`, + fileMetadata: { + title: clipMetadata.title, + artist: `Twitch Clip by @${clipMetadata.broadcaster.login}, clipped by @${clipMetadata.curator.login}`, + }, + filename: `twitchclip_${clipMetadata.id}_${format.quality}p.mp4`, + audioFilename: `twitchclip_${clipMetadata.id}_audio` } -} \ No newline at end of file +} +async function getVideo(obj) { + let req_metadata = await fetch(gqlURL, { + method: "POST", + headers: clientIdHead, + body: JSON.stringify([ + { + "operationName": "VideoMetadata", + "variables": { + "channelLogin": "", + "videoID": obj.vodId + }, + "extensions": { + "persistedQuery": { + "version": 1, + "sha256Hash": "226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687" + } + } + } + ]) + }).then((r) => { return r.status === 200 ? r.json() : false; }).catch(() => { return false }); + if (!req_metadata) return { error: 'ErrorCouldntFetch' }; + + let vodMetadata = req_metadata[0].data.video; + + if (vodMetadata.previewThumbnailURL.endsWith('/404_processing_{width}x{height}.png')) return { error: 'ErrorLiveVideo' }; + if (vodMetadata.lengthSeconds > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + if (!vodMetadata.owner) return { error: 'ErrorEmptyDownload' }; + + let req_token = await fetch(gqlURL, { + method: "POST", + headers: clientIdHead, + body: JSON.stringify({ + query: `{ + videoPlaybackAccessToken( + id: "${obj.vodId}", + params: { + platform: "web", + playerBackend: "mediaplayer", + playerType: "site" + } + ) + { + value + signature + } + }` + }) + }).then((r) => { return r.status === 200 ? r.json() : false; }).catch(() => { return false }); + if (!req_token) return { error: 'ErrorCouldntFetch' }; + + let req_m3u8 = await fetch( + `${m3u8URL}/vod/${obj.vodId}.m3u8?${ + new URLSearchParams({ + allow_source: 'true', + allow_audio_only: 'true', + allow_spectre: 'true', + player: 'twitchweb', + playlist_include_framerate: 'true', + nauth: req_token.data.videoPlaybackAccessToken.value, + nauthsig: req_token.data.videoPlaybackAccessToken.signature + } + )}`, { + headers: clientIdHead + } + ).then((r) => { return r.status === 200 ? r.text() : false; }).catch(() => { return false }); + + if (!req_m3u8) return { error: 'ErrorCouldntFetch' }; + + let formats = getM3U8Formats(req_m3u8); + let generalMeta = { + title: vodMetadata.title, + artist: `Twitch Broadcast by @${vodMetadata.owner.login}`, + } + if (obj.isAudioOnly) { + return { + type: "render", + isM3U8: true, + time: vodMetadata.lengthSeconds * 1000, + urls: formats.find(f => f.id === 'audio_only').url, + audioFilename: `twitchvod_${obj.vodId}_audio`, + fileMetadata: generalMeta + } + } else { + let format = formats.find(f => f.resolution && f.resolution[1] === obj.quality) || formats[0]; + return { + urls: format.url, + isM3U8: true, + time: vodMetadata.lengthSeconds * 1000, + filename: `twitchvod_${obj.vodId}_${format.resolution[0]}x${format.resolution[1]}.mp4`, + fileMetadata: generalMeta + } + } +} +export default async function (obj) { + let response = { error: 'ErrorEmptyDownload' }; + if (obj.clipId) { + response = await getClip(obj) + } else if (obj.vodId) { + response = await getVideo(obj) + } + return response +} diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index f904eaf..e65b4f3 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -73,11 +73,11 @@ "patterns": [":id", "o/:id", "e/:id", "s/:id"], "enabled": true }, - "twitch": { - "alias": "twitch vods & videos & clips", + "twitch": { + "alias": "twitch clips, videos & vods", "tld": "tv", "patterns": ["videos/:video", ":channel/clip/:clip"], "enabled": true } - } + } } diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index cdc2ca4..19fca2e 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -22,12 +22,12 @@ export const testers = { || (patternMatch["id"] && patternMatch["id"].length < 21 && patternMatch["user"] && patternMatch["user"].length <= 32)), "vimeo": (patternMatch) => ((patternMatch["id"] && patternMatch["id"].length <= 11)), - + "soundcloud": (patternMatch) => (patternMatch["author"]?.length <= 25 && patternMatch["song"]?.length <= 255) || (patternMatch["shortLink"] && patternMatch["shortLink"].length <= 32), "instagram": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length <= 12), - + "vine": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length <= 12), "pinterest": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length <= 128), diff --git a/src/modules/sub/utils.js b/src/modules/sub/utils.js index 6f6886c..b38fc39 100644 --- a/src/modules/sub/utils.js +++ b/src/modules/sub/utils.js @@ -70,12 +70,6 @@ export function cleanURL(url, host) { url = url.replaceAll(forbiddenChars[i], '') } url = url.replace('https//', 'https://') - if (url.includes('youtube.com/shorts/')) { - url = url.split('?')[0].replace('shorts/', 'watch?v='); - } - if (url.includes('clips.twitch.tv')) { - url = url.split('?')[0].replace('clips.twitch.tv/', 'twitch.tv/_/clip/'); - } return url.slice(0, 128) } export function cleanString(string) { @@ -155,3 +149,52 @@ export function cleanHTML(html) { clean = clean.replace(/\n/g, ''); return clean } + +export function parseM3U8Line(line) { + let result = {}; + let str = '', inQuotes = false, keyName = null, escaping = false; + + for (let i = 0; i < line.length; i++) { + const char = line[i]; + if (char === '"' && !escaping) { + inQuotes = !inQuotes; + continue + } else if (char === ',' && !escaping && !inQuotes) { + if (!keyName) break; + result[keyName] = str; + keyName = null; + str = ''; + continue + } else if (char === '\\' && !escaping) { + escaping = true; + continue + } else if (char === '=' && !escaping && !inQuotes) { + keyName = str; + str = ''; + continue + } + + str += char; + escaping = false + } + + if (keyName) result[keyName] = str; + return result +} + +export function getM3U8Formats(m3u8body) { + const formatLines = m3u8body.split('\n').slice(2); + let formats = []; + + for (let i = 0; i < formatLines.length; i += 3) { + const mediaLine = parseM3U8Line(formatLines[i].split(':')[1]); + const streamLine = parseM3U8Line(formatLines[i + 1].split(':')[1]); + formats.push({ + id: mediaLine['GROUP-ID'], + name: mediaLine.NAME, + resolution: streamLine.RESOLUTION ? streamLine.RESOLUTION.split('x') : null, + url: formatLines[i + 2] + }) + } + return formats +} From f7247b87f0088c1c62b83e55750dbf7c8aac1c4d Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 16 Sep 2023 16:31:02 +0600 Subject: [PATCH 4/6] fix wording --- README.md | 2 +- src/modules/processing/servicesConfig.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ec91b41..6638c72 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Paste the link, get the video, move on. It's that simple. Just how it should be. | Streamable | ✅ | ✅ | ✅ | | | TikTok | ✅ | ✅ | ✅ | Supports downloads of: videos with or without watermark, images from slideshow without watermark, full (original) audios. | | Tumblr | ✅ | ✅ | ✅ | Support for audio file downloads. | -| Twitch Clips/VODs | ✅ | ✅ | ✅ | | +| Twitch Clips & Videos | ✅ | ✅ | ✅ | | | Twitter/X * | ✅ | ✅ | ✅ | Ability to pick what to save from multi-media tweets. | | Vimeo | ✅ | ✅ | ✅ | Audio downloads are only available for dash files. | | Vine Archive | ✅ | ✅ | ✅ | | diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index e65b4f3..288c1d1 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -74,7 +74,7 @@ "enabled": true }, "twitch": { - "alias": "twitch clips, videos & vods", + "alias": "twitch clips & videos", "tld": "tv", "patterns": ["videos/:video", ":channel/clip/:clip"], "enabled": true From ad8a9c454d1cca12552822beee7a057f84294cee Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 16 Sep 2023 17:58:43 +0600 Subject: [PATCH 5/6] remove vods there's no point in downloading entire streams. people can clip what they need and download that instead! --- README.md | 2 +- src/modules/processing/match.js | 1 - src/modules/processing/services/twitch.js | 106 +----------------- src/modules/processing/servicesConfig.json | 4 +- .../processing/servicesPatternTesters.js | 2 +- 5 files changed, 5 insertions(+), 110 deletions(-) diff --git a/README.md b/README.md index 6638c72..ade04cd 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Paste the link, get the video, move on. It's that simple. Just how it should be. | Streamable | ✅ | ✅ | ✅ | | | TikTok | ✅ | ✅ | ✅ | Supports downloads of: videos with or without watermark, images from slideshow without watermark, full (original) audios. | | Tumblr | ✅ | ✅ | ✅ | Support for audio file downloads. | -| Twitch Clips & Videos | ✅ | ✅ | ✅ | | +| Twitch Clips | ✅ | ✅ | ✅ | | | Twitter/X * | ✅ | ✅ | ✅ | Ability to pick what to save from multi-media tweets. | | Vimeo | ✅ | ✅ | ✅ | Audio downloads are only available for dash files. | | Vine Archive | ✅ | ✅ | ✅ | | diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 5c34b6d..7b769dc 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -125,7 +125,6 @@ export default async function (host, patternMatch, url, lang, obj) { break; case "twitch": r = await twitch({ - vodId: patternMatch["video"] ? patternMatch["video"] : false, clipId: patternMatch["clip"] ? patternMatch["clip"] : false, quality: obj.vQuality, isAudioOnly: obj.isAudioOnly diff --git a/src/modules/processing/services/twitch.js b/src/modules/processing/services/twitch.js index 31b239e..6d7e6d9 100644 --- a/src/modules/processing/services/twitch.js +++ b/src/modules/processing/services/twitch.js @@ -1,11 +1,9 @@ import { maxVideoDuration } from "../../config.js"; -import { getM3U8Formats } from "../../sub/utils.js"; const gqlURL = "https://gql.twitch.tv/gql"; -const m3u8URL = "https://usher.ttvnw.net"; const clientIdHead = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" }; -async function getClip(obj) { +export default async function (obj) { let req_metadata = await fetch(gqlURL, { method: "POST", headers: clientIdHead, @@ -76,105 +74,3 @@ async function getClip(obj) { audioFilename: `twitchclip_${clipMetadata.id}_audio` } } -async function getVideo(obj) { - let req_metadata = await fetch(gqlURL, { - method: "POST", - headers: clientIdHead, - body: JSON.stringify([ - { - "operationName": "VideoMetadata", - "variables": { - "channelLogin": "", - "videoID": obj.vodId - }, - "extensions": { - "persistedQuery": { - "version": 1, - "sha256Hash": "226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687" - } - } - } - ]) - }).then((r) => { return r.status === 200 ? r.json() : false; }).catch(() => { return false }); - if (!req_metadata) return { error: 'ErrorCouldntFetch' }; - - let vodMetadata = req_metadata[0].data.video; - - if (vodMetadata.previewThumbnailURL.endsWith('/404_processing_{width}x{height}.png')) return { error: 'ErrorLiveVideo' }; - if (vodMetadata.lengthSeconds > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; - if (!vodMetadata.owner) return { error: 'ErrorEmptyDownload' }; - - let req_token = await fetch(gqlURL, { - method: "POST", - headers: clientIdHead, - body: JSON.stringify({ - query: `{ - videoPlaybackAccessToken( - id: "${obj.vodId}", - params: { - platform: "web", - playerBackend: "mediaplayer", - playerType: "site" - } - ) - { - value - signature - } - }` - }) - }).then((r) => { return r.status === 200 ? r.json() : false; }).catch(() => { return false }); - if (!req_token) return { error: 'ErrorCouldntFetch' }; - - let req_m3u8 = await fetch( - `${m3u8URL}/vod/${obj.vodId}.m3u8?${ - new URLSearchParams({ - allow_source: 'true', - allow_audio_only: 'true', - allow_spectre: 'true', - player: 'twitchweb', - playlist_include_framerate: 'true', - nauth: req_token.data.videoPlaybackAccessToken.value, - nauthsig: req_token.data.videoPlaybackAccessToken.signature - } - )}`, { - headers: clientIdHead - } - ).then((r) => { return r.status === 200 ? r.text() : false; }).catch(() => { return false }); - - if (!req_m3u8) return { error: 'ErrorCouldntFetch' }; - - let formats = getM3U8Formats(req_m3u8); - let generalMeta = { - title: vodMetadata.title, - artist: `Twitch Broadcast by @${vodMetadata.owner.login}`, - } - if (obj.isAudioOnly) { - return { - type: "render", - isM3U8: true, - time: vodMetadata.lengthSeconds * 1000, - urls: formats.find(f => f.id === 'audio_only').url, - audioFilename: `twitchvod_${obj.vodId}_audio`, - fileMetadata: generalMeta - } - } else { - let format = formats.find(f => f.resolution && f.resolution[1] === obj.quality) || formats[0]; - return { - urls: format.url, - isM3U8: true, - time: vodMetadata.lengthSeconds * 1000, - filename: `twitchvod_${obj.vodId}_${format.resolution[0]}x${format.resolution[1]}.mp4`, - fileMetadata: generalMeta - } - } -} -export default async function (obj) { - let response = { error: 'ErrorEmptyDownload' }; - if (obj.clipId) { - response = await getClip(obj) - } else if (obj.vodId) { - response = await getVideo(obj) - } - return response -} diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index 288c1d1..47f501f 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -74,9 +74,9 @@ "enabled": true }, "twitch": { - "alias": "twitch clips & videos", + "alias": "twitch clips", "tld": "tv", - "patterns": ["videos/:video", ":channel/clip/:clip"], + "patterns": [":channel/clip/:clip"], "enabled": true } } diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index 19fca2e..a22b2d5 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -34,5 +34,5 @@ export const testers = { "streamable": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length === 6), - "twitch": (patternMatch) => ((patternMatch["channel"] && patternMatch["clip"] && patternMatch["clip"].length <= 100 || patternMatch["video"] && patternMatch["video"].length <= 10)), + "twitch": (patternMatch) => ((patternMatch["channel"] && patternMatch["clip"] && patternMatch["clip"].length <= 100)), } From e0af5ef1cf5eea1668cf524251fb168013081fbb Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 16 Sep 2023 18:00:07 +0600 Subject: [PATCH 6/6] one more fix --- src/modules/stream/types.js | 2 +- src/test/tests.json | 28 ---------------------------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index d844ccc..50cd11c 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -152,7 +152,7 @@ export function streamVideoOnly(streamInfo, res) { '-c', 'copy' ] if (streamInfo.mute) args.push('-an'); - if (streamInfo.service === "vimeo" || streamInfo.service === "twitch") args.push('-bsf:a', 'aac_adtstoasc'); + if (streamInfo.service === "vimeo") args.push('-bsf:a', 'aac_adtstoasc'); if (format === "mp4") args.push('-movflags', 'faststart+frag_keyframe+empty_moov'); args.push('-f', format, 'pipe:3'); const ffmpegProcess = spawn(ffmpeg, args, { diff --git a/src/test/tests.json b/src/test/tests.json index 08d3c9a..80b05c9 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -1099,33 +1099,5 @@ "code": 200, "status": "stream" } - }, { - "name": "video", - "url": "https://twitch.tv/videos/1315890970", - "params": {}, - "expected": { - "code": 200, - "status": "stream" - } - }, { - "name": "video (isAudioOnly)", - "url": "https://twitch.tv/videos/1315890970", - "params": { - "isAudioOnly": true - }, - "expected": { - "code": 200, - "status": "stream" - } - }, { - "name": "video (isAudioMuted)", - "url": "https://twitch.tv/videos/1315890970", - "params": { - "isAudioMuted": true - }, - "expected": { - "code": 200, - "status": "stream" - } }] }