From ad9b6ebdd38f5a8711621f09dd6cb5c0b6d4e8ee Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 16 Sep 2023 16:27:53 +0600 Subject: [PATCH] twitch clean up --- src/modules/processing/hostOverrides.js | 8 + src/modules/processing/match.js | 6 +- src/modules/processing/matchActionDecider.js | 2 +- src/modules/processing/services/twitch.js | 377 ++++++++---------- src/modules/processing/servicesConfig.json | 6 +- .../processing/servicesPatternTesters.js | 4 +- src/modules/sub/utils.js | 55 ++- 7 files changed, 233 insertions(+), 225 deletions(-) diff --git a/src/modules/processing/hostOverrides.js b/src/modules/processing/hostOverrides.js index a9f2fc2..88553e3 100644 --- a/src/modules/processing/hostOverrides.js +++ b/src/modules/processing/hostOverrides.js @@ -8,6 +8,9 @@ export default function (inHost, inURL) { url = url.split("?")[0].replace("www.", ""); url = `https://youtube.com/watch?v=${url.replace("https://youtube.com/live/", "")}` } + if (url.includes('youtube.com/shorts/')) { + url = url.split('?')[0].replace('shorts/', 'watch?v='); + } break; case "youtu": if (url.startsWith("https://youtu.be/")) { @@ -32,6 +35,11 @@ export default function (inHost, inURL) { url = url.replace(url.split('/')[5], '') } break; + case "twitch": + if (url.includes('clips.twitch.tv')) { + url = url.split('?')[0].replace('clips.twitch.tv/', 'twitch.tv/_/clip/'); + } + break; } return { host: host, diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 6425f93..5c34b6d 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -127,10 +127,10 @@ export default async function (host, patternMatch, url, lang, obj) { r = await twitch({ vodId: patternMatch["video"] ? patternMatch["video"] : false, clipId: patternMatch["clip"] ? patternMatch["clip"] : false, - lang: lang, quality: obj.vQuality, - isAudioOnly: obj.isAudioOnly, - format: obj.vFormat + quality: obj.vQuality, + isAudioOnly: obj.isAudioOnly }); + break; default: return apiJSON(0, { t: errorUnsupported(lang) }); } diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index 258fd81..82eb341 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -55,7 +55,7 @@ export default function(r, host, audioFormat, isAudioOnly, lang, isAudioMuted, d case "tiktok": params = { type: "bridge" }; break; - + case "vine": case "instagram": case "tumblr": diff --git a/src/modules/processing/services/twitch.js b/src/modules/processing/services/twitch.js index bd8a869..31b239e 100644 --- a/src/modules/processing/services/twitch.js +++ b/src/modules/processing/services/twitch.js @@ -1,223 +1,180 @@ import { maxVideoDuration } from "../../config.js"; +import { getM3U8Formats } from "../../sub/utils.js"; const gqlURL = "https://gql.twitch.tv/gql"; const m3u8URL = "https://usher.ttvnw.net"; +const clientIdHead = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" }; -function parseM3U8Line(line) { - const result = {}; - - let str = '', inQuotes = false, keyName = null, escaping = false; - for (let i = 0; i < line.length; i++) { - const char = line[i]; - if (char === '"' && !escaping) { - inQuotes = !inQuotes; - continue; - } else if (char === ',' && !escaping && !inQuotes) { - if (!keyName) break; - result[keyName] = str; - keyName = null; - str = ''; - continue; - } else if (char === '\\' && !escaping) { - escaping = true; - continue; - } else if (char === '=' && !escaping && !inQuotes) { - keyName = str; - str = ''; - continue; - } - - str += char; - escaping = false; - } - - if (keyName) result[keyName] = str; - return result; -} - -function getM3U8Formats(m3u8body) { - let formats = []; - const formatLines = m3u8body.split('\n').slice(2); - - for (let i = 0; i < formatLines.length; i += 3) { - const mediaLine = parseM3U8Line(formatLines[i].split(':')[1]); - const streamLine = parseM3U8Line(formatLines[i + 1].split(':')[1]); - formats.push({ - id: mediaLine['GROUP-ID'], - name: mediaLine.NAME, - resolution: streamLine.RESOLUTION ? streamLine.RESOLUTION.split('x') : null, - url: formatLines[i + 2] - }); - } - return formats; -}; - -export default async function(obj) { - try { - let _headers = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" }; - - if (!obj.clipId && !obj.vodId) return { error: 'ErrorCantGetID' }; - - if (obj.vodId) { - const req_metadata = await fetch(gqlURL, { - method: "POST", - headers: _headers, - body: JSON.stringify([ - { - "operationName": "VideoMetadata", - "variables": { - "channelLogin": "", - "videoID": obj.vodId - }, - "extensions": { - "persistedQuery": { - "version": 1, - "sha256Hash": "226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687" - } - } - } - ]) - }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); - if (!req_metadata) return { error: 'ErrorCouldntFetch' }; - const vodMetadata = req_metadata[0].data.video; - - if (vodMetadata.previewThumbnailURL.endsWith('/404_processing_{width}x{height}.png')) return { error: 'ErrorLiveVideo' }; - if (vodMetadata.lengthSeconds > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; - if (!vodMetadata.owner) return { error: 'ErrorEmptyDownload' }; // Streamer was banned... - - const req_token = await fetch(gqlURL, { - method: "POST", - headers: _headers, - body: JSON.stringify({ - query: `{ - videoPlaybackAccessToken( - id: "${obj.vodId}", - params: { - platform: "web", - playerBackend: "mediaplayer", - playerType: "site" - } - ) - { - value - signature - } - }` - }) - }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); - if (!req_token) return { error: 'ErrorCouldntFetch' }; - - const access_token = req_token.data.videoPlaybackAccessToken; - const req_m3u8 = await fetch(`${m3u8URL}/vod/${obj.vodId}.m3u8?${new URLSearchParams({ - allow_source: 'true', - allow_audio_only: 'true', - allow_spectre: 'true', - player: 'twitchweb', - playlist_include_framerate: 'true', - nauth: access_token.value, - nauthsig: access_token.signature - })}`, { - headers: _headers - }).then((r) => { return r.status == 200 ? r.text() : false;}).catch(() => {return false}); - if (!req_m3u8) return { error: 'ErrorCouldntFetch' }; - - const formats = getM3U8Formats(req_m3u8); - const generalMeta = { - title: vodMetadata.title, - artist: `Twitch Broadcast by @${vodMetadata.owner.login}`, - } - - if (!obj.isAudioOnly) { - const format = formats.find(f => f.resolution && f.resolution[1] == obj.quality) || formats[0]; - - return { - urls: format.url, - isM3U8: true, - time: vodMetadata.lengthSeconds * 1000, - filename: `twitchvod_${obj.vodId}_${format.resolution[0]}x${format.resolution[1]}.mp4` - }; - } else { - return { - type: "render", - isM3U8: true, - time: vodMetadata.lengthSeconds * 1000, - urls: formats.find(f => f.id === 'audio_only').url, - audioFilename: `twitchvod_${obj.vodId}_audio`, - fileMetadata: generalMeta +async function getClip(obj) { + let req_metadata = await fetch(gqlURL, { + method: "POST", + headers: clientIdHead, + body: JSON.stringify({ + query: `{ + clip(slug: "${obj.clipId}") { + broadcaster { + login + } + createdAt + curator { + login + } + durationSeconds + id + medium: thumbnailURL(width: 480, height: 272) + title + videoQualities { + quality + sourceURL } } - } else if (obj.clipId) { - const req_metadata = await fetch(gqlURL, { - method: "POST", - headers: _headers, - body: JSON.stringify({ - query: `{ - clip(slug: "${obj.clipId}") { - broadcaster { - login - } - createdAt - curator { - login - } - durationSeconds - id - medium: thumbnailURL(width: 480, height: 272) - title - videoQualities { - quality - sourceURL - } - } - }` - }) - }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); - if (!req_metadata) return { error: 'ErrorCouldntFetch' }; - const clipMetadata = req_metadata.data.clip; - if (clipMetadata.durationSeconds > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; - if (!clipMetadata.videoQualities || !clipMetadata.broadcaster) return { error: 'ErrorEmptyDownload' }; // Streamer was banned... + }` + }) + }).then((r) => { return r.status === 200 ? r.json() : false; }).catch(() => { return false }); + if (!req_metadata) return { error: 'ErrorCouldntFetch' }; - const req_token = await fetch(gqlURL, { - method: "POST", - headers: _headers, - body: JSON.stringify([ - { - "operationName": "VideoAccessToken_Clip", - "variables": { - "slug": obj.clipId - }, - "extensions": { - "persistedQuery": { - "version": 1, - "sha256Hash": "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11" - } - } - } - ]) - }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); - if (!req_token) return { error: 'ErrorCouldntFetch' }; + let clipMetadata = req_metadata.data.clip; - const generalMeta = { - title: clipMetadata.title, - artist: `Twitch Clip by @${clipMetadata.broadcaster.login}, clipped by @${clipMetadata.curator.login}`, + if (clipMetadata.durationSeconds > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + if (!clipMetadata.videoQualities || !clipMetadata.broadcaster) return { error: 'ErrorEmptyDownload' }; + + let req_token = await fetch(gqlURL, { + method: "POST", + headers: clientIdHead, + body: JSON.stringify([ + { + "operationName": "VideoAccessToken_Clip", + "variables": { + "slug": obj.clipId + }, + "extensions": { + "persistedQuery": { + "version": 1, + "sha256Hash": "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11" + } + } } + ]) + }).then((r) => { return r.status === 200 ? r.json() : false; }).catch(() => { return false }); - const access_token = req_token[0].data.clip.playbackAccessToken; - const formats = clipMetadata.videoQualities; - const format = formats.find(f => f.quality == obj.quality) || formats[0]; + if (!req_token) return { error: 'ErrorCouldntFetch' }; - return { - type: "bridge", - urls: `${format.sourceURL}?${new URLSearchParams({ - sig: access_token.signature, - token: access_token.value - })}`, - filename: `twitchclip_${clipMetadata.id}_${format.quality}.mp4`, - audioFilename: `twitchclip_${clipMetadata.id}_audio`, - fileMetadata: generalMeta - }; - } - } catch (err) { - return { error: 'ErrorBadFetch' }; + let formats = clipMetadata.videoQualities; + let format = formats.find(f => f.quality === obj.quality) || formats[0]; + + return { + type: "bridge", + urls: `${format.sourceURL}?${new URLSearchParams({ + sig: req_token[0].data.clip.playbackAccessToken.signature, + token: req_token[0].data.clip.playbackAccessToken.value + })}`, + fileMetadata: { + title: clipMetadata.title, + artist: `Twitch Clip by @${clipMetadata.broadcaster.login}, clipped by @${clipMetadata.curator.login}`, + }, + filename: `twitchclip_${clipMetadata.id}_${format.quality}p.mp4`, + audioFilename: `twitchclip_${clipMetadata.id}_audio` } -} \ No newline at end of file +} +async function getVideo(obj) { + let req_metadata = await fetch(gqlURL, { + method: "POST", + headers: clientIdHead, + body: JSON.stringify([ + { + "operationName": "VideoMetadata", + "variables": { + "channelLogin": "", + "videoID": obj.vodId + }, + "extensions": { + "persistedQuery": { + "version": 1, + "sha256Hash": "226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687" + } + } + } + ]) + }).then((r) => { return r.status === 200 ? r.json() : false; }).catch(() => { return false }); + if (!req_metadata) return { error: 'ErrorCouldntFetch' }; + + let vodMetadata = req_metadata[0].data.video; + + if (vodMetadata.previewThumbnailURL.endsWith('/404_processing_{width}x{height}.png')) return { error: 'ErrorLiveVideo' }; + if (vodMetadata.lengthSeconds > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + if (!vodMetadata.owner) return { error: 'ErrorEmptyDownload' }; + + let req_token = await fetch(gqlURL, { + method: "POST", + headers: clientIdHead, + body: JSON.stringify({ + query: `{ + videoPlaybackAccessToken( + id: "${obj.vodId}", + params: { + platform: "web", + playerBackend: "mediaplayer", + playerType: "site" + } + ) + { + value + signature + } + }` + }) + }).then((r) => { return r.status === 200 ? r.json() : false; }).catch(() => { return false }); + if (!req_token) return { error: 'ErrorCouldntFetch' }; + + let req_m3u8 = await fetch( + `${m3u8URL}/vod/${obj.vodId}.m3u8?${ + new URLSearchParams({ + allow_source: 'true', + allow_audio_only: 'true', + allow_spectre: 'true', + player: 'twitchweb', + playlist_include_framerate: 'true', + nauth: req_token.data.videoPlaybackAccessToken.value, + nauthsig: req_token.data.videoPlaybackAccessToken.signature + } + )}`, { + headers: clientIdHead + } + ).then((r) => { return r.status === 200 ? r.text() : false; }).catch(() => { return false }); + + if (!req_m3u8) return { error: 'ErrorCouldntFetch' }; + + let formats = getM3U8Formats(req_m3u8); + let generalMeta = { + title: vodMetadata.title, + artist: `Twitch Broadcast by @${vodMetadata.owner.login}`, + } + if (obj.isAudioOnly) { + return { + type: "render", + isM3U8: true, + time: vodMetadata.lengthSeconds * 1000, + urls: formats.find(f => f.id === 'audio_only').url, + audioFilename: `twitchvod_${obj.vodId}_audio`, + fileMetadata: generalMeta + } + } else { + let format = formats.find(f => f.resolution && f.resolution[1] === obj.quality) || formats[0]; + return { + urls: format.url, + isM3U8: true, + time: vodMetadata.lengthSeconds * 1000, + filename: `twitchvod_${obj.vodId}_${format.resolution[0]}x${format.resolution[1]}.mp4`, + fileMetadata: generalMeta + } + } +} +export default async function (obj) { + let response = { error: 'ErrorEmptyDownload' }; + if (obj.clipId) { + response = await getClip(obj) + } else if (obj.vodId) { + response = await getVideo(obj) + } + return response +} diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index f904eaf..e65b4f3 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -73,11 +73,11 @@ "patterns": [":id", "o/:id", "e/:id", "s/:id"], "enabled": true }, - "twitch": { - "alias": "twitch vods & videos & clips", + "twitch": { + "alias": "twitch clips, videos & vods", "tld": "tv", "patterns": ["videos/:video", ":channel/clip/:clip"], "enabled": true } - } + } } diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index cdc2ca4..19fca2e 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -22,12 +22,12 @@ export const testers = { || (patternMatch["id"] && patternMatch["id"].length < 21 && patternMatch["user"] && patternMatch["user"].length <= 32)), "vimeo": (patternMatch) => ((patternMatch["id"] && patternMatch["id"].length <= 11)), - + "soundcloud": (patternMatch) => (patternMatch["author"]?.length <= 25 && patternMatch["song"]?.length <= 255) || (patternMatch["shortLink"] && patternMatch["shortLink"].length <= 32), "instagram": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length <= 12), - + "vine": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length <= 12), "pinterest": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length <= 128), diff --git a/src/modules/sub/utils.js b/src/modules/sub/utils.js index 6f6886c..b38fc39 100644 --- a/src/modules/sub/utils.js +++ b/src/modules/sub/utils.js @@ -70,12 +70,6 @@ export function cleanURL(url, host) { url = url.replaceAll(forbiddenChars[i], '') } url = url.replace('https//', 'https://') - if (url.includes('youtube.com/shorts/')) { - url = url.split('?')[0].replace('shorts/', 'watch?v='); - } - if (url.includes('clips.twitch.tv')) { - url = url.split('?')[0].replace('clips.twitch.tv/', 'twitch.tv/_/clip/'); - } return url.slice(0, 128) } export function cleanString(string) { @@ -155,3 +149,52 @@ export function cleanHTML(html) { clean = clean.replace(/\n/g, ''); return clean } + +export function parseM3U8Line(line) { + let result = {}; + let str = '', inQuotes = false, keyName = null, escaping = false; + + for (let i = 0; i < line.length; i++) { + const char = line[i]; + if (char === '"' && !escaping) { + inQuotes = !inQuotes; + continue + } else if (char === ',' && !escaping && !inQuotes) { + if (!keyName) break; + result[keyName] = str; + keyName = null; + str = ''; + continue + } else if (char === '\\' && !escaping) { + escaping = true; + continue + } else if (char === '=' && !escaping && !inQuotes) { + keyName = str; + str = ''; + continue + } + + str += char; + escaping = false + } + + if (keyName) result[keyName] = str; + return result +} + +export function getM3U8Formats(m3u8body) { + const formatLines = m3u8body.split('\n').slice(2); + let formats = []; + + for (let i = 0; i < formatLines.length; i += 3) { + const mediaLine = parseM3U8Line(formatLines[i].split(':')[1]); + const streamLine = parseM3U8Line(formatLines[i + 1].split(':')[1]); + formats.push({ + id: mediaLine['GROUP-ID'], + name: mediaLine.NAME, + resolution: streamLine.RESOLUTION ? streamLine.RESOLUTION.split('x') : null, + url: formatLines[i + 2] + }) + } + return formats +}