vimeo support revamp and bug fixes

- completely reworked vimeo module.
- added support for audio downloads from vimeo.
- added support for chop type of dash for vimeo.
- added ability to choose between progressive and dash vimeo downloads. both to api and settings on frontend.
- added support for single m3u8 playlists. will be useful for future additions and is currently used for vimeo.
- proper error is now shown if there are no matching vimeo videos found
- temporarily disabled douyin support because bytedance killed off old endpoint.
- fixed the issue related to periods in tiktok usernames. (closes #96)
- fixed error text value patching in match module.
- fixed video stream removal for audio only option, wouldn't work in some edge cases.
- minor clean up.
This commit is contained in:
wukko 2023-03-15 22:18:31 +06:00
parent f6ee934949
commit 6e9f9efa28
16 changed files with 149 additions and 117 deletions

View file

@ -12,7 +12,8 @@ let switchers = {
"vCodec": ["h264", "av1", "vp9"],
"vQuality": ["1080", "max", "2160", "1440", "720", "480", "360"],
"aFormat": ["mp3", "best", "ogg", "wav", "opus"],
"dubLang": ["original", "auto"]
"dubLang": ["original", "auto"],
"vimeoDash": ["false", "true"]
}
let checkboxes = ["disableTikTokWatermark", "fullTikTokAudio", "muteAudio"];
let exceptions = { // used for mobile devices
@ -340,6 +341,7 @@ async function download(url) {
} else if (sGet("dubLang") === "custom") {
req.dubLang = true
}
if (sGet("vimeoDash") === "true") req.vimeoDash = true;
if (sGet("audioMode") === "true") {
req.isAudioOnly = true;
req.isNoTTWatermark = true; // video tiktok no watermark

View file

@ -112,10 +112,12 @@
"ErrorYTUnavailable": "this youtube video is unavailable or age restricted. i am currently unable to download videos with sensitive content. try another one!",
"ErrorYTTryOtherCodec": "i couldn't find anything to download with your settings. try another codec or quality!\n\nnote: youtube api sometimes acts unexpectedly. blame google for this, not me.",
"SettingsCodecSubtitle": "youtube codec",
"SettingsCodecDescription": "h264: generally better player support, but quality tops out at 1080p.\nav1: low player support, but supports 8k & HDR.\nvp9: usually highest bitrate, preserves most detail. supports 4k & HDR.\n\nif you want best editor/player/social media compatibility, pick h264.",
"SettingsCodecDescription": "h264: generally better player support, but quality tops out at 1080p.\nav1: low player support, but supports 8k & HDR.\nvp9: usually highest bitrate, preserves most detail. supports 4k & HDR.\n\npick h264 if you want best editor/player/social media compatibility.",
"SettingsAudioDub": "youtube audio track",
"SettingsAudioDubDescription": "defines which audio track will be used. if dubbed track isn't available, original video language is used instead.\n\noriginal: original video language is used.\nauto: default browser (and {appName}) language is used.",
"SettingsDubDefault": "original",
"SettingsDubAuto": "auto"
"SettingsDubAuto": "auto",
"SettingsVimeoPrefer": "vimeo downloads type",
"SettingsVimeoPreferDescription": "progressive: direct file link to vimeo's cdn. max quality is 1080p.\ndash: video and audio are merged by {appName} into one file. max quality is 4k.\n\npick progressive if you want best editor/player/social media compatibility. sometimes progressive downloads aren't available, and then dash is used instead."
}
}

View file

@ -9,15 +9,11 @@ import match from "./processing/match.js";
export async function getJSON(originalURL, lang, obj) {
try {
let url = decodeURIComponent(originalURL);
if (url.startsWith('http://')) {
return apiJSON(0, { t: errorUnsupported(lang) });
}
let hostname = url.replace("https://", "").replace(' ', '').split('&')[0].split("/")[0].split("."),
host = hostname[hostname.length - 2],
patternMatch;
let patternMatch, url = decodeURIComponent(originalURL),
hostname = new URL(url).hostname.split('.'),
host = hostname[hostname.length - 2];
if (!url.startsWith('https://')) return apiJSON(0, { t: errorUnsupported(lang) });
// TO-DO: bring all tests into one unified module instead of placing them in several places
switch(host) {
case "youtu":
host = "youtube";
@ -25,27 +21,27 @@ export async function getJSON(originalURL, lang, obj) {
break;
case "goo":
if (url.substring(0, 30) === "https://soundcloud.app.goo.gl/") {
host = "soundcloud"
host = "soundcloud";
url = `https://soundcloud.com/${url.replace("https://soundcloud.app.goo.gl/", "").split('/')[0]}`
}
break;
case "tumblr":
if (!url.includes("blog/view")) {
if (url.slice(-1) === '/') url = url.slice(0, -1);
url = url.replace(url.split('/')[5], '');
url = url.replace(url.split('/')[5], '')
}
break;
}
if (!(host && host.length < 20 && host in patterns && patterns[host]["enabled"])) return apiJSON(0, { t: errorUnsupported(lang) });
for (let i in patterns[host]["patterns"]) {
patternMatch = new UrlPattern(patterns[host]["patterns"][i]).match(cleanURL(url, host).split(".com/")[1]);
if (patternMatch) break;
patternMatch = new UrlPattern(patterns[host]["patterns"][i]).match(cleanURL(url, host).split(`.${patterns[host]['tld'] ? patterns[host]['tld'] : "com"}/`)[1].replace('.', ''));
if (patternMatch) break
}
if (!patternMatch) return apiJSON(0, { t: errorUnsupported(lang) });
return await match(host, patternMatch, url, lang, obj);
return await match(host, patternMatch, url, lang, obj)
} catch (e) {
return apiJSON(0, { t: loc(lang, 'ErrorSomethingWentWrong') });
return apiJSON(0, { t: loc(lang, 'ErrorSomethingWentWrong') })
}
}

View file

@ -250,6 +250,20 @@ export default function(obj) {
}]
})
})
+ settingsCategory({
name: t('SettingsVimeoPrefer'),
body: switcher({
name: "vimeoDash",
explanation: t('SettingsVimeoPreferDescription'),
items: [{
"action": "false",
"text": "progressive"
}, {
"action": "true",
"text": "dash"
}]
})
})
}, {
name: "audio",
title: `${emoji("🎶")} ${t('SettingsAudioTab')}`,

View file

@ -21,7 +21,7 @@ export default async function (host, patternMatch, url, lang, obj) {
let r, isAudioOnly = !!obj.isAudioOnly;
if (!testers[host]) return apiJSON(0, { t: errorUnsupported(lang) });
if (!(testers[host](patternMatch))) return apiJSON(0, { t: brokenLink(lang) });
if (!(testers[host](patternMatch))) return apiJSON(0, { t: brokenLink(lang, host) });
switch (host) {
case "twitter":
@ -87,7 +87,9 @@ export default async function (host, patternMatch, url, lang, obj) {
case "vimeo":
r = await vimeo({
id: patternMatch["id"].slice(0, 11),
quality: obj.vQuality
quality: obj.vQuality,
isAudioOnly: isAudioOnly,
forceDash: isAudioOnly ? true : obj.vimeoDash
});
break;
case "soundcloud":

View file

@ -14,6 +14,7 @@ export default function(r, host, ip, audioFormat, isAudioOnly, lang, isAudioMute
params = {}
if (!isAudioOnly && !r.picker && !isAudioMuted) action = "video";
if (r.isM3U8) action = "singleM3U8";
if (isAudioOnly && !r.picker) action = "audio";
if (r.picker) action = "picker";
if (isAudioMuted) action = "muteVideo";
@ -57,7 +58,9 @@ export default function(r, host, ip, audioFormat, isAudioOnly, lang, isAudioMute
break;
}
break;
case "singleM3U8":
params = { type: "videoM3U8" }
break;
case "muteVideo":
params = {
type: Array.isArray(r.urls) ? "bridge" : "mute",
@ -89,7 +92,7 @@ export default function(r, host, ip, audioFormat, isAudioOnly, lang, isAudioMute
break;
case "audio":
if ((host === "reddit" && r.typeId === 1) || (host === "vimeo" && !r.filename) || audioIgnore.includes(host)) return apiJSON(0, { t: loc(lang, 'ErrorEmptyDownload') });
if ((host === "reddit" && r.typeId === 1) || audioIgnore.includes(host)) return apiJSON(0, { t: loc(lang, 'ErrorEmptyDownload') });
let processType = "render";
let copy = false;
@ -120,6 +123,10 @@ export default function(r, host, ip, audioFormat, isAudioOnly, lang, isAudioMute
copy = false
}
}
if (r.isM3U8 || host === "vimeo") {
copy = false;
processType = "render"
}
params = {
type: processType,

View file

@ -54,8 +54,8 @@ export default async function(obj) {
let clientId = await findClientID();
if (!clientId) return { error: 'ErrorSoundCloudNoClientId' };
let fileUrlBase = json.media.transcodings[0]["url"].replace("/hls", "/progressive")
let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`;
let fileUrlBase = json.media.transcodings[0]["url"].replace("/hls", "/progressive"),
fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`;
if (fileUrl.substring(0, 54) !== "https://api-v2.soundcloud.com/media/soundcloud:tracks:") return { error: 'ErrorEmptyDownload' };
if (json.duration > maxVideoDuration) return { error: ['ErrorLengthAudioConvert', maxVideoDuration / 60000] };

View file

@ -4,11 +4,11 @@ const userAgent = genericUserAgent.split(' Chrome/1')[0],
config = {
tiktok: {
short: "https://vt.tiktok.com/",
api: "https://api2.musical.ly/aweme/v1/feed/?aweme_id={postId}&version_code=262&app_name=musical_ly&channel=App&device_id=null&os_version=14.4.2&device_platform=iphone&device_type=iPhone9&region=US&carrier_region=US",
api: "https://api2.musical.ly/aweme/v1/feed/?aweme_id={postId}&version_code=262&app_name=musical_ly&channel=App&device_id=null&os_version=14.4.2&device_platform=iphone&device_type=iPhone9&region=US&carrier_region=US"
},
douyin: {
short: "https://v.douyin.com/",
api: "https://www.iesdouyin.com/aweme/v1/web/aweme/detail/?aweme_id={postId}",
api: "https://www.iesdouyin.com/aweme/v1/web/aweme/detail/?aweme_id={postId}"
}
}

View file

@ -81,8 +81,8 @@ export default async function(obj) {
).then((r) =>{ return r.status === 200 ? r.json() : false }).catch(() => { return false });
if (!streamStatus) return { error: 'ErrorCouldntFetch' };
let participants = AudioSpaceById.data.audioSpace.participants.speakers;
let listOfParticipants = `Twitter Space speakers: `;
let participants = AudioSpaceById.data.audioSpace.participants.speakers,
listOfParticipants = `Twitter Space speakers: `;
for (let i in participants) { listOfParticipants += `@${participants[i]["twitter_screen_name"]}, ` }
listOfParticipants = listOfParticipants.slice(0, -2);

View file

@ -2,44 +2,50 @@ import { maxVideoDuration } from "../../config.js";
const resolutionMatch = {
"3840": "2160",
"2732": "1440",
"2048": "1080",
"1920": "1080",
"1366": "720",
"1280": "720",
"960": "480"
"960": "480",
"640": "360",
"426": "240"
}
// ^ vimeo you're fucked in the head for this ^
const qualityMatch = {
"2160": "4K",
"1440": "2K",
"480": "540",
"4K": "2160",
"2K": "1440",
"540": "480"
}
export default async function(obj) {
let quality = obj.quality === "max" ? "9000" : obj.quality;
if (!quality || obj.isAudioOnly) quality = "9000";
let api = await fetch(`https://player.vimeo.com/video/${obj.id}/config`).then((r) => { return r.json() }).catch(() => { return false });
if (!api) return { error: 'ErrorCouldntFetch' };
let downloadType = "dash";
if (JSON.stringify(api).includes('"progressive":[{')) downloadType = "progressive";
if (!obj.forceDash && JSON.stringify(api).includes('"progressive":[{')) downloadType = "progressive";
switch(downloadType) {
case "progressive":
if (downloadType !== "dash") {
if (qualityMatch[quality]) quality = qualityMatch[quality];
let all = api["request"]["files"]["progressive"].sort((a, b) => Number(b.width) - Number(a.width));
let best = all[0];
try {
if (obj.quality !== "max") {
let pref = parseInt(obj.quality, 10)
for (let i in all) {
let currQuality = parseInt(all[i]["quality"].replace('p', ''), 10)
if (currQuality === pref) {
best = all[i];
break
}
if (currQuality < pref) {
best = all[i-1];
break
}
}
}
} catch (e) {
best = all[0]
let bestQuality = all[0]["quality"].split('p')[0];
bestQuality = qualityMatch[bestQuality] ? qualityMatch[bestQuality] : bestQuality;
if (Number(quality) < Number(bestQuality)) best = all.find(i => i["quality"].split('p')[0] === quality);
if (!best) return { error: 'ErrorEmptyDownload' };
return { urls: best["url"], audioFilename: `vimeo_${obj.id}_audio`, filename: `vimeo_${obj.id}_${best["width"]}x${best["height"]}.mp4` }
}
return { urls: best["url"], filename: `tumblr_${obj.id}.mp4` };
case "dash":
if (api.video.duration > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
let masterJSONURL = api["request"]["files"]["dash"]["cdns"]["akfire_interconnect_quic"]["url"];
@ -51,34 +57,29 @@ export default async function(obj) {
let type = "parcel";
if (masterJSON.base_url === "../") type = "chop";
let masterJSON_Video = masterJSON.video.sort((a, b) => Number(b.width) - Number(a.width));
let masterJSON_Audio = masterJSON.audio.sort((a, b) => Number(b.bitrate) - Number(a.bitrate)).filter((a)=> {if (a['mime_type'] === "audio/mp4") return true;});
let bestVideo = masterJSON_Video[0], bestAudio = masterJSON_Audio[0];
let masterJSON_Video = masterJSON.video.sort((a, b) => Number(b.width) - Number(a.width)),
bestVideo = masterJSON_Video[0];
if (Number(quality) < Number(resolutionMatch[bestVideo["width"]])) bestVideo = masterJSON_Video.find(i => resolutionMatch[i["width"]] === quality);
let videoUrl, audioUrl, baseUrl = masterJSONURL.split("/sep/")[0];
switch (type) {
case "parcel":
if (obj.quality !== "max") {
let pref = parseInt(obj.quality, 10)
for (let i in masterJSON_Video) {
let currQuality = parseInt(resolutionMatch[masterJSON_Video[i]["width"]], 10)
if (currQuality < pref) {
break;
} else if (String(currQuality) === String(pref)) {
bestVideo = masterJSON_Video[i]
}
}
}
let baseUrl = masterJSONURL.split("/sep/")[0];
let videoUrl = `${baseUrl}/parcel/video/${bestVideo.index_segment.split('?')[0]}`,
let masterJSON_Audio = masterJSON.audio.sort((a, b) => Number(b.bitrate) - Number(a.bitrate)).filter((a) => { if (a['mime_type'] === "audio/mp4") return true }),
bestAudio = masterJSON_Audio[0];
videoUrl = `${baseUrl}/parcel/video/${bestVideo.index_segment.split('?')[0]}`,
audioUrl = `${baseUrl}/parcel/audio/${bestAudio.index_segment.split('?')[0]}`;
return { urls: [videoUrl, audioUrl], audioFilename: `vimeo_${obj.id}_audio`, filename: `vimeo_${obj.id}_${bestVideo["width"]}x${bestVideo["height"]}.mp4` }
case "chop": // TO-DO: support for chop stream type
default:
break;
case "chop":
videoUrl = `${baseUrl}/sep/video/${bestVideo.id}/master.m3u8`;
break;
}
if (videoUrl) {
return {
urls: audioUrl ? [videoUrl, audioUrl] : videoUrl,
isM3U8: audioUrl ? false : true,
audioFilename: `vimeo_${obj.id}_audio`,
filename: `vimeo_${obj.id}_${bestVideo["width"]}x${bestVideo["height"]}.mp4`
}
}
return { error: 'ErrorEmptyDownload' }
}
default:
return { error: 'ErrorEmptyDownload' }
}
}

View file

@ -10,8 +10,7 @@ const representationMatch = {
"360": 2,
"240": 1,
"144": 0
}
const resolutionMatch = {
}, resolutionMatch = {
"3840": "2160",
"2560": "1440",
"1920": "1080",
@ -30,16 +29,16 @@ export default async function(o) {
if (!html) return { error: 'ErrorCouldntFetch' };
if (!html.includes(`{"lang":`)) return { error: 'ErrorEmptyDownload' };
let quality = o.quality === "max" ? 7 : representationMatch[o.quality];
let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]);
let quality = o.quality === "max" ? 7 : representationMatch[o.quality],
js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]);
if (Number(js.mvData.is_active_live) !== 0) return { error: 'ErrorLiveVideo' };
if (js.mvData.duration > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
let mpd = JSON.parse(xml2json(js.player.params[0]["manifest"], { compact: true, spaces: 4 }));
let repr = mpd.MPD.Period.AdaptationSet.Representation ? mpd.MPD.Period.AdaptationSet.Representation : mpd.MPD.Period.AdaptationSet[0]["Representation"];
let bestQuality = repr[repr.length - 1];
let resolutionPick = Number(bestQuality._attributes.width) > Number(bestQuality._attributes.height) ? 'width': 'height'
let mpd = JSON.parse(xml2json(js.player.params[0]["manifest"], { compact: true, spaces: 4 })),
repr = mpd.MPD.Period.AdaptationSet.Representation ? mpd.MPD.Period.AdaptationSet.Representation : mpd.MPD.Period.AdaptationSet[0]["Representation"],
bestQuality = repr[repr.length - 1],
resolutionPick = Number(bestQuality._attributes.width) > Number(bestQuality._attributes.height) ? 'width': 'height';
if (Number(bestQuality._attributes.id) > Number(quality)) bestQuality = repr[quality];
if (bestQuality) return {

View file

@ -44,8 +44,8 @@ export default async function(o) {
if (!bestQuality && !o.isAudioOnly || !hasAudio) return { error: 'ErrorYTTryOtherCodec' };
if (info.basic_info.duration > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
let checkBestAudio = (i) => (i["has_audio"] && !i["has_video"]);
let audio = adaptive_formats.find(i => checkBestAudio(i) && i["is_original"]);
let checkBestAudio = (i) => (i["has_audio"] && !i["has_video"]),
audio = adaptive_formats.find(i => checkBestAudio(i) && i["is_original"]);
if (o.dubLang) {
let dubbedAudio = adaptive_formats.find(i => checkBestAudio(i) && i["language"] === o.dubLang);
@ -74,11 +74,11 @@ export default async function(o) {
return r
}
let checkSingle = (i) => ((i['quality_label'].split('p')[0] === quality || i['quality_label'].split('p')[0] === bestQuality) && i["mime_type"].includes(c[o.format].codec));
let checkBestVideo = (i) => (i['quality_label'].split('p')[0] === bestQuality && !i["has_audio"] && i["has_video"]);
let checkRightVideo = (i) => (i['quality_label'].split('p')[0] === quality && !i["has_audio"] && i["has_video"]);
let checkSingle = (i) => ((i['quality_label'].split('p')[0] === quality || i['quality_label'].split('p')[0] === bestQuality) && i["mime_type"].includes(c[o.format].codec)),
checkBestVideo = (i) => (i['quality_label'].split('p')[0] === bestQuality && !i["has_audio"] && i["has_video"]),
checkRightVideo = (i) => (i['quality_label'].split('p')[0] === quality && !i["has_audio"] && i["has_video"]);
if (!o.isAudioOnly && !o.isAudioMuted) {
if (!o.isAudioOnly && !o.isAudioMuted && o.format === 'h264') {
let single = info.streaming_data.formats.find(i => checkSingle(i));
if (single) return {
type: "bridge",

View file

@ -40,11 +40,12 @@
"douyin": {
"alias": "douyin videos & audio",
"patterns": ["video/:postId", ":id"],
"enabled": true
"enabled": false
},
"vimeo": {
"patterns": [":id"],
"enabled": true
"enabled": true,
"bestAudio": "mp3"
},
"soundcloud": {
"patterns": [":author/:song/s-:accessKey", ":author/:song", ":shortLink"],

View file

@ -17,6 +17,7 @@ export default function(res, ip, id, hmac, exp) {
case "render":
streamLiveRender(streamInfo, res);
break;
case "videoM3U8":
case "mute":
streamVideoOnly(streamInfo, res);
break;

View file

@ -92,11 +92,15 @@ export function streamAudioOnly(streamInfo, res) {
args.push('-vn')
}
args = args.concat(metadataManager(streamInfo.metadata))
} else {
args.push('-vn')
}
let arg = streamInfo.copy ? ffmpegArgs["copy"] : ffmpegArgs["audio"]
args = args.concat(arg)
let arg = streamInfo.copy ? ffmpegArgs["copy"] : ffmpegArgs["audio"];
args = args.concat(arg);
if (ffmpegArgs[streamInfo.audioFormat]) args = args.concat(ffmpegArgs[streamInfo.audioFormat]);
args.push('-f', streamInfo.audioFormat === "m4a" ? "ipod" : streamInfo.audioFormat, 'pipe:3');
const ffmpegProcess = spawn(ffmpeg, args, {
windowsHide: true,
stdio: [
@ -126,9 +130,11 @@ export function streamVideoOnly(streamInfo, res) {
let format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1], args = [
'-loglevel', '-8',
'-i', streamInfo.urls,
'-c', 'copy', '-an'
'-c', 'copy'
]
if (format === "mp4") args.push('-movflags', 'faststart+frag_keyframe+empty_moov')
if (streamInfo.mute) args.push('-an');
if (streamInfo.service === "vimeo") args.push('-bsf:a', 'aac_adtstoasc');
if (format === "mp4") args.push('-movflags', 'faststart+frag_keyframe+empty_moov');
args.push('-f', format, 'pipe:3');
const ffmpegProcess = spawn(ffmpeg, args, {
windowsHide: true,
@ -138,7 +144,7 @@ export function streamVideoOnly(streamInfo, res) {
],
});
res.setHeader('Connection', 'keep-alive');
res.setHeader('Content-Disposition', `attachment; filename="${streamInfo.filename.split('.')[0]}_mute.${format}"`);
res.setHeader('Content-Disposition', `attachment; filename="${streamInfo.filename.split('.')[0]}${streamInfo.mute ? '_mute' : ''}.${format}"`);
ffmpegProcess.stdio[3].pipe(res);
ffmpegProcess.on('disconnect', () => ffmpegProcess.kill());

View file

@ -6,7 +6,7 @@ let apiVar = {
vQuality: ["max", "4320", "2160", "1440", "1080", "720", "480", "360", "240", "144"],
aFormat: ["best", "mp3", "ogg", "wav", "opus"]
},
booleanOnly: ["isAudioOnly", "isNoTTWatermark", "isTTFullAudio", "isAudioMuted", "dubLang"]
booleanOnly: ["isAudioOnly", "isNoTTWatermark", "isTTFullAudio", "isAudioMuted", "dubLang", "vimeoDash"]
}
export function apiJSON(type, obj) {
@ -104,7 +104,8 @@ export function checkJSONPost(obj) {
isNoTTWatermark: false,
isTTFullAudio: false,
isAudioMuted: false,
dubLang: false
dubLang: false,
vimeoDash: false
}
try {
let objKeys = Object.keys(obj);