From ca0bc9f3958ff7746911cc8ea053dcbde2d12043 Mon Sep 17 00:00:00 2001 From: zjy4fun <168287641+zjy4fun@users.noreply.github.com> Date: Tue, 11 Feb 2025 16:42:07 +0800 Subject: [PATCH 01/17] api/ok: fix author not being handled properly (#1009) --- api/src/processing/services/ok.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/processing/services/ok.js b/api/src/processing/services/ok.js index 10fb785b..cfe18e49 100644 --- a/api/src/processing/services/ok.js +++ b/api/src/processing/services/ok.js @@ -44,7 +44,7 @@ export default async function(o) { let fileMetadata = { title: videoData.movie.title.trim(), - author: (videoData.author?.name || videoData.compilationTitle).trim(), + author: (videoData.author?.name || videoData.compilationTitle)?.trim(), } if (bestVideo) return { From 20b1d9ab30d139968d879fc1325f92fdb8d6724b Mon Sep 17 00:00:00 2001 From: Hk-Gosuto Date: Tue, 11 Feb 2025 16:44:06 +0800 Subject: [PATCH 02/17] web/youtube-lang: add zh, zh-Hans, and zh-Hant language codes (#1076) --- web/src/lib/settings/youtube-lang.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/web/src/lib/settings/youtube-lang.ts b/web/src/lib/settings/youtube-lang.ts index 1a777552..9c470547 100644 --- a/web/src/lib/settings/youtube-lang.ts +++ b/web/src/lib/settings/youtube-lang.ts @@ -72,6 +72,9 @@ export const youtubeLanguages = [ "ur", "uz", "vi", + "zh", + "zh-Hans", + "zh-Hant", "zh-CN", "zh-HK", "zh-TW", From d6b0fbc8ecb26717b165e06ee7d3ad5f34386bec Mon Sep 17 00:00:00 2001 From: Satya Ananda Date: Tue, 11 Feb 2025 15:52:20 +0700 Subject: [PATCH 03/17] api/url: extract loom video id from longer links (#832) Co-authored-by: jj --- api/src/processing/url.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/api/src/processing/url.js b/api/src/processing/url.js index f4a5c5b1..82299999 100644 --- a/api/src/processing/url.js +++ b/api/src/processing/url.js @@ -98,6 +98,14 @@ function aliasURL(url) { if (url.hostname === 'xhslink.com' && parts.length === 3) { url = new URL(`https://www.xiaohongshu.com/a/${parts[2]}`); } + break; + + case "loom": + const idPart = parts[parts.length - 1]; + if (idPart.length > 32) { + url.pathname = `/share/${idPart.slice(-32)}`; + } + break; } return url; From 8f57881a681e08a7a94f6d3bca4758efdaa841fa Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 10 Feb 2025 21:48:30 +0000 Subject: [PATCH 04/17] api/test: use proxy from external proxy env if available --- api/src/util/test.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/src/util/test.js b/api/src/util/test.js index 9457c1ca..25c822cc 100644 --- a/api/src/util/test.js +++ b/api/src/util/test.js @@ -4,6 +4,7 @@ import { env } from "../config.js"; import { runTest } from "../misc/run-test.js"; import { loadJSON } from "../misc/load-from-fs.js"; import { Red, Bright } from "../misc/console-text.js"; +import { setGlobalDispatcher, ProxyAgent } from "undici"; import { randomizeCiphers } from "../misc/randomize-ciphers.js"; import { services } from "../processing/service-config.js"; @@ -64,6 +65,10 @@ const printHeader = (service, padLen) => { console.log(service + '='.repeat(50)); } +if (env.externalProxy) { + setGlobalDispatcher(new ProxyAgent(env.externalProxy)); +} + const action = process.argv[2]; switch (action) { case "get-services": From a06bad161a7eb9fbcd74346b0a0e3b89b8ac0d48 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 10 Feb 2025 21:55:11 +0000 Subject: [PATCH 05/17] api/test: add env for configuring ignored tests --- api/src/util/test.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/src/util/test.js b/api/src/util/test.js index 25c822cc..7dd0450d 100644 --- a/api/src/util/test.js +++ b/api/src/util/test.js @@ -14,7 +14,11 @@ const getTests = (service) => loadJSON(getTestPath(service)); // services that are known to frequently fail due to external // factors (e.g. rate limiting) -const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube', 'vk', 'twitter', 'reddit']); +const finnicky = new Set( + typeof process.env.TEST_IGNORE_SERVICES === 'string' + ? process.env.TEST_IGNORE_SERVICES.split(',') + : ['bilibili', 'instagram', 'facebook', 'youtube', 'vk', 'twitter', 'reddit'] +); const runTestsFor = async (service) => { const tests = getTests(service); From 71d17cc31d1a740cc6f3b4fc744e4769673cb2dd Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 10 Feb 2025 21:55:20 +0000 Subject: [PATCH 06/17] ci: use external proxy for tests --- .github/workflows/test-services.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-services.yml b/.github/workflows/test-services.yml index 77242cb8..993fbcb4 100644 --- a/.github/workflows/test-services.yml +++ b/.github/workflows/test-services.yml @@ -31,3 +31,6 @@ jobs: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 - run: pnpm i --frozen-lockfile && node api/src/util/test run-tests-for ${{ matrix.service }} + env: + API_EXTERNAL_PROXY: ${{ secrets.API_EXTERNAL_PROXY }} + TEST_IGNORE_SERVICES: "none" From 0426621cf5b78f3eb85c1b3282a6648fd311d5a4 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 10 Feb 2025 21:59:11 +0000 Subject: [PATCH 07/17] api/test: fix twitter bookmarked photo link --- api/src/util/tests/twitter.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/util/tests/twitter.json b/api/src/util/tests/twitter.json index 0024d097..7d6be8ad 100644 --- a/api/src/util/tests/twitter.json +++ b/api/src/util/tests/twitter.json @@ -203,11 +203,11 @@ }, { "name": "bookmarked photo", - "url": "https://twitter.com/i/bookmarks?post_id=1837430141179289876", + "url": "https://twitter.com/i/bookmarks?post_id=1887450602164396149", "params": {}, "expected": { "code": 200, "status": "redirect" } } -] \ No newline at end of file +] From d79950b15f91a01eecf5a83ba7ec2d58e5bfcec9 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 10 Feb 2025 22:03:10 +0000 Subject: [PATCH 08/17] api/test: clear a bunch of canFails --- api/src/util/tests/facebook.json | 4 +--- api/src/util/tests/xiaohongshu.json | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/api/src/util/tests/facebook.json b/api/src/util/tests/facebook.json index 876ac7fe..d0c8cc7b 100644 --- a/api/src/util/tests/facebook.json +++ b/api/src/util/tests/facebook.json @@ -29,7 +29,6 @@ { "name": "shortlink video", "url": "https://fb.watch/r1K6XHMfGT/", - "canFail": true, "params": {}, "expected": { "code": 200, @@ -39,7 +38,6 @@ { "name": "reel video", "url": "https://web.facebook.com/reel/730293269054758", - "canFail": true, "params": {}, "expected": { "code": 200, @@ -64,4 +62,4 @@ "status": "redirect" } } -] \ No newline at end of file +] diff --git a/api/src/util/tests/xiaohongshu.json b/api/src/util/tests/xiaohongshu.json index de632a77..0cca9393 100644 --- a/api/src/util/tests/xiaohongshu.json +++ b/api/src/util/tests/xiaohongshu.json @@ -48,7 +48,6 @@ { "name": "short link, wrong id", "url": "https://xhslink.com/a/aaaaaa", - "canFail": true, "params": {}, "expected": { "code": 400, From fcb5023c23cced10f8648f6e10b6aba43dcb000d Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 10 Feb 2025 22:04:57 +0000 Subject: [PATCH 09/17] api/test: always randomize ciphers and override envs --- api/src/util/test.js | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/api/src/util/test.js b/api/src/util/test.js index 7dd0450d..a11130a7 100644 --- a/api/src/util/test.js +++ b/api/src/util/test.js @@ -73,6 +73,10 @@ if (env.externalProxy) { setGlobalDispatcher(new ProxyAgent(env.externalProxy)); } +env.streamLifespan = 10000; +env.apiURL = 'http://x/'; +randomizeCiphers(); + const action = process.argv[2]; switch (action) { case "get-services": @@ -95,9 +99,6 @@ switch (action) { break; case "run-tests-for": - env.streamLifespan = 10000; - env.apiURL = 'http://x/'; - randomizeCiphers(); try { const { softFails } = await runTestsFor(process.argv[3]); @@ -113,10 +114,6 @@ switch (action) { const maxHeaderLen = Object.keys(services).reduce((n, v) => v.length > n ? v.length : n, 0); const failCounters = {}; - env.streamLifespan = 10000; - env.apiURL = 'http://x/'; - randomizeCiphers(); - for (const service in services) { printHeader(service, maxHeaderLen); const { fails, softFails } = await runTestsFor(service); From 92061f2e82f5fc7b5342a16b76fb15849eae71d1 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 10 Feb 2025 22:06:30 +0000 Subject: [PATCH 10/17] api/run-test: print error code for unexpected errors --- api/src/misc/run-test.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/src/misc/run-test.js b/api/src/misc/run-test.js index 2dc1a28a..6dd08183 100644 --- a/api/src/misc/run-test.js +++ b/api/src/misc/run-test.js @@ -23,6 +23,10 @@ export async function runTest(url, params, expect) { if (expect.status !== result.body.status) { const detail = `${expect.status} (expected) != ${result.body.status} (actual)`; error.push(`status mismatch: ${detail}`); + + if (result.body.status === 'error') { + error.push(`error code: ${result.body?.error?.code}`); + } } if (expect.errorCode && expect.errorCode !== result.body?.error?.code) { From d8eda230e8bd2f987ed89e1ace6e11983cd9b861 Mon Sep 17 00:00:00 2001 From: jj Date: Mon, 10 Feb 2025 22:10:35 +0000 Subject: [PATCH 11/17] api/test: fix more twitter tests --- api/src/util/tests/twitter.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/api/src/util/tests/twitter.json b/api/src/util/tests/twitter.json index 7d6be8ad..4fc5900f 100644 --- a/api/src/util/tests/twitter.json +++ b/api/src/util/tests/twitter.json @@ -102,9 +102,8 @@ }, { "name": "retweeted video", - "url": "https://twitter.com/uwukko/status/1696901469633421344", + "url": "https://twitter.com/schlizzawg/status/1869017025055793405", "params": {}, - "canFail": true, "expected": { "code": 200, "status": "redirect" @@ -145,7 +144,7 @@ "params": {}, "expected": { "code": 200, - "status": "redirect" + "status": "tunnel" } }, { @@ -207,7 +206,7 @@ "params": {}, "expected": { "code": 200, - "status": "redirect" + "status": "tunnel" } } ] From 366279a3bc319aa1f34f6dc5fbc4932597772b99 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 11 Feb 2025 15:25:01 +0600 Subject: [PATCH 12/17] web/PickerDialog: don't render an item if it has no url --- web/src/components/dialog/PickerDialog.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web/src/components/dialog/PickerDialog.svelte b/web/src/components/dialog/PickerDialog.svelte index 1fa5f5f5..f2dc4fb0 100644 --- a/web/src/components/dialog/PickerDialog.svelte +++ b/web/src/components/dialog/PickerDialog.svelte @@ -50,7 +50,9 @@
{#if items} {#each items as item, i} - + {#if item.url} + + {/if} {/each} {/if}
From f76d40bec4b919c877a1e370aecbb471d33f2a55 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 11 Feb 2025 15:29:23 +0600 Subject: [PATCH 13/17] web/PickerItem: make sure the item url is valid --- web/src/components/dialog/PickerItem.svelte | 22 +++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/web/src/components/dialog/PickerItem.svelte b/web/src/components/dialog/PickerItem.svelte index 4a515a6e..04e301ec 100644 --- a/web/src/components/dialog/PickerItem.svelte +++ b/web/src/components/dialog/PickerItem.svelte @@ -14,18 +14,28 @@ export let number: number; let imageLoaded = false; - const isTunnel = new URL(item.url).pathname === "/tunnel"; + + let validUrl = false; + try { + new URL(item.url); + validUrl = true; + } catch {} + + const isTunnel = validUrl && new URL(item.url).pathname === "/tunnel"; $: itemType = item.type ?? "photo";