diff --git a/.github/workflows/test-services.yml b/.github/workflows/test-services.yml index 77242cb8..993fbcb4 100644 --- a/.github/workflows/test-services.yml +++ b/.github/workflows/test-services.yml @@ -31,3 +31,6 @@ jobs: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 - run: pnpm i --frozen-lockfile && node api/src/util/test run-tests-for ${{ matrix.service }} + env: + API_EXTERNAL_PROXY: ${{ secrets.API_EXTERNAL_PROXY }} + TEST_IGNORE_SERVICES: "none" diff --git a/api/package.json b/api/package.json index b46f0a6a..0660b0d8 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "10.7.2", + "version": "10.7.3", "author": "imput", "exports": "./src/cobalt.js", "type": "module", diff --git a/api/src/misc/run-test.js b/api/src/misc/run-test.js index 2dc1a28a..6dd08183 100644 --- a/api/src/misc/run-test.js +++ b/api/src/misc/run-test.js @@ -23,6 +23,10 @@ export async function runTest(url, params, expect) { if (expect.status !== result.body.status) { const detail = `${expect.status} (expected) != ${result.body.status} (actual)`; error.push(`status mismatch: ${detail}`); + + if (result.body.status === 'error') { + error.push(`error code: ${result.body?.error?.code}`); + } } if (expect.errorCode && expect.errorCode !== result.body?.error?.code) { diff --git a/api/src/processing/services/instagram.js b/api/src/processing/services/instagram.js index eab4776e..9cc7dbdf 100644 --- a/api/src/processing/services/instagram.js +++ b/api/src/processing/services/instagram.js @@ -305,12 +305,12 @@ export default function instagram(obj) { if (sidecar) { const picker = sidecar.edges.filter(e => e.node?.display_url) .map((e, i) => { - const type = e.node?.is_video ? "video" : "photo"; + const type = e.node?.is_video && e.node?.video_url ? "video" : "photo"; let url; - if (type === 'video') { + if (type === "video") { url = e.node?.video_url; - } else if (type === 'photo') { + } else if (type === "photo") { url = e.node?.display_url; } diff --git a/api/src/processing/services/ok.js b/api/src/processing/services/ok.js index 10fb785b..cfe18e49 100644 --- a/api/src/processing/services/ok.js +++ b/api/src/processing/services/ok.js @@ -44,7 +44,7 @@ export default async function(o) { let fileMetadata = { title: videoData.movie.title.trim(), - author: (videoData.author?.name || videoData.compilationTitle).trim(), + author: (videoData.author?.name || videoData.compilationTitle)?.trim(), } if (bestVideo) return { diff --git a/api/src/processing/url.js b/api/src/processing/url.js index f4a5c5b1..82299999 100644 --- a/api/src/processing/url.js +++ b/api/src/processing/url.js @@ -98,6 +98,14 @@ function aliasURL(url) { if (url.hostname === 'xhslink.com' && parts.length === 3) { url = new URL(`https://www.xiaohongshu.com/a/${parts[2]}`); } + break; + + case "loom": + const idPart = parts[parts.length - 1]; + if (idPart.length > 32) { + url.pathname = `/share/${idPart.slice(-32)}`; + } + break; } return url; diff --git a/api/src/util/test.js b/api/src/util/test.js index 9457c1ca..a11130a7 100644 --- a/api/src/util/test.js +++ b/api/src/util/test.js @@ -4,6 +4,7 @@ import { env } from "../config.js"; import { runTest } from "../misc/run-test.js"; import { loadJSON } from "../misc/load-from-fs.js"; import { Red, Bright } from "../misc/console-text.js"; +import { setGlobalDispatcher, ProxyAgent } from "undici"; import { randomizeCiphers } from "../misc/randomize-ciphers.js"; import { services } from "../processing/service-config.js"; @@ -13,7 +14,11 @@ const getTests = (service) => loadJSON(getTestPath(service)); // services that are known to frequently fail due to external // factors (e.g. rate limiting) -const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube', 'vk', 'twitter', 'reddit']); +const finnicky = new Set( + typeof process.env.TEST_IGNORE_SERVICES === 'string' + ? process.env.TEST_IGNORE_SERVICES.split(',') + : ['bilibili', 'instagram', 'facebook', 'youtube', 'vk', 'twitter', 'reddit'] +); const runTestsFor = async (service) => { const tests = getTests(service); @@ -64,6 +69,14 @@ const printHeader = (service, padLen) => { console.log(service + '='.repeat(50)); } +if (env.externalProxy) { + setGlobalDispatcher(new ProxyAgent(env.externalProxy)); +} + +env.streamLifespan = 10000; +env.apiURL = 'http://x/'; +randomizeCiphers(); + const action = process.argv[2]; switch (action) { case "get-services": @@ -86,9 +99,6 @@ switch (action) { break; case "run-tests-for": - env.streamLifespan = 10000; - env.apiURL = 'http://x/'; - randomizeCiphers(); try { const { softFails } = await runTestsFor(process.argv[3]); @@ -104,10 +114,6 @@ switch (action) { const maxHeaderLen = Object.keys(services).reduce((n, v) => v.length > n ? v.length : n, 0); const failCounters = {}; - env.streamLifespan = 10000; - env.apiURL = 'http://x/'; - randomizeCiphers(); - for (const service in services) { printHeader(service, maxHeaderLen); const { fails, softFails } = await runTestsFor(service); diff --git a/api/src/util/tests/facebook.json b/api/src/util/tests/facebook.json index 876ac7fe..d0c8cc7b 100644 --- a/api/src/util/tests/facebook.json +++ b/api/src/util/tests/facebook.json @@ -29,7 +29,6 @@ { "name": "shortlink video", "url": "https://fb.watch/r1K6XHMfGT/", - "canFail": true, "params": {}, "expected": { "code": 200, @@ -39,7 +38,6 @@ { "name": "reel video", "url": "https://web.facebook.com/reel/730293269054758", - "canFail": true, "params": {}, "expected": { "code": 200, @@ -64,4 +62,4 @@ "status": "redirect" } } -] \ No newline at end of file +] diff --git a/api/src/util/tests/twitter.json b/api/src/util/tests/twitter.json index 0024d097..4fc5900f 100644 --- a/api/src/util/tests/twitter.json +++ b/api/src/util/tests/twitter.json @@ -102,9 +102,8 @@ }, { "name": "retweeted video", - "url": "https://twitter.com/uwukko/status/1696901469633421344", + "url": "https://twitter.com/schlizzawg/status/1869017025055793405", "params": {}, - "canFail": true, "expected": { "code": 200, "status": "redirect" @@ -145,7 +144,7 @@ "params": {}, "expected": { "code": 200, - "status": "redirect" + "status": "tunnel" } }, { @@ -203,11 +202,11 @@ }, { "name": "bookmarked photo", - "url": "https://twitter.com/i/bookmarks?post_id=1837430141179289876", + "url": "https://twitter.com/i/bookmarks?post_id=1887450602164396149", "params": {}, "expected": { "code": 200, - "status": "redirect" + "status": "tunnel" } } -] \ No newline at end of file +] diff --git a/api/src/util/tests/xiaohongshu.json b/api/src/util/tests/xiaohongshu.json index de632a77..0cca9393 100644 --- a/api/src/util/tests/xiaohongshu.json +++ b/api/src/util/tests/xiaohongshu.json @@ -48,7 +48,6 @@ { "name": "short link, wrong id", "url": "https://xhslink.com/a/aaaaaa", - "canFail": true, "params": {}, "expected": { "code": 400, diff --git a/web/package.json b/web/package.json index 9f51e006..954eb5d7 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "@imput/cobalt-web", - "version": "10.6", + "version": "10.7.3", "type": "module", "private": true, "scripts": { diff --git a/web/src/components/dialog/PickerDialog.svelte b/web/src/components/dialog/PickerDialog.svelte index 1fa5f5f5..b84ea4ad 100644 --- a/web/src/components/dialog/PickerDialog.svelte +++ b/web/src/components/dialog/PickerDialog.svelte @@ -50,7 +50,9 @@
{#if items} {#each items as item, i} - + {#if item?.url} + + {/if} {/each} {/if}
diff --git a/web/src/components/dialog/PickerItem.svelte b/web/src/components/dialog/PickerItem.svelte index 4a515a6e..04e301ec 100644 --- a/web/src/components/dialog/PickerItem.svelte +++ b/web/src/components/dialog/PickerItem.svelte @@ -14,18 +14,28 @@ export let number: number; let imageLoaded = false; - const isTunnel = new URL(item.url).pathname === "/tunnel"; + + let validUrl = false; + try { + new URL(item.url); + validUrl = true; + } catch {} + + const isTunnel = validUrl && new URL(item.url).pathname === "/tunnel"; $: itemType = item.type ?? "photo";