From f5facc7d399e72499379dbcf76d0ed2405f7e5b7 Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 1 Aug 2024 07:43:51 +0600 Subject: [PATCH 01/10] youtube: catch cipher errors not the first and seemingly not the last time it's gonna happen --- src/localization/languages/en.json | 3 ++- src/modules/processing/services/youtube.js | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/localization/languages/en.json b/src/localization/languages/en.json index 2b10f41d..8db4585b 100644 --- a/src/localization/languages/en.json +++ b/src/localization/languages/en.json @@ -160,6 +160,7 @@ "ErrorYTAgeRestrict": "this youtube video is age-restricted, so i can't see it. try another one!", "ErrorYTLogin": "couldn't get this youtube video because it requires an account to view.\n\nthis limitation is done by google to seemingly stop scraping, affecting all 3rd party tools and even their own clients.\n\ntry again, but if issue persists, {ContactLink}.", "ErrorYTRateLimit": "i got rate limited by youtube. try again in a few seconds, but if issue persists, {ContactLink}.", - "ErrorInvalidAcceptHeader": "invalid accept header" + "ErrorInvalidAcceptHeader": "invalid accept header", + "ErrorYoutubeDecipher": "youtube updated its decipher algorithm, so i can't get the video info. try again, but if it still doesn't work, please create an issue on github." } } diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 5aa201ce..d3ee5602 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -50,7 +50,9 @@ const transformSessionData = (cookie) => { const cloneInnertube = async (customFetch) => { const innertube = await ytBase; if (innertube instanceof Error) { - throw innertube; + if (innertube?.message?.endsWith("decipher algorithm")) { + return { error: "ErrorYoutubeDecipher" } + } else throw innertube; } const session = new Session( @@ -101,6 +103,7 @@ export default async function(o) { dispatcher: o.dispatcher }) ); + if (yt.error) return yt; const quality = o.quality === "max" ? "9000" : o.quality; From 6626d74746b88420ec80e6b707a8d7cd01a4f5d9 Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 1 Aug 2024 07:51:26 +0600 Subject: [PATCH 02/10] package: use set youtube.js commit from github --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 87e11661..6afe977b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,7 @@ "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", - "youtubei.js": "^10.2.0" + "youtubei.js": "git://github.com/LuanRT/YouTube.js.git#04369be620e452ab3c4b2d14991fc55773995137" }, "engines": { "node": ">=18" @@ -1124,8 +1124,8 @@ }, "node_modules/youtubei.js": { "version": "10.2.0", - "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-10.2.0.tgz", - "integrity": "sha512-JLKW9AHQ1qrTwBbre1aDkH8UJFmNcc4+kOSaVou5jSY7AzfFPFJK0yvX6afnLst0UVC9wfXHrLiNx93sutVErA==", + "resolved": "git+ssh://git@github.com/LuanRT/YouTube.js.git#04369be620e452ab3c4b2d14991fc55773995137", + "integrity": "sha512-rFjkH7A8ZvQRmCCxIj/AfWHIkEZTQI5N2VN2Oag77oEHF3HYKMNwQkHPVSDNR/qy61cJVvEoP5Azuk0GK7+VFQ==", "funding": [ "https://github.com/sponsors/LuanRT" ], diff --git a/package.json b/package.json index 58fdec83..5900a034 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", - "youtubei.js": "^10.2.0" + "youtubei.js": "git://github.com/LuanRT/YouTube.js.git#04369be620e452ab3c4b2d14991fc55773995137" }, "optionalDependencies": { "freebind": "^0.2.2" From 4f100b626bc848e3bdffd8f243c9650ea95e5f81 Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 1 Aug 2024 08:21:52 +0600 Subject: [PATCH 03/10] workflows/docker: bump version of build & push action --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ee3d757b..ae83b419 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -48,7 +48,7 @@ jobs: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Build and push Docker image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . platforms: linux/amd64,linux/arm64,linux/arm/v7 From 38e43d4a8b4e7f61b6bad9a846c0eac6f21b0101 Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 1 Aug 2024 08:54:26 +0600 Subject: [PATCH 04/10] workflows/docker: remove arm64 and arm7 builds temporary, github action builds hang for 30 FUCKING MINUTES and i need to deploy the patch --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ae83b419..196c4190 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -51,7 +51,7 @@ jobs: uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64,linux/arm64,linux/arm/v7 + platforms: linux/amd64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 50f312fc71a55e3bfbb32d39fe1f32cf938526a0 Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 1 Aug 2024 16:02:24 +0600 Subject: [PATCH 05/10] package: update youtubei.js to version 10.3.0 --- package-lock.json | 18 ++++++++++-------- package.json | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6afe977b..5f09ef3e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,7 @@ "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", - "youtubei.js": "git://github.com/LuanRT/YouTube.js.git#04369be620e452ab3c4b2d14991fc55773995137" + "youtubei.js": "^10.3.0" }, "engines": { "node": ">=18" @@ -76,6 +76,7 @@ "version": "8.12.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==", + "license": "MIT", "bin": { "acorn": "bin/acorn" }, @@ -683,12 +684,13 @@ } }, "node_modules/jintr": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jintr/-/jintr-2.0.0.tgz", - "integrity": "sha512-RiVlevxttZ4eHEYB2dXKXDXluzHfRuw0DJQGsYuKCc5IvZj5/GbOakeqVX+Bar/G9kTty9xDJREcxukurkmYLA==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/jintr/-/jintr-2.1.1.tgz", + "integrity": "sha512-89cwX4ouogeDGOBsEVsVYsnWWvWjchmwXBB4kiBhmjOKw19FiOKhNhMhpxhTlK2ctl7DS+d/ethfmuBpzoNNgA==", "funding": [ "https://github.com/sponsors/LuanRT" ], + "license": "MIT", "dependencies": { "acorn": "^8.8.0" } @@ -1123,15 +1125,15 @@ } }, "node_modules/youtubei.js": { - "version": "10.2.0", - "resolved": "git+ssh://git@github.com/LuanRT/YouTube.js.git#04369be620e452ab3c4b2d14991fc55773995137", - "integrity": "sha512-rFjkH7A8ZvQRmCCxIj/AfWHIkEZTQI5N2VN2Oag77oEHF3HYKMNwQkHPVSDNR/qy61cJVvEoP5Azuk0GK7+VFQ==", + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-10.3.0.tgz", + "integrity": "sha512-tLmeJCECK2xF2hZZtF2nEqirdKVNLFSDpa0LhTaXY3tngtL7doQXyy7M2CLueramDTlmCnFaW+rctHirTPFaRQ==", "funding": [ "https://github.com/sponsors/LuanRT" ], "license": "MIT", "dependencies": { - "jintr": "^2.0.0", + "jintr": "^2.1.1", "tslib": "^2.5.0", "undici": "^5.19.1" } diff --git a/package.json b/package.json index 5900a034..f41b27cb 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", - "youtubei.js": "git://github.com/LuanRT/YouTube.js.git#04369be620e452ab3c4b2d14991fc55773995137" + "youtubei.js": "^10.3.0" }, "optionalDependencies": { "freebind": "^0.2.2" From f57c08785eac719b3f0e0f0323b250839026ecb5 Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 1 Aug 2024 16:08:05 +0600 Subject: [PATCH 06/10] workflows/docker: add arm build targets back --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 196c4190..cc1bbf68 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,7 +21,7 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - + - name: Log in to the Container registry uses: docker/login-action@v3 with: @@ -51,7 +51,7 @@ jobs: uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64,linux/arm/v7 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 650c14f2da309311431f31271f345355046b2cef Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 2 Aug 2024 18:25:07 +0600 Subject: [PATCH 07/10] youtube: always use the iOS client --- src/modules/processing/services/youtube.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index d3ee5602..1bf4d4ce 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -119,7 +119,7 @@ export default async function(o) { } try { - info = await yt.getBasicInfo(o.id, yt.session.logged_in ? 'ANDROID' : 'IOS'); + info = await yt.getBasicInfo(o.id, 'IOS'); } catch(e) { if (e?.message === 'This video is unavailable') { return { error: 'ErrorCouldntFetch' }; From 7f12f4c2d3c35a74f05664355d24678e4df16b24 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 2 Aug 2024 18:44:09 +0600 Subject: [PATCH 08/10] youtube: return the android client ios client just kills streams most of the time for some reason --- src/modules/processing/services/youtube.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index 1bf4d4ce..d3ee5602 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -119,7 +119,7 @@ export default async function(o) { } try { - info = await yt.getBasicInfo(o.id, 'IOS'); + info = await yt.getBasicInfo(o.id, yt.session.logged_in ? 'ANDROID' : 'IOS'); } catch(e) { if (e?.message === 'This video is unavailable') { return { error: 'ErrorCouldntFetch' }; From 05ba1f09b68898ef673c2cd629ef28b577e8cf4f Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Fri, 2 Aug 2024 16:37:23 +0000 Subject: [PATCH 09/10] youtube: periodically refresh innertube player --- src/modules/processing/services/youtube.js | 35 ++++++++++++++-------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/modules/processing/services/youtube.js b/src/modules/processing/services/youtube.js index d3ee5602..9bfc3919 100644 --- a/src/modules/processing/services/youtube.js +++ b/src/modules/processing/services/youtube.js @@ -6,7 +6,9 @@ import { env } from "../../config.js"; import { cleanString } from "../../sub/utils.js"; import { getCookie, updateCookieValues } from "../cookie/manager.js"; -const ytBase = Innertube.create().catch(e => e); +const PLAYER_REFRESH_PERIOD = 1000 * 60 * 15; // ms + +let innertube, lastRefreshedAt; const codecMatch = { h264: { @@ -48,11 +50,12 @@ const transformSessionData = (cookie) => { } const cloneInnertube = async (customFetch) => { - const innertube = await ytBase; - if (innertube instanceof Error) { - if (innertube?.message?.endsWith("decipher algorithm")) { - return { error: "ErrorYoutubeDecipher" } - } else throw innertube; + const shouldRefreshPlayer = lastRefreshedAt + PLAYER_REFRESH_PERIOD < new Date(); + if (!innertube || shouldRefreshPlayer) { + innertube = await Innertube.create({ + fetch: customFetch + }); + lastRefreshedAt = +new Date(); } const session = new Session( @@ -97,13 +100,19 @@ const cloneInnertube = async (customFetch) => { } export default async function(o) { - const yt = await cloneInnertube( - (input, init) => fetch(input, { - ...init, - dispatcher: o.dispatcher - }) - ); - if (yt.error) return yt; + let yt; + try { + yt = await cloneInnertube( + (input, init) => fetch(input, { + ...init, + dispatcher: o.dispatcher + }) + ); + } catch(e) { + if (e.message?.endsWith("decipher algorithm")) { + return { error: "ErrorYoutubeDecipher" } + } else throw e; + } const quality = o.quality === "max" ? "9000" : o.quality; From afa33c404355e1a00d884a2db837233bca54f7f6 Mon Sep 17 00:00:00 2001 From: dumbmoron Date: Sat, 3 Aug 2024 16:46:45 +0000 Subject: [PATCH 10/10] ci: add build action for `develop` tag --- .github/workflows/docker-develop.yml | 55 ++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/docker-develop.yml diff --git a/.github/workflows/docker-develop.yml b/.github/workflows/docker-develop.yml new file mode 100644 index 00000000..e89eeae0 --- /dev/null +++ b/.github/workflows/docker-develop.yml @@ -0,0 +1,55 @@ +name: Build Docker development image + +on: + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Get release metadata + id: release-meta + run: | + version=$(cat package.json | jq -r .version) + echo "commit_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "version=$version" >> $GITHUB_OUTPUT + echo "major_version=$(echo "$version" | cut -d. -f1)" >> $GITHUB_OUTPUT + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + tags: type=raw,value=develop + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max