Whitelist yt-dlp plugin load at runtime, #build

Changed:
- Added whitelist logic for yt-dlp plugins loading
- Added stop on bot
- Bump newest yt-dlp

.github/workflows/unit_tests.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.13'
 
       - name: Cache pip
         uses: actions/cache@v4

Dockerfile

@@ -25,7 +25,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 # install requirements
 COPY ./backend/requirements.txt /requirements.txt
-RUN pip install --user -r requirements.txt
+COPY ./backend/requirements.plugins.txt /requirements.plugins.txt
+RUN pip install --user -r /requirements.txt \
+    && python -m pip install --target /opt/yt_plugins/bgutil -r /requirements.plugins.txt
 
 # build ffmpeg
 FROM python:3.13.11-slim-trixie AS ffmpeg-builder
@@ -46,6 +48,7 @@ COPY --from=denoland/deno:bin /deno /usr/local/bin/deno
 
 # copy build requirements
 COPY --from=builder /root/.local /root/.local
+COPY --from=builder /opt/yt_plugins /opt/yt_plugins
 ENV PATH=/root/.local/bin:$PATH
 
 # copy ffmpeg

backend/download/src/yt_dlp_base.py

@@ -12,7 +12,7 @@ from os import path
 import yt_dlp
 from appsettings.src.config import AppConfig
 from common.src.env_settings import EnvironmentSettings
-from common.src.helper import deep_merge
+from common.src.helper import deep_merge, rand_sleep
 from common.src.ta_redis import RedisArchivist
 from django.conf import settings
 
@@ -20,6 +20,11 @@ from django.conf import settings
 class YtWrap:
     """wrap calls to yt"""
 
+    BOT_MESSAGES = [
+        "not a bot",
+    ]
+    BOT_ERROR_LOG = "YouTube bot detection, abort!"
+
     OBS_BASE = {
         "default_search": "ytsearch",
         "quiet": True,
@@ -29,6 +34,7 @@ class YtWrap:
         "cachedir": path.abspath(
             path.join(EnvironmentSettings.CACHE_DIR, "ytdlp")
         ),
+        "plugin_dirs": [],
     }
 
     def __init__(self, obs_request, config=False):
@@ -72,17 +78,9 @@ class YtWrap:
                     }
                 },
             )
-            return
-
-        # https://github.com/Brainicism/bgutil-ytdlp-pot-provider/pull/185
-        deep_merge(
-            self.obs,
-            {
-                "extractor_args": {
-                    "youtubepot-bgutilhttp": {"disable": ["True"]}
-                }
-            },
-        )
+            if EnvironmentSettings.APP_DIR == "/app":
+                # container internal only
+                self.obs["plugin_dirs"].append("/opt/yt_plugins/bgutil")
 
     def download(self, url):
         """make download request"""
@@ -94,6 +92,10 @@ class YtWrap:
                 print(f"{url}: failed to download with message {err}")
                 if "Temporary failure in name resolution" in str(err):
                     raise ConnectionError("lost the internet, abort!") from err
+                if any(m in str(err) for m in self.BOT_MESSAGES):
+                    print(self.BOT_ERROR_LOG)
+                    rand_sleep(self.config)
+                    raise ConnectionError(self.BOT_ERROR_LOG) from err
 
                 return False, str(err)
 
@@ -122,6 +124,10 @@ class YtWrap:
                 print(f"{url}: failed to get info from youtube: {err}")
                 if "Temporary failure in name resolution" in str(err):
                     raise ConnectionError("lost the internet, abort!") from err
+                if any(m in str(err) for m in self.BOT_MESSAGES):
+                    print(self.BOT_ERROR_LOG)
+                    rand_sleep(self.config)
+                    raise ConnectionError(self.BOT_ERROR_LOG) from err
 
                 return None, str(err)
 

backend/requirements.plugins.txt

@@ -0,0 +1,2 @@
+# install plugins in separate folder for runtime whitelisting
+bgutil-ytdlp-pot-provider==1.3.1

backend/requirements.txt

@@ -1,5 +1,4 @@
-apprise==1.9.7
-bgutil-ytdlp-pot-provider @ git+https://github.com/bbilly1/bgutil-ytdlp-pot-provider@f9e694f0b040ec3074301e8ffccad0f9cdb68ed1#subdirectory=plugin
+apprise==1.9.8
 celery==5.6.2
 deepdiff==8.6.1
 django-auth-ldap==5.3.0
@@ -9,8 +8,8 @@ Django==6.0.3
 djangorestframework==3.16.1
 drf-spectacular==0.28.0  # rc:ignore
 Pillow==12.1.1
-redis==7.2.1
+redis==7.3.0
 requests==2.32.5
 ryd-client==0.0.6
-uvicorn==0.41.0
-yt-dlp[default]==2026.3.3
+uvicorn==0.42.0
+yt-dlp[default]==2026.3.17

frontend/package.json

@@ -31,7 +31,7 @@
     "prettier": "3.7.4",
     "typescript": "^5.9.3",
     "typescript-eslint": "^8.51.0",
-    "vite": ">=7.3.0",
+    "vite": "^7.3.0",
     "vite-plugin-checker": "^0.12.0"
   }
 }

requirements-dev.txt

@@ -1,5 +1,6 @@
+-r backend/requirements.plugins.txt
 -r backend/requirements.txt
-ipython==9.10.0
+ipython==9.11.0
 pre-commit==4.5.1
 pylint-django==2.7.0
 pylint==4.0.5