diff --git a/backend/appsettings/serializers.py b/backend/appsettings/serializers.py index f26bc73d..80f71062 100644 --- a/backend/appsettings/serializers.py +++ b/backend/appsettings/serializers.py @@ -55,6 +55,12 @@ class AppConfigDownloadsSerializer( ) cookie_import = serializers.BooleanField() pot_provider_url = serializers.CharField(allow_null=True) + gluetun_url = serializers.CharField(allow_null=True) + gluetun_control_url = serializers.CharField(allow_null=True) + gluetun_control_key = serializers.CharField(allow_null=True) + gluetun_swap = serializers.BooleanField() + gluetun_attempts = serializers.IntegerField(allow_null=True) + gluetun_sleep = serializers.IntegerField(allow_null=True) throttledratelimit = serializers.IntegerField(allow_null=True) extractor_lang = serializers.CharField(allow_null=True) integrate_ryd = serializers.BooleanField() diff --git a/backend/appsettings/src/config.py b/backend/appsettings/src/config.py index 8bd81579..a0b52952 100644 --- a/backend/appsettings/src/config.py +++ b/backend/appsettings/src/config.py @@ -42,6 +42,8 @@ class DownloadsConfigType(TypedDict): comment_sort: Literal["top", "new"] | None cookie_import: bool pot_provider_url: str | None + gluetun_url: str | None + gluetun_swap: bool throttledratelimit: int | None extractor_lang: str | None integrate_ryd: bool @@ -91,6 +93,12 @@ class AppConfig: "comment_sort": "top", "cookie_import": False, "pot_provider_url": None, + "gluetun_url": None, + "gluetun_control_url": None, + "gluetun_control_key": None, + "gluetun_swap": False, + "gluetun_attempts": 0, + "gluetun_sleep": 0, "throttledratelimit": None, "extractor_lang": None, "integrate_ryd": False, diff --git a/backend/download/src/yt_dlp_base.py b/backend/download/src/yt_dlp_base.py index ea65ca77..d668251e 100644 --- a/backend/download/src/yt_dlp_base.py +++ b/backend/download/src/yt_dlp_base.py @@ -4,19 +4,20 @@ functionality: - handle yt-dlp errors """ +import time from datetime import datetime from http import cookiejar from io import StringIO from os import path import yt_dlp +import requests from appsettings.src.config import AppConfig from common.src.env_settings import EnvironmentSettings from common.src.helper import deep_merge, rand_sleep from common.src.ta_redis import RedisArchivist from django.conf import settings - class YtWrap: """wrap calls to yt""" @@ -81,23 +82,65 @@ class YtWrap: if EnvironmentSettings.APP_DIR == "/app": # container internal only self.obs["plugin_dirs"].append("/opt/yt_plugins/bgutil") + + def _add_gluetun_url(self): + """add gluetun proxy url""" + if gluetun_url := self.config["downloads"].get("gluetun_url"): + deep_merge( + self.obs, + { + "proxy": [gluetun_url] + }, + ) def download(self, url): """make download request""" self.obs.update({"check_formats": "selected"}) with yt_dlp.YoutubeDL(self.obs) as ydl: - try: - ydl.download([url]) - except yt_dlp.utils.DownloadError as err: - print(f"{url}: failed to download with message {err}") - if "Temporary failure in name resolution" in str(err): - raise ConnectionError("lost the internet, abort!") from err - if any(m in str(err) for m in self.BOT_MESSAGES): - print(self.BOT_ERROR_LOG) - rand_sleep(self.config) - raise ConnectionError(self.BOT_ERROR_LOG) from err + max_attempts = self.config["downloads"].get("gluetun_attempts") + inf = False + if max_attempts == 0 or max_attempts == None: + inf = True + max_attempts = 0 + attempt = 0 + while attempt < max_attempts or inf: + try: + ydl.download([url]) + break + except yt_dlp.utils.DownloadError as err: + print(f"{url}: failed to download with message {err}") + if "Temporary failure in name resolution" in str(err): + raise ConnectionError("lost the internet, abort!") from err + if any(m in str(err) for m in self.BOT_MESSAGES): + print(self.BOT_ERROR_LOG) + if self.config["downloads"].get("gluetun_swap"): + control_url=f"{self.config['downloads'].get('gluetun_control_url')}/v1/vpn/status" + headers={ + "Content-Type": "application/json", + "X-API-Key": self.config['downloads'].get('gluetun_control_key') + } + requests.put( + url=control_url, + headers=headers, + json={"status": "stopped"} + ) + requests.put( + url=control_url, + headers=headers, + json={"status": "running"} + ) + sleep_amount = self.config['downloads'].get('gluetun_sleep') + if sleep_amount == 0: + sleep_amount = 30 + time.sleep(sleep_amount) + attempt += 1 + if (attempt == max_attempts): + raise ConnectionError("Reached maximum IP attempts!") from err + continue + rand_sleep(self.config) + raise ConnectionError(self.BOT_ERROR_LOG) from err - return False, str(err) + return False, str(err) self._validate_cookie() @@ -109,27 +152,59 @@ class YtWrap: returns response, error """ with yt_dlp.YoutubeDL(self.obs) as ydl: - try: - response = ydl.extract_info(url) - except cookiejar.LoadError as err: - print(f"cookie file is invalid: {err}") - return None, str(err) - except yt_dlp.utils.ExtractorError as err: - print(f"{url}: failed to extract: {err}, continue...") - return None, str(err) - except yt_dlp.utils.DownloadError as err: - if "This channel does not have a" in str(err): - return None, None + max_attempts = self.config["downloads"].get("gluetun_attempts") + inf = False + if max_attempts == 0 or max_attempts == None: + inf = True + max_attempts = 0 + attempt = 0 + while attempt < max_attempts or inf: + try: + response = ydl.extract_info(url) + break + except cookiejar.LoadError as err: + print(f"cookie file is invalid: {err}") + return None, str(err) + except yt_dlp.utils.ExtractorError as err: + print(f"{url}: failed to extract: {err}, continue...") + return None, str(err) + except yt_dlp.utils.DownloadError as err: + if "This channel does not have a" in str(err): + return None, None - print(f"{url}: failed to get info from youtube: {err}") - if "Temporary failure in name resolution" in str(err): - raise ConnectionError("lost the internet, abort!") from err - if any(m in str(err) for m in self.BOT_MESSAGES): - print(self.BOT_ERROR_LOG) - rand_sleep(self.config) - raise ConnectionError(self.BOT_ERROR_LOG) from err + print(f"{url}: failed to get info from youtube: {err}") + if "Temporary failure in name resolution" in str(err): + raise ConnectionError("lost the internet, abort!") from err + if any(m in str(err) for m in self.BOT_MESSAGES): + print(self.BOT_ERROR_LOG) + if self.config["downloads"].get("gluetun_swap"): + control_url=f"{self.config['downloads'].get('gluetun_control_url')}/v1/vpn/status" + headers={ + "Content-Type": "application/json", + "X-API-Key": self.config['downloads'].get('gluetun_control_key') + } + requests.put( + url=control_url, + headers=headers, + json={"status": "stopped"} + ) + requests.put( + url=control_url, + headers=headers, + json={"status": "running"} + ) + sleep_amount = self.config['downloads'].get('gluetun_sleep') + if sleep_amount == 0: + sleep_amount = 30 + time.sleep(sleep_amount) + attempt += 1 + if (attempt == max_attempts): + raise ConnectionError("Reached maximum IP attempts!") from err + continue + rand_sleep(self.config) + raise ConnectionError(self.BOT_ERROR_LOG) from err - return None, str(err) + return None, str(err) self._validate_cookie() diff --git a/frontend/src/api/loader/loadAppsettingsConfig.ts b/frontend/src/api/loader/loadAppsettingsConfig.ts index 47b1ba3a..34e15532 100644 --- a/frontend/src/api/loader/loadAppsettingsConfig.ts +++ b/frontend/src/api/loader/loadAppsettingsConfig.ts @@ -23,6 +23,12 @@ export type AppSettingsConfigType = { comment_sort: string; cookie_import: boolean; pot_provider_url: string | null; + gluetun_url: string | null; + gluetun_control_url: string | null; + gluetun_control_key: string | null; + gluetun_swap: boolean; + gluetun_attempts: number | null; + gluetun_sleep: number | null; throttledratelimit: number | null; extractor_lang: string | null; integrate_ryd: boolean; diff --git a/frontend/src/pages/SettingsApplication.tsx b/frontend/src/pages/SettingsApplication.tsx index d5a56e9f..d04c09ce 100644 --- a/frontend/src/pages/SettingsApplication.tsx +++ b/frontend/src/pages/SettingsApplication.tsx @@ -71,6 +71,14 @@ const SettingsApplication = () => { const [showCookieForm, setShowCookieForm] = useState(false); const [potProviderUrl, setPotProviderUrl] = useState(null); + // Gluetun + const [gluetunUrl, setGluetunUrl] = useState(null); + const [gluetunControlUrl, setGluetunControlUrl] = useState(null); + const [gluetunControlKey, setGluetunControlKey] = useState(null); + const [gluetunSwap, setGluetunSwap] = useState(false); + const [gluetunAttempts, setGluetunAttempts] = useState(null); + const [gluetunSleep, setGluetunSleep] = useState(null); + // Integrations const [showApiToken, setShowApiToken] = useState(false); const [downloadDislikes, setDownloadDislikes] = useState(false); @@ -125,6 +133,14 @@ const SettingsApplication = () => { // Cookie setPotProviderUrl(appSettingsConfigData?.downloads.pot_provider_url || null); + // Gluetun + setGluetunUrl(appSettingsConfigData?.downloads.gluetun_url || null); + setGluetunControlUrl(appSettingsConfigData?.downloads.gluetun_control_url || null); + setGluetunControlKey(appSettingsConfigData?.downloads.gluetun_control_key || null); + setGluetunSwap(appSettingsConfigData?.downloads.gluetun_swap || false); + setGluetunAttempts(appSettingsConfigData?.downloads.gluetun_attempts || null); + setGluetunSleep(appSettingsConfigData?.downloads.gluetun_sleep || null); + // Integrations setDownloadDislikes(appSettingsConfigData?.downloads.integrate_ryd || false); setEnableSponsorBlock(appSettingsConfigData?.downloads.integrate_sponsorblock || false); @@ -753,6 +769,129 @@ const SettingsApplication = () => { /> +
+

Gluetun

+ {userConfig.show_help_text && ( +
+

+ Use a VPN for downloads via Gluetun +

+
    +
  • + Using a VPN can help get work around blocked requests if you don't + have or want to use an account. +
    • +
  • + This expects you to use and setup{' '} + + Gluetun. + +
    • +
  • + Enabling Swap IPs will change the VPN's IP if it's blocked from + downloading regular videos. +
    • +
  • + Using Swap IPs requires you to configure a control server and use + an API key, see{' '} + + Gluetun's Documentation. + +
    • +
  • + Setting "Swap IP attempts" to 0 or leaving it empty will try + swapping the IP infinitely until it can download. Otherwise, it + will attempt to swap the IP only for a certain number of attempts. +
    • +
  • + Setting "Sleep Interval" will wait that amount of time after + swapping the IP before attempting to download again. Default is 30. +
    • +
+
+ )} +
+
+

Gluetun Proxy URL

+
+ +
+
+
+

Gluetun Control Server URL

+
+ +
+
+
+

Gluetun Control Server API Key

+
+ +
+
+
+

Swap IPs

+
+ +
+
+
+

Swap IP Attempts

+
+ +
+
+
+

Sleep Interval

+
+ +
+

Integrations

{userConfig.show_help_text && ( diff --git a/frontend/src/stores/AppSettingsStore.ts b/frontend/src/stores/AppSettingsStore.ts index 1f8654e2..e5cf310e 100644 --- a/frontend/src/stores/AppSettingsStore.ts +++ b/frontend/src/stores/AppSettingsStore.ts @@ -30,6 +30,12 @@ export const useAppSettingsStore = create(set => ({ comment_sort: 'asc', cookie_import: false, pot_provider_url: null, + gluetun_url: null, + gluetun_control_url: null, + gluetun_control_key: null, + gluetun_swap: false, + gluetun_attempts: null, + gluetun_sleep: null, throttledratelimit: null, extractor_lang: null, integrate_ryd: false,