diff --git a/backend/appsettings/serializers.py b/backend/appsettings/serializers.py index b7e1670f..3f145696 100644 --- a/backend/appsettings/serializers.py +++ b/backend/appsettings/serializers.py @@ -57,7 +57,6 @@ class AppConfigDownloadsSerializer( pot_provider_url = serializers.CharField(allow_null=True) potoken = serializers.BooleanField() throttledratelimit = serializers.IntegerField(allow_null=True) - extractor_args = serializers.CharField(allow_null=True) extractor_lang = serializers.CharField(allow_null=True) integrate_ryd = serializers.BooleanField() integrate_sponsorblock = serializers.BooleanField() diff --git a/backend/appsettings/src/config.py b/backend/appsettings/src/config.py index 8cabd641..f557b459 100644 --- a/backend/appsettings/src/config.py +++ b/backend/appsettings/src/config.py @@ -44,7 +44,6 @@ class DownloadsConfigType(TypedDict): pot_provider_url: str | None potoken: bool throttledratelimit: int | None - extractor_args: str | None extractor_lang: str | None integrate_ryd: bool integrate_sponsorblock: bool @@ -95,7 +94,6 @@ class AppConfig: "pot_provider_url": None, "potoken": False, "throttledratelimit": None, - "extractor_args": None, "extractor_lang": None, "integrate_ryd": False, "integrate_sponsorblock": False, diff --git a/backend/download/src/yt_dlp_base.py b/backend/download/src/yt_dlp_base.py index 11348a72..462e12c4 100644 --- a/backend/download/src/yt_dlp_base.py +++ b/backend/download/src/yt_dlp_base.py @@ -4,7 +4,6 @@ functionality: - handle yt-dlp errors """ -import re from datetime import datetime from http import cookiejar from io import StringIO @@ -15,124 +14,6 @@ from common.src.ta_redis import RedisArchivist from django.conf import settings -class ExtractorArgsParser: - """ - Parse extractor_args string following yt-dlp format. - - Format: "extractor1:key1=val1,val2;key2=val3 extractor2:key3=val4" - - - Whitespace separates multiple extractors - - Colon (:) separates extractor name from key-value pairs - - Semicolon (;) separates multiple key-value pairs - - Equals (=) separates key from values - - Comma (,) separates multiple values (escaped commas \\, are preserved) - - Example: - >>> parser = ExtractorArgsParser() - >>> result = parser.parse("youtube:key1=val1,val2;key2=val3") - >>> # Returns: {"youtube": {"key1": ["val1", "val2"], - >>> # "key2": ["val3"]}} - """ - - @staticmethod - def _parse_key_values(key, vals=""): - """ - Parse a single key=values pair following yt-dlp logic. - - Args: - key: The key name (will be normalized) - vals: Comma-separated values (escaped commas preserved) - - Returns: - tuple: (normalized_key, list_of_values) - """ - # Normalize key: strip, lowercase, replace hyphens with underscores - normalized_key = key.strip().lower().replace("-", "_") - - # Split values by comma, but preserve escaped commas - # yt-dlp uses: re.split(r'(?>> parser.parse("youtube:lang=en,es;key2=val generic:key3=val3") - {'youtube': {'lang': ['en', 'es'], 'key2': ['val']}, - 'generic': {'key3': ['val3']}} - """ - if not extractor_args_str or not extractor_args_str.strip(): - return {} - - result = {} - - # Split by whitespace to get individual extractor specifications - extractor_specs = extractor_args_str.strip().split() - - for spec in extractor_specs: - # Split by first colon to separate extractor from args - if ":" not in spec: - print( - "[extractor_args] Warning: Invalid format " - f"(missing colon): {spec}" - ) - continue - - extractor_name, args_part = spec.split(":", 1) - extractor_name = extractor_name.strip() - - if not extractor_name: - print( - "[extractor_args] Warning: Empty extractor name " - f"in: {spec}" - ) - continue - - # Initialize extractor dict if not exists - if extractor_name not in result: - result[extractor_name] = {} - - # Split args by semicolon to get individual key=value pairs - key_value_pairs = args_part.split(";") - - for pair in key_value_pairs: - if not pair.strip(): - continue - - # Split by first equals sign - if "=" not in pair: - print( - "[extractor_args] Warning: Invalid pair " - f"(missing =): {pair}" - ) - continue - - key_part, value_part = pair.split("=", 1) - normalized_key, value_list = self._parse_key_values( - key_part, value_part - ) - - result[extractor_name][normalized_key] = value_list - - return result - - class YtWrap: """wrap calls to yt""" @@ -156,7 +37,7 @@ class YtWrap: if self.config: self._add_cookie() self._add_potoken() - self._add_extractor_args() + self._add_potoken_url() if getattr(settings, "DEBUG", False): del self.obs["quiet"] @@ -183,41 +64,20 @@ class YtWrap: } ) - def _add_extractor_args(self): - """add custom extractor_args from config if present""" - extractor_args_str = self.config["downloads"].get("extractor_args") - pot_provider_url = self.config["downloads"].get("pot_provider_url") - - # If pot_provider_url is set, append it to extractor_args - if pot_provider_url: - pot_arg = f"youtubepot-bgutilhttp:base_url={pot_provider_url}" - if extractor_args_str: - extractor_args_str = f"{extractor_args_str} {pot_arg}" - else: - extractor_args_str = pot_arg - - if not extractor_args_str: - return - - # Parse the extractor_args string - parser = ExtractorArgsParser() - parsed_args = parser.parse(extractor_args_str) - - if not parsed_args: - return - - # Merge with existing extractor_args if present - if "extractor_args" not in self.obs: - self.obs["extractor_args"] = {} - - for extractor_name, args_dict in parsed_args.items(): - if extractor_name not in self.obs["extractor_args"]: - self.obs["extractor_args"][extractor_name] = {} - - # Merge the arguments, with parsed args taking precedence - self.obs["extractor_args"][extractor_name].update(args_dict) - - print(f"[extractor_args] Applied custom args: {parsed_args}") + def _add_potoken_url(self): + """add bgutils token url""" + if pot_provider_url := self.config["downloads"].get( + "pot_provider_url" + ): + self.obs.update( + { + "extractor_args": { + "youtubepot-bgutilhttp": { + "base_url": [pot_provider_url] + } + } + } + ) def download(self, url): """make download request""" diff --git a/backend/user/migrations/0001_initial.py b/backend/user/migrations/0001_initial.py index e328a0f9..3b965203 100644 --- a/backend/user/migrations/0001_initial.py +++ b/backend/user/migrations/0001_initial.py @@ -40,8 +40,7 @@ class Migration(migrations.Migration): models.BooleanField( default=False, help_text=( - "Designates that this user has all permissions " - "without explicitly assigning them." + "Designates that this user has all permissions without explicitly assigning them." ), verbose_name="superuser status", ), @@ -53,9 +52,7 @@ class Migration(migrations.Migration): models.ManyToManyField( blank=True, help_text=( - "The groups this user belongs to. A user will " - "get all permissions granted to each of their " - "groups." + "The groups this user belongs to. A user will get all permissions granted to each of their groups." ), related_name="user_set", related_query_name="user", diff --git a/frontend/src/api/loader/loadAppsettingsConfig.ts b/frontend/src/api/loader/loadAppsettingsConfig.ts index a7d363fb..fb05d00c 100644 --- a/frontend/src/api/loader/loadAppsettingsConfig.ts +++ b/frontend/src/api/loader/loadAppsettingsConfig.ts @@ -25,7 +25,6 @@ export type AppSettingsConfigType = { pot_provider_url: string | null; potoken: boolean; throttledratelimit: number | null; - extractor_args: string | null; extractor_lang: string | null; integrate_ryd: boolean; integrate_sponsorblock: boolean; diff --git a/frontend/src/pages/SettingsApplication.tsx b/frontend/src/pages/SettingsApplication.tsx index 47837889..2d2153a0 100644 --- a/frontend/src/pages/SettingsApplication.tsx +++ b/frontend/src/pages/SettingsApplication.tsx @@ -56,7 +56,6 @@ const SettingsApplication = () => { // Download Format const [downloadsFormat, setDownloadsFormat] = useState(null); const [downloadsFormatSort, setDownloadsFormatSort] = useState(null); - const [downloadsExtractorArgs, setDownloadsExtractorArgs] = useState(null); const [downloadsExtractorLang, setDownloadsExtractorLang] = useState(null); const [embedMetadata, setEmbedMetadata] = useState(false); @@ -115,7 +114,6 @@ const SettingsApplication = () => { // Download Format setDownloadsFormat(appSettingsConfigData?.downloads.format || null); setDownloadsFormatSort(appSettingsConfigData?.downloads.format_sort || null); - setDownloadsExtractorArgs(appSettingsConfigData?.downloads.extractor_args || null); setDownloadsExtractorLang(appSettingsConfigData?.downloads.extractor_lang || null); setEmbedMetadata(appSettingsConfigData?.downloads.add_metadata || false); @@ -463,28 +461,6 @@ const SettingsApplication = () => { -
  • - Extractor Arguments to be passed at runtime -
      -
    • Some extractors accept additional arguments
      • -
    • Separate multiple extractors with a space
      • -
    • - - EXTRACTOR1:ARG1=VAL1,VAL2;ARG2=VAL3 EXTRACTOR2:ARG3=VAL4 - -
      • -
    • - More details{' '} - - here - - . -
      • -
    -
  • Extractor language will change how a video gets indexed. Index language configuration @@ -535,19 +511,6 @@ const SettingsApplication = () => { updateCallback={handleUpdateConfig} /> -
    -
    -

    Extractor Arguments

    -
    - -

    Extractor Language

    diff --git a/frontend/src/stores/AppSettingsStore.ts b/frontend/src/stores/AppSettingsStore.ts index 995d0f46..9db60607 100644 --- a/frontend/src/stores/AppSettingsStore.ts +++ b/frontend/src/stores/AppSettingsStore.ts @@ -32,7 +32,6 @@ export const useAppSettingsStore = create(set => ({ pot_provider_url: null, potoken: false, throttledratelimit: null, - extractor_args: null, extractor_lang: null, integrate_ryd: false, integrate_sponsorblock: false,