Add potoken provider (#1076)

* added extractor_args

* added extractor_args

* added extractor_args within AppConfig

* added extractor_args

* added extractor_args

* added extractor_args

* created ExtractorArgsParser

* forgot the settings-box-wrapper for extractor_args

* fixed linting errors

* Update extractor arguments format in settings, example of multiples

* added bgutil-ytdlp-pot-provider

* UI changes to support pot_provider_url

* if pot_provide_url, append to extractor_args

* reworded PO Token Provider URL, to match yt-dlp wording.

* changed pot token provider url link to User Guide

* simplify, remove generic extractor arg parsing

* revert package.json changes

* revert package.json changes, take 2

* revert package.json changes, take 3

---------

Co-authored-by: Simon <simobilleter@gmail.com>

backend/appsettings/serializers.py

@@ -54,6 +54,7 @@ class AppConfigDownloadsSerializer(
         choices=["top", "new"], allow_null=True
     )
     cookie_import = serializers.BooleanField()
+    pot_provider_url = serializers.CharField(allow_null=True)
     potoken = serializers.BooleanField()
     throttledratelimit = serializers.IntegerField(allow_null=True)
     extractor_lang = serializers.CharField(allow_null=True)

backend/appsettings/src/config.py

@@ -41,6 +41,7 @@ class DownloadsConfigType(TypedDict):
     comment_max: str | None
     comment_sort: Literal["top", "new"] | None
     cookie_import: bool
+    pot_provider_url: str | None
     potoken: bool
     throttledratelimit: int | None
     extractor_lang: str | None
@@ -90,6 +91,7 @@ class AppConfig:
             "comment_max": None,
             "comment_sort": "top",
             "cookie_import": False,
+            "pot_provider_url": None,
             "potoken": False,
             "throttledratelimit": None,
             "extractor_lang": None,

backend/download/src/yt_dlp_base.py

@@ -37,6 +37,7 @@ class YtWrap:
         if self.config:
             self._add_cookie()
             self._add_potoken()
+            self._add_potoken_url()
 
         if getattr(settings, "DEBUG", False):
             del self.obs["quiet"]
@@ -63,6 +64,21 @@ class YtWrap:
                 }
             )
 
+    def _add_potoken_url(self):
+        """add bgutils token url"""
+        if pot_provider_url := self.config["downloads"].get(
+            "pot_provider_url"
+        ):
+            self.obs.update(
+                {
+                    "extractor_args": {
+                        "youtubepot-bgutilhttp": {
+                            "base_url": [pot_provider_url]
+                        }
+                    }
+                }
+            )
+
     def download(self, url):
         """make download request"""
         self.obs.update({"check_formats": "selected"})

backend/requirements.txt

@@ -1,4 +1,5 @@
 apprise==1.9.7
+bgutil-ytdlp-pot-provider==1.2.2
 celery==5.6.2
 deepdiff==8.6.1
 django-auth-ldap==5.3.0

backend/user/migrations/0001_initial.py

@@ -39,7 +39,9 @@ class Migration(migrations.Migration):
                     "is_superuser",
                     models.BooleanField(
                         default=False,
-                        help_text="Designates that this user has all permissions without explicitly assigning them.",
+                        help_text=(
+                            "Designates that this user has all permissions without explicitly assigning them."
+                        ),
                         verbose_name="superuser status",
                     ),
                 ),
@@ -49,7 +51,9 @@ class Migration(migrations.Migration):
                     "groups",
                     models.ManyToManyField(
                         blank=True,
-                        help_text="The groups this user belongs to. A user will get all permissions granted to each of their groups.",
+                        help_text=(
+                            "The groups this user belongs to. A user will get all permissions granted to each of their groups."
+                        ),
                         related_name="user_set",
                         related_query_name="user",
                         to="auth.group",

frontend/src/api/loader/loadAppsettingsConfig.ts

@@ -22,6 +22,7 @@ export type AppSettingsConfigType = {
     comment_max: string | null;
     comment_sort: string;
     cookie_import: boolean;
+    pot_provider_url: string | null;
     potoken: boolean;
     throttledratelimit: number | null;
     extractor_lang: string | null;

frontend/src/pages/SettingsApplication.tsx

@@ -73,6 +73,7 @@ const SettingsApplication = () => {
   const [showCookieForm, setShowCookieForm] = useState<boolean>(false);
   const [poTokenFormData, setPoTokenFormData] = useState<string>('web+');
   const [showPoTokenForm, setShowPoTokenForm] = useState<boolean>(false);
+  const [potProviderUrl, setPotProviderUrl] = useState<string | null>(null);
 
   // Integrations
   const [showApiToken, setShowApiToken] = useState(false);
@@ -125,6 +126,9 @@ const SettingsApplication = () => {
     setCommentsMax(appSettingsConfigData?.downloads.comment_max || null);
     setCommentsSort(appSettingsConfigData?.downloads.comment_sort || '');
 
+    // Cookie
+    setPotProviderUrl(appSettingsConfigData?.downloads.pot_provider_url || null);
+
     // Integrations
     setDownloadDislikes(appSettingsConfigData?.downloads.integrate_ryd || false);
     setEnableSponsorBlock(appSettingsConfigData?.downloads.integrate_sponsorblock || false);
@@ -701,6 +705,16 @@ const SettingsApplication = () => {
                         PO guide
                       </a>
                     </li>
+                    <li>
+                      The PO Token Provider URL running external to tubearchivist. Make sure to
+                      review{' '}
+                      <a
+                        target="_blank"
+                        href="https://docs.tubearchivist.com/settings/application/#po-token-provider-url"
+                      >
+                        User Guide
+                      </a>
+                    </li>
                   </ul>
                 </div>
               )}
@@ -800,6 +814,19 @@ const SettingsApplication = () => {
                   )}
                 </div>
               </div>
+              <div className="settings-box-wrapper">
+                <div>
+                  <p>PO Token Provider URL</p>
+                </div>
+                <InputConfig
+                  type="text"
+                  name="downloads.pot_provider_url"
+                  value={potProviderUrl}
+                  setValue={setPotProviderUrl}
+                  oldValue={appSettingsConfig.downloads.pot_provider_url}
+                  updateCallback={handleUpdateConfig}
+                />
+              </div>
             </div>
             <div className="info-box-item">
               <h2 id="sntegrations">Integrations</h2>

frontend/src/stores/AppSettingsStore.ts

@@ -29,6 +29,7 @@ export const useAppSettingsStore = create<AppSettingsState>(set => ({
       comment_max: null,
       comment_sort: 'asc',
       cookie_import: false,
+      pot_provider_url: null,
       potoken: false,
       throttledratelimit: null,
       extractor_lang: null,