Fix startup migration, #build

Changed:
- Fixed startup migration for nested dict update by query
- Added pot provider plugin noop default
- Fixed timeout in json backups fetching
- Fixed date parser fload in search processing

CONTRIBUTING.md

@@ -134,6 +134,7 @@ The documentation available at [docs.tubearchivist.com](https://docs.tubearchivi
 This codebase is set up to be developed natively outside of docker as well as in a docker container. Developing outside of a docker container can be convenient, as IDE and hot reload usually works out of the box. But testing inside of a container is still essential, as there are subtle differences, especially when working with the filesystem and networking between containers.
 
 Note:
+- This project doesn't look for contributions to this to cover additional dev setup environments from new contributors. If you are a regular contributor and you see ways to improve this, please reach out on Discord first.
 - Subtitles currently fail to load with `DJANGO_DEBUG=True`, that is due to incorrect `Content-Type` error set by Django's static file implementation. That's only if you run the Django dev server, Nginx sets the correct headers in the container.
 
 ### Native Instruction

backend/appsettings/serializers_mb.py

@@ -29,3 +29,4 @@ class MembershipProfileSerializer(serializers.Serializer):
     sponsor_tier = SponsortierSerializer()
     subscription_count = serializers.IntegerField()
     subscription_is_max = serializers.BooleanField()
+    is_connected = serializers.BooleanField()

backend/appsettings/src/backup.py

@@ -21,7 +21,7 @@ class ElasticBackup:
     """dump index to nd-json files for later bulk import"""
 
     INDEX_SIZE_CONF = {
-        "comment": 200,
+        "comment": 100,
         "subtitle": 10000,
     }
     CACHE_DIR = EnvironmentSettings.CACHE_DIR
@@ -64,6 +64,7 @@ class ElasticBackup:
             "callback": BackupCallback,
             "task": self.task,
             "total": self._get_total(index_name),
+            "timeout": 30,
         }
 
         if size_overwrite := self.INDEX_SIZE_CONF.get(index_name):

backend/common/src/es_connect.py

@@ -148,6 +148,7 @@ class IndexPaginate:
     - callback: obj, Class implementing run method callback for every loop
     - task: task object to send notification
     - total: int, total items in index for progress message
+    - timeout: int, overwrite timeout in get request
     """
 
     DEFAULT_SIZE = 500
@@ -191,7 +192,11 @@ class IndexPaginate:
         all_results = []
         counter = 0
         while True:
-            response, _ = ElasticWrap("_search").get(data=self.data)
+            get_kwargs = {"data": self.data}
+            if timeout_overwrite := self.kwargs.get("timeout"):
+                get_kwargs.update({"timeout": timeout_overwrite})
+
+            response, _ = ElasticWrap("_search").get(**get_kwargs)
             all_hits = response["hits"]["hits"]
             if not all_hits:
                 break

backend/common/src/helper.py

@@ -103,13 +103,15 @@ def requests_headers() -> dict[str, str]:
     return {"User-Agent": template}
 
 
-def date_parser(timestamp: int | str | None) -> str | None:
+def date_parser(timestamp: int | float | str | None) -> str | None:
     """return formatted date string"""
     if timestamp is None:
         return None
 
     if isinstance(timestamp, int):
         date_obj = datetime.fromtimestamp(timestamp, tz=timezone.utc)
+    elif isinstance(timestamp, float):
+        date_obj = datetime.fromtimestamp(int(timestamp), tz=timezone.utc)
     elif isinstance(timestamp, str) and timestamp.isdigit():
         date_obj = datetime.fromtimestamp(int(timestamp), tz=timezone.utc)
     elif isinstance(timestamp, str):

backend/common/tests/test_src/test_helper.py

@@ -33,6 +33,13 @@ def test_date_parser_with_digit():
     assert date_parser(timestamp) == expected_date
 
 
+def test_date_parser_with_float():
+    """iso timestamp"""
+    date_float = 1766210400.0
+    expected_date = "2025-12-20T06:00:00+00:00"
+    assert date_parser(date_float) == expected_date
+
+
 def test_date_parser_with_str():
     """iso timestamp"""
     date_str = "2021-05-21"

backend/config/management/commands/ta_startup.py

@@ -376,14 +376,15 @@ class Command(BaseCommand):
                     "lang": "painless",
                 },
             )
+            source = f"""
+                if (ctx._source.containsKey('channel'))
+                {{ctx._source.channel.remove('{field}');}}
+            """
             self._run_migration(
                 index_name="ta_video",
                 desc=f"fix missing data type for field channel.{field}",
                 query={"term": {f"channel.{field}": {"value": False}}},
-                script={
-                    "source": f"ctx._source.remove('channel.{field}')",
-                    "lang": "painless",
-                },
+                script={"source": source, "lang": "painless"},
             )
 
     def _mig_fix_channel_description(self) -> None:

backend/download/src/yt_dlp_base.py

@@ -89,6 +89,17 @@ class YtWrap:
                     }
                 },
             )
+            return
+
+        # https://github.com/Brainicism/bgutil-ytdlp-pot-provider/pull/185
+        deep_merge(
+            self.obs,
+            {
+                "extractor_args": {
+                    "youtubepot-bgutilhttp": {"disable": ["True"]}
+                }
+            },
+        )
 
     def download(self, url):
         """make download request"""

backend/requirements.txt

@@ -1,5 +1,5 @@
 apprise==1.9.7
-bgutil-ytdlp-pot-provider==1.2.2
+bgutil-ytdlp-pot-provider @ git+https://github.com/bbilly1/bgutil-ytdlp-pot-provider@c3f7bf4c99f8eb62015cb1c4142324f3d681cf07#subdirectory=plugin
 celery==5.6.2
 deepdiff==8.6.1
 django-auth-ldap==5.3.0
@@ -13,4 +13,4 @@ redis==7.1.0
 requests==2.32.5
 ryd-client==0.0.6
 uvicorn==0.40.0
-yt-dlp[default] @ git+https://github.com/yt-dlp/yt-dlp@23b846506378a6a9c9a0958382d37f943f7cfa51
+yt-dlp[default]==2026.1.29

frontend/src/components/MembershipAppsettings.tsx

@@ -24,6 +24,7 @@ type ProfileResponseType = {
   sponsor_tier: SponsorTierType;
   subscription_count: number;
   subscription_is_max: boolean;
+  is_connected: boolean;
 };
 
 export default function MembershipAppsettings({ show_help_text }: { show_help_text: boolean }) {
@@ -136,6 +137,13 @@ export default function MembershipAppsettings({ show_help_text }: { show_help_te
               .
             </li>
             <li>Click on validate to verify everything is working.</li>
+            <li>
+              Setup the{' '}
+              <a href="https://github.com/tubearchivist/members" target="_blank">
+                Client Container
+              </a>
+              .
+            </li>
             <li>
               If you are subscribed to less channels than your sponsor tier allows, you can directly
               sync all your subscriptions here.
@@ -206,6 +214,19 @@ export default function MembershipAppsettings({ show_help_text }: { show_help_te
                     <br />
                     Subscriptions: {profileResponse.subscription_count}/
                     {profileResponse.sponsor_tier.max_subs}
+                    <br />
+                    Socket:{' '}
+                    {profileResponse.is_connected ? (
+                      'Established'
+                    ) : (
+                      <>
+                        Not established. Make sure the{' '}
+                        <a href="https://github.com/tubearchivist/members" target="_blank">
+                          Client Container
+                        </a>{' '}
+                        is connected.
+                      </>
+                    )}
                   </p>
                 </>
               )}