Update yt-dlp, small fixes, #build

Changed:
- bumped yt-dlp
- improved index alias creation
- add password from file lookup
- removed old manual POT field
- embed metadata fixes

Dockerfile

@@ -1,11 +1,11 @@
 # multi stage to build tube archivist
 # build python wheel, download and extract ffmpeg, copy into final image
 
-FROM node:22.12.0-alpine AS npm-builder
+FROM node:22.13.0-alpine AS npm-builder
 COPY frontend/package.json frontend/package-lock.json /
 RUN npm i
 
-FROM node:22.12.0-alpine AS node-builder
+FROM node:22.13.0-alpine AS node-builder
 
 # RUN npm config set registry https://registry.npmjs.org/
 

backend/appsettings/serializers.py

@@ -55,7 +55,6 @@ class AppConfigDownloadsSerializer(
     )
     cookie_import = serializers.BooleanField()
     pot_provider_url = serializers.CharField(allow_null=True)
-    potoken = serializers.BooleanField()
     throttledratelimit = serializers.IntegerField(allow_null=True)
     extractor_lang = serializers.CharField(allow_null=True)
     integrate_ryd = serializers.BooleanField()
@@ -94,12 +93,6 @@ class CookieUpdateSerializer(serializers.Serializer):
     cookie = serializers.CharField()
 
 
-class PoTokenSerializer(serializers.Serializer):
-    """serialize PO token"""
-
-    potoken = serializers.CharField()
-
-
 class RescanFileSystemConfig(serializers.Serializer):
     """serialize rescan filesystem config"""
 

backend/appsettings/src/config.py

@@ -42,7 +42,6 @@ class DownloadsConfigType(TypedDict):
     comment_sort: Literal["top", "new"] | None
     cookie_import: bool
     pot_provider_url: str | None
-    potoken: bool
     throttledratelimit: int | None
     extractor_lang: str | None
     integrate_ryd: bool
@@ -92,7 +91,6 @@ class AppConfig:
             "comment_sort": "top",
             "cookie_import": False,
             "pot_provider_url": None,
-            "potoken": False,
             "throttledratelimit": None,
             "extractor_lang": None,
             "integrate_ryd": False,
@@ -179,6 +177,34 @@ class AppConfig:
 
         return updated
 
+    def clear_old_keys(self) -> list[str]:
+        """clear old unused keys"""
+        cleared = []
+        for key, value in self.config.items():
+            if key not in self.CONFIG_DEFAULTS:
+                # complete key removed
+                self.config.pop(key)
+                cleared.append(str({key: value}))
+                continue
+
+            expected_keys = set(
+                self.CONFIG_DEFAULTS[key].keys()  # type: ignore
+            )
+            is_keys = set(self.config[key].keys())
+
+            for to_delete in is_keys - expected_keys:
+                self.config[key].pop(to_delete)
+                cleared.append(f"{key}.{to_delete}")
+
+        if not cleared:
+            return []
+
+        response, status_code = ElasticWrap(self.ES_PATH).post(self.config)
+        if not status_code == 200:
+            print(response)
+
+        return cleared
+
 
 class ReleaseVersion:
     """compare local version with remote version"""

backend/appsettings/src/index_setup.py

@@ -13,6 +13,7 @@ from appsettings.src.snapshot import ElasticSnapshot
 from common.src.es_connect import ElasticWrap
 from common.src.helper import get_mapping
 from deepdiff import DeepDiff
+from deepdiff.model import DiffLevel
 from django.conf import settings
 
 
@@ -113,6 +114,8 @@ class ElasticIndex:
         now_set = self.details["settings"]["index"]
 
         for key, value in self.expected_set.items():
+            if key == "number_of_replicas":
+                continue
             if key not in now_set.keys():
                 print(key, value)
                 return True
@@ -150,24 +153,37 @@ class ElasticIndex:
             return MappingAction.REINDEX
 
         added = diff.get("dictionary_item_added", [])
-        has_additions = bool(added)
+        reindex_from_added = self._needs_reindex(diff_items=added)
+        if reindex_from_added:
+            return MappingAction.REINDEX
 
-        for item in diff.get("values_changed", []):
+        removed = diff.get("dictionary_item_removed", [])
-            path = item.path(output_format="list")
+        reindex_from_removed = self._needs_reindex(diff_items=removed)
-            if not path:
+        if reindex_from_removed:
-                continue
+            return MappingAction.REINDEX
 
-            if path[-1] in self.REINDEX_KEYS:
+        changed = diff.get("values_changed", [])
-                return MappingAction.REINDEX
+        reindex_from_changed = self._needs_reindex(diff_items=changed)
+        if reindex_from_changed:
+            return MappingAction.REINDEX
 
-            # compatible addition
+        if added or changed:
-            has_additions = True
-
-        if has_additions:
             return MappingAction.PUT_MAPPING
 
         return MappingAction.NOOP
 
+    def _needs_reindex(self, diff_items: list[DiffLevel]) -> bool:
+        """check if diff has fields that need reindex"""
+        for item in diff_items:
+            path = item.path(output_format="list")
+            if not path:
+                return False
+
+            if path[-1] in self.REINDEX_KEYS:
+                return True
+
+        return False
+
     def _get_fields_to_delete(self, diff: DeepDiff) -> set[str]:
         """fields to remove during next reindex"""
         removed_fields = set()
@@ -192,7 +208,7 @@ class ElasticIndex:
         self.create_blank(new_version=new_version)
         self.reindex(new_version=new_version, removed_fields=removed_fields)
         self.delete_index(by_version=current_version)
-        self.create_alias(new_version=new_version, old_version=current_version)
+        self.create_alias(new_version=new_version)
 
     def delete_index(self, by_version: int | None):
         """delete index passed as argument"""
@@ -257,7 +273,7 @@ class ElasticIndex:
             print(f"{status_code}: {response}")
             raise ValueError("reindex failed failed")
 
-    def create_alias(self, new_version: int, old_version: int | None = None):
+    def create_alias(self, new_version: int):
         """create aliast for moved index"""
         index_new = f"{self.index_namespace}_v{new_version}"
         index_old = None
@@ -273,16 +289,6 @@ class ElasticIndex:
                 },
             ]
         }
-        if old_version:
-            index_old = f"{self.index_namespace}_v{old_version}"
-            data["actions"].append(
-                {
-                    "remove": {
-                        "index": index_old,
-                        "alias": self.index_namespace,
-                    }
-                }
-            )
 
         message = f"create new alias {index_new}"
         if index_old:
@@ -292,7 +298,7 @@ class ElasticIndex:
         if settings.DEBUG:
             print(f"create alias with data: {data}")
 
-        response, status_code = ElasticWrap("_alias").put(data=data)
+        response, status_code = ElasticWrap("_aliases").post(data=data)
         if status_code not in [200, 201]:
             print(f"{status_code}: {response}")
             raise ValueError("alias update failed")

backend/appsettings/src/reindex.py

@@ -7,7 +7,7 @@ functionality:
 import json
 import os
 from datetime import datetime
-from typing import Callable, TypedDict
+from typing import TypedDict
 
 from appsettings.src.config import AppConfig
 from channel.src.index import YoutubeChannel
@@ -277,7 +277,6 @@ class Reindex(ReindexBase):
 
     def reindex_type(self, name: str, index_config: ReindexConfigType) -> None:
         """reindex all of a single index"""
-        reindex = self._get_reindex_map(index_config["index_name"])
         queue = RedisQueue(index_config["queue_name"])
         while True:
             total = queue.max_score()
@@ -288,44 +287,48 @@ class Reindex(ReindexBase):
             if self.task:
                 self._notify(name, total, idx)
 
-            reindex(youtube_id)
+            index_name = index_config["index_name"]
+            if index_name == "ta_vide":
+                video = self.reindex_single_video(youtube_id)
+                if video:
+                    self._reindex_video_related(video)
+
+            elif index_name == "ta_channel":
+                self._reindex_single_channel(channel_id=youtube_id)
+            elif index_name == "ta_playlist":
+                self._reindex_single_playlist(playlist_id=youtube_id)
+
             rand_sleep(self.config)
 
-    def _get_reindex_map(self, index_name: str) -> Callable:
-        """return def to run for index"""
-        def_map = {
-            "ta_video": self._reindex_single_video,
-            "ta_channel": self._reindex_single_channel,
-            "ta_playlist": self._reindex_single_playlist,
-        }
-
-        return def_map[index_name]
-
     def _notify(self, name: str, total: int, idx: int) -> None:
         """send notification back to task"""
         message = [f"Reindexing {name.title()}s {idx}/{total}"]
         progress = idx / total
         self.task.send_progress(message, progress=progress)
 
-    def _reindex_single_video(self, youtube_id: str) -> None:
+    def reindex_single_video(self, youtube_id: str) -> YoutubeVideo | None:
         """refresh data for single video"""
         video = YoutubeVideo(youtube_id)
 
         # read current state
         video.get_from_es()
         if not video.json_data:
-            return
+            return None
 
         es_meta = video.json_data.copy()
 
         # get new
-        media_url = os.path.join(
+        media_url: str | bool = os.path.join(
             EnvironmentSettings.MEDIA_DIR, es_meta["media_url"]
         )
+        if not os.path.exists(media_url):
+            # fallback to cache path
+            media_url = False
+
         video.build_json(media_path=media_url)
         if not video.youtube_meta:
             video.deactivate()
-            return
+            return None
 
         video.delete_subtitles(subtitles=es_meta.get("subtitles"))
         video.check_subtitles()
@@ -339,15 +342,19 @@ class Reindex(ReindexBase):
             video.json_data["playlist"] = es_meta.get("playlist")
 
         video.upload_to_es()
+        self.processed["videos"] += 1
 
-        thumb_handler = ThumbManager(youtube_id)
+        return video
+
+    def _reindex_video_related(self, video: YoutubeVideo) -> None:
+        """reindex video related metadata and fields"""
+        thumb_handler = ThumbManager(video.youtube_id)
         thumb_handler.delete_video_thumb()
         thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
 
-        Comments(youtube_id, config=self.config).reindex_comments()
+        Comments(video.youtube_id, config=self.config).reindex_comments()
         video.get_from_es()
         video.embed_metadata()
-        self.processed["videos"] += 1
 
     def _reindex_single_channel(self, channel_id: str) -> None:
         """refresh channel data and sync to videos"""

backend/appsettings/urls.py

@@ -34,11 +34,6 @@ urlpatterns = [
         views.CookieView.as_view(),
         name="api-cookie",
     ),
-    path(
-        "potoken/",
-        views.POTokenView.as_view(),
-        name="api-potoken",
-    ),
     path(
         "token/",
         views.TokenView.as_view(),

backend/appsettings/views.py

@@ -6,7 +6,6 @@ from appsettings.serializers import (
     CookieUpdateSerializer,
     CookieValidationSerializer,
     ManualImportConfig,
-    PoTokenSerializer,
     RescanFileSystemConfig,
     SnapshotCreateResponseSerializer,
     SnapshotItemSerializer,
@@ -24,7 +23,7 @@ from common.serializers import (
 from common.src.ta_redis import RedisArchivist
 from common.views_base import AdminOnly, AdminWriteOnly, ApiBaseView
 from django.conf import settings
-from download.src.yt_dlp_base import CookieHandler, POTokenHandler
+from download.src.yt_dlp_base import CookieHandler
 from drf_spectacular.utils import OpenApiResponse, extend_schema
 from rest_framework.authtoken.models import Token
 from rest_framework.response import Response
@@ -292,73 +291,6 @@ class CookieView(ApiBaseView):
         return validation
 
 
-class POTokenView(ApiBaseView):
-    """resolves to /api/appsettings/potoken/
-    GET: get potoken
-    POST: update potoken
-    DELETE: revoke potoken
-    """
-
-    permission_classes = [AdminOnly]
-
-    @extend_schema(
-        responses={
-            200: OpenApiResponse(PoTokenSerializer()),
-            404: OpenApiResponse(
-                ErrorResponseSerializer(), description="PO token not found"
-            ),
-        }
-    )
-    def get(self, request):
-        """get PO token"""
-        config = AppConfig().config
-        potoken = POTokenHandler(config).get()
-        if not potoken:
-            error = ErrorResponseSerializer({"error": "PO token not found"})
-            return Response(error.data, status=404)
-
-        serializer = PoTokenSerializer(data={"potoken": potoken})
-        serializer.is_valid(raise_exception=True)
-
-        return Response(serializer.data)
-
-    @extend_schema(
-        responses={
-            200: OpenApiResponse(PoTokenSerializer()),
-            400: OpenApiResponse(
-                ErrorResponseSerializer(), description="Bad request"
-            ),
-        }
-    )
-    def post(self, request):
-        """Update PO token"""
-        serializer = PoTokenSerializer(data=request.data)
-        serializer.is_valid(raise_exception=True)
-        validated_data = serializer.validated_data
-        if not validated_data:
-            error = ErrorResponseSerializer(
-                {"error": "missing PO token key in request data"}
-            )
-            return Response(error.data, status=400)
-
-        config = AppConfig().config
-        new_token = validated_data["potoken"]
-
-        POTokenHandler(config).set_token(new_token)
-        return Response(serializer.data)
-
-    @extend_schema(
-        responses={
-            204: OpenApiResponse(description="PO token revoked"),
-        },
-    )
-    def delete(self, request):
-        """delete PO token"""
-        config = AppConfig().config
-        POTokenHandler(config).revoke_token()
-        return Response(status=204)
-
-
 class SnapshotApiListView(ApiBaseView):
     """resolves to /api/appsettings/snapshot/
     GET: returns snapshot config plus list of existing snapshots

backend/common/src/env_settings.py

@@ -4,7 +4,7 @@ Functionality:
 - encapsulate persistence of application properties
 """
 
-from os import environ
+from os import environ, path
 
 try:
     from dotenv import load_dotenv
@@ -15,6 +15,32 @@ except ModuleNotFoundError:
     pass
 
 
+def get_password_from_file(env_var_name) -> str:
+    """get password from file"""
+
+    env_var_file: str = env_var_name + "_FILE"
+
+    env_var_name_val = environ.get(env_var_name)
+    env_var_path_val = environ.get(env_var_file)
+
+    if env_var_name_val is not None:
+        return str(env_var_name_val)
+
+    if env_var_path_val is None:
+        print(f"either {env_var_name} or {env_var_file} must be set")
+        return ""
+
+    is_path = path.isfile(env_var_path_val)
+    if not is_path:
+        print(f"{env_var_path_val} is not a path")
+        return ""
+
+    with open(env_var_path_val, "r", encoding="utf-8") as f:
+        file_content = f.read().strip()
+
+    return file_content
+
+
 class EnvironmentSettings:
     """
     Handle settings for the application that are driven from the environment.
@@ -29,7 +55,7 @@ class EnvironmentSettings:
     TA_PORT: int = int(environ.get("TA_PORT", False))
     TA_BACKEND_PORT: int = int(environ.get("TA_BACKEND_PORT", False))
     TA_USERNAME: str = str(environ.get("TA_USERNAME"))
-    TA_PASSWORD: str = str(environ.get("TA_PASSWORD"))
+    TA_PASSWORD: str = get_password_from_file("TA_PASSWORD")
 
     # Application Paths
     MEDIA_DIR: str = str(environ.get("TA_MEDIA_DIR", "/youtube"))
@@ -42,7 +68,7 @@ class EnvironmentSettings:
 
     # ElasticSearch
     ES_URL: str = str(environ.get("ES_URL"))
-    ES_PASS: str = str(environ.get("ELASTIC_PASSWORD"))
+    ES_PASS: str = get_password_from_file("ELASTIC_PASSWORD")
     ES_USER: str = str(environ.get("ELASTIC_USER", "elastic"))
     ES_SNAPSHOT_DIR: str = str(
         environ.get(

backend/common/src/es_connect.py

@@ -149,6 +149,7 @@ class IndexPaginate:
     - task: task object to send notification
     - total: int, total items in index for progress message
     - timeout: int, overwrite timeout in get request
+    - pit_keep_alive: int, overwrite pit valid
     """
 
     DEFAULT_SIZE = 500
@@ -169,7 +170,8 @@ class IndexPaginate:
 
     def get_pit(self):
         """get pit for index"""
-        path = f"{self.index_name}/_pit?keep_alive=15m"
+        keep_alive = self.kwargs.get("pit_keep_alive", 15)
+        path = f"{self.index_name}/_pit?keep_alive={keep_alive}m"
         response, _ = ElasticWrap(path).post()
         self.pit_id = response["id"]
 

backend/common/src/index_generic.py

@@ -53,11 +53,11 @@ class YouTubeItem:
             obs_request, self.config
         ).extract(url)
 
-    def get_from_es(self):
+    def get_from_es(self, print_error: bool = True) -> None:
         """get indexed data from elastic search"""
         print(f"{self.youtube_id}: get metadata from es")
-        response, _ = ElasticWrap(f"{self.es_path}").get()
+        resp, _ = ElasticWrap(f"{self.es_path}").get(print_error=print_error)
-        source = response.get("_source")
+        source = resp.get("_source")
         self.json_data = source
 
     def upload_to_es(self):

backend/common/src/searching.py

@@ -31,13 +31,13 @@ class SearchForm:
         fulltext_results = []
         if search_results:
             for result in search_results:
-                if result["_index"] == "ta_video":
+                if result["_index"].startswith("ta_video"):
                     video_results.append(result)
-                elif result["_index"] == "ta_channel":
+                elif result["_index"].startswith("ta_channel"):
                     channel_results.append(result)
-                elif result["_index"] == "ta_playlist":
+                elif result["_index"].startswith("ta_playlist"):
                     playlist_results.append(result)
-                elif result["_index"] == "ta_subtitle":
+                elif result["_index"].startswith("ta_subtitle"):
                     fulltext_results.append(result)
 
         all_results = {

backend/config/management/commands/ta_envcheck.py

@@ -61,6 +61,10 @@ EXPECTED_ENV_VARS = [
     "ES_URL",
     "TA_HOST",
 ]
+FILE_FALLBACK = [
+    "ELASTIC_PASSWORD",
+    "TA_PASSWORD",
+]
 UNEXPECTED_ENV_VARS = {
     "TA_UWSGI_PORT": "Has been replaced with 'TA_BACKEND_PORT'",
     "REDIS_HOST": "Has been replaced with 'REDIS_CON' connection string",
@@ -139,11 +143,16 @@ class Command(BaseCommand):
         self.stdout.write("[1] checking expected env vars")
         env = os.environ
         for var in EXPECTED_ENV_VARS:
-            if not env.get(var):
+            if var in env:
-                message = f"    🗙 expected env var {var} not set\n    {INST}"
+                continue
-                self.stdout.write(self.style.ERROR(message))
+
-                sleep(60)
+            if var in FILE_FALLBACK and f"{var}_FILE" in env:
-                raise CommandError(message)
+                continue
+
+            message = f"    🗙 expected env var {var} not set\n    {INST}"
+            self.stdout.write(self.style.ERROR(message))
+            sleep(60)
+            raise CommandError(message)
 
         message = "    ✓ all expected env vars are set"
         self.stdout.write(self.style.SUCCESS(message))

backend/config/management/commands/ta_startup.py

@@ -280,6 +280,12 @@ class Command(BaseCommand):
                     self.style.SUCCESS(f"    added new default: {new_default}")
                 )
 
+            cleared = AppConfig().clear_old_keys()
+            for removed_key in cleared:
+                self.stdout.write(
+                    self.style.SUCCESS(f"    removed old key: {removed_key}")
+                )
+
             return
 
         if status_code != 404:

backend/config/settings.py

@@ -221,7 +221,7 @@ CORS_EXPOSE_HEADERS = ["X-Start-Timestamp"]
 
 # TA application settings
 TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
-TA_VERSION = "v0.5.9"
+TA_VERSION = "v0.5.10-unstable"
 try:
     TA_START = RedisArchivist().get_message_str("STARTTIMESTAMP")
 except ValueError:

backend/download/src/yt_dlp_base.py

@@ -42,7 +42,6 @@ class YtWrap:
         deep_merge(self.obs, self.obs_request)
         if self.config:
             self._add_cookie()
-            self._add_potoken()
             self._add_potoken_url()
 
         if getattr(settings, "DEBUG", False):
@@ -58,22 +57,6 @@ class YtWrap:
 
         self.obs["cookiefile"] = cookie_io
 
-    def _add_potoken(self):
-        """add potoken if enabled"""
-        if self.config["downloads"].get("potoken"):
-            potoken = POTokenHandler(self.config).get()
-            deep_merge(
-                self.obs,
-                {
-                    "extractor_args": {
-                        "youtube": {
-                            "po_token": [potoken],
-                            "player-client": ["mweb", "default"],
-                        },
-                    }
-                },
-            )
-
     def _add_potoken_url(self):
         """add bgutils token url"""
         if pot_provider_url := self.config["downloads"].get(
@@ -248,27 +231,3 @@ class CookieHandler:
             "validated_str": now.strftime("%Y-%m-%d %H:%M"),
         }
         RedisArchivist().set_message("cookie:valid", message, expire=3600)
-
-
-class POTokenHandler:
-    """handle po token"""
-
-    REDIS_KEY = "potoken"
-
-    def __init__(self, config):
-        self.config = config
-
-    def get(self) -> str | None:
-        """get PO token"""
-        potoken = RedisArchivist().get_message_str(self.REDIS_KEY)
-        return potoken
-
-    def set_token(self, new_token: str) -> None:
-        """set new PO token"""
-        RedisArchivist().set_message(self.REDIS_KEY, new_token)
-        AppConfig().update_config({"downloads": {"potoken": True}})
-
-    def revoke_token(self) -> None:
-        """revoke token"""
-        RedisArchivist().del_message(self.REDIS_KEY)
-        AppConfig().update_config({"downloads": {"potoken": False}})

backend/playlist/src/index.py

@@ -314,7 +314,10 @@ class YoutubePlaylist(YouTubeItem):
         self.del_in_es()
 
     def is_custom_playlist(self):
-        self.get_from_es()
+        """check if is custom playlist"""
+        if not self.json_data:
+            self.get_from_es()
+
         return self.json_data["playlist_type"] == "custom"
 
     def delete_videos_metadata(self, channel_id=None):
@@ -392,13 +395,16 @@ class YoutubePlaylist(YouTubeItem):
         return True
 
     def remove_playlist_from_video(self, video_id):
+        """remove playlist id from video metadata"""
         video = ta_video.YoutubeVideo(video_id)
         video.get_from_es()
         if video.json_data is not None and "playlist" in video.json_data:
-            video.json_data["playlist"].remove(self.youtube_id)
+            if self.youtube_id in video.json_data["playlist"]:
-            video.upload_to_es()
+                video.json_data["playlist"].remove(self.youtube_id)
+                video.upload_to_es()
 
     def move_video(self, video_id, action, hide_watched=False):
+        """move video within custion playlist based on action"""
         self.get_from_es()
         video_index = self.get_video_index(video_id)
         playlist = self.json_data["playlist_entries"]

backend/requirements.txt

@@ -1,5 +1,5 @@
 apprise==1.9.7
-bgutil-ytdlp-pot-provider @ git+https://github.com/bbilly1/bgutil-ytdlp-pot-provider@c3f7bf4c99f8eb62015cb1c4142324f3d681cf07#subdirectory=plugin
+bgutil-ytdlp-pot-provider @ git+https://github.com/bbilly1/bgutil-ytdlp-pot-provider@47c4f8c21c6748d4522c82de0e78a2f1e38714a7#subdirectory=plugin
 celery==5.6.2
 deepdiff==8.6.1
 django-auth-ldap==5.3.0
@@ -8,9 +8,9 @@ django-cors-headers==4.9.0
 Django==5.2.11
 djangorestframework==3.16.1
 drf-spectacular==0.28.0  # rc:ignore
-Pillow==12.1.0
+Pillow==12.1.1
-redis==7.1.0
+redis==7.2.1
 requests==2.32.5
 ryd-client==0.0.6
-uvicorn==0.40.0
+uvicorn==0.41.0
-yt-dlp[default]==2026.2.4
+yt-dlp[default]==2026.2.21

backend/video/src/index.py

@@ -406,6 +406,10 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
             self.json_data["subtitles"] = indexed
             return
 
+        if not self.youtube_meta:
+            print(f"{self.youtube_id}: skip subtitle check without metadata")
+            return
+
         handler = YoutubeSubtitle(self)
         subtitles = handler.get_subtitles()
         if subtitles:
@@ -465,8 +469,10 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         video = MP4(file_path)
         video["\xa9nam"] = [title]  # title
         video["\xa9ART"] = [artist]  # artist
-        video["desc"] = [description]  # description
+        if description:
-        video["ldes"] = [description]  # synopsis
+            video["desc"] = [description]  # description
+            video["ldes"] = [description]  # synopsis
+
         video["----:com.tubearchivist:ta"] = [to_embed.encode("utf-8")]
         video.save()
 
@@ -508,8 +514,16 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
 def index_new_video(youtube_id, video_type=VideoTypeEnum.VIDEOS):
     """combined classes to create new video in index"""
+    from appsettings.src.reindex import Reindex
+
     video = YoutubeVideo(youtube_id, video_type=video_type)
-    video.build_json()
+    video.get_from_es(print_error=False)
+    if video.json_data:
+        # reindex only for force redownload
+        video = Reindex().reindex_single_video(youtube_id=youtube_id)
+    else:
+        video.build_json()
+
     if not video.json_data:
         raise ValueError("failed to get metadata for " + youtube_id)
 

backend/video/src/meta_embed.py

@@ -48,6 +48,7 @@ class MetadataEmbed:
             callback=MetadataEmbedCallback,
             task=self.task,
             total=self._get_total(),
+            pit_keep_alive=1000,
         )
         _ = paginate.get_results()
 

frontend/.nvmrc

@@ -1 +1 @@
-22.12.0
+22.13.0

frontend/src/api/actions/deletePoToken.ts

@@ -1,9 +0,0 @@
-import APIClient from '../../functions/APIClient';
-
-const deletePoToken = async () => {
-  return APIClient('/api/appsettings/potoken/', {
-    method: 'DELETE',
-  });
-};
-
-export default deletePoToken;

frontend/src/api/actions/updatePoToken.ts

@@ -1,10 +0,0 @@
-import APIClient from '../../functions/APIClient';
-
-const updatePoToken = async (potoken: string) => {
-  return APIClient('/api/appsettings/potoken/', {
-    method: 'POST',
-    body: { potoken },
-  });
-};
-
-export default updatePoToken;

frontend/src/api/loader/loadAppsettingsConfig.ts

@@ -23,7 +23,6 @@ export type AppSettingsConfigType = {
     comment_sort: string;
     cookie_import: boolean;
     pot_provider_url: string | null;
-    potoken: boolean;
     throttledratelimit: number | null;
     extractor_lang: string | null;
     integrate_ryd: boolean;

frontend/src/pages/SettingsApplication.tsx

@@ -17,8 +17,6 @@ import updateCookie from '../api/actions/updateCookie';
 import loadCookie, { CookieStateType } from '../api/loader/loadCookie';
 import deleteCookie from '../api/actions/deleteCookie';
 import validateCookie from '../api/actions/validateCookie';
-import deletePoToken from '../api/actions/deletePoToken';
-import updatePoToken from '../api/actions/updatePoToken';
 import { useUserConfigStore } from '../stores/UserConfigStore';
 import MembershipAppsettings from '../components/MembershipAppsettings';
 
@@ -71,8 +69,6 @@ const SettingsApplication = () => {
   // Cookie
   const [cookieFormData, setCookieFormData] = useState<string>('');
   const [showCookieForm, setShowCookieForm] = useState<boolean>(false);
-  const [poTokenFormData, setPoTokenFormData] = useState<string>('web+');
-  const [showPoTokenForm, setShowPoTokenForm] = useState<boolean>(false);
   const [potProviderUrl, setPotProviderUrl] = useState<string | null>(null);
 
   // Integrations
@@ -172,18 +168,6 @@ const SettingsApplication = () => {
     setRefresh(true);
   };
 
-  const handlePoTokenRevoke = async () => {
-    await deletePoToken();
-    setRefresh(true);
-  };
-
-  const handlePoTokenUpdate = async () => {
-    await updatePoToken(poTokenFormData);
-    setPoTokenFormData('web+');
-    setShowPoTokenForm(false);
-    setRefresh(true);
-  };
-
   useEffect(() => {
     // eslint-disable-next-line react-hooks/set-state-in-effect
     fetchData();
@@ -695,16 +679,6 @@ const SettingsApplication = () => {
                       </a>
                       .
                     </li>
-                    <li>
-                      The PO Token <i>(Proof of origin token)</i> can authenticate your request.
-                      Make sure to read the{' '}
-                      <a
-                        target="_blank"
-                        href="https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide"
-                      >
-                        PO guide
-                      </a>
-                    </li>
                     <li>
                       The PO Token Provider URL running external to tubearchivist. Make sure to
                       review{' '}
@@ -765,55 +739,6 @@ const SettingsApplication = () => {
                   )}
                 </div>
               </div>
-              <div className="settings-box-wrapper">
-                <div>
-                  <p>Add PO Token</p>
-                </div>
-                <div>
-                  {response?.appSettingsConfig?.downloads.potoken ? (
-                    <>
-                      <p>PO Token enabled.</p>
-                      <button onClick={handlePoTokenRevoke} className="danger-button">
-                        Revoke
-                      </button>
-                    </>
-                  ) : (
-                    <p>PO Token disabled</p>
-                  )}
-                  {showPoTokenForm ? (
-                    <div>
-                      <input
-                        type="text"
-                        value={poTokenFormData}
-                        onChange={async e => {
-                          setPoTokenFormData(e.target.value);
-                        }}
-                      />
-                      {poTokenFormData !== 'web+' && (
-                        <button onClick={handlePoTokenUpdate}>Update</button>
-                      )}
-                      <button
-                        onClick={() => {
-                          setShowPoTokenForm(false);
-                          setPoTokenFormData('web+');
-                        }}
-                      >
-                        Cancel
-                      </button>
-                    </div>
-                  ) : (
-                    <div>
-                      <button
-                        onClick={async () => {
-                          setShowPoTokenForm(true);
-                        }}
-                      >
-                        Update PO Token
-                      </button>
-                    </div>
-                  )}
-                </div>
-              </div>
               <div className="settings-box-wrapper">
                 <div>
                   <p>PO Token Provider URL</p>

frontend/src/stores/AppSettingsStore.ts

@@ -30,7 +30,6 @@ export const useAppSettingsStore = create<AppSettingsState>(set => ({
       comment_sort: 'asc',
       cookie_import: false,
       pot_provider_url: null,
-      potoken: false,
       throttledratelimit: null,
       extractor_lang: null,
       integrate_ryd: false,

requirements-dev.txt

@@ -2,8 +2,8 @@
 ipython==9.10.0
 pre-commit==4.5.1
 pylint-django==2.7.0
-pylint==4.0.4
+pylint==4.0.5
-pytest-django==4.11.1
+pytest-django==4.12.0
 pytest==9.0.2
 python-dotenv==1.2.1
 requirementscheck==0.1.0