use reindex for force redownload to preserve user metadata

backend/appsettings/src/reindex.py

@@ -7,7 +7,7 @@ functionality:
 import json
 import os
 from datetime import datetime
-from typing import Callable, TypedDict
+from typing import TypedDict
 
 from appsettings.src.config import AppConfig
 from channel.src.index import YoutubeChannel
@@ -277,7 +277,6 @@ class Reindex(ReindexBase):
 
     def reindex_type(self, name: str, index_config: ReindexConfigType) -> None:
         """reindex all of a single index"""
-        reindex = self._get_reindex_map(index_config["index_name"])
         queue = RedisQueue(index_config["queue_name"])
         while True:
             total = queue.max_score()
@@ -288,44 +287,48 @@ class Reindex(ReindexBase):
             if self.task:
                 self._notify(name, total, idx)
 
-            reindex(youtube_id)
+            index_name = index_config["index_name"]
+            if index_name == "ta_vide":
+                video = self.reindex_single_video(youtube_id)
+                if video:
+                    self._reindex_video_related(video)
+
+            elif index_name == "ta_channel":
+                self._reindex_single_channel(channel_id=youtube_id)
+            elif index_name == "ta_playlist":
+                self._reindex_single_playlist(playlist_id=youtube_id)
+
             rand_sleep(self.config)
 
-    def _get_reindex_map(self, index_name: str) -> Callable:
-        """return def to run for index"""
-        def_map = {
-            "ta_video": self._reindex_single_video,
-            "ta_channel": self._reindex_single_channel,
-            "ta_playlist": self._reindex_single_playlist,
-        }
-
-        return def_map[index_name]
-
     def _notify(self, name: str, total: int, idx: int) -> None:
         """send notification back to task"""
         message = [f"Reindexing {name.title()}s {idx}/{total}"]
         progress = idx / total
         self.task.send_progress(message, progress=progress)
 
-    def _reindex_single_video(self, youtube_id: str) -> None:
+    def reindex_single_video(self, youtube_id: str) -> YoutubeVideo | None:
         """refresh data for single video"""
         video = YoutubeVideo(youtube_id)
 
         # read current state
         video.get_from_es()
         if not video.json_data:
-            return
+            return None
 
         es_meta = video.json_data.copy()
 
         # get new
-        media_url = os.path.join(
+        media_url: str | bool = os.path.join(
             EnvironmentSettings.MEDIA_DIR, es_meta["media_url"]
         )
+        if not os.path.exists(media_url):
+            # fallback to cache path
+            media_url = False
+
         video.build_json(media_path=media_url)
         if not video.youtube_meta:
             video.deactivate()
-            return
+            return None
 
         video.delete_subtitles(subtitles=es_meta.get("subtitles"))
         video.check_subtitles()
@@ -339,15 +342,19 @@ class Reindex(ReindexBase):
             video.json_data["playlist"] = es_meta.get("playlist")
 
         video.upload_to_es()
+        self.processed["videos"] += 1
 
-        thumb_handler = ThumbManager(youtube_id)
+        return video
+
+    def _reindex_video_related(self, video: YoutubeVideo) -> None:
+        """reindex video related metadata and fields"""
+        thumb_handler = ThumbManager(video.youtube_id)
         thumb_handler.delete_video_thumb()
         thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
 
-        Comments(youtube_id, config=self.config).reindex_comments()
+        Comments(video.youtube_id, config=self.config).reindex_comments()
         video.get_from_es()
         video.embed_metadata()
-        self.processed["videos"] += 1
 
     def _reindex_single_channel(self, channel_id: str) -> None:
         """refresh channel data and sync to videos"""

backend/common/src/index_generic.py

@@ -53,11 +53,11 @@ class YouTubeItem:
             obs_request, self.config
         ).extract(url)
 
-    def get_from_es(self):
+    def get_from_es(self, print_error: bool = True) -> None:
         """get indexed data from elastic search"""
         print(f"{self.youtube_id}: get metadata from es")
-        response, _ = ElasticWrap(f"{self.es_path}").get()
+        resp, _ = ElasticWrap(f"{self.es_path}").get(print_error=print_error)
-        source = response.get("_source")
+        source = resp.get("_source")
         self.json_data = source
 
     def upload_to_es(self):

backend/video/src/index.py

@@ -406,6 +406,10 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
             self.json_data["subtitles"] = indexed
             return
 
+        if not self.youtube_meta:
+            print(f"{self.youtube_id}: skip subtitle check without metadata")
+            return
+
         handler = YoutubeSubtitle(self)
         subtitles = handler.get_subtitles()
         if subtitles:
@@ -508,8 +512,16 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
 def index_new_video(youtube_id, video_type=VideoTypeEnum.VIDEOS):
     """combined classes to create new video in index"""
+    from appsettings.src.reindex import Reindex
+
     video = YoutubeVideo(youtube_id, video_type=video_type)
-    video.build_json()
+    video.get_from_es(print_error=False)
+    if video.json_data:
+        # reindex only for force redownload
+        video = Reindex().reindex_single_video(youtube_id=youtube_id)
+    else:
+        video.build_json()
+
     if not video.json_data:
         raise ValueError("failed to get metadata for " + youtube_id)