add full metadata embedding

backend/appsettings/src/reindex.py

@@ -345,6 +345,7 @@ class Reindex(ReindexBase):
         thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
 
         Comments(youtube_id, config=self.config).reindex_comments()
+        video.embed_metadata()
         self.processed["videos"] += 1
 
     def _reindex_single_channel(self, channel_id: str) -> None:

backend/download/src/yt_dlp_handler.py

@@ -188,22 +188,11 @@ class VideoDownloader(DownloaderBase):
         postprocessors = []
 
         if self.config["downloads"]["add_metadata"]:
+            # full metadata is added in DownloadPostProcess
             postprocessors.append(
                 {
                     "key": "FFmpegMetadata",
                     "add_chapters": True,
-                    "add_metadata": True,
-                }
-            )
-            postprocessors.append(
-                {
-                    "key": "MetadataFromField",
-                    "formats": [
-                        "%(title)s:%(meta_title)s",
-                        "%(uploader)s:%(meta_artist)s",
-                        ":(?P<album>)",
-                    ],
-                    "when": "pre_process",
                 }
             )
 
@@ -303,6 +292,9 @@ class DownloadPostProcess(DownloaderBase):
         self.refresh_playlist()
         self.match_videos()
         self.get_comments()
+        self.embed_metadata()
+
+        RedisQueue(self.VIDEO_QUEUE).clear()
 
     def auto_delete_all(self):
         """handle auto delete"""
@@ -488,6 +480,26 @@ class DownloadPostProcess(DownloaderBase):
         video_queue = RedisQueue(self.VIDEO_QUEUE)
         comment_list = CommentList(task=self.task)
         comment_list.add(video_ids=video_queue.get_all())
-
-        video_queue.clear()
         comment_list.index()
+
+    def embed_metadata(self):
+        """embed metadata in media file"""
+        if not self.config["downloads"].get("add_metadata"):
+            return
+
+        queue = RedisQueue(self.VIDEO_QUEUE)
+        total = queue.max_score()
+        video_ids = queue.get_all()
+
+        for idx, youtube_id in enumerate(video_ids):
+            YoutubeVideo(youtube_id).embed_metadata()
+
+            if not self.task:
+                continue
+
+            message = [
+                "Post Processing Videos.",
+                f"Embed metadata: - {idx}/{total}",
+            ]
+            progress = idx / total
+            self.task.send_progress(message, progress=progress)

backend/video/src/index.py

@@ -4,6 +4,7 @@ functionality:
 - index and update in es
 """
 
+import json
 import os
 from datetime import datetime
 
@@ -14,6 +15,7 @@ from common.src.helper import get_duration_sec, get_duration_str, randomizor
 from common.src.index_generic import YouTubeItem
 from django.conf import settings
 from download.src.thumbnails import ThumbManager
+from mutagen.mp4 import MP4
 from playlist.src import index as ta_playlist
 from ryd_client import ryd_client
 from user.src.user_config import UserConfig
@@ -420,6 +422,61 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
         return subtitles
 
+    def embed_metadata(self):
+        """embed metadata for video"""
+        if not self.config["downloads"].get("add_metadata"):
+            return
+
+        print(f"{self.youtube_id}: embed metadata")
+        if not self.json_data:
+            self.get_from_es()
+
+        video_base = EnvironmentSettings.MEDIA_DIR
+        media_url = self.json_data.get("media_url")
+        file_path = os.path.join(video_base, media_url)
+
+        title = self.json_data["title"]
+        artist = self.json_data["channel"]["channel_name"]
+        description = self.json_data["description"]
+        to_embed = self._get_to_embed()
+
+        video = MP4(file_path)
+        video["\xa9nam"] = [title]  # title
+        video["\xa9ART"] = [artist]  # artist
+        video["desc"] = [description]  # description
+        video["ldes"] = [description]  # synopsis
+        video["----:com.tubearchivist:ta"] = [to_embed.encode("utf-8")]
+        video.save()
+
+    def _get_to_embed(self) -> str:
+        """get metadata json str to embed"""
+        comments = None
+        if self.json_data.get("comment_count"):
+            comments = Comments(self.youtube_id).get_es_comments()
+
+        subtitles = None
+        if self.json_data.get("subtitles"):
+            subtitles = YoutubeSubtitle(video=self).get_es_subtitles()
+
+        playlists = None
+        if self.json_data.get("playlist"):
+            playlists = []
+            for playlist_id in self.json_data["playlist"]:
+                playlist = ta_playlist.YoutubePlaylist(playlist_id)
+                playlist.get_from_es()
+                playlists.append(playlist.json_data)
+
+        to_embed = json.dumps(
+            {
+                "video": self.json_data,
+                "comments": comments,
+                "subtitles": subtitles,
+                "playlists": playlists,
+            }
+        )
+
+        return to_embed
+
 
 def index_new_video(youtube_id, video_type=VideoTypeEnum.VIDEOS):
     """combined classes to create new video in index"""

backend/video/src/subtitle.py

@@ -13,7 +13,7 @@ from operator import itemgetter
 
 import requests
 from common.src.env_settings import EnvironmentSettings
-from common.src.es_connect import ElasticWrap
+from common.src.es_connect import ElasticWrap, IndexPaginate
 from common.src.helper import rand_sleep, requests_headers
 from yt_dlp.utils import orderedSet_from_options
 
@@ -93,6 +93,14 @@ class YoutubeSubtitle:
 
         return candidate_subtitles
 
+    def get_es_subtitles(self) -> list[dict]:
+        """get subtitles from elastic"""
+        data = {
+            "query": {"term": {"youtube_id": {"value": self.video.youtube_id}}}
+        }
+        response = IndexPaginate("ta_subtitle", data).get_results()
+        return response
+
     def download_subtitles(self, relevant_subtitles):
         """download subtitle files to archive"""
         subtitle_list = ", ".join(map(itemgetter("lang"), relevant_subtitles))