diff --git a/backend/appsettings/src/reindex.py b/backend/appsettings/src/reindex.py index b8569c08..234fd7fa 100644 --- a/backend/appsettings/src/reindex.py +++ b/backend/appsettings/src/reindex.py @@ -345,6 +345,7 @@ class Reindex(ReindexBase): thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"]) Comments(youtube_id, config=self.config).reindex_comments() + video.embed_metadata() self.processed["videos"] += 1 def _reindex_single_channel(self, channel_id: str) -> None: diff --git a/backend/download/src/yt_dlp_handler.py b/backend/download/src/yt_dlp_handler.py index 2221390d..ee14e2c9 100644 --- a/backend/download/src/yt_dlp_handler.py +++ b/backend/download/src/yt_dlp_handler.py @@ -188,22 +188,11 @@ class VideoDownloader(DownloaderBase): postprocessors = [] if self.config["downloads"]["add_metadata"]: + # full metadata is added in DownloadPostProcess postprocessors.append( { "key": "FFmpegMetadata", "add_chapters": True, - "add_metadata": True, - } - ) - postprocessors.append( - { - "key": "MetadataFromField", - "formats": [ - "%(title)s:%(meta_title)s", - "%(uploader)s:%(meta_artist)s", - ":(?P)", - ], - "when": "pre_process", } ) @@ -303,6 +292,9 @@ class DownloadPostProcess(DownloaderBase): self.refresh_playlist() self.match_videos() self.get_comments() + self.embed_metadata() + + RedisQueue(self.VIDEO_QUEUE).clear() def auto_delete_all(self): """handle auto delete""" @@ -488,6 +480,26 @@ class DownloadPostProcess(DownloaderBase): video_queue = RedisQueue(self.VIDEO_QUEUE) comment_list = CommentList(task=self.task) comment_list.add(video_ids=video_queue.get_all()) - - video_queue.clear() comment_list.index() + + def embed_metadata(self): + """embed metadata in media file""" + if not self.config["downloads"].get("add_metadata"): + return + + queue = RedisQueue(self.VIDEO_QUEUE) + total = queue.max_score() + video_ids = queue.get_all() + + for idx, youtube_id in enumerate(video_ids): + YoutubeVideo(youtube_id).embed_metadata() + + if not self.task: + continue + + message = [ + "Post Processing Videos.", + f"Embed metadata: - {idx}/{total}", + ] + progress = idx / total + self.task.send_progress(message, progress=progress) diff --git a/backend/video/src/index.py b/backend/video/src/index.py index 2aded46f..e33ba37c 100644 --- a/backend/video/src/index.py +++ b/backend/video/src/index.py @@ -4,6 +4,7 @@ functionality: - index and update in es """ +import json import os from datetime import datetime @@ -14,6 +15,7 @@ from common.src.helper import get_duration_sec, get_duration_str, randomizor from common.src.index_generic import YouTubeItem from django.conf import settings from download.src.thumbnails import ThumbManager +from mutagen.mp4 import MP4 from playlist.src import index as ta_playlist from ryd_client import ryd_client from user.src.user_config import UserConfig @@ -420,6 +422,61 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): return subtitles + def embed_metadata(self): + """embed metadata for video""" + if not self.config["downloads"].get("add_metadata"): + return + + print(f"{self.youtube_id}: embed metadata") + if not self.json_data: + self.get_from_es() + + video_base = EnvironmentSettings.MEDIA_DIR + media_url = self.json_data.get("media_url") + file_path = os.path.join(video_base, media_url) + + title = self.json_data["title"] + artist = self.json_data["channel"]["channel_name"] + description = self.json_data["description"] + to_embed = self._get_to_embed() + + video = MP4(file_path) + video["\xa9nam"] = [title] # title + video["\xa9ART"] = [artist] # artist + video["desc"] = [description] # description + video["ldes"] = [description] # synopsis + video["----:com.tubearchivist:ta"] = [to_embed.encode("utf-8")] + video.save() + + def _get_to_embed(self) -> str: + """get metadata json str to embed""" + comments = None + if self.json_data.get("comment_count"): + comments = Comments(self.youtube_id).get_es_comments() + + subtitles = None + if self.json_data.get("subtitles"): + subtitles = YoutubeSubtitle(video=self).get_es_subtitles() + + playlists = None + if self.json_data.get("playlist"): + playlists = [] + for playlist_id in self.json_data["playlist"]: + playlist = ta_playlist.YoutubePlaylist(playlist_id) + playlist.get_from_es() + playlists.append(playlist.json_data) + + to_embed = json.dumps( + { + "video": self.json_data, + "comments": comments, + "subtitles": subtitles, + "playlists": playlists, + } + ) + + return to_embed + def index_new_video(youtube_id, video_type=VideoTypeEnum.VIDEOS): """combined classes to create new video in index""" diff --git a/backend/video/src/subtitle.py b/backend/video/src/subtitle.py index 26370b08..42e686c9 100644 --- a/backend/video/src/subtitle.py +++ b/backend/video/src/subtitle.py @@ -13,7 +13,7 @@ from operator import itemgetter import requests from common.src.env_settings import EnvironmentSettings -from common.src.es_connect import ElasticWrap +from common.src.es_connect import ElasticWrap, IndexPaginate from common.src.helper import rand_sleep, requests_headers from yt_dlp.utils import orderedSet_from_options @@ -93,6 +93,14 @@ class YoutubeSubtitle: return candidate_subtitles + def get_es_subtitles(self) -> list[dict]: + """get subtitles from elastic""" + data = { + "query": {"term": {"youtube_id": {"value": self.video.youtube_id}}} + } + response = IndexPaginate("ta_subtitle", data).get_results() + return response + def download_subtitles(self, relevant_subtitles): """download subtitle files to archive""" subtitle_list = ", ".join(map(itemgetter("lang"), relevant_subtitles))