From 94db7252ea07c3bfbdb3cc56c317ada45a5646b2 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 29 Oct 2025 10:11:55 +0700 Subject: [PATCH 1/7] remove middleware outdated inheritance --- backend/config/middleware.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/backend/config/middleware.py b/backend/config/middleware.py index 8d85e60f..77e64b83 100644 --- a/backend/config/middleware.py +++ b/backend/config/middleware.py @@ -1,11 +1,13 @@ """middleware""" from django.conf import settings -from django.utils.deprecation import MiddlewareMixin -class StartTimeMiddleware(MiddlewareMixin): - """add a start time header""" +class StartTimeMiddleware: + """set start time header""" + + def __init__(self, get_response): + self.get_response = get_response def __call__(self, request): response = self.get_response(request) From 815790e39733e025417c1c50a4c9d4b8deda6372 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 29 Oct 2025 18:54:07 +0700 Subject: [PATCH 2/7] add full metadata embedding --- backend/appsettings/src/reindex.py | 1 + backend/download/src/yt_dlp_handler.py | 40 +++++++++++------- backend/video/src/index.py | 57 ++++++++++++++++++++++++++ backend/video/src/subtitle.py | 10 ++++- 4 files changed, 93 insertions(+), 15 deletions(-) diff --git a/backend/appsettings/src/reindex.py b/backend/appsettings/src/reindex.py index b8569c08..234fd7fa 100644 --- a/backend/appsettings/src/reindex.py +++ b/backend/appsettings/src/reindex.py @@ -345,6 +345,7 @@ class Reindex(ReindexBase): thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"]) Comments(youtube_id, config=self.config).reindex_comments() + video.embed_metadata() self.processed["videos"] += 1 def _reindex_single_channel(self, channel_id: str) -> None: diff --git a/backend/download/src/yt_dlp_handler.py b/backend/download/src/yt_dlp_handler.py index 2221390d..ee14e2c9 100644 --- a/backend/download/src/yt_dlp_handler.py +++ b/backend/download/src/yt_dlp_handler.py @@ -188,22 +188,11 @@ class VideoDownloader(DownloaderBase): postprocessors = [] if self.config["downloads"]["add_metadata"]: + # full metadata is added in DownloadPostProcess postprocessors.append( { "key": "FFmpegMetadata", "add_chapters": True, - "add_metadata": True, - } - ) - postprocessors.append( - { - "key": "MetadataFromField", - "formats": [ - "%(title)s:%(meta_title)s", - "%(uploader)s:%(meta_artist)s", - ":(?P)", - ], - "when": "pre_process", } ) @@ -303,6 +292,9 @@ class DownloadPostProcess(DownloaderBase): self.refresh_playlist() self.match_videos() self.get_comments() + self.embed_metadata() + + RedisQueue(self.VIDEO_QUEUE).clear() def auto_delete_all(self): """handle auto delete""" @@ -488,6 +480,26 @@ class DownloadPostProcess(DownloaderBase): video_queue = RedisQueue(self.VIDEO_QUEUE) comment_list = CommentList(task=self.task) comment_list.add(video_ids=video_queue.get_all()) - - video_queue.clear() comment_list.index() + + def embed_metadata(self): + """embed metadata in media file""" + if not self.config["downloads"].get("add_metadata"): + return + + queue = RedisQueue(self.VIDEO_QUEUE) + total = queue.max_score() + video_ids = queue.get_all() + + for idx, youtube_id in enumerate(video_ids): + YoutubeVideo(youtube_id).embed_metadata() + + if not self.task: + continue + + message = [ + "Post Processing Videos.", + f"Embed metadata: - {idx}/{total}", + ] + progress = idx / total + self.task.send_progress(message, progress=progress) diff --git a/backend/video/src/index.py b/backend/video/src/index.py index 2aded46f..e33ba37c 100644 --- a/backend/video/src/index.py +++ b/backend/video/src/index.py @@ -4,6 +4,7 @@ functionality: - index and update in es """ +import json import os from datetime import datetime @@ -14,6 +15,7 @@ from common.src.helper import get_duration_sec, get_duration_str, randomizor from common.src.index_generic import YouTubeItem from django.conf import settings from download.src.thumbnails import ThumbManager +from mutagen.mp4 import MP4 from playlist.src import index as ta_playlist from ryd_client import ryd_client from user.src.user_config import UserConfig @@ -420,6 +422,61 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): return subtitles + def embed_metadata(self): + """embed metadata for video""" + if not self.config["downloads"].get("add_metadata"): + return + + print(f"{self.youtube_id}: embed metadata") + if not self.json_data: + self.get_from_es() + + video_base = EnvironmentSettings.MEDIA_DIR + media_url = self.json_data.get("media_url") + file_path = os.path.join(video_base, media_url) + + title = self.json_data["title"] + artist = self.json_data["channel"]["channel_name"] + description = self.json_data["description"] + to_embed = self._get_to_embed() + + video = MP4(file_path) + video["\xa9nam"] = [title] # title + video["\xa9ART"] = [artist] # artist + video["desc"] = [description] # description + video["ldes"] = [description] # synopsis + video["----:com.tubearchivist:ta"] = [to_embed.encode("utf-8")] + video.save() + + def _get_to_embed(self) -> str: + """get metadata json str to embed""" + comments = None + if self.json_data.get("comment_count"): + comments = Comments(self.youtube_id).get_es_comments() + + subtitles = None + if self.json_data.get("subtitles"): + subtitles = YoutubeSubtitle(video=self).get_es_subtitles() + + playlists = None + if self.json_data.get("playlist"): + playlists = [] + for playlist_id in self.json_data["playlist"]: + playlist = ta_playlist.YoutubePlaylist(playlist_id) + playlist.get_from_es() + playlists.append(playlist.json_data) + + to_embed = json.dumps( + { + "video": self.json_data, + "comments": comments, + "subtitles": subtitles, + "playlists": playlists, + } + ) + + return to_embed + def index_new_video(youtube_id, video_type=VideoTypeEnum.VIDEOS): """combined classes to create new video in index""" diff --git a/backend/video/src/subtitle.py b/backend/video/src/subtitle.py index 26370b08..42e686c9 100644 --- a/backend/video/src/subtitle.py +++ b/backend/video/src/subtitle.py @@ -13,7 +13,7 @@ from operator import itemgetter import requests from common.src.env_settings import EnvironmentSettings -from common.src.es_connect import ElasticWrap +from common.src.es_connect import ElasticWrap, IndexPaginate from common.src.helper import rand_sleep, requests_headers from yt_dlp.utils import orderedSet_from_options @@ -93,6 +93,14 @@ class YoutubeSubtitle: return candidate_subtitles + def get_es_subtitles(self) -> list[dict]: + """get subtitles from elastic""" + data = { + "query": {"term": {"youtube_id": {"value": self.video.youtube_id}}} + } + response = IndexPaginate("ta_subtitle", data).get_results() + return response + def download_subtitles(self, relevant_subtitles): """download subtitle files to archive""" subtitle_list = ", ".join(map(itemgetter("lang"), relevant_subtitles)) From 79d994e460c0d6cb22c196d3d8eea7b0424ffd8a Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 29 Oct 2025 19:01:56 +0700 Subject: [PATCH 3/7] handle embed error --- backend/video/src/index.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/backend/video/src/index.py b/backend/video/src/index.py index e33ba37c..0dbb2aae 100644 --- a/backend/video/src/index.py +++ b/backend/video/src/index.py @@ -431,9 +431,16 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): if not self.json_data: self.get_from_es() + if not self.json_data: + print(f"{self.youtube_id}: skip embed, video not indexed") + return + video_base = EnvironmentSettings.MEDIA_DIR media_url = self.json_data.get("media_url") file_path = os.path.join(video_base, media_url) + if not os.path.exists(file_path): + print(f"{self.youtube_id}: skip embed, file not found") + return title = self.json_data["title"] artist = self.json_data["channel"]["channel_name"] From e1393fc67c4cc007d96bb50e9fc6ceaabf5fb730 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 29 Oct 2025 22:31:30 +0700 Subject: [PATCH 4/7] add metadata embed task and callback --- backend/task/src/task_config.py | 8 +++ backend/task/tasks.py | 14 ++++++ backend/video/src/meta_embed.py | 51 ++++++++++++++++++++ frontend/src/api/actions/updateTaskByName.ts | 1 + frontend/src/pages/SettingsActions.tsx | 19 ++++++++ 5 files changed, 93 insertions(+) create mode 100644 backend/video/src/meta_embed.py diff --git a/backend/task/src/task_config.py b/backend/task/src/task_config.py index 5c3dc433..9fd494e9 100644 --- a/backend/task/src/task_config.py +++ b/backend/task/src/task_config.py @@ -87,6 +87,13 @@ RESYNC_THUMBS: TaskItemConfig = { "api_stop": False, } +RESYNC_METADATA: TaskItemConfig = { + "title": "Sync Metadata to Media Files", + "group": "setting:thumbnailsync", + "api_start": True, + "api_stop": False, +} + INDEX_PLAYLISTS: TaskItemConfig = { "title": "Index Channel Playlist", "group": "channel:indexplaylist", @@ -119,6 +126,7 @@ TASK_CONFIG: dict[str, TaskItemConfig] = { "rescan_filesystem": RESCAN_FILESYSTEM, "thumbnail_check": THUMBNAIL_CHECK, "resync_thumbs": RESYNC_THUMBS, + "resync_metadata": RESYNC_METADATA, "index_playlists": INDEX_PLAYLISTS, "subscribe_to": SUBSCRIBE_TO, "version_check": VERSION_CHECK, diff --git a/backend/task/tasks.py b/backend/task/tasks.py index 64fb5ab8..995e1c85 100644 --- a/backend/task/tasks.py +++ b/backend/task/tasks.py @@ -24,6 +24,7 @@ from download.src.yt_dlp_handler import VideoDownloader from task.src.notify import Notifications from task.src.task_config import TASK_CONFIG from task.src.task_manager import TaskManager +from video.src.meta_embed import MetadataEmbed class BaseTask(Task): @@ -302,6 +303,19 @@ def re_sync_thumbs(self): ThumbFilesystem(task=self).embed() +@shared_task(bind=True, name="resync_metadata", base=BaseTask) +def re_sync_metadata(self): + """resync metadata to media files""" + manager = TaskManager() + if manager.is_pending(self): + print(f"[task][{self.name}] metadata re-embed is already running") + self.send_progress(["Metadata re-embed is already running."]) + return + + manager.init(self) + MetadataEmbed(task=self).embed() + + @shared_task(bind=True, name="subscribe_to", base=BaseTask) def subscribe_to(self, url_str: str, expected_type: str | bool = False): """ diff --git a/backend/video/src/meta_embed.py b/backend/video/src/meta_embed.py new file mode 100644 index 00000000..0a7f62e1 --- /dev/null +++ b/backend/video/src/meta_embed.py @@ -0,0 +1,51 @@ +"""bulk metadata embedding""" + +from common.src.es_connect import ElasticWrap, IndexPaginate +from video.src.index import YoutubeVideo + + +class MetadataEmbed: + """sync metadata to videos in bulk""" + + INDEX_NAME = "ta_video" + + def __init__(self, task=False): + self.task = task + + def embed(self): + """entry point""" + data = { + "query": {"match_all": {}}, + "_source": ["youtube_id"], + } + paginate = IndexPaginate( + index_name=self.INDEX_NAME, + data=data, + size=200, + callback=MetadataEmbedCallback, + task=self.task, + total=self._get_total(), + ) + _ = paginate.get_results() + + def _get_total(self): + """get total documents in index""" + path = f"{self.INDEX_NAME}/_count" + response, _ = ElasticWrap(path).get() + + return response.get("count") + + +class MetadataEmbedCallback: + """callback for metadata embed""" + + def __init__(self, source, index_name, counter=0): + self.source = source + self.index_name = index_name + self.counter = counter + + def run(self): + """run embed""" + for video in self.source: + youtube_id = video["_source"]["youtube_id"] + YoutubeVideo(youtube_id).embed_metadata() diff --git a/frontend/src/api/actions/updateTaskByName.ts b/frontend/src/api/actions/updateTaskByName.ts index 786c1a26..7053dca9 100644 --- a/frontend/src/api/actions/updateTaskByName.ts +++ b/frontend/src/api/actions/updateTaskByName.ts @@ -5,6 +5,7 @@ type TaskNamesType = | 'update_subscribed' | 'manual_import' | 'resync_thumbs' + | 'resync_metadata' | 'rescan_filesystem'; const updateTaskByName = async (taskName: TaskNamesType) => { diff --git a/frontend/src/pages/SettingsActions.tsx b/frontend/src/pages/SettingsActions.tsx index 8d7b44cf..a226019d 100644 --- a/frontend/src/pages/SettingsActions.tsx +++ b/frontend/src/pages/SettingsActions.tsx @@ -13,6 +13,7 @@ const SettingsActions = () => { const [deletePending, setDeletePending] = useState(false); const [processingImports, setProcessingImports] = useState(false); const [reEmbed, setReEmbed] = useState(false); + const [reSyncMeta, setReSyncMeta] = useState(false); const [backupStarted, setBackupStarted] = useState(false); const [isRestoringBackup, setIsRestoringBackup] = useState(false); const [reScanningFileSystem, setReScanningFileSystem] = useState(false); @@ -44,6 +45,7 @@ const SettingsActions = () => { deletePending || processingImports || reEmbed || + reSyncMeta || backupStarted || isRestoringBackup || reScanningFileSystem @@ -53,6 +55,7 @@ const SettingsActions = () => { setDeletePending(false); setProcessingImports(false); setReEmbed(false); + setReSyncMeta(false); setBackupStarted(false); setIsRestoringBackup(false); setReScanningFileSystem(false); @@ -104,6 +107,22 @@ const SettingsActions = () => { )} +
+

Embed metadata into media file

+

Embed metadata into media files as mp4 tags.

+
+ {reSyncMeta &&

Processing video metadata

} + {!reSyncMeta && ( +
+

ZIP file index backup

From b6872d063fdea9b98051482650dd9ce7e48526fe Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 1 Nov 2025 10:56:55 +0700 Subject: [PATCH 5/7] round TA_START for multithread startup --- backend/config/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/config/settings.py b/backend/config/settings.py index 04b41879..b5fb184c 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -228,7 +228,7 @@ CORS_EXPOSE_HEADERS = ["X-Start-Timestamp"] # TA application settings TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist" TA_VERSION = "v0.5.8-unstable" -TA_START = str(int(datetime.now().timestamp())) +TA_START = str(int(datetime.now().timestamp() // 10 * 10)) # round it # API REST_FRAMEWORK = { From 3e7baebec97b657f60f58f585e1ca2409c02d383 Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 1 Nov 2025 11:26:56 +0700 Subject: [PATCH 6/7] add deno, pin yt-dlp to master --- Dockerfile | 2 ++ backend/requirements.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c4319f71..16138904 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,6 +42,8 @@ ARG INSTALL_DEBUG ENV PYTHONUNBUFFERED=1 +COPY --from=denoland/deno:bin /deno /usr/local/bin/deno + # copy build requirements COPY --from=builder /root/.local /root/.local ENV PATH=/root/.local/bin:$PATH diff --git a/backend/requirements.txt b/backend/requirements.txt index ac0d1f7b..8bfb5703 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -12,4 +12,4 @@ requests==2.32.5 ryd-client==0.0.6 uvicorn==0.38.0 whitenoise==6.11.0 -yt-dlp[default]==2025.10.22 +yt-dlp[default] @ git+https://github.com/yt-dlp/yt-dlp@6224a3898821965a7d6a2cb9cc2de40a0fd6e6bc From d5cfbd03ca3cac16bf36ad13720b28b417b88794 Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 1 Nov 2025 12:26:59 +0700 Subject: [PATCH 7/7] clean up debug output --- backend/download/src/queue.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/download/src/queue.py b/backend/download/src/queue.py index dbd4116e..e03a8871 100644 --- a/backend/download/src/queue.py +++ b/backend/download/src/queue.py @@ -462,8 +462,8 @@ class PendingList(PendingIndex): response, status_code = ElasticWrap("_bulk").post( query_str, ndjson=True ) - print(response) - if status_code != 200: + if status_code not in [200, 201]: + print(response) self._notify_fail(status_code) elif response.get("errors", False): failed_video_ids = []