diff --git a/backend/common/src/search_processor.py b/backend/common/src/search_processor.py index e5c0cac4..e0ceaefc 100644 --- a/backend/common/src/search_processor.py +++ b/backend/common/src/search_processor.py @@ -182,17 +182,23 @@ class SearchProcess: def _process_comment(self, comment_dict): """run on all comments, create reply thread""" - all_comments = comment_dict["comment_comments"] - processed_comments = [] + comment_tree = [] + lookup = {} + for comment in comment_dict["comment_comments"]: + comment = comment.copy() + comment["comment_replies"] = [] + lookup[comment["comment_id"]] = comment - for comment in all_comments: - if comment["comment_parent"] == "root": - comment.update({"comment_replies": []}) - processed_comments.append(comment) + for comment in lookup.values(): + parent = comment.get("comment_parent") + if parent == "root": + comment_tree.append(comment) else: - processed_comments[-1]["comment_replies"].append(comment) + parent_node = lookup.get(parent) + if parent_node: + parent_node["comment_replies"].append(comment) - return processed_comments + return comment_tree def _process_subtitle(self, result): """take complete result dict to extract highlight""" diff --git a/backend/config/management/commands/ta_index_channel_tabs.py b/backend/config/management/commands/ta_index_channel_tabs.py deleted file mode 100644 index 339843e8..00000000 --- a/backend/config/management/commands/ta_index_channel_tabs.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -migration for 0.5.4 to 0.5.5 -index channel_tabs for subscribed channels -""" - -import time - -from channel.src.index import YoutubeChannel -from common.src.helper import get_channels -from django.core.management.base import BaseCommand - - -class Command(BaseCommand): - """command""" - - def handle(self, *args, **kwargs): - """handle task""" - - self.stdout.write("channel tags initial index") - - channels = get_channels(subscribed_only=True, source=["channel_id"]) - - for es_channel in channels: - channel = YoutubeChannel(es_channel["channel_id"]) - channel.get_from_es() - channel_name = channel.json_data["channel_name"] - channel_tabs = channel.get_channel_tabs() - channel.json_data["channel_tabs"] = channel_tabs - channel.upload_to_es() - channel.sync_to_videos() - self.stdout.write( - self.style.SUCCESS( - f" ✓ updated '{channel_name}' tabs: {channel_tabs}" - ) - ) - time.sleep(5) diff --git a/backend/config/management/commands/ta_startup.py b/backend/config/management/commands/ta_startup.py index a0eb6863..7c7ae349 100644 --- a/backend/config/management/commands/ta_startup.py +++ b/backend/config/management/commands/ta_startup.py @@ -37,8 +37,6 @@ TOPIC = """ class Command(BaseCommand): """command framework""" - # pylint: disable=no-member - def handle(self, *args, **options): """run all commands""" self.stdout.write(TOPIC) @@ -53,10 +51,10 @@ class Command(BaseCommand): self._update_schedule_tz() self._init_app_config() self._set_ta_startup_time() - self._mig_fix_download_channel_indexed() self._mig_add_default_playlist_sort() self._mig_set_channel_tabs() self._mig_set_video_channel_tabs() + self._mig_fix_playlist_description() def _make_folders(self): """make expected cache folders""" @@ -271,40 +269,6 @@ class Command(BaseCommand): self.style.SUCCESS(f" ✓ set timestamp to {message}.") ) - def _mig_fix_download_channel_indexed(self) -> None: - """migrate from v0.5.2 to 0.5.3, fix missing channel_indexed""" - self.stdout.write("[MIGRATION] fix incorrect video channel tags types") - path = "ta_download/_update_by_query" - data = { - "query": { - "bool": { - "must_not": [{"exists": {"field": "channel_indexed"}}] - } - }, - "script": { - "source": "ctx._source.channel_indexed = false", - "lang": "painless", - }, - } - response, status_code = ElasticWrap(path).post(data) - if status_code in [200, 201]: - updated = response.get("updated") - if updated: - self.stdout.write( - self.style.SUCCESS(f" ✓ fixed {updated} queued videos") - ) - else: - self.stdout.write( - self.style.SUCCESS(" no queued videos to fix") - ) - return - - message = " 🗙 failed to fix video channel tags" - self.stdout.write(self.style.ERROR(message)) - self.stdout.write(response) - sleep(60) - raise CommandError(message) - def _mig_add_default_playlist_sort(self) -> None: """migrate from 0.5.4 to 0.5.5 set default playlist sortorder""" self.stdout.write("[MIGRATION] set default playlist sort order") @@ -408,3 +372,34 @@ class Command(BaseCommand): self.stdout.write(response) sleep(60) raise CommandError(message) + + def _mig_fix_playlist_description(self) -> None: + """migrate from 0.5.8 to 0.5.9 fix playlist desc null data type""" + self.stdout.write("[MIGRATION] fix playlist description data type") + + path = "ta_playlist/_update_by_query" + data = { + "query": {"term": {"playlist_description": {"value": False}}}, + "script": { + "source": "ctx._source.playlist_description = null", + "lang": "painless", + }, + } + response, status_code = ElasticWrap(path).post(data) + if status_code in [200, 201]: + updated = response.get("updated") + if updated: + self.stdout.write( + self.style.SUCCESS(f" ✓ updated {updated} playlists") + ) + else: + self.stdout.write( + self.style.SUCCESS(" no playlists need updating") + ) + return + + message = " 🗙 failed to fix playlist description null data type" + self.stdout.write(self.style.ERROR(message)) + self.stdout.write(response) + sleep(60) + raise CommandError(message) diff --git a/backend/download/src/thumbnails.py b/backend/download/src/thumbnails.py index c4efa66f..0b6a7c44 100644 --- a/backend/download/src/thumbnails.py +++ b/backend/download/src/thumbnails.py @@ -234,38 +234,41 @@ class ThumbManager(ThumbManagerBase): banner_path = os.path.join( self.CHANNEL_DIR, f"{channel_id}_banner.jpg" ) - if os.path.exists(banner_path): - with open(banner_path, "rb") as f: - banner_data = f.read() - - video["----:com.tubearchivist:channel_banner"] = [ - MP4Cover(banner_data, imageformat=MP4Cover.FORMAT_JPEG) - ] + self._embed_art_item(video, "channel_banner", art_path=banner_path) channel_icon_path = os.path.join( self.CHANNEL_DIR, f"{channel_id}_thumb.jpg" ) - if os.path.exists(channel_icon_path): - with open(channel_icon_path, "rb") as f: - icon_data = f.read() - - video["----:com.tubearchivist:channel_icon"] = [ - MP4Cover(icon_data, imageformat=MP4Cover.FORMAT_JPEG) - ] + self._embed_art_item(video, "channel_icon", art_path=channel_icon_path) channel_tv_path = os.path.join( self.CHANNEL_DIR, f"{channel_id}_tvart.jpg" ) - if os.path.exists(channel_tv_path): - with open(channel_tv_path, "rb") as f: - tv_data = f.read() + self._embed_art_item(video, "channel_tv", art_path=channel_tv_path) - video["----:com.tubearchivist:channel_tv"] = [ - MP4Cover(tv_data, imageformat=MP4Cover.FORMAT_JPEG) - ] + playlist_ids = json_data.get("playlist", []) + for plalyist_id in playlist_ids: + playlist_path = os.path.join( + self.PLAYLIST_DIR, f"{plalyist_id}.jpg" + ) + self._embed_art_item( + video, f"playlist_{plalyist_id}", art_path=playlist_path + ) video.save() + def _embed_art_item(self, video, key, art_path): + """embed single item""" + if not os.path.exists(art_path): + return + + with open(art_path, "rb") as f: + art_data = f.read() + + video[f"----:com.tubearchivist:{key}"] = [ + MP4Cover(art_data, imageformat=MP4Cover.FORMAT_JPEG) + ] + def delete_video_thumb(self): """delete video thumbnail if exists""" thumb_path = self.vid_thumb_path() @@ -501,7 +504,7 @@ class ThumbFilesystem: paginate = IndexPaginate( index_name=self.INDEX_NAME, data=data, - size=200, + size=100, callback=EmbedCallback, task=self.task, total=self._get_total(), diff --git a/backend/download/src/yt_dlp_handler.py b/backend/download/src/yt_dlp_handler.py index 08017a3c..9459ea3b 100644 --- a/backend/download/src/yt_dlp_handler.py +++ b/backend/download/src/yt_dlp_handler.py @@ -429,7 +429,7 @@ class DownloadPostProcess(DownloaderBase): channel = YoutubeChannel(channel_id) channel.get_from_es() overwrites = channel.get_overwrites() - if "index_playlists" in overwrites: + if overwrites.get("index_playlists"): channel.get_all_playlists() to_add = [i[0] for i in channel.all_playlists] RedisQueue(self.PLAYLIST_QUEUE).add_list(to_add) diff --git a/backend/playlist/serializers.py b/backend/playlist/serializers.py index cb2a12c7..6293b483 100644 --- a/backend/playlist/serializers.py +++ b/backend/playlist/serializers.py @@ -11,7 +11,7 @@ class PlaylistEntrySerializer(serializers.Serializer): youtube_id = serializers.CharField() title = serializers.CharField() - uploader = serializers.CharField() + uploader = serializers.CharField(allow_null=True) idx = serializers.IntegerField() downloaded = serializers.BooleanField() @@ -22,7 +22,7 @@ class PlaylistSerializer(serializers.Serializer): playlist_active = serializers.BooleanField() playlist_channel = serializers.CharField() playlist_channel_id = serializers.CharField() - playlist_description = serializers.CharField() + playlist_description = serializers.CharField(allow_null=True) playlist_entries = PlaylistEntrySerializer(many=True) playlist_id = serializers.CharField() playlist_last_refresh = serializers.CharField() diff --git a/backend/playlist/src/index.py b/backend/playlist/src/index.py index d2e6e301..2f96ba8d 100644 --- a/backend/playlist/src/index.py +++ b/backend/playlist/src/index.py @@ -81,7 +81,7 @@ class YoutubePlaylist(YouTubeItem): "playlist_channel": self.youtube_meta["channel"], "playlist_channel_id": self.youtube_meta["channel_id"], "playlist_thumbnail": playlist_thumbnail, - "playlist_description": self.youtube_meta["description"] or False, + "playlist_description": self.youtube_meta["description"] or None, "playlist_last_refresh": int(datetime.now().timestamp()), "playlist_type": "regular", } diff --git a/backend/requirements.txt b/backend/requirements.txt index 136e49af..a310c1f5 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -11,4 +11,4 @@ redis==7.1.0 requests==2.32.5 ryd-client==0.0.6 uvicorn==0.38.0 -yt-dlp[default]==2025.12.8 +yt-dlp[default] @ git+https://github.com/yt-dlp/yt-dlp@468aa6a9b431194949ede9eaad8f33e314a288d6 diff --git a/backend/task/src/task_config.py b/backend/task/src/task_config.py index 8d8e3a7e..6442dd22 100644 --- a/backend/task/src/task_config.py +++ b/backend/task/src/task_config.py @@ -80,13 +80,6 @@ THUMBNAIL_CHECK: TaskItemConfig = { "api_stop": False, } -RESYNC_THUMBS: TaskItemConfig = { - "title": "Sync Thumbnails to Media Files", - "group": "setting:thumbnailsync", - "api_start": True, - "api_stop": False, -} - RESYNC_METADATA: TaskItemConfig = { "title": "Sync Metadata to Media Files", "group": "setting:thumbnailsync", @@ -125,7 +118,6 @@ TASK_CONFIG: dict[str, TaskItemConfig] = { "restore_backup": RESTORE_BACKUP, "rescan_filesystem": RESCAN_FILESYSTEM, "thumbnail_check": THUMBNAIL_CHECK, - "resync_thumbs": RESYNC_THUMBS, "resync_metadata": RESYNC_METADATA, "index_playlists": INDEX_PLAYLISTS, "subscribe_to": SUBSCRIBE_TO, diff --git a/backend/task/tasks.py b/backend/task/tasks.py index 8b31b8dc..1b4293b3 100644 --- a/backend/task/tasks.py +++ b/backend/task/tasks.py @@ -19,7 +19,7 @@ from common.src.ta_redis import RedisArchivist from common.src.urlparser import ParsedURLType, Parser from download.src.queue import PendingList from download.src.subscriptions import SubscriptionHandler, SubscriptionScanner -from download.src.thumbnails import ThumbFilesystem, ThumbValidator +from download.src.thumbnails import ThumbValidator from download.src.yt_dlp_handler import VideoDownloader from task.src.notify import Notifications from task.src.task_config import TASK_CONFIG @@ -294,19 +294,6 @@ def thumbnail_check(self): thumbnail.clean_up() -@shared_task(bind=True, name="resync_thumbs", base=BaseTask) -def re_sync_thumbs(self): - """sync thumbnails to mediafiles""" - manager = TaskManager() - if manager.is_pending(self): - print(f"[task][{self.name}] thumb re-embed is already running") - self.send_progress(["Thumbnail re-embed is already running."]) - return - - manager.init(self) - ThumbFilesystem(task=self).embed() - - @shared_task(bind=True, name="resync_metadata", base=BaseTask) def re_sync_metadata(self): """resync metadata to media files""" diff --git a/backend/user/src/user_config.py b/backend/user/src/user_config.py index 8cd5c01d..b4543b3a 100644 --- a/backend/user/src/user_config.py +++ b/backend/user/src/user_config.py @@ -40,7 +40,7 @@ class UserConfig: _DEFAULT_USER_SETTINGS = UserConfigType( stylesheet="dark.css", - page_size=12, + page_size=25, sort_by="published", sort_order="desc", view_style_home="grid", diff --git a/backend/video/serializers.py b/backend/video/serializers.py index 5a5d7e3e..1cdf167c 100644 --- a/backend/video/serializers.py +++ b/backend/video/serializers.py @@ -4,6 +4,7 @@ from channel.serializers import ChannelSerializer from common.serializers import PaginationSerializer +from drf_spectacular.utils import extend_schema_field from rest_framework import serializers from video.src.constants import OrderEnum, SortEnum, VideoTypeEnum, WatchedEnum @@ -107,7 +108,7 @@ class VideoSerializer(serializers.Serializer): sponsorblock = SponsorBlockSerializer(allow_null=True, required=False) stats = StatsSerializer() streams = StreamItemSerializer(many=True) - subtitles = SubtitleItemSerializer(many=True) + subtitles = SubtitleItemSerializer(many=True, required=False) tags = serializers.ListField(child=serializers.CharField()) title = serializers.CharField() vid_last_refresh = serializers.CharField() @@ -142,22 +143,6 @@ class VideoListQuerySerializer(serializers.Serializer): height = serializers.IntegerField(required=False) -class CommentThreadItemSerializer(serializers.Serializer): - """serialize comment thread item""" - - comment_id = serializers.CharField() - comment_text = serializers.CharField() - comment_timestamp = serializers.IntegerField() - comment_time_text = serializers.CharField() - comment_likecount = serializers.IntegerField() - comment_is_favorited = serializers.BooleanField() - comment_author = serializers.CharField() - comment_author_id = serializers.CharField() - comment_author_thumbnail = serializers.URLField() - comment_author_is_uploader = serializers.BooleanField() - comment_parent = serializers.CharField() - - class CommentItemSerializer(serializers.Serializer): """serialize comment item""" @@ -172,7 +157,14 @@ class CommentItemSerializer(serializers.Serializer): comment_author_thumbnail = serializers.URLField() comment_author_is_uploader = serializers.BooleanField() comment_parent = serializers.CharField() - comment_replies = CommentThreadItemSerializer(many=True, required=False) + comment_replies = serializers.SerializerMethodField() + + @extend_schema_field(serializers.ListField()) + def get_comment_replies(self, obj): + """recursive replies""" + return CommentItemSerializer( + obj.get("comment_replies", []), many=True + ).data class CommentsSerializer(serializers.Serializer): diff --git a/backend/video/src/meta_embed.py b/backend/video/src/meta_embed.py index a568d04f..21f9b3f7 100644 --- a/backend/video/src/meta_embed.py +++ b/backend/video/src/meta_embed.py @@ -15,6 +15,8 @@ from common.src.env_settings import EnvironmentSettings from common.src.es_connect import ElasticWrap, IndexPaginate from download.src.thumbnails import ThumbManager from mutagen.mp4 import MP4, MP4FreeForm +from playlist.serializers import PlaylistSerializer +from playlist.src.index import YoutubePlaylist from video.serializers import ( CommentsSerializer, SubtitleFragmentSerializer, @@ -104,6 +106,8 @@ class IndexFromEmbed: self.archive_video(video) self.index_subtitles(json_embed, video) self.index_comments(json_embed) + self.restore_artwork(video) + self.index_playlists(json_embed, video) return video.json_data @@ -207,6 +211,64 @@ class IndexFromEmbed: if self.HOST_UID and self.HOST_GID: os.chown(new_path, self.HOST_UID, self.HOST_GID) + def index_playlists(self, json_embed, video): + """index playlists""" + playlist_data = json_embed.get("playlists") + if not playlist_data or not isinstance(playlist_data, list): + return + + serializer = PlaylistSerializer(data=playlist_data, many=True) + is_valid = serializer.is_valid() + if not is_valid: + err = serializer.errors + raise ValueError( + f"[{self.file_path}] playlist serializer failed: {err}" + ) + + expected = video.json_data.get("playlist", []) + video_mutagen = MP4(self.file_path) + + for playlist_data in serializer.data: + playlist_id = playlist_data["playlist_id"] + if playlist_id not in expected: + continue + + json_data = self._process_embedded_playlist(playlist_data) + if not json_data: + continue + + playlist_art = os.path.join( + self.CACHE_DIR, "playlists", f"{playlist_id}.jpg" + ) + self._restore_art_item( + video_mutagen, + "----:com.tubearchivist:playlist_{plalyist_id}", + playlist_art, + ) + + def _process_embedded_playlist(self, playlist_data) -> dict | None: + """process single embedded playlist""" + if not self.use_user_conf: + if playlist_data["playlist_type"] == "custom": + # custom playlist is user conf + return None + + playlist = YoutubePlaylist(youtube_id=playlist_data["playlist_id"]) + playlist.get_from_es() + if playlist.json_data: + # already indexed + return None + + playlist_data_clean = dict(playlist_data) + if not self.use_user_conf: + playlist_data_clean["playlist_subscribed"] = False + playlist_data_clean["playlist_sort_order"] = "top" + + playlist.json_data = playlist_data_clean + playlist.upload_to_es() + + return playlist.json_data + def restore_artwork(self, video): """restore artwork if needed""" video_mutagen = MP4(self.file_path) @@ -249,6 +311,10 @@ class IndexFromEmbed: # is not embedded return + art_folder = os.path.dirname(target_path) + if not os.path.exists(art_folder): + os.mkdir(art_folder) + with open(target_path, "wb") as f: f.write(bytes(art_item[0])) diff --git a/frontend/src/api/actions/createTaskSchedule.ts b/frontend/src/api/actions/createTaskSchedule.ts index 346ad3e2..0d2e1348 100644 --- a/frontend/src/api/actions/createTaskSchedule.ts +++ b/frontend/src/api/actions/createTaskSchedule.ts @@ -10,7 +10,6 @@ export type TaskScheduleNameType = | 'restore_backup' | 'rescan_filesystem' | 'thumbnail_check' - | 'resync_thumbs' | 'index_playlists' | 'subscribe_to' | 'version_check'; diff --git a/frontend/src/api/actions/updateTaskByName.ts b/frontend/src/api/actions/updateTaskByName.ts index d4bfd104..a55f25d9 100644 --- a/frontend/src/api/actions/updateTaskByName.ts +++ b/frontend/src/api/actions/updateTaskByName.ts @@ -1,6 +1,6 @@ import APIClient from '../../functions/APIClient'; -type TaskNamesType = 'download_pending' | 'update_subscribed' | 'resync_thumbs' | 'resync_metadata'; +type TaskNamesType = 'download_pending' | 'update_subscribed' | 'resync_metadata'; const updateTaskByName = async (taskName: TaskNamesType) => { return APIClient(`/api/task/by-name/${taskName}/`, { diff --git a/frontend/src/api/loader/loadPlaylistById.ts b/frontend/src/api/loader/loadPlaylistById.ts index 5a339baa..63a79184 100644 --- a/frontend/src/api/loader/loadPlaylistById.ts +++ b/frontend/src/api/loader/loadPlaylistById.ts @@ -12,7 +12,7 @@ export type PlaylistType = { playlist_active: boolean; playlist_channel: string; playlist_channel_id: string; - playlist_description: string; + playlist_description: string | null; playlist_entries: PlaylistEntryType[]; playlist_sort_order: 'top' | 'bottom'; playlist_id: string; diff --git a/frontend/src/components/CommentBox.tsx b/frontend/src/components/CommentBox.tsx index f7f71379..f2df2b9b 100644 --- a/frontend/src/components/CommentBox.tsx +++ b/frontend/src/components/CommentBox.tsx @@ -6,20 +6,6 @@ import Linkify from './Linkify'; import formatNumbers from '../functions/formatNumbers'; import Button from './Button'; -export type CommentReplyType = { - comment_id: string; - comment_text: string; - comment_timestamp: number; - comment_time_text: string; - comment_likecount: number; - comment_is_favorited: false; - comment_author: string; - comment_author_id: string; - comment_author_thumbnail: string; - comment_author_is_uploader: boolean; - comment_parent: string; -}; - export type CommentsType = { comment_id: string; comment_text: string; @@ -32,16 +18,17 @@ export type CommentsType = { comment_author_thumbnail: string; comment_author_is_uploader: boolean; comment_parent: string; - comment_replies?: CommentReplyType[]; + comment_replies: CommentsType[]; }; type CommentBoxProps = { comment: CommentsType; + showThread?: boolean; onTimestampClick?: (seconds: number) => void; }; -const CommentBox = ({ comment, onTimestampClick }: CommentBoxProps) => { - const [showSubComments, setShowSubComments] = useState(false); +const CommentBox = ({ comment, showThread = false, onTimestampClick }: CommentBoxProps) => { + const [showSubComments, setShowSubComments] = useState(showThread); const hasSubComments = comment.comment_replies !== undefined && comment.comment_replies.length > 0; @@ -93,10 +80,14 @@ const CommentBox = ({ comment, onTimestampClick }: CommentBoxProps) => {
{showSubComments && - comment.comment_replies?.map(comment => { + comment.comment_replies.map(comment => { return ( - + ); })} diff --git a/frontend/src/pages/Playlist.tsx b/frontend/src/pages/Playlist.tsx index 45f2bdee..31cf00ef 100644 --- a/frontend/src/pages/Playlist.tsx +++ b/frontend/src/pages/Playlist.tsx @@ -330,7 +330,7 @@ const Playlist = () => {
- {playlist.playlist_description !== 'False' && ( + {playlist.playlist_description !== null && (

{ const [deleteIgnored, setDeleteIgnored] = useState(false); const [deletePending, setDeletePending] = useState(false); const [processingImports, setProcessingImports] = useState(false); - const [reEmbed, setReEmbed] = useState(false); const [reSyncMeta, setReSyncMeta] = useState(false); const [backupStarted, setBackupStarted] = useState(false); const [isRestoringBackup, setIsRestoringBackup] = useState(false); @@ -51,7 +50,6 @@ const SettingsActions = () => { deleteIgnored || deletePending || processingImports || - reEmbed || reSyncMeta || backupStarted || isRestoringBackup || @@ -61,7 +59,6 @@ const SettingsActions = () => { setDeleteIgnored(false); setDeletePending(false); setProcessingImports(false); - setReEmbed(false); setReSyncMeta(false); setBackupStarted(false); setIsRestoringBackup(false); @@ -118,22 +115,6 @@ const SettingsActions = () => { )}

-
-

Embed thumbnails into media file.

-

Set extracted youtube thumbnail as cover art of the media file.

-
- {reEmbed &&

Processing thumbnails

} - {!reEmbed && ( -
-

Embed metadata into media file

Embed metadata into media files as mp4 tags.

diff --git a/frontend/src/pages/SettingsApplication.tsx b/frontend/src/pages/SettingsApplication.tsx index 5cfacf6a..755e0eaa 100644 --- a/frontend/src/pages/SettingsApplication.tsx +++ b/frontend/src/pages/SettingsApplication.tsx @@ -633,17 +633,17 @@ const SettingsApplication = () => { Download and index comments. Browsable on the video detail page. Example: