Embedding improvements, comment tree, #build

Changed:
- Improve embedding handling
- Improve restore from embedding
- Handle full artwork embed and restore
- Implement comment tree
This commit is contained in:
Simon 2025-12-31 13:02:21 +07:00
commit 7462a76729
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
20 changed files with 171 additions and 195 deletions

View File

@ -182,17 +182,23 @@ class SearchProcess:
def _process_comment(self, comment_dict):
"""run on all comments, create reply thread"""
all_comments = comment_dict["comment_comments"]
processed_comments = []
comment_tree = []
lookup = {}
for comment in comment_dict["comment_comments"]:
comment = comment.copy()
comment["comment_replies"] = []
lookup[comment["comment_id"]] = comment
for comment in all_comments:
if comment["comment_parent"] == "root":
comment.update({"comment_replies": []})
processed_comments.append(comment)
for comment in lookup.values():
parent = comment.get("comment_parent")
if parent == "root":
comment_tree.append(comment)
else:
processed_comments[-1]["comment_replies"].append(comment)
parent_node = lookup.get(parent)
if parent_node:
parent_node["comment_replies"].append(comment)
return processed_comments
return comment_tree
def _process_subtitle(self, result):
"""take complete result dict to extract highlight"""

View File

@ -1,36 +0,0 @@
"""
migration for 0.5.4 to 0.5.5
index channel_tabs for subscribed channels
"""
import time
from channel.src.index import YoutubeChannel
from common.src.helper import get_channels
from django.core.management.base import BaseCommand
class Command(BaseCommand):
"""command"""
def handle(self, *args, **kwargs):
"""handle task"""
self.stdout.write("channel tags initial index")
channels = get_channels(subscribed_only=True, source=["channel_id"])
for es_channel in channels:
channel = YoutubeChannel(es_channel["channel_id"])
channel.get_from_es()
channel_name = channel.json_data["channel_name"]
channel_tabs = channel.get_channel_tabs()
channel.json_data["channel_tabs"] = channel_tabs
channel.upload_to_es()
channel.sync_to_videos()
self.stdout.write(
self.style.SUCCESS(
f" ✓ updated '{channel_name}' tabs: {channel_tabs}"
)
)
time.sleep(5)

View File

@ -37,8 +37,6 @@ TOPIC = """
class Command(BaseCommand):
"""command framework"""
# pylint: disable=no-member
def handle(self, *args, **options):
"""run all commands"""
self.stdout.write(TOPIC)
@ -53,10 +51,10 @@ class Command(BaseCommand):
self._update_schedule_tz()
self._init_app_config()
self._set_ta_startup_time()
self._mig_fix_download_channel_indexed()
self._mig_add_default_playlist_sort()
self._mig_set_channel_tabs()
self._mig_set_video_channel_tabs()
self._mig_fix_playlist_description()
def _make_folders(self):
"""make expected cache folders"""
@ -271,40 +269,6 @@ class Command(BaseCommand):
self.style.SUCCESS(f" ✓ set timestamp to {message}.")
)
def _mig_fix_download_channel_indexed(self) -> None:
"""migrate from v0.5.2 to 0.5.3, fix missing channel_indexed"""
self.stdout.write("[MIGRATION] fix incorrect video channel tags types")
path = "ta_download/_update_by_query"
data = {
"query": {
"bool": {
"must_not": [{"exists": {"field": "channel_indexed"}}]
}
},
"script": {
"source": "ctx._source.channel_indexed = false",
"lang": "painless",
},
}
response, status_code = ElasticWrap(path).post(data)
if status_code in [200, 201]:
updated = response.get("updated")
if updated:
self.stdout.write(
self.style.SUCCESS(f" ✓ fixed {updated} queued videos")
)
else:
self.stdout.write(
self.style.SUCCESS(" no queued videos to fix")
)
return
message = " 🗙 failed to fix video channel tags"
self.stdout.write(self.style.ERROR(message))
self.stdout.write(response)
sleep(60)
raise CommandError(message)
def _mig_add_default_playlist_sort(self) -> None:
"""migrate from 0.5.4 to 0.5.5 set default playlist sortorder"""
self.stdout.write("[MIGRATION] set default playlist sort order")
@ -408,3 +372,34 @@ class Command(BaseCommand):
self.stdout.write(response)
sleep(60)
raise CommandError(message)
def _mig_fix_playlist_description(self) -> None:
"""migrate from 0.5.8 to 0.5.9 fix playlist desc null data type"""
self.stdout.write("[MIGRATION] fix playlist description data type")
path = "ta_playlist/_update_by_query"
data = {
"query": {"term": {"playlist_description": {"value": False}}},
"script": {
"source": "ctx._source.playlist_description = null",
"lang": "painless",
},
}
response, status_code = ElasticWrap(path).post(data)
if status_code in [200, 201]:
updated = response.get("updated")
if updated:
self.stdout.write(
self.style.SUCCESS(f" ✓ updated {updated} playlists")
)
else:
self.stdout.write(
self.style.SUCCESS(" no playlists need updating")
)
return
message = " 🗙 failed to fix playlist description null data type"
self.stdout.write(self.style.ERROR(message))
self.stdout.write(response)
sleep(60)
raise CommandError(message)

View File

@ -234,38 +234,41 @@ class ThumbManager(ThumbManagerBase):
banner_path = os.path.join(
self.CHANNEL_DIR, f"{channel_id}_banner.jpg"
)
if os.path.exists(banner_path):
with open(banner_path, "rb") as f:
banner_data = f.read()
video["----:com.tubearchivist:channel_banner"] = [
MP4Cover(banner_data, imageformat=MP4Cover.FORMAT_JPEG)
]
self._embed_art_item(video, "channel_banner", art_path=banner_path)
channel_icon_path = os.path.join(
self.CHANNEL_DIR, f"{channel_id}_thumb.jpg"
)
if os.path.exists(channel_icon_path):
with open(channel_icon_path, "rb") as f:
icon_data = f.read()
video["----:com.tubearchivist:channel_icon"] = [
MP4Cover(icon_data, imageformat=MP4Cover.FORMAT_JPEG)
]
self._embed_art_item(video, "channel_icon", art_path=channel_icon_path)
channel_tv_path = os.path.join(
self.CHANNEL_DIR, f"{channel_id}_tvart.jpg"
)
if os.path.exists(channel_tv_path):
with open(channel_tv_path, "rb") as f:
tv_data = f.read()
self._embed_art_item(video, "channel_tv", art_path=channel_tv_path)
video["----:com.tubearchivist:channel_tv"] = [
MP4Cover(tv_data, imageformat=MP4Cover.FORMAT_JPEG)
]
playlist_ids = json_data.get("playlist", [])
for plalyist_id in playlist_ids:
playlist_path = os.path.join(
self.PLAYLIST_DIR, f"{plalyist_id}.jpg"
)
self._embed_art_item(
video, f"playlist_{plalyist_id}", art_path=playlist_path
)
video.save()
def _embed_art_item(self, video, key, art_path):
"""embed single item"""
if not os.path.exists(art_path):
return
with open(art_path, "rb") as f:
art_data = f.read()
video[f"----:com.tubearchivist:{key}"] = [
MP4Cover(art_data, imageformat=MP4Cover.FORMAT_JPEG)
]
def delete_video_thumb(self):
"""delete video thumbnail if exists"""
thumb_path = self.vid_thumb_path()
@ -501,7 +504,7 @@ class ThumbFilesystem:
paginate = IndexPaginate(
index_name=self.INDEX_NAME,
data=data,
size=200,
size=100,
callback=EmbedCallback,
task=self.task,
total=self._get_total(),

View File

@ -429,7 +429,7 @@ class DownloadPostProcess(DownloaderBase):
channel = YoutubeChannel(channel_id)
channel.get_from_es()
overwrites = channel.get_overwrites()
if "index_playlists" in overwrites:
if overwrites.get("index_playlists"):
channel.get_all_playlists()
to_add = [i[0] for i in channel.all_playlists]
RedisQueue(self.PLAYLIST_QUEUE).add_list(to_add)

View File

@ -11,7 +11,7 @@ class PlaylistEntrySerializer(serializers.Serializer):
youtube_id = serializers.CharField()
title = serializers.CharField()
uploader = serializers.CharField()
uploader = serializers.CharField(allow_null=True)
idx = serializers.IntegerField()
downloaded = serializers.BooleanField()
@ -22,7 +22,7 @@ class PlaylistSerializer(serializers.Serializer):
playlist_active = serializers.BooleanField()
playlist_channel = serializers.CharField()
playlist_channel_id = serializers.CharField()
playlist_description = serializers.CharField()
playlist_description = serializers.CharField(allow_null=True)
playlist_entries = PlaylistEntrySerializer(many=True)
playlist_id = serializers.CharField()
playlist_last_refresh = serializers.CharField()

View File

@ -81,7 +81,7 @@ class YoutubePlaylist(YouTubeItem):
"playlist_channel": self.youtube_meta["channel"],
"playlist_channel_id": self.youtube_meta["channel_id"],
"playlist_thumbnail": playlist_thumbnail,
"playlist_description": self.youtube_meta["description"] or False,
"playlist_description": self.youtube_meta["description"] or None,
"playlist_last_refresh": int(datetime.now().timestamp()),
"playlist_type": "regular",
}

View File

@ -11,4 +11,4 @@ redis==7.1.0
requests==2.32.5
ryd-client==0.0.6
uvicorn==0.38.0
yt-dlp[default]==2025.12.8
yt-dlp[default] @ git+https://github.com/yt-dlp/yt-dlp@468aa6a9b431194949ede9eaad8f33e314a288d6

View File

@ -80,13 +80,6 @@ THUMBNAIL_CHECK: TaskItemConfig = {
"api_stop": False,
}
RESYNC_THUMBS: TaskItemConfig = {
"title": "Sync Thumbnails to Media Files",
"group": "setting:thumbnailsync",
"api_start": True,
"api_stop": False,
}
RESYNC_METADATA: TaskItemConfig = {
"title": "Sync Metadata to Media Files",
"group": "setting:thumbnailsync",
@ -125,7 +118,6 @@ TASK_CONFIG: dict[str, TaskItemConfig] = {
"restore_backup": RESTORE_BACKUP,
"rescan_filesystem": RESCAN_FILESYSTEM,
"thumbnail_check": THUMBNAIL_CHECK,
"resync_thumbs": RESYNC_THUMBS,
"resync_metadata": RESYNC_METADATA,
"index_playlists": INDEX_PLAYLISTS,
"subscribe_to": SUBSCRIBE_TO,

View File

@ -19,7 +19,7 @@ from common.src.ta_redis import RedisArchivist
from common.src.urlparser import ParsedURLType, Parser
from download.src.queue import PendingList
from download.src.subscriptions import SubscriptionHandler, SubscriptionScanner
from download.src.thumbnails import ThumbFilesystem, ThumbValidator
from download.src.thumbnails import ThumbValidator
from download.src.yt_dlp_handler import VideoDownloader
from task.src.notify import Notifications
from task.src.task_config import TASK_CONFIG
@ -294,19 +294,6 @@ def thumbnail_check(self):
thumbnail.clean_up()
@shared_task(bind=True, name="resync_thumbs", base=BaseTask)
def re_sync_thumbs(self):
"""sync thumbnails to mediafiles"""
manager = TaskManager()
if manager.is_pending(self):
print(f"[task][{self.name}] thumb re-embed is already running")
self.send_progress(["Thumbnail re-embed is already running."])
return
manager.init(self)
ThumbFilesystem(task=self).embed()
@shared_task(bind=True, name="resync_metadata", base=BaseTask)
def re_sync_metadata(self):
"""resync metadata to media files"""

View File

@ -40,7 +40,7 @@ class UserConfig:
_DEFAULT_USER_SETTINGS = UserConfigType(
stylesheet="dark.css",
page_size=12,
page_size=25,
sort_by="published",
sort_order="desc",
view_style_home="grid",

View File

@ -4,6 +4,7 @@
from channel.serializers import ChannelSerializer
from common.serializers import PaginationSerializer
from drf_spectacular.utils import extend_schema_field
from rest_framework import serializers
from video.src.constants import OrderEnum, SortEnum, VideoTypeEnum, WatchedEnum
@ -107,7 +108,7 @@ class VideoSerializer(serializers.Serializer):
sponsorblock = SponsorBlockSerializer(allow_null=True, required=False)
stats = StatsSerializer()
streams = StreamItemSerializer(many=True)
subtitles = SubtitleItemSerializer(many=True)
subtitles = SubtitleItemSerializer(many=True, required=False)
tags = serializers.ListField(child=serializers.CharField())
title = serializers.CharField()
vid_last_refresh = serializers.CharField()
@ -142,22 +143,6 @@ class VideoListQuerySerializer(serializers.Serializer):
height = serializers.IntegerField(required=False)
class CommentThreadItemSerializer(serializers.Serializer):
"""serialize comment thread item"""
comment_id = serializers.CharField()
comment_text = serializers.CharField()
comment_timestamp = serializers.IntegerField()
comment_time_text = serializers.CharField()
comment_likecount = serializers.IntegerField()
comment_is_favorited = serializers.BooleanField()
comment_author = serializers.CharField()
comment_author_id = serializers.CharField()
comment_author_thumbnail = serializers.URLField()
comment_author_is_uploader = serializers.BooleanField()
comment_parent = serializers.CharField()
class CommentItemSerializer(serializers.Serializer):
"""serialize comment item"""
@ -172,7 +157,14 @@ class CommentItemSerializer(serializers.Serializer):
comment_author_thumbnail = serializers.URLField()
comment_author_is_uploader = serializers.BooleanField()
comment_parent = serializers.CharField()
comment_replies = CommentThreadItemSerializer(many=True, required=False)
comment_replies = serializers.SerializerMethodField()
@extend_schema_field(serializers.ListField())
def get_comment_replies(self, obj):
"""recursive replies"""
return CommentItemSerializer(
obj.get("comment_replies", []), many=True
).data
class CommentsSerializer(serializers.Serializer):

View File

@ -15,6 +15,8 @@ from common.src.env_settings import EnvironmentSettings
from common.src.es_connect import ElasticWrap, IndexPaginate
from download.src.thumbnails import ThumbManager
from mutagen.mp4 import MP4, MP4FreeForm
from playlist.serializers import PlaylistSerializer
from playlist.src.index import YoutubePlaylist
from video.serializers import (
CommentsSerializer,
SubtitleFragmentSerializer,
@ -104,6 +106,8 @@ class IndexFromEmbed:
self.archive_video(video)
self.index_subtitles(json_embed, video)
self.index_comments(json_embed)
self.restore_artwork(video)
self.index_playlists(json_embed, video)
return video.json_data
@ -207,6 +211,64 @@ class IndexFromEmbed:
if self.HOST_UID and self.HOST_GID:
os.chown(new_path, self.HOST_UID, self.HOST_GID)
def index_playlists(self, json_embed, video):
"""index playlists"""
playlist_data = json_embed.get("playlists")
if not playlist_data or not isinstance(playlist_data, list):
return
serializer = PlaylistSerializer(data=playlist_data, many=True)
is_valid = serializer.is_valid()
if not is_valid:
err = serializer.errors
raise ValueError(
f"[{self.file_path}] playlist serializer failed: {err}"
)
expected = video.json_data.get("playlist", [])
video_mutagen = MP4(self.file_path)
for playlist_data in serializer.data:
playlist_id = playlist_data["playlist_id"]
if playlist_id not in expected:
continue
json_data = self._process_embedded_playlist(playlist_data)
if not json_data:
continue
playlist_art = os.path.join(
self.CACHE_DIR, "playlists", f"{playlist_id}.jpg"
)
self._restore_art_item(
video_mutagen,
"----:com.tubearchivist:playlist_{plalyist_id}",
playlist_art,
)
def _process_embedded_playlist(self, playlist_data) -> dict | None:
"""process single embedded playlist"""
if not self.use_user_conf:
if playlist_data["playlist_type"] == "custom":
# custom playlist is user conf
return None
playlist = YoutubePlaylist(youtube_id=playlist_data["playlist_id"])
playlist.get_from_es()
if playlist.json_data:
# already indexed
return None
playlist_data_clean = dict(playlist_data)
if not self.use_user_conf:
playlist_data_clean["playlist_subscribed"] = False
playlist_data_clean["playlist_sort_order"] = "top"
playlist.json_data = playlist_data_clean
playlist.upload_to_es()
return playlist.json_data
def restore_artwork(self, video):
"""restore artwork if needed"""
video_mutagen = MP4(self.file_path)
@ -249,6 +311,10 @@ class IndexFromEmbed:
# is not embedded
return
art_folder = os.path.dirname(target_path)
if not os.path.exists(art_folder):
os.mkdir(art_folder)
with open(target_path, "wb") as f:
f.write(bytes(art_item[0]))

View File

@ -10,7 +10,6 @@ export type TaskScheduleNameType =
| 'restore_backup'
| 'rescan_filesystem'
| 'thumbnail_check'
| 'resync_thumbs'
| 'index_playlists'
| 'subscribe_to'
| 'version_check';

View File

@ -1,6 +1,6 @@
import APIClient from '../../functions/APIClient';
type TaskNamesType = 'download_pending' | 'update_subscribed' | 'resync_thumbs' | 'resync_metadata';
type TaskNamesType = 'download_pending' | 'update_subscribed' | 'resync_metadata';
const updateTaskByName = async (taskName: TaskNamesType) => {
return APIClient(`/api/task/by-name/${taskName}/`, {

View File

@ -12,7 +12,7 @@ export type PlaylistType = {
playlist_active: boolean;
playlist_channel: string;
playlist_channel_id: string;
playlist_description: string;
playlist_description: string | null;
playlist_entries: PlaylistEntryType[];
playlist_sort_order: 'top' | 'bottom';
playlist_id: string;

View File

@ -6,20 +6,6 @@ import Linkify from './Linkify';
import formatNumbers from '../functions/formatNumbers';
import Button from './Button';
export type CommentReplyType = {
comment_id: string;
comment_text: string;
comment_timestamp: number;
comment_time_text: string;
comment_likecount: number;
comment_is_favorited: false;
comment_author: string;
comment_author_id: string;
comment_author_thumbnail: string;
comment_author_is_uploader: boolean;
comment_parent: string;
};
export type CommentsType = {
comment_id: string;
comment_text: string;
@ -32,16 +18,17 @@ export type CommentsType = {
comment_author_thumbnail: string;
comment_author_is_uploader: boolean;
comment_parent: string;
comment_replies?: CommentReplyType[];
comment_replies: CommentsType[];
};
type CommentBoxProps = {
comment: CommentsType;
showThread?: boolean;
onTimestampClick?: (seconds: number) => void;
};
const CommentBox = ({ comment, onTimestampClick }: CommentBoxProps) => {
const [showSubComments, setShowSubComments] = useState(false);
const CommentBox = ({ comment, showThread = false, onTimestampClick }: CommentBoxProps) => {
const [showSubComments, setShowSubComments] = useState(showThread);
const hasSubComments =
comment.comment_replies !== undefined && comment.comment_replies.length > 0;
@ -93,10 +80,14 @@ const CommentBox = ({ comment, onTimestampClick }: CommentBoxProps) => {
<div className="comments-replies" style={{ display: 'block' }}>
{showSubComments &&
comment.comment_replies?.map(comment => {
comment.comment_replies.map(comment => {
return (
<Fragment key={comment.comment_id}>
<CommentBox comment={comment} onTimestampClick={onTimestampClick} />
<CommentBox
comment={comment}
onTimestampClick={onTimestampClick}
showThread={showSubComments}
/>
</Fragment>
);
})}

View File

@ -330,7 +330,7 @@ const Playlist = () => {
</div>
</div>
{playlist.playlist_description !== 'False' && (
{playlist.playlist_description !== null && (
<div className="description-box">
<p
id={descriptionExpanded ? 'text-expand-expanded' : 'text-expand'}

View File

@ -15,7 +15,6 @@ const SettingsActions = () => {
const [deleteIgnored, setDeleteIgnored] = useState(false);
const [deletePending, setDeletePending] = useState(false);
const [processingImports, setProcessingImports] = useState(false);
const [reEmbed, setReEmbed] = useState(false);
const [reSyncMeta, setReSyncMeta] = useState(false);
const [backupStarted, setBackupStarted] = useState(false);
const [isRestoringBackup, setIsRestoringBackup] = useState(false);
@ -51,7 +50,6 @@ const SettingsActions = () => {
deleteIgnored ||
deletePending ||
processingImports ||
reEmbed ||
reSyncMeta ||
backupStarted ||
isRestoringBackup ||
@ -61,7 +59,6 @@ const SettingsActions = () => {
setDeleteIgnored(false);
setDeletePending(false);
setProcessingImports(false);
setReEmbed(false);
setReSyncMeta(false);
setBackupStarted(false);
setIsRestoringBackup(false);
@ -118,22 +115,6 @@ const SettingsActions = () => {
)}
</div>
</div>
<div className="settings-group">
<h2>Embed thumbnails into media file.</h2>
<p>Set extracted youtube thumbnail as cover art of the media file.</p>
<div id="re-embed">
{reEmbed && <p>Processing thumbnails</p>}
{!reEmbed && (
<Button
label="Start process"
onClick={async () => {
await updateTaskByName('resync_thumbs');
setReEmbed(true);
}}
/>
)}
</div>
</div>
<div className="settings-group">
<h2>Embed metadata into media file</h2>
<p>Embed metadata into media files as mp4 tags.</p>

View File

@ -633,17 +633,17 @@ const SettingsApplication = () => {
Download and index comments. Browsable on the video detail page. Example:
<ul>
<li>
<span className="settings-current">all,100,all,30</span>: Get 100
max-parents and 30 max-replies-per-thread.
<span className="settings-current">all,100,all,30,all</span>: Get 100
max-parents and 30 max-replies-per-thread at any depth.
</li>
<li>
<span className="settings-current">1000,all,all,50</span>: Get a total of
1000 comments over all, 50 replies per thread.
<span className="settings-current">1000,all,all,50,2</span>: Get a total
of 1000 comments over all, 50 replies per thread, only 2 levels of depth.
</li>
<li>
Values are in the format:{' '}
<span className="settings-current">
max-comments,max-parents,max-replies,max-replies-per-thread
max-comments,max-parents,max-replies,max-replies-per-thread,max-depth
</span>
.
</li>