From 2ec9f9e78b644a21b506be0346b6a95b14d76528 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 4 Jan 2026 14:21:56 +0700 Subject: [PATCH] serializer improvements for video, mapping definitions --- backend/appsettings/index_mapping.json | 268 +++++++++--------- backend/common/src/search_processor.py | 1 + .../config/management/commands/ta_startup.py | 31 +- backend/video/serializers.py | 17 +- backend/video/src/index.py | 31 +- backend/video/src/media_streams.py | 26 +- backend/video/src/subtitle.py | 11 +- 7 files changed, 215 insertions(+), 170 deletions(-) diff --git a/backend/appsettings/index_mapping.json b/backend/appsettings/index_mapping.json index c5aa5ee5..f989eab1 100644 --- a/backend/appsettings/index_mapping.json +++ b/backend/appsettings/index_mapping.json @@ -110,17 +110,17 @@ { "index_name": "video", "expected_map": { - "vid_thumb_url": { - "type": "text", - "index": false + "active": { + "type": "boolean" }, - "vid_thumb_base64": { + "category": { "type": "text", - "index": false - }, - "date_downloaded": { - "type": "date", - "format": "epoch_second" + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } }, "channel": { "properties": { @@ -211,84 +211,22 @@ } } }, + "comment_count": { + "type": "long" + }, + "date_downloaded": { + "type": "date", + "format": "epoch_second" + }, "description": { "type": "text" }, - "media_url": { - "type": "keyword", - "index": false - }, "media_size": { "type": "long" }, - "tags": { - "type": "text", - "analyzer": "english", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "title": { - "type": "text", - "analyzer": "english", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256, - "normalizer": "to_lower" - }, - "search_as_you_type": { - "type": "search_as_you_type", - "doc_values": false, - "max_shingle_size": 3 - } - } - }, - "vid_last_refresh": { - "type": "date", - "format": "epoch_second" - }, - "youtube_id": { - "type": "keyword" - }, - "vid_type": { - "type": "keyword" - }, - "published": { - "type": "date", - "format": "epoch_second||strict_date_optional_time" - }, - "playlist": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256, - "normalizer": "to_lower" - } - } - }, - "comment_count": { - "type": "long" - }, - "stats": { - "properties": { - "average_rating": { - "type": "float" - }, - "dislike_count": { - "type": "long" - }, - "like_count": { - "type": "long" - }, - "view_count": { - "type": "long" - } - } + "media_url": { + "type": "keyword", + "index": false }, "player": { "properties": { @@ -308,68 +246,32 @@ } } }, - "subtitles": { - "properties": { - "ext": { + "playlist": { + "type": "text", + "fields": { + "keyword": { "type": "keyword", - "index": false - }, - "lang": { - "type": "keyword", - "index": false - }, - "media_url": { - "type": "keyword", - "index": false - }, - "name": { - "type": "keyword" - }, - "source": { - "type": "keyword" - }, - "url": { - "type": "keyword", - "index": false + "ignore_above": 256, + "normalizer": "to_lower" } } }, - "streams": { - "properties": { - "type": { - "type": "keyword", - "index": false - }, - "index": { - "type": "short", - "index": false - }, - "codec": { - "type": "text" - }, - "width": { - "type": "short" - }, - "height": { - "type": "short" - }, - "bitrate": { - "type": "integer" - } - } + "published": { + "type": "date", + "format": "epoch_second||strict_date_optional_time" }, "sponsorblock": { "properties": { - "last_refresh": { - "type": "date", - "format": "epoch_second" - }, "has_unlocked": { "type": "boolean" }, "is_enabled": { "type": "boolean" }, + "last_refresh": { + "type": "date", + "format": "epoch_second" + }, "segments": { "properties": { "UUID": { @@ -405,6 +307,112 @@ } } } + }, + "stats": { + "properties": { + "average_rating": { + "type": "float" + }, + "dislike_count": { + "type": "long" + }, + "like_count": { + "type": "long" + }, + "view_count": { + "type": "long" + } + } + }, + "streams": { + "properties": { + "bitrate": { + "type": "integer" + }, + "codec": { + "type": "text" + }, + "height": { + "type": "short" + }, + "index": { + "type": "short", + "index": false + }, + "type": { + "type": "keyword", + "index": false + }, + "width": { + "type": "short" + } + } + }, + "subtitles": { + "properties": { + "ext": { + "type": "keyword", + "index": false + }, + "lang": { + "type": "keyword", + "index": false + }, + "media_url": { + "type": "keyword", + "index": false + }, + "name": { + "type": "keyword" + }, + "source": { + "type": "keyword" + }, + "url": { + "type": "keyword", + "index": false + } + } + }, + "tags": { + "type": "text", + "analyzer": "english", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "title": { + "type": "text", + "analyzer": "english", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "to_lower" + }, + "search_as_you_type": { + "type": "search_as_you_type", + "doc_values": false, + "max_shingle_size": 3 + } + } + }, + "vid_last_refresh": { + "type": "date", + "format": "epoch_second" + }, + "vid_thumb_url": { + "type": "text", + "index": false + }, + "vid_type": { + "type": "keyword" + }, + "youtube_id": { + "type": "keyword" } }, "expected_set": { diff --git a/backend/common/src/search_processor.py b/backend/common/src/search_processor.py index 83e311c5..02f71957 100644 --- a/backend/common/src/search_processor.py +++ b/backend/common/src/search_processor.py @@ -138,6 +138,7 @@ class SearchProcess: "vid_last_refresh": vid_last_refresh, "published": published, "vid_thumb_url": f"{cache_root}/{vid_thumb_url}", + "description": video_dict.get("description"), } ) diff --git a/backend/config/management/commands/ta_startup.py b/backend/config/management/commands/ta_startup.py index 2638eb8b..f10be9b7 100644 --- a/backend/config/management/commands/ta_startup.py +++ b/backend/config/management/commands/ta_startup.py @@ -14,7 +14,7 @@ from appsettings.src.index_setup import ElasticIndexWrap from appsettings.src.snapshot import ElasticSnapshot from channel.src.index import YoutubeChannel from common.src.env_settings import EnvironmentSettings -from common.src.es_connect import ElasticWrap +from common.src.es_connect import ElasticWrap, IndexPaginate from common.src.helper import clear_dl_cache, get_channels from common.src.ta_redis import RedisArchivist from django.core.management.base import BaseCommand, CommandError @@ -25,6 +25,7 @@ from task.src.config_schedule import ScheduleBuilder from task.src.task_manager import TaskManager from task.tasks import version_check from video.src.constants import VideoTypeEnum +from video.src.index import YoutubeVideo TOPIC = """ @@ -59,6 +60,7 @@ class Command(BaseCommand): self._mig_fix_missing_stats() self._mig_fix_channel_art_types() self._mig_fix_channel_description() + self._mig_fix_video_description() def _make_folders(self): """make expected cache folders""" @@ -409,6 +411,33 @@ class Command(BaseCommand): noop_msg = " no items needed updating" self.stdout.write(self.style.SUCCESS(noop_msg)) + def _mig_fix_video_description(self) -> None: + """migrate from 0.5.8 to 0.5.9, fix video desc null value""" + desc = "fix video description null value" + self.stdout.write(f"[MIGRATION] run {desc}") + + data = {"_source": ["youtube_id", "description"]} + videos = IndexPaginate("ta_video", data=data).get_results() + + counter = 0 + for video_response in videos: + if not video_response.get("description") == "": + continue + + video = YoutubeVideo(youtube_id=video_response["youtube_id"]) + video.get_from_es() + video.json_data.pop("description") + video.upload_to_es() + + counter += 1 + + if counter: + suc_msg = f" ✓ updated {counter} videos" + self.stdout.write(self.style.SUCCESS(suc_msg)) + else: + noop_msg = " no items needed updating" + self.stdout.write(self.style.SUCCESS(noop_msg)) + def _run_migration( self, index_name: str, desc: str, query: dict, script: dict ): diff --git a/backend/video/serializers.py b/backend/video/serializers.py index 99639829..410a8ea8 100644 --- a/backend/video/serializers.py +++ b/backend/video/serializers.py @@ -16,6 +16,7 @@ class PlayerSerializer(serializers.Serializer): watched_date = serializers.IntegerField(required=False) duration = serializers.IntegerField() duration_str = serializers.CharField() + progress = serializers.FloatField(required=False) position = serializers.FloatField(required=False) @@ -53,12 +54,12 @@ class StatsSerializer(serializers.Serializer): class StreamItemSerializer(serializers.Serializer): """serialize stream item""" - index = serializers.IntegerField() - codec = serializers.CharField() bitrate = serializers.IntegerField() + codec = serializers.CharField() + height = serializers.IntegerField(required=False) + index = serializers.IntegerField() type = serializers.ChoiceField(choices=["video", "audio"]) width = serializers.IntegerField(required=False) - height = serializers.IntegerField(required=False) class SubtitleFragmentSerializer(serializers.Serializer): @@ -81,12 +82,12 @@ class SubtitleFragmentSerializer(serializers.Serializer): class SubtitleItemSerializer(serializers.Serializer): """serialize subtitle item""" - ext = serializers.ChoiceField(choices=["json3"]) - name = serializers.CharField() - source = serializers.ChoiceField(choices=["user", "auto"]) + ext = serializers.ChoiceField(choices=["json3", "vtt"]) lang = serializers.CharField() media_url = serializers.CharField() - url = serializers.URLField() + name = serializers.CharField() + source = serializers.ChoiceField(choices=["user", "auto"]) + url = serializers.URLField(allow_null=True) class VideoSerializer(serializers.Serializer): @@ -97,7 +98,7 @@ class VideoSerializer(serializers.Serializer): channel = ChannelSerializer(required=False) comment_count = serializers.IntegerField(allow_null=True, required=False) date_downloaded = serializers.IntegerField() - description = serializers.CharField() + description = serializers.CharField(allow_null=True) media_size = serializers.IntegerField() media_url = serializers.CharField() player = PlayerSerializer() diff --git a/backend/video/src/index.py b/backend/video/src/index.py index 437c1e57..69f7ac5c 100644 --- a/backend/video/src/index.py +++ b/backend/video/src/index.py @@ -198,26 +198,25 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): def process_youtube_meta(self): """extract relevant fields from youtube""" self._validate_id() - # extract self.channel_id = self.youtube_meta["channel_id"] last_refresh = int(datetime.now().timestamp()) - # build json_data basics self.json_data = { - "title": self.youtube_meta["title"], - "description": self.youtube_meta.get("description", ""), - "category": self.youtube_meta.get("categories", []), - "vid_thumb_url": self.youtube_meta["thumbnail"], - "tags": self.youtube_meta.get("tags", []), - "published": self._build_published(), - "vid_last_refresh": last_refresh, - "date_downloaded": last_refresh, - "youtube_id": self.youtube_id, - # Using .value to make json encodable - "vid_type": self.video_type.value, "active": True, + "category": self.youtube_meta.get("categories", []), + "date_downloaded": last_refresh, + "published": self._build_published(), + "tags": self.youtube_meta.get("tags", []), + "title": self.youtube_meta["title"], + "vid_last_refresh": last_refresh, + "vid_thumb_url": self.youtube_meta["thumbnail"], + "vid_type": self.video_type.value, + "youtube_id": self.youtube_id, } - def _build_published(self): + if description := self.youtube_meta.get("description"): + self.json_data["description"] = description + + def _build_published(self) -> int | str: """build published date or timestamp""" timestamp = self.youtube_meta.get("timestamp") if timestamp and isinstance(timestamp, int): @@ -288,9 +287,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): self.json_data.update( { "player": { - "watched": False, "duration": duration, "duration_str": get_duration_str(duration), + "watched": False, } } ) @@ -412,7 +411,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): subtitle_media_url = f"{base_name}.{lang}.vtt" to_add = { "ext": "vtt", - "url": False, + "url": None, "name": lang, "lang": lang, "source": "file", diff --git a/backend/video/src/media_streams.py b/backend/video/src/media_streams.py index 0d8c1823..93e1f983 100644 --- a/backend/video/src/media_streams.py +++ b/backend/video/src/media_streams.py @@ -12,7 +12,7 @@ class MediaStreamExtractor: self.media_path = media_path self.metadata = [] - def extract_metadata(self): + def extract_metadata(self) -> list[dict]: """entry point to extract metadata""" cmd = [ @@ -38,17 +38,15 @@ class MediaStreamExtractor: return self.metadata - def process_stream(self, stream): + def process_stream(self, stream) -> None: """parse stream to metadata""" codec_type = stream.get("codec_type") if codec_type == "video": self._extract_video_metadata(stream) elif codec_type == "audio": self._extract_audio_metadata(stream) - else: - return - def _extract_video_metadata(self, stream): + def _extract_video_metadata(self, stream) -> None: """parse video metadata""" if "bit_rate" not in stream: # is probably thumbnail @@ -56,26 +54,26 @@ class MediaStreamExtractor: self.metadata.append( { - "type": "video", - "index": stream["index"], + "bitrate": int(stream.get("bit_rate", 0)), "codec": stream["codec_name"], - "width": stream["width"], "height": stream["height"], - "bitrate": int(stream["bit_rate"]), + "index": stream["index"], + "type": "video", + "width": stream["width"], } ) - def _extract_audio_metadata(self, stream): + def _extract_audio_metadata(self, stream) -> None: """extract audio metadata""" self.metadata.append( { - "type": "audio", - "index": stream["index"], - "codec": stream.get("codec_name", "undefined"), "bitrate": int(stream.get("bit_rate", 0)), + "codec": stream.get("codec_name", "undefined"), + "index": stream["index"], + "type": "audio", } ) - def get_file_size(self): + def get_file_size(self) -> int: """get filesize in bytes""" return stat(self.media_path).st_size diff --git a/backend/video/src/subtitle.py b/backend/video/src/subtitle.py index 68868eb9..a3c51566 100644 --- a/backend/video/src/subtitle.py +++ b/backend/video/src/subtitle.py @@ -149,7 +149,16 @@ class YoutubeSubtitle: query_str = parser.create_bulk_import(documents) self.index_subtitle(query_str) - indexed.append(subtitle) + indexed.append( + { + "ext": "json3", + "name": subtitle["name"], + "source": subtitle["source"], + "lang": subtitle["lang"], + "media_url": subtitle["media_url"], + "url": subtitle["url"], + } + ) rand_sleep(self.video.config) return indexed