From 6e346efcb3f12e584d0c3343db1caea81d4edae6 Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 3 Jan 2026 15:44:08 +0700 Subject: [PATCH] serialize and validate channel data --- backend/appsettings/index_mapping.json | 146 ++++++++++-------- backend/channel/serializers.py | 11 +- backend/channel/src/index.py | 88 +++++------ backend/common/src/search_processor.py | 19 ++- .../config/management/commands/ta_startup.py | 58 ++++++- backend/download/src/thumbnails.py | 6 +- frontend/src/pages/ChannelAbout.tsx | 5 - frontend/src/pages/Channels.tsx | 1 - 8 files changed, 203 insertions(+), 131 deletions(-) diff --git a/backend/appsettings/index_mapping.json b/backend/appsettings/index_mapping.json index 8014aef8..c5aa5ee5 100644 --- a/backend/appsettings/index_mapping.json +++ b/backend/appsettings/index_mapping.json @@ -12,6 +12,28 @@ "channel_id": { "type": "keyword" }, + "channel_active": { + "type": "boolean" + }, + "channel_banner_url": { + "type": "keyword", + "index": false + }, + "channel_thumb_url": { + "type": "keyword", + "index": false + }, + "channel_tvart_url": { + "type": "keyword", + "index": false + }, + "channel_description": { + "type": "text" + }, + "channel_last_refresh": { + "type": "date", + "format": "epoch_second" + }, "channel_name": { "type": "text", "analyzer": "english", @@ -28,38 +50,6 @@ } } }, - "channel_banner_url": { - "type": "keyword", - "index": false - }, - "channel_tvart_url": { - "type": "keyword", - "index": false - }, - "channel_thumb_url": { - "type": "keyword", - "index": false - }, - "channel_description": { - "type": "text" - }, - "channel_last_refresh": { - "type": "date", - "format": "epoch_second" - }, - "channel_tags": { - "type": "text", - "analyzer": "english", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "channel_tabs": { - "type": "keyword" - }, "channel_overwrites": { "properties": { "download_format": { @@ -84,6 +74,25 @@ "type": "long" } } + }, + "channel_subs": { + "type": "long" + }, + "channel_subscribed": { + "type": "boolean" + }, + "channel_tags": { + "type": "text", + "analyzer": "english", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "channel_tabs": { + "type": "keyword" } }, "expected_set": { @@ -118,6 +127,28 @@ "channel_id": { "type": "keyword" }, + "channel_active": { + "type": "boolean" + }, + "channel_banner_url": { + "type": "keyword", + "index": false + }, + "channel_thumb_url": { + "type": "keyword", + "index": false + }, + "channel_tvart_url": { + "type": "keyword", + "index": false + }, + "channel_description": { + "type": "text" + }, + "channel_last_refresh": { + "type": "date", + "format": "epoch_second" + }, "channel_name": { "type": "text", "analyzer": "english", @@ -134,38 +165,6 @@ } } }, - "channel_banner_url": { - "type": "keyword", - "index": false - }, - "channel_tvart_url": { - "type": "keyword", - "index": false - }, - "channel_thumb_url": { - "type": "keyword", - "index": false - }, - "channel_description": { - "type": "text" - }, - "channel_last_refresh": { - "type": "date", - "format": "epoch_second" - }, - "channel_tags": { - "type": "text", - "analyzer": "english", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "channel_tabs": { - "type": "keyword" - }, "channel_overwrites": { "properties": { "download_format": { @@ -190,6 +189,25 @@ "type": "long" } } + }, + "channel_subs": { + "type": "long" + }, + "channel_subscribed": { + "type": "boolean" + }, + "channel_tags": { + "type": "text", + "analyzer": "english", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "channel_tabs": { + "type": "keyword" } } }, diff --git a/backend/channel/serializers.py b/backend/channel/serializers.py index 1f364a97..d2dc4184 100644 --- a/backend/channel/serializers.py +++ b/backend/channel/serializers.py @@ -34,10 +34,12 @@ class ChannelSerializer(serializers.Serializer): channel_id = serializers.CharField() channel_active = serializers.BooleanField() - channel_banner_url = serializers.CharField() - channel_thumb_url = serializers.CharField() - channel_tvart_url = serializers.CharField() - channel_description = serializers.CharField() + channel_banner_url = serializers.CharField(allow_null=True, required=False) + channel_thumb_url = serializers.CharField(allow_null=True, required=False) + channel_tvart_url = serializers.CharField(allow_null=True, required=False) + channel_description = serializers.CharField( + allow_null=True, required=False + ) channel_last_refresh = serializers.CharField() channel_name = serializers.CharField() channel_overwrites = ChannelOverwriteSerializer(required=False) @@ -49,7 +51,6 @@ class ChannelSerializer(serializers.Serializer): channel_tabs = serializers.ListField( child=serializers.ChoiceField(VideoTypeEnum.values_known()) ) - channel_views = serializers.IntegerField() _index = serializers.CharField(required=False) _score = serializers.IntegerField(required=False) diff --git a/backend/channel/src/index.py b/backend/channel/src/index.py index 37f2e625..390f9023 100644 --- a/backend/channel/src/index.py +++ b/backend/channel/src/index.py @@ -57,54 +57,56 @@ class YoutubeChannel(YouTubeItem): """extract relevant fields""" self.youtube_meta["thumbnails"].reverse() channel_name = self.youtube_meta["uploader"] or self.youtube_meta["id"] + description = self.youtube_meta.get("description") or None self.json_data = { "channel_active": True, - "channel_description": self.youtube_meta.get("description", ""), + "channel_description": description, "channel_id": self.youtube_id, "channel_last_refresh": int(datetime.now().timestamp()), "channel_name": channel_name, - "channel_subs": self.youtube_meta.get("channel_follower_count", 0), + "channel_subs": self.youtube_meta.get("channel_follower_count") + or 0, "channel_subscribed": False, "channel_tags": self.youtube_meta.get("tags", []), - "channel_banner_url": self._get_banner_art(), - "channel_thumb_url": self._get_thumb_art(), - "channel_tvart_url": self._get_tv_art(), - "channel_views": self.youtube_meta.get("view_count") or 0, "channel_tabs": self.get_channel_tabs(), } - def _get_thumb_art(self): + self._get_thumb_art() + self._get_tv_art() + self._get_banner_art() + + def _get_thumb_art(self) -> None: """extract thumb art""" for i in self.youtube_meta["thumbnails"]: if not i.get("width"): continue if i.get("width") == i.get("height"): - return i["url"] + self.json_data["channel_thumb_url"] = i["url"] + return - return False - - def _get_tv_art(self): + def _get_tv_art(self) -> None: """extract tv artwork""" for i in self.youtube_meta["thumbnails"]: if i.get("id") == "banner_uncropped": - return i["url"] + self.json_data["channel_tvart_url"] = i["url"] + return for i in self.youtube_meta["thumbnails"]: if not i.get("width"): continue if i["width"] // i["height"] < 2 and not i["width"] == i["height"]: - return i["url"] + self.json_data["channel_tvart_url"] = i["url"] + return - return False + return - def _get_banner_art(self): + def _get_banner_art(self) -> None: """extract banner artwork""" for i in self.youtube_meta["thumbnails"]: if not i.get("width"): continue if i["width"] // i["height"] > 5: - return i["url"] - - return False + self.json_data["channel_banner_url"] = i["url"] + return def get_channel_tabs(self) -> list[str]: """get channel tabs""" @@ -132,51 +134,39 @@ class YoutubeChannel(YouTubeItem): self.json_data = { "channel_active": False, "channel_last_refresh": int(datetime.now().timestamp()), - "channel_subs": fallback.get("channel_follower_count", 0), + "channel_subs": fallback.get("channel_follower_count") or 0, "channel_name": fallback["uploader"], - "channel_banner_url": False, - "channel_tvart_url": False, "channel_id": self.youtube_id, "channel_subscribed": False, "channel_tags": [], - "channel_description": "", - "channel_thumb_url": False, - "channel_views": 0, } def get_channel_art(self): """download channel art for new channels""" urls = ( - self.json_data["channel_thumb_url"], - self.json_data["channel_banner_url"], - self.json_data["channel_tvart_url"], + self.json_data.get("channel_thumb_url"), + self.json_data.get("channel_banner_url"), + self.json_data.get("channel_tvart_url"), ) ThumbManager(self.youtube_id, item_type="channel").download(urls) def sync_to_videos(self): """sync new channel_dict to all videos of channel""" - # add ingest pipeline - processors = [] - for field, value in self.json_data.items(): - if value is None: - line = { - "script": { - "lang": "painless", - "source": f"ctx['{field}'] = null;", - } - } - else: - line = {"set": {"field": "channel." + field, "value": value}} - - processors.append(line) - - data = {"description": self.youtube_id, "processors": processors} - ingest_path = f"_ingest/pipeline/{self.youtube_id}" - _, _ = ElasticWrap(ingest_path).put(data) - # apply pipeline - data = {"query": {"match": {"channel.channel_id": self.youtube_id}}} - update_path = f"ta_video/_update_by_query?pipeline={self.youtube_id}" - _, _ = ElasticWrap(update_path).post(data) + data = { + "query": { + "term": {"channel.channel_id": {"value": self.youtube_id}}, + }, + "script": { + "lang": "painless", + "params": {"channel": self.json_data}, + "source": "ctx._source.channel = params.channel", + }, + } + update_path = "ta_video/_update_by_query" + response, status_code = ElasticWrap(update_path).post(data) + if status_code not in [200, 201]: + print(f"sync to videos failed with status code {status_code}") + print(response) def change_subscribe(self, new_subscribe_state: bool): """change subscribe status""" diff --git a/backend/common/src/search_processor.py b/backend/common/src/search_processor.py index e0ceaefc..83e311c5 100644 --- a/backend/common/src/search_processor.py +++ b/backend/common/src/search_processor.py @@ -89,12 +89,25 @@ class SearchProcess: channel_dict.update( { "channel_last_refresh": date_str, - "channel_banner_url": f"{art_base}_banner.jpg", - "channel_thumb_url": f"{art_base}_thumb.jpg", - "channel_tvart_url": f"{art_base}_tvart.jpg", + "channel_description": channel_dict.get("channel_description"), } ) + if channel_dict.get("channel_banner_url"): + channel_dict["channel_banner_url"] = f"{art_base}_banner.jpg" + else: + channel_dict["channel_banner_url"] = None + + if channel_dict.get("channel_thumb_url"): + channel_dict["channel_thumb_url"] = f"{art_base}_thumb.jpg" + else: + channel_dict["channel_thumb_url"] = None + + if channel_dict.get("channel_tvart_url"): + channel_dict["channel_tvart_url"] = f"{art_base}_tvart.jpg" + else: + channel_dict["channel_tvart_url"] = None + return dict(sorted(channel_dict.items())) def _process_video(self, video_dict): diff --git a/backend/config/management/commands/ta_startup.py b/backend/config/management/commands/ta_startup.py index 0caaac07..2638eb8b 100644 --- a/backend/config/management/commands/ta_startup.py +++ b/backend/config/management/commands/ta_startup.py @@ -12,9 +12,10 @@ from time import sleep from appsettings.src.config import AppConfig, ReleaseVersion from appsettings.src.index_setup import ElasticIndexWrap from appsettings.src.snapshot import ElasticSnapshot +from channel.src.index import YoutubeChannel from common.src.env_settings import EnvironmentSettings from common.src.es_connect import ElasticWrap -from common.src.helper import clear_dl_cache +from common.src.helper import clear_dl_cache, get_channels from common.src.ta_redis import RedisArchivist from django.core.management.base import BaseCommand, CommandError from django.utils import dateformat @@ -56,6 +57,8 @@ class Command(BaseCommand): self._mig_set_video_channel_tabs() self._mig_fix_playlist_description() self._mig_fix_missing_stats() + self._mig_fix_channel_art_types() + self._mig_fix_channel_description() def _make_folders(self): """make expected cache folders""" @@ -353,6 +356,59 @@ class Command(BaseCommand): }, ) + def _mig_fix_channel_art_types(self) -> None: + """migrate from 0.5.8 to 0.5.9, fix channel artwork types""" + fields = [ + "channel_banner_url", + "channel_thumb_url", + "channel_tvart_url", + ] + for field in fields: + self._run_migration( + index_name="ta_channel", + desc=f"fix missing data type for field {field}", + query={"term": {field: {"value": False}}}, + script={ + "source": f"ctx._source.remove('{field}')", + "lang": "painless", + }, + ) + self._run_migration( + index_name="ta_video", + desc=f"fix missing data type for field channel.{field}", + query={"term": {f"channel.{field}": {"value": False}}}, + script={ + "source": f"ctx._source.remove('channel.{field}')", + "lang": "painless", + }, + ) + + def _mig_fix_channel_description(self) -> None: + """migrate from 0.5.8 to 0.5.9, fix channel desc null value""" + desc = "fix channel description null value" + self.stdout.write(f"[MIGRATION] run {desc}") + channels = get_channels( + subscribed_only=False, source=["channel_description", "channel_id"] + ) + counter = 0 + for channel_response in channels: + if not channel_response.get("channel_description") == "": + continue + + channel = YoutubeChannel(youtube_id=channel_response["channel_id"]) + channel.get_from_es() + channel.json_data.pop("channel_description") + channel.upload_to_es() + channel.sync_to_videos() + counter += 1 + + if counter: + suc_msg = f" ✓ updated {counter} channels with videos" + self.stdout.write(self.style.SUCCESS(suc_msg)) + else: + noop_msg = " no items needed updating" + self.stdout.write(self.style.SUCCESS(noop_msg)) + def _run_migration( self, index_name: str, desc: str, query: dict, script: dict ): diff --git a/backend/download/src/thumbnails.py b/backend/download/src/thumbnails.py index 0b6a7c44..87e88e64 100644 --- a/backend/download/src/thumbnails.py +++ b/backend/download/src/thumbnails.py @@ -324,9 +324,9 @@ class ValidatorCallback: """check if all channel artwork is there""" for channel in self.source: urls = ( - channel["_source"]["channel_thumb_url"], - channel["_source"]["channel_banner_url"], - channel["_source"].get("channel_tvart_url", False), + channel["_source"].get("channel_thumb_url"), + channel["_source"].get("channel_banner_url"), + channel["_source"].get("channel_tvart_url"), ) handler = ThumbManager(channel["_source"]["channel_id"]) handler.download_channel_art(urls, skip_existing=True) diff --git a/frontend/src/pages/ChannelAbout.tsx b/frontend/src/pages/ChannelAbout.tsx index 54898b92..3e966182 100644 --- a/frontend/src/pages/ChannelAbout.tsx +++ b/frontend/src/pages/ChannelAbout.tsx @@ -8,7 +8,6 @@ import Routes from '../configuration/routes/RouteList'; import queueReindex, { ReindexType, ReindexTypeEnum } from '../api/actions/queueReindex'; import formatDate from '../functions/formatDates'; import PaginationDummy from '../components/PaginationDummy'; -import FormattedNumber from '../components/FormattedNumber'; import Button from '../components/Button'; import updateChannelOverwrites from '../api/actions/updateChannelOverwrite'; import useIsAdmin from '../functions/useIsAdmin'; @@ -145,10 +144,6 @@ const ChannelAbout = () => {
- {channel.channel_views > 0 && ( - - )} - {isAdmin && ( <>
diff --git a/frontend/src/pages/Channels.tsx b/frontend/src/pages/Channels.tsx index 2a6fb5f3..e5b7212d 100644 --- a/frontend/src/pages/Channels.tsx +++ b/frontend/src/pages/Channels.tsx @@ -41,7 +41,6 @@ export type ChannelType = { channel_tags?: string[]; channel_thumb_url: string; channel_tvart_url: string; - channel_views: number; }; const Channels = () => {