diff --git a/backend/appsettings/index_mapping.json b/backend/appsettings/index_mapping.json index 72845492..03d43deb 100644 --- a/backend/appsettings/index_mapping.json +++ b/backend/appsettings/index_mapping.json @@ -62,6 +62,9 @@ } } }, + "channel_tabs": { + "type": "keyword" + }, "channel_overwrites": { "properties": { "download_format": { @@ -165,6 +168,9 @@ } } }, + "channel_tabs": { + "type": "keyword" + }, "channel_overwrites": { "properties": { "download_format": { diff --git a/backend/channel/src/index.py b/backend/channel/src/index.py index 17bf9526..ab14b820 100644 --- a/backend/channel/src/index.py +++ b/backend/channel/src/index.py @@ -7,12 +7,14 @@ functionality: import os from datetime import datetime +from channel.src.remote_query import get_last_channel_videos from common.src.env_settings import EnvironmentSettings from common.src.es_connect import ElasticWrap, IndexPaginate from common.src.helper import rand_sleep from common.src.index_generic import YouTubeItem from download.src.thumbnails import ThumbManager from download.src.yt_dlp_base import YtWrap +from video.src.constants import VideoTypeEnum class YoutubeChannel(YouTubeItem): @@ -68,6 +70,7 @@ class YoutubeChannel(YouTubeItem): "channel_thumb_url": self._get_thumb_art(), "channel_tvart_url": self._get_tv_art(), "channel_views": self.youtube_meta.get("view_count") or 0, + "channel_tabs": self.get_channel_tabs(), } def _get_thumb_art(self): @@ -103,6 +106,31 @@ class YoutubeChannel(YouTubeItem): return False + def get_channel_tabs(self) -> list[str]: + """get channel tabs""" + tabs = VideoTypeEnum.values_known() + config_cp = self.config.copy() + config_cp["subscriptions"] = { + "channel_size": 1, + "live_channel_size": 1, + "shorts_channel_size": 1, + } + tabs = [] + for query_filter in VideoTypeEnum: + if query_filter == VideoTypeEnum.UNKNOWN: + continue + + videos = get_last_channel_videos( + channel_id=self.youtube_id, + config=config_cp, + limit=True, + query_filter=query_filter, + ) + if videos: + tabs.append(query_filter.value) + + return tabs + def _video_fallback(self, fallback): """use video metadata as fallback""" print(f"{self.youtube_id}: fallback to video metadata") diff --git a/backend/channel/src/remote_query.py b/backend/channel/src/remote_query.py index a38d1cce..17168467 100644 --- a/backend/channel/src/remote_query.py +++ b/backend/channel/src/remote_query.py @@ -12,7 +12,9 @@ class VideoQueryBuilder: self.channel_overwrites = channel_overwrites or {} def build_queries( - self, video_type: VideoTypeEnum | None, limit: bool = True + self, + video_type: VideoTypeEnum | list[VideoTypeEnum] | None, + limit: bool = True, ) -> list[tuple[VideoTypeEnum, int | None]]: """Build queries for all or specific video type.""" query_methods = { @@ -22,13 +24,21 @@ class VideoQueryBuilder: } if video_type: - # build query for specific type - query_method = query_methods.get(video_type) - if query_method: + # build query for specific type/s + if not isinstance(video_type, list): + video_type = [video_type] + + queries = [] + for video_type_item in video_type: + query_method = query_methods.get(video_type_item) + if not query_method: + continue + query = query_method(limit) if query[1] != 0: - return [query] - return [] + queries.append(query) + + return queries # Build and return queries for all video types queries = [] diff --git a/backend/config/management/commands/ta_index_channel_tags.py b/backend/config/management/commands/ta_index_channel_tags.py new file mode 100644 index 00000000..bb1c913f --- /dev/null +++ b/backend/config/management/commands/ta_index_channel_tags.py @@ -0,0 +1,36 @@ +""" +migration for 0.5.4 to 0.5.5 +index channel_tags for subscribed channels +""" + +import time + +from channel.src.index import YoutubeChannel +from common.src.helper import get_channels +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + """command""" + + def handle(self, *args, **kwargs): + """handle task""" + + self.stdout.write("channel tags initial index") + + channels = get_channels(subscribed_only=True, source=["channel_id"]) + + for es_channel in channels: + channel = YoutubeChannel(es_channel["channel_id"]) + channel.get_from_es() + channel_name = channel.json_data["channel_name"] + channel_tabs = channel.get_channel_tabs() + channel.json_data["channel_tabs"] = channel_tabs + channel.upload_to_es() + channel.sync_to_videos() + self.stdout.write( + self.style.SUCCESS( + f" ✓ updated '{channel_name}' tabs: {channel_tabs}" + ) + ) + time.sleep(5) diff --git a/backend/config/management/commands/ta_startup.py b/backend/config/management/commands/ta_startup.py index 40557f2d..12004bfb 100644 --- a/backend/config/management/commands/ta_startup.py +++ b/backend/config/management/commands/ta_startup.py @@ -23,6 +23,7 @@ from task.models import CustomPeriodicTask from task.src.config_schedule import ScheduleBuilder from task.src.task_manager import TaskManager from task.tasks import version_check +from video.src.constants import VideoTypeEnum TOPIC = """ @@ -53,6 +54,7 @@ class Command(BaseCommand): self._init_app_config() self._mig_fix_download_channel_indexed() self._mig_add_default_playlist_sort() + self._mig_set_channel_tabs() def _make_folders(self): """make expected cache folders""" @@ -323,3 +325,37 @@ class Command(BaseCommand): self.stdout.write(response) sleep(60) raise CommandError(message) + + def _mig_set_channel_tabs(self) -> None: + """migrate from 0.5.4 to 0.5.5 set initial channel tabs""" + self.stdout.write("[MIGRATION] set default channel_tabs") + + path = "ta_channel/_update_by_query" + tabs = VideoTypeEnum.values_known() + data = { + "query": { + "bool": {"must_not": [{"exists": {"field": "channel_tabs"}}]} + }, + "script": { + "source": f"ctx._source.channel_tabs = {tabs}", + "lang": "painless", + }, + } + response, status_code = ElasticWrap(path).post(data) + if status_code in [200, 201]: + updated = response.get("updated") + if updated: + self.stdout.write( + self.style.SUCCESS(f" ✓ updated {updated} channels") + ) + else: + self.stdout.write( + self.style.SUCCESS(" no channels need updating") + ) + return + + message = " 🗙 failed to set default channel_tabs" + self.stdout.write(self.style.ERROR(message)) + self.stdout.write(response) + sleep(60) + raise CommandError(message) diff --git a/backend/download/src/subscriptions.py b/backend/download/src/subscriptions.py index 5d84ba03..ea4e1fe3 100644 --- a/backend/download/src/subscriptions.py +++ b/backend/download/src/subscriptions.py @@ -28,7 +28,8 @@ class ChannelSubscription: self.task.send_progress(["Looking up channels."]) all_channels = get_channels( - subscribed_only=True, source=["channel_id", "channel_overwrites"] + subscribed_only=True, + source=["channel_id", "channel_overwrites", "channel_tabs"], ) if not all_channels: return 0 @@ -36,10 +37,15 @@ class ChannelSubscription: all_channel_urls: list[ParsedURLType] = [] for channel in all_channels: + channel_tabs = channel["channel_tabs"] + if not channel_tabs: + continue + + enum = [getattr(VideoTypeEnum, i.upper()) for i in channel_tabs] queries = VideoQueryBuilder( config=self.config, channel_overwrites=channel.get("channel_overwrites", {}), - ).build_queries(video_type=None) + ).build_queries(video_type=enum) for query in queries: all_channel_urls.append(