add channel_tabs indexing

backend/appsettings/index_mapping.json

@@ -62,6 +62,9 @@
                         }
                     }
                 },
+                "channel_tabs": {
+                    "type": "keyword"
+                },
                 "channel_overwrites": {
                     "properties": {
                         "download_format": {
@@ -165,6 +168,9 @@
                                 }
                             }
                         },
+                        "channel_tabs": {
+                            "type": "keyword"
+                        },
                         "channel_overwrites": {
                             "properties": {
                                 "download_format": {

backend/channel/src/index.py

@@ -7,12 +7,14 @@ functionality:
 import os
 from datetime import datetime
 
+from channel.src.remote_query import get_last_channel_videos
 from common.src.env_settings import EnvironmentSettings
 from common.src.es_connect import ElasticWrap, IndexPaginate
 from common.src.helper import rand_sleep
 from common.src.index_generic import YouTubeItem
 from download.src.thumbnails import ThumbManager
 from download.src.yt_dlp_base import YtWrap
+from video.src.constants import VideoTypeEnum
 
 
 class YoutubeChannel(YouTubeItem):
@@ -68,6 +70,7 @@ class YoutubeChannel(YouTubeItem):
             "channel_thumb_url": self._get_thumb_art(),
             "channel_tvart_url": self._get_tv_art(),
             "channel_views": self.youtube_meta.get("view_count") or 0,
+            "channel_tabs": self.get_channel_tabs(),
         }
 
     def _get_thumb_art(self):
@@ -103,6 +106,31 @@ class YoutubeChannel(YouTubeItem):
 
         return False
 
+    def get_channel_tabs(self) -> list[str]:
+        """get channel tabs"""
+        tabs = VideoTypeEnum.values_known()
+        config_cp = self.config.copy()
+        config_cp["subscriptions"] = {
+            "channel_size": 1,
+            "live_channel_size": 1,
+            "shorts_channel_size": 1,
+        }
+        tabs = []
+        for query_filter in VideoTypeEnum:
+            if query_filter == VideoTypeEnum.UNKNOWN:
+                continue
+
+            videos = get_last_channel_videos(
+                channel_id=self.youtube_id,
+                config=config_cp,
+                limit=True,
+                query_filter=query_filter,
+            )
+            if videos:
+                tabs.append(query_filter.value)
+
+        return tabs
+
     def _video_fallback(self, fallback):
         """use video metadata as fallback"""
         print(f"{self.youtube_id}: fallback to video metadata")

backend/channel/src/remote_query.py

@@ -12,7 +12,9 @@ class VideoQueryBuilder:
         self.channel_overwrites = channel_overwrites or {}
 
     def build_queries(
-        self, video_type: VideoTypeEnum | None, limit: bool = True
+        self,
+        video_type: VideoTypeEnum | list[VideoTypeEnum] | None,
+        limit: bool = True,
     ) -> list[tuple[VideoTypeEnum, int | None]]:
         """Build queries for all or specific video type."""
         query_methods = {
@@ -22,13 +24,21 @@ class VideoQueryBuilder:
         }
 
         if video_type:
-            # build query for specific type
-            query_method = query_methods.get(video_type)
-            if query_method:
+            # build query for specific type/s
+            if not isinstance(video_type, list):
+                video_type = [video_type]
+
+            queries = []
+            for video_type_item in video_type:
+                query_method = query_methods.get(video_type_item)
+                if not query_method:
+                    continue
+
                 query = query_method(limit)
                 if query[1] != 0:
-                    return [query]
-                return []
+                    queries.append(query)
+
+            return queries
 
         # Build and return queries for all video types
         queries = []

backend/config/management/commands/ta_index_channel_tags.py

@@ -0,0 +1,36 @@
+"""
+migration for 0.5.4 to 0.5.5
+index channel_tags for subscribed channels
+"""
+
+import time
+
+from channel.src.index import YoutubeChannel
+from common.src.helper import get_channels
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    """command"""
+
+    def handle(self, *args, **kwargs):
+        """handle task"""
+
+        self.stdout.write("channel tags initial index")
+
+        channels = get_channels(subscribed_only=True, source=["channel_id"])
+
+        for es_channel in channels:
+            channel = YoutubeChannel(es_channel["channel_id"])
+            channel.get_from_es()
+            channel_name = channel.json_data["channel_name"]
+            channel_tabs = channel.get_channel_tabs()
+            channel.json_data["channel_tabs"] = channel_tabs
+            channel.upload_to_es()
+            channel.sync_to_videos()
+            self.stdout.write(
+                self.style.SUCCESS(
+                    f"    ✓ updated '{channel_name}' tabs: {channel_tabs}"
+                )
+            )
+            time.sleep(5)

backend/config/management/commands/ta_startup.py

@@ -23,6 +23,7 @@ from task.models import CustomPeriodicTask
 from task.src.config_schedule import ScheduleBuilder
 from task.src.task_manager import TaskManager
 from task.tasks import version_check
+from video.src.constants import VideoTypeEnum
 
 TOPIC = """
 
@@ -53,6 +54,7 @@ class Command(BaseCommand):
         self._init_app_config()
         self._mig_fix_download_channel_indexed()
         self._mig_add_default_playlist_sort()
+        self._mig_set_channel_tabs()
 
     def _make_folders(self):
         """make expected cache folders"""
@@ -323,3 +325,37 @@ class Command(BaseCommand):
         self.stdout.write(response)
         sleep(60)
         raise CommandError(message)
+
+    def _mig_set_channel_tabs(self) -> None:
+        """migrate from 0.5.4 to 0.5.5 set initial channel tabs"""
+        self.stdout.write("[MIGRATION] set default channel_tabs")
+
+        path = "ta_channel/_update_by_query"
+        tabs = VideoTypeEnum.values_known()
+        data = {
+            "query": {
+                "bool": {"must_not": [{"exists": {"field": "channel_tabs"}}]}
+            },
+            "script": {
+                "source": f"ctx._source.channel_tabs = {tabs}",
+                "lang": "painless",
+            },
+        }
+        response, status_code = ElasticWrap(path).post(data)
+        if status_code in [200, 201]:
+            updated = response.get("updated")
+            if updated:
+                self.stdout.write(
+                    self.style.SUCCESS(f"    ✓ updated {updated} channels")
+                )
+            else:
+                self.stdout.write(
+                    self.style.SUCCESS("    no channels need updating")
+                )
+            return
+
+        message = "    🗙 failed to set default channel_tabs"
+        self.stdout.write(self.style.ERROR(message))
+        self.stdout.write(response)
+        sleep(60)
+        raise CommandError(message)

backend/download/src/subscriptions.py

@@ -28,7 +28,8 @@ class ChannelSubscription:
             self.task.send_progress(["Looking up channels."])
 
         all_channels = get_channels(
-            subscribed_only=True, source=["channel_id", "channel_overwrites"]
+            subscribed_only=True,
+            source=["channel_id", "channel_overwrites", "channel_tabs"],
         )
         if not all_channels:
             return 0
@@ -36,10 +37,15 @@ class ChannelSubscription:
         all_channel_urls: list[ParsedURLType] = []
 
         for channel in all_channels:
+            channel_tabs = channel["channel_tabs"]
+            if not channel_tabs:
+                continue
+
+            enum = [getattr(VideoTypeEnum, i.upper()) for i in channel_tabs]
             queries = VideoQueryBuilder(
                 config=self.config,
                 channel_overwrites=channel.get("channel_overwrites", {}),
-            ).build_queries(video_type=None)
+            ).build_queries(video_type=enum)
 
             for query in queries:
                 all_channel_urls.append(