diff --git a/backend/appsettings/index_mapping.json b/backend/appsettings/index_mapping.json index 72845492..eaeaf325 100644 --- a/backend/appsettings/index_mapping.json +++ b/backend/appsettings/index_mapping.json @@ -62,6 +62,9 @@ } } }, + "channel_tabs": { + "type": "keyword" + }, "channel_overwrites": { "properties": { "download_format": { @@ -107,10 +110,6 @@ "type": "text", "index": false }, - "vid_thumb_base64": { - "type": "text", - "index": false - }, "date_downloaded": { "type": "date", "format": "epoch_second" @@ -165,6 +164,9 @@ } } }, + "channel_tabs": { + "type": "keyword" + }, "channel_overwrites": { "properties": { "download_format": { diff --git a/backend/channel/serializers.py b/backend/channel/serializers.py index 7b2a9ec8..81464957 100644 --- a/backend/channel/serializers.py +++ b/backend/channel/serializers.py @@ -4,6 +4,7 @@ from common.serializers import PaginationSerializer, ValidateUnknownFieldsMixin from rest_framework import serializers +from video.src.constants import VideoTypeEnum class ChannelOverwriteSerializer( @@ -45,6 +46,9 @@ class ChannelSerializer(serializers.Serializer): channel_tags = serializers.ListField( child=serializers.CharField(), required=False ) + channel_tabs = serializers.ListField( + child=serializers.ChoiceField(VideoTypeEnum.values_known()) + ) channel_views = serializers.IntegerField() _index = serializers.CharField(required=False) _score = serializers.IntegerField(required=False) diff --git a/backend/channel/src/index.py b/backend/channel/src/index.py index 17bf9526..ab14b820 100644 --- a/backend/channel/src/index.py +++ b/backend/channel/src/index.py @@ -7,12 +7,14 @@ functionality: import os from datetime import datetime +from channel.src.remote_query import get_last_channel_videos from common.src.env_settings import EnvironmentSettings from common.src.es_connect import ElasticWrap, IndexPaginate from common.src.helper import rand_sleep from common.src.index_generic import YouTubeItem from download.src.thumbnails import ThumbManager from download.src.yt_dlp_base import YtWrap +from video.src.constants import VideoTypeEnum class YoutubeChannel(YouTubeItem): @@ -68,6 +70,7 @@ class YoutubeChannel(YouTubeItem): "channel_thumb_url": self._get_thumb_art(), "channel_tvart_url": self._get_tv_art(), "channel_views": self.youtube_meta.get("view_count") or 0, + "channel_tabs": self.get_channel_tabs(), } def _get_thumb_art(self): @@ -103,6 +106,31 @@ class YoutubeChannel(YouTubeItem): return False + def get_channel_tabs(self) -> list[str]: + """get channel tabs""" + tabs = VideoTypeEnum.values_known() + config_cp = self.config.copy() + config_cp["subscriptions"] = { + "channel_size": 1, + "live_channel_size": 1, + "shorts_channel_size": 1, + } + tabs = [] + for query_filter in VideoTypeEnum: + if query_filter == VideoTypeEnum.UNKNOWN: + continue + + videos = get_last_channel_videos( + channel_id=self.youtube_id, + config=config_cp, + limit=True, + query_filter=query_filter, + ) + if videos: + tabs.append(query_filter.value) + + return tabs + def _video_fallback(self, fallback): """use video metadata as fallback""" print(f"{self.youtube_id}: fallback to video metadata") diff --git a/backend/channel/src/remote_query.py b/backend/channel/src/remote_query.py index a38d1cce..17168467 100644 --- a/backend/channel/src/remote_query.py +++ b/backend/channel/src/remote_query.py @@ -12,7 +12,9 @@ class VideoQueryBuilder: self.channel_overwrites = channel_overwrites or {} def build_queries( - self, video_type: VideoTypeEnum | None, limit: bool = True + self, + video_type: VideoTypeEnum | list[VideoTypeEnum] | None, + limit: bool = True, ) -> list[tuple[VideoTypeEnum, int | None]]: """Build queries for all or specific video type.""" query_methods = { @@ -22,13 +24,21 @@ class VideoQueryBuilder: } if video_type: - # build query for specific type - query_method = query_methods.get(video_type) - if query_method: + # build query for specific type/s + if not isinstance(video_type, list): + video_type = [video_type] + + queries = [] + for video_type_item in video_type: + query_method = query_methods.get(video_type_item) + if not query_method: + continue + query = query_method(limit) if query[1] != 0: - return [query] - return [] + queries.append(query) + + return queries # Build and return queries for all video types queries = [] diff --git a/backend/common/src/search_processor.py b/backend/common/src/search_processor.py index e5c0cac4..6c1d410b 100644 --- a/backend/common/src/search_processor.py +++ b/backend/common/src/search_processor.py @@ -160,6 +160,7 @@ class SearchProcess: "playlist_last_refresh": playlist_last_refresh, } ) + print(playlist_dict) return dict(sorted(playlist_dict.items())) diff --git a/backend/config/management/commands/ta_config_backup.py b/backend/config/management/commands/ta_config_backup.py deleted file mode 100644 index 40802ddd..00000000 --- a/backend/config/management/commands/ta_config_backup.py +++ /dev/null @@ -1,76 +0,0 @@ -"""backup config for sqlite reset and restore""" - -import json -from pathlib import Path - -from django.contrib.auth import get_user_model -from django.core.management.base import BaseCommand -from home.models import CustomPeriodicTask -from home.src.ta.settings import EnvironmentSettings -from rest_framework.authtoken.models import Token - -User = get_user_model() - - -class Command(BaseCommand): - """export""" - - help = "Exports all users and their auth tokens to a JSON file" - FILE = Path(EnvironmentSettings.CACHE_DIR) / "backup" / "migration.json" - - def handle(self, *args, **kwargs): - """entry point""" - - data = { - "user_data": self.get_users(), - "schedule_data": self.get_schedules(), - } - - with open(self.FILE, "w", encoding="utf-8") as json_file: - json_file.write(json.dumps(data)) - - def get_users(self): - """get users""" - - users = User.objects.all() - - user_data = [] - - for user in users: - user_info = { - "username": user.name, - "is_staff": user.is_staff, - "is_superuser": user.is_superuser, - "password": user.password, - "tokens": [], - } - - try: - token = Token.objects.get(user=user) - user_info["tokens"] = [token.key] - except Token.DoesNotExist: - user_info["tokens"] = [] - - user_data.append(user_info) - - return user_data - - def get_schedules(self): - """get schedules""" - - all_schedules = CustomPeriodicTask.objects.all() - schedule_data = [] - - for schedule in all_schedules: - schedule_info = { - "name": schedule.name, - "crontab": { - "minute": schedule.crontab.minute, - "hour": schedule.crontab.hour, - "day_of_week": schedule.crontab.day_of_week, - }, - } - - schedule_data.append(schedule_info) - - return schedule_data diff --git a/backend/config/management/commands/ta_config_restore.py b/backend/config/management/commands/ta_config_restore.py deleted file mode 100644 index 0918627a..00000000 --- a/backend/config/management/commands/ta_config_restore.py +++ /dev/null @@ -1,89 +0,0 @@ -"""restore config from backup""" - -import json -from pathlib import Path - -from common.src.env_settings import EnvironmentSettings -from django.core.management.base import BaseCommand -from django_celery_beat.models import CrontabSchedule -from rest_framework.authtoken.models import Token -from task.models import CustomPeriodicTask -from task.src.task_config import TASK_CONFIG -from user.models import Account - - -class Command(BaseCommand): - """export""" - - help = "Exports all users and their auth tokens to a JSON file" - FILE = Path(EnvironmentSettings.CACHE_DIR) / "backup" / "migration.json" - - def handle(self, *args, **options): - """handle""" - self.stdout.write("restore users and schedules") - data = self.get_config() - self.restore_users(data["user_data"]) - self.restore_schedules(data["schedule_data"]) - self.stdout.write( - self.style.SUCCESS( - " ✓ restore completed. Please restart the container." - ) - ) - - def get_config(self) -> dict: - """get config from backup""" - with open(self.FILE, "r", encoding="utf-8") as json_file: - data = json.loads(json_file.read()) - - self.stdout.write( - self.style.SUCCESS(f" ✓ json file found: {self.FILE}") - ) - - return data - - def restore_users(self, user_data: list[dict]) -> None: - """restore users from config""" - self.stdout.write("delete existing users") - Account.objects.all().delete() - - self.stdout.write("recreate users") - for user_info in user_data: - user = Account.objects.create( - name=user_info["username"], - is_staff=user_info["is_staff"], - is_superuser=user_info["is_superuser"], - password=user_info["password"], - ) - for token in user_info["tokens"]: - Token.objects.create(user=user, key=token) - - self.stdout.write( - self.style.SUCCESS( - f" ✓ recreated user with name: {user_info['username']}" - ) - ) - - def restore_schedules(self, schedule_data: list[dict]) -> None: - """restore schedules""" - self.stdout.write("delete existing schedules") - CustomPeriodicTask.objects.all().delete() - - self.stdout.write("recreate schedules") - for schedule in schedule_data: - task_name = schedule["name"] - description = TASK_CONFIG[task_name].get("title") - crontab, _ = CrontabSchedule.objects.get_or_create( - minute=schedule["crontab"]["minute"], - hour=schedule["crontab"]["hour"], - day_of_week=schedule["crontab"]["day_of_week"], - timezone=EnvironmentSettings.TZ, - ) - task = CustomPeriodicTask.objects.create( - name=task_name, - task=task_name, - description=description, - crontab=crontab, - ) - self.stdout.write( - self.style.SUCCESS(f" ✓ recreated schedule: {task}") - ) diff --git a/backend/config/management/commands/ta_index_channel_tags.py b/backend/config/management/commands/ta_index_channel_tags.py new file mode 100644 index 00000000..bb1c913f --- /dev/null +++ b/backend/config/management/commands/ta_index_channel_tags.py @@ -0,0 +1,36 @@ +""" +migration for 0.5.4 to 0.5.5 +index channel_tags for subscribed channels +""" + +import time + +from channel.src.index import YoutubeChannel +from common.src.helper import get_channels +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + """command""" + + def handle(self, *args, **kwargs): + """handle task""" + + self.stdout.write("channel tags initial index") + + channels = get_channels(subscribed_only=True, source=["channel_id"]) + + for es_channel in channels: + channel = YoutubeChannel(es_channel["channel_id"]) + channel.get_from_es() + channel_name = channel.json_data["channel_name"] + channel_tabs = channel.get_channel_tabs() + channel.json_data["channel_tabs"] = channel_tabs + channel.upload_to_es() + channel.sync_to_videos() + self.stdout.write( + self.style.SUCCESS( + f" ✓ updated '{channel_name}' tabs: {channel_tabs}" + ) + ) + time.sleep(5) diff --git a/backend/config/management/commands/ta_startup.py b/backend/config/management/commands/ta_startup.py index 4b75f05b..12004bfb 100644 --- a/backend/config/management/commands/ta_startup.py +++ b/backend/config/management/commands/ta_startup.py @@ -19,11 +19,11 @@ from common.src.ta_redis import RedisArchivist from django.core.management.base import BaseCommand, CommandError from django.utils import dateformat from django_celery_beat.models import CrontabSchedule, PeriodicTasks -from redis.exceptions import ResponseError from task.models import CustomPeriodicTask from task.src.config_schedule import ScheduleBuilder from task.src.task_manager import TaskManager from task.tasks import version_check +from video.src.constants import VideoTypeEnum TOPIC = """ @@ -49,14 +49,12 @@ class Command(BaseCommand): self._version_check() self._index_setup() self._snapshot_check() - self._mig_app_settings() self._create_default_schedules() self._update_schedule_tz() self._init_app_config() - self._mig_channel_tags() - self._mig_video_channel_tags() self._mig_fix_download_channel_indexed() self._mig_add_default_playlist_sort() + self._mig_set_channel_tabs() def _make_folders(self): """make expected cache folders""" @@ -159,39 +157,6 @@ class Command(BaseCommand): self.stdout.write("[7] setup snapshots") ElasticSnapshot().setup() - def _mig_app_settings(self) -> None: - """update from v0.4.13 to v0.5.0, migrate application settings""" - self.stdout.write("[MIGRATION] move appconfig to ES") - try: - config = RedisArchivist().get_message("config") - except ResponseError: - self.stdout.write( - self.style.SUCCESS(" Redis does not support JSON decoding") - ) - return - - if not config or config == {"status": False}: - self.stdout.write( - self.style.SUCCESS(" no config values to migrate") - ) - return - - path = "ta_config/_doc/appsettings" - response, status_code = ElasticWrap(path).post(config) - - if status_code in [200, 201]: - self.stdout.write( - self.style.SUCCESS(" ✓ migrated appconfig to ES") - ) - RedisArchivist().del_message("config", save=True) - return - - message = " 🗙 failed to migrate app config" - self.stdout.write(self.style.ERROR(message)) - self.stdout.write(response) - sleep(60) - raise CommandError(message) - def _create_default_schedules(self) -> None: """create default schedules for new installations""" self.stdout.write("[8] create initial schedules") @@ -293,70 +258,6 @@ class Command(BaseCommand): self.style.SUCCESS(f" Status code: {status_code}") ) - def _mig_channel_tags(self) -> None: - """update from v0.4.13 to v0.5.0, migrate incorrect data types""" - self.stdout.write("[MIGRATION] fix incorrect channel tags types") - path = "ta_channel/_update_by_query" - data = { - "query": {"match": {"channel_tags": False}}, - "script": { - "source": "ctx._source.channel_tags = []", - "lang": "painless", - }, - } - response, status_code = ElasticWrap(path).post(data) - if status_code in [200, 201]: - updated = response.get("updated") - if updated: - self.stdout.write( - self.style.SUCCESS(f" ✓ fixed {updated} channel tags") - ) - else: - self.stdout.write( - self.style.SUCCESS(" no channel tags needed fixing") - ) - return - - message = " 🗙 failed to fix channel tags" - self.stdout.write(self.style.ERROR(message)) - self.stdout.write(response) - sleep(60) - raise CommandError(message) - - def _mig_video_channel_tags(self) -> None: - """update from v0.4.13 to v0.5.0, migrate incorrect data types""" - self.stdout.write("[MIGRATION] fix incorrect video channel tags types") - path = "ta_video/_update_by_query" - data = { - "query": {"match": {"channel.channel_tags": False}}, - "script": { - "source": "ctx._source.channel.channel_tags = []", - "lang": "painless", - }, - } - response, status_code = ElasticWrap(path).post(data) - if status_code in [200, 201]: - updated = response.get("updated") - if updated: - self.stdout.write( - self.style.SUCCESS( - f" ✓ fixed {updated} video channel tags" - ) - ) - else: - self.stdout.write( - self.style.SUCCESS( - " no video channel tags needed fixing" - ) - ) - return - - message = " 🗙 failed to fix video channel tags" - self.stdout.write(self.style.ERROR(message)) - self.stdout.write(response) - sleep(60) - raise CommandError(message) - def _mig_fix_download_channel_indexed(self) -> None: """migrate from v0.5.2 to 0.5.3, fix missing channel_indexed""" self.stdout.write("[MIGRATION] fix incorrect video channel tags types") @@ -424,3 +325,37 @@ class Command(BaseCommand): self.stdout.write(response) sleep(60) raise CommandError(message) + + def _mig_set_channel_tabs(self) -> None: + """migrate from 0.5.4 to 0.5.5 set initial channel tabs""" + self.stdout.write("[MIGRATION] set default channel_tabs") + + path = "ta_channel/_update_by_query" + tabs = VideoTypeEnum.values_known() + data = { + "query": { + "bool": {"must_not": [{"exists": {"field": "channel_tabs"}}]} + }, + "script": { + "source": f"ctx._source.channel_tabs = {tabs}", + "lang": "painless", + }, + } + response, status_code = ElasticWrap(path).post(data) + if status_code in [200, 201]: + updated = response.get("updated") + if updated: + self.stdout.write( + self.style.SUCCESS(f" ✓ updated {updated} channels") + ) + else: + self.stdout.write( + self.style.SUCCESS(" no channels need updating") + ) + return + + message = " 🗙 failed to set default channel_tabs" + self.stdout.write(self.style.ERROR(message)) + self.stdout.write(response) + sleep(60) + raise CommandError(message) diff --git a/backend/download/src/queue.py b/backend/download/src/queue.py index 8bf59e83..25f6a8e9 100644 --- a/backend/download/src/queue.py +++ b/backend/download/src/queue.py @@ -165,7 +165,7 @@ class PendingList(PendingIndex): else: raise ValueError(f"invalid url_type: {entry}") - def _add_video(self, url, vid_type): + def _add_video(self, url, vid_type) -> dict | None: """add video to list""" if self.auto_start and url in set( i["youtube_id"] for i in self.all_pending @@ -175,10 +175,11 @@ class PendingList(PendingIndex): if url in self.missing_videos or url in self.to_skip: print(f"{url}: skipped adding already indexed video to download.") - else: - to_add = self._parse_video(url, vid_type) - if to_add: - self.missing_videos.append(to_add) + return None + + to_add = self._parse_video(url, vid_type) + if to_add: + self.missing_videos.append(to_add) return to_add diff --git a/backend/download/src/subscriptions.py b/backend/download/src/subscriptions.py index 5d84ba03..ea4e1fe3 100644 --- a/backend/download/src/subscriptions.py +++ b/backend/download/src/subscriptions.py @@ -28,7 +28,8 @@ class ChannelSubscription: self.task.send_progress(["Looking up channels."]) all_channels = get_channels( - subscribed_only=True, source=["channel_id", "channel_overwrites"] + subscribed_only=True, + source=["channel_id", "channel_overwrites", "channel_tabs"], ) if not all_channels: return 0 @@ -36,10 +37,15 @@ class ChannelSubscription: all_channel_urls: list[ParsedURLType] = [] for channel in all_channels: + channel_tabs = channel["channel_tabs"] + if not channel_tabs: + continue + + enum = [getattr(VideoTypeEnum, i.upper()) for i in channel_tabs] queries = VideoQueryBuilder( config=self.config, channel_overwrites=channel.get("channel_overwrites", {}), - ).build_queries(video_type=None) + ).build_queries(video_type=enum) for query in queries: all_channel_urls.append( diff --git a/backend/playlist/src/index.py b/backend/playlist/src/index.py index 165860dc..e676fbbb 100644 --- a/backend/playlist/src/index.py +++ b/backend/playlist/src/index.py @@ -325,6 +325,7 @@ class YoutubePlaylist(YouTubeItem): self.delete_metadata() def create(self, name): + """create custom playlist""" self.json_data = { "playlist_id": self.youtube_id, "playlist_active": False, @@ -337,6 +338,7 @@ class YoutubePlaylist(YouTubeItem): "playlist_description": False, "playlist_thumbnail": False, "playlist_subscribed": False, + "playlist_sort_order": "top", } self.upload_to_es() self.get_playlist_art() diff --git a/backend/playlist/src/query_building.py b/backend/playlist/src/query_building.py index 9b93f640..3f5593e5 100644 --- a/backend/playlist/src/query_building.py +++ b/backend/playlist/src/query_building.py @@ -45,7 +45,7 @@ class QueryBuilder: type_parsed = getattr(PlaylistTypesEnum, playlist_type.upper()).value - return {"match": {"playlist_type.keyword": type_parsed}} + return {"match": {"playlist_type": type_parsed}} def parse_sort(self) -> dict: """return sort""" diff --git a/backend/playlist/tests/test_src/test_query_building.py b/backend/playlist/tests/test_src/test_query_building.py index 66a8ec22..a3e16cd3 100644 --- a/backend/playlist/tests/test_src/test_query_building.py +++ b/backend/playlist/tests/test_src/test_query_building.py @@ -27,4 +27,4 @@ def test_parse_type(): qb.parse_type("invalid") result = qb.parse_type("custom") - assert result == {"match": {"playlist_type.keyword": "custom"}} + assert result == {"match": {"playlist_type": "custom"}} diff --git a/backend/playlist/views.py b/backend/playlist/views.py index bb756081..d7a4561d 100644 --- a/backend/playlist/views.py +++ b/backend/playlist/views.py @@ -239,6 +239,12 @@ class PlaylistApiView(ApiBaseView): error = ErrorResponseSerializer({"error": "playlist not found"}) return Response(error.data, status=404) + if self.response["playlist_type"] == "custom": + error = ErrorResponseSerializer( + {"error": f"playlist with ID {playlist_id} is custom"} + ) + return Response(error.data, status=400) + subscribed = validated_data.get("playlist_subscribed") sort_order = validated_data.get("playlist_sort_order") diff --git a/backend/video/src/index.py b/backend/video/src/index.py index 34e23a8c..5b1e8c38 100644 --- a/backend/video/src/index.py +++ b/backend/video/src/index.py @@ -174,15 +174,12 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): # extract self.channel_id = self.youtube_meta["channel_id"] last_refresh = int(datetime.now().timestamp()) - # base64_blur = ThumbManager().get_base64_blur(self.youtube_id) - base64_blur = False # build json_data basics self.json_data = { "title": self.youtube_meta["title"], "description": self.youtube_meta.get("description", ""), "category": self.youtube_meta.get("categories", []), "vid_thumb_url": self.youtube_meta["thumbnail"], - "vid_thumb_base64": base64_blur, "tags": self.youtube_meta.get("tags", []), "published": self._build_published(), "vid_last_refresh": last_refresh, diff --git a/frontend/src/pages/Home.tsx b/frontend/src/pages/Home.tsx index 291725a6..be016f6e 100644 --- a/frontend/src/pages/Home.tsx +++ b/frontend/src/pages/Home.tsx @@ -74,7 +74,6 @@ export type VideoType = { tags: string[]; title: string; vid_last_refresh: string; - vid_thumb_base64: boolean; vid_thumb_url: string; vid_type: string; youtube_id: string; diff --git a/frontend/src/pages/Playlist.tsx b/frontend/src/pages/Playlist.tsx index 6d1227d4..9e19a669 100644 --- a/frontend/src/pages/Playlist.tsx +++ b/frontend/src/pages/Playlist.tsx @@ -288,31 +288,33 @@ const Playlist = () => { /> )} -
- Switch sort order: -
- { - const newSortOrder = - playlist.playlist_sort_order === 'top' ? 'bottom' : 'top'; - await updatePlaylistSortOrder(playlist.playlist_id, newSortOrder); - setRefresh(true); - }} - /> - {playlist.playlist_sort_order === 'bottom' ? ( - - ) : ( - - )} + {playlist.playlist_type !== 'custom' && ( +
+ Switch sort order: +
+ { + const newSortOrder = + playlist.playlist_sort_order === 'top' ? 'bottom' : 'top'; + await updatePlaylistSortOrder(playlist.playlist_id, newSortOrder); + setRefresh(true); + }} + /> + {playlist.playlist_sort_order === 'bottom' ? ( + + ) : ( + + )} +
-
+ )}