diff --git a/backend/appsettings/serializers.py b/backend/appsettings/serializers.py index 22f3fb44..8130c19d 100644 --- a/backend/appsettings/serializers.py +++ b/backend/appsettings/serializers.py @@ -21,10 +21,14 @@ class AppConfigSubSerializer( ): """serialize app config subscriptions""" - channel_size = serializers.IntegerField(required=False) - live_channel_size = serializers.IntegerField(required=False) - shorts_channel_size = serializers.IntegerField(required=False) - playlist_size = serializers.IntegerField(required=False) + channel_size = serializers.IntegerField(required=False, allow_null=True) + live_channel_size = serializers.IntegerField( + required=False, allow_null=True + ) + shorts_channel_size = serializers.IntegerField( + required=False, allow_null=True + ) + playlist_size = serializers.IntegerField(required=False, allow_null=True) auto_start = serializers.BooleanField(required=False) extract_flat = serializers.BooleanField(required=False) diff --git a/backend/channel/src/remote_query.py b/backend/channel/src/remote_query.py index 17168467..90cbde22 100644 --- a/backend/channel/src/remote_query.py +++ b/backend/channel/src/remote_query.py @@ -84,7 +84,9 @@ class VideoQueryBuilder: limit: bool, ) -> tuple[VideoTypeEnum, int | None]: """Generic query for video page scraping.""" - if not limit: + app_config_size = self.config["subscriptions"].get(config_key) + if not limit or app_config_size is None: + # treat None as unlimited return (video_type, None) if ( @@ -94,8 +96,8 @@ class VideoQueryBuilder: overwrite = self.channel_overwrites[overwrite_key] return (video_type, overwrite) - if overwrite := self.config["subscriptions"].get(config_key): - return (video_type, overwrite) + if app_config_size: + return (video_type, app_config_size) return (video_type, 0) diff --git a/backend/channel/tests/__init__.py b/backend/channel/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/channel/tests/test_src/__init__.py b/backend/channel/tests/test_src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/channel/tests/test_src/test_remote_query.py b/backend/channel/tests/test_src/test_remote_query.py new file mode 100644 index 00000000..a123d75c --- /dev/null +++ b/backend/channel/tests/test_src/test_remote_query.py @@ -0,0 +1,117 @@ +"""test video query building""" + +# pylint: disable=redefined-outer-name + +from enum import Enum + +import pytest +from channel.src.remote_query import VideoQueryBuilder +from video.src.constants import VideoTypeEnum + + +@pytest.fixture +def default_config(): + """from appsettings""" + return { + "subscriptions": { + "channel_size": 5, + "live_channel_size": 3, + "shorts_channel_size": 2, + } + } + + +@pytest.fixture +def empty_overwrites(): + """from channel overwrites""" + return {} + + +@pytest.fixture +def overwrites(): + """from channel overwrites""" + return { + "subscriptions_channel_size": 10, + "subscriptions_live_channel_size": 0, + "subscriptions_shorts_channel_size": None, + } + + +def test_build_all_queries_with_limit(default_config, empty_overwrites): + """default, empty overwrite""" + builder = VideoQueryBuilder(default_config, empty_overwrites) + result = builder.build_queries(None, limit=True) + expected = [ + (VideoTypeEnum.VIDEOS, 5), + (VideoTypeEnum.STREAMS, 3), + (VideoTypeEnum.SHORTS, 2), + ] + assert result == expected + + +def test_build_all_queries_without_limit(default_config, empty_overwrites): + """limit disabled""" + builder = VideoQueryBuilder(default_config, empty_overwrites) + result = builder.build_queries(None, limit=False) + expected = [ + (VideoTypeEnum.VIDEOS, None), + (VideoTypeEnum.STREAMS, None), + (VideoTypeEnum.SHORTS, None), + ] + assert result == expected + + +def test_build_specific_query(default_config, empty_overwrites): + """single vid_type""" + builder = VideoQueryBuilder(default_config, empty_overwrites) + result = builder.build_queries(VideoTypeEnum.VIDEOS) + assert result == [(VideoTypeEnum.VIDEOS, 5)] + + +def test_build_multiple_queries(default_config, empty_overwrites): + """vid_type list""" + builder = VideoQueryBuilder(default_config, empty_overwrites) + result = builder.build_queries( + [VideoTypeEnum.VIDEOS, VideoTypeEnum.SHORTS] + ) + assert result == [(VideoTypeEnum.VIDEOS, 5), (VideoTypeEnum.SHORTS, 2)] + + +def test_overwrite_applied(default_config, overwrites): + """with overwrite from channel config""" + builder = VideoQueryBuilder(default_config, overwrites) + result = builder.build_queries(None, limit=True) + expected = [ + (VideoTypeEnum.VIDEOS, 10), # Overwritten + # STREAMS is overwritten to 0, should be excluded + (VideoTypeEnum.SHORTS, 2), # None in overwrite, fallback to config + ] + assert result == expected + + +def test_no_limit_ignores_config_and_overwrites(default_config, overwrites): + """no limit single vid_type""" + builder = VideoQueryBuilder(default_config, overwrites) + result = builder.build_queries([VideoTypeEnum.STREAMS], limit=False) + assert result == [(VideoTypeEnum.STREAMS, None)] + + +def test_zero_query_not_included(default_config): + """overwrite to zero to disable""" + overwrites = {"subscriptions_live_channel_size": 0} + builder = VideoQueryBuilder(default_config, overwrites) + result = builder.build_queries([VideoTypeEnum.STREAMS], limit=True) + assert not result # Should be skipped due to 0 + + +def test_invalid_video_type_is_ignored(default_config): + """invalid enum""" + builder = VideoQueryBuilder(default_config) + + class FakeEnum(Enum): + """invalid""" + + INVALID = "invalid" + + result = builder.build_queries([FakeEnum.INVALID], limit=True) + assert not result diff --git a/backend/download/src/queue.py b/backend/download/src/queue.py index 25f6a8e9..f5554a18 100644 --- a/backend/download/src/queue.py +++ b/backend/download/src/queue.py @@ -381,7 +381,11 @@ class PendingList(PendingIndex): def __extract_vid_type(self, video_data) -> str: """build vid type""" - if "vid_type" in video_data: + if ( + "vid_type" in video_data + and video_data["vid_type"] + and str(video_data["vid_type"]) in VideoTypeEnum.values_known() + ): return str(video_data["vid_type"]) if video_data.get("live_status") == "was_live": diff --git a/backend/download/src/subscriptions.py b/backend/download/src/subscriptions.py index ea4e1fe3..d18f5399 100644 --- a/backend/download/src/subscriptions.py +++ b/backend/download/src/subscriptions.py @@ -34,28 +34,7 @@ class ChannelSubscription: if not all_channels: return 0 - all_channel_urls: list[ParsedURLType] = [] - - for channel in all_channels: - channel_tabs = channel["channel_tabs"] - if not channel_tabs: - continue - - enum = [getattr(VideoTypeEnum, i.upper()) for i in channel_tabs] - queries = VideoQueryBuilder( - config=self.config, - channel_overwrites=channel.get("channel_overwrites", {}), - ).build_queries(video_type=enum) - - for query in queries: - all_channel_urls.append( - ParsedURLType( - type="channel", - url=channel["channel_id"], - vid_type=query[0], - limit=query[1], - ) - ) + all_channel_urls = self._process_channel_urls(all_channels) if self.task: self.task.send_progress([f"Scanning {len(all_channels)} channels"]) @@ -70,6 +49,34 @@ class ChannelSubscription: return added + def _process_channel_urls(self, all_channels: list[dict]): + """process channels, build queries""" + + all_channel_urls: list[ParsedURLType] = [] + + for channel in all_channels: + channel_tabs = channel["channel_tabs"] + if not channel_tabs: + continue + + enums = [getattr(VideoTypeEnum, i.upper()) for i in channel_tabs] + queries = VideoQueryBuilder( + config=self.config, + channel_overwrites=channel.get("channel_overwrites", {}), + ).build_queries(video_type=enums) + + for query in queries: + all_channel_urls.append( + ParsedURLType( + type="channel", + url=channel["channel_id"], + vid_type=query[0], + limit=query[1], + ) + ) + + return all_channel_urls + class PlaylistSubscription: """scan subscribed playlists for videos to add to pending""" diff --git a/frontend/src/pages/SettingsApplication.tsx b/frontend/src/pages/SettingsApplication.tsx index 718f5d1a..25847766 100644 --- a/frontend/src/pages/SettingsApplication.tsx +++ b/frontend/src/pages/SettingsApplication.tsx @@ -96,9 +96,9 @@ const SettingsApplication = () => { const { data: cookieStateResponseData } = cookieStateResponse ?? {}; // Subscriptions - setVideoPageSize(appSettingsConfigData?.subscriptions.channel_size || null); - setLivePageSize(appSettingsConfigData?.subscriptions.live_channel_size || null); - setShortPageSize(appSettingsConfigData?.subscriptions.shorts_channel_size || null); + setVideoPageSize(appSettingsConfigData?.subscriptions.channel_size ?? null); + setLivePageSize(appSettingsConfigData?.subscriptions.live_channel_size ?? null); + setShortPageSize(appSettingsConfigData?.subscriptions.shorts_channel_size ?? null); setPlaylistPageSize(appSettingsConfigData?.subscriptions.playlist_size || null); setIsAutostart(appSettingsConfigData?.subscriptions.auto_start || false); setIsExtractFlat(appSettingsConfigData?.subscriptions.extract_flat || false);