diff --git a/backend/appsettings/index_mapping.json b/backend/appsettings/index_mapping.json index 79c257bf..6ee68647 100644 --- a/backend/appsettings/index_mapping.json +++ b/backend/appsettings/index_mapping.json @@ -93,6 +93,15 @@ }, "channel_tabs": { "type": "keyword" + }, + "channel_media_size": { + "type": "long" + }, + "channel_video_count": { + "type": "long" + }, + "channel_media_duration": { + "type": "long" } }, "expected_set": { @@ -208,6 +217,15 @@ }, "channel_tabs": { "type": "keyword" + }, + "channel_media_size": { + "type": "long" + }, + "channel_video_count": { + "type": "long" + }, + "channel_media_duration": { + "type": "long" } } }, diff --git a/backend/channel/serializers.py b/backend/channel/serializers.py index d2dc4184..ed4ca78b 100644 --- a/backend/channel/serializers.py +++ b/backend/channel/serializers.py @@ -51,6 +51,9 @@ class ChannelSerializer(serializers.Serializer): channel_tabs = serializers.ListField( child=serializers.ChoiceField(VideoTypeEnum.values_known()) ) + channel_media_size = serializers.IntegerField(required=False) + channel_video_count = serializers.IntegerField(required=False) + channel_media_duration = serializers.IntegerField(required=False) _index = serializers.CharField(required=False) _score = serializers.IntegerField(required=False) @@ -69,6 +72,24 @@ class ChannelListQuerySerializer(serializers.Serializer): choices=["subscribed", "unsubscribed"], required=False ) page = serializers.IntegerField(required=False) + view = serializers.ChoiceField( + choices=["list", "grid", "table"], required=False + ) + sort = serializers.ChoiceField( + choices=[ + "name", + "subscribers", + "video_count", + "duration", + "media_size", + "last_refresh", + ], + required=False, + default="name", + ) + order = serializers.ChoiceField( + choices=["asc", "desc"], required=False, default="asc" + ) class ChannelUpdateSerializer(serializers.Serializer): @@ -78,21 +99,6 @@ class ChannelUpdateSerializer(serializers.Serializer): channel_overwrites = ChannelOverwriteSerializer(required=False) -class ChannelAggBucketSerializer(serializers.Serializer): - """serialize channel agg bucket""" - - value = serializers.IntegerField() - value_str = serializers.CharField(required=False) - - -class ChannelAggSerializer(serializers.Serializer): - """serialize channel aggregation""" - - total_items = ChannelAggBucketSerializer() - total_size = ChannelAggBucketSerializer() - total_duration = ChannelAggBucketSerializer() - - class ChannelNavSerializer(serializers.Serializer): """serialize channel navigation""" diff --git a/backend/channel/src/index.py b/backend/channel/src/index.py index 390f9023..b337c9f2 100644 --- a/backend/channel/src/index.py +++ b/backend/channel/src/index.py @@ -168,6 +168,31 @@ class YoutubeChannel(YouTubeItem): print(f"sync to videos failed with status code {status_code}") print(response) + def update_channel_stats(self) -> None: + """Recompute and store media stats on the channel document.""" + data = { + "query": { + "term": {"channel.channel_id": {"value": self.youtube_id}} + }, + "aggs": { + "total_items": {"value_count": {"field": "youtube_id"}}, + "total_size": {"sum": {"field": "media_size"}}, + "total_duration": {"sum": {"field": "player.duration"}}, + }, + "size": 0, + } + response, _ = ElasticWrap("ta_video/_search").get(data=data) + aggs = response.get("aggregations", {}) + + update = { + "doc": { + "channel_video_count": int(aggs["total_items"]["value"]), + "channel_media_size": int(aggs["total_size"]["value"]), + "channel_media_duration": int(aggs["total_duration"]["value"]), + } + } + ElasticWrap(f"ta_channel/_update/{self.youtube_id}").post(update) + def change_subscribe(self, new_subscribe_state: bool): """change subscribe status""" if not self.json_data: diff --git a/backend/channel/urls.py b/backend/channel/urls.py index 4b4ec53e..6661fef5 100644 --- a/backend/channel/urls.py +++ b/backend/channel/urls.py @@ -19,11 +19,6 @@ urlpatterns = [ views.ChannelApiView.as_view(), name="api-channel", ), - path( - "/aggs/", - views.ChannelAggsApiView.as_view(), - name="api-channel-aggs", - ), path( "/nav/", views.ChannelNavApiView.as_view(), diff --git a/backend/channel/views.py b/backend/channel/views.py index eb8a694d..c0d97b8d 100644 --- a/backend/channel/views.py +++ b/backend/channel/views.py @@ -1,7 +1,6 @@ """all channel API views""" from channel.serializers import ( - ChannelAggSerializer, ChannelListQuerySerializer, ChannelListSerializer, ChannelNavSerializer, @@ -33,6 +32,15 @@ class ChannelApiListView(ApiBaseView): valid_filter = ["subscribed"] permission_classes = [AdminWriteOnly] + sort_field_map = { + "name": "channel_name.keyword", + "subscribers": "channel_subs", + "video_count": "channel_video_count", + "duration": "channel_media_duration", + "media_size": "channel_media_size", + "last_refresh": "channel_last_refresh", + } + @extend_schema( responses={ 200: OpenApiResponse(ChannelListSerializer()), @@ -41,14 +49,15 @@ class ChannelApiListView(ApiBaseView): ) def get(self, request): """get request""" - self.data.update( - {"sort": [{"channel_name.keyword": {"order": "asc"}}]} - ) - serializer = ChannelListQuerySerializer(data=request.query_params) serializer.is_valid(raise_exception=True) validated_data = serializer.validated_data + sort_by = validated_data.get("sort", "name") + sort_order = validated_data.get("order", "asc") + es_sort_field = self.sort_field_map[sort_by] + self.data["sort"] = [{es_sort_field: {"order": sort_order}}] + must_list = [] query_filter = validated_data.get("filter") if query_filter is not None: @@ -179,38 +188,6 @@ class ChannelApiView(ApiBaseView): return Response(error.data, status=404) -class ChannelAggsApiView(ApiBaseView): - """resolves to /api/channel//aggs/ - GET: get channel aggregations - """ - - search_base = "ta_video/_search" - - @extend_schema( - responses={ - 200: OpenApiResponse(ChannelAggSerializer()), - }, - ) - def get(self, request, channel_id): - """get channel aggregations""" - self.data.update( - { - "query": { - "term": {"channel.channel_id": {"value": channel_id}} - }, - "aggs": { - "total_items": {"value_count": {"field": "youtube_id"}}, - "total_size": {"sum": {"field": "media_size"}}, - "total_duration": {"sum": {"field": "player.duration"}}, - }, - } - ) - self.get_aggs() - serializer = ChannelAggSerializer(self.response) - - return Response(serializer.data) - - class ChannelNavApiView(ApiBaseView): """resolves to /api/channel//nav/ GET: get channel nav diff --git a/backend/config/management/commands/ta_startup.py b/backend/config/management/commands/ta_startup.py index e65a701a..5b62cdf1 100644 --- a/backend/config/management/commands/ta_startup.py +++ b/backend/config/management/commands/ta_startup.py @@ -66,6 +66,7 @@ class Command(BaseCommand): self._mig_fix_channel_art_types() self._mig_fix_channel_description() self._mig_fix_video_description() + self._mig_populate_channel_stats() @property def skip_migrations(self) -> bool: @@ -479,6 +480,34 @@ class Command(BaseCommand): noop_msg = " no items needed updating" self.stdout.write(self.style.SUCCESS(noop_msg)) + def _mig_populate_channel_stats(self) -> None: + """populate channel size/count/duration for channels missing stats""" + desc = "populate channel media stats" + self.stdout.write(f"[MIGRATION] run {desc}") + channels = get_channels( + subscribed_only=False, source=["channel_id", "channel_media_size"] + ) + counter = 0 + for channel_response in channels: + if channel_response.get("channel_media_size") is not None: + continue + + YoutubeChannel( + channel_response["channel_id"] + ).update_channel_stats() + counter += 1 + + if counter: + self.stdout.write( + self.style.SUCCESS( + f" ✓ populated stats for {counter} channels" + ) + ) + else: + self.stdout.write( + self.style.SUCCESS(" no channels needed updating") + ) + def _run_migration( self, index_name: str, desc: str, query: dict, script: dict ): diff --git a/backend/download/src/yt_dlp_handler.py b/backend/download/src/yt_dlp_handler.py index 768ef643..4300b9ce 100644 --- a/backend/download/src/yt_dlp_handler.py +++ b/backend/download/src/yt_dlp_handler.py @@ -87,6 +87,7 @@ class VideoDownloader(DownloaderBase): self._notify(video_data, "Move downloaded file to archive") self.move_to_archive(vid_dict) self._delete_from_pending(youtube_id) + YoutubeChannel(channel_id).update_channel_stats() downloaded += 1 # post processing diff --git a/backend/stats/src/aggs.py b/backend/stats/src/aggs.py index 57cdd18d..1d5f6fb7 100644 --- a/backend/stats/src/aggs.py +++ b/backend/stats/src/aggs.py @@ -320,54 +320,42 @@ class DownloadHist(AggBase): return response -class BiggestChannel(AggBase): - """get channel aggregations""" +class BiggestChannel: + """get biggest channels by precomputed channel stats""" + + path = "ta_channel/_search" + sort_field_map = { + "doc_count": "channel_video_count", + "duration": "channel_media_duration", + "media_size": "channel_media_size", + } def __init__(self, order): - self.data["aggs"][self.name]["multi_terms"]["order"] = {order: "desc"} - - name = "channel_stats" - path = "ta_video/_search" - data = { - "size": 0, - "aggs": { - name: { - "multi_terms": { - "terms": [ - {"field": "channel.channel_name.keyword"}, - {"field": "channel.channel_id"}, - ], - "order": {"doc_count": "desc"}, - }, - "aggs": { - "doc_count": {"value_count": {"field": "_index"}}, - "duration": {"sum": {"field": "player.duration"}}, - "media_size": {"sum": {"field": "media_size"}}, - }, - }, - }, - } - order_choices = ["doc_count", "duration", "media_size"] + self.order = order def process(self): - """process aggregation, order_by validated in the view""" - - aggregations = self.get() - if not aggregations: + """get channels sorted by order, order_by validated in the view""" + data = { + "size": 10, + "sort": [{self.sort_field_map[self.order]: {"order": "desc"}}], + } + response, _ = ElasticWrap(self.path).get(data=data) + hits = response.get("hits", {}).get("hits") + if not hits: return None - buckets = aggregations[self.name]["buckets"] - response = [ { - "id": i["key"][1], - "name": i["key"][0].title(), - "doc_count": i["doc_count"]["value"], - "duration": i["duration"]["value"], - "duration_str": get_duration_str(int(i["duration"]["value"])), - "media_size": i["media_size"]["value"], + "id": hit["_source"]["channel_id"], + "name": hit["_source"]["channel_name"], + "doc_count": hit["_source"].get("channel_video_count", 0), + "duration": hit["_source"].get("channel_media_duration", 0), + "duration_str": get_duration_str( + hit["_source"].get("channel_media_duration", 0) + ), + "media_size": hit["_source"].get("channel_media_size", 0), } - for i in buckets + for hit in hits ] return response diff --git a/backend/user/serializers.py b/backend/user/serializers.py index 4a54aa4c..d1727a82 100644 --- a/backend/user/serializers.py +++ b/backend/user/serializers.py @@ -34,7 +34,9 @@ class UserMeConfigSerializer(serializers.Serializer): view_style_home = serializers.ChoiceField( choices=["grid", "list", "table"] ) - view_style_channel = serializers.ChoiceField(choices=["grid", "list"]) + view_style_channel = serializers.ChoiceField( + choices=["grid", "list", "table"] + ) view_style_downloads = serializers.ChoiceField(choices=["grid", "list"]) view_style_playlist = serializers.ChoiceField(choices=["grid", "list"]) vid_type_filter = serializers.ChoiceField( diff --git a/backend/video/src/index.py b/backend/video/src/index.py index 00250d17..123b44ba 100644 --- a/backend/video/src/index.py +++ b/backend/video/src/index.py @@ -337,10 +337,12 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): except FileNotFoundError: print(f"{self.youtube_id}: failed {media_url}, continue.") + channel_id = self.json_data["channel"]["channel_id"] self.del_in_playlists() self.del_in_es() self.delete_subtitles() self.delete_comments() + ta_channel.YoutubeChannel(channel_id).update_channel_stats() def del_in_playlists(self): """remove downloaded in playlist""" diff --git a/frontend/src/api/loader/loadChannelAggs.ts b/frontend/src/api/loader/loadChannelAggs.ts deleted file mode 100644 index a479ba6c..00000000 --- a/frontend/src/api/loader/loadChannelAggs.ts +++ /dev/null @@ -1,20 +0,0 @@ -import APIClient from '../../functions/APIClient'; - -export type ChannelAggsType = { - total_items: { - value: number; - }; - total_size: { - value: number; - }; - total_duration: { - value: number; - value_str: string; - }; -}; - -const loadChannelAggs = async (channelId: string) => { - return APIClient(`/api/channel/${channelId}/aggs/`); -}; - -export default loadChannelAggs; diff --git a/frontend/src/api/loader/loadChannelList.ts b/frontend/src/api/loader/loadChannelList.ts index a46a9574..ec5f3cb2 100644 --- a/frontend/src/api/loader/loadChannelList.ts +++ b/frontend/src/api/loader/loadChannelList.ts @@ -1,7 +1,9 @@ import { PaginationType } from '../../components/Pagination'; +import { ViewStylesType } from '../../configuration/constants/ViewStyle'; import APIClient from '../../functions/APIClient'; import { ChannelType } from '../../pages/Channels'; import { ConfigType } from '../../pages/Home'; +import { SortOrderType } from './loadVideoListByPage'; export type ChannelsListResponse = { data: ChannelType[]; @@ -9,13 +11,30 @@ export type ChannelsListResponse = { config?: ConfigType; }; -const loadChannelList = async (page: number, showSubscribed: boolean | null) => { +export type ChannelSortByType = + | 'name' + | 'subscribers' + | 'video_count' + | 'duration' + | 'media_size' + | 'last_refresh'; + +const loadChannelList = async ( + page: number, + showSubscribed: boolean | null, + viewStyle?: ViewStylesType, + sort?: ChannelSortByType, + order?: SortOrderType, +) => { const searchParams = new URLSearchParams(); if (page) searchParams.append('page', page.toString()); if (showSubscribed !== null) { searchParams.append('filter', showSubscribed ? 'subscribed' : 'unsubscribed'); } + if (viewStyle) searchParams.append('view', viewStyle); + if (sort) searchParams.append('sort', sort); + if (order) searchParams.append('order', order); const endpoint = `/api/channel/${searchParams.toString() ? `?${searchParams.toString()}` : ''}`; diff --git a/frontend/src/components/ChannelList.tsx b/frontend/src/components/ChannelList.tsx index 7df96c8e..db000839 100644 --- a/frontend/src/components/ChannelList.tsx +++ b/frontend/src/components/ChannelList.tsx @@ -3,22 +3,52 @@ import { ChannelType } from '../pages/Channels'; import Routes from '../configuration/routes/RouteList'; import updateChannelSubscription from '../api/actions/updateChannelSubscription'; import formatDate from '../functions/formatDates'; +import formatTime from '../functions/formatTime'; +import humanFileSize from '../functions/humanFileSize'; +import { FileSizeUnits } from '../api/actions/updateUserConfig'; import FormattedNumber from './FormattedNumber'; import Button from './Button'; import ChannelIcon from './ChannelIcon'; import ChannelBanner from './ChannelBanner'; import LoadingIndicator from './LoadingIndicator'; import { useUserConfigStore } from '../stores/UserConfigStore'; +import { ViewStylesEnum } from '../configuration/constants/ViewStyle'; +import { ChannelSortByType } from '../api/loader/loadChannelList'; +import { SortOrderType } from '../api/loader/loadVideoListByPage'; type ChannelListProps = { channelList: ChannelType[] | undefined; refreshChannelList: (refresh: boolean) => void; + sortBy?: ChannelSortByType; + sortOrder?: SortOrderType; + onSort?: (column: ChannelSortByType) => void; }; -const ChannelList = ({ channelList, refreshChannelList }: ChannelListProps) => { +const ChannelList = ({ + channelList, + refreshChannelList, + sortBy, + sortOrder, + onSort, +}: ChannelListProps) => { const { userConfig } = useUserConfigStore(); const viewStyle = userConfig.view_style_channel; + const useSiUnits = userConfig.file_size_unit === FileSizeUnits.Metric; + const getSortIndicator = (column: ChannelSortByType) => { + if (sortBy !== column) return ''; + return sortOrder === 'asc' ? ' \u25B2' : ' \u25BC'; + }; + const renderSortableHeader = (column: ChannelSortByType, label: string) => ( + onSort?.(column)} + style={{ cursor: 'pointer' }} + className={sortBy === column ? 'sort-active' : ''} + > + {label} + {getSortIndicator(column)} + + ); if (!channelList) { return ; } @@ -26,6 +56,84 @@ const ChannelList = ({ channelList, refreshChannelList }: ChannelListProps) => { return

No channels found...

; } + if (viewStyle === ViewStylesEnum.Table) { + return ( +
+ + + + {renderSortableHeader('name', 'Channel')} + {renderSortableHeader('subscribers', 'Subscribers')} + {renderSortableHeader('video_count', 'Videos')} + {renderSortableHeader('duration', 'Duration')} + {renderSortableHeader('media_size', 'Media size')} + {renderSortableHeader('last_refresh', 'Last refreshed')} + + + + + {channelList.map(channel => { + return ( + + + + + + + + + + ); + })} + +
Status
+
+
+ + + +
+
+ {channel.channel_name} +
+
+ 		+ {channel.channel_subs !== null ? ( + + ) : ( + '-' + )} + {channel.channel_video_count ?? 0}{formatTime(channel.channel_media_duration ?? 0)}{humanFileSize(channel.channel_media_size ?? 0, useSiUnits)}{formatDate(channel.channel_last_refresh)} + {channel.channel_subscribed ? ( +
+
+ ); + } + return ( <> {channelList.map(channel => { diff --git a/frontend/src/pages/ChannelVideo.tsx b/frontend/src/pages/ChannelVideo.tsx index 47d34860..cfe30d68 100644 --- a/frontend/src/pages/ChannelVideo.tsx +++ b/frontend/src/pages/ChannelVideo.tsx @@ -22,8 +22,8 @@ import loadVideoListByFilter, { WatchTypes, WatchTypesEnum, } from '../api/loader/loadVideoListByPage'; -import loadChannelAggs, { ChannelAggsType } from '../api/loader/loadChannelAggs'; import humanFileSize from '../functions/humanFileSize'; +import formatTime from '../functions/formatTime'; import { useUserConfigStore } from '../stores/UserConfigStore'; import { FileSizeUnits } from '../api/actions/updateUserConfig'; import { ApiResponseType } from '../functions/APIClient'; @@ -51,16 +51,12 @@ const ChannelVideo = ({ videoType }: ChannelVideoProps) => { const [channelResponse, setChannelResponse] = useState>(); const [videoResponse, setVideoReponse] = useState>(); - const [videoAggsResponse, setVideoAggsResponse] = useState>(); - const { data: channelResponseData } = channelResponse ?? {}; const { data: videoResponseData } = videoResponse ?? {}; - const { data: videoAggsResponseData } = videoAggsResponse ?? {}; const channel = channelResponseData; const videoList = videoResponseData?.data; const pagination = videoResponseData?.paginate; - const videoAggs = videoAggsResponseData; const hasVideos = videoResponseData?.data?.length !== 0; const useSiUnits = userConfig.file_size_unit === FileSizeUnits.Metric; @@ -87,11 +83,8 @@ const ChannelVideo = ({ videoType }: ChannelVideoProps) => { type: videoType, height: filterHeight, }); - const channelAggs = await loadChannelAggs(channelId); - setChannelResponse(channelResponse); setVideoReponse(videos); - setVideoAggsResponse(channelAggs); setRefresh(false); })(); }, [ @@ -123,13 +116,13 @@ const ChannelVideo = ({ videoType }: ChannelVideoProps) => { setRefresh={setRefresh} />
- {videoAggs && ( + {channel.channel_video_count !== undefined && ( <>

- {videoAggs.total_items.value} videos |{' '} - {videoAggs.total_duration.value_str} playback{' '} + {channel.channel_video_count} videos |{' '} + {formatTime(channel.channel_media_duration ?? 0)} playback{' '} | Total size{' '} - {humanFileSize(videoAggs.total_size.value, useSiUnits)} + {humanFileSize(channel.channel_media_size ?? 0, useSiUnits)}

- +
{pagination && ( diff --git a/frontend/src/style.css b/frontend/src/style.css index 74f0113a..9f0c7745 100644 --- a/frontend/src/style.css +++ b/frontend/src/style.css @@ -1080,6 +1080,10 @@ video:-webkit-full-screen { gap: 1rem; } +.channel-list.table { + display: block; +} + .channel-item.list { padding-bottom: 1rem; } @@ -1088,6 +1092,52 @@ video:-webkit-full-screen { display: block; } +.channel-item.table { + overflow-x: auto; + -webkit-overflow-scrolling: touch; +} + +.channel-item.table table { + min-width: 700px; + width: 100%; + border-collapse: collapse; + position: relative; +} + +.channel-item.table th, +.channel-item.table td { + padding: 8px 12px; + text-align: left; + white-space: nowrap; +} + +.channel-item.table th { + font-weight: 600; + border-bottom: 1px solid #ddd; +} + +.channel-item.table td { + border-bottom: 1px solid #eee; +} + +.channel-item.table td.no-nowrap { + white-space: normal; + word-break: keep-all; + min-width: 160px; +} + +.channel-item.table td img, +.channel-item.table th img { + width: 40px; + height: 40px; +} + +.channel-table-title { + display: inline-flex; + align-items: center; + gap: 10px; +} + .channel-banner img { width: 100%; aspect-ratio: 18 / 3;