RomanNum3ral
/
tubearchivist
mirror of https://github.com/tubearchivist/tubearchivist.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge afae28a26b into ed19ffa020

This commit is contained in:
Caleb Rogers 2026-07-06 20:15:41 +08:00 committed by GitHub
parent ed19ffa020 afae28a26b
commit 7bdcd51a87
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 380 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
backend/appsettings/index_mapping.json
View File

@ -93,6 +93,15 @@
},
"channel_tabs": {
"type": "keyword"
},
"channel_media_size": {
"type": "long"
},
"channel_video_count": {
"type": "long"
},
"channel_media_duration": {
"type": "long"
}
},
"expected_set": {
@ -208,6 +217,15 @@
},
"channel_tabs": {
"type": "keyword"
},
"channel_media_size": {
"type": "long"
},
"channel_video_count": {
"type": "long"
},
"channel_media_duration": {
"type": "long"
}
}
},

36
backend/channel/serializers.py
View File

@ -51,6 +51,9 @@ class ChannelSerializer(serializers.Serializer):
channel_tabs = serializers.ListField(
child=serializers.ChoiceField(VideoTypeEnum.values_known())
)
channel_media_size = serializers.IntegerField(required=False)
channel_video_count = serializers.IntegerField(required=False)
channel_media_duration = serializers.IntegerField(required=False)
_index = serializers.CharField(required=False)
_score = serializers.IntegerField(required=False)
@ -69,6 +72,24 @@ class ChannelListQuerySerializer(serializers.Serializer):
choices=["subscribed", "unsubscribed"], required=False
)
page = serializers.IntegerField(required=False)
view = serializers.ChoiceField(
choices=["list", "grid", "table"], required=False
)
sort = serializers.ChoiceField(
choices=[
"name",
"subscribers",
"video_count",
"duration",
"media_size",
"last_refresh",
],
required=False,
default="name",
)
order = serializers.ChoiceField(
choices=["asc", "desc"], required=False, default="asc"
)
class ChannelUpdateSerializer(serializers.Serializer):
@ -78,21 +99,6 @@ class ChannelUpdateSerializer(serializers.Serializer):
channel_overwrites = ChannelOverwriteSerializer(required=False)
class ChannelAggBucketSerializer(serializers.Serializer):
"""serialize channel agg bucket"""
value = serializers.IntegerField()
value_str = serializers.CharField(required=False)
class ChannelAggSerializer(serializers.Serializer):
"""serialize channel aggregation"""
total_items = ChannelAggBucketSerializer()
total_size = ChannelAggBucketSerializer()
total_duration = ChannelAggBucketSerializer()
class ChannelNavSerializer(serializers.Serializer):
"""serialize channel navigation"""

25
backend/channel/src/index.py
View File

@ -168,6 +168,31 @@ class YoutubeChannel(YouTubeItem):
print(f"sync to videos failed with status code {status_code}")
print(response)
def update_channel_stats(self) -> None:
"""Recompute and store media stats on the channel document."""
data = {
"query": {
"term": {"channel.channel_id": {"value": self.youtube_id}}
},
"aggs": {
"total_items": {"value_count": {"field": "youtube_id"}},
"total_size": {"sum": {"field": "media_size"}},
"total_duration": {"sum": {"field": "player.duration"}},
},
"size": 0,
}
response, _ = ElasticWrap("ta_video/_search").get(data=data)
aggs = response.get("aggregations", {})
update = {
"doc": {
"channel_video_count": int(aggs["total_items"]["value"]),
"channel_media_size": int(aggs["total_size"]["value"]),
"channel_media_duration": int(aggs["total_duration"]["value"]),
}
}
ElasticWrap(f"ta_channel/_update/{self.youtube_id}").post(update)
def change_subscribe(self, new_subscribe_state: bool):
"""change subscribe status"""
if not self.json_data:

5
backend/channel/urls.py
View File

@ -19,11 +19,6 @@ urlpatterns = [
views.ChannelApiView.as_view(),
name="api-channel",
),
path(
"<slug:channel_id>/aggs/",
views.ChannelAggsApiView.as_view(),
name="api-channel-aggs",
),
path(
"<slug:channel_id>/nav/",
views.ChannelNavApiView.as_view(),

51
backend/channel/views.py
View File

@ -1,7 +1,6 @@
"""all channel API views"""
from channel.serializers import (
ChannelAggSerializer,
ChannelListQuerySerializer,
ChannelListSerializer,
ChannelNavSerializer,
@ -33,6 +32,15 @@ class ChannelApiListView(ApiBaseView):
valid_filter = ["subscribed"]
permission_classes = [AdminWriteOnly]
sort_field_map = {
"name": "channel_name.keyword",
"subscribers": "channel_subs",
"video_count": "channel_video_count",
"duration": "channel_media_duration",
"media_size": "channel_media_size",
"last_refresh": "channel_last_refresh",
}
@extend_schema(
responses={
200: OpenApiResponse(ChannelListSerializer()),
@ -41,14 +49,15 @@ class ChannelApiListView(ApiBaseView):
)
def get(self, request):
"""get request"""
self.data.update(
{"sort": [{"channel_name.keyword": {"order": "asc"}}]}
)
serializer = ChannelListQuerySerializer(data=request.query_params)
serializer.is_valid(raise_exception=True)
validated_data = serializer.validated_data
sort_by = validated_data.get("sort", "name")
sort_order = validated_data.get("order", "asc")
es_sort_field = self.sort_field_map[sort_by]
self.data["sort"] = [{es_sort_field: {"order": sort_order}}]
must_list = []
query_filter = validated_data.get("filter")
if query_filter is not None:
@ -179,38 +188,6 @@ class ChannelApiView(ApiBaseView):
return Response(error.data, status=404)
class ChannelAggsApiView(ApiBaseView):
"""resolves to /api/channel/<channel_id>/aggs/
GET: get channel aggregations
"""
search_base = "ta_video/_search"
@extend_schema(
responses={
200: OpenApiResponse(ChannelAggSerializer()),
},
)
def get(self, request, channel_id):
"""get channel aggregations"""
self.data.update(
{
"query": {
"term": {"channel.channel_id": {"value": channel_id}}
},
"aggs": {
"total_items": {"value_count": {"field": "youtube_id"}},
"total_size": {"sum": {"field": "media_size"}},
"total_duration": {"sum": {"field": "player.duration"}},
},
}
)
self.get_aggs()
serializer = ChannelAggSerializer(self.response)
return Response(serializer.data)
class ChannelNavApiView(ApiBaseView):
"""resolves to /api/channel/<channel_id>/nav/
GET: get channel nav

29
backend/config/management/commands/ta_startup.py
View File

@ -66,6 +66,7 @@ class Command(BaseCommand):
self._mig_fix_channel_art_types()
self._mig_fix_channel_description()
self._mig_fix_video_description()
self._mig_populate_channel_stats()
@property
def skip_migrations(self) -> bool:
@ -479,6 +480,34 @@ class Command(BaseCommand):
noop_msg = " no items needed updating"
self.stdout.write(self.style.SUCCESS(noop_msg))
def _mig_populate_channel_stats(self) -> None:
"""populate channel size/count/duration for channels missing stats"""
desc = "populate channel media stats"
self.stdout.write(f"[MIGRATION] run {desc}")
channels = get_channels(
subscribed_only=False, source=["channel_id", "channel_media_size"]
)
counter = 0
for channel_response in channels:
if channel_response.get("channel_media_size") is not None:
continue
YoutubeChannel(
channel_response["channel_id"]
).update_channel_stats()
counter += 1
if counter:
self.stdout.write(
self.style.SUCCESS(
f" ✓ populated stats for {counter} channels"
)
)
else:
self.stdout.write(
self.style.SUCCESS(" no channels needed updating")
)
def _run_migration(
self, index_name: str, desc: str, query: dict, script: dict
):

1
backend/download/src/yt_dlp_handler.py
View File

@ -87,6 +87,7 @@ class VideoDownloader(DownloaderBase):
self._notify(video_data, "Move downloaded file to archive")
self.move_to_archive(vid_dict)
self._delete_from_pending(youtube_id)
YoutubeChannel(channel_id).update_channel_stats()
downloaded += 1
# post processing

66
backend/stats/src/aggs.py
View File

@ -320,54 +320,42 @@ class DownloadHist(AggBase):
return response
class BiggestChannel(AggBase):
"""get channel aggregations"""
class BiggestChannel:
"""get biggest channels by precomputed channel stats"""
path = "ta_channel/_search"
sort_field_map = {
"doc_count": "channel_video_count",
"duration": "channel_media_duration",
"media_size": "channel_media_size",
}
def __init__(self, order):
self.data["aggs"][self.name]["multi_terms"]["order"] = {order: "desc"}
name = "channel_stats"
path = "ta_video/_search"
data = {
"size": 0,
"aggs": {
name: {
"multi_terms": {
"terms": [
{"field": "channel.channel_name.keyword"},
{"field": "channel.channel_id"},
],
"order": {"doc_count": "desc"},
},
"aggs": {
"doc_count": {"value_count": {"field": "_index"}},
"duration": {"sum": {"field": "player.duration"}},
"media_size": {"sum": {"field": "media_size"}},
},
},
},
}
order_choices = ["doc_count", "duration", "media_size"]
self.order = order
def process(self):
"""process aggregation, order_by validated in the view"""
aggregations = self.get()
if not aggregations:
"""get channels sorted by order, order_by validated in the view"""
data = {
"size": 10,
"sort": [{self.sort_field_map[self.order]: {"order": "desc"}}],
}
response, _ = ElasticWrap(self.path).get(data=data)
hits = response.get("hits", {}).get("hits")
if not hits:
return None
buckets = aggregations[self.name]["buckets"]
response = [
{
"id": i["key"][1],
"name": i["key"][0].title(),
"doc_count": i["doc_count"]["value"],
"duration": i["duration"]["value"],
"duration_str": get_duration_str(int(i["duration"]["value"])),
"media_size": i["media_size"]["value"],
"id": hit["_source"]["channel_id"],
"name": hit["_source"]["channel_name"],
"doc_count": hit["_source"].get("channel_video_count", 0),
"duration": hit["_source"].get("channel_media_duration", 0),
"duration_str": get_duration_str(
hit["_source"].get("channel_media_duration", 0)
),
"media_size": hit["_source"].get("channel_media_size", 0),
}
for i in buckets
for hit in hits
]
return response

4
backend/user/serializers.py
View File

@ -34,7 +34,9 @@ class UserMeConfigSerializer(serializers.Serializer):
view_style_home = serializers.ChoiceField(
choices=["grid", "list", "table"]
)
view_style_channel = serializers.ChoiceField(choices=["grid", "list"])
view_style_channel = serializers.ChoiceField(
choices=["grid", "list", "table"]
)
view_style_downloads = serializers.ChoiceField(choices=["grid", "list"])
view_style_playlist = serializers.ChoiceField(choices=["grid", "list"])
vid_type_filter = serializers.ChoiceField(

2
backend/video/src/index.py
View File

@ -337,10 +337,12 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
except FileNotFoundError:
print(f"{self.youtube_id}: failed {media_url}, continue.")
channel_id = self.json_data["channel"]["channel_id"]
self.del_in_playlists()
self.del_in_es()
self.delete_subtitles()
self.delete_comments()
ta_channel.YoutubeChannel(channel_id).update_channel_stats()
def del_in_playlists(self):
"""remove downloaded in playlist"""

20
frontend/src/api/loader/loadChannelAggs.ts
View File

@ -1,20 +0,0 @@
import APIClient from '../../functions/APIClient';
export type ChannelAggsType = {
total_items: {
value: number;
};
total_size: {
value: number;
};
total_duration: {
value: number;
value_str: string;
};
};
const loadChannelAggs = async (channelId: string) => {
return APIClient<ChannelAggsType>(`/api/channel/${channelId}/aggs/`);
};
export default loadChannelAggs;

21
frontend/src/api/loader/loadChannelList.ts
View File

@ -1,7 +1,9 @@
import { PaginationType } from '../../components/Pagination';
import { ViewStylesType } from '../../configuration/constants/ViewStyle';
import APIClient from '../../functions/APIClient';
import { ChannelType } from '../../pages/Channels';
import { ConfigType } from '../../pages/Home';
import { SortOrderType } from './loadVideoListByPage';
export type ChannelsListResponse = {
data: ChannelType[];
@ -9,13 +11,30 @@ export type ChannelsListResponse = {
config?: ConfigType;
};
const loadChannelList = async (page: number, showSubscribed: boolean | null) => {
export type ChannelSortByType =
| 'name'
| 'subscribers'
| 'video_count'
| 'duration'
| 'media_size'
| 'last_refresh';
const loadChannelList = async (
page: number,
showSubscribed: boolean | null,
viewStyle?: ViewStylesType,
sort?: ChannelSortByType,
order?: SortOrderType,
) => {
const searchParams = new URLSearchParams();
if (page) searchParams.append('page', page.toString());
if (showSubscribed !== null) {
searchParams.append('filter', showSubscribed ? 'subscribed' : 'unsubscribed');
}
if (viewStyle) searchParams.append('view', viewStyle);
if (sort) searchParams.append('sort', sort);
if (order) searchParams.append('order', order);
const endpoint = `/api/channel/${searchParams.toString() ? `?${searchParams.toString()}` : ''}`;

110
frontend/src/components/ChannelList.tsx
View File

@ -3,22 +3,52 @@ import { ChannelType } from '../pages/Channels';
import Routes from '../configuration/routes/RouteList';
import updateChannelSubscription from '../api/actions/updateChannelSubscription';
import formatDate from '../functions/formatDates';
import formatTime from '../functions/formatTime';
import humanFileSize from '../functions/humanFileSize';
import { FileSizeUnits } from '../api/actions/updateUserConfig';
import FormattedNumber from './FormattedNumber';
import Button from './Button';
import ChannelIcon from './ChannelIcon';
import ChannelBanner from './ChannelBanner';
import LoadingIndicator from './LoadingIndicator';
import { useUserConfigStore } from '../stores/UserConfigStore';
import { ViewStylesEnum } from '../configuration/constants/ViewStyle';
import { ChannelSortByType } from '../api/loader/loadChannelList';
import { SortOrderType } from '../api/loader/loadVideoListByPage';
type ChannelListProps = {
channelList: ChannelType[] | undefined;
refreshChannelList: (refresh: boolean) => void;
sortBy?: ChannelSortByType;
sortOrder?: SortOrderType;
onSort?: (column: ChannelSortByType) => void;
};
const ChannelList = ({ channelList, refreshChannelList }: ChannelListProps) => {
const ChannelList = ({
channelList,
refreshChannelList,
sortBy,
sortOrder,
onSort,
}: ChannelListProps) => {
const { userConfig } = useUserConfigStore();
const viewStyle = userConfig.view_style_channel;
const useSiUnits = userConfig.file_size_unit === FileSizeUnits.Metric;
const getSortIndicator = (column: ChannelSortByType) => {
if (sortBy !== column) return '';
return sortOrder === 'asc' ? ' \u25B2' : ' \u25BC';
};
const renderSortableHeader = (column: ChannelSortByType, label: string) => (
<th
onClick={() => onSort?.(column)}
style={{ cursor: 'pointer' }}
className={sortBy === column ? 'sort-active' : ''}
>
{label}
{getSortIndicator(column)}
</th>
);
if (!channelList) {
return <LoadingIndicator />;
}
@ -26,6 +56,84 @@ const ChannelList = ({ channelList, refreshChannelList }: ChannelListProps) => {
return <h2>No channels found...</h2>;
}
if (viewStyle === ViewStylesEnum.Table) {
return (
<div className={`channel-item ${viewStyle}`}>
<table>
<thead>
<tr>
{renderSortableHeader('name', 'Channel')}
{renderSortableHeader('subscribers', 'Subscribers')}
{renderSortableHeader('video_count', 'Videos')}
{renderSortableHeader('duration', 'Duration')}
{renderSortableHeader('media_size', 'Media size')}
{renderSortableHeader('last_refresh', 'Last refreshed')}
<th>Status</th>
</tr>
</thead>
<tbody>
{channelList.map(channel => {
return (
<tr key={channel.channel_id}>
<td className="no-nowrap">
<div className="channel-table-title">
<div className="round-img">
<Link to={Routes.Channel(channel.channel_id)}>
<ChannelIcon
channelId={channel.channel_id}
channelThumbUrl={channel.channel_thumb_url}
/>
</Link>
</div>
<div>
<Link to={Routes.Channel(channel.channel_id)}>{channel.channel_name}</Link>
</div>
</div>
</td>
<td>
{channel.channel_subs !== null ? (
<FormattedNumber text="" number={channel.channel_subs} />
) : (
'-'
)}
</td>
<td>{channel.channel_video_count ?? 0}</td>
<td>{formatTime(channel.channel_media_duration ?? 0)}</td>
<td>{humanFileSize(channel.channel_media_size ?? 0, useSiUnits)}</td>
<td>{formatDate(channel.channel_last_refresh)}</td>
<td>
{channel.channel_subscribed ? (
<Button
label="Unsubscribe"
className="unsubscribe"
type="button"
title={`Unsubscribe from ${channel.channel_name}`}
onClick={async () => {
await updateChannelSubscription(channel.channel_id, false);
refreshChannelList(true);
}}
/>
) : (
<Button
label="Subscribe"
type="button"
title={`Subscribe to ${channel.channel_name}`}
onClick={async () => {
await updateChannelSubscription(channel.channel_id, true);
refreshChannelList(true);
}}
/>
)}
</td>
</tr>
);
})}
</tbody>
</table>
</div>
);
}
return (
<>
{channelList.map(channel => {

17
frontend/src/pages/ChannelVideo.tsx
View File

@ -22,8 +22,8 @@ import loadVideoListByFilter, {
WatchTypes,
WatchTypesEnum,
} from '../api/loader/loadVideoListByPage';
import loadChannelAggs, { ChannelAggsType } from '../api/loader/loadChannelAggs';
import humanFileSize from '../functions/humanFileSize';
import formatTime from '../functions/formatTime';
import { useUserConfigStore } from '../stores/UserConfigStore';
import { FileSizeUnits } from '../api/actions/updateUserConfig';
import { ApiResponseType } from '../functions/APIClient';
@ -51,16 +51,12 @@ const ChannelVideo = ({ videoType }: ChannelVideoProps) => {
const [channelResponse, setChannelResponse] = useState<ApiResponseType<ChannelResponseType>>();
const [videoResponse, setVideoReponse] =
useState<ApiResponseType<VideoListByFilterResponseType>>();
const [videoAggsResponse, setVideoAggsResponse] = useState<ApiResponseType<ChannelAggsType>>();
const { data: channelResponseData } = channelResponse ?? {};
const { data: videoResponseData } = videoResponse ?? {};
const { data: videoAggsResponseData } = videoAggsResponse ?? {};
const channel = channelResponseData;
const videoList = videoResponseData?.data;
const pagination = videoResponseData?.paginate;
const videoAggs = videoAggsResponseData;
const hasVideos = videoResponseData?.data?.length !== 0;
const useSiUnits = userConfig.file_size_unit === FileSizeUnits.Metric;
@ -87,11 +83,8 @@ const ChannelVideo = ({ videoType }: ChannelVideoProps) => {
type: videoType,
height: filterHeight,
});
const channelAggs = await loadChannelAggs(channelId);
setChannelResponse(channelResponse);
setVideoReponse(videos);
setVideoAggsResponse(channelAggs);
setRefresh(false);
})();
}, [
@ -123,13 +116,13 @@ const ChannelVideo = ({ videoType }: ChannelVideoProps) => {
setRefresh={setRefresh}
/>
<div className="info-box-item">
{videoAggs && (
{channel.channel_video_count !== undefined && (
<>
<p>
{videoAggs.total_items.value} videos <span className="space-carrot">|</span>{' '}
{videoAggs.total_duration.value_str} playback{' '}
{channel.channel_video_count} videos <span className="space-carrot">|</span>{' '}
{formatTime(channel.channel_media_duration ?? 0)} playback{' '}
<span className="space-carrot">|</span> Total size{' '}
{humanFileSize(videoAggs.total_size.value, useSiUnits)}
{humanFileSize(channel.channel_media_size ?? 0, useSiUnits)}
</p>
<div className="button-box">
<Button

60
frontend/src/pages/Channels.tsx
View File

@ -1,7 +1,12 @@
import { useOutletContext } from 'react-router-dom';
import loadChannelList, { ChannelsListResponse } from '../api/loader/loadChannelList';
import loadChannelList, {
ChannelsListResponse,
ChannelSortByType,
} from '../api/loader/loadChannelList';
import { SortOrderType } from '../api/loader/loadVideoListByPage';
import iconGridView from '/img/icon-gridview.svg';
import iconListView from '/img/icon-listview.svg';
import iconTableView from '/img/icon-tableview.svg';
import iconAdd from '/img/icon-add.svg';
import iconFilter from '/img/icon-filter.svg';
import { useEffect, useState } from 'react';
@ -41,6 +46,9 @@ export type ChannelType = {
channel_tags?: string[];
channel_thumb_url: string;
channel_tvart_url: string;
channel_media_size?: number;
channel_video_count?: number;
channel_media_duration?: number;
};
const Channels = () => {
@ -61,6 +69,19 @@ const Channels = () => {
const channels = channelListResponseData?.data;
const pagination = channelListResponseData?.paginate;
const [sortBy, setSortBy] = useState<ChannelSortByType>('name');
const [sortOrder, setSortOrder] = useState<SortOrderType>('asc');
const viewStyle = userConfig.view_style_channel;
const handleSort = (column: ChannelSortByType) => {
if (sortBy === column) {
setSortOrder(sortOrder === 'asc' ? 'desc' : 'asc');
} else {
setSortBy(column);
setSortOrder('asc');
}
};
const handleUserConfigUpdate = async (config: Partial<UserConfigType>) => {
const updatedUserConfig = await updateUserConfig(config);
const { data: updatedUserConfigData } = updatedUserConfig;
@ -72,13 +93,27 @@ const Channels = () => {
useEffect(() => {
(async () => {
const channelListResponse = await loadChannelList(currentPage, userConfig.show_subed_only);
const channelListResponse = await loadChannelList(
currentPage,
userConfig.show_subed_only,
viewStyle,
sortBy,
sortOrder,
);
setChannelListResponse(channelListResponse);
setShowNotification(false);
setRefresh(false);
})();
}, [refresh, userConfig.show_subed_only, currentPage, pagination?.current_page]);
}, [
refresh,
userConfig.show_subed_only,
currentPage,
pagination?.current_page,
viewStyle,
sortBy,
sortOrder,
]);
return (
<>
@ -194,11 +229,28 @@ const Channels = () => {
data-value="list"
alt="list view"
/>
<img
src={iconTableView}
onClick={() => {
handleUserConfigUpdate({
view_style_channel: ViewStylesEnum.Table as ViewStylesType,
});
}}
data-origin="channel"
data-value="table"
alt="table view"
/>
</div>
</div>
<div className={`channel-list ${userConfig.view_style_channel}`}>
<ChannelList channelList={channels} refreshChannelList={setRefresh} />
<ChannelList
channelList={channels}
refreshChannelList={setRefresh}
sortBy={sortBy}
sortOrder={sortOrder}
onSort={handleSort}
/>
</div>
{pagination && (

50
frontend/src/style.css
View File

@ -1080,6 +1080,10 @@ video:-webkit-full-screen {
gap: 1rem;
}
.channel-list.table {
display: block;
}
.channel-item.list {
padding-bottom: 1rem;
}
@ -1088,6 +1092,52 @@ video:-webkit-full-screen {
display: block;
}
.channel-item.table {
overflow-x: auto;
-webkit-overflow-scrolling: touch;
}
.channel-item.table table {
min-width: 700px;
width: 100%;
border-collapse: collapse;
position: relative;
}
.channel-item.table th,
.channel-item.table td {
padding: 8px 12px;
text-align: left;
white-space: nowrap;
}
.channel-item.table th {
font-weight: 600;
border-bottom: 1px solid #ddd;
}
.channel-item.table td {
border-bottom: 1px solid #eee;
}
.channel-item.table td.no-nowrap {
white-space: normal;
word-break: keep-all;
min-width: 160px;
}
.channel-item.table td img,
.channel-item.table th img {
width: 40px;
height: 40px;
}
.channel-table-title {
display: inline-flex;
align-items: center;
gap: 10px;
}
.channel-banner img {
width: 100%;
aspect-ratio: 18 / 3;