diff --git a/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml b/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml index fbdc7998..f84b1ab8 100644 --- a/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml +++ b/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml @@ -10,3 +10,5 @@ body: options: - label: I understand that this issue will be closed without comment. required: true + - label: I will resist the temptation and I will not submit this issue. If I submit this, I understand I might get blocked from this repo. + required: true diff --git a/.gitignore b/.gitignore index 812a03ba..474bdc25 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ backend/.env # JavaScript stuff node_modules + +.editorconfig diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c2f62d79..2a4425a9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,17 +16,17 @@ Welcome, and thanks for showing interest in improving Tube Archivist! --- ## Beta Testing -Be the first to help test new features and improvements and provide feedback! There are regular `:unstable` builds for easy access. That's for the tinkerers and the breave. Ideally use a testing environment first, before a release be the first to install it on your main system. +Be the first to help test new features/improvements and provide feedback! Regular `:unstable` builds are available for early access. These are for the tinkerers and the brave. Ideally, use a testing environment first, before upgrading your main installation. -There is always something that can get missed during development. Look at the commit messages tagged with `#build`, these are the unstable builds and give a quick overview what has changed. +There is always something that can get missed during development. Look at the commit messages tagged with `#build` - these are the unstable builds and give a quick overview of what has changed. - Test the features mentioned, play around, try to break it. -- Test the update path by installing the `:latest` release first, the upgrade to `:unstable` to check for any errors. +- Test the update path by installing the `:latest` release first, then upgrade to `:unstable` to check for any errors. - Test the unstable build on a fresh install. -Then provide feedback, if there is a problem but also if there is no problem. Reach out on [Discord](https://tubearchivist.com/discord) in the `#beta-testing` channel with your findings. +Then provide feedback - even if you don't encounter any issues! You can do this in the `#beta-testing` channel on the [Discord](https://tubearchivist.com/discord) Discord server. -This will help with a smooth update for the regular release. Plus you get to test things out early! +This helps ensure a smooth update for the stable release. Plus you get to test things out early! ## How to open an issue Please read this carefully before opening any [issue](https://github.com/tubearchivist/tubearchivist/issues) on GitHub. diff --git a/Dockerfile b/Dockerfile index b8260679..c4319f71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,24 @@ # multi stage to build tube archivist # build python wheel, download and extract ffmpeg, copy into final image +FROM node:lts-alpine AS npm-builder +COPY frontend/package.json frontend/package-lock.json / +RUN npm i + FROM node:lts-alpine AS node-builder # RUN npm config set registry https://registry.npmjs.org/ +COPY --from=npm-builder ./node_modules /frontend/node_modules COPY ./frontend /frontend - WORKDIR /frontend -RUN npm i + RUN npm run build:deploy WORKDIR / # First stage to build python wheel -FROM python:3.11.8-slim-bookworm AS builder +FROM python:3.11.13-slim-bookworm AS builder RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential gcc libldap2-dev libsasl2-dev libssl-dev git @@ -24,7 +28,7 @@ COPY ./backend/requirements.txt /requirements.txt RUN pip install --user -r requirements.txt # build ffmpeg -FROM python:3.11.8-slim-bookworm AS ffmpeg-builder +FROM python:3.11.13-slim-bookworm AS ffmpeg-builder ARG TARGETPLATFORM @@ -32,7 +36,7 @@ COPY docker_assets/ffmpeg_download.py ffmpeg_download.py RUN python ffmpeg_download.py $TARGETPLATFORM # build final image -FROM python:3.11.8-slim-bookworm AS tubearchivist +FROM python:3.11.13-slim-bookworm AS tubearchivist ARG INSTALL_DEBUG @@ -54,9 +58,9 @@ RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recomm # install debug tools for testing environment RUN if [ "$INSTALL_DEBUG" ] ; then \ - apt-get -y update && apt-get -y install --no-install-recommends \ - vim htop bmon net-tools iputils-ping procps lsof \ - && pip install --user ipython pytest pytest-django \ + apt-get -y update && apt-get -y install --no-install-recommends \ + vim htop bmon net-tools iputils-ping procps lsof \ + && pip install --user ipython pytest pytest-django \ ; fi # make folders diff --git a/README.md b/README.md index 7d3b2a96..9c50cd8c 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ All environment variables are explained in detail in the docs [here](https://doc | ELASTIC_USER | Change the default ElasticSearch user | Optional | | TA_LDAP | Configure TA to use LDAP Authentication | [Read more](https://docs.tubearchivist.com/configuration/ldap/) | | DISABLE_STATIC_AUTH | Remove authentication from media files, (Google Cast...) | [Read more](https://docs.tubearchivist.com/installation/env-vars/#disable_static_auth) | +| TA_AUTO_UPDATE_YTDLP | Configure TA to automatically install the latest yt-dlp on container start | Optional | | DJANGO_DEBUG | Return additional error messages, for debug only | Optional | | TA_LOGIN_AUTH_MODE | Configure the order of login authentication backends (Default: single) | Optional | diff --git a/backend/appsettings/index_mapping.json b/backend/appsettings/index_mapping.json index d1bbd500..72845492 100644 --- a/backend/appsettings/index_mapping.json +++ b/backend/appsettings/index_mapping.json @@ -461,6 +461,15 @@ "playlist_description": { "type": "text" }, + "playlist_subscribed": { + "type": "boolean" + }, + "playlist_type": { + "type": "keyword" + }, + "playlist_active": { + "type": "boolean" + }, "playlist_name": { "type": "text", "analyzer": "english", @@ -497,6 +506,9 @@ "type": "date", "format": "epoch_second" }, + "playlist_sort_order": { + "type": "keyword" + }, "playlist_entries": { "properties": { "downloaded": { diff --git a/backend/appsettings/serializers.py b/backend/appsettings/serializers.py index 5e6e434b..a690f37d 100644 --- a/backend/appsettings/serializers.py +++ b/backend/appsettings/serializers.py @@ -25,6 +25,7 @@ class AppConfigSubSerializer( live_channel_size = serializers.IntegerField(required=False) shorts_channel_size = serializers.IntegerField(required=False) auto_start = serializers.BooleanField(required=False) + extract_flat = serializers.BooleanField(required=False) class AppConfigDownloadsSerializer( diff --git a/backend/appsettings/src/config.py b/backend/appsettings/src/config.py index 3c14a2c0..52c69bb6 100644 --- a/backend/appsettings/src/config.py +++ b/backend/appsettings/src/config.py @@ -22,6 +22,7 @@ class SubscriptionsConfigType(TypedDict): live_channel_size: int shorts_channel_size: int auto_start: bool + extract_flat: bool class DownloadsConfigType(TypedDict): @@ -73,6 +74,7 @@ class AppConfig: "live_channel_size": 50, "shorts_channel_size": 50, "auto_start": False, + "extract_flat": False, }, "downloads": { "limit_speed": None, diff --git a/backend/appsettings/src/reindex.py b/backend/appsettings/src/reindex.py index 56fd339e..20f82425 100644 --- a/backend/appsettings/src/reindex.py +++ b/backend/appsettings/src/reindex.py @@ -11,11 +11,11 @@ from typing import Callable, TypedDict from appsettings.src.config import AppConfig from channel.src.index import YoutubeChannel +from channel.src.remote_query import get_last_channel_videos from common.src.env_settings import EnvironmentSettings from common.src.es_connect import ElasticWrap, IndexPaginate from common.src.helper import rand_sleep from common.src.ta_redis import RedisQueue -from download.src.subscriptions import ChannelSubscription from download.src.thumbnails import ThumbManager from download.src.yt_dlp_base import CookieHandler from playlist.src.index import YoutubePlaylist @@ -376,7 +376,7 @@ class Reindex(ReindexBase): channel.upload_to_es() channel.sync_to_videos() - ChannelFullScan(channel_id).scan() + ChannelFullScan(channel_id, self.config).scan() self.processed["channels"] += 1 def _reindex_single_playlist(self, playlist_id: str) -> None: @@ -493,13 +493,11 @@ class ReindexProgress(ReindexBase): class ChannelFullScan: - """ - update from v0.3.0 to v0.3.1 - full scan of channel to fix vid_type mismatch - """ + """full scan of channel to fix vid_type mismatch""" - def __init__(self, channel_id): + def __init__(self, channel_id, config): self.channel_id = channel_id + self.config = config self.to_update = False def scan(self): @@ -510,12 +508,14 @@ class ChannelFullScan: self.to_update = [] for video in all_local_videos: video_id = video["youtube_id"] - remote_match = [i for i in all_remote_videos if i[0] == video_id] + remote_match = [ + i for i in all_remote_videos if i["id"] == video_id + ] if not remote_match: print(f"{video_id}: no remote match found") continue - expected_type = remote_match[0][-1] + expected_type = remote_match[0]["vid_type"] if video["vid_type"] != expected_type: self.to_update.append( { @@ -528,9 +528,8 @@ class ChannelFullScan: def _get_all_remote(self): """get all channel videos""" - sub = ChannelSubscription() - all_remote_videos = sub.get_last_youtube_videos( - self.channel_id, limit=False + all_remote_videos = get_last_channel_videos( + self.channel_id, self.config, limit=False ) return all_remote_videos diff --git a/backend/channel/src/index.py b/backend/channel/src/index.py index 65503ef7..17bf9526 100644 --- a/backend/channel/src/index.py +++ b/backend/channel/src/index.py @@ -4,7 +4,6 @@ functionality: - index and update in es """ -import json import os from datetime import datetime @@ -121,27 +120,6 @@ class YoutubeChannel(YouTubeItem): "channel_thumb_url": False, "channel_views": 0, } - self._info_json_fallback() - - def _info_json_fallback(self): - """read channel info.json for additional metadata""" - info_json = os.path.join( - EnvironmentSettings.CACHE_DIR, - "import", - f"{self.youtube_id}.info.json", - ) - if os.path.exists(info_json): - print(f"{self.youtube_id}: read info.json file") - with open(info_json, "r", encoding="utf-8") as f: - content = json.loads(f.read()) - - self.json_data.update( - { - "channel_subs": content.get("channel_follower_count", 0), - "channel_description": content.get("description", False), - } - ) - os.remove(info_json) def get_channel_art(self): """download channel art for new channels""" @@ -177,48 +155,15 @@ class YoutubeChannel(YouTubeItem): update_path = f"ta_video/_update_by_query?pipeline={self.youtube_id}" _, _ = ElasticWrap(update_path).post(data) - def get_folder_path(self): - """get folder where media files get stored""" - folder_path = os.path.join( - EnvironmentSettings.MEDIA_DIR, - self.json_data["channel_id"], - ) - return folder_path + def change_subscribe(self, new_subscribe_state: bool): + """change subscribe status""" + if not self.json_data: + self.build_json() - def delete_es_videos(self): - """delete all channel documents from elasticsearch""" - data = { - "query": { - "term": {"channel.channel_id": {"value": self.youtube_id}} - } - } - _, _ = ElasticWrap("ta_video/_delete_by_query").post(data) - - def delete_es_comments(self): - """delete all comments from this channel""" - data = { - "query": { - "term": {"comment_channel_id": {"value": self.youtube_id}} - } - } - _, _ = ElasticWrap("ta_comment/_delete_by_query").post(data) - - def delete_es_subtitles(self): - """delete all subtitles from this channel""" - data = { - "query": { - "term": {"subtitle_channel_id": {"value": self.youtube_id}} - } - } - _, _ = ElasticWrap("ta_subtitle/_delete_by_query").post(data) - - def delete_playlists(self): - """delete all indexed playlist from es""" - from playlist.src.index import YoutubePlaylist - - all_playlists = self.get_indexed_playlists() - for playlist in all_playlists: - YoutubePlaylist(playlist["playlist_id"]).delete_metadata() + self.json_data["channel_subscribed"] = new_subscribe_state + self.upload_to_es() + self.sync_to_videos() + return self.json_data def delete_channel(self): """delete channel and all videos""" @@ -227,24 +172,7 @@ class YoutubeChannel(YouTubeItem): if not self.json_data: raise FileNotFoundError - folder_path = self.get_folder_path() - print(f"{self.youtube_id}: delete all media files") - try: - all_videos = os.listdir(folder_path) - for video in all_videos: - video_path = os.path.join(folder_path, video) - os.remove(video_path) - os.rmdir(folder_path) - except FileNotFoundError: - print(f"no videos found for {folder_path}") - - print(f"{self.youtube_id}: delete indexed playlists") - self.delete_playlists() - print(f"{self.youtube_id}: delete indexed videos") - self.delete_es_videos() - self.delete_es_comments() - self.delete_es_subtitles() - self.del_in_es() + ChannelDelete(json_data=self.json_data).delete() def index_channel_playlists(self): """add all playlists of channel to index""" @@ -266,6 +194,21 @@ class YoutubeChannel(YouTubeItem): print("add playlist: " + playlist[1]) rand_sleep(self.config) + def get_all_playlists(self): + """get all playlists owned by this channel""" + url = ( + f"https://www.youtube.com/channel/{self.youtube_id}" + + "/playlists?view=1&sort=dd&shelf_id=0" + ) + obs = {"skip_download": True, "extract_flat": True} + playlists, _ = YtWrap(obs, self.config).extract(url) + if not playlists: + self.all_playlists = [] + return + + all_entries = [(i["id"], i["title"]) for i in playlists["entries"]] + self.all_playlists = all_entries + def _notify_single_playlist(self, idx, total): """send notification""" channel_name = self.json_data["channel_name"] @@ -294,34 +237,6 @@ class YoutubeChannel(YouTubeItem): all_videos = IndexPaginate("ta_video", data).get_results() return all_videos - def get_all_playlists(self): - """get all playlists owned by this channel""" - url = ( - f"https://www.youtube.com/channel/{self.youtube_id}" - + "/playlists?view=1&sort=dd&shelf_id=0" - ) - obs = {"skip_download": True, "extract_flat": True} - playlists = YtWrap(obs, self.config).extract(url) - if not playlists: - self.all_playlists = [] - return - - all_entries = [(i["id"], i["title"]) for i in playlists["entries"]] - self.all_playlists = all_entries - - def get_indexed_playlists(self, active_only=False): - """get all indexed playlists from channel""" - must_list = [ - {"term": {"playlist_channel_id": {"value": self.youtube_id}}} - ] - if active_only: - must_list.append({"term": {"playlist_active": {"value": True}}}) - - data = {"query": {"bool": {"must": must_list}}} - - all_playlists = IndexPaginate("ta_playlist", data).get_results() - return all_playlists - def get_overwrites(self) -> dict: """get all per channel overwrites""" return self.json_data.get("channel_overwrites", {}) @@ -352,6 +267,93 @@ class YoutubeChannel(YouTubeItem): self.json_data["channel_overwrites"] = to_write +class ChannelDelete(YouTubeItem): + """delete and cleanup""" + + index_name = "ta_channel" + + def __init__(self, json_data): + super().__init__(youtube_id=json_data["channel_id"]) + self.json_data = json_data + + def delete(self): + """delete channel and all videos""" + folder_path = self._get_folder_path() + print(f"{self.youtube_id}: delete all media files") + try: + all_videos = os.listdir(folder_path) + for video in all_videos: + video_path = os.path.join(folder_path, video) + os.remove(video_path) + os.rmdir(folder_path) + except FileNotFoundError: + print(f"no videos found for {folder_path}") + + print(f"{self.youtube_id}: delete indexed playlists") + self._delete_playlists() + print(f"{self.youtube_id}: delete indexed videos") + self._delete_es_videos() + self._delete_es_comments() + self._delete_es_subtitles() + self.del_in_es() + + def _get_folder_path(self): + """get folder where media files get stored""" + folder_path = os.path.join( + EnvironmentSettings.MEDIA_DIR, + self.json_data["channel_id"], + ) + return folder_path + + def _delete_es_videos(self): + """delete all channel documents from elasticsearch""" + data = { + "query": { + "term": {"channel.channel_id": {"value": self.youtube_id}} + } + } + _, _ = ElasticWrap("ta_video/_delete_by_query").post(data) + + def _delete_es_comments(self): + """delete all comments from this channel""" + data = { + "query": { + "term": {"comment_channel_id": {"value": self.youtube_id}} + } + } + _, _ = ElasticWrap("ta_comment/_delete_by_query").post(data) + + def _delete_es_subtitles(self): + """delete all subtitles from this channel""" + data = { + "query": { + "term": {"subtitle_channel_id": {"value": self.youtube_id}} + } + } + _, _ = ElasticWrap("ta_subtitle/_delete_by_query").post(data) + + def _delete_playlists(self): + """delete all indexed playlist from es""" + from playlist.src.index import YoutubePlaylist + + all_playlists = self._get_indexed_playlists() + for playlist in all_playlists: + YoutubePlaylist(playlist["playlist_id"]).delete_metadata() + + def _get_indexed_playlists(self, active_only=False): + """get all indexed playlists from channel""" + must_list = [ + {"term": {"playlist_channel_id": {"value": self.youtube_id}}} + ] + if active_only: + must_list.append({"term": {"playlist_active": {"value": True}}}) + + data = {"query": {"bool": {"must": must_list}}} + + all_playlists = IndexPaginate("ta_playlist", data).get_results() + return all_playlists + + def channel_overwrites(channel_id, overwrites): """collection to overwrite settings per channel""" channel = YoutubeChannel(channel_id) diff --git a/backend/channel/src/remote_query.py b/backend/channel/src/remote_query.py new file mode 100644 index 00000000..a38d1cce --- /dev/null +++ b/backend/channel/src/remote_query.py @@ -0,0 +1,124 @@ +"""build queries for video extraction from channel subscriptions""" + +from download.src.yt_dlp_base import YtWrap +from video.src.constants import VideoTypeEnum + + +class VideoQueryBuilder: + """Build queries for yt-dlp.""" + + def __init__(self, config: dict, channel_overwrites: dict | None = None): + self.config = config + self.channel_overwrites = channel_overwrites or {} + + def build_queries( + self, video_type: VideoTypeEnum | None, limit: bool = True + ) -> list[tuple[VideoTypeEnum, int | None]]: + """Build queries for all or specific video type.""" + query_methods = { + VideoTypeEnum.VIDEOS: self.videos_query, + VideoTypeEnum.STREAMS: self.streams_query, + VideoTypeEnum.SHORTS: self.shorts_query, + } + + if video_type: + # build query for specific type + query_method = query_methods.get(video_type) + if query_method: + query = query_method(limit) + if query[1] != 0: + return [query] + return [] + + # Build and return queries for all video types + queries = [] + for build_query in query_methods.values(): + query = build_query(limit) + if query[1] != 0: + queries.append(query) + + return queries + + def videos_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]: + """Build query for videos.""" + return self._build_generic_query( + video_type=VideoTypeEnum.VIDEOS, + overwrite_key="subscriptions_channel_size", + config_key="channel_size", + limit=limit, + ) + + def streams_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]: + """Build query for streams.""" + return self._build_generic_query( + video_type=VideoTypeEnum.STREAMS, + overwrite_key="subscriptions_live_channel_size", + config_key="live_channel_size", + limit=limit, + ) + + def shorts_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]: + """Build query for shorts.""" + return self._build_generic_query( + video_type=VideoTypeEnum.SHORTS, + overwrite_key="subscriptions_shorts_channel_size", + config_key="shorts_channel_size", + limit=limit, + ) + + def _build_generic_query( + self, + video_type: VideoTypeEnum, + overwrite_key: str, + config_key: str, + limit: bool, + ) -> tuple[VideoTypeEnum, int | None]: + """Generic query for video page scraping.""" + if not limit: + return (video_type, None) + + if ( + overwrite_key in self.channel_overwrites + and self.channel_overwrites[overwrite_key] is not None + ): + overwrite = self.channel_overwrites[overwrite_key] + return (video_type, overwrite) + + if overwrite := self.config["subscriptions"].get(config_key): + return (video_type, overwrite) + + return (video_type, 0) + + +def get_last_channel_videos( + channel_id, + config, + limit=None, + query_filter=None, + channel_overwrites=None, +): + """get a list of last videos from channel""" + query_handler = VideoQueryBuilder(config, channel_overwrites) + queries = query_handler.build_queries(query_filter) + last_videos = [] + + for vid_type_enum, limit_amount in queries: + obs = { + "skip_download": True, + "extract_flat": True, + } + vid_type = vid_type_enum.value + + if limit is not None: + obs.update({"playlist_items": f":{limit_amount}:1"}) + + url = f"https://www.youtube.com/channel/{channel_id}/{vid_type}" + channel_query, _ = YtWrap(obs, config).extract(url) + if not channel_query: + continue + + for entry in channel_query["entries"]: + entry["vid_type"] = vid_type + last_videos.append(entry) + + return last_videos diff --git a/backend/channel/views.py b/backend/channel/views.py index f915ecec..1f0daf57 100644 --- a/backend/channel/views.py +++ b/backend/channel/views.py @@ -14,7 +14,6 @@ from channel.src.nav import ChannelNav from common.serializers import ErrorResponseSerializer from common.src.urlparser import Parser from common.views_base import AdminWriteOnly, ApiBaseView -from download.src.subscriptions import ChannelSubscription from drf_spectacular.utils import ( OpenApiParameter, OpenApiResponse, @@ -89,9 +88,7 @@ class ChannelApiListView(ApiBaseView): def _unsubscribe(channel_id: str): """unsubscribe""" print(f"[{channel_id}] unsubscribe from channel") - ChannelSubscription().change_subscribe( - channel_id, channel_subscribed=False - ) + YoutubeChannel(channel_id).change_subscribe(new_subscribe_state=False) class ChannelApiView(ApiBaseView): @@ -146,7 +143,9 @@ class ChannelApiView(ApiBaseView): subscribed = validated_data.get("channel_subscribed") if subscribed is not None: - ChannelSubscription().change_subscribe(channel_id, subscribed) + YoutubeChannel(channel_id).change_subscribe( + new_subscribe_state=subscribed + ) overwrites = validated_data.get("channel_overwrites") if overwrites: diff --git a/backend/common/src/es_connect.py b/backend/common/src/es_connect.py index f7a59e3a..b2d11635 100644 --- a/backend/common/src/es_connect.py +++ b/backend/common/src/es_connect.py @@ -56,7 +56,7 @@ class ElasticWrap: return response.json(), response.status_code def post( - self, data: bool | dict = False, ndjson: bool = False + self, data: bool | dict | str = False, ndjson: bool = False ) -> tuple[dict, int]: """post data to es""" diff --git a/backend/common/src/helper.py b/backend/common/src/helper.py index fcf75dfe..cf647a4e 100644 --- a/backend/common/src/helper.py +++ b/backend/common/src/helper.py @@ -103,8 +103,11 @@ def requests_headers() -> dict[str, str]: return {"User-Agent": template} -def date_parser(timestamp: int | str) -> str: +def date_parser(timestamp: int | str | None) -> str | None: """return formatted date string""" + if timestamp is None: + return None + if isinstance(timestamp, int): date_obj = datetime.fromtimestamp(timestamp, tz=timezone.utc) elif isinstance(timestamp, str): @@ -152,9 +155,13 @@ def is_shorts(youtube_id: str) -> bool: """check if youtube_id is a shorts video, bot not it it's not a shorts""" shorts_url = f"https://www.youtube.com/shorts/{youtube_id}" cookies = {"SOCS": "CAI"} - response = requests.head( - shorts_url, cookies=cookies, headers=requests_headers(), timeout=10 - ) + try: + response = requests.head( + shorts_url, cookies=cookies, headers=requests_headers(), timeout=10 + ) + except requests.exceptions.RequestException: + # assume video on error + return False return response.status_code == 200 @@ -184,11 +191,12 @@ def get_duration_sec(file_path: str) -> int: return duration_sec -def get_duration_str(seconds: int) -> str: +def get_duration_str(seconds: int | float) -> str: """Return a human-readable duration string from seconds.""" if not seconds: return "NA" + seconds = int(seconds) units = [("y", 31536000), ("d", 86400), ("h", 3600), ("m", 60), ("s", 1)] duration_parts = [] @@ -284,6 +292,50 @@ def get_channel_overwrites() -> dict[str, dict[str, Any]]: return overwrites +def get_channels( + subscribed_only: bool, source: list[str] | None = None +) -> list[dict]: + """get a list of all channels""" + data = { + "sort": [{"channel_name.keyword": {"order": "asc"}}], + } + if subscribed_only: + query = {"term": {"channel_subscribed": {"value": True}}} + else: + query = {"match_all": {}} + + data["query"] = query # type: ignore + + if source: + data["_source"] = source # type: ignore + + all_channels = IndexPaginate("ta_channel", data).get_results() + + return all_channels + + +def get_playlists( + subscribed_only: bool, source: list[str] | None = None +) -> list[dict]: + """get list of playlists""" + + data = { + "sort": [{"playlist_channel.keyword": {"order": "desc"}}], + } + + must_list = [{"term": {"playlist_active": {"value": True}}}] + if subscribed_only: + must_list.append({"term": {"playlist_subscribed": {"value": True}}}) + + data = {"query": {"bool": {"must": must_list}}} # type: ignore + if source: + data["_source"] = source # type: ignore + + all_playlists = IndexPaginate("ta_playlist", data).get_results() + + return all_playlists + + def calc_is_watched(duration: float, position: float) -> bool: """considered watched based on duration position""" diff --git a/backend/common/src/index_generic.py b/backend/common/src/index_generic.py index 450c1f88..354f83ea 100644 --- a/backend/common/src/index_generic.py +++ b/backend/common/src/index_generic.py @@ -26,6 +26,7 @@ class YouTubeItem: self.youtube_id = youtube_id self.es_path = f"{self.index_name}/_doc/{youtube_id}" self.config = AppConfig().config + self.error = None self.youtube_meta = False self.json_data = False @@ -33,17 +34,24 @@ class YouTubeItem: """build youtube url""" return self.yt_base + self.youtube_id - def get_from_youtube(self): + def get_from_youtube(self, obs_overwrite: dict | None = None): """use yt-dlp to get meta data from youtube""" print(f"{self.youtube_id}: get metadata from youtube") obs_request = self.yt_obs.copy() if self.config["downloads"]["extractor_lang"]: langs = self.config["downloads"]["extractor_lang"] langs_list = [i.strip() for i in langs.split(",")] - obs_request["extractor_args"] = {"youtube": {"lang": langs_list}} + obs_request["extractor_args"] = { + "youtube": {"lang": langs_list} + } # type: ignore + + if obs_overwrite: + obs_request.update(obs_overwrite) url = self.build_yt_url() - self.youtube_meta = YtWrap(obs_request, self.config).extract(url) + self.youtube_meta, self.error = YtWrap( + obs_request, self.config + ).extract(url) def get_from_es(self): """get indexed data from elastic search""" diff --git a/backend/common/src/search_processor.py b/backend/common/src/search_processor.py index 714d33b0..e5c0cac4 100644 --- a/backend/common/src/search_processor.py +++ b/backend/common/src/search_processor.py @@ -165,15 +165,17 @@ class SearchProcess: def _process_download(self, download_dict): """run on single download item""" - video_id = download_dict["youtube_id"] - cache_root = EnvironmentSettings().get_cache_root() - vid_thumb_url = ThumbManager(video_id).vid_thumb_path() - published = date_parser(download_dict["published"]) + vid_thumb_url = None + if download_dict.get("vid_thumb_url"): + video_id = download_dict["youtube_id"] + cache_root = EnvironmentSettings().get_cache_root() + relative_path = ThumbManager(video_id).vid_thumb_path() + vid_thumb_url = f"{cache_root}/{relative_path}" download_dict.update( { - "vid_thumb_url": f"{cache_root}/{vid_thumb_url}", - "published": published, + "vid_thumb_url": vid_thumb_url, + "published": date_parser(download_dict["published"]), } ) return dict(sorted(download_dict.items())) diff --git a/backend/common/src/urlparser.py b/backend/common/src/urlparser.py index 3db07ad6..c3b80690 100644 --- a/backend/common/src/urlparser.py +++ b/backend/common/src/urlparser.py @@ -4,6 +4,7 @@ Functionality: - identify vid_type if possible """ +from typing import Literal, NotRequired, TypedDict from urllib.parse import parse_qs, urlparse from common.src.ta_redis import RedisArchivist @@ -11,19 +12,28 @@ from download.src.yt_dlp_base import YtWrap from video.src.constants import VideoTypeEnum +class ParsedURLType(TypedDict): + """represents single parsed url""" + + type: Literal["video", "channel", "playlist"] + url: str + vid_type: VideoTypeEnum + limit: NotRequired[int | None] + + class Parser: """ take a multi line string and detect valid youtube ids channel handle lookup is cached, can be disabled for unittests """ - def __init__(self, url_str, use_cache=True): + def __init__(self, url_str: str, use_cache: bool = True): self.url_list = [i.strip() for i in url_str.split()] self.use_cache = use_cache - def parse(self): + def parse(self) -> list[ParsedURLType]: """parse the list""" - ids = [] + ids: list[ParsedURLType] = [] for url in self.url_list: parsed = urlparse(url) if parsed.netloc: @@ -124,9 +134,9 @@ class Parser: "extract_flat": True, "playlistend": 0, } - url_info = YtWrap(obs_request).extract(url) + url_info, error = YtWrap(obs_request).extract(url) if not url_info: - raise ValueError(f"failed to retrieve content from URL: {url}") + raise ValueError(f"failed to retrieve URL: {error}") channel_id = url_info.get("channel_id", False) if channel_id: diff --git a/backend/common/src/watched.py b/backend/common/src/watched.py index 5cdef464..27e135c2 100644 --- a/backend/common/src/watched.py +++ b/backend/common/src/watched.py @@ -28,6 +28,11 @@ class WatchState: self.change_vid_state() return + if url_type == "channel": + self.reset_channel_progress() + if url_type == "playlist": + self.reset_playlist_progress() + self._add_pipeline() path = f"ta_video/_update_by_query?pipeline=watch_{self.youtube_id}" data = self._build_update_data(url_type) @@ -53,6 +58,31 @@ class WatchState: print(response) raise ValueError("failed to mark video as watched") + def reset_channel_progress(self): + """reset channel progress positions""" + from channel.src.index import YoutubeChannel + + videos = YoutubeChannel(self.youtube_id).get_channel_videos() + video_ids = [i["youtube_id"] for i in videos] + self._reset_list(video_ids) + + def reset_playlist_progress(self): + """reset playlist progress positions""" + from playlist.src.index import YoutubePlaylist + + videos = YoutubePlaylist(self.youtube_id).get_playlist_videos() + video_ids = [i["youtube_id"] for i in videos] + self._reset_list(video_ids) + + def _reset_list(self, video_ids: list[str]): + """reset list of video ids""" + redis_con = RedisArchivist() + all_ids = redis_con.list_keys(f"{self.user_id}:progress") + for progress_id in all_ids: + video_id = progress_id.split(":")[-1] + if video_id in video_ids: + redis_con.del_message(progress_id) + def _build_update_data(self, url_type): """build update by query data based on url_type""" term_key_map = { diff --git a/backend/config/management/commands/ta_startup.py b/backend/config/management/commands/ta_startup.py index 980354b1..4b75f05b 100644 --- a/backend/config/management/commands/ta_startup.py +++ b/backend/config/management/commands/ta_startup.py @@ -56,6 +56,7 @@ class Command(BaseCommand): self._mig_channel_tags() self._mig_video_channel_tags() self._mig_fix_download_channel_indexed() + self._mig_add_default_playlist_sort() def _make_folders(self): """make expected cache folders""" @@ -389,3 +390,37 @@ class Command(BaseCommand): self.stdout.write(response) sleep(60) raise CommandError(message) + + def _mig_add_default_playlist_sort(self) -> None: + """migrate from 0.5.4 to 0.5.5 set default playlist sortorder""" + self.stdout.write("[MIGRATION] set default playlist sort order") + path = "ta_playlist/_update_by_query" + data = { + "query": { + "bool": { + "must_not": [{"exists": {"field": "playlist_sort_order"}}] + } + }, + "script": { + "source": "ctx._source.playlist_sort_order = 'top'", + "lang": "painless", + }, + } + response, status_code = ElasticWrap(path).post(data) + if status_code in [200, 201]: + updated = response.get("updated") + if updated: + self.stdout.write( + self.style.SUCCESS(f" ✓ updated {updated} playlists") + ) + else: + self.stdout.write( + self.style.SUCCESS(" no playlists need updating") + ) + return + + message = " 🗙 failed to set default playlist sort order" + self.stdout.write(self.style.ERROR(message)) + self.stdout.write(response) + sleep(60) + raise CommandError(message) diff --git a/backend/config/settings.py b/backend/config/settings.py index b2d9318f..abcc4aa8 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -242,7 +242,9 @@ if bool(environ.get("TA_ENABLE_AUTH_PROXY")): # Configure Authentication Backend Combinations _login_auth_mode = (environ.get("TA_LOGIN_AUTH_MODE") or "single").casefold() if _login_auth_mode == "local": - AUTHENTICATION_BACKENDS = ("django.contrib.auth.backends.ModelBackend",) + AUTHENTICATION_BACKENDS: tuple = ( + "django.contrib.auth.backends.ModelBackend", + ) elif _login_auth_mode == "ldap": AUTHENTICATION_BACKENDS = ("django_auth_ldap.backend.LDAPBackend",) elif _login_auth_mode == "forwardauth": @@ -314,7 +316,7 @@ CORS_ALLOW_HEADERS = list(default_headers) + [ # TA application settings TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist" -TA_VERSION = "v0.5.3-unstable" +TA_VERSION = "v0.5.5-unstable" # API REST_FRAMEWORK = { diff --git a/backend/download/serializers.py b/backend/download/serializers.py index 4ae1b180..b838b69c 100644 --- a/backend/download/serializers.py +++ b/backend/download/serializers.py @@ -15,11 +15,11 @@ class DownloadItemSerializer(serializers.Serializer): channel_indexed = serializers.BooleanField() channel_name = serializers.CharField() duration = serializers.CharField() - published = serializers.CharField() + published = serializers.CharField(allow_null=True) status = serializers.ChoiceField(choices=["pending", "ignore"]) - timestamp = serializers.IntegerField() + timestamp = serializers.IntegerField(allow_null=True) title = serializers.CharField() - vid_thumb_url = serializers.CharField() + vid_thumb_url = serializers.CharField(allow_null=True) vid_type = serializers.ChoiceField(choices=VideoTypeEnum.values()) youtube_id = serializers.CharField() message = serializers.CharField(required=False) @@ -42,14 +42,23 @@ class DownloadListQuerySerializer( filter = serializers.ChoiceField( choices=["pending", "ignore"], required=False ) + vid_type = serializers.ChoiceField( + choices=VideoTypeEnum.values_known(), required=False + ) channel = serializers.CharField(required=False, help_text="channel ID") page = serializers.IntegerField(required=False) + q = serializers.CharField(required=False, help_text="Search Query") + error = serializers.BooleanField(required=False, allow_null=True) class DownloadListQueueDeleteQuerySerializer(serializers.Serializer): """serialize bulk delete download queue query string""" filter = serializers.ChoiceField(choices=["pending", "ignore"]) + channel = serializers.CharField(required=False, help_text="channel ID") + vid_type = serializers.ChoiceField( + choices=VideoTypeEnum.values_known(), required=False + ) class AddDownloadItemSerializer(serializers.Serializer): @@ -69,6 +78,23 @@ class AddToDownloadQuerySerializer(serializers.Serializer): """add to queue query serializer""" autostart = serializers.BooleanField(required=False) + flat = serializers.BooleanField(required=False) + + +class BulkUpdateDowloadQuerySerializer(serializers.Serializer): + """serialize bulk update query""" + + filter = serializers.ChoiceField(choices=["pending", "ignore", "priority"]) + channel = serializers.CharField(required=False) + vid_type = serializers.ChoiceField( + choices=VideoTypeEnum.values_known(), required=False + ) + + +class BulkUpdateDowloadDataSerializer(serializers.Serializer): + """serialize data""" + + status = serializers.ChoiceField(choices=["pending", "ignore", "priority"]) class DownloadQueueItemUpdateSerializer(serializers.Serializer): diff --git a/backend/download/src/queue.py b/backend/download/src/queue.py index 7110ceaf..a35b985b 100644 --- a/backend/download/src/queue.py +++ b/backend/download/src/queue.py @@ -4,16 +4,25 @@ Functionality: - linked with ta_dowload index """ +import json from datetime import datetime from appsettings.src.config import AppConfig +from channel.src.index import YoutubeChannel +from channel.src.remote_query import get_last_channel_videos from common.src.es_connect import ElasticWrap, IndexPaginate -from common.src.helper import get_duration_str, is_shorts, rand_sleep -from download.src.subscriptions import ChannelSubscription +from common.src.helper import ( + get_channels, + get_duration_str, + is_shorts, + rand_sleep, +) +from common.src.urlparser import ParsedURLType +from download.src.queue_interact import PendingInteract from download.src.thumbnails import ThumbManager -from download.src.yt_dlp_base import YtWrap from playlist.src.index import YoutubePlaylist from video.src.constants import VideoTypeEnum +from video.src.index import YoutubeVideo class PendingIndex: @@ -61,11 +70,7 @@ class PendingIndex: """get a list of all channels indexed""" self.all_channels = [] self.channel_overwrites = {} - data = { - "query": {"match_all": {}}, - "sort": [{"channel_id": {"order": "asc"}}], - } - channels = IndexPaginate("ta_channel", data).get_results() + channels = get_channels(subscribed_only=False) for channel in channels: channel_id = channel["channel_id"] @@ -88,67 +93,6 @@ class PendingIndex: self.video_overwrites.update({video_id: overwrites}) -class PendingInteract: - """interact with items in download queue""" - - def __init__(self, youtube_id=False, status=False): - self.youtube_id = youtube_id - self.status = status - - def delete_item(self): - """delete single item from pending""" - path = f"ta_download/_doc/{self.youtube_id}" - _, _ = ElasticWrap(path).delete(refresh=True) - - def delete_by_status(self): - """delete all matching item by status""" - data = {"query": {"term": {"status": {"value": self.status}}}} - path = "ta_download/_delete_by_query" - _, _ = ElasticWrap(path).post(data=data) - - def update_status(self): - """update status of pending item""" - if self.status == "priority": - data = { - "doc": { - "status": "pending", - "auto_start": True, - "message": None, - } - } - else: - data = {"doc": {"status": self.status}} - - path = f"ta_download/_update/{self.youtube_id}/?refresh=true" - _, _ = ElasticWrap(path).post(data=data) - - def get_item(self): - """return pending item dict""" - path = f"ta_download/_doc/{self.youtube_id}" - response, status_code = ElasticWrap(path).get() - return response["_source"], status_code - - def get_channel(self): - """ - get channel metadata from queue to not depend on channel to be indexed - """ - data = { - "size": 1, - "query": {"term": {"channel_id": {"value": self.youtube_id}}}, - } - response, _ = ElasticWrap("ta_download/_search").get(data=data) - hits = response["hits"]["hits"] - if not hits: - channel_name = "NA" - else: - channel_name = hits[0]["_source"].get("channel_name", "NA") - - return { - "channel_id": self.youtube_id, - "channel_name": channel_name, - } - - class PendingList(PendingIndex): """manage the pending videos list""" @@ -159,126 +103,365 @@ class PendingList(PendingIndex): "check_formats": None, } - def __init__(self, youtube_ids=False, task=False): + def __init__( + self, + youtube_ids: list[ParsedURLType], + task=None, + auto_start=False, + flat=False, + ): super().__init__() self.config = AppConfig().config self.youtube_ids = youtube_ids self.task = task + self.auto_start = auto_start + self.flat = flat self.to_skip = False - self.missing_videos = False + self.missing_videos: list[dict] = [] - def parse_url_list(self, auto_start=False): + def parse_url_list(self): """extract youtube ids from list""" - self.missing_videos = [] self.get_download() self.get_indexed() + self.get_channels() total = len(self.youtube_ids) for idx, entry in enumerate(self.youtube_ids): - self._process_entry(entry, auto_start=auto_start) - if not self.task: - continue + if self.task: + self.task.send_progress( + message_lines=[f"Extracting items {idx + 1}/{total}"], + progress=(idx + 1) / total, + ) - self.task.send_progress( - message_lines=[f"Extracting items {idx + 1}/{total}"], - progress=(idx + 1) / total, - ) + self._process_entry(entry, idx, total) + if self.task and self.task.is_stopped(): + break - def _process_entry(self, entry, auto_start=False): + rand_sleep(self.config) + + def _process_entry(self, entry: dict, idx_url: int, total_url: int): """process single entry from url list""" - vid_type = self._get_vid_type(entry) if entry["type"] == "video": - self._add_video(entry["url"], vid_type, auto_start=auto_start) + self._add_video( + entry["url"], entry["vid_type"], idx_url, total_url + ) elif entry["type"] == "channel": - self._parse_channel(entry["url"], vid_type) + self._parse_channel(entry) elif entry["type"] == "playlist": - self._parse_playlist(entry["url"]) + self._parse_playlist(entry["url"], entry.get("limit")) else: raise ValueError(f"invalid url_type: {entry}") - @staticmethod - def _get_vid_type(entry): - """add vid type enum if available""" - vid_type_str = entry.get("vid_type") - if not vid_type_str: - return VideoTypeEnum.UNKNOWN - - return VideoTypeEnum(vid_type_str) - - def _add_video(self, url, vid_type, auto_start=False): + def _add_video(self, url, vid_type, idx, total): """add video to list""" - if auto_start and url in set( + if self.auto_start and url in set( i["youtube_id"] for i in self.all_pending ): PendingInteract(youtube_id=url, status="priority").update_status() return - if url not in self.missing_videos and url not in self.to_skip: - self.missing_videos.append((url, vid_type)) - else: + if url in self.missing_videos or url in self.to_skip: print(f"{url}: skipped adding already indexed video to download.") + else: + to_add = self._parse_video( + url, + vid_type, + url_type="video", + idx=idx, + total=total, + ) + if to_add: + self.missing_videos.append(to_add) - def _parse_channel(self, url, vid_type): - """add all videos of channel to list""" - video_results = ChannelSubscription().get_last_youtube_videos( - url, limit=False, query_filter=vid_type + def _parse_channel(self, entry): + """parse channel""" + url = entry["url"] + vid_type = entry["vid_type"] + if isinstance(vid_type, str): + # lookup enum + vid_type = getattr(VideoTypeEnum, vid_type.upper()) + + limit = entry.get("limit") + video_results = get_last_channel_videos( + channel_id=url, + config=self.config, + limit=limit, + query_filter=vid_type, ) - for video_id, _, vid_type in video_results: - self._add_video(video_id, vid_type) - - def _parse_playlist(self, url): - """add all videos of playlist to list""" - playlist = YoutubePlaylist(url) - is_active = playlist.update_playlist() - if not is_active: - message = f"{playlist.youtube_id}: failed to extract metadata" - print(message) - raise ValueError(message) - - entries = playlist.json_data["playlist_entries"] - to_add = [i["youtube_id"] for i in entries if not i["downloaded"]] - if not to_add: + if not video_results: + print(f"{url}: no videos to add from channel, skipping") return - for video_id in to_add: - # match vid_type later - self._add_video(video_id, VideoTypeEnum.UNKNOWN) + channel_handler = YoutubeChannel(url) + channel_handler.build_json(upload=False) + if not channel_handler.json_data: + print(f"{url}: channel metadata extraction failed, skipping") + return - def add_to_pending(self, status="pending", auto_start=False): - """add missing videos to pending list""" - self.get_channels() + total = len(video_results) + for idx, video_data in enumerate(video_results): + to_add = self.__parse_channel_video( + video_data, vid_type, idx, total, channel_handler + ) + if to_add: + self.missing_videos.append(to_add) - total = len(self.missing_videos) - videos_added = [] - for idx, (youtube_id, vid_type) in enumerate(self.missing_videos): if self.task and self.task.is_stopped(): break - print(f"{youtube_id}: [{idx + 1}/{total}]: add to queue") - self._notify_add(idx, total) - video_details = self.get_youtube_details(youtube_id, vid_type) - if not video_details: - rand_sleep(self.config) - continue + def __parse_channel_video( + self, video_data, vid_type, idx, total, channel_handler + ) -> dict | None: + """parse video of channel""" + video_id = video_data["id"] + if video_id in self.to_skip: + return None - video_details.update( - { - "status": status, - "auto_start": auto_start, - } + if self.flat: + if not video_data.get("channel"): + channel_name = channel_handler.json_data["channel_name"] + video_data["channel"] = channel_name + + if not video_data.get("channel_id"): + channel_id = channel_handler.json_data["channel_id"] + video_data["channel_id"] = channel_id + + to_add = self._parse_entry( + youtube_id=video_id, + video_data=video_data, + url_type="channel", + idx=idx, + total=total, + ) + else: + to_add = self._parse_video( + video_id, + vid_type, + url_type="channel", + idx=idx, + total=total, ) - url = video_details["vid_thumb_url"] - ThumbManager(youtube_id).download_video_thumb(url) - es_url = f"ta_download/_doc/{youtube_id}" - _, _ = ElasticWrap(es_url).put(video_details) - videos_added.append(youtube_id) + return to_add - if idx != total: - rand_sleep(self.config) + def _parse_playlist(self, url: str, limit: int | None): + """fast parse playlist""" + playlist = YoutubePlaylist(url) + playlist.update_playlist(limit=limit) + if not playlist.youtube_meta: + print(f"{url}: playlist metadata extraction failed, skipping") + return - return videos_added + video_results = playlist.youtube_meta["entries"] - def _notify_add(self, idx, total): + total = len(video_results) + for idx, video_data in enumerate(video_results): + video_id = video_data["id"] + if video_id in self.to_skip: + continue + + if self.task and self.task.is_stopped(): + break + + if self.flat: + if not video_data.get("channel"): + video_data["channel"] = playlist.youtube_meta["channel"] + + if not video_data.get("channel_id"): + channel_id = playlist.youtube_meta["channel_id"] + video_data["channel_id"] = channel_id + + to_add = self._parse_entry( + video_id, + video_data, + url_type="playlist", + idx=idx, + total=total, + ) + else: + to_add = self._parse_video( + video_id, + vid_type=None, + url_type="playlist", + idx=idx, + total=total, + ) + + if to_add: + self.missing_videos.append(to_add) + + def _parse_video(self, url, vid_type, url_type, idx, total): + """parse video""" + video = YoutubeVideo(youtube_id=url) + video.get_from_youtube() + + if not video.youtube_meta: + print(f"{url}: video metadata extraction failed, skipping") + if self.task: + self.task.send_progress( + message_lines=[ + "Video extraction failed.", + f"{video.error}", + ], + level="error", + ) + return None + + video.youtube_meta["vid_type"] = vid_type + to_add = self._parse_entry( + youtube_id=url, + video_data=video.youtube_meta, + url_type=url_type, + idx=idx, + total=total, + ) + ThumbManager(item_id=url).download_video_thumb(to_add["vid_thumb_url"]) + rand_sleep(self.config) + + return to_add + + def _parse_entry( + self, + youtube_id: str, + video_data: dict, + url_type: str, + idx: int, + total: int, + ) -> dict | None: + """parse entry""" + if video_data.get("id") != youtube_id: + # skip premium videos with different id or redirects + print(f"{youtube_id}: skipping redirect, id not matching") + return None + + if video_data.get("live_status") in ["is_upcoming", "is_live"]: + print(f"{youtube_id}: skip is_upcoming or is_live") + return None + + to_add = { + "youtube_id": video_data["id"], + "title": video_data["title"], + "vid_thumb_url": self.__extract_thumb(video_data), + "duration": get_duration_str(video_data.get("duration", 0)), + "published": self.__extract_published(video_data), + "timestamp": int(datetime.now().timestamp()), + "vid_type": self.__extract_vid_type(video_data), + "channel_name": video_data["channel"], + "channel_id": video_data["channel_id"], + "channel_indexed": video_data["channel_id"] in self.all_channels, + } + + self._notify_progress(url_type, video_data["title"], idx, total) + + return to_add + + def __extract_thumb(self, video_data) -> str | None: + """extract thumb""" + if "thumbnail" in video_data: + return video_data["thumbnail"] + + return None + + def __extract_published(self, video_data) -> str | int | None: + """build published date or timestamp""" + timestamp = video_data.get("timestamp") + if timestamp: + return timestamp + + upload_date = video_data.get("upload_date") + if not upload_date: + return None + + upload_date_time = datetime.strptime(upload_date, "%Y%m%d") + published = upload_date_time.strftime("%Y-%m-%d") + + return published + + def __extract_vid_type(self, video_data) -> str: + """build vid type""" + if "vid_type" in video_data: + return video_data["vid_type"] + + if video_data.get("live_status") == "was_live": + return VideoTypeEnum.STREAMS.value + + if video_data.get("width", 0) > video_data.get("height", 0): + return VideoTypeEnum.VIDEOS.value + + duration = video_data.get("duration") + if duration and isinstance(duration, int): + if duration > 3 * 60: + return VideoTypeEnum.VIDEOS.value + + if is_shorts(video_data["id"]): + return VideoTypeEnum.SHORTS.value + + return VideoTypeEnum.VIDEOS.value + + def add_to_pending(self, status="pending") -> int: + """add missing videos to pending list""" + + total = len(self.missing_videos) + + if not self.missing_videos: + self._notify_empty() + return 0 + + self._notify_start(total) + bulk_list = [] + for video_entry in self.missing_videos: + video_entry.update( + { + "status": status, + "auto_start": self.auto_start, + } + ) + video_id = video_entry["youtube_id"] + action = {"index": {"_index": "ta_download", "_id": video_id}} + bulk_list.append(json.dumps(action)) + bulk_list.append(json.dumps(video_entry)) + + # add last newline + bulk_list.append("\n") + query_str = "\n".join(bulk_list) + _, status_code = ElasticWrap("_bulk").post(query_str, ndjson=True) + if status_code != 200: + self._notify_fail(status_code) + else: + self._notify_done(total) + + return len(self.missing_videos) + + def _notify_progress(self, url_type, name, idx, total): + """notify extraction progress""" + if not self.task: + return + + if self.flat: + second_line = f"Bulk processing {total} items." + else: + second_line = f"Processing video {idx + 1}/{total}." + + self.task.send_progress( + message_lines=[ + f"Extracting '{name}' from {url_type.title()}.", + second_line, + ], + progress=(idx + 1) / total, + ) + + def _notify_empty(self): + """notify nothing to add""" + if not self.task: + return + + self.task.send_progress( + message_lines=[ + "Extracting videos completed.", + "No new videos found to add.", + ] + ) + + def _notify_start(self, total): """send notification for adding videos to download queue""" if not self.task: return @@ -286,82 +469,31 @@ class PendingList(PendingIndex): self.task.send_progress( message_lines=[ "Adding new videos to download queue.", - f"Extracting items {idx + 1}/{total}", - ], - progress=(idx + 1) / total, + f"Bulk adding {total} videos", + ] ) - def get_youtube_details(self, youtube_id, vid_type=VideoTypeEnum.VIDEOS): - """get details from youtubedl for single pending video""" - vid = YtWrap(self.yt_obs, self.config).extract(youtube_id) - if not vid: - return False + def _notify_done(self, total): + """send done notification""" + if not self.task: + return - if vid.get("id") != youtube_id: - # skip premium videos with different id - print(f"{youtube_id}: skipping premium video, id not matching") - return False - # stop if video is streaming live now - if vid["live_status"] in ["is_upcoming", "is_live"]: - print(f"{youtube_id}: skip is_upcoming or is_live") - return False + self.task.send_progress( + message_lines=[ + "Adding new videos to the queue completed.", + f"Added {total} videos.", + ] + ) - if vid["live_status"] == "was_live": - vid_type = VideoTypeEnum.STREAMS - else: - if self._check_shorts(vid): - vid_type = VideoTypeEnum.SHORTS - else: - vid_type = VideoTypeEnum.VIDEOS + def _notify_fail(self, status_code): + """failed to add""" + if not self.task: + return - if not vid.get("channel"): - print(f"{youtube_id}: skip video not part of channel") - return False - - return self._parse_youtube_details(vid, vid_type) - - @staticmethod - def _check_shorts(vid): - """check if vid is shorts video""" - if vid["width"] > vid["height"]: - return False - - duration = vid.get("duration") - if duration and isinstance(duration, int): - if duration > 3 * 60: - return False - - return is_shorts(vid["id"]) - - def _parse_youtube_details(self, vid, vid_type=VideoTypeEnum.VIDEOS): - """parse response""" - vid_id = vid.get("id") - - # build dict - youtube_details = { - "youtube_id": vid_id, - "channel_name": vid["channel"], - "vid_thumb_url": vid["thumbnail"], - "title": vid["title"], - "channel_id": vid["channel_id"], - "duration": get_duration_str(vid["duration"]), - "published": self._build_published(vid), - "timestamp": int(datetime.now().timestamp()), - "vid_type": vid_type.value, - "channel_indexed": vid["channel_id"] in self.all_channels, - } - - return youtube_details - - @staticmethod - def _build_published(vid): - """build published date or timestamp""" - timestamp = vid["timestamp"] - if timestamp: - return timestamp - - upload_date = vid["upload_date"] - upload_date_time = datetime.strptime(upload_date, "%Y%m%d") - published = upload_date_time.strftime("%Y-%m-%d") - - return published + self.task.send_progress( + message_lines=[ + "Adding extracted videos failed.", + f"Status code: {status_code}", + ], + level="error", + ) diff --git a/backend/download/src/queue_interact.py b/backend/download/src/queue_interact.py new file mode 100644 index 00000000..260b5e5a --- /dev/null +++ b/backend/download/src/queue_interact.py @@ -0,0 +1,100 @@ +"""interact with queue items""" + +from common.src.es_connect import ElasticWrap + + +class PendingInteract: + """interact with items in download queue""" + + def __init__(self, youtube_id=False, status=False): + self.youtube_id = youtube_id + self.status = status + + def delete_item(self): + """delete single item from pending""" + path = f"ta_download/_doc/{self.youtube_id}" + _, _ = ElasticWrap(path).delete(refresh=True) + + def delete_bulk(self, channel_id: str | None, vid_type: str | None): + """delete all matching item by status""" + must_list = [{"term": {"status": {"value": self.status}}}] + if channel_id: + must_list.append({"term": {"channel_id": {"value": channel_id}}}) + + if vid_type: + must_list.append({"term": {"vid_type": {"value": vid_type}}}) + + data = {"query": {"bool": {"must": must_list}}} + + path = "ta_download/_delete_by_query?refresh=true" + _, _ = ElasticWrap(path).post(data=data) + + def update_bulk( + self, channel_id: str | None, vid_type: str | None, new_status: str + ): + """update status in bulk""" + must_list = [{"term": {"status": {"value": self.status}}}] + if channel_id: + must_list.append({"term": {"channel_id": {"value": channel_id}}}) + + if vid_type: + must_list.append({"term": {"vid_type": {"value": vid_type}}}) + + if new_status == "priority": + source = """ + ctx._source.status = 'pending'; + ctx._source.auto_start = true; + ctx._source.message = null; + """ + else: + source = f"ctx._source.status = '{new_status}'" + + data = { + "query": {"bool": {"must": must_list}}, + "script": {"source": source, "lang": "painless"}, + } + + path = "ta_download/_update_by_query?refresh=true" + _, _ = ElasticWrap(path).post(data) + + def update_status(self): + """update status of pending item""" + if self.status == "priority": + data = { + "doc": { + "status": "pending", + "auto_start": True, + "message": None, + } + } + else: + data = {"doc": {"status": self.status}} + + path = f"ta_download/_update/{self.youtube_id}/?refresh=true" + _, _ = ElasticWrap(path).post(data=data) + + def get_item(self): + """return pending item dict""" + path = f"ta_download/_doc/{self.youtube_id}" + response, status_code = ElasticWrap(path).get() + return response["_source"], status_code + + def get_channel(self): + """ + get channel metadata from queue to not depend on channel to be indexed + """ + data = { + "size": 1, + "query": {"term": {"channel_id": {"value": self.youtube_id}}}, + } + response, _ = ElasticWrap("ta_download/_search").get(data=data) + hits = response["hits"]["hits"] + if not hits: + channel_name = "NA" + else: + channel_name = hits[0]["_source"].get("channel_name", "NA") + + return { + "channel_id": self.youtube_id, + "channel_name": channel_name, + } diff --git a/backend/download/src/subscriptions.py b/backend/download/src/subscriptions.py index abc0a4eb..3c8df7dc 100644 --- a/backend/download/src/subscriptions.py +++ b/backend/download/src/subscriptions.py @@ -6,324 +6,97 @@ Functionality: from appsettings.src.config import AppConfig from channel.src.index import YoutubeChannel -from common.src.es_connect import IndexPaginate -from common.src.helper import is_missing, rand_sleep -from common.src.urlparser import Parser -from download.src.thumbnails import ThumbManager -from download.src.yt_dlp_base import YtWrap +from channel.src.remote_query import VideoQueryBuilder +from common.src.helper import get_channels, get_playlists +from common.src.urlparser import ParsedURLType, Parser +from download.src.queue import PendingList from playlist.src.index import YoutubePlaylist from video.src.constants import VideoTypeEnum from video.src.index import YoutubeVideo class ChannelSubscription: - """manage the list of channels subscribed""" + """scan subscribed channels to find missing videos to add to pending""" - def __init__(self, task=False): + def __init__(self, task=None): self.config = AppConfig().config self.task = task - @staticmethod - def get_channels(subscribed_only=True): - """get a list of all channels subscribed to""" - data = { - "sort": [{"channel_name.keyword": {"order": "asc"}}], - } - if subscribed_only: - data["query"] = {"term": {"channel_subscribed": {"value": True}}} - else: - data["query"] = {"match_all": {}} - - all_channels = IndexPaginate("ta_channel", data).get_results() - - return all_channels - - def get_last_youtube_videos( - self, - channel_id, - limit=True, - query_filter=None, - channel_overwrites=None, - ): - """get a list of last videos from channel""" - query_handler = VideoQueryBuilder(self.config, channel_overwrites) - queries = query_handler.build_queries(query_filter) - last_videos = [] - - for vid_type_enum, limit_amount in queries: - obs = { - "skip_download": True, - "extract_flat": True, - } - vid_type = vid_type_enum.value - - if limit: - obs["playlistend"] = limit_amount - - url = f"https://www.youtube.com/channel/{channel_id}/{vid_type}" - channel_query = YtWrap(obs, self.config).extract(url) - if not channel_query: - continue - - last_videos.extend( - [ - (i["id"], i["title"], vid_type) - for i in channel_query["entries"] - ] - ) - - return last_videos - - def find_missing(self): - """add missing videos from subscribed channels to pending""" - all_channels = self.get_channels() + def find_missing(self) -> int: + """find missing videos from channel subscriptions""" + all_channels = get_channels( + subscribed_only=True, source=["channel_id", "channel_overwrites"] + ) if not all_channels: - return False + return 0 - missing_videos = [] + all_channel_urls: list[ParsedURLType] = [] - total = len(all_channels) - for idx, channel in enumerate(all_channels): - channel_id = channel["channel_id"] - print(f"{channel_id}: find missing videos.") - last_videos = self.get_last_youtube_videos( - channel_id, - channel_overwrites=channel.get("channel_overwrites"), - ) + for channel in all_channels: + queries = VideoQueryBuilder( + config=self.config, + channel_overwrites=channel.get("channel_overwrites", {}), + ).build_queries(video_type=None) - if last_videos: - ids_to_add = is_missing([i[0] for i in last_videos]) - for video_id, _, vid_type in last_videos: - if video_id in ids_to_add: - missing_videos.append((video_id, vid_type)) + for query in queries: + all_channel_urls.append( + ParsedURLType( + type="channel", + url=channel["channel_id"], + vid_type=query[0], + limit=query[1], + ) + ) - if not self.task: - continue - - if self.task.is_stopped(): - self.task.send_progress(["Received Stop signal."]) - break - - self.task.send_progress( - message_lines=[f"Scanning Channel {idx + 1}/{total}"], - progress=(idx + 1) / total, - ) - rand_sleep(self.config) - - return missing_videos - - @staticmethod - def change_subscribe(channel_id, channel_subscribed): - """subscribe or unsubscribe from channel and update""" - channel = YoutubeChannel(channel_id) - channel.build_json() - channel.json_data["channel_subscribed"] = channel_subscribed - channel.upload_to_es() - channel.sync_to_videos() - - return channel.json_data - - -class VideoQueryBuilder: - """Build queries for yt-dlp.""" - - def __init__(self, config: dict, channel_overwrites: dict | None = None): - self.config = config - self.channel_overwrites = channel_overwrites or {} - - def build_queries( - self, video_type: VideoTypeEnum | None, limit: bool = True - ) -> list[tuple[VideoTypeEnum, int | None]]: - """Build queries for all or specific video type.""" - query_methods = { - VideoTypeEnum.VIDEOS: self.videos_query, - VideoTypeEnum.STREAMS: self.streams_query, - VideoTypeEnum.SHORTS: self.shorts_query, - } - - if video_type: - # build query for specific type - query_method = query_methods.get(video_type) - if query_method: - query = query_method(limit) - if query[1] != 0: - return [query] - return [] - - # Build and return queries for all video types - queries = [] - for build_query in query_methods.values(): - query = build_query(limit) - if query[1] != 0: - queries.append(query) - - return queries - - def videos_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]: - """Build query for videos.""" - return self._build_generic_query( - video_type=VideoTypeEnum.VIDEOS, - overwrite_key="subscriptions_channel_size", - config_key="channel_size", - limit=limit, + pending_handler = PendingList( + youtube_ids=all_channel_urls, + task=self.task, + auto_start=self.config["subscriptions"].get("auto_start", False), + flat=self.config["subscriptions"].get("extract_flat", False), ) + pending_handler.parse_url_list() + added = pending_handler.add_to_pending() - def streams_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]: - """Build query for streams.""" - return self._build_generic_query( - video_type=VideoTypeEnum.STREAMS, - overwrite_key="subscriptions_live_channel_size", - config_key="live_channel_size", - limit=limit, - ) - - def shorts_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]: - """Build query for shorts.""" - return self._build_generic_query( - video_type=VideoTypeEnum.SHORTS, - overwrite_key="subscriptions_shorts_channel_size", - config_key="shorts_channel_size", - limit=limit, - ) - - def _build_generic_query( - self, - video_type: VideoTypeEnum, - overwrite_key: str, - config_key: str, - limit: bool, - ) -> tuple[VideoTypeEnum, int | None]: - """Generic query for video page scraping.""" - if not limit: - return (video_type, None) - - if ( - overwrite_key in self.channel_overwrites - and self.channel_overwrites[overwrite_key] is not None - ): - overwrite = self.channel_overwrites[overwrite_key] - return (video_type, overwrite) - - if overwrite := self.config["subscriptions"].get(config_key): - return (video_type, overwrite) - - return (video_type, 0) + return added class PlaylistSubscription: - """manage the playlist download functionality""" + """scan subscribed playlists for videos to add to pending""" - def __init__(self, task=False): + def __init__(self, task=None): self.config = AppConfig().config self.task = task - @staticmethod - def get_playlists(subscribed_only=True): - """get a list of all active playlists""" - data = { - "sort": [{"playlist_channel.keyword": {"order": "desc"}}], - } - data["query"] = { - "bool": {"must": [{"term": {"playlist_active": {"value": True}}}]} - } - if subscribed_only: - data["query"]["bool"]["must"].append( - {"term": {"playlist_subscribed": {"value": True}}} - ) - - all_playlists = IndexPaginate("ta_playlist", data).get_results() - - return all_playlists - - def process_url_str(self, new_playlists, subscribed=True): - """process playlist subscribe form url_str""" - for idx, playlist in enumerate(new_playlists): - playlist_id = playlist["url"] - if not playlist["type"] == "playlist": - print(f"{playlist_id} not a playlist, skipping...") - continue - - playlist_h = YoutubePlaylist(playlist_id) - playlist_h.build_json() - if not playlist_h.json_data: - message = f"{playlist_h.youtube_id}: failed to extract data" - print(message) - raise ValueError(message) - - playlist_h.json_data["playlist_subscribed"] = subscribed - playlist_h.upload_to_es() - playlist_h.add_vids_to_playlist() - self.channel_validate(playlist_h.json_data["playlist_channel_id"]) - - url = playlist_h.json_data["playlist_thumbnail"] - thumb = ThumbManager(playlist_id, item_type="playlist") - thumb.download_playlist_thumb(url) - - if self.task: - self.task.send_progress( - message_lines=[ - f"Processing {idx + 1} of {len(new_playlists)}" - ], - progress=(idx + 1) / len(new_playlists), - ) - - @staticmethod - def channel_validate(channel_id): - """make sure channel of playlist is there""" - channel = YoutubeChannel(channel_id) - channel.build_json(upload=True) - - @staticmethod - def change_subscribe(playlist_id, subscribe_status): - """change the subscribe status of a playlist""" - playlist = YoutubePlaylist(playlist_id) - playlist.build_json() - playlist.json_data["playlist_subscribed"] = subscribe_status - playlist.upload_to_es() - return playlist.json_data - - def find_missing(self): - """find videos in subscribed playlists not downloaded yet""" - all_playlists = [i["playlist_id"] for i in self.get_playlists()] + def find_missing(self) -> int: + """find missing""" + all_playlists = get_playlists( + subscribed_only=True, source=["playlist_id"] + ) if not all_playlists: - return False + return 0 - missing_videos = [] - total = len(all_playlists) - for idx, playlist_id in enumerate(all_playlists): - playlist = YoutubePlaylist(playlist_id) - is_active = playlist.update_playlist() - if not is_active: - playlist.deactivate() - continue - - playlist_entries = playlist.json_data["playlist_entries"] - size_limit = self.config["subscriptions"]["channel_size"] - if size_limit: - del playlist_entries[size_limit:] - - to_check = [ - i["youtube_id"] - for i in playlist_entries - if i["downloaded"] is False - ] - needs_downloading = is_missing(to_check) - missing_videos.extend(needs_downloading) - - if not self.task: - continue - - if self.task.is_stopped(): - self.task.send_progress(["Received Stop signal."]) - break - - self.task.send_progress( - message_lines=[f"Scanning Playlists {idx + 1}/{total}"], - progress=(idx + 1) / total, + size_limit = self.config["subscriptions"]["channel_size"] + all_playlist_urls: list[ParsedURLType] = [] + for playlist in all_playlists: + all_playlist_urls.append( + ParsedURLType( + type="playlist", + url=playlist["playlist_id"], + vid_type=VideoTypeEnum.UNKNOWN, + limit=size_limit, + ) ) - rand_sleep(self.config) - return missing_videos + pending_handler = PendingList( + youtube_ids=all_playlist_urls, + task=self.task, + auto_start=self.config["subscriptions"].get("auto_start", False), + flat=self.config["subscriptions"].get("extract_flat", False), + ) + pending_handler.parse_url_list() + added = pending_handler.add_to_pending() + + return added class SubscriptionScanner: @@ -339,40 +112,12 @@ class SubscriptionScanner: if self.task: self.task.send_progress(["Rescanning channels and playlists."]) - self.missing_videos = [] - self.scan_channels() + added = 0 + added += ChannelSubscription(task=self.task).find_missing() if self.task and not self.task.is_stopped(): - self.scan_playlists() + added += PlaylistSubscription(task=self.task).find_missing() - return self.missing_videos - - def scan_channels(self): - """get missing from channels""" - channel_handler = ChannelSubscription(task=self.task) - missing = channel_handler.find_missing() - if not missing: - return - - for vid_id, vid_type in missing: - self.missing_videos.append( - {"type": "video", "vid_type": vid_type, "url": vid_id} - ) - - def scan_playlists(self): - """get missing from playlists""" - playlist_handler = PlaylistSubscription(task=self.task) - missing = playlist_handler.find_missing() - if not missing: - return - - for i in missing: - self.missing_videos.append( - { - "type": "video", - "vid_type": VideoTypeEnum.VIDEOS.value, - "url": i, - } - ) + return added class SubscriptionHandler: @@ -404,7 +149,8 @@ class SubscriptionHandler: f"expected {expected_type} url but got {item.get('type')}" ) - PlaylistSubscription().process_url_str([item]) + playlist = YoutubePlaylist(item["url"]) + playlist.change_subscribe(new_subscribe_state=True) return if item["type"] == "video": @@ -427,9 +173,7 @@ class SubscriptionHandler: def _subscribe(self, channel_id): """subscribe to channel""" - _ = ChannelSubscription().change_subscribe( - channel_id, channel_subscribed=True - ) + YoutubeChannel(channel_id).change_subscribe(new_subscribe_state=True) def _notify(self, idx, item, total): """send notification message to redis""" diff --git a/backend/download/src/yt_dlp_base.py b/backend/download/src/yt_dlp_base.py index c80f9d7d..68332da0 100644 --- a/backend/download/src/yt_dlp_base.py +++ b/backend/download/src/yt_dlp_base.py @@ -80,30 +80,33 @@ class YtWrap: return True, True - def extract(self, url): - """make extract request""" + def extract(self, url) -> tuple[dict | None, str | None]: + """ + make extract request + returns response, error + """ with yt_dlp.YoutubeDL(self.obs) as ydl: try: response = ydl.extract_info(url) except cookiejar.LoadError as err: print(f"cookie file is invalid: {err}") - return False + return None, str(err) except yt_dlp.utils.ExtractorError as err: print(f"{url}: failed to extract: {err}, continue...") - return False + return None, str(err) except yt_dlp.utils.DownloadError as err: if "This channel does not have a" in str(err): - return False + return None, None print(f"{url}: failed to get info from youtube: {err}") if "Temporary failure in name resolution" in str(err): raise ConnectionError("lost the internet, abort!") from err - return False + return None, str(err) self._validate_cookie() - return response + return response, None def _validate_cookie(self): """check cookie and write it back for next use""" @@ -146,7 +149,7 @@ class CookieHandler: AppConfig().update_config({"downloads": {"cookie_import": False}}) print("[cookie]: revoked") - def validate(self): + def validate(self) -> bool: """validate cookie using the liked videos playlist""" validation = RedisArchivist().get_message_dict("cookie:valid") if validation: @@ -159,8 +162,8 @@ class CookieHandler: "extract_flat": True, } validator = YtWrap(obs_request, self.config) - response = bool(validator.extract("LL")) - self.store_validation(response) + response, error = validator.extract("LL") + self.store_validation(bool(response)) # update in redis to avoid expiring modified = validator.obs["cookiefile"].getvalue().strip("\x00") @@ -173,15 +176,15 @@ class CookieHandler: "status": "message:download", "level": "error", "title": "Cookie validation failed, exiting...", - "message": "", + "message": error, } RedisArchivist().set_message( "message:download", mess_dict, expire=4 ) print("[cookie]: validation failed, exiting...") - print(f"[cookie]: validation success: {response}") - return response + print(f"[cookie]: validation success: {bool(response)}") + return bool(response) @staticmethod def store_validation(response): diff --git a/backend/download/src/yt_dlp_handler.py b/backend/download/src/yt_dlp_handler.py index 6348e3ab..f74af4eb 100644 --- a/backend/download/src/yt_dlp_handler.py +++ b/backend/download/src/yt_dlp_handler.py @@ -16,12 +16,12 @@ from common.src.env_settings import EnvironmentSettings from common.src.es_connect import ElasticWrap, IndexPaginate from common.src.helper import ( get_channel_overwrites, + get_playlists, ignore_filelist, rand_sleep, ) from common.src.ta_redis import RedisQueue from download.src.queue import PendingList -from download.src.subscriptions import PlaylistSubscription from download.src.yt_dlp_base import YtWrap from playlist.src.index import YoutubePlaylist from video.src.comments import CommentList @@ -403,8 +403,8 @@ class DownloadPostProcess(DownloaderBase): def _add_playlist_sub(self): """add subscribed playlists to refresh""" - subs = PlaylistSubscription().get_playlists() - to_add = [i["playlist_id"] for i in subs] + playlists = get_playlists(subscribed_only=True, source=["playlist_id"]) + to_add = [i["playlist_id"] for i in playlists] RedisQueue(self.PLAYLIST_QUEUE).add_list(to_add) def _add_channel_playlists(self): diff --git a/backend/download/views.py b/backend/download/views.py index 6ef1f7ff..05f3deb5 100644 --- a/backend/download/views.py +++ b/backend/download/views.py @@ -8,6 +8,8 @@ from common.views_base import AdminOnly, ApiBaseView from download.serializers import ( AddToDownloadListSerializer, AddToDownloadQuerySerializer, + BulkUpdateDowloadDataSerializer, + BulkUpdateDowloadQuerySerializer, DownloadAggsSerializer, DownloadItemSerializer, DownloadListQuerySerializer, @@ -15,7 +17,7 @@ from download.serializers import ( DownloadListSerializer, DownloadQueueItemUpdateSerializer, ) -from download.src.queue import PendingInteract +from download.src.queue_interact import PendingInteract from drf_spectacular.utils import OpenApiResponse, extend_schema from rest_framework.response import Response from task.tasks import download_pending, extrac_dl @@ -65,6 +67,22 @@ class DownloadApiListView(ApiBaseView): {"term": {"channel_id": {"value": filter_channel}}} ) + vid_type_filter = validated_data.get("vid_type") + if vid_type_filter: + must_list.append( + {"term": {"vid_type": {"value": vid_type_filter}}} + ) + + search_query = validated_data.get("q") + if search_query: + must_list.append({"match_phrase_prefix": {"title": search_query}}) + + if validated_data.get("error") is not None: + operator = "must" if validated_data["error"] else "must_not" + must_list.append( + {"bool": {operator: [{"exists": {"field": "message"}}]}} + ) + self.data["query"] = {"bool": {"must": must_list}} self.get_document_list(request) @@ -99,12 +117,13 @@ class DownloadApiListView(ApiBaseView): validated_query = query_serializer.validated_data auto_start = validated_query.get("autostart") - print(f"auto_start: {auto_start}") + flat = validated_query.get("flat", False) + print(f"auto_start: {auto_start}, flat: {flat}") to_add = validated_data["data"] pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"] url_str = " ".join(pending) - task = extrac_dl.delay(url_str, auto_start=auto_start) + task = extrac_dl.delay(url_str, auto_start=auto_start, flat=flat) message = { "message": "add to queue task started", @@ -114,6 +133,38 @@ class DownloadApiListView(ApiBaseView): return Response(response_serializer.data) + @staticmethod + @extend_schema( + request=BulkUpdateDowloadDataSerializer(), + parameters=[BulkUpdateDowloadQuerySerializer()], + responses={204: OpenApiResponse(description="Status updated")}, + ) + def patch(request): + """bulk update status""" + data_serializer = BulkUpdateDowloadDataSerializer(data=request.data) + data_serializer.is_valid(raise_exception=True) + validated_data = data_serializer.validated_data + + new_status = validated_data["status"] + + query_serializer = BulkUpdateDowloadQuerySerializer( + data=request.query_params + ) + query_serializer.is_valid(raise_exception=True) + validated_query = query_serializer.validated_data + status_filter = validated_query.get("filter") + channel = validated_query.get("channel") + vid_type = validated_query.get("vid_type") + + PendingInteract(status=status_filter).update_bulk( + channel_id=channel, vid_type=vid_type, new_status=new_status + ) + + if new_status == "priority": + download_pending.delay(auto_only=True) + + return Response(status=204) + @extend_schema( parameters=[DownloadListQueueDeleteQuerySerializer()], responses={ @@ -132,9 +183,18 @@ class DownloadApiListView(ApiBaseView): validated_query = serializer.validated_data query_filter = validated_query["filter"] + channel = validated_query.get("channel") + vid_type = validated_query.get("vid_type") message = f"delete queue by status: {query_filter}" + if channel: + message += f" - filter by channel: {channel}" + if vid_type: + message += f" - filter by vid_type: {vid_type}" + print(message) - PendingInteract(status=query_filter).delete_by_status() + PendingInteract(status=query_filter).delete_bulk( + channel_id=channel, vid_type=vid_type + ) return Response(status=204) diff --git a/backend/playlist/serializers.py b/backend/playlist/serializers.py index 9362f5b6..741c770d 100644 --- a/backend/playlist/serializers.py +++ b/backend/playlist/serializers.py @@ -28,6 +28,7 @@ class PlaylistSerializer(serializers.Serializer): playlist_last_refresh = serializers.CharField() playlist_name = serializers.CharField() playlist_subscribed = serializers.BooleanField() + playlist_sort_order = serializers.ChoiceField(choices=["top", "bottom"]) playlist_thumbnail = serializers.CharField() playlist_type = serializers.ChoiceField(choices=["regular", "custom"]) _index = serializers.CharField(required=False) @@ -68,7 +69,10 @@ class PlaylistBulkAddSerializer(serializers.Serializer): class PlaylistSingleUpdate(serializers.Serializer): """update state of single playlist""" - playlist_subscribed = serializers.BooleanField() + playlist_subscribed = serializers.BooleanField(required=False) + playlist_sort_order = serializers.ChoiceField( + choices=["top", "bottom"], required=False + ) class PlaylistListCustomPostSerializer(serializers.Serializer): diff --git a/backend/playlist/src/index.py b/backend/playlist/src/index.py index 3533205a..165860dc 100644 --- a/backend/playlist/src/index.py +++ b/backend/playlist/src/index.py @@ -30,16 +30,24 @@ class YoutubePlaylist(YouTubeItem): self.all_members = False self.nav = False - def build_json(self, scrape=False): + def build_json(self, scrape=False, limit: int | None = None): """collection to create json_data""" self.get_from_es() if self.json_data: subscribed = self.json_data.get("playlist_subscribed") + playlist_sort_order = self.json_data.get("playlist_sort_order") else: subscribed = False + playlist_sort_order = "top" + + limit_str = limit if limit is not None else "" + sort_order = 1 if playlist_sort_order == "top" else -1 + playlist_items = f":{limit_str}:{sort_order}" if scrape or not self.json_data: - self.get_from_youtube() + self.get_from_youtube( + obs_overwrite={"playlist_items": playlist_items} + ) if not self.youtube_meta: self.json_data = False return @@ -48,8 +56,13 @@ class YoutubePlaylist(YouTubeItem): self._ensure_channel() ids_found = self.get_local_vids() self.get_entries(ids_found) - self.json_data["playlist_entries"] = self.all_members - self.json_data["playlist_subscribed"] = subscribed + self.json_data.update( + { + "playlist_entries": self.all_members, + "playlist_subscribed": subscribed, + "playlist_sort_order": playlist_sort_order, + } + ) def process_youtube_meta(self): """extract relevant fields from youtube""" @@ -59,6 +72,9 @@ class YoutubePlaylist(YouTubeItem): print(f"{self.youtube_id}: thumbnail extraction failed") playlist_thumbnail = False + if not self.youtube_meta.get("channel_id"): + raise ValueError("Failed to extract Channel ID for Playlist") + self.json_data = { "playlist_id": self.youtube_id, "playlist_active": True, @@ -79,6 +95,18 @@ class YoutubePlaylist(YouTubeItem): channel_handler = YoutubeChannel(channel_id) channel_handler.build_json(upload=True) + def get_playlist_videos(self): + """get all playlist videos""" + data = { + "query": { + "term": {"playlist.keyword": {"value": self.youtube_id}} + }, + "_source": ["youtube_id"], + } + result = IndexPaginate("ta_video", data).get_results() + + return result + def get_local_vids(self) -> list[str]: """get local video ids from youtube entries""" entries = self.youtube_meta["entries"] @@ -111,6 +139,13 @@ class YoutubePlaylist(YouTubeItem): url = self.json_data["playlist_thumbnail"] ThumbManager(self.youtube_id, item_type="playlist").download(url) + def change_subscribe(self, new_subscribe_state: bool): + """change subscribe status""" + self.build_json() + self.json_data["playlist_subscribed"] = new_subscribe_state + self.upload_to_es() + return self.json_data + def add_vids_to_playlist(self): """sync the playlist id to videos""" script = ( @@ -144,14 +179,7 @@ class YoutubePlaylist(YouTubeItem): def remove_vids_from_playlist(self): """remove playlist ids from videos if needed""" needed = [i["youtube_id"] for i in self.json_data["playlist_entries"]] - data = { - "query": {"match": {"playlist": self.youtube_id}}, - "_source": ["youtube_id"], - } - data = { - "query": {"term": {"playlist.keyword": {"value": self.youtube_id}}} - } - result = IndexPaginate("ta_video", data).get_results() + result = self.get_playlist_videos() to_remove = [ i["youtube_id"] for i in result if i["youtube_id"] not in needed ] @@ -170,9 +198,9 @@ class YoutubePlaylist(YouTubeItem): if status_code == 200: print(f"{self.youtube_id}: removed {video_id} from playlist") - def update_playlist(self, skip_on_empty=False): + def update_playlist(self, skip_on_empty=False, limit: int | None = None): """update metadata for playlist with data from YouTube""" - self.build_json(scrape=True) + self.build_json(scrape=True, limit=limit) if not self.json_data: # return false to deactivate return False @@ -190,6 +218,15 @@ class YoutubePlaylist(YouTubeItem): self.get_playlist_art() return True + def change_sort_order(self, new_sort_order): + """update sort order of playlist""" + playlist = YoutubePlaylist(self.youtube_id) + playlist.build_json() + playlist.json_data["playlist_sort_order"] = new_sort_order + playlist.upload_to_es() + + return playlist.json_data + def build_nav(self, youtube_id): """find next and previous in playlist of a given youtube_id""" cache_root = EnvironmentSettings().get_cache_root() diff --git a/backend/playlist/views.py b/backend/playlist/views.py index 6ec102ff..bb756081 100644 --- a/backend/playlist/views.py +++ b/backend/playlist/views.py @@ -7,7 +7,6 @@ from common.serializers import ( ErrorResponseSerializer, ) from common.views_base import AdminWriteOnly, ApiBaseView -from download.src.subscriptions import PlaylistSubscription from drf_spectacular.utils import OpenApiResponse, extend_schema from playlist.serializers import ( PlaylistBulkAddSerializer, @@ -240,9 +239,25 @@ class PlaylistApiView(ApiBaseView): error = ErrorResponseSerializer({"error": "playlist not found"}) return Response(error.data, status=404) - subscribed = validated_data["playlist_subscribed"] - playlist_sub = PlaylistSubscription() - json_data = playlist_sub.change_subscribe(playlist_id, subscribed) + subscribed = validated_data.get("playlist_subscribed") + sort_order = validated_data.get("playlist_sort_order") + + json_data = None + if subscribed is not None: + json_data = YoutubePlaylist(playlist_id).change_subscribe( + new_subscribe_state=subscribed + ) + + if sort_order: + json_data = YoutubePlaylist(playlist_id).change_sort_order( + new_sort_order=sort_order + ) + + if not json_data: + error = ErrorResponseSerializer( + {"error": "expect playlist_subscribed or playlist_sort_order"} + ) + return Response(error.data, status=400) response_serializer = PlaylistSerializer(json_data) return Response(response_serializer.data) diff --git a/backend/requirements-dev.txt b/backend/requirements-dev.txt index a5ee0352..30696ad3 100644 --- a/backend/requirements-dev.txt +++ b/backend/requirements-dev.txt @@ -4,7 +4,7 @@ pre-commit==4.2.0 pylint-django==2.6.1 pylint==3.3.7 pytest-django==4.11.1 -pytest==8.4.0 -python-dotenv==1.1.0 +pytest==8.4.1 +python-dotenv==1.1.1 requirementscheck==0.0.6 -types-requests==2.32.0.20250602 +types-requests==2.32.4.20250611 diff --git a/backend/requirements.txt b/backend/requirements.txt index fa21380e..b939b37f 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -3,13 +3,13 @@ celery==5.5.3 django-auth-ldap==5.2.0 django-celery-beat==2.8.1 django-cors-headers==4.7.0 -Django==5.2.2 +Django==5.2.3 djangorestframework==3.16.0 drf-spectacular==0.28.0 Pillow==11.2.1 redis==6.2.0 -requests==2.32.3 +requests==2.32.4 ryd-client==0.0.6 -uvicorn==0.34.3 +uvicorn==0.35.0 whitenoise==6.9.0 -yt-dlp[default]==2025.5.22 +yt-dlp[default]==2025.6.30 diff --git a/backend/task/tasks.py b/backend/task/tasks.py index d879cb9e..63a843f0 100644 --- a/backend/task/tasks.py +++ b/backend/task/tasks.py @@ -39,10 +39,10 @@ class BaseTask(Task): RedisArchivist().set_message(key, message, expire=20) def on_success(self, retval, task_id, args, kwargs): - """callback task completed successfully""" + """callback task completed""" print(f"{task_id} success callback") message, key = self._build_message() - message.update({"messages": ["Task completed successfully"]}) + message.update({"messages": ["Task completed"]}) RedisArchivist().set_message(key, message, expire=5) def before_start(self, task_id, args, kwargs): @@ -58,9 +58,11 @@ class BaseTask(Task): task_title = TASK_CONFIG.get(self.name).get("title") Notifications(self.name).send(task_id, task_title) - def send_progress(self, message_lines, progress=False, title=False): + def send_progress( + self, message_lines, progress=False, title=False, level="info" + ): """send progress message""" - message, key = self._build_message() + message, key = self._build_message(level=level) message.update( { "messages": message_lines, @@ -101,13 +103,13 @@ def update_subscribed(self): manager.init(self) handler = SubscriptionScanner(task=self) - missing_videos = handler.scan() + added = handler.scan() auto_start = handler.auto_start - if missing_videos: - print(missing_videos) - extrac_dl.delay(missing_videos, auto_start=auto_start) - message = f"Found {len(missing_videos)} videos to add to the queue." - return message + if added: + if auto_start: + download_pending.delay(auto_only=True) + + return f"Found {added} videos to add to the queue." return None @@ -147,7 +149,9 @@ def download_pending(self, auto_only=False): @shared_task(name="extract_download", bind=True, base=BaseTask) -def extrac_dl(self, youtube_ids, auto_start=False, status="pending"): +def extrac_dl( + self, youtube_ids, auto_start=False, flat=False, status="pending" +): """parse list passed and add to pending""" TaskManager().init(self) if isinstance(youtube_ids, str): @@ -155,17 +159,17 @@ def extrac_dl(self, youtube_ids, auto_start=False, status="pending"): else: to_add = youtube_ids - pending_handler = PendingList(youtube_ids=to_add, task=self) - pending_handler.parse_url_list(auto_start=auto_start) - videos_added = pending_handler.add_to_pending( - status=status, auto_start=auto_start + pending_handler = PendingList( + youtube_ids=to_add, task=self, auto_start=auto_start, flat=flat ) + pending_handler.parse_url_list() + videos_added = pending_handler.add_to_pending(status=status) if auto_start: download_pending.delay(auto_only=True) if videos_added: - return f"added {len(videos_added)} Videos to Queue" + return f"added {videos_added} Videos to Queue" return None @@ -259,7 +263,7 @@ def rescan_filesystem(self): handler = Scanner(task=self) handler.scan() handler.apply() - ThumbValidator(task=self).validate() + thumbnail_check.delay() @shared_task(bind=True, name="thumbnail_check", base=BaseTask) diff --git a/backend/video/src/comments.py b/backend/video/src/comments.py index 2d13448c..4b145a75 100644 --- a/backend/video/src/comments.py +++ b/backend/video/src/comments.py @@ -79,7 +79,9 @@ class Comments: def get_yt_comments(self): """get comments from youtube""" yt_obs = self.build_yt_obs() - info_json = YtWrap(yt_obs, config=self.config).extract(self.youtube_id) + info_json, _ = YtWrap(yt_obs, config=self.config).extract( + self.youtube_id + ) if not info_json: return False, False diff --git a/backend/video/src/index.py b/backend/video/src/index.py index 01b8435b..34e23a8c 100644 --- a/backend/video/src/index.py +++ b/backend/video/src/index.py @@ -10,10 +10,10 @@ from datetime import datetime import requests from channel.src import index as ta_channel from common.src.env_settings import EnvironmentSettings -from common.src.es_connect import ElasticWrap from common.src.helper import get_duration_sec, get_duration_str, randomizor from common.src.index_generic import YouTubeItem from django.conf import settings +from download.src.thumbnails import ThumbManager from playlist.src import index as ta_playlist from ryd_client import ryd_client from user.src.user_config import UserConfig @@ -195,7 +195,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): def _build_published(self): """build published date or timestamp""" - timestamp = self.youtube_meta["timestamp"] + timestamp = self.youtube_meta.get("timestamp") if timestamp: return timestamp @@ -394,12 +394,6 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): return subtitles - def update_media_url(self): - """update only media_url in es for reindex channel rename""" - data = {"doc": {"media_url": self.json_data["media_url"]}} - path = f"{self.index_name}/_update/{self.youtube_id}" - _, _ = ElasticWrap(path).post(data=data) - def index_new_video(youtube_id, video_type=VideoTypeEnum.VIDEOS): """combined classes to create new video in index""" @@ -409,5 +403,8 @@ def index_new_video(youtube_id, video_type=VideoTypeEnum.VIDEOS): raise ValueError("failed to get metadata for " + youtube_id) video.check_subtitles() + url = video.json_data["vid_thumb_url"] + ThumbManager(item_id=video.youtube_id).download_video_thumb(url=url) video.upload_to_es() + return video.json_data diff --git a/backend/video/views.py b/backend/video/views.py index a679552c..ef5c011a 100644 --- a/backend/video/views.py +++ b/backend/video/views.py @@ -227,7 +227,8 @@ class VideoProgressView(ApiBaseView): expire = False current_progress.update({"watched": watched}) - redis_con.set_message(key, current_progress, expire=expire) + if position > 5: + redis_con.set_message(key, current_progress, expire=expire) response_serializer = PlayerSerializer(current_progress) diff --git a/docker-compose.yml b/docker-compose.yml index a906ce7b..5a5bc2f3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.5' - services: tubearchivist: container_name: tubearchivist diff --git a/docker_assets/run.sh b/docker_assets/run.sh index 9f065d7b..b09bee22 100644 --- a/docker_assets/run.sh +++ b/docker_assets/run.sh @@ -9,6 +9,15 @@ else LOGLEVEL="INFO" fi +# update yt-dlp if needed +if [[ "${TA_AUTO_UPDATE_YTDLP,,}" =~ ^(release|nightly)$ ]]; then + echo "Updating yt-dlp..." + preflag=$([[ "${TA_AUTO_UPDATE_YTDLP,,}" == "nightly" ]] && echo "--pre" || echo "") + python -m pip install --target=/root/.local/bin --upgrade $preflag "yt-dlp[default]" || { + echo "yt-dlp update failed" + } +fi + # stop on pending manual migration python manage.py ta_stop_on_error diff --git a/frontend/package-lock.json b/frontend/package-lock.json index a5be2f31..44cefb06 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -11,24 +11,24 @@ "dompurify": "^3.2.6", "react": "^19.1.0", "react-dom": "^19.1.0", - "react-router-dom": "^7.6.0", - "zustand": "^5.0.5" + "react-router-dom": "^7.6.3", + "zustand": "^5.0.6" }, "devDependencies": { - "@types/react": "^19.1.5", - "@types/react-dom": "^19.1.5", + "@types/react": "^19.1.8", + "@types/react-dom": "^19.1.6", "@typescript-eslint/eslint-plugin": "^8.32.1", "@typescript-eslint/parser": "^8.32.1", - "@vitejs/plugin-react-swc": "^3.10.0", - "eslint": "^9.27.0", + "@vitejs/plugin-react-swc": "^3.10.2", + "eslint": "^9.30.1", "eslint-config-prettier": "^10.1.5", "eslint-plugin-react-hooks": "^5.2.0", "eslint-plugin-react-refresh": "^0.4.20", - "globals": "^16.1.0", - "prettier": "3.5.3", + "globals": "^16.3.0", + "prettier": "3.6.2", "typescript": "^5.8.3", "typescript-eslint": "^8.32.1", - "vite": ">=6.3.5", + "vite": ">=7.0.0", "vite-plugin-checker": "^0.9.3" } }, @@ -512,9 +512,9 @@ } }, "node_modules/@eslint/config-array": { - "version": "0.20.0", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.20.0.tgz", - "integrity": "sha512-fxlS1kkIjx8+vy2SjuCB94q3htSNrufYTXubwiBFeaQHbH6Ipi43gFJq2zCMt6PHhImH3Xmr0NksKDvchWlpQQ==", + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.0.tgz", + "integrity": "sha512-ENIdc4iLu0d93HeYirvKmrzshzofPw6VkZRKQGe9Nv46ZnWUzcF1xV01dcvEg/1wXUR61OmmlSfyeyO7EvjLxQ==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -527,9 +527,9 @@ } }, "node_modules/@eslint/config-array/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, "license": "MIT", "dependencies": { @@ -551,9 +551,9 @@ } }, "node_modules/@eslint/config-helpers": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.2.1.tgz", - "integrity": "sha512-RI17tsD2frtDu/3dmI7QRrD4bedNKPM08ziRYaC5AhkGrzIAJelm9kJU1TznK+apx6V+cqRz8tfpEeG3oIyjxw==", + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.3.0.tgz", + "integrity": "sha512-ViuymvFmcJi04qdZeDc2whTHryouGcDlaxPqarTD0ZE10ISpxGUVZGZDx4w01upyIynL3iu6IXH2bS1NhclQMw==", "dev": true, "license": "Apache-2.0", "engines": { @@ -598,9 +598,9 @@ } }, "node_modules/@eslint/eslintrc/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, "license": "MIT", "dependencies": { @@ -635,9 +635,9 @@ } }, "node_modules/@eslint/js": { - "version": "9.27.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.27.0.tgz", - "integrity": "sha512-G5JD9Tu5HJEu4z2Uo4aHY2sLV64B7CDMXxFzqzjl3NKd6RVzSXNoE80jk7Y0lJkTTkjiIhBAqmlYwjuBY3tvpA==", + "version": "9.30.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.30.1.tgz", + "integrity": "sha512-zXhuECFlyep42KZUhWjfvsmXGX39W8K8LFb8AWXM9gSV9dQB+MrJGLKvW6Zw0Ggnbpw0VHTtrhFXYe3Gym18jg==", "dev": true, "license": "MIT", "engines": { @@ -776,16 +776,16 @@ } }, "node_modules/@rolldown/pluginutils": { - "version": "1.0.0-beta.9", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.9.tgz", - "integrity": "sha512-e9MeMtVWo186sgvFFJOPGy7/d2j2mZhLJIdVW0C/xDluuOvymEATqz6zKsP0ZmXGzQtqlyjz5sC1sYQUoJG98w==", + "version": "1.0.0-beta.11", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.11.tgz", + "integrity": "sha512-L/gAA/hyCSuzTF1ftlzUSI/IKr2POHsv1Dd78GfqkR83KMNuswWD61JxGV2L7nRwBBBSDr6R1gCkdTmoN7W4ag==", "dev": true, "license": "MIT" }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.39.0.tgz", - "integrity": "sha512-lGVys55Qb00Wvh8DMAocp5kIcaNzEFTmGhfFd88LfaogYTRKrdxgtlO5H6S49v2Nd8R2C6wLOal0qv6/kCkOwA==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.44.1.tgz", + "integrity": "sha512-JAcBr1+fgqx20m7Fwe1DxPUl/hPkee6jA6Pl7n1v2EFiktAHenTaXl5aIFjUIEsfn9w3HE4gK1lEgNGMzBDs1w==", "cpu": [ "arm" ], @@ -797,9 +797,9 @@ ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.39.0.tgz", - "integrity": "sha512-It9+M1zE31KWfqh/0cJLrrsCPiF72PoJjIChLX+rEcujVRCb4NLQ5QzFkzIZW8Kn8FTbvGQBY5TkKBau3S8cCQ==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.44.1.tgz", + "integrity": "sha512-RurZetXqTu4p+G0ChbnkwBuAtwAbIwJkycw1n6GvlGlBuS4u5qlr5opix8cBAYFJgaY05TWtM+LaoFggUmbZEQ==", "cpu": [ "arm64" ], @@ -811,9 +811,9 @@ ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.39.0.tgz", - "integrity": "sha512-lXQnhpFDOKDXiGxsU9/l8UEGGM65comrQuZ+lDcGUx+9YQ9dKpF3rSEGepyeR5AHZ0b5RgiligsBhWZfSSQh8Q==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.44.1.tgz", + "integrity": "sha512-fM/xPesi7g2M7chk37LOnmnSTHLG/v2ggWqKj3CCA1rMA4mm5KVBT1fNoswbo1JhPuNNZrVwpTvlCVggv8A2zg==", "cpu": [ "arm64" ], @@ -825,9 +825,9 @@ ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.39.0.tgz", - "integrity": "sha512-mKXpNZLvtEbgu6WCkNij7CGycdw9cJi2k9v0noMb++Vab12GZjFgUXD69ilAbBh034Zwn95c2PNSz9xM7KYEAQ==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.44.1.tgz", + "integrity": "sha512-gDnWk57urJrkrHQ2WVx9TSVTH7lSlU7E3AFqiko+bgjlh78aJ88/3nycMax52VIVjIm3ObXnDL2H00e/xzoipw==", "cpu": [ "x64" ], @@ -839,9 +839,9 @@ ] }, "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.39.0.tgz", - "integrity": "sha512-jivRRlh2Lod/KvDZx2zUR+I4iBfHcu2V/BA2vasUtdtTN2Uk3jfcZczLa81ESHZHPHy4ih3T/W5rPFZ/hX7RtQ==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.44.1.tgz", + "integrity": "sha512-wnFQmJ/zPThM5zEGcnDcCJeYJgtSLjh1d//WuHzhf6zT3Md1BvvhJnWoy+HECKu2bMxaIcfWiu3bJgx6z4g2XA==", "cpu": [ "arm64" ], @@ -853,9 +853,9 @@ ] }, "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.39.0.tgz", - "integrity": "sha512-8RXIWvYIRK9nO+bhVz8DwLBepcptw633gv/QT4015CpJ0Ht8punmoHU/DuEd3iw9Hr8UwUV+t+VNNuZIWYeY7Q==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.44.1.tgz", + "integrity": "sha512-uBmIxoJ4493YATvU2c0upGz87f99e3wop7TJgOA/bXMFd2SvKCI7xkxY/5k50bv7J6dw1SXT4MQBQSLn8Bb/Uw==", "cpu": [ "x64" ], @@ -867,9 +867,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.39.0.tgz", - "integrity": "sha512-mz5POx5Zu58f2xAG5RaRRhp3IZDK7zXGk5sdEDj4o96HeaXhlUwmLFzNlc4hCQi5sGdR12VDgEUqVSHer0lI9g==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.44.1.tgz", + "integrity": "sha512-n0edDmSHlXFhrlmTK7XBuwKlG5MbS7yleS1cQ9nn4kIeW+dJH+ExqNgQ0RrFRew8Y+0V/x6C5IjsHrJmiHtkxQ==", "cpu": [ "arm" ], @@ -881,9 +881,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.39.0.tgz", - "integrity": "sha512-+YDwhM6gUAyakl0CD+bMFpdmwIoRDzZYaTWV3SDRBGkMU/VpIBYXXEvkEcTagw/7VVkL2vA29zU4UVy1mP0/Yw==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.44.1.tgz", + "integrity": "sha512-8WVUPy3FtAsKSpyk21kV52HCxB+me6YkbkFHATzC2Yd3yuqHwy2lbFL4alJOLXKljoRw08Zk8/xEj89cLQ/4Nw==", "cpu": [ "arm" ], @@ -895,9 +895,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.39.0.tgz", - "integrity": "sha512-EKf7iF7aK36eEChvlgxGnk7pdJfzfQbNvGV/+l98iiMwU23MwvmV0Ty3pJ0p5WQfm3JRHOytSIqD9LB7Bq7xdQ==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.44.1.tgz", + "integrity": "sha512-yuktAOaeOgorWDeFJggjuCkMGeITfqvPgkIXhDqsfKX8J3jGyxdDZgBV/2kj/2DyPaLiX6bPdjJDTu9RB8lUPQ==", "cpu": [ "arm64" ], @@ -909,9 +909,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.39.0.tgz", - "integrity": "sha512-vYanR6MtqC7Z2SNr8gzVnzUul09Wi1kZqJaek3KcIlI/wq5Xtq4ZPIZ0Mr/st/sv/NnaPwy/D4yXg5x0B3aUUA==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.44.1.tgz", + "integrity": "sha512-W+GBM4ifET1Plw8pdVaecwUgxmiH23CfAUj32u8knq0JPFyK4weRy6H7ooxYFD19YxBulL0Ktsflg5XS7+7u9g==", "cpu": [ "arm64" ], @@ -923,9 +923,9 @@ ] }, "node_modules/@rollup/rollup-linux-loongarch64-gnu": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.39.0.tgz", - "integrity": "sha512-NMRUT40+h0FBa5fb+cpxtZoGAggRem16ocVKIv5gDB5uLDgBIwrIsXlGqYbLwW8YyO3WVTk1FkFDjMETYlDqiw==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.44.1.tgz", + "integrity": "sha512-1zqnUEMWp9WrGVuVak6jWTl4fEtrVKfZY7CvcBmUUpxAJ7WcSowPSAWIKa/0o5mBL/Ij50SIf9tuirGx63Ovew==", "cpu": [ "loong64" ], @@ -937,9 +937,9 @@ ] }, "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.39.0.tgz", - "integrity": "sha512-0pCNnmxgduJ3YRt+D+kJ6Ai/r+TaePu9ZLENl+ZDV/CdVczXl95CbIiwwswu4L+K7uOIGf6tMo2vm8uadRaICQ==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.44.1.tgz", + "integrity": "sha512-Rl3JKaRu0LHIx7ExBAAnf0JcOQetQffaw34T8vLlg9b1IhzcBgaIdnvEbbsZq9uZp3uAH+JkHd20Nwn0h9zPjA==", "cpu": [ "ppc64" ], @@ -951,9 +951,9 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.39.0.tgz", - "integrity": "sha512-t7j5Zhr7S4bBtksT73bO6c3Qa2AV/HqiGlj9+KB3gNF5upcVkx+HLgxTm8DK4OkzsOYqbdqbLKwvGMhylJCPhQ==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.44.1.tgz", + "integrity": "sha512-j5akelU3snyL6K3N/iX7otLBIl347fGwmd95U5gS/7z6T4ftK288jKq3A5lcFKcx7wwzb5rgNvAg3ZbV4BqUSw==", "cpu": [ "riscv64" ], @@ -965,9 +965,9 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-musl": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.39.0.tgz", - "integrity": "sha512-m6cwI86IvQ7M93MQ2RF5SP8tUjD39Y7rjb1qjHgYh28uAPVU8+k/xYWvxRO3/tBN2pZkSMa5RjnPuUIbrwVxeA==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.44.1.tgz", + "integrity": "sha512-ppn5llVGgrZw7yxbIm8TTvtj1EoPgYUAbfw0uDjIOzzoqlZlZrLJ/KuiE7uf5EpTpCTrNt1EdtzF0naMm0wGYg==", "cpu": [ "riscv64" ], @@ -979,9 +979,9 @@ ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.39.0.tgz", - "integrity": "sha512-iRDJd2ebMunnk2rsSBYlsptCyuINvxUfGwOUldjv5M4tpa93K8tFMeYGpNk2+Nxl+OBJnBzy2/JCscGeO507kA==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.44.1.tgz", + "integrity": "sha512-Hu6hEdix0oxtUma99jSP7xbvjkUM/ycke/AQQ4EC5g7jNRLLIwjcNwaUy95ZKBJJwg1ZowsclNnjYqzN4zwkAw==", "cpu": [ "s390x" ], @@ -993,9 +993,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.39.0.tgz", - "integrity": "sha512-t9jqYw27R6Lx0XKfEFe5vUeEJ5pF3SGIM6gTfONSMb7DuG6z6wfj2yjcoZxHg129veTqU7+wOhY6GX8wmf90dA==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.44.1.tgz", + "integrity": "sha512-EtnsrmZGomz9WxK1bR5079zee3+7a+AdFlghyd6VbAjgRJDbTANJ9dcPIPAi76uG05micpEL+gPGmAKYTschQw==", "cpu": [ "x64" ], @@ -1007,9 +1007,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.39.0.tgz", - "integrity": "sha512-ThFdkrFDP55AIsIZDKSBWEt/JcWlCzydbZHinZ0F/r1h83qbGeenCt/G/wG2O0reuENDD2tawfAj2s8VK7Bugg==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.44.1.tgz", + "integrity": "sha512-iAS4p+J1az6Usn0f8xhgL4PaU878KEtutP4hqw52I4IO6AGoyOkHCxcc4bqufv1tQLdDWFx8lR9YlwxKuv3/3g==", "cpu": [ "x64" ], @@ -1021,9 +1021,9 @@ ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.39.0.tgz", - "integrity": "sha512-jDrLm6yUtbOg2TYB3sBF3acUnAwsIksEYjLeHL+TJv9jg+TmTwdyjnDex27jqEMakNKf3RwwPahDIt7QXCSqRQ==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.44.1.tgz", + "integrity": "sha512-NtSJVKcXwcqozOl+FwI41OH3OApDyLk3kqTJgx8+gp6On9ZEt5mYhIsKNPGuaZr3p9T6NWPKGU/03Vw4CNU9qg==", "cpu": [ "arm64" ], @@ -1035,9 +1035,9 @@ ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.39.0.tgz", - "integrity": "sha512-6w9uMuza+LbLCVoNKL5FSLE7yvYkq9laSd09bwS0tMjkwXrmib/4KmoJcrKhLWHvw19mwU+33ndC69T7weNNjQ==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.44.1.tgz", + "integrity": "sha512-JYA3qvCOLXSsnTR3oiyGws1Dm0YTuxAAeaYGVlGpUsHqloPcFjPg+X0Fj2qODGLNwQOAcCiQmHub/V007kiH5A==", "cpu": [ "ia32" ], @@ -1049,9 +1049,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.39.0.tgz", - "integrity": "sha512-yAkUOkIKZlK5dl7u6dg897doBgLXmUHhIINM2c+sND3DZwnrdQkkSiDh7N75Ll4mM4dxSkYfXqU9fW3lLkMFug==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.44.1.tgz", + "integrity": "sha512-J8o22LuF0kTe7m+8PvW9wk3/bRq5+mRo5Dqo6+vXb7otCm3TPhYOJqOaQtGU9YMWQSL3krMnoOxMr0+9E6F3Ug==", "cpu": [ "x64" ], @@ -1063,15 +1063,15 @@ ] }, "node_modules/@swc/core": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core/-/core-1.11.29.tgz", - "integrity": "sha512-g4mThMIpWbNhV8G2rWp5a5/Igv8/2UFRJx2yImrLGMgrDDYZIopqZ/z0jZxDgqNA1QDx93rpwNF7jGsxVWcMlA==", + "version": "1.12.9", + "resolved": "https://registry.npmjs.org/@swc/core/-/core-1.12.9.tgz", + "integrity": "sha512-O+LfT2JlVMsIMWG9x+rdxg8GzpzeGtCZQfXV7cKc1PjIKUkLFf1QJ7okuseA4f/9vncu37dQ2ZcRrPKy0Ndd5g==", "dev": true, "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { "@swc/counter": "^0.1.3", - "@swc/types": "^0.1.21" + "@swc/types": "^0.1.23" }, "engines": { "node": ">=10" @@ -1081,16 +1081,16 @@ "url": "https://opencollective.com/swc" }, "optionalDependencies": { - "@swc/core-darwin-arm64": "1.11.29", - "@swc/core-darwin-x64": "1.11.29", - "@swc/core-linux-arm-gnueabihf": "1.11.29", - "@swc/core-linux-arm64-gnu": "1.11.29", - "@swc/core-linux-arm64-musl": "1.11.29", - "@swc/core-linux-x64-gnu": "1.11.29", - "@swc/core-linux-x64-musl": "1.11.29", - "@swc/core-win32-arm64-msvc": "1.11.29", - "@swc/core-win32-ia32-msvc": "1.11.29", - "@swc/core-win32-x64-msvc": "1.11.29" + "@swc/core-darwin-arm64": "1.12.9", + "@swc/core-darwin-x64": "1.12.9", + "@swc/core-linux-arm-gnueabihf": "1.12.9", + "@swc/core-linux-arm64-gnu": "1.12.9", + "@swc/core-linux-arm64-musl": "1.12.9", + "@swc/core-linux-x64-gnu": "1.12.9", + "@swc/core-linux-x64-musl": "1.12.9", + "@swc/core-win32-arm64-msvc": "1.12.9", + "@swc/core-win32-ia32-msvc": "1.12.9", + "@swc/core-win32-x64-msvc": "1.12.9" }, "peerDependencies": { "@swc/helpers": ">=0.5.17" @@ -1102,9 +1102,9 @@ } }, "node_modules/@swc/core-darwin-arm64": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-darwin-arm64/-/core-darwin-arm64-1.11.29.tgz", - "integrity": "sha512-whsCX7URzbuS5aET58c75Dloby3Gtj/ITk2vc4WW6pSDQKSPDuONsIcZ7B2ng8oz0K6ttbi4p3H/PNPQLJ4maQ==", + "version": "1.12.9", + "resolved": "https://registry.npmjs.org/@swc/core-darwin-arm64/-/core-darwin-arm64-1.12.9.tgz", + "integrity": "sha512-GACFEp4nD6V+TZNR2JwbMZRHB+Yyvp14FrcmB6UCUYmhuNWjkxi+CLnEvdbuiKyQYv0zA+TRpCHZ+whEs6gwfA==", "cpu": [ "arm64" ], @@ -1119,9 +1119,9 @@ } }, "node_modules/@swc/core-darwin-x64": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-darwin-x64/-/core-darwin-x64-1.11.29.tgz", - "integrity": "sha512-S3eTo/KYFk+76cWJRgX30hylN5XkSmjYtCBnM4jPLYn7L6zWYEPajsFLmruQEiTEDUg0gBEWLMNyUeghtswouw==", + "version": "1.12.9", + "resolved": "https://registry.npmjs.org/@swc/core-darwin-x64/-/core-darwin-x64-1.12.9.tgz", + "integrity": "sha512-hv2kls7Ilkm2EpeJz+I9MCil7pGS3z55ZAgZfxklEuYsxpICycxeH+RNRv4EraggN44ms+FWCjtZFu0LGg2V3g==", "cpu": [ "x64" ], @@ -1136,9 +1136,9 @@ } }, "node_modules/@swc/core-linux-arm-gnueabihf": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm-gnueabihf/-/core-linux-arm-gnueabihf-1.11.29.tgz", - "integrity": "sha512-o9gdshbzkUMG6azldHdmKklcfrcMx+a23d/2qHQHPDLUPAN+Trd+sDQUYArK5Fcm7TlpG4sczz95ghN0DMkM7g==", + "version": "1.12.9", + "resolved": "https://registry.npmjs.org/@swc/core-linux-arm-gnueabihf/-/core-linux-arm-gnueabihf-1.12.9.tgz", + "integrity": "sha512-od9tDPiG+wMU9wKtd6y3nYJdNqgDOyLdgRRcrj1/hrbHoUPOM8wZQZdwQYGarw63iLXGgsw7t5HAF9Yc51ilFA==", "cpu": [ "arm" ], @@ -1153,9 +1153,9 @@ } }, "node_modules/@swc/core-linux-arm64-gnu": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-gnu/-/core-linux-arm64-gnu-1.11.29.tgz", - "integrity": "sha512-sLoaciOgUKQF1KX9T6hPGzvhOQaJn+3DHy4LOHeXhQqvBgr+7QcZ+hl4uixPKTzxk6hy6Hb0QOvQEdBAAR1gXw==", + "version": "1.12.9", + "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-gnu/-/core-linux-arm64-gnu-1.12.9.tgz", + "integrity": "sha512-6qx1ka9LHcLzxIgn2Mros+CZLkHK2TawlXzi/h7DJeNnzi8F1Hw0Yzjp8WimxNCg6s2n+o3jnmin1oXB7gg8rw==", "cpu": [ "arm64" ], @@ -1170,9 +1170,9 @@ } }, "node_modules/@swc/core-linux-arm64-musl": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-musl/-/core-linux-arm64-musl-1.11.29.tgz", - "integrity": "sha512-PwjB10BC0N+Ce7RU/L23eYch6lXFHz7r3NFavIcwDNa/AAqywfxyxh13OeRy+P0cg7NDpWEETWspXeI4Ek8otw==", + "version": "1.12.9", + "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-musl/-/core-linux-arm64-musl-1.12.9.tgz", + "integrity": "sha512-yghFZWKPVVGbUdqiD7ft23G0JX6YFGDJPz9YbLLAwGuKZ9th3/jlWoQDAw1Naci31LQhVC+oIji6ozihSuwB2A==", "cpu": [ "arm64" ], @@ -1187,9 +1187,9 @@ } }, "node_modules/@swc/core-linux-x64-gnu": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-gnu/-/core-linux-x64-gnu-1.11.29.tgz", - "integrity": "sha512-i62vBVoPaVe9A3mc6gJG07n0/e7FVeAvdD9uzZTtGLiuIfVfIBta8EMquzvf+POLycSk79Z6lRhGPZPJPYiQaA==", + "version": "1.12.9", + "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-gnu/-/core-linux-x64-gnu-1.12.9.tgz", + "integrity": "sha512-SFUxyhWLZRNL8QmgGNqdi2Q43PNyFVkRZ2zIif30SOGFSxnxcf2JNeSeBgKIGVgaLSuk6xFVVCtJ3KIeaStgRg==", "cpu": [ "x64" ], @@ -1204,9 +1204,9 @@ } }, "node_modules/@swc/core-linux-x64-musl": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.11.29.tgz", - "integrity": "sha512-YER0XU1xqFdK0hKkfSVX1YIyCvMDI7K07GIpefPvcfyNGs38AXKhb2byySDjbVxkdl4dycaxxhRyhQ2gKSlsFQ==", + "version": "1.12.9", + "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.12.9.tgz", + "integrity": "sha512-9FB0wM+6idCGTI20YsBNBg9xSWtkDBymnpaTCsZM3qDc0l4uOpJMqbfWhQvp17x7r/ulZfb2QY8RDvQmCL6AcQ==", "cpu": [ "x64" ], @@ -1221,9 +1221,9 @@ } }, "node_modules/@swc/core-win32-arm64-msvc": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.11.29.tgz", - "integrity": "sha512-po+WHw+k9g6FAg5IJ+sMwtA/fIUL3zPQ4m/uJgONBATCVnDDkyW6dBA49uHNVtSEvjvhuD8DVWdFP847YTcITw==", + "version": "1.12.9", + "resolved": "https://registry.npmjs.org/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.12.9.tgz", + "integrity": "sha512-zHOusMVbOH9ik5RtRrMiGzLpKwxrPXgXkBm3SbUCa65HAdjV33NZ0/R9Rv1uPESALtEl2tzMYLUxYA5ECFDFhA==", "cpu": [ "arm64" ], @@ -1238,9 +1238,9 @@ } }, "node_modules/@swc/core-win32-ia32-msvc": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-win32-ia32-msvc/-/core-win32-ia32-msvc-1.11.29.tgz", - "integrity": "sha512-h+NjOrbqdRBYr5ItmStmQt6x3tnhqgwbj9YxdGPepbTDamFv7vFnhZR0YfB3jz3UKJ8H3uGJ65Zw1VsC+xpFkg==", + "version": "1.12.9", + "resolved": "https://registry.npmjs.org/@swc/core-win32-ia32-msvc/-/core-win32-ia32-msvc-1.12.9.tgz", + "integrity": "sha512-aWZf0PqE0ot7tCuhAjRkDFf41AzzSQO0x2xRfTbnhpROp57BRJ/N5eee1VULO/UA2PIJRG7GKQky5bSGBYlFug==", "cpu": [ "ia32" ], @@ -1255,9 +1255,9 @@ } }, "node_modules/@swc/core-win32-x64-msvc": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.11.29.tgz", - "integrity": "sha512-Q8cs2BDV9wqDvqobkXOYdC+pLUSEpX/KvI0Dgfun1F+LzuLotRFuDhrvkU9ETJA6OnD2+Fn/ieHgloiKA/Mn/g==", + "version": "1.12.9", + "resolved": "https://registry.npmjs.org/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.12.9.tgz", + "integrity": "sha512-C25fYftXOras3P3anSUeXXIpxmEkdAcsIL9yrr0j1xepTZ/yKwpnQ6g3coj8UXdeJy4GTVlR6+Ow/QiBgZQNOg==", "cpu": [ "x64" ], @@ -1279,9 +1279,9 @@ "license": "Apache-2.0" }, "node_modules/@swc/types": { - "version": "0.1.21", - "resolved": "https://registry.npmjs.org/@swc/types/-/types-0.1.21.tgz", - "integrity": "sha512-2YEtj5HJVbKivud9N4bpPBAyZhj4S2Ipe5LkUG94alTpr7in/GU/EARgPAd3BwU+YOmFVJC2+kjqhGRi3r0ZpQ==", + "version": "0.1.23", + "resolved": "https://registry.npmjs.org/@swc/types/-/types-0.1.23.tgz", + "integrity": "sha512-u1iIVZV9Q0jxY+yM2vw/hZGDNudsN85bBpTqzAQ9rzkxW9D+e3aEM4Han+ow518gSewkXgjmEK0BD79ZcNVgPw==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -1289,9 +1289,9 @@ } }, "node_modules/@types/estree": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.7.tgz", - "integrity": "sha512-w28IoSUCJpidD/TGviZwwMJckNESJZXFu7NBZ5YJ4mEUnNraUn9Pm8HSZm/jDF1pDWYKspWE7oVphigUPRakIQ==", + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", "dev": true, "license": "MIT" }, @@ -1303,9 +1303,9 @@ "license": "MIT" }, "node_modules/@types/react": { - "version": "19.1.5", - "resolved": "https://registry.npmjs.org/@types/react/-/react-19.1.5.tgz", - "integrity": "sha512-piErsCVVbpMMT2r7wbawdZsq4xMvIAhQuac2gedQHysu1TZYEigE6pnFfgZT+/jQnrRuF5r+SHzuehFjfRjr4g==", + "version": "19.1.8", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.1.8.tgz", + "integrity": "sha512-AwAfQ2Wa5bCx9WP8nZL2uMZWod7J7/JSplxbTmBQ5ms6QpqNYm672H0Vu9ZVKVngQ+ii4R/byguVEUZQyeg44g==", "devOptional": true, "license": "MIT", "dependencies": { @@ -1313,9 +1313,9 @@ } }, "node_modules/@types/react-dom": { - "version": "19.1.5", - "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.1.5.tgz", - "integrity": "sha512-CMCjrWucUBZvohgZxkjd6S9h0nZxXjzus6yDfUb+xLxYM7VvjKNH1tQrE9GWLql1XoOP4/Ds3bwFqShHUYraGg==", + "version": "19.1.6", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.1.6.tgz", + "integrity": "sha512-4hOiT/dwO8Ko0gV1m/TJZYk3y0KBnY9vzDh7W+DH17b2HFSOGgdj33dhihPeuy3l0q23+4e+hoXHV6hCC4dCXw==", "dev": true, "license": "MIT", "peerDependencies": { @@ -1533,23 +1533,23 @@ } }, "node_modules/@vitejs/plugin-react-swc": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/@vitejs/plugin-react-swc/-/plugin-react-swc-3.10.0.tgz", - "integrity": "sha512-ZmkdHw3wo/o/Rk05YsXZs/DJAfY2CdQ5DUAjoWji+PEr+hYADdGMCGgEAILbiKj+CjspBTuTACBcWDrmC8AUfw==", + "version": "3.10.2", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react-swc/-/plugin-react-swc-3.10.2.tgz", + "integrity": "sha512-xD3Rdvrt5LgANug7WekBn1KhcvLn1H3jNBfJRL3reeOIua/WnZOEV5qi5qIBq5T8R0jUDmRtxuvk4bPhzGHDWw==", "dev": true, "license": "MIT", "dependencies": { - "@rolldown/pluginutils": "1.0.0-beta.9", - "@swc/core": "^1.11.22" + "@rolldown/pluginutils": "1.0.0-beta.11", + "@swc/core": "^1.11.31" }, "peerDependencies": { - "vite": "^4 || ^5 || ^6" + "vite": "^4 || ^5 || ^6 || ^7.0.0-beta.0" } }, "node_modules/acorn": { - "version": "8.14.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.1.tgz", - "integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==", + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", "bin": { @@ -1630,9 +1630,9 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, "license": "MIT", "dependencies": { @@ -1722,6 +1722,15 @@ "dev": true, "license": "MIT" }, + "node_modules/cookie": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.0.2.tgz", + "integrity": "sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -1833,19 +1842,19 @@ } }, "node_modules/eslint": { - "version": "9.27.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.27.0.tgz", - "integrity": "sha512-ixRawFQuMB9DZ7fjU3iGGganFDp3+45bPOdaRurcFHSXO1e/sYwUX/FtQZpLZJR6SjMoJH8hR2pPEAfDyCoU2Q==", + "version": "9.30.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.30.1.tgz", + "integrity": "sha512-zmxXPNMOXmwm9E0yQLi5uqXHs7uq2UIiqEKo3Gq+3fwo1XrJ+hijAZImyF7hclW3E6oHz43Yk3RP8at6OTKflQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.12.1", - "@eslint/config-array": "^0.20.0", - "@eslint/config-helpers": "^0.2.1", + "@eslint/config-array": "^0.21.0", + "@eslint/config-helpers": "^0.3.0", "@eslint/core": "^0.14.0", "@eslint/eslintrc": "^3.3.1", - "@eslint/js": "9.27.0", + "@eslint/js": "9.30.1", "@eslint/plugin-kit": "^0.3.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", @@ -1857,9 +1866,9 @@ "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", - "eslint-scope": "^8.3.0", - "eslint-visitor-keys": "^4.2.0", - "espree": "^10.3.0", + "eslint-scope": "^8.4.0", + "eslint-visitor-keys": "^4.2.1", + "espree": "^10.4.0", "esquery": "^1.5.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", @@ -1933,9 +1942,9 @@ } }, "node_modules/eslint-scope": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.3.0.tgz", - "integrity": "sha512-pUNxi75F8MJ/GdeKtVLSbYg4ZI34J6C0C7sbL4YOp2exGwen7ZsuBqKzUhXd0qMQ362yET3z+uPwKeg/0C2XCQ==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", + "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", "dev": true, "license": "BSD-2-Clause", "dependencies": { @@ -1963,9 +1972,9 @@ } }, "node_modules/eslint/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, "license": "MIT", "dependencies": { @@ -1974,9 +1983,9 @@ } }, "node_modules/eslint/node_modules/eslint-visitor-keys": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.0.tgz", - "integrity": "sha512-UyLnSehNt62FFhSwjZlHmeokpRK59rcz29j+F1/aDgbkbRTk7wIc9XzdoasMUbRNKDM0qQt/+BJ4BrpFeABemw==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", "dev": true, "license": "Apache-2.0", "engines": { @@ -2000,15 +2009,15 @@ } }, "node_modules/espree": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-10.3.0.tgz", - "integrity": "sha512-0QYC8b24HWY8zjRnDTL6RiHfDbAWn63qb4LMj1Z4b076A4une81+z03Kg7l7mn/48PUTqoLptSXez8oknU8Clg==", + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", "dev": true, "license": "BSD-2-Clause", "dependencies": { - "acorn": "^8.14.0", + "acorn": "^8.15.0", "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^4.2.0" + "eslint-visitor-keys": "^4.2.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2018,9 +2027,9 @@ } }, "node_modules/espree/node_modules/eslint-visitor-keys": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.0.tgz", - "integrity": "sha512-UyLnSehNt62FFhSwjZlHmeokpRK59rcz29j+F1/aDgbkbRTk7wIc9XzdoasMUbRNKDM0qQt/+BJ4BrpFeABemw==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", "dev": true, "license": "Apache-2.0", "engines": { @@ -2230,9 +2239,9 @@ } }, "node_modules/globals": { - "version": "16.1.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-16.1.0.tgz", - "integrity": "sha512-aibexHNbb/jiUSObBgpHLj+sIuUmJnYcgXBlrfsiDZ9rt4aF2TFRbyLgZ2iFQuVZ1K5Mx3FVkbKRSgKrbK3K2g==", + "version": "16.3.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-16.3.0.tgz", + "integrity": "sha512-bqWEnJ1Nt3neqx2q5SFfGS8r/ahumIakg3HcwtNlrVlwXIeNumWn/c7Pn/wKzGhf6SaW6H6uWXLqC30STCMchQ==", "dev": true, "license": "MIT", "engines": { @@ -2631,9 +2640,9 @@ } }, "node_modules/postcss": { - "version": "8.5.3", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.3.tgz", - "integrity": "sha512-dle9A3yYxlBSrt8Fu+IpjGT8SY8hN0mlaA6GY8t0P5PjIOZemULz/E2Bnm/2dcUOena75OTNkHI76uZBNUUq3A==", + "version": "8.5.6", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", + "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", "dev": true, "funding": [ { @@ -2651,7 +2660,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.8", + "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -2670,9 +2679,9 @@ } }, "node_modules/prettier": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.3.tgz", - "integrity": "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==", + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz", + "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", "dev": true, "license": "MIT", "bin": { @@ -2738,9 +2747,9 @@ } }, "node_modules/react-router": { - "version": "7.6.0", - "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.6.0.tgz", - "integrity": "sha512-GGufuHIVCJDbnIAXP3P9Sxzq3UUsddG3rrI3ut1q6m0FI6vxVBF3JoPQ38+W/blslLH4a5Yutp8drkEpXoddGQ==", + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.6.3.tgz", + "integrity": "sha512-zf45LZp5skDC6I3jDLXQUu0u26jtuP4lEGbc7BbdyxenBN1vJSTA18czM2D+h5qyMBuMrD+9uB+mU37HIoKGRA==", "license": "MIT", "dependencies": { "cookie": "^1.0.1", @@ -2760,12 +2769,12 @@ } }, "node_modules/react-router-dom": { - "version": "7.6.0", - "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.6.0.tgz", - "integrity": "sha512-DYgm6RDEuKdopSyGOWZGtDfSm7Aofb8CCzgkliTjtu/eDuB0gcsv6qdFhhi8HdtmA+KHkt5MfZ5K2PdzjugYsA==", + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.6.3.tgz", + "integrity": "sha512-DiWJm9qdUAmiJrVWaeJdu4TKu13+iB/8IEi0EW/XgaHCjW/vWGrwzup0GVvaMteuZjKnh5bEvJP/K0MDnzawHw==", "license": "MIT", "dependencies": { - "react-router": "7.6.0" + "react-router": "7.6.3" }, "engines": { "node": ">=20.0.0" @@ -2775,15 +2784,6 @@ "react-dom": ">=18" } }, - "node_modules/react-router/node_modules/cookie": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.0.2.tgz", - "integrity": "sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, "node_modules/readdirp": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", @@ -2820,13 +2820,13 @@ } }, "node_modules/rollup": { - "version": "4.39.0", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.39.0.tgz", - "integrity": "sha512-thI8kNc02yNvnmJp8dr3fNWJ9tCONDhp6TV35X6HkKGGs9E6q7YWCHbe5vKiTa7TAiNcFEmXKj3X/pG2b3ci0g==", + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.44.1.tgz", + "integrity": "sha512-x8H8aPvD+xbl0Do8oez5f5o8eMS3trfCghc4HhLAnCkj7Vl0d1JWGs0UF/D886zLW2rOj2QymV/JcSSsw+XDNg==", "dev": true, "license": "MIT", "dependencies": { - "@types/estree": "1.0.7" + "@types/estree": "1.0.8" }, "bin": { "rollup": "dist/bin/rollup" @@ -2836,26 +2836,26 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.39.0", - "@rollup/rollup-android-arm64": "4.39.0", - "@rollup/rollup-darwin-arm64": "4.39.0", - "@rollup/rollup-darwin-x64": "4.39.0", - "@rollup/rollup-freebsd-arm64": "4.39.0", - "@rollup/rollup-freebsd-x64": "4.39.0", - "@rollup/rollup-linux-arm-gnueabihf": "4.39.0", - "@rollup/rollup-linux-arm-musleabihf": "4.39.0", - "@rollup/rollup-linux-arm64-gnu": "4.39.0", - "@rollup/rollup-linux-arm64-musl": "4.39.0", - "@rollup/rollup-linux-loongarch64-gnu": "4.39.0", - "@rollup/rollup-linux-powerpc64le-gnu": "4.39.0", - "@rollup/rollup-linux-riscv64-gnu": "4.39.0", - "@rollup/rollup-linux-riscv64-musl": "4.39.0", - "@rollup/rollup-linux-s390x-gnu": "4.39.0", - "@rollup/rollup-linux-x64-gnu": "4.39.0", - "@rollup/rollup-linux-x64-musl": "4.39.0", - "@rollup/rollup-win32-arm64-msvc": "4.39.0", - "@rollup/rollup-win32-ia32-msvc": "4.39.0", - "@rollup/rollup-win32-x64-msvc": "4.39.0", + "@rollup/rollup-android-arm-eabi": "4.44.1", + "@rollup/rollup-android-arm64": "4.44.1", + "@rollup/rollup-darwin-arm64": "4.44.1", + "@rollup/rollup-darwin-x64": "4.44.1", + "@rollup/rollup-freebsd-arm64": "4.44.1", + "@rollup/rollup-freebsd-x64": "4.44.1", + "@rollup/rollup-linux-arm-gnueabihf": "4.44.1", + "@rollup/rollup-linux-arm-musleabihf": "4.44.1", + "@rollup/rollup-linux-arm64-gnu": "4.44.1", + "@rollup/rollup-linux-arm64-musl": "4.44.1", + "@rollup/rollup-linux-loongarch64-gnu": "4.44.1", + "@rollup/rollup-linux-powerpc64le-gnu": "4.44.1", + "@rollup/rollup-linux-riscv64-gnu": "4.44.1", + "@rollup/rollup-linux-riscv64-musl": "4.44.1", + "@rollup/rollup-linux-s390x-gnu": "4.44.1", + "@rollup/rollup-linux-x64-gnu": "4.44.1", + "@rollup/rollup-linux-x64-musl": "4.44.1", + "@rollup/rollup-win32-arm64-msvc": "4.44.1", + "@rollup/rollup-win32-ia32-msvc": "4.44.1", + "@rollup/rollup-win32-x64-msvc": "4.44.1", "fsevents": "~2.3.2" } }, @@ -2991,9 +2991,9 @@ "license": "MIT" }, "node_modules/tinyglobby": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.13.tgz", - "integrity": "sha512-mEwzpUgrLySlveBwEVDMKk5B57bhLPYovRfPAXD5gA/98Opn0rCDj3GtLwFvCvH5RK9uPCExUROW5NjDwvqkxw==", + "version": "0.2.14", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.14.tgz", + "integrity": "sha512-tX5e7OM1HnYr2+a2C/4V0htOcSQcoSTH9KgJnVvNm5zm/cyEWKJ7j7YutsH9CxMdtOkkLFy2AHrMci9IM8IPZQ==", "dev": true, "license": "MIT", "dependencies": { @@ -3135,24 +3135,24 @@ } }, "node_modules/vite": { - "version": "6.3.5", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.3.5.tgz", - "integrity": "sha512-cZn6NDFE7wdTpINgs++ZJ4N49W2vRp8LCKrn3Ob1kYNtOo21vfDoaV5GzBfLU4MovSAB8uNRm4jgzVQZ+mBzPQ==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.0.0.tgz", + "integrity": "sha512-ixXJB1YRgDIw2OszKQS9WxGHKwLdCsbQNkpJN171udl6szi/rIySHL6/Os3s2+oE4P/FLD4dxg4mD7Wust+u5g==", "dev": true, "license": "MIT", "dependencies": { "esbuild": "^0.25.0", - "fdir": "^6.4.4", + "fdir": "^6.4.6", "picomatch": "^4.0.2", - "postcss": "^8.5.3", - "rollup": "^4.34.9", - "tinyglobby": "^0.2.13" + "postcss": "^8.5.6", + "rollup": "^4.40.0", + "tinyglobby": "^0.2.14" }, "bin": { "vite": "bin/vite.js" }, "engines": { - "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + "node": "^20.19.0 || >=22.12.0" }, "funding": { "url": "https://github.com/vitejs/vite?sponsor=1" @@ -3161,14 +3161,14 @@ "fsevents": "~2.3.3" }, "peerDependencies": { - "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "@types/node": "^20.19.0 || >=22.12.0", "jiti": ">=1.21.0", - "less": "*", + "less": "^4.0.0", "lightningcss": "^1.21.0", - "sass": "*", - "sass-embedded": "*", - "stylus": "*", - "sugarss": "*", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" @@ -3285,9 +3285,9 @@ } }, "node_modules/vite/node_modules/fdir": { - "version": "6.4.4", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.4.tgz", - "integrity": "sha512-1NZP+GK4GfuAv3PqKvxQRDMjdSRZjnkq7KfhlNrCNNlZ0ygQFpebfrnfnq/W7fpUnAv9aGWmY1zKx7FYL3gwhg==", + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.6.tgz", + "integrity": "sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==", "dev": true, "license": "MIT", "peerDependencies": { @@ -3359,9 +3359,9 @@ } }, "node_modules/zustand": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/zustand/-/zustand-5.0.5.tgz", - "integrity": "sha512-mILtRfKW9xM47hqxGIxCv12gXusoY/xTSHBYApXozR0HmQv299whhBeeAcRy+KrPPybzosvJBCOmVjq6x12fCg==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-5.0.6.tgz", + "integrity": "sha512-ihAqNeUVhe0MAD+X8M5UzqyZ9k3FFZLBTtqo6JLPwV53cbRB/mJwBI0PxcIgqhBBHlEs8G45OTDTMq3gNcLq3A==", "license": "MIT", "engines": { "node": ">=12.20.0" diff --git a/frontend/package.json b/frontend/package.json index 6d205445..2407f73b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -14,24 +14,24 @@ "dompurify": "^3.2.6", "react": "^19.1.0", "react-dom": "^19.1.0", - "react-router-dom": "^7.6.0", - "zustand": "^5.0.5" + "react-router-dom": "^7.6.3", + "zustand": "^5.0.6" }, "devDependencies": { - "@types/react": "^19.1.5", - "@types/react-dom": "^19.1.5", + "@types/react": "^19.1.8", + "@types/react-dom": "^19.1.6", "@typescript-eslint/eslint-plugin": "^8.32.1", "@typescript-eslint/parser": "^8.32.1", - "@vitejs/plugin-react-swc": "^3.10.0", - "eslint": "^9.27.0", + "@vitejs/plugin-react-swc": "^3.10.2", + "eslint": "^9.30.1", "eslint-config-prettier": "^10.1.5", "eslint-plugin-react-hooks": "^5.2.0", "eslint-plugin-react-refresh": "^0.4.20", - "globals": "^16.1.0", - "prettier": "3.5.3", + "globals": "^16.3.0", + "prettier": "3.6.2", "typescript": "^5.8.3", "typescript-eslint": "^8.32.1", - "vite": ">=6.3.5", + "vite": ">=7.0.0", "vite-plugin-checker": "^0.9.3" } } diff --git a/frontend/src/api/actions/deleteAppriseNotificationUrl.ts b/frontend/src/api/actions/deleteAppriseNotificationUrl.ts index 5018b79f..33153c49 100644 --- a/frontend/src/api/actions/deleteAppriseNotificationUrl.ts +++ b/frontend/src/api/actions/deleteAppriseNotificationUrl.ts @@ -6,10 +6,10 @@ type AppriseTaskNameType = | 'download_pending' | 'check_reindex'; -const deleteAppriseNotificationUrl = async (taskName: AppriseTaskNameType) => { +const deleteAppriseNotificationUrl = async (taskName: AppriseTaskNameType, url: string) => { return APIClient('/api/task/notification/', { method: 'DELETE', - body: { task_name: taskName }, + body: { task_name: taskName, url: url }, }); }; diff --git a/frontend/src/api/actions/deleteDownloadQueueByFilter.ts b/frontend/src/api/actions/deleteDownloadQueueByFilter.ts index 65a01a3c..9d4f5902 100644 --- a/frontend/src/api/actions/deleteDownloadQueueByFilter.ts +++ b/frontend/src/api/actions/deleteDownloadQueueByFilter.ts @@ -2,9 +2,15 @@ import APIClient from '../../functions/APIClient'; type FilterType = 'ignore' | 'pending'; -const deleteDownloadQueueByFilter = async (filter: FilterType) => { +const deleteDownloadQueueByFilter = async ( + filter: FilterType, + channel: string | null, + vid_type: string | null, +) => { const searchParams = new URLSearchParams(); if (filter) searchParams.append('filter', filter); + if (channel) searchParams.append('channel', channel); + if (vid_type) searchParams.append('vid_type', vid_type); return APIClient(`/api/download/?${searchParams.toString()}`, { method: 'DELETE', diff --git a/frontend/src/api/actions/updateDownloadQueue.ts b/frontend/src/api/actions/updateDownloadQueue.ts index 071707df..beb14120 100644 --- a/frontend/src/api/actions/updateDownloadQueue.ts +++ b/frontend/src/api/actions/updateDownloadQueue.ts @@ -1,6 +1,6 @@ import APIClient from '../../functions/APIClient'; -const updateDownloadQueue = async (youtubeIdStrings: string, autostart: boolean) => { +const updateDownloadQueue = async (youtubeIdStrings: string, autostart: boolean, flat: boolean) => { const urls = []; const containsMultiple = youtubeIdStrings.includes('\n'); @@ -16,12 +16,12 @@ const updateDownloadQueue = async (youtubeIdStrings: string, autostart: boolean) urls.push({ youtube_id: youtubeIdStrings, status: 'pending' }); } - let params = ''; - if (autostart) { - params = '?autostart=true'; - } + const searchParams = new URLSearchParams(); + if (autostart) searchParams.append('autostart', 'true'); + if (flat) searchParams.append('flat', 'true'); + const endpoint = `/api/download/${searchParams.toString() ? `?${searchParams.toString()}` : ''}`; - return APIClient(`/api/download/${params}`, { + return APIClient(endpoint, { method: 'POST', body: { data: [...urls] }, }); diff --git a/frontend/src/api/actions/updateDownloadQueueByFilter.ts b/frontend/src/api/actions/updateDownloadQueueByFilter.ts new file mode 100644 index 00000000..1410360e --- /dev/null +++ b/frontend/src/api/actions/updateDownloadQueueByFilter.ts @@ -0,0 +1,23 @@ +import APIClient from '../../functions/APIClient'; + +type FilterType = 'ignore' | 'pending'; +export type DownloadQueueStatus = 'ignore' | 'pending' | 'priority'; + +const updateDownloadQueueByFilter = async ( + filter: FilterType, + channel: string | null, + vid_type: string | null, + status: DownloadQueueStatus, +) => { + const searchParams = new URLSearchParams(); + if (filter) searchParams.append('filter', filter); + if (channel) searchParams.append('channel', channel); + if (vid_type) searchParams.append('vid_type', vid_type); + + return APIClient(`/api/download/?${searchParams.toString()}`, { + method: 'PATCH', + body: { status: status }, + }); +}; + +export default updateDownloadQueueByFilter; diff --git a/frontend/src/api/actions/updatePlaylistSortOrder.ts b/frontend/src/api/actions/updatePlaylistSortOrder.ts new file mode 100644 index 00000000..a7234da2 --- /dev/null +++ b/frontend/src/api/actions/updatePlaylistSortOrder.ts @@ -0,0 +1,10 @@ +import APIClient from '../../functions/APIClient'; + +const updatePlaylistSortOrder = async (playlistId: string, newSortOrder: 'top' | 'bottom') => { + return APIClient(`/api/playlist/${playlistId}/`, { + method: 'POST', + body: { playlist_sort_order: newSortOrder }, + }); +}; + +export default updatePlaylistSortOrder; diff --git a/frontend/src/api/loader/loadAppsettingsConfig.ts b/frontend/src/api/loader/loadAppsettingsConfig.ts index 44f22e8a..3a6489fd 100644 --- a/frontend/src/api/loader/loadAppsettingsConfig.ts +++ b/frontend/src/api/loader/loadAppsettingsConfig.ts @@ -6,6 +6,7 @@ export type AppSettingsConfigType = { live_channel_size: number | null; shorts_channel_size: number | null; auto_start: boolean; + extract_flat: boolean; }; downloads: { limit_speed: number | null; diff --git a/frontend/src/api/loader/loadDownloadQueue.ts b/frontend/src/api/loader/loadDownloadQueue.ts index d74f4bae..367638ab 100644 --- a/frontend/src/api/loader/loadDownloadQueue.ts +++ b/frontend/src/api/loader/loadDownloadQueue.ts @@ -1,11 +1,21 @@ import APIClient from '../../functions/APIClient'; import { DownloadResponseType } from '../../pages/Download'; -const loadDownloadQueue = async (page: number, channelId: string | null, showIgnored: boolean) => { +const loadDownloadQueue = async ( + page: number, + channelId: string | null, + vid_type: string | null, + errorFilterFromUrl: string | null, + showIgnored: boolean, + search: string, +) => { const searchParams = new URLSearchParams(); if (page) searchParams.append('page', page.toString()); if (channelId) searchParams.append('channel', channelId); + if (vid_type) searchParams.append('vid_type', vid_type); + if (search) searchParams.append('q', encodeURIComponent(search)); + if (errorFilterFromUrl !== null) searchParams.append('error', errorFilterFromUrl); searchParams.append('filter', showIgnored ? 'ignore' : 'pending'); const endpoint = `/api/download/${searchParams.toString() ? `?${searchParams.toString()}` : ''}`; diff --git a/frontend/src/api/loader/loadPlaylistById.ts b/frontend/src/api/loader/loadPlaylistById.ts index b800410f..5a339baa 100644 --- a/frontend/src/api/loader/loadPlaylistById.ts +++ b/frontend/src/api/loader/loadPlaylistById.ts @@ -14,6 +14,7 @@ export type PlaylistType = { playlist_channel_id: string; playlist_description: string; playlist_entries: PlaylistEntryType[]; + playlist_sort_order: 'top' | 'bottom'; playlist_id: string; playlist_last_refresh: string; playlist_name: string; diff --git a/frontend/src/components/DownloadListItem.tsx b/frontend/src/components/DownloadListItem.tsx index bb0a82e3..cd540c87 100644 --- a/frontend/src/components/DownloadListItem.tsx +++ b/frontend/src/components/DownloadListItem.tsx @@ -47,14 +47,19 @@ const DownloadListItem = ({ download, setRefresh }: DownloadListItemProps) => { {!download.channel_indexed && {download.channel_name}} - +

{download.title}

- Published: {formatDate(download.published)} | Duration: {download.duration} |{' '} - {download.youtube_id} + {download.published && Published: {formatDate(download.published)} | } + Duration: {download.duration} | + {download.youtube_id}

{download.message &&

{download.message}

} diff --git a/frontend/src/components/InputConfig.tsx b/frontend/src/components/InputConfig.tsx index a378a5fb..cf1bfa13 100644 --- a/frontend/src/components/InputConfig.tsx +++ b/frontend/src/components/InputConfig.tsx @@ -1,4 +1,5 @@ import { useState } from 'react'; +import LoadingIndicator from './LoadingIndicator'; type InputTextProps = { type: 'text' | 'number'; @@ -51,13 +52,7 @@ const InputConfig = ({ type, name, value, setValue, oldValue, updateCallback }: )} {oldValue !== null && } - {loading && ( - <> -
-
-
- - )} + {loading && } {success && ✅}
diff --git a/frontend/src/components/LoadingIndicator.tsx b/frontend/src/components/LoadingIndicator.tsx new file mode 100644 index 00000000..0eaeaeb4 --- /dev/null +++ b/frontend/src/components/LoadingIndicator.tsx @@ -0,0 +1,9 @@ +const LoadingIndicator = () => { + return ( +
+
+
+ ); +}; + +export default LoadingIndicator; diff --git a/frontend/src/components/SearchExampleQueries.tsx b/frontend/src/components/SearchExampleQueries.tsx index 2e17792e..1c181c6e 100644 --- a/frontend/src/components/SearchExampleQueries.tsx +++ b/frontend/src/components/SearchExampleQueries.tsx @@ -95,8 +95,8 @@ const SearchExampleQueries = () => { full: — search in video subtitles