diff --git a/backend/appsettings/serializers.py b/backend/appsettings/serializers.py index 5e6e434b..a690f37d 100644 --- a/backend/appsettings/serializers.py +++ b/backend/appsettings/serializers.py @@ -25,6 +25,7 @@ class AppConfigSubSerializer( live_channel_size = serializers.IntegerField(required=False) shorts_channel_size = serializers.IntegerField(required=False) auto_start = serializers.BooleanField(required=False) + extract_flat = serializers.BooleanField(required=False) class AppConfigDownloadsSerializer( diff --git a/backend/appsettings/src/config.py b/backend/appsettings/src/config.py index 3c14a2c0..52c69bb6 100644 --- a/backend/appsettings/src/config.py +++ b/backend/appsettings/src/config.py @@ -22,6 +22,7 @@ class SubscriptionsConfigType(TypedDict): live_channel_size: int shorts_channel_size: int auto_start: bool + extract_flat: bool class DownloadsConfigType(TypedDict): @@ -73,6 +74,7 @@ class AppConfig: "live_channel_size": 50, "shorts_channel_size": 50, "auto_start": False, + "extract_flat": False, }, "downloads": { "limit_speed": None, diff --git a/backend/appsettings/src/reindex.py b/backend/appsettings/src/reindex.py index e1cedcca..20f82425 100644 --- a/backend/appsettings/src/reindex.py +++ b/backend/appsettings/src/reindex.py @@ -11,11 +11,11 @@ from typing import Callable, TypedDict from appsettings.src.config import AppConfig from channel.src.index import YoutubeChannel +from channel.src.remote_query import get_last_channel_videos from common.src.env_settings import EnvironmentSettings from common.src.es_connect import ElasticWrap, IndexPaginate from common.src.helper import rand_sleep from common.src.ta_redis import RedisQueue -from download.src.subscriptions import ChannelSubscription from download.src.thumbnails import ThumbManager from download.src.yt_dlp_base import CookieHandler from playlist.src.index import YoutubePlaylist @@ -376,7 +376,7 @@ class Reindex(ReindexBase): channel.upload_to_es() channel.sync_to_videos() - ChannelFullScan(channel_id).scan() + ChannelFullScan(channel_id, self.config).scan() self.processed["channels"] += 1 def _reindex_single_playlist(self, playlist_id: str) -> None: @@ -495,8 +495,9 @@ class ReindexProgress(ReindexBase): class ChannelFullScan: """full scan of channel to fix vid_type mismatch""" - def __init__(self, channel_id): + def __init__(self, channel_id, config): self.channel_id = channel_id + self.config = config self.to_update = False def scan(self): @@ -527,9 +528,8 @@ class ChannelFullScan: def _get_all_remote(self): """get all channel videos""" - sub = ChannelSubscription() - all_remote_videos = sub.get_last_youtube_videos( - self.channel_id, limit=False + all_remote_videos = get_last_channel_videos( + self.channel_id, self.config, limit=False ) return all_remote_videos diff --git a/backend/channel/src/index.py b/backend/channel/src/index.py index bf36b5a3..17bf9526 100644 --- a/backend/channel/src/index.py +++ b/backend/channel/src/index.py @@ -155,48 +155,15 @@ class YoutubeChannel(YouTubeItem): update_path = f"ta_video/_update_by_query?pipeline={self.youtube_id}" _, _ = ElasticWrap(update_path).post(data) - def get_folder_path(self): - """get folder where media files get stored""" - folder_path = os.path.join( - EnvironmentSettings.MEDIA_DIR, - self.json_data["channel_id"], - ) - return folder_path + def change_subscribe(self, new_subscribe_state: bool): + """change subscribe status""" + if not self.json_data: + self.build_json() - def delete_es_videos(self): - """delete all channel documents from elasticsearch""" - data = { - "query": { - "term": {"channel.channel_id": {"value": self.youtube_id}} - } - } - _, _ = ElasticWrap("ta_video/_delete_by_query").post(data) - - def delete_es_comments(self): - """delete all comments from this channel""" - data = { - "query": { - "term": {"comment_channel_id": {"value": self.youtube_id}} - } - } - _, _ = ElasticWrap("ta_comment/_delete_by_query").post(data) - - def delete_es_subtitles(self): - """delete all subtitles from this channel""" - data = { - "query": { - "term": {"subtitle_channel_id": {"value": self.youtube_id}} - } - } - _, _ = ElasticWrap("ta_subtitle/_delete_by_query").post(data) - - def delete_playlists(self): - """delete all indexed playlist from es""" - from playlist.src.index import YoutubePlaylist - - all_playlists = self.get_indexed_playlists() - for playlist in all_playlists: - YoutubePlaylist(playlist["playlist_id"]).delete_metadata() + self.json_data["channel_subscribed"] = new_subscribe_state + self.upload_to_es() + self.sync_to_videos() + return self.json_data def delete_channel(self): """delete channel and all videos""" @@ -205,24 +172,7 @@ class YoutubeChannel(YouTubeItem): if not self.json_data: raise FileNotFoundError - folder_path = self.get_folder_path() - print(f"{self.youtube_id}: delete all media files") - try: - all_videos = os.listdir(folder_path) - for video in all_videos: - video_path = os.path.join(folder_path, video) - os.remove(video_path) - os.rmdir(folder_path) - except FileNotFoundError: - print(f"no videos found for {folder_path}") - - print(f"{self.youtube_id}: delete indexed playlists") - self.delete_playlists() - print(f"{self.youtube_id}: delete indexed videos") - self.delete_es_videos() - self.delete_es_comments() - self.delete_es_subtitles() - self.del_in_es() + ChannelDelete(json_data=self.json_data).delete() def index_channel_playlists(self): """add all playlists of channel to index""" @@ -244,6 +194,21 @@ class YoutubeChannel(YouTubeItem): print("add playlist: " + playlist[1]) rand_sleep(self.config) + def get_all_playlists(self): + """get all playlists owned by this channel""" + url = ( + f"https://www.youtube.com/channel/{self.youtube_id}" + + "/playlists?view=1&sort=dd&shelf_id=0" + ) + obs = {"skip_download": True, "extract_flat": True} + playlists, _ = YtWrap(obs, self.config).extract(url) + if not playlists: + self.all_playlists = [] + return + + all_entries = [(i["id"], i["title"]) for i in playlists["entries"]] + self.all_playlists = all_entries + def _notify_single_playlist(self, idx, total): """send notification""" channel_name = self.json_data["channel_name"] @@ -272,34 +237,6 @@ class YoutubeChannel(YouTubeItem): all_videos = IndexPaginate("ta_video", data).get_results() return all_videos - def get_all_playlists(self): - """get all playlists owned by this channel""" - url = ( - f"https://www.youtube.com/channel/{self.youtube_id}" - + "/playlists?view=1&sort=dd&shelf_id=0" - ) - obs = {"skip_download": True, "extract_flat": True} - playlists, _ = YtWrap(obs, self.config).extract(url) - if not playlists: - self.all_playlists = [] - return - - all_entries = [(i["id"], i["title"]) for i in playlists["entries"]] - self.all_playlists = all_entries - - def get_indexed_playlists(self, active_only=False): - """get all indexed playlists from channel""" - must_list = [ - {"term": {"playlist_channel_id": {"value": self.youtube_id}}} - ] - if active_only: - must_list.append({"term": {"playlist_active": {"value": True}}}) - - data = {"query": {"bool": {"must": must_list}}} - - all_playlists = IndexPaginate("ta_playlist", data).get_results() - return all_playlists - def get_overwrites(self) -> dict: """get all per channel overwrites""" return self.json_data.get("channel_overwrites", {}) @@ -330,6 +267,93 @@ class YoutubeChannel(YouTubeItem): self.json_data["channel_overwrites"] = to_write +class ChannelDelete(YouTubeItem): + """delete and cleanup""" + + index_name = "ta_channel" + + def __init__(self, json_data): + super().__init__(youtube_id=json_data["channel_id"]) + self.json_data = json_data + + def delete(self): + """delete channel and all videos""" + folder_path = self._get_folder_path() + print(f"{self.youtube_id}: delete all media files") + try: + all_videos = os.listdir(folder_path) + for video in all_videos: + video_path = os.path.join(folder_path, video) + os.remove(video_path) + os.rmdir(folder_path) + except FileNotFoundError: + print(f"no videos found for {folder_path}") + + print(f"{self.youtube_id}: delete indexed playlists") + self._delete_playlists() + print(f"{self.youtube_id}: delete indexed videos") + self._delete_es_videos() + self._delete_es_comments() + self._delete_es_subtitles() + self.del_in_es() + + def _get_folder_path(self): + """get folder where media files get stored""" + folder_path = os.path.join( + EnvironmentSettings.MEDIA_DIR, + self.json_data["channel_id"], + ) + return folder_path + + def _delete_es_videos(self): + """delete all channel documents from elasticsearch""" + data = { + "query": { + "term": {"channel.channel_id": {"value": self.youtube_id}} + } + } + _, _ = ElasticWrap("ta_video/_delete_by_query").post(data) + + def _delete_es_comments(self): + """delete all comments from this channel""" + data = { + "query": { + "term": {"comment_channel_id": {"value": self.youtube_id}} + } + } + _, _ = ElasticWrap("ta_comment/_delete_by_query").post(data) + + def _delete_es_subtitles(self): + """delete all subtitles from this channel""" + data = { + "query": { + "term": {"subtitle_channel_id": {"value": self.youtube_id}} + } + } + _, _ = ElasticWrap("ta_subtitle/_delete_by_query").post(data) + + def _delete_playlists(self): + """delete all indexed playlist from es""" + from playlist.src.index import YoutubePlaylist + + all_playlists = self._get_indexed_playlists() + for playlist in all_playlists: + YoutubePlaylist(playlist["playlist_id"]).delete_metadata() + + def _get_indexed_playlists(self, active_only=False): + """get all indexed playlists from channel""" + must_list = [ + {"term": {"playlist_channel_id": {"value": self.youtube_id}}} + ] + if active_only: + must_list.append({"term": {"playlist_active": {"value": True}}}) + + data = {"query": {"bool": {"must": must_list}}} + + all_playlists = IndexPaginate("ta_playlist", data).get_results() + return all_playlists + + def channel_overwrites(channel_id, overwrites): """collection to overwrite settings per channel""" channel = YoutubeChannel(channel_id) diff --git a/backend/channel/src/remote_query.py b/backend/channel/src/remote_query.py new file mode 100644 index 00000000..a38d1cce --- /dev/null +++ b/backend/channel/src/remote_query.py @@ -0,0 +1,124 @@ +"""build queries for video extraction from channel subscriptions""" + +from download.src.yt_dlp_base import YtWrap +from video.src.constants import VideoTypeEnum + + +class VideoQueryBuilder: + """Build queries for yt-dlp.""" + + def __init__(self, config: dict, channel_overwrites: dict | None = None): + self.config = config + self.channel_overwrites = channel_overwrites or {} + + def build_queries( + self, video_type: VideoTypeEnum | None, limit: bool = True + ) -> list[tuple[VideoTypeEnum, int | None]]: + """Build queries for all or specific video type.""" + query_methods = { + VideoTypeEnum.VIDEOS: self.videos_query, + VideoTypeEnum.STREAMS: self.streams_query, + VideoTypeEnum.SHORTS: self.shorts_query, + } + + if video_type: + # build query for specific type + query_method = query_methods.get(video_type) + if query_method: + query = query_method(limit) + if query[1] != 0: + return [query] + return [] + + # Build and return queries for all video types + queries = [] + for build_query in query_methods.values(): + query = build_query(limit) + if query[1] != 0: + queries.append(query) + + return queries + + def videos_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]: + """Build query for videos.""" + return self._build_generic_query( + video_type=VideoTypeEnum.VIDEOS, + overwrite_key="subscriptions_channel_size", + config_key="channel_size", + limit=limit, + ) + + def streams_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]: + """Build query for streams.""" + return self._build_generic_query( + video_type=VideoTypeEnum.STREAMS, + overwrite_key="subscriptions_live_channel_size", + config_key="live_channel_size", + limit=limit, + ) + + def shorts_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]: + """Build query for shorts.""" + return self._build_generic_query( + video_type=VideoTypeEnum.SHORTS, + overwrite_key="subscriptions_shorts_channel_size", + config_key="shorts_channel_size", + limit=limit, + ) + + def _build_generic_query( + self, + video_type: VideoTypeEnum, + overwrite_key: str, + config_key: str, + limit: bool, + ) -> tuple[VideoTypeEnum, int | None]: + """Generic query for video page scraping.""" + if not limit: + return (video_type, None) + + if ( + overwrite_key in self.channel_overwrites + and self.channel_overwrites[overwrite_key] is not None + ): + overwrite = self.channel_overwrites[overwrite_key] + return (video_type, overwrite) + + if overwrite := self.config["subscriptions"].get(config_key): + return (video_type, overwrite) + + return (video_type, 0) + + +def get_last_channel_videos( + channel_id, + config, + limit=None, + query_filter=None, + channel_overwrites=None, +): + """get a list of last videos from channel""" + query_handler = VideoQueryBuilder(config, channel_overwrites) + queries = query_handler.build_queries(query_filter) + last_videos = [] + + for vid_type_enum, limit_amount in queries: + obs = { + "skip_download": True, + "extract_flat": True, + } + vid_type = vid_type_enum.value + + if limit is not None: + obs.update({"playlist_items": f":{limit_amount}:1"}) + + url = f"https://www.youtube.com/channel/{channel_id}/{vid_type}" + channel_query, _ = YtWrap(obs, config).extract(url) + if not channel_query: + continue + + for entry in channel_query["entries"]: + entry["vid_type"] = vid_type + last_videos.append(entry) + + return last_videos diff --git a/backend/channel/views.py b/backend/channel/views.py index f915ecec..1f0daf57 100644 --- a/backend/channel/views.py +++ b/backend/channel/views.py @@ -14,7 +14,6 @@ from channel.src.nav import ChannelNav from common.serializers import ErrorResponseSerializer from common.src.urlparser import Parser from common.views_base import AdminWriteOnly, ApiBaseView -from download.src.subscriptions import ChannelSubscription from drf_spectacular.utils import ( OpenApiParameter, OpenApiResponse, @@ -89,9 +88,7 @@ class ChannelApiListView(ApiBaseView): def _unsubscribe(channel_id: str): """unsubscribe""" print(f"[{channel_id}] unsubscribe from channel") - ChannelSubscription().change_subscribe( - channel_id, channel_subscribed=False - ) + YoutubeChannel(channel_id).change_subscribe(new_subscribe_state=False) class ChannelApiView(ApiBaseView): @@ -146,7 +143,9 @@ class ChannelApiView(ApiBaseView): subscribed = validated_data.get("channel_subscribed") if subscribed is not None: - ChannelSubscription().change_subscribe(channel_id, subscribed) + YoutubeChannel(channel_id).change_subscribe( + new_subscribe_state=subscribed + ) overwrites = validated_data.get("channel_overwrites") if overwrites: diff --git a/backend/common/src/helper.py b/backend/common/src/helper.py index bfb5e474..cf647a4e 100644 --- a/backend/common/src/helper.py +++ b/backend/common/src/helper.py @@ -292,6 +292,50 @@ def get_channel_overwrites() -> dict[str, dict[str, Any]]: return overwrites +def get_channels( + subscribed_only: bool, source: list[str] | None = None +) -> list[dict]: + """get a list of all channels""" + data = { + "sort": [{"channel_name.keyword": {"order": "asc"}}], + } + if subscribed_only: + query = {"term": {"channel_subscribed": {"value": True}}} + else: + query = {"match_all": {}} + + data["query"] = query # type: ignore + + if source: + data["_source"] = source # type: ignore + + all_channels = IndexPaginate("ta_channel", data).get_results() + + return all_channels + + +def get_playlists( + subscribed_only: bool, source: list[str] | None = None +) -> list[dict]: + """get list of playlists""" + + data = { + "sort": [{"playlist_channel.keyword": {"order": "desc"}}], + } + + must_list = [{"term": {"playlist_active": {"value": True}}}] + if subscribed_only: + must_list.append({"term": {"playlist_subscribed": {"value": True}}}) + + data = {"query": {"bool": {"must": must_list}}} # type: ignore + if source: + data["_source"] = source # type: ignore + + all_playlists = IndexPaginate("ta_playlist", data).get_results() + + return all_playlists + + def calc_is_watched(duration: float, position: float) -> bool: """considered watched based on duration position""" diff --git a/backend/common/src/urlparser.py b/backend/common/src/urlparser.py index 3c75c82f..c3b80690 100644 --- a/backend/common/src/urlparser.py +++ b/backend/common/src/urlparser.py @@ -4,6 +4,7 @@ Functionality: - identify vid_type if possible """ +from typing import Literal, NotRequired, TypedDict from urllib.parse import parse_qs, urlparse from common.src.ta_redis import RedisArchivist @@ -11,19 +12,28 @@ from download.src.yt_dlp_base import YtWrap from video.src.constants import VideoTypeEnum +class ParsedURLType(TypedDict): + """represents single parsed url""" + + type: Literal["video", "channel", "playlist"] + url: str + vid_type: VideoTypeEnum + limit: NotRequired[int | None] + + class Parser: """ take a multi line string and detect valid youtube ids channel handle lookup is cached, can be disabled for unittests """ - def __init__(self, url_str, use_cache=True): + def __init__(self, url_str: str, use_cache: bool = True): self.url_list = [i.strip() for i in url_str.split()] self.use_cache = use_cache - def parse(self): + def parse(self) -> list[ParsedURLType]: """parse the list""" - ids = [] + ids: list[ParsedURLType] = [] for url in self.url_list: parsed = urlparse(url) if parsed.netloc: diff --git a/backend/download/src/queue.py b/backend/download/src/queue.py index 1451e307..a35b985b 100644 --- a/backend/download/src/queue.py +++ b/backend/download/src/queue.py @@ -9,10 +9,16 @@ from datetime import datetime from appsettings.src.config import AppConfig from channel.src.index import YoutubeChannel +from channel.src.remote_query import get_last_channel_videos from common.src.es_connect import ElasticWrap, IndexPaginate -from common.src.helper import get_duration_str, is_shorts, rand_sleep +from common.src.helper import ( + get_channels, + get_duration_str, + is_shorts, + rand_sleep, +) +from common.src.urlparser import ParsedURLType from download.src.queue_interact import PendingInteract -from download.src.subscriptions import ChannelSubscription from download.src.thumbnails import ThumbManager from playlist.src.index import YoutubePlaylist from video.src.constants import VideoTypeEnum @@ -64,11 +70,7 @@ class PendingIndex: """get a list of all channels indexed""" self.all_channels = [] self.channel_overwrites = {} - data = { - "query": {"match_all": {}}, - "sort": [{"channel_id": {"order": "asc"}}], - } - channels = IndexPaginate("ta_channel", data).get_results() + channels = get_channels(subscribed_only=False) for channel in channels: channel_id = channel["channel_id"] @@ -102,7 +104,11 @@ class PendingList(PendingIndex): } def __init__( - self, youtube_ids=False, task=False, auto_start=False, flat=False + self, + youtube_ids: list[ParsedURLType], + task=None, + auto_start=False, + flat=False, ): super().__init__() self.config = AppConfig().config @@ -111,7 +117,7 @@ class PendingList(PendingIndex): self.auto_start = auto_start self.flat = flat self.to_skip = False - self.missing_videos = [] + self.missing_videos: list[dict] = [] def parse_url_list(self): """extract youtube ids from list""" @@ -139,9 +145,9 @@ class PendingList(PendingIndex): entry["url"], entry["vid_type"], idx_url, total_url ) elif entry["type"] == "channel": - self._parse_channel(entry["url"], entry["vid_type"]) + self._parse_channel(entry) elif entry["type"] == "playlist": - self._parse_playlist(entry["url"]) + self._parse_playlist(entry["url"], entry.get("limit")) else: raise ValueError(f"invalid url_type: {entry}") @@ -166,11 +172,20 @@ class PendingList(PendingIndex): if to_add: self.missing_videos.append(to_add) - def _parse_channel(self, url, vid_type): + def _parse_channel(self, entry): """parse channel""" - query_filter = getattr(VideoTypeEnum, vid_type.upper()) - video_results = ChannelSubscription().get_last_youtube_videos( - url, limit=False, query_filter=query_filter + url = entry["url"] + vid_type = entry["vid_type"] + if isinstance(vid_type, str): + # lookup enum + vid_type = getattr(VideoTypeEnum, vid_type.upper()) + + limit = entry.get("limit") + video_results = get_last_channel_videos( + channel_id=url, + config=self.config, + limit=limit, + query_filter=vid_type, ) if not video_results: print(f"{url}: no videos to add from channel, skipping") @@ -184,45 +199,54 @@ class PendingList(PendingIndex): total = len(video_results) for idx, video_data in enumerate(video_results): - video_id = video_data["id"] - if video_id in self.to_skip: - continue + to_add = self.__parse_channel_video( + video_data, vid_type, idx, total, channel_handler + ) + if to_add: + self.missing_videos.append(to_add) if self.task and self.task.is_stopped(): break - if self.flat: - if not video_data.get("channel"): - channel_name = channel_handler.json_data["channel_name"] - video_data["channel"] = channel_name + def __parse_channel_video( + self, video_data, vid_type, idx, total, channel_handler + ) -> dict | None: + """parse video of channel""" + video_id = video_data["id"] + if video_id in self.to_skip: + return None - if not video_data.get("channel_id"): - channel_id = channel_handler.json_data["channel_id"] - video_data["channel_id"] = channel_id + if self.flat: + if not video_data.get("channel"): + channel_name = channel_handler.json_data["channel_name"] + video_data["channel"] = channel_name - to_add = self._parse_entry( - youtube_id=video_id, - video_data=video_data, - url_type="channel", - idx=idx, - total=total, - ) - else: - to_add = self._parse_video( - video_id, - vid_type, - url_type="channel", - idx=idx, - total=total, - ) + if not video_data.get("channel_id"): + channel_id = channel_handler.json_data["channel_id"] + video_data["channel_id"] = channel_id - if to_add: - self.missing_videos.append(to_add) + to_add = self._parse_entry( + youtube_id=video_id, + video_data=video_data, + url_type="channel", + idx=idx, + total=total, + ) + else: + to_add = self._parse_video( + video_id, + vid_type, + url_type="channel", + idx=idx, + total=total, + ) - def _parse_playlist(self, url): + return to_add + + def _parse_playlist(self, url: str, limit: int | None): """fast parse playlist""" playlist = YoutubePlaylist(url) - playlist.update_playlist() + playlist.update_playlist(limit=limit) if not playlist.youtube_meta: print(f"{url}: playlist metadata extraction failed, skipping") return diff --git a/backend/download/src/subscriptions.py b/backend/download/src/subscriptions.py index 65080dd7..3c8df7dc 100644 --- a/backend/download/src/subscriptions.py +++ b/backend/download/src/subscriptions.py @@ -6,323 +6,97 @@ Functionality: from appsettings.src.config import AppConfig from channel.src.index import YoutubeChannel -from common.src.es_connect import IndexPaginate -from common.src.helper import is_missing, rand_sleep -from common.src.urlparser import Parser -from download.src.thumbnails import ThumbManager -from download.src.yt_dlp_base import YtWrap +from channel.src.remote_query import VideoQueryBuilder +from common.src.helper import get_channels, get_playlists +from common.src.urlparser import ParsedURLType, Parser +from download.src.queue import PendingList from playlist.src.index import YoutubePlaylist from video.src.constants import VideoTypeEnum from video.src.index import YoutubeVideo class ChannelSubscription: - """manage the list of channels subscribed""" + """scan subscribed channels to find missing videos to add to pending""" - def __init__(self, task=False): + def __init__(self, task=None): self.config = AppConfig().config self.task = task - @staticmethod - def get_channels(subscribed_only=True): - """get a list of all channels subscribed to""" - data = { - "sort": [{"channel_name.keyword": {"order": "asc"}}], - } - if subscribed_only: - data["query"] = {"term": {"channel_subscribed": {"value": True}}} - else: - data["query"] = {"match_all": {}} - - all_channels = IndexPaginate("ta_channel", data).get_results() - - return all_channels - - def get_last_youtube_videos( - self, - channel_id, - limit=True, - query_filter=None, - channel_overwrites=None, - ): - """get a list of last videos from channel""" - query_handler = VideoQueryBuilder(self.config, channel_overwrites) - queries = query_handler.build_queries(query_filter) - last_videos = [] - - for vid_type_enum, limit_amount in queries: - obs = { - "skip_download": True, - "extract_flat": True, - } - vid_type = vid_type_enum.value - - if limit: - obs["playlistend"] = limit_amount - - url = f"https://www.youtube.com/channel/{channel_id}/{vid_type}" - channel_query, _ = YtWrap(obs, self.config).extract(url) - if not channel_query: - continue - - for entry in channel_query["entries"]: - entry["vid_type"] = vid_type - last_videos.append(entry) - - return last_videos - - def find_missing(self): - """add missing videos from subscribed channels to pending""" - all_channels = self.get_channels() + def find_missing(self) -> int: + """find missing videos from channel subscriptions""" + all_channels = get_channels( + subscribed_only=True, source=["channel_id", "channel_overwrites"] + ) if not all_channels: - return False + return 0 - missing_videos = [] + all_channel_urls: list[ParsedURLType] = [] - total = len(all_channels) - for idx, channel in enumerate(all_channels): - channel_id = channel["channel_id"] - print(f"{channel_id}: find missing videos.") - last_videos = self.get_last_youtube_videos( - channel_id, - channel_overwrites=channel.get("channel_overwrites"), - ) + for channel in all_channels: + queries = VideoQueryBuilder( + config=self.config, + channel_overwrites=channel.get("channel_overwrites", {}), + ).build_queries(video_type=None) - if last_videos: - ids_to_add = is_missing([i[0] for i in last_videos]) - for video_entry in last_videos: - video_id = video_entry["id"] - vid_type = video_entry["vid_type"] - if video_id in ids_to_add: - missing_videos.append((video_id, vid_type)) + for query in queries: + all_channel_urls.append( + ParsedURLType( + type="channel", + url=channel["channel_id"], + vid_type=query[0], + limit=query[1], + ) + ) - if not self.task: - continue - - if self.task.is_stopped(): - self.task.send_progress(["Received Stop signal."]) - break - - self.task.send_progress( - message_lines=[f"Scanning Channel {idx + 1}/{total}"], - progress=(idx + 1) / total, - ) - rand_sleep(self.config) - - return missing_videos - - @staticmethod - def change_subscribe(channel_id, channel_subscribed): - """subscribe or unsubscribe from channel and update""" - channel = YoutubeChannel(channel_id) - channel.build_json() - channel.json_data["channel_subscribed"] = channel_subscribed - channel.upload_to_es() - channel.sync_to_videos() - - return channel.json_data - - -class VideoQueryBuilder: - """Build queries for yt-dlp.""" - - def __init__(self, config: dict, channel_overwrites: dict | None = None): - self.config = config - self.channel_overwrites = channel_overwrites or {} - - def build_queries( - self, video_type: VideoTypeEnum | None, limit: bool = True - ) -> list[tuple[VideoTypeEnum, int | None]]: - """Build queries for all or specific video type.""" - query_methods = { - VideoTypeEnum.VIDEOS: self.videos_query, - VideoTypeEnum.STREAMS: self.streams_query, - VideoTypeEnum.SHORTS: self.shorts_query, - } - - if video_type: - # build query for specific type - query_method = query_methods.get(video_type) - if query_method: - query = query_method(limit) - if query[1] != 0: - return [query] - return [] - - # Build and return queries for all video types - queries = [] - for build_query in query_methods.values(): - query = build_query(limit) - if query[1] != 0: - queries.append(query) - - return queries - - def videos_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]: - """Build query for videos.""" - return self._build_generic_query( - video_type=VideoTypeEnum.VIDEOS, - overwrite_key="subscriptions_channel_size", - config_key="channel_size", - limit=limit, + pending_handler = PendingList( + youtube_ids=all_channel_urls, + task=self.task, + auto_start=self.config["subscriptions"].get("auto_start", False), + flat=self.config["subscriptions"].get("extract_flat", False), ) + pending_handler.parse_url_list() + added = pending_handler.add_to_pending() - def streams_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]: - """Build query for streams.""" - return self._build_generic_query( - video_type=VideoTypeEnum.STREAMS, - overwrite_key="subscriptions_live_channel_size", - config_key="live_channel_size", - limit=limit, - ) - - def shorts_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]: - """Build query for shorts.""" - return self._build_generic_query( - video_type=VideoTypeEnum.SHORTS, - overwrite_key="subscriptions_shorts_channel_size", - config_key="shorts_channel_size", - limit=limit, - ) - - def _build_generic_query( - self, - video_type: VideoTypeEnum, - overwrite_key: str, - config_key: str, - limit: bool, - ) -> tuple[VideoTypeEnum, int | None]: - """Generic query for video page scraping.""" - if not limit: - return (video_type, None) - - if ( - overwrite_key in self.channel_overwrites - and self.channel_overwrites[overwrite_key] is not None - ): - overwrite = self.channel_overwrites[overwrite_key] - return (video_type, overwrite) - - if overwrite := self.config["subscriptions"].get(config_key): - return (video_type, overwrite) - - return (video_type, 0) + return added class PlaylistSubscription: - """manage the playlist download functionality""" + """scan subscribed playlists for videos to add to pending""" - def __init__(self, task=False): + def __init__(self, task=None): self.config = AppConfig().config self.task = task - @staticmethod - def get_playlists(subscribed_only=True): - """get a list of all active playlists""" - data = { - "sort": [{"playlist_channel.keyword": {"order": "desc"}}], - } - data["query"] = { - "bool": {"must": [{"term": {"playlist_active": {"value": True}}}]} - } - if subscribed_only: - data["query"]["bool"]["must"].append( - {"term": {"playlist_subscribed": {"value": True}}} - ) - - all_playlists = IndexPaginate("ta_playlist", data).get_results() - - return all_playlists - - def process_url_str(self, new_playlists, subscribed=True): - """process playlist subscribe form url_str""" - for idx, playlist in enumerate(new_playlists): - playlist_id = playlist["url"] - if not playlist["type"] == "playlist": - print(f"{playlist_id} not a playlist, skipping...") - continue - - playlist_h = YoutubePlaylist(playlist_id) - playlist_h.build_json() - if not playlist_h.json_data: - message = f"{playlist_h.youtube_id}: failed to extract data" - print(message) - raise ValueError(message) - - playlist_h.json_data["playlist_subscribed"] = subscribed - playlist_h.upload_to_es() - playlist_h.add_vids_to_playlist() - self.channel_validate(playlist_h.json_data["playlist_channel_id"]) - - url = playlist_h.json_data["playlist_thumbnail"] - thumb = ThumbManager(playlist_id, item_type="playlist") - thumb.download_playlist_thumb(url) - - if self.task: - self.task.send_progress( - message_lines=[ - f"Processing {idx + 1} of {len(new_playlists)}" - ], - progress=(idx + 1) / len(new_playlists), - ) - - @staticmethod - def channel_validate(channel_id): - """make sure channel of playlist is there""" - channel = YoutubeChannel(channel_id) - channel.build_json(upload=True) - - @staticmethod - def change_subscribe(playlist_id, subscribe_status): - """change the subscribe status of a playlist""" - playlist = YoutubePlaylist(playlist_id) - playlist.build_json() - playlist.json_data["playlist_subscribed"] = subscribe_status - playlist.upload_to_es() - return playlist.json_data - - def find_missing(self): - """find videos in subscribed playlists not downloaded yet""" - all_playlists = [i["playlist_id"] for i in self.get_playlists()] + def find_missing(self) -> int: + """find missing""" + all_playlists = get_playlists( + subscribed_only=True, source=["playlist_id"] + ) if not all_playlists: - return False + return 0 - missing_videos = [] - total = len(all_playlists) - for idx, playlist_id in enumerate(all_playlists): - playlist = YoutubePlaylist(playlist_id) - is_active = playlist.update_playlist() - if not is_active: - playlist.deactivate() - continue - - playlist_entries = playlist.json_data["playlist_entries"] - size_limit = self.config["subscriptions"]["channel_size"] - if size_limit: - del playlist_entries[size_limit:] - - to_check = [ - i["youtube_id"] - for i in playlist_entries - if i["downloaded"] is False - ] - needs_downloading = is_missing(to_check) - missing_videos.extend(needs_downloading) - - if not self.task: - continue - - if self.task.is_stopped(): - self.task.send_progress(["Received Stop signal."]) - break - - self.task.send_progress( - message_lines=[f"Scanning Playlists {idx + 1}/{total}"], - progress=(idx + 1) / total, + size_limit = self.config["subscriptions"]["channel_size"] + all_playlist_urls: list[ParsedURLType] = [] + for playlist in all_playlists: + all_playlist_urls.append( + ParsedURLType( + type="playlist", + url=playlist["playlist_id"], + vid_type=VideoTypeEnum.UNKNOWN, + limit=size_limit, + ) ) - rand_sleep(self.config) - return missing_videos + pending_handler = PendingList( + youtube_ids=all_playlist_urls, + task=self.task, + auto_start=self.config["subscriptions"].get("auto_start", False), + flat=self.config["subscriptions"].get("extract_flat", False), + ) + pending_handler.parse_url_list() + added = pending_handler.add_to_pending() + + return added class SubscriptionScanner: @@ -338,40 +112,12 @@ class SubscriptionScanner: if self.task: self.task.send_progress(["Rescanning channels and playlists."]) - self.missing_videos = [] - self.scan_channels() + added = 0 + added += ChannelSubscription(task=self.task).find_missing() if self.task and not self.task.is_stopped(): - self.scan_playlists() + added += PlaylistSubscription(task=self.task).find_missing() - return self.missing_videos - - def scan_channels(self): - """get missing from channels""" - channel_handler = ChannelSubscription(task=self.task) - missing = channel_handler.find_missing() - if not missing: - return - - for vid_id, vid_type in missing: - self.missing_videos.append( - {"type": "video", "vid_type": vid_type, "url": vid_id} - ) - - def scan_playlists(self): - """get missing from playlists""" - playlist_handler = PlaylistSubscription(task=self.task) - missing = playlist_handler.find_missing() - if not missing: - return - - for i in missing: - self.missing_videos.append( - { - "type": "video", - "vid_type": VideoTypeEnum.VIDEOS.value, - "url": i, - } - ) + return added class SubscriptionHandler: @@ -403,7 +149,8 @@ class SubscriptionHandler: f"expected {expected_type} url but got {item.get('type')}" ) - PlaylistSubscription().process_url_str([item]) + playlist = YoutubePlaylist(item["url"]) + playlist.change_subscribe(new_subscribe_state=True) return if item["type"] == "video": @@ -426,9 +173,7 @@ class SubscriptionHandler: def _subscribe(self, channel_id): """subscribe to channel""" - _ = ChannelSubscription().change_subscribe( - channel_id, channel_subscribed=True - ) + YoutubeChannel(channel_id).change_subscribe(new_subscribe_state=True) def _notify(self, idx, item, total): """send notification message to redis""" diff --git a/backend/download/src/yt_dlp_handler.py b/backend/download/src/yt_dlp_handler.py index 6348e3ab..f74af4eb 100644 --- a/backend/download/src/yt_dlp_handler.py +++ b/backend/download/src/yt_dlp_handler.py @@ -16,12 +16,12 @@ from common.src.env_settings import EnvironmentSettings from common.src.es_connect import ElasticWrap, IndexPaginate from common.src.helper import ( get_channel_overwrites, + get_playlists, ignore_filelist, rand_sleep, ) from common.src.ta_redis import RedisQueue from download.src.queue import PendingList -from download.src.subscriptions import PlaylistSubscription from download.src.yt_dlp_base import YtWrap from playlist.src.index import YoutubePlaylist from video.src.comments import CommentList @@ -403,8 +403,8 @@ class DownloadPostProcess(DownloaderBase): def _add_playlist_sub(self): """add subscribed playlists to refresh""" - subs = PlaylistSubscription().get_playlists() - to_add = [i["playlist_id"] for i in subs] + playlists = get_playlists(subscribed_only=True, source=["playlist_id"]) + to_add = [i["playlist_id"] for i in playlists] RedisQueue(self.PLAYLIST_QUEUE).add_list(to_add) def _add_channel_playlists(self): diff --git a/backend/playlist/src/index.py b/backend/playlist/src/index.py index d0d938d3..165860dc 100644 --- a/backend/playlist/src/index.py +++ b/backend/playlist/src/index.py @@ -30,7 +30,7 @@ class YoutubePlaylist(YouTubeItem): self.all_members = False self.nav = False - def build_json(self, scrape=False): + def build_json(self, scrape=False, limit: int | None = None): """collection to create json_data""" self.get_from_es() if self.json_data: @@ -40,7 +40,9 @@ class YoutubePlaylist(YouTubeItem): subscribed = False playlist_sort_order = "top" - playlist_items = "::1" if playlist_sort_order == "top" else "::-1" + limit_str = limit if limit is not None else "" + sort_order = 1 if playlist_sort_order == "top" else -1 + playlist_items = f":{limit_str}:{sort_order}" if scrape or not self.json_data: self.get_from_youtube( @@ -137,6 +139,13 @@ class YoutubePlaylist(YouTubeItem): url = self.json_data["playlist_thumbnail"] ThumbManager(self.youtube_id, item_type="playlist").download(url) + def change_subscribe(self, new_subscribe_state: bool): + """change subscribe status""" + self.build_json() + self.json_data["playlist_subscribed"] = new_subscribe_state + self.upload_to_es() + return self.json_data + def add_vids_to_playlist(self): """sync the playlist id to videos""" script = ( @@ -189,9 +198,9 @@ class YoutubePlaylist(YouTubeItem): if status_code == 200: print(f"{self.youtube_id}: removed {video_id} from playlist") - def update_playlist(self, skip_on_empty=False): + def update_playlist(self, skip_on_empty=False, limit: int | None = None): """update metadata for playlist with data from YouTube""" - self.build_json(scrape=True) + self.build_json(scrape=True, limit=limit) if not self.json_data: # return false to deactivate return False diff --git a/backend/playlist/views.py b/backend/playlist/views.py index 1ce9261a..bb756081 100644 --- a/backend/playlist/views.py +++ b/backend/playlist/views.py @@ -7,7 +7,6 @@ from common.serializers import ( ErrorResponseSerializer, ) from common.views_base import AdminWriteOnly, ApiBaseView -from download.src.subscriptions import PlaylistSubscription from drf_spectacular.utils import OpenApiResponse, extend_schema from playlist.serializers import ( PlaylistBulkAddSerializer, @@ -245,8 +244,9 @@ class PlaylistApiView(ApiBaseView): json_data = None if subscribed is not None: - playlist_sub = PlaylistSubscription() - json_data = playlist_sub.change_subscribe(playlist_id, subscribed) + json_data = YoutubePlaylist(playlist_id).change_subscribe( + new_subscribe_state=subscribed + ) if sort_order: json_data = YoutubePlaylist(playlist_id).change_sort_order( diff --git a/backend/task/tasks.py b/backend/task/tasks.py index cb76b8e8..63a843f0 100644 --- a/backend/task/tasks.py +++ b/backend/task/tasks.py @@ -103,13 +103,13 @@ def update_subscribed(self): manager.init(self) handler = SubscriptionScanner(task=self) - missing_videos = handler.scan() + added = handler.scan() auto_start = handler.auto_start - if missing_videos: - print(missing_videos) - extrac_dl.delay(missing_videos, auto_start=auto_start) - message = f"Found {len(missing_videos)} videos to add to the queue." - return message + if added: + if auto_start: + download_pending.delay(auto_only=True) + + return f"Found {added} videos to add to the queue." return None diff --git a/frontend/src/api/loader/loadAppsettingsConfig.ts b/frontend/src/api/loader/loadAppsettingsConfig.ts index 44f22e8a..3a6489fd 100644 --- a/frontend/src/api/loader/loadAppsettingsConfig.ts +++ b/frontend/src/api/loader/loadAppsettingsConfig.ts @@ -6,6 +6,7 @@ export type AppSettingsConfigType = { live_channel_size: number | null; shorts_channel_size: number | null; auto_start: boolean; + extract_flat: boolean; }; downloads: { limit_speed: number | null; diff --git a/frontend/src/pages/SettingsApplication.tsx b/frontend/src/pages/SettingsApplication.tsx index 594ef29c..d4a45f7f 100644 --- a/frontend/src/pages/SettingsApplication.tsx +++ b/frontend/src/pages/SettingsApplication.tsx @@ -42,6 +42,7 @@ const SettingsApplication = () => { const [livePageSize, setLivePageSize] = useState(null); const [shortPageSize, setShortPageSize] = useState(null); const [isAutostart, setIsAutostart] = useState(false); + const [isExtractFlat, setIsExtractFlat] = useState(false); // Downloads const [currentDownloadSpeed, setCurrentDownloadSpeed] = useState(null); @@ -98,6 +99,7 @@ const SettingsApplication = () => { setLivePageSize(appSettingsConfigData?.subscriptions.live_channel_size || null); setShortPageSize(appSettingsConfigData?.subscriptions.shorts_channel_size || null); setIsAutostart(appSettingsConfigData?.subscriptions.auto_start || false); + setIsExtractFlat(appSettingsConfigData?.subscriptions.extract_flat || false); // Downloads setCurrentDownloadSpeed(appSettingsConfigData?.downloads.limit_speed || null); @@ -212,6 +214,10 @@ const SettingsApplication = () => { Autostart automatically starts downloading videos from subscriptions with priority. +
  • + Fast add extracts and adds videos in bulk. That is much faster but is not able + to extract as much metadata during adding to the queue. +
  • )} @@ -264,6 +270,16 @@ const SettingsApplication = () => { updateCallback={handleUpdateConfig} /> +
    +
    +

    Fast add

    +
    + +

    Downloads

    diff --git a/frontend/src/stores/AppSettingsStore.ts b/frontend/src/stores/AppSettingsStore.ts index 2166e7e5..d3410431 100644 --- a/frontend/src/stores/AppSettingsStore.ts +++ b/frontend/src/stores/AppSettingsStore.ts @@ -13,6 +13,7 @@ export const useAppSettingsStore = create(set => ({ live_channel_size: null, shorts_channel_size: null, auto_start: false, + extract_flat: false, }, downloads: { limit_speed: null,