RomanNum3ral
/
tubearchivist
mirror of https://github.com/tubearchivist/tubearchivist.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

implement bulk add subscriptions in appsettings

This commit is contained in:
Simon 2025-07-11 16:45:00 +07:00
parent 05ce2a7034
commit fa7643e903
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
17 changed files with 486 additions and 486 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backend/appsettings/serializers.py
View File

@ -25,6 +25,7 @@ class AppConfigSubSerializer(
live_channel_size = serializers.IntegerField(required=False)
shorts_channel_size = serializers.IntegerField(required=False)
auto_start = serializers.BooleanField(required=False)
extract_flat = serializers.BooleanField(required=False)
class AppConfigDownloadsSerializer(

2
backend/appsettings/src/config.py
View File

@ -22,6 +22,7 @@ class SubscriptionsConfigType(TypedDict):
live_channel_size: int
shorts_channel_size: int
auto_start: bool
extract_flat: bool
class DownloadsConfigType(TypedDict):
@ -73,6 +74,7 @@ class AppConfig:
"live_channel_size": 50,
"shorts_channel_size": 50,
"auto_start": False,
"extract_flat": False,
},
"downloads": {
"limit_speed": None,

12
backend/appsettings/src/reindex.py
View File

@ -11,11 +11,11 @@ from typing import Callable, TypedDict
from appsettings.src.config import AppConfig
from channel.src.index import YoutubeChannel
from channel.src.remote_query import get_last_channel_videos
from common.src.env_settings import EnvironmentSettings
from common.src.es_connect import ElasticWrap, IndexPaginate
from common.src.helper import rand_sleep
from common.src.ta_redis import RedisQueue
from download.src.subscriptions import ChannelSubscription
from download.src.thumbnails import ThumbManager
from download.src.yt_dlp_base import CookieHandler
from playlist.src.index import YoutubePlaylist
@ -376,7 +376,7 @@ class Reindex(ReindexBase):
channel.upload_to_es()
channel.sync_to_videos()
ChannelFullScan(channel_id).scan()
ChannelFullScan(channel_id, self.config).scan()
self.processed["channels"] += 1
def _reindex_single_playlist(self, playlist_id: str) -> None:
@ -495,8 +495,9 @@ class ReindexProgress(ReindexBase):
class ChannelFullScan:
"""full scan of channel to fix vid_type mismatch"""
def __init__(self, channel_id):
def __init__(self, channel_id, config):
self.channel_id = channel_id
self.config = config
self.to_update = False
def scan(self):
@ -527,9 +528,8 @@ class ChannelFullScan:
def _get_all_remote(self):
"""get all channel videos"""
sub = ChannelSubscription()
all_remote_videos = sub.get_last_youtube_videos(
self.channel_id, limit=False
all_remote_videos = get_last_channel_videos(
self.channel_id, self.config, limit=False
)
return all_remote_videos

198
backend/channel/src/index.py
View File

@ -155,48 +155,15 @@ class YoutubeChannel(YouTubeItem):
update_path = f"ta_video/_update_by_query?pipeline={self.youtube_id}"
_, _ = ElasticWrap(update_path).post(data)
def get_folder_path(self):
"""get folder where media files get stored"""
folder_path = os.path.join(
EnvironmentSettings.MEDIA_DIR,
self.json_data["channel_id"],
)
return folder_path
def change_subscribe(self, new_subscribe_state: bool):
"""change subscribe status"""
if not self.json_data:
self.build_json()
def delete_es_videos(self):
"""delete all channel documents from elasticsearch"""
data = {
"query": {
"term": {"channel.channel_id": {"value": self.youtube_id}}
}
}
_, _ = ElasticWrap("ta_video/_delete_by_query").post(data)
def delete_es_comments(self):
"""delete all comments from this channel"""
data = {
"query": {
"term": {"comment_channel_id": {"value": self.youtube_id}}
}
}
_, _ = ElasticWrap("ta_comment/_delete_by_query").post(data)
def delete_es_subtitles(self):
"""delete all subtitles from this channel"""
data = {
"query": {
"term": {"subtitle_channel_id": {"value": self.youtube_id}}
}
}
_, _ = ElasticWrap("ta_subtitle/_delete_by_query").post(data)
def delete_playlists(self):
"""delete all indexed playlist from es"""
from playlist.src.index import YoutubePlaylist
all_playlists = self.get_indexed_playlists()
for playlist in all_playlists:
YoutubePlaylist(playlist["playlist_id"]).delete_metadata()
self.json_data["channel_subscribed"] = new_subscribe_state
self.upload_to_es()
self.sync_to_videos()
return self.json_data
def delete_channel(self):
"""delete channel and all videos"""
@ -205,24 +172,7 @@ class YoutubeChannel(YouTubeItem):
if not self.json_data:
raise FileNotFoundError
folder_path = self.get_folder_path()
print(f"{self.youtube_id}: delete all media files")
try:
all_videos = os.listdir(folder_path)
for video in all_videos:
video_path = os.path.join(folder_path, video)
os.remove(video_path)
os.rmdir(folder_path)
except FileNotFoundError:
print(f"no videos found for {folder_path}")
print(f"{self.youtube_id}: delete indexed playlists")
self.delete_playlists()
print(f"{self.youtube_id}: delete indexed videos")
self.delete_es_videos()
self.delete_es_comments()
self.delete_es_subtitles()
self.del_in_es()
ChannelDelete(json_data=self.json_data).delete()
def index_channel_playlists(self):
"""add all playlists of channel to index"""
@ -244,6 +194,21 @@ class YoutubeChannel(YouTubeItem):
print("add playlist: " + playlist[1])
rand_sleep(self.config)
def get_all_playlists(self):
"""get all playlists owned by this channel"""
url = (
f"https://www.youtube.com/channel/{self.youtube_id}"
+ "/playlists?view=1&sort=dd&shelf_id=0"
)
obs = {"skip_download": True, "extract_flat": True}
playlists, _ = YtWrap(obs, self.config).extract(url)
if not playlists:
self.all_playlists = []
return
all_entries = [(i["id"], i["title"]) for i in playlists["entries"]]
self.all_playlists = all_entries
def _notify_single_playlist(self, idx, total):
"""send notification"""
channel_name = self.json_data["channel_name"]
@ -272,34 +237,6 @@ class YoutubeChannel(YouTubeItem):
all_videos = IndexPaginate("ta_video", data).get_results()
return all_videos
def get_all_playlists(self):
"""get all playlists owned by this channel"""
url = (
f"https://www.youtube.com/channel/{self.youtube_id}"
+ "/playlists?view=1&sort=dd&shelf_id=0"
)
obs = {"skip_download": True, "extract_flat": True}
playlists, _ = YtWrap(obs, self.config).extract(url)
if not playlists:
self.all_playlists = []
return
all_entries = [(i["id"], i["title"]) for i in playlists["entries"]]
self.all_playlists = all_entries
def get_indexed_playlists(self, active_only=False):
"""get all indexed playlists from channel"""
must_list = [
{"term": {"playlist_channel_id": {"value": self.youtube_id}}}
]
if active_only:
must_list.append({"term": {"playlist_active": {"value": True}}})
data = {"query": {"bool": {"must": must_list}}}
all_playlists = IndexPaginate("ta_playlist", data).get_results()
return all_playlists
def get_overwrites(self) -> dict:
"""get all per channel overwrites"""
return self.json_data.get("channel_overwrites", {})
@ -330,6 +267,93 @@ class YoutubeChannel(YouTubeItem):
self.json_data["channel_overwrites"] = to_write
class ChannelDelete(YouTubeItem):
"""delete and cleanup"""
index_name = "ta_channel"
def __init__(self, json_data):
super().__init__(youtube_id=json_data["channel_id"])
self.json_data = json_data
def delete(self):
"""delete channel and all videos"""
folder_path = self._get_folder_path()
print(f"{self.youtube_id}: delete all media files")
try:
all_videos = os.listdir(folder_path)
for video in all_videos:
video_path = os.path.join(folder_path, video)
os.remove(video_path)
os.rmdir(folder_path)
except FileNotFoundError:
print(f"no videos found for {folder_path}")
print(f"{self.youtube_id}: delete indexed playlists")
self._delete_playlists()
print(f"{self.youtube_id}: delete indexed videos")
self._delete_es_videos()
self._delete_es_comments()
self._delete_es_subtitles()
self.del_in_es()
def _get_folder_path(self):
"""get folder where media files get stored"""
folder_path = os.path.join(
EnvironmentSettings.MEDIA_DIR,
self.json_data["channel_id"],
)
return folder_path
def _delete_es_videos(self):
"""delete all channel documents from elasticsearch"""
data = {
"query": {
"term": {"channel.channel_id": {"value": self.youtube_id}}
}
}
_, _ = ElasticWrap("ta_video/_delete_by_query").post(data)
def _delete_es_comments(self):
"""delete all comments from this channel"""
data = {
"query": {
"term": {"comment_channel_id": {"value": self.youtube_id}}
}
}
_, _ = ElasticWrap("ta_comment/_delete_by_query").post(data)
def _delete_es_subtitles(self):
"""delete all subtitles from this channel"""
data = {
"query": {
"term": {"subtitle_channel_id": {"value": self.youtube_id}}
}
}
_, _ = ElasticWrap("ta_subtitle/_delete_by_query").post(data)
def _delete_playlists(self):
"""delete all indexed playlist from es"""
from playlist.src.index import YoutubePlaylist
all_playlists = self._get_indexed_playlists()
for playlist in all_playlists:
YoutubePlaylist(playlist["playlist_id"]).delete_metadata()
def _get_indexed_playlists(self, active_only=False):
"""get all indexed playlists from channel"""
must_list = [
{"term": {"playlist_channel_id": {"value": self.youtube_id}}}
]
if active_only:
must_list.append({"term": {"playlist_active": {"value": True}}})
data = {"query": {"bool": {"must": must_list}}}
all_playlists = IndexPaginate("ta_playlist", data).get_results()
return all_playlists
def channel_overwrites(channel_id, overwrites):
"""collection to overwrite settings per channel"""
channel = YoutubeChannel(channel_id)

124
backend/channel/src/remote_query.py Normal file
View File

@ -0,0 +1,124 @@
"""build queries for video extraction from channel subscriptions"""
from download.src.yt_dlp_base import YtWrap
from video.src.constants import VideoTypeEnum
class VideoQueryBuilder:
"""Build queries for yt-dlp."""
def __init__(self, config: dict, channel_overwrites: dict | None = None):
self.config = config
self.channel_overwrites = channel_overwrites or {}
def build_queries(
self, video_type: VideoTypeEnum | None, limit: bool = True
) -> list[tuple[VideoTypeEnum, int | None]]:
"""Build queries for all or specific video type."""
query_methods = {
VideoTypeEnum.VIDEOS: self.videos_query,
VideoTypeEnum.STREAMS: self.streams_query,
VideoTypeEnum.SHORTS: self.shorts_query,
}
if video_type:
# build query for specific type
query_method = query_methods.get(video_type)
if query_method:
query = query_method(limit)
if query[1] != 0:
return [query]
return []
# Build and return queries for all video types
queries = []
for build_query in query_methods.values():
query = build_query(limit)
if query[1] != 0:
queries.append(query)
return queries
def videos_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
"""Build query for videos."""
return self._build_generic_query(
video_type=VideoTypeEnum.VIDEOS,
overwrite_key="subscriptions_channel_size",
config_key="channel_size",
limit=limit,
)
def streams_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
"""Build query for streams."""
return self._build_generic_query(
video_type=VideoTypeEnum.STREAMS,
overwrite_key="subscriptions_live_channel_size",
config_key="live_channel_size",
limit=limit,
)
def shorts_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
"""Build query for shorts."""
return self._build_generic_query(
video_type=VideoTypeEnum.SHORTS,
overwrite_key="subscriptions_shorts_channel_size",
config_key="shorts_channel_size",
limit=limit,
)
def _build_generic_query(
self,
video_type: VideoTypeEnum,
overwrite_key: str,
config_key: str,
limit: bool,
) -> tuple[VideoTypeEnum, int | None]:
"""Generic query for video page scraping."""
if not limit:
return (video_type, None)
if (
overwrite_key in self.channel_overwrites
and self.channel_overwrites[overwrite_key] is not None
):
overwrite = self.channel_overwrites[overwrite_key]
return (video_type, overwrite)
if overwrite := self.config["subscriptions"].get(config_key):
return (video_type, overwrite)
return (video_type, 0)
def get_last_channel_videos(
channel_id,
config,
limit=None,
query_filter=None,
channel_overwrites=None,
):
"""get a list of last videos from channel"""
query_handler = VideoQueryBuilder(config, channel_overwrites)
queries = query_handler.build_queries(query_filter)
last_videos = []
for vid_type_enum, limit_amount in queries:
obs = {
"skip_download": True,
"extract_flat": True,
}
vid_type = vid_type_enum.value
if limit is not None:
obs.update({"playlist_items": f":{limit_amount}:1"})
url = f"https://www.youtube.com/channel/{channel_id}/{vid_type}"
channel_query, _ = YtWrap(obs, config).extract(url)
if not channel_query:
continue
for entry in channel_query["entries"]:
entry["vid_type"] = vid_type
last_videos.append(entry)
return last_videos

9
backend/channel/views.py
View File

@ -14,7 +14,6 @@ from channel.src.nav import ChannelNav
from common.serializers import ErrorResponseSerializer
from common.src.urlparser import Parser
from common.views_base import AdminWriteOnly, ApiBaseView
from download.src.subscriptions import ChannelSubscription
from drf_spectacular.utils import (
OpenApiParameter,
OpenApiResponse,
@ -89,9 +88,7 @@ class ChannelApiListView(ApiBaseView):
def _unsubscribe(channel_id: str):
"""unsubscribe"""
print(f"[{channel_id}] unsubscribe from channel")
ChannelSubscription().change_subscribe(
channel_id, channel_subscribed=False
)
YoutubeChannel(channel_id).change_subscribe(new_subscribe_state=False)
class ChannelApiView(ApiBaseView):
@ -146,7 +143,9 @@ class ChannelApiView(ApiBaseView):
subscribed = validated_data.get("channel_subscribed")
if subscribed is not None:
ChannelSubscription().change_subscribe(channel_id, subscribed)
YoutubeChannel(channel_id).change_subscribe(
new_subscribe_state=subscribed
)
overwrites = validated_data.get("channel_overwrites")
if overwrites:

44
backend/common/src/helper.py
View File

@ -292,6 +292,50 @@ def get_channel_overwrites() -> dict[str, dict[str, Any]]:
return overwrites
def get_channels(
subscribed_only: bool, source: list[str] | None = None
) -> list[dict]:
"""get a list of all channels"""
data = {
"sort": [{"channel_name.keyword": {"order": "asc"}}],
}
if subscribed_only:
query = {"term": {"channel_subscribed": {"value": True}}}
else:
query = {"match_all": {}}
data["query"] = query # type: ignore
if source:
data["_source"] = source # type: ignore
all_channels = IndexPaginate("ta_channel", data).get_results()
return all_channels
def get_playlists(
subscribed_only: bool, source: list[str] | None = None
) -> list[dict]:
"""get list of playlists"""
data = {
"sort": [{"playlist_channel.keyword": {"order": "desc"}}],
}
must_list = [{"term": {"playlist_active": {"value": True}}}]
if subscribed_only:
must_list.append({"term": {"playlist_subscribed": {"value": True}}})
data = {"query": {"bool": {"must": must_list}}} # type: ignore
if source:
data["_source"] = source # type: ignore
all_playlists = IndexPaginate("ta_playlist", data).get_results()
return all_playlists
def calc_is_watched(duration: float, position: float) -> bool:
"""considered watched based on duration position"""

16
backend/common/src/urlparser.py
View File

@ -4,6 +4,7 @@ Functionality:
- identify vid_type if possible
"""
from typing import Literal, NotRequired, TypedDict
from urllib.parse import parse_qs, urlparse
from common.src.ta_redis import RedisArchivist
@ -11,19 +12,28 @@ from download.src.yt_dlp_base import YtWrap
from video.src.constants import VideoTypeEnum
class ParsedURLType(TypedDict):
"""represents single parsed url"""
type: Literal["video", "channel", "playlist"]
url: str
vid_type: VideoTypeEnum
limit: NotRequired[int | None]
class Parser:
"""
take a multi line string and detect valid youtube ids
channel handle lookup is cached, can be disabled for unittests
"""
def __init__(self, url_str, use_cache=True):
def __init__(self, url_str: str, use_cache: bool = True):
self.url_list = [i.strip() for i in url_str.split()]
self.use_cache = use_cache
def parse(self):
def parse(self) -> list[ParsedURLType]:
"""parse the list"""
ids = []
ids: list[ParsedURLType] = []
for url in self.url_list:
parsed = urlparse(url)
if parsed.netloc:

112
backend/download/src/queue.py
View File

@ -9,10 +9,16 @@ from datetime import datetime
from appsettings.src.config import AppConfig
from channel.src.index import YoutubeChannel
from channel.src.remote_query import get_last_channel_videos
from common.src.es_connect import ElasticWrap, IndexPaginate
from common.src.helper import get_duration_str, is_shorts, rand_sleep
from common.src.helper import (
get_channels,
get_duration_str,
is_shorts,
rand_sleep,
)
from common.src.urlparser import ParsedURLType
from download.src.queue_interact import PendingInteract
from download.src.subscriptions import ChannelSubscription
from download.src.thumbnails import ThumbManager
from playlist.src.index import YoutubePlaylist
from video.src.constants import VideoTypeEnum
@ -64,11 +70,7 @@ class PendingIndex:
"""get a list of all channels indexed"""
self.all_channels = []
self.channel_overwrites = {}
data = {
"query": {"match_all": {}},
"sort": [{"channel_id": {"order": "asc"}}],
}
channels = IndexPaginate("ta_channel", data).get_results()
channels = get_channels(subscribed_only=False)
for channel in channels:
channel_id = channel["channel_id"]
@ -102,7 +104,11 @@ class PendingList(PendingIndex):
}
def __init__(
self, youtube_ids=False, task=False, auto_start=False, flat=False
self,
youtube_ids: list[ParsedURLType],
task=None,
auto_start=False,
flat=False,
):
super().__init__()
self.config = AppConfig().config
@ -111,7 +117,7 @@ class PendingList(PendingIndex):
self.auto_start = auto_start
self.flat = flat
self.to_skip = False
self.missing_videos = []
self.missing_videos: list[dict] = []
def parse_url_list(self):
"""extract youtube ids from list"""
@ -139,9 +145,9 @@ class PendingList(PendingIndex):
entry["url"], entry["vid_type"], idx_url, total_url
)
elif entry["type"] == "channel":
self._parse_channel(entry["url"], entry["vid_type"])
self._parse_channel(entry)
elif entry["type"] == "playlist":
self._parse_playlist(entry["url"])
self._parse_playlist(entry["url"], entry.get("limit"))
else:
raise ValueError(f"invalid url_type: {entry}")
@ -166,11 +172,20 @@ class PendingList(PendingIndex):
if to_add:
self.missing_videos.append(to_add)
def _parse_channel(self, url, vid_type):
def _parse_channel(self, entry):
"""parse channel"""
query_filter = getattr(VideoTypeEnum, vid_type.upper())
video_results = ChannelSubscription().get_last_youtube_videos(
url, limit=False, query_filter=query_filter
url = entry["url"]
vid_type = entry["vid_type"]
if isinstance(vid_type, str):
# lookup enum
vid_type = getattr(VideoTypeEnum, vid_type.upper())
limit = entry.get("limit")
video_results = get_last_channel_videos(
channel_id=url,
config=self.config,
limit=limit,
query_filter=vid_type,
)
if not video_results:
print(f"{url}: no videos to add from channel, skipping")
@ -184,45 +199,54 @@ class PendingList(PendingIndex):
total = len(video_results)
for idx, video_data in enumerate(video_results):
video_id = video_data["id"]
if video_id in self.to_skip:
continue
to_add = self.__parse_channel_video(
video_data, vid_type, idx, total, channel_handler
)
if to_add:
self.missing_videos.append(to_add)
if self.task and self.task.is_stopped():
break
if self.flat:
if not video_data.get("channel"):
channel_name = channel_handler.json_data["channel_name"]
video_data["channel"] = channel_name
def __parse_channel_video(
self, video_data, vid_type, idx, total, channel_handler
) -> dict | None:
"""parse video of channel"""
video_id = video_data["id"]
if video_id in self.to_skip:
return None
if not video_data.get("channel_id"):
channel_id = channel_handler.json_data["channel_id"]
video_data["channel_id"] = channel_id
if self.flat:
if not video_data.get("channel"):
channel_name = channel_handler.json_data["channel_name"]
video_data["channel"] = channel_name
to_add = self._parse_entry(
youtube_id=video_id,
video_data=video_data,
url_type="channel",
idx=idx,
total=total,
)
else:
to_add = self._parse_video(
video_id,
vid_type,
url_type="channel",
idx=idx,
total=total,
)
if not video_data.get("channel_id"):
channel_id = channel_handler.json_data["channel_id"]
video_data["channel_id"] = channel_id
if to_add:
self.missing_videos.append(to_add)
to_add = self._parse_entry(
youtube_id=video_id,
video_data=video_data,
url_type="channel",
idx=idx,
total=total,
)
else:
to_add = self._parse_video(
video_id,
vid_type,
url_type="channel",
idx=idx,
total=total,
)
def _parse_playlist(self, url):
return to_add
def _parse_playlist(self, url: str, limit: int | None):
"""fast parse playlist"""
playlist = YoutubePlaylist(url)
playlist.update_playlist()
playlist.update_playlist(limit=limit)
if not playlist.youtube_meta:
print(f"{url}: playlist metadata extraction failed, skipping")
return

395
backend/download/src/subscriptions.py
View File

@ -6,323 +6,97 @@ Functionality:
from appsettings.src.config import AppConfig
from channel.src.index import YoutubeChannel
from common.src.es_connect import IndexPaginate
from common.src.helper import is_missing, rand_sleep
from common.src.urlparser import Parser
from download.src.thumbnails import ThumbManager
from download.src.yt_dlp_base import YtWrap
from channel.src.remote_query import VideoQueryBuilder
from common.src.helper import get_channels, get_playlists
from common.src.urlparser import ParsedURLType, Parser
from download.src.queue import PendingList
from playlist.src.index import YoutubePlaylist
from video.src.constants import VideoTypeEnum
from video.src.index import YoutubeVideo
class ChannelSubscription:
"""manage the list of channels subscribed"""
"""scan subscribed channels to find missing videos to add to pending"""
def __init__(self, task=False):
def __init__(self, task=None):
self.config = AppConfig().config
self.task = task
@staticmethod
def get_channels(subscribed_only=True):
"""get a list of all channels subscribed to"""
data = {
"sort": [{"channel_name.keyword": {"order": "asc"}}],
}
if subscribed_only:
data["query"] = {"term": {"channel_subscribed": {"value": True}}}
else:
data["query"] = {"match_all": {}}
all_channels = IndexPaginate("ta_channel", data).get_results()
return all_channels
def get_last_youtube_videos(
self,
channel_id,
limit=True,
query_filter=None,
channel_overwrites=None,
):
"""get a list of last videos from channel"""
query_handler = VideoQueryBuilder(self.config, channel_overwrites)
queries = query_handler.build_queries(query_filter)
last_videos = []
for vid_type_enum, limit_amount in queries:
obs = {
"skip_download": True,
"extract_flat": True,
}
vid_type = vid_type_enum.value
if limit:
obs["playlistend"] = limit_amount
url = f"https://www.youtube.com/channel/{channel_id}/{vid_type}"
channel_query, _ = YtWrap(obs, self.config).extract(url)
if not channel_query:
continue
for entry in channel_query["entries"]:
entry["vid_type"] = vid_type
last_videos.append(entry)
return last_videos
def find_missing(self):
"""add missing videos from subscribed channels to pending"""
all_channels = self.get_channels()
def find_missing(self) -> int:
"""find missing videos from channel subscriptions"""
all_channels = get_channels(
subscribed_only=True, source=["channel_id", "channel_overwrites"]
)
if not all_channels:
return False
return 0
missing_videos = []
all_channel_urls: list[ParsedURLType] = []
total = len(all_channels)
for idx, channel in enumerate(all_channels):
channel_id = channel["channel_id"]
print(f"{channel_id}: find missing videos.")
last_videos = self.get_last_youtube_videos(
channel_id,
channel_overwrites=channel.get("channel_overwrites"),
)
for channel in all_channels:
queries = VideoQueryBuilder(
config=self.config,
channel_overwrites=channel.get("channel_overwrites", {}),
).build_queries(video_type=None)
if last_videos:
ids_to_add = is_missing([i[0] for i in last_videos])
for video_entry in last_videos:
video_id = video_entry["id"]
vid_type = video_entry["vid_type"]
if video_id in ids_to_add:
missing_videos.append((video_id, vid_type))
for query in queries:
all_channel_urls.append(
ParsedURLType(
type="channel",
url=channel["channel_id"],
vid_type=query[0],
limit=query[1],
)
)
if not self.task:
continue
if self.task.is_stopped():
self.task.send_progress(["Received Stop signal."])
break
self.task.send_progress(
message_lines=[f"Scanning Channel {idx + 1}/{total}"],
progress=(idx + 1) / total,
)
rand_sleep(self.config)
return missing_videos
@staticmethod
def change_subscribe(channel_id, channel_subscribed):
"""subscribe or unsubscribe from channel and update"""
channel = YoutubeChannel(channel_id)
channel.build_json()
channel.json_data["channel_subscribed"] = channel_subscribed
channel.upload_to_es()
channel.sync_to_videos()
return channel.json_data
class VideoQueryBuilder:
"""Build queries for yt-dlp."""
def __init__(self, config: dict, channel_overwrites: dict | None = None):
self.config = config
self.channel_overwrites = channel_overwrites or {}
def build_queries(
self, video_type: VideoTypeEnum | None, limit: bool = True
) -> list[tuple[VideoTypeEnum, int | None]]:
"""Build queries for all or specific video type."""
query_methods = {
VideoTypeEnum.VIDEOS: self.videos_query,
VideoTypeEnum.STREAMS: self.streams_query,
VideoTypeEnum.SHORTS: self.shorts_query,
}
if video_type:
# build query for specific type
query_method = query_methods.get(video_type)
if query_method:
query = query_method(limit)
if query[1] != 0:
return [query]
return []
# Build and return queries for all video types
queries = []
for build_query in query_methods.values():
query = build_query(limit)
if query[1] != 0:
queries.append(query)
return queries
def videos_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
"""Build query for videos."""
return self._build_generic_query(
video_type=VideoTypeEnum.VIDEOS,
overwrite_key="subscriptions_channel_size",
config_key="channel_size",
limit=limit,
pending_handler = PendingList(
youtube_ids=all_channel_urls,
task=self.task,
auto_start=self.config["subscriptions"].get("auto_start", False),
flat=self.config["subscriptions"].get("extract_flat", False),
)
pending_handler.parse_url_list()
added = pending_handler.add_to_pending()
def streams_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
"""Build query for streams."""
return self._build_generic_query(
video_type=VideoTypeEnum.STREAMS,
overwrite_key="subscriptions_live_channel_size",
config_key="live_channel_size",
limit=limit,
)
def shorts_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]:
"""Build query for shorts."""
return self._build_generic_query(
video_type=VideoTypeEnum.SHORTS,
overwrite_key="subscriptions_shorts_channel_size",
config_key="shorts_channel_size",
limit=limit,
)
def _build_generic_query(
self,
video_type: VideoTypeEnum,
overwrite_key: str,
config_key: str,
limit: bool,
) -> tuple[VideoTypeEnum, int | None]:
"""Generic query for video page scraping."""
if not limit:
return (video_type, None)
if (
overwrite_key in self.channel_overwrites
and self.channel_overwrites[overwrite_key] is not None
):
overwrite = self.channel_overwrites[overwrite_key]
return (video_type, overwrite)
if overwrite := self.config["subscriptions"].get(config_key):
return (video_type, overwrite)
return (video_type, 0)
return added
class PlaylistSubscription:
"""manage the playlist download functionality"""
"""scan subscribed playlists for videos to add to pending"""
def __init__(self, task=False):
def __init__(self, task=None):
self.config = AppConfig().config
self.task = task
@staticmethod
def get_playlists(subscribed_only=True):
"""get a list of all active playlists"""
data = {
"sort": [{"playlist_channel.keyword": {"order": "desc"}}],
}
data["query"] = {
"bool": {"must": [{"term": {"playlist_active": {"value": True}}}]}
}
if subscribed_only:
data["query"]["bool"]["must"].append(
{"term": {"playlist_subscribed": {"value": True}}}
)
all_playlists = IndexPaginate("ta_playlist", data).get_results()
return all_playlists
def process_url_str(self, new_playlists, subscribed=True):
"""process playlist subscribe form url_str"""
for idx, playlist in enumerate(new_playlists):
playlist_id = playlist["url"]
if not playlist["type"] == "playlist":
print(f"{playlist_id} not a playlist, skipping...")
continue
playlist_h = YoutubePlaylist(playlist_id)
playlist_h.build_json()
if not playlist_h.json_data:
message = f"{playlist_h.youtube_id}: failed to extract data"
print(message)
raise ValueError(message)
playlist_h.json_data["playlist_subscribed"] = subscribed
playlist_h.upload_to_es()
playlist_h.add_vids_to_playlist()
self.channel_validate(playlist_h.json_data["playlist_channel_id"])
url = playlist_h.json_data["playlist_thumbnail"]
thumb = ThumbManager(playlist_id, item_type="playlist")
thumb.download_playlist_thumb(url)
if self.task:
self.task.send_progress(
message_lines=[
f"Processing {idx + 1} of {len(new_playlists)}"
],
progress=(idx + 1) / len(new_playlists),
)
@staticmethod
def channel_validate(channel_id):
"""make sure channel of playlist is there"""
channel = YoutubeChannel(channel_id)
channel.build_json(upload=True)
@staticmethod
def change_subscribe(playlist_id, subscribe_status):
"""change the subscribe status of a playlist"""
playlist = YoutubePlaylist(playlist_id)
playlist.build_json()
playlist.json_data["playlist_subscribed"] = subscribe_status
playlist.upload_to_es()
return playlist.json_data
def find_missing(self):
"""find videos in subscribed playlists not downloaded yet"""
all_playlists = [i["playlist_id"] for i in self.get_playlists()]
def find_missing(self) -> int:
"""find missing"""
all_playlists = get_playlists(
subscribed_only=True, source=["playlist_id"]
)
if not all_playlists:
return False
return 0
missing_videos = []
total = len(all_playlists)
for idx, playlist_id in enumerate(all_playlists):
playlist = YoutubePlaylist(playlist_id)
is_active = playlist.update_playlist()
if not is_active:
playlist.deactivate()
continue
playlist_entries = playlist.json_data["playlist_entries"]
size_limit = self.config["subscriptions"]["channel_size"]
if size_limit:
del playlist_entries[size_limit:]
to_check = [
i["youtube_id"]
for i in playlist_entries
if i["downloaded"] is False
]
needs_downloading = is_missing(to_check)
missing_videos.extend(needs_downloading)
if not self.task:
continue
if self.task.is_stopped():
self.task.send_progress(["Received Stop signal."])
break
self.task.send_progress(
message_lines=[f"Scanning Playlists {idx + 1}/{total}"],
progress=(idx + 1) / total,
size_limit = self.config["subscriptions"]["channel_size"]
all_playlist_urls: list[ParsedURLType] = []
for playlist in all_playlists:
all_playlist_urls.append(
ParsedURLType(
type="playlist",
url=playlist["playlist_id"],
vid_type=VideoTypeEnum.UNKNOWN,
limit=size_limit,
)
)
rand_sleep(self.config)
return missing_videos
pending_handler = PendingList(
youtube_ids=all_playlist_urls,
task=self.task,
auto_start=self.config["subscriptions"].get("auto_start", False),
flat=self.config["subscriptions"].get("extract_flat", False),
)
pending_handler.parse_url_list()
added = pending_handler.add_to_pending()
return added
class SubscriptionScanner:
@ -338,40 +112,12 @@ class SubscriptionScanner:
if self.task:
self.task.send_progress(["Rescanning channels and playlists."])
self.missing_videos = []
self.scan_channels()
added = 0
added += ChannelSubscription(task=self.task).find_missing()
if self.task and not self.task.is_stopped():
self.scan_playlists()
added += PlaylistSubscription(task=self.task).find_missing()
return self.missing_videos
def scan_channels(self):
"""get missing from channels"""
channel_handler = ChannelSubscription(task=self.task)
missing = channel_handler.find_missing()
if not missing:
return
for vid_id, vid_type in missing:
self.missing_videos.append(
{"type": "video", "vid_type": vid_type, "url": vid_id}
)
def scan_playlists(self):
"""get missing from playlists"""
playlist_handler = PlaylistSubscription(task=self.task)
missing = playlist_handler.find_missing()
if not missing:
return
for i in missing:
self.missing_videos.append(
{
"type": "video",
"vid_type": VideoTypeEnum.VIDEOS.value,
"url": i,
}
)
return added
class SubscriptionHandler:
@ -403,7 +149,8 @@ class SubscriptionHandler:
f"expected {expected_type} url but got {item.get('type')}"
)
PlaylistSubscription().process_url_str([item])
playlist = YoutubePlaylist(item["url"])
playlist.change_subscribe(new_subscribe_state=True)
return
if item["type"] == "video":
@ -426,9 +173,7 @@ class SubscriptionHandler:
def _subscribe(self, channel_id):
"""subscribe to channel"""
_ = ChannelSubscription().change_subscribe(
channel_id, channel_subscribed=True
)
YoutubeChannel(channel_id).change_subscribe(new_subscribe_state=True)
def _notify(self, idx, item, total):
"""send notification message to redis"""

6
backend/download/src/yt_dlp_handler.py
View File

@ -16,12 +16,12 @@ from common.src.env_settings import EnvironmentSettings
from common.src.es_connect import ElasticWrap, IndexPaginate
from common.src.helper import (
get_channel_overwrites,
get_playlists,
ignore_filelist,
rand_sleep,
)
from common.src.ta_redis import RedisQueue
from download.src.queue import PendingList
from download.src.subscriptions import PlaylistSubscription
from download.src.yt_dlp_base import YtWrap
from playlist.src.index import YoutubePlaylist
from video.src.comments import CommentList
@ -403,8 +403,8 @@ class DownloadPostProcess(DownloaderBase):
def _add_playlist_sub(self):
"""add subscribed playlists to refresh"""
subs = PlaylistSubscription().get_playlists()
to_add = [i["playlist_id"] for i in subs]
playlists = get_playlists(subscribed_only=True, source=["playlist_id"])
to_add = [i["playlist_id"] for i in playlists]
RedisQueue(self.PLAYLIST_QUEUE).add_list(to_add)
def _add_channel_playlists(self):

17
backend/playlist/src/index.py
View File

@ -30,7 +30,7 @@ class YoutubePlaylist(YouTubeItem):
self.all_members = False
self.nav = False
def build_json(self, scrape=False):
def build_json(self, scrape=False, limit: int | None = None):
"""collection to create json_data"""
self.get_from_es()
if self.json_data:
@ -40,7 +40,9 @@ class YoutubePlaylist(YouTubeItem):
subscribed = False
playlist_sort_order = "top"
playlist_items = "::1" if playlist_sort_order == "top" else "::-1"
limit_str = limit if limit is not None else ""
sort_order = 1 if playlist_sort_order == "top" else -1
playlist_items = f":{limit_str}:{sort_order}"
if scrape or not self.json_data:
self.get_from_youtube(
@ -137,6 +139,13 @@ class YoutubePlaylist(YouTubeItem):
url = self.json_data["playlist_thumbnail"]
ThumbManager(self.youtube_id, item_type="playlist").download(url)
def change_subscribe(self, new_subscribe_state: bool):
"""change subscribe status"""
self.build_json()
self.json_data["playlist_subscribed"] = new_subscribe_state
self.upload_to_es()
return self.json_data
def add_vids_to_playlist(self):
"""sync the playlist id to videos"""
script = (
@ -189,9 +198,9 @@ class YoutubePlaylist(YouTubeItem):
if status_code == 200:
print(f"{self.youtube_id}: removed {video_id} from playlist")
def update_playlist(self, skip_on_empty=False):
def update_playlist(self, skip_on_empty=False, limit: int | None = None):
"""update metadata for playlist with data from YouTube"""
self.build_json(scrape=True)
self.build_json(scrape=True, limit=limit)
if not self.json_data:
# return false to deactivate
return False

6
backend/playlist/views.py
View File

@ -7,7 +7,6 @@ from common.serializers import (
ErrorResponseSerializer,
)
from common.views_base import AdminWriteOnly, ApiBaseView
from download.src.subscriptions import PlaylistSubscription
from drf_spectacular.utils import OpenApiResponse, extend_schema
from playlist.serializers import (
PlaylistBulkAddSerializer,
@ -245,8 +244,9 @@ class PlaylistApiView(ApiBaseView):
json_data = None
if subscribed is not None:
playlist_sub = PlaylistSubscription()
json_data = playlist_sub.change_subscribe(playlist_id, subscribed)
json_data = YoutubePlaylist(playlist_id).change_subscribe(
new_subscribe_state=subscribed
)
if sort_order:
json_data = YoutubePlaylist(playlist_id).change_sort_order(

12
backend/task/tasks.py
View File

@ -103,13 +103,13 @@ def update_subscribed(self):
manager.init(self)
handler = SubscriptionScanner(task=self)
missing_videos = handler.scan()
added = handler.scan()
auto_start = handler.auto_start
if missing_videos:
print(missing_videos)
extrac_dl.delay(missing_videos, auto_start=auto_start)
message = f"Found {len(missing_videos)} videos to add to the queue."
return message
if added:
if auto_start:
download_pending.delay(auto_only=True)
return f"Found {added} videos to add to the queue."
return None

1
frontend/src/api/loader/loadAppsettingsConfig.ts
View File

@ -6,6 +6,7 @@ export type AppSettingsConfigType = {
live_channel_size: number | null;
shorts_channel_size: number | null;
auto_start: boolean;
extract_flat: boolean;
};
downloads: {
limit_speed: number | null;

16
frontend/src/pages/SettingsApplication.tsx
View File

@ -42,6 +42,7 @@ const SettingsApplication = () => {
const [livePageSize, setLivePageSize] = useState<number | null>(null);
const [shortPageSize, setShortPageSize] = useState<number | null>(null);
const [isAutostart, setIsAutostart] = useState<boolean>(false);
const [isExtractFlat, setIsExtractFlat] = useState<boolean>(false);
// Downloads
const [currentDownloadSpeed, setCurrentDownloadSpeed] = useState<number | null>(null);
@ -98,6 +99,7 @@ const SettingsApplication = () => {
setLivePageSize(appSettingsConfigData?.subscriptions.live_channel_size || null);
setShortPageSize(appSettingsConfigData?.subscriptions.shorts_channel_size || null);
setIsAutostart(appSettingsConfigData?.subscriptions.auto_start || false);
setIsExtractFlat(appSettingsConfigData?.subscriptions.extract_flat || false);
// Downloads
setCurrentDownloadSpeed(appSettingsConfigData?.downloads.limit_speed || null);
@ -212,6 +214,10 @@ const SettingsApplication = () => {
Autostart automatically starts downloading videos from subscriptions with
priority.
</li>
<li>
Fast add extracts and adds videos in bulk. That is much faster but is not able
to extract as much metadata during adding to the queue.
</li>
</ul>
</div>
)}
@ -264,6 +270,16 @@ const SettingsApplication = () => {
updateCallback={handleUpdateConfig}
/>
</div>
<div className="settings-box-wrapper">
<div>
<p>Fast add</p>
</div>
<ToggleConfig
name="subscriptions.extract_flat"
value={isExtractFlat}
updateCallback={handleUpdateConfig}
/>
</div>
</div>
<div className="info-box-item">
<h2 id="downloads">Downloads</h2>

1
frontend/src/stores/AppSettingsStore.ts
View File

@ -13,6 +13,7 @@ export const useAppSettingsStore = create<AppSettingsState>(set => ({
live_channel_size: null,
shorts_channel_size: null,
auto_start: false,
extract_flat: false,
},
downloads: {
limit_speed: null,