diff --git a/backend/appsettings/src/config.py b/backend/appsettings/src/config.py index 24c3c13f..c3ca6168 100644 --- a/backend/appsettings/src/config.py +++ b/backend/appsettings/src/config.py @@ -74,7 +74,7 @@ class AppConfig: }, "downloads": { "limit_speed": False, - "sleep_interval": 3, + "sleep_interval": 10, "autodelete_days": False, "format": False, "format_sort": False, diff --git a/backend/appsettings/src/reindex.py b/backend/appsettings/src/reindex.py index 9af73d97..d6259b81 100644 --- a/backend/appsettings/src/reindex.py +++ b/backend/appsettings/src/reindex.py @@ -14,6 +14,7 @@ from appsettings.src.config import AppConfig from channel.src.index import YoutubeChannel from common.src.env_settings import EnvironmentSettings from common.src.es_connect import ElasticWrap, IndexPaginate +from common.src.helper import get_sleep from common.src.ta_redis import RedisQueue from download.src.subscriptions import ChannelSubscription from download.src.thumbnails import ThumbManager @@ -289,8 +290,7 @@ class Reindex(ReindexBase): self._notify(name, total, idx) reindex(youtube_id) - sleep_interval = self.config["downloads"].get("sleep_interval", 0) - sleep(sleep_interval) + sleep(get_sleep(self.config)) def _get_reindex_map(self, index_name: str) -> Callable: """return def to run for index""" diff --git a/backend/common/src/helper.py b/backend/common/src/helper.py index a7db4f3e..86cb657d 100644 --- a/backend/common/src/helper.py +++ b/backend/common/src/helper.py @@ -40,6 +40,18 @@ def randomizor(length: int) -> str: return "".join(random.choice(pool) for i in range(length)) +def get_sleep(config) -> int: + """get randomized sleep""" + sleep_config = config["downloads"].get("sleep_interval") + if not sleep_config: + return 0 + + rand_sleep = random.randrange( + int(sleep_config * 0.5), int(sleep_config * 1.5) + ) + return rand_sleep + + def requests_headers() -> dict[str, str]: """build header with random user agent for requests outside of yt-dlp""" diff --git a/backend/download/src/queue.py b/backend/download/src/queue.py index e2777959..f3dc2cca 100644 --- a/backend/download/src/queue.py +++ b/backend/download/src/queue.py @@ -10,7 +10,7 @@ from time import sleep from appsettings.src.config import AppConfig from common.src.es_connect import ElasticWrap, IndexPaginate -from common.src.helper import get_duration_str, is_shorts +from common.src.helper import get_duration_str, get_sleep, is_shorts from download.src.subscriptions import ChannelSubscription from download.src.thumbnails import ThumbManager from download.src.yt_dlp_base import YtWrap @@ -246,9 +246,7 @@ class PendingList(PendingIndex): total = len(self.missing_videos) videos_added = [] - sleep_interval = self.config["downloads"].get("sleep_interval", 0) for idx, (youtube_id, vid_type) in enumerate(self.missing_videos): - sleep(sleep_interval) if self.task and self.task.is_stopped(): break @@ -277,6 +275,8 @@ class PendingList(PendingIndex): self._ingest_bulk(bulk_list) bulk_list = [] + sleep(get_sleep(self.config)) + self._ingest_bulk(bulk_list) return videos_added diff --git a/backend/download/src/subscriptions.py b/backend/download/src/subscriptions.py index 833746ff..1abc2d2c 100644 --- a/backend/download/src/subscriptions.py +++ b/backend/download/src/subscriptions.py @@ -4,10 +4,12 @@ Functionality: - handle playlist subscriptions """ +from time import sleep + from appsettings.src.config import AppConfig from channel.src.index import YoutubeChannel from common.src.es_connect import IndexPaginate -from common.src.helper import is_missing +from common.src.helper import get_sleep, is_missing from common.src.urlparser import Parser from download.src.thumbnails import ThumbManager from download.src.yt_dlp_base import YtWrap @@ -108,6 +110,7 @@ class ChannelSubscription: message_lines=[f"Scanning Channel {idx + 1}/{total}"], progress=(idx + 1) / total, ) + sleep(get_sleep(self.config)) return missing_videos @@ -317,6 +320,7 @@ class PlaylistSubscription: message_lines=[f"Scanning Playlists {idx + 1}/{total}"], progress=(idx + 1) / total, ) + sleep(get_sleep(self.config)) return missing_videos