dynamic sleep

backend/appsettings/src/config.py

@@ -74,7 +74,7 @@ class AppConfig:
         },
         "downloads": {
             "limit_speed": False,
-            "sleep_interval": 3,
+            "sleep_interval": 10,
             "autodelete_days": False,
             "format": False,
             "format_sort": False,

backend/appsettings/src/reindex.py

@@ -14,6 +14,7 @@ from appsettings.src.config import AppConfig
 from channel.src.index import YoutubeChannel
 from common.src.env_settings import EnvironmentSettings
 from common.src.es_connect import ElasticWrap, IndexPaginate
+from common.src.helper import get_sleep
 from common.src.ta_redis import RedisQueue
 from download.src.subscriptions import ChannelSubscription
 from download.src.thumbnails import ThumbManager
@@ -289,8 +290,7 @@ class Reindex(ReindexBase):
                 self._notify(name, total, idx)
 
             reindex(youtube_id)
-            sleep_interval = self.config["downloads"].get("sleep_interval", 0)
-            sleep(sleep_interval)
+            sleep(get_sleep(self.config))
 
     def _get_reindex_map(self, index_name: str) -> Callable:
         """return def to run for index"""

backend/common/src/helper.py

@@ -40,6 +40,18 @@ def randomizor(length: int) -> str:
     return "".join(random.choice(pool) for i in range(length))
 
 
+def get_sleep(config) -> int:
+    """get randomized sleep"""
+    sleep_config = config["downloads"].get("sleep_interval")
+    if not sleep_config:
+        return 0
+
+    rand_sleep = random.randrange(
+        int(sleep_config * 0.5), int(sleep_config * 1.5)
+    )
+    return rand_sleep
+
+
 def requests_headers() -> dict[str, str]:
     """build header with random user agent for requests outside of yt-dlp"""
 

backend/download/src/queue.py

@@ -10,7 +10,7 @@ from time import sleep
 
 from appsettings.src.config import AppConfig
 from common.src.es_connect import ElasticWrap, IndexPaginate
-from common.src.helper import get_duration_str, is_shorts
+from common.src.helper import get_duration_str, get_sleep, is_shorts
 from download.src.subscriptions import ChannelSubscription
 from download.src.thumbnails import ThumbManager
 from download.src.yt_dlp_base import YtWrap
@@ -246,9 +246,7 @@ class PendingList(PendingIndex):
 
         total = len(self.missing_videos)
         videos_added = []
-        sleep_interval = self.config["downloads"].get("sleep_interval", 0)
         for idx, (youtube_id, vid_type) in enumerate(self.missing_videos):
-            sleep(sleep_interval)
             if self.task and self.task.is_stopped():
                 break
 
@@ -277,6 +275,8 @@ class PendingList(PendingIndex):
                 self._ingest_bulk(bulk_list)
                 bulk_list = []
 
+            sleep(get_sleep(self.config))
+
         self._ingest_bulk(bulk_list)
 
         return videos_added

backend/download/src/subscriptions.py

@@ -4,10 +4,12 @@ Functionality:
 - handle playlist subscriptions
 """
 
+from time import sleep
+
 from appsettings.src.config import AppConfig
 from channel.src.index import YoutubeChannel
 from common.src.es_connect import IndexPaginate
-from common.src.helper import is_missing
+from common.src.helper import get_sleep, is_missing
 from common.src.urlparser import Parser
 from download.src.thumbnails import ThumbManager
 from download.src.yt_dlp_base import YtWrap
@@ -108,6 +110,7 @@ class ChannelSubscription:
                 message_lines=[f"Scanning Channel {idx + 1}/{total}"],
                 progress=(idx + 1) / total,
             )
+            sleep(get_sleep(self.config))
 
         return missing_videos
 
@@ -317,6 +320,7 @@ class PlaylistSubscription:
                 message_lines=[f"Scanning Playlists {idx + 1}/{total}"],
                 progress=(idx + 1) / total,
             )
+            sleep(get_sleep(self.config))
 
         return missing_videos