diff --git a/backend/appsettings/src/reindex.py b/backend/appsettings/src/reindex.py index d6259b81..bb4b26e2 100644 --- a/backend/appsettings/src/reindex.py +++ b/backend/appsettings/src/reindex.py @@ -7,14 +7,13 @@ functionality: import json import os from datetime import datetime -from time import sleep from typing import Callable, TypedDict from appsettings.src.config import AppConfig from channel.src.index import YoutubeChannel from common.src.env_settings import EnvironmentSettings from common.src.es_connect import ElasticWrap, IndexPaginate -from common.src.helper import get_sleep +from common.src.helper import rand_sleep from common.src.ta_redis import RedisQueue from download.src.subscriptions import ChannelSubscription from download.src.thumbnails import ThumbManager @@ -290,7 +289,7 @@ class Reindex(ReindexBase): self._notify(name, total, idx) reindex(youtube_id) - sleep(get_sleep(self.config)) + rand_sleep(self.config) def _get_reindex_map(self, index_name: str) -> Callable: """return def to run for index""" diff --git a/backend/common/src/helper.py b/backend/common/src/helper.py index 86cb657d..3a487e63 100644 --- a/backend/common/src/helper.py +++ b/backend/common/src/helper.py @@ -9,6 +9,7 @@ import random import string import subprocess from datetime import datetime +from time import sleep from typing import Any from urllib.parse import urlparse @@ -40,16 +41,14 @@ def randomizor(length: int) -> str: return "".join(random.choice(pool) for i in range(length)) -def get_sleep(config) -> int: - """get randomized sleep""" +def rand_sleep(config) -> None: + """randomized sleep based on config""" sleep_config = config["downloads"].get("sleep_interval") if not sleep_config: - return 0 + return - rand_sleep = random.randrange( - int(sleep_config * 0.5), int(sleep_config * 1.5) - ) - return rand_sleep + secs = random.randrange(int(sleep_config * 0.5), int(sleep_config * 1.5)) + sleep(secs) def requests_headers() -> dict[str, str]: diff --git a/backend/download/src/queue.py b/backend/download/src/queue.py index 9f8889ab..cf0e40d9 100644 --- a/backend/download/src/queue.py +++ b/backend/download/src/queue.py @@ -5,11 +5,10 @@ Functionality: """ from datetime import datetime -from time import sleep from appsettings.src.config import AppConfig from common.src.es_connect import ElasticWrap, IndexPaginate -from common.src.helper import get_duration_str, get_sleep, is_shorts +from common.src.helper import get_duration_str, is_shorts, rand_sleep from download.src.subscriptions import ChannelSubscription from download.src.thumbnails import ThumbManager from download.src.yt_dlp_base import YtWrap @@ -267,8 +266,8 @@ class PendingList(PendingIndex): _, _ = ElasticWrap(es_url).put(video_details) videos_added.append(youtube_id) - if len(videos_added) != total: - sleep(get_sleep(self.config)) + if idx != total: + rand_sleep(self.config) return videos_added diff --git a/backend/download/src/subscriptions.py b/backend/download/src/subscriptions.py index 0c07eb1a..abc0a4eb 100644 --- a/backend/download/src/subscriptions.py +++ b/backend/download/src/subscriptions.py @@ -4,12 +4,10 @@ Functionality: - handle playlist subscriptions """ -from time import sleep - from appsettings.src.config import AppConfig from channel.src.index import YoutubeChannel from common.src.es_connect import IndexPaginate -from common.src.helper import get_sleep, is_missing +from common.src.helper import is_missing, rand_sleep from common.src.urlparser import Parser from download.src.thumbnails import ThumbManager from download.src.yt_dlp_base import YtWrap @@ -110,7 +108,7 @@ class ChannelSubscription: message_lines=[f"Scanning Channel {idx + 1}/{total}"], progress=(idx + 1) / total, ) - sleep(get_sleep(self.config)) + rand_sleep(self.config) return missing_videos @@ -323,7 +321,7 @@ class PlaylistSubscription: message_lines=[f"Scanning Playlists {idx + 1}/{total}"], progress=(idx + 1) / total, ) - sleep(get_sleep(self.config)) + rand_sleep(self.config) return missing_videos diff --git a/backend/video/src/comments.py b/backend/video/src/comments.py index 5e3df655..2d13448c 100644 --- a/backend/video/src/comments.py +++ b/backend/video/src/comments.py @@ -6,11 +6,10 @@ Functionality: """ from datetime import datetime -from time import sleep from appsettings.src.config import AppConfig from common.src.es_connect import ElasticWrap -from common.src.helper import get_sleep +from common.src.helper import rand_sleep from common.src.ta_redis import RedisQueue from download.src.yt_dlp_base import YtWrap @@ -221,7 +220,7 @@ class CommentList: if comment.json_data: comment.upload_comments() - sleep(get_sleep(self.config)) + rand_sleep(self.config) def notify(self, idx, total_videos): """send notification on task"""