diff --git a/backend/download/src/yt_dlp_base.py b/backend/download/src/yt_dlp_base.py index a18e5ef0..b3ce3e16 100644 --- a/backend/download/src/yt_dlp_base.py +++ b/backend/download/src/yt_dlp_base.py @@ -82,23 +82,24 @@ class YtWrap: def extract(self, url): """make extract request""" - try: - response = yt_dlp.YoutubeDL(self.obs).extract_info(url) - except cookiejar.LoadError as err: - print(f"cookie file is invalid: {err}") - return False - except yt_dlp.utils.ExtractorError as err: - print(f"{url}: failed to extract with message: {err}, continue...") - return False - except yt_dlp.utils.DownloadError as err: - if "This channel does not have a" in str(err): + with yt_dlp.YoutubeDL(self.obs) as ydl: + try: + response = ydl.extract_info(url) + except cookiejar.LoadError as err: + print(f"cookie file is invalid: {err}") return False + except yt_dlp.utils.ExtractorError as err: + print(f"{url}: failed to extract: {err}, continue...") + return False + except yt_dlp.utils.DownloadError as err: + if "This channel does not have a" in str(err): + return False - print(f"{url}: failed to get info from youtube with message {err}") - if "Temporary failure in name resolution" in str(err): - raise ConnectionError("lost the internet, abort!") from err + print(f"{url}: failed to get info from youtube: {err}") + if "Temporary failure in name resolution" in str(err): + raise ConnectionError("lost the internet, abort!") from err - return False + return False self._validate_cookie() diff --git a/docker_assets/run.sh b/docker_assets/run.sh index 283b70f0..33029f1c 100644 --- a/docker_assets/run.sh +++ b/docker_assets/run.sh @@ -20,7 +20,11 @@ python manage.py ta_startup # start all tasks nginx & -celery -A task.celery worker --loglevel=INFO --max-tasks-per-child 10 & +celery -A task.celery worker \ + --loglevel=INFO \ + --concurrency 4 \ + --max-tasks-per-child 5 \ + --max-memory-per-child 150000 & celery -A task beat --loglevel=INFO \ --scheduler django_celery_beat.schedulers:DatabaseScheduler & python backend_start.py