From c687c0fd96cf835f99f4c15fc7a5477132a0efbe Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 17 Jul 2024 19:09:31 +0200 Subject: [PATCH] make application run outside of docker --- .gitignore | 5 +-- CONTRIBUTING.md | 34 +++++++++++++++++-- deploy.sh | 3 +- tubearchivist/api/src/search_processor.py | 19 +++++++---- tubearchivist/config/settings.py | 8 +++++ tubearchivist/home/src/es/snapshot.py | 7 ++-- tubearchivist/home/src/ta/helper.py | 8 +++-- tubearchivist/home/src/ta/settings.py | 22 ++++++++++++ .../home/templates/home/channel.html | 4 +-- .../home/templates/home/channel_id.html | 4 +-- .../templates/home/channel_id_playlist.html | 2 +- .../home/templates/home/playlist.html | 2 +- .../home/templates/home/playlist_id.html | 2 +- tubearchivist/requirements-dev.txt | 2 ++ 14 files changed, 97 insertions(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index 682dd836..41dd08d9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,9 @@ __pycache__ .venv -# django testing db -db.sqlite3 +# django testing +**/static/volume +**/.env # vscode custom conf .vscode diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d86d2047..23a896b7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -138,9 +138,39 @@ The documentation available at [docs.tubearchivist.com](https://docs.tubearchivi ## Development Environment -I have learned the hard way, that working on a dockerized application outside of docker is very error prone and in general not a good idea. So if you want to test your changes, it's best to run them in a docker testing environment. You might be able to run the application directly, but this document assumes you're using docker. +I have learned the hard way, that working on a dockerized application outside of docker is very error prone and in general not a good idea. So if you want to test your changes, it's best to run them in a docker testing environment. But for quick development, running the application outside of docker, can also be helpful. -### Instructions +### Native Instruction + +For convenience, it's recommended to still run Redis and ES in a docker container. + +Set up your virtual environment and install the requirements defined in `requirements-dev.txt`. + +There are options built in to load environment variables from a file using `load_dotenv`. Example `.env` file to place in the same folder as `manage.py`: + +``` +TA_HOST="localhost" +TA_USERNAME=tubearchivist +TA_PASSWORD=verysecret +TA_MEDIA_DIR="static/volume/media" +TA_CACHE_DIR="static/volume/cache" +TA_APP_DIR="." +REDIS_HOST=localhost +ES_URL="http://localhost:9200" +ELASTIC_PASSWORD=verysecret +TZ=America/New_York +DJANGO_DEBUG=True +``` + +Than from look at the container startup script `run.sh`, make sure all needed migrations ran, then to start the dev server from the same folder as `manage.py` run: + +```bash +python manage.py runserver +``` + +You'll probably also want to have a Celery worker instance running, refer to `run.sh` for that. The Beat Scheduler might not be needed. + +### Docker Instructions Set up docker on your development machine. diff --git a/deploy.sh b/deploy.sh index 6087b03f..d8badf0d 100755 --- a/deploy.sh +++ b/deploy.sh @@ -51,6 +51,7 @@ function sync_test { --exclude "**/cache" \ --exclude "**/__pycache__/" \ --exclude "**/.pytest_cache/" \ + --exclude "**/static/volume" \ --exclude ".venv" \ --exclude "db.sqlite3" \ --exclude ".mypy_cache" \ @@ -92,7 +93,7 @@ function validate { echo "running black" black --force-exclude "migrations/*" --diff --color --check -l 79 "$check_path" echo "running codespell" - codespell --skip="./.git,./.venv,./package.json,./package-lock.json,./node_modules,./.mypy_cache" "$check_path" + codespell --skip="./.git,./.venv,./package.json,./package-lock.json,./node_modules,./.mypy_cache,**/static/volume" "$check_path" echo "running flake8" flake8 "$check_path" --exclude "migrations,.venv" --count --max-complexity=10 \ --max-line-length=79 --show-source --statistics diff --git a/tubearchivist/api/src/search_processor.py b/tubearchivist/api/src/search_processor.py index a7891c96..05993369 100644 --- a/tubearchivist/api/src/search_processor.py +++ b/tubearchivist/api/src/search_processor.py @@ -14,8 +14,6 @@ from home.src.ta.settings import EnvironmentSettings class SearchProcess: """process search results""" - CACHE_DIR = EnvironmentSettings.CACHE_DIR - def __init__(self, response): self.response = response self.processed = False @@ -66,7 +64,8 @@ class SearchProcess: def _process_channel(channel_dict): """run on single channel""" channel_id = channel_dict["channel_id"] - art_base = f"/cache/channels/{channel_id}" + cache_root = EnvironmentSettings().get_cache_root() + art_base = f"{cache_root}/channels/{channel_id}" date_str = date_parser(channel_dict["channel_last_refresh"]) channel_dict.update( { @@ -93,13 +92,16 @@ class SearchProcess: url = video_dict["subtitles"][idx]["media_url"] video_dict["subtitles"][idx]["media_url"] = f"/media/{url}" + cache_root = EnvironmentSettings().get_cache_root() + media_root = EnvironmentSettings().get_media_root() + video_dict.update( { "channel": channel, - "media_url": f"/media/{media_url}", + "media_url": f"{media_root}/{media_url}", "vid_last_refresh": vid_last_refresh, "published": published, - "vid_thumb_url": f"{self.CACHE_DIR}/{vid_thumb_url}", + "vid_thumb_url": f"{cache_root}/{vid_thumb_url}", } ) @@ -112,9 +114,11 @@ class SearchProcess: playlist_last_refresh = date_parser( playlist_dict["playlist_last_refresh"] ) + cache_root = EnvironmentSettings().get_cache_root() + playlist_thumbnail = f"{cache_root}/playlists/{playlist_id}.jpg" playlist_dict.update( { - "playlist_thumbnail": f"/cache/playlists/{playlist_id}.jpg", + "playlist_thumbnail": playlist_thumbnail, "playlist_last_refresh": playlist_last_refresh, } ) @@ -124,12 +128,13 @@ class SearchProcess: def _process_download(self, download_dict): """run on single download item""" video_id = download_dict["youtube_id"] + cache_root = EnvironmentSettings().get_cache_root() vid_thumb_url = ThumbManager(video_id).vid_thumb_path() published = date_parser(download_dict["published"]) download_dict.update( { - "vid_thumb_url": f"{self.CACHE_DIR}/{vid_thumb_url}", + "vid_thumb_url": f"{cache_root}/{vid_thumb_url}", "published": published, } ) diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index a74aef86..39742ab9 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -20,6 +20,14 @@ from django_auth_ldap.config import LDAPSearch from home.src.ta.helper import ta_host_parser from home.src.ta.settings import EnvironmentSettings +try: + from dotenv import load_dotenv + + load_dotenv(".env") +except ModuleNotFoundError: + pass + + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent diff --git a/tubearchivist/home/src/es/snapshot.py b/tubearchivist/home/src/es/snapshot.py index 0cff51e6..5d946097 100644 --- a/tubearchivist/home/src/es/snapshot.py +++ b/tubearchivist/home/src/es/snapshot.py @@ -251,10 +251,9 @@ class ElasticSnapshot: @staticmethod def _date_converter(date_utc): """convert datetime string""" - expected_format = "%Y-%m-%dT%H:%M:%S.%fZ" - date = datetime.strptime(date_utc, expected_format) - local_datetime = date.replace(tzinfo=ZoneInfo("localtime")) - converted = local_datetime.astimezone(ZoneInfo(EnvironmentSettings.TZ)) + date = datetime.strptime(date_utc, "%Y-%m-%dT%H:%M:%S.%fZ") + utc_date = date.replace(tzinfo=ZoneInfo("UTC")) + converted = utc_date.astimezone(ZoneInfo(EnvironmentSettings.TZ)) converted_str = converted.strftime("%Y-%m-%d %H:%M") return converted_str diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index 767b4a00..5e069804 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -213,10 +213,14 @@ def ta_host_parser(ta_host: str) -> tuple[list[str], list[str]]: return allowed_hosts, csrf_trusted_origins -def get_stylesheets(): +def get_stylesheets() -> list: """Get all valid stylesheets from /static/css""" app_root = EnvironmentSettings.APP_DIR - stylesheets = os.listdir(os.path.join(app_root, "static/css")) + try: + stylesheets = os.listdir(os.path.join(app_root, "static/css")) + except FileNotFoundError: + return [] + stylesheets.remove("style.css") stylesheets.sort() stylesheets = list(filter(lambda x: x.endswith(".css"), stylesheets)) diff --git a/tubearchivist/home/src/ta/settings.py b/tubearchivist/home/src/ta/settings.py index 16f36994..1347d410 100644 --- a/tubearchivist/home/src/ta/settings.py +++ b/tubearchivist/home/src/ta/settings.py @@ -6,6 +6,14 @@ Functionality: from os import environ +try: + from dotenv import load_dotenv + + print("loading local dotenv") + load_dotenv(".env") +except ModuleNotFoundError: + pass + class EnvironmentSettings: """ @@ -44,6 +52,20 @@ class EnvironmentSettings: ) ES_DISABLE_VERIFY_SSL: bool = bool(environ.get("ES_DISABLE_VERIFY_SSL")) + def get_cache_root(self): + """get root for web server""" + if self.CACHE_DIR.startswith("/"): + return self.CACHE_DIR + + return f"/{self.CACHE_DIR}" + + def get_media_root(self): + """get root for media folder""" + if self.MEDIA_DIR.startswith("/"): + return self.MEDIA_DIR + + return f"/{self.MEDIA_DIR}" + def print_generic(self): """print generic env vars""" print( diff --git a/tubearchivist/home/templates/home/channel.html b/tubearchivist/home/templates/home/channel.html index 83a163d3..ce5eab54 100644 --- a/tubearchivist/home/templates/home/channel.html +++ b/tubearchivist/home/templates/home/channel.html @@ -46,14 +46,14 @@
- {{ channel.channel_id }}-banner + {{ channel.channel_id }}-banner
- channel-thumb + channel-thumb
diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index 6c8b6539..367074e9 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -6,7 +6,7 @@
- channel_banner + channel_banner

Videos

 @@ -31,7 +31,7 @@
- channel-thumb + channel-thumb
diff --git a/tubearchivist/home/templates/home/channel_id_playlist.html b/tubearchivist/home/templates/home/channel_id_playlist.html index 52b99800..ba9341f2 100644 --- a/tubearchivist/home/templates/home/channel_id_playlist.html +++ b/tubearchivist/home/templates/home/channel_id_playlist.html @@ -49,7 +49,7 @@
- {{ playlist.playlist_id }}-thumbnail + {{ playlist.playlist_id }}-thumbnail
diff --git a/tubearchivist/home/templates/home/playlist.html b/tubearchivist/home/templates/home/playlist.html index ea7babab..6983372f 100644 --- a/tubearchivist/home/templates/home/playlist.html +++ b/tubearchivist/home/templates/home/playlist.html @@ -52,7 +52,7 @@
- {{ playlist.playlist_id }}-thumbnail + {{ playlist.playlist_id }}-thumbnail
diff --git a/tubearchivist/home/templates/home/playlist_id.html b/tubearchivist/home/templates/home/playlist_id.html index 9a234c1d..942ab8ea 100644 --- a/tubearchivist/home/templates/home/playlist_id.html +++ b/tubearchivist/home/templates/home/playlist_id.html @@ -13,7 +13,7 @@
- channel-thumb + channel-thumb
diff --git a/tubearchivist/requirements-dev.txt b/tubearchivist/requirements-dev.txt index 02778fba..6f76c2ad 100644 --- a/tubearchivist/requirements-dev.txt +++ b/tubearchivist/requirements-dev.txt @@ -2,9 +2,11 @@ black codespell flake8 +ipython isort pylint pylint-django pytest pytest-django +python-dotenv types-requests