make application run outside of docker

.gitignore

@@ -2,8 +2,9 @@
 __pycache__
 .venv
 
-# django testing db
-db.sqlite3
+# django testing
+**/static/volume
+**/.env
 
 # vscode custom conf
 .vscode

CONTRIBUTING.md

@@ -138,9 +138,39 @@ The documentation available at [docs.tubearchivist.com](https://docs.tubearchivi
 
 ## Development Environment
 
-I have learned the hard way, that working on a dockerized application outside of docker is very error prone and in general not a good idea. So if you want to test your changes, it's best to run them in a docker testing environment. You might be able to run the application directly, but this document assumes you're using docker.
+I have learned the hard way, that working on a dockerized application outside of docker is very error prone and in general not a good idea. So if you want to test your changes, it's best to run them in a docker testing environment. But for quick development, running the application outside of docker, can also be helpful.
 
-### Instructions
+### Native Instruction
+
+For convenience, it's recommended to still run Redis and ES in a docker container.
+
+Set up your virtual environment and install the requirements defined in `requirements-dev.txt`.
+
+There are options built in to load environment variables from a file using `load_dotenv`. Example `.env` file to place in the same folder as `manage.py`:
+
+```
+TA_HOST="localhost"
+TA_USERNAME=tubearchivist
+TA_PASSWORD=verysecret
+TA_MEDIA_DIR="static/volume/media"
+TA_CACHE_DIR="static/volume/cache"
+TA_APP_DIR="."
+REDIS_HOST=localhost
+ES_URL="http://localhost:9200"
+ELASTIC_PASSWORD=verysecret
+TZ=America/New_York
+DJANGO_DEBUG=True
+```
+
+Than from look at the container startup script `run.sh`, make sure all needed migrations ran, then to start the dev server from the same folder as `manage.py` run:
+
+```bash
+python manage.py runserver
+```
+
+You'll probably also want to have a Celery worker instance running, refer to `run.sh` for that. The Beat Scheduler might not be needed.
+
+### Docker Instructions
 
 Set up docker on your development machine.
 

deploy.sh

@@ -51,6 +51,7 @@ function sync_test {
         --exclude "**/cache" \
         --exclude "**/__pycache__/" \
         --exclude "**/.pytest_cache/" \
+        --exclude "**/static/volume" \
         --exclude ".venv" \
         --exclude "db.sqlite3" \
         --exclude ".mypy_cache" \
@@ -92,7 +93,7 @@ function validate {
     echo "running black"
     black --force-exclude "migrations/*" --diff --color --check -l 79 "$check_path"
     echo "running codespell"
-    codespell --skip="./.git,./.venv,./package.json,./package-lock.json,./node_modules,./.mypy_cache" "$check_path"
+    codespell --skip="./.git,./.venv,./package.json,./package-lock.json,./node_modules,./.mypy_cache,**/static/volume" "$check_path"
     echo "running flake8"
     flake8 "$check_path" --exclude "migrations,.venv" --count --max-complexity=10 \
         --max-line-length=79 --show-source --statistics

tubearchivist/api/src/search_processor.py

@@ -14,8 +14,6 @@ from home.src.ta.settings import EnvironmentSettings
 class SearchProcess:
     """process search results"""
 
-    CACHE_DIR = EnvironmentSettings.CACHE_DIR
-
     def __init__(self, response):
         self.response = response
         self.processed = False
@@ -66,7 +64,8 @@ class SearchProcess:
     def _process_channel(channel_dict):
         """run on single channel"""
         channel_id = channel_dict["channel_id"]
-        art_base = f"/cache/channels/{channel_id}"
+        cache_root = EnvironmentSettings().get_cache_root()
+        art_base = f"{cache_root}/channels/{channel_id}"
         date_str = date_parser(channel_dict["channel_last_refresh"])
         channel_dict.update(
             {
@@ -93,13 +92,16 @@ class SearchProcess:
                 url = video_dict["subtitles"][idx]["media_url"]
                 video_dict["subtitles"][idx]["media_url"] = f"/media/{url}"
 
+        cache_root = EnvironmentSettings().get_cache_root()
+        media_root = EnvironmentSettings().get_media_root()
+
         video_dict.update(
             {
                 "channel": channel,
-                "media_url": f"/media/{media_url}",
+                "media_url": f"{media_root}/{media_url}",
                 "vid_last_refresh": vid_last_refresh,
                 "published": published,
-                "vid_thumb_url": f"{self.CACHE_DIR}/{vid_thumb_url}",
+                "vid_thumb_url": f"{cache_root}/{vid_thumb_url}",
             }
         )
 
@@ -112,9 +114,11 @@ class SearchProcess:
         playlist_last_refresh = date_parser(
             playlist_dict["playlist_last_refresh"]
         )
+        cache_root = EnvironmentSettings().get_cache_root()
+        playlist_thumbnail = f"{cache_root}/playlists/{playlist_id}.jpg"
         playlist_dict.update(
             {
-                "playlist_thumbnail": f"/cache/playlists/{playlist_id}.jpg",
+                "playlist_thumbnail": playlist_thumbnail,
                 "playlist_last_refresh": playlist_last_refresh,
             }
         )
@@ -124,12 +128,13 @@ class SearchProcess:
     def _process_download(self, download_dict):
         """run on single download item"""
         video_id = download_dict["youtube_id"]
+        cache_root = EnvironmentSettings().get_cache_root()
         vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
         published = date_parser(download_dict["published"])
 
         download_dict.update(
             {
-                "vid_thumb_url": f"{self.CACHE_DIR}/{vid_thumb_url}",
+                "vid_thumb_url": f"{cache_root}/{vid_thumb_url}",
                 "published": published,
             }
         )

tubearchivist/config/settings.py

@@ -20,6 +20,14 @@ from django_auth_ldap.config import LDAPSearch
 from home.src.ta.helper import ta_host_parser
 from home.src.ta.settings import EnvironmentSettings
 
+try:
+    from dotenv import load_dotenv
+
+    load_dotenv(".env")
+except ModuleNotFoundError:
+    pass
+
+
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
 

tubearchivist/home/src/es/snapshot.py

@@ -251,10 +251,9 @@ class ElasticSnapshot:
     @staticmethod
     def _date_converter(date_utc):
         """convert datetime string"""
-        expected_format = "%Y-%m-%dT%H:%M:%S.%fZ"
-        date = datetime.strptime(date_utc, expected_format)
-        local_datetime = date.replace(tzinfo=ZoneInfo("localtime"))
-        converted = local_datetime.astimezone(ZoneInfo(EnvironmentSettings.TZ))
+        date = datetime.strptime(date_utc, "%Y-%m-%dT%H:%M:%S.%fZ")
+        utc_date = date.replace(tzinfo=ZoneInfo("UTC"))
+        converted = utc_date.astimezone(ZoneInfo(EnvironmentSettings.TZ))
         converted_str = converted.strftime("%Y-%m-%d %H:%M")
 
         return converted_str

tubearchivist/home/src/ta/helper.py

@@ -213,10 +213,14 @@ def ta_host_parser(ta_host: str) -> tuple[list[str], list[str]]:
     return allowed_hosts, csrf_trusted_origins
 
 
-def get_stylesheets():
+def get_stylesheets() -> list:
     """Get all valid stylesheets from /static/css"""
     app_root = EnvironmentSettings.APP_DIR
-    stylesheets = os.listdir(os.path.join(app_root, "static/css"))
+    try:
+        stylesheets = os.listdir(os.path.join(app_root, "static/css"))
+    except FileNotFoundError:
+        return []
+
     stylesheets.remove("style.css")
     stylesheets.sort()
     stylesheets = list(filter(lambda x: x.endswith(".css"), stylesheets))

tubearchivist/home/src/ta/settings.py

@@ -6,6 +6,14 @@ Functionality:
 
 from os import environ
 
+try:
+    from dotenv import load_dotenv
+
+    print("loading local dotenv")
+    load_dotenv(".env")
+except ModuleNotFoundError:
+    pass
+
 
 class EnvironmentSettings:
     """
@@ -44,6 +52,20 @@ class EnvironmentSettings:
     )
     ES_DISABLE_VERIFY_SSL: bool = bool(environ.get("ES_DISABLE_VERIFY_SSL"))
 
+    def get_cache_root(self):
+        """get root for web server"""
+        if self.CACHE_DIR.startswith("/"):
+            return self.CACHE_DIR
+
+        return f"/{self.CACHE_DIR}"
+
+    def get_media_root(self):
+        """get root for media folder"""
+        if self.MEDIA_DIR.startswith("/"):
+            return self.MEDIA_DIR
+
+        return f"/{self.MEDIA_DIR}"
+
     def print_generic(self):
         """print generic env vars"""
         print(

tubearchivist/home/templates/home/channel.html

@@ -46,14 +46,14 @@
                 <div class="channel-item {{ view_style }}">
                     <div class="channel-banner {{ view_style }}">
                         <a href="{% url 'channel_id' channel.channel_id %}">
-                            <img src="/cache/channels/{{ channel.channel_id }}_banner.jpg" alt="{{ channel.channel_id }}-banner">
+                            <img src="{{ channel.channel_banner_url }}" alt="{{ channel.channel_id }}-banner">
                         </a>
                     </div>
                     <div class="info-box info-box-2 {{ view_style }}">
                         <div class="info-box-item">
                             <div class="round-img">
                                 <a href="{% url 'channel_id' channel.channel_id %}">
-                                    <img src="/cache/channels/{{ channel.channel_id }}_thumb.jpg" alt="channel-thumb">
+                                    <img src="{{ channel.channel_thumb_url }}" alt="channel-thumb">
                                 </a>
                             </div>
                             <div>

tubearchivist/home/templates/home/channel_id.html

@@ -6,7 +6,7 @@
 
 <div class="boxed-content">
     <div class="channel-banner">
-        <a href="/channel/{{ channel_info.channel_id }}/"><img src="/cache/channels/{{ channel_info.channel_id }}_banner.jpg" alt="channel_banner"></a>
+        <a href="/channel/{{ channel_info.channel_id }}/"><img src="{{ channel_info.channel_banner_url }}" alt="channel_banner"></a>
     </div>
     <div class="info-box-item child-page-nav">
         <a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
@@ -31,7 +31,7 @@
         <div class="info-box-item">
             <div class="round-img">
                 <a href="{% url 'channel_id' channel_info.channel_id %}">
-                    <img src="/cache/channels/{{ channel_info.channel_id }}_thumb.jpg" alt="channel-thumb">
+                    <img src="{{ channel_info.channel_thumb_url }}" alt="channel-thumb">
                 </a>
             </div>
             <div>

tubearchivist/home/templates/home/channel_id_playlist.html

@@ -49,7 +49,7 @@
             <div class="playlist-item {{ view_style }}">
                 <div class="playlist-thumbnail">
                     <a href="{% url 'playlist_id' playlist.playlist_id %}">
-                        <img src="/cache/playlists/{{ playlist.playlist_id }}.jpg" alt="{{ playlist.playlist_id }}-thumbnail">
+                        <img src="{{ playlist.playlist_thumbnail }}" alt="{{ playlist.playlist_id }}-thumbnail">
                     </a>
                 </div>
                 <div class="playlist-desc {{ view_style }}">

tubearchivist/home/templates/home/playlist.html

@@ -52,7 +52,7 @@
             <div class="playlist-item {{ view_style }}">
                 <div class="playlist-thumbnail">
                     <a href="{% url 'playlist_id' playlist.playlist_id %}">
-                        <img src="/cache/playlists/{{ playlist.playlist_id }}.jpg" alt="{{ playlist.playlist_id }}-thumbnail">
+                        <img src="{{ playlist.playlist_thumbnail }}" alt="{{ playlist.playlist_id }}-thumbnail">
                     </a>
                 </div>
                 <div class="playlist-desc {{ view_style }}">

tubearchivist/home/templates/home/playlist_id.html

@@ -13,7 +13,7 @@
 	        <div class="info-box-item">
 	            <div class="round-img">
 	                <a href="{% url 'channel_id' channel_info.channel_id %}">
-	                    <img src="/cache/channels/{{ channel_info.channel_id }}_thumb.jpg" alt="channel-thumb">
+	                    <img src="{{ channel_info.channel_thumb_url }}" alt="channel-thumb">
 	                </a>
 	            </div>
 	            <div>

tubearchivist/requirements-dev.txt

@@ -2,9 +2,11 @@
 black
 codespell
 flake8
+ipython
 isort
 pylint
 pylint-django
 pytest
 pytest-django
+python-dotenv
 types-requests