consolidate in common app

tubearchivist/README.md

@@ -0,0 +1,14 @@
+# Django Setup
+
+## Apps
+
+### appsettings
+### channel
+### common
+### config
+### download
+### playlist
+### stats
+### task
+### user
+### video

tubearchivist/api/README.md

@@ -1,3 +0,0 @@
-# TubeArchivist API
-
-All API documentation has moved to [docs.tubearchivist.com](https://docs.tubearchivist.com/).

tubearchivist/api/admin.py

@@ -1,3 +0,0 @@
-from django.contrib import admin  # noqa: F401
-
-# Register your models here.

tubearchivist/api/apps.py

@@ -1,10 +0,0 @@
-"""apps file for api package"""
-
-from django.apps import AppConfig
-
-
-class ApiConfig(AppConfig):
-    """app config"""
-
-    default_auto_field = "django.db.models.BigAutoField"
-    name = "api"

tubearchivist/api/models.py

@@ -1,3 +0,0 @@
-"""api models"""
-
-# from django.db import models

tubearchivist/api/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase  # noqa: F401
-
-# Create your tests here.

tubearchivist/api/views.py

@@ -1,209 +0,0 @@
-"""all API views"""
-
-from api.src.search_processor import SearchProcess
-from appsettings.src.reindex import ReindexProgress
-from home.src.es.connect import ElasticWrap
-from home.src.frontend.searching import SearchForm
-from home.src.frontend.watched import WatchState
-from home.src.index.generic import Pagination
-from home.src.ta.config import AppConfig, ReleaseVersion
-from home.src.ta.settings import EnvironmentSettings
-from home.src.ta.ta_redis import RedisArchivist
-from rest_framework import permissions
-from rest_framework.authentication import (
-    SessionAuthentication,
-    TokenAuthentication,
-)
-from rest_framework.response import Response
-from rest_framework.views import APIView
-from task.tasks import check_reindex
-
-
-def check_admin(user):
-    """check for admin permission for restricted views"""
-    return user.is_staff or user.groups.filter(name="admin").exists()
-
-
-class AdminOnly(permissions.BasePermission):
-    """allow only admin"""
-
-    def has_permission(self, request, view):
-        return check_admin(request.user)
-
-
-class AdminWriteOnly(permissions.BasePermission):
-    """allow only admin writes"""
-
-    def has_permission(self, request, view):
-        if request.method in permissions.SAFE_METHODS:
-            return permissions.IsAuthenticated().has_permission(request, view)
-
-        return check_admin(request.user)
-
-
-class ApiBaseView(APIView):
-    """base view to inherit from"""
-
-    authentication_classes = [SessionAuthentication, TokenAuthentication]
-    permission_classes = [permissions.IsAuthenticated]
-    search_base = ""
-    data = ""
-
-    def __init__(self):
-        super().__init__()
-        self.response = {
-            "data": False,
-            "config": {
-                "enable_cast": EnvironmentSettings.ENABLE_CAST,
-                "downloads": AppConfig().config["downloads"],
-            },
-        }
-        self.data = {"query": {"match_all": {}}}
-        self.status_code = False
-        self.context = False
-        self.pagination_handler = False
-
-    def get_document(self, document_id):
-        """get single document from es"""
-        path = f"{self.search_base}{document_id}"
-        response, status_code = ElasticWrap(path).get()
-        try:
-            self.response["data"] = SearchProcess(response).process()
-        except KeyError:
-            print(f"item not found: {document_id}")
-            self.response["data"] = False
-        self.status_code = status_code
-
-    def initiate_pagination(self, request):
-        """set initial pagination values"""
-        self.pagination_handler = Pagination(request)
-        self.data.update(
-            {
-                "size": self.pagination_handler.pagination["page_size"],
-                "from": self.pagination_handler.pagination["page_from"],
-            }
-        )
-
-    def get_document_list(self, request, pagination=True):
-        """get a list of results"""
-        if pagination:
-            self.initiate_pagination(request)
-
-        es_handler = ElasticWrap(self.search_base)
-        response, status_code = es_handler.get(data=self.data)
-        self.response["data"] = SearchProcess(response).process()
-        if self.response["data"]:
-            self.status_code = status_code
-        else:
-            self.status_code = 404
-
-        if pagination:
-            self.pagination_handler.validate(
-                response["hits"]["total"]["value"]
-            )
-            self.response["paginate"] = self.pagination_handler.pagination
-
-
-class PingView(ApiBaseView):
-    """resolves to /api/ping/
-    GET: test your connection
-    """
-
-    @staticmethod
-    def get(request):
-        """get pong"""
-        data = {
-            "response": "pong",
-            "user": request.user.id,
-            "version": ReleaseVersion().get_local_version(),
-        }
-        return Response(data)
-
-
-class RefreshView(ApiBaseView):
-    """resolves to /api/refresh/
-    GET: get refresh progress
-    POST: start a manual refresh task
-    """
-
-    permission_classes = [AdminOnly]
-
-    def get(self, request):
-        """handle get request"""
-        request_type = request.GET.get("type")
-        request_id = request.GET.get("id")
-
-        if request_id and not request_type:
-            return Response({"status": "Bad Request"}, status=400)
-
-        try:
-            progress = ReindexProgress(
-                request_type=request_type, request_id=request_id
-            ).get_progress()
-        except ValueError:
-            return Response({"status": "Bad Request"}, status=400)
-
-        return Response(progress)
-
-    def post(self, request):
-        """handle post request"""
-        data = request.data
-        extract_videos = bool(request.GET.get("extract_videos", False))
-        check_reindex.delay(data=data, extract_videos=extract_videos)
-
-        return Response(data)
-
-
-class WatchedView(ApiBaseView):
-    """resolves to /api/watched/
-    POST: change watched state of video, channel or playlist
-    """
-
-    def post(self, request):
-        """change watched state"""
-        youtube_id = request.data.get("id")
-        is_watched = request.data.get("is_watched")
-
-        if not youtube_id or is_watched is None:
-            message = {"message": "missing id or is_watched"}
-            return Response(message, status=400)
-
-        WatchState(youtube_id, is_watched).change()
-        return Response({"message": "success"}, status=200)
-
-
-class SearchView(ApiBaseView):
-    """resolves to /api/search/
-    GET: run a search with the string in the ?query parameter
-    """
-
-    @staticmethod
-    def get(request):
-        """handle get request
-        search through all indexes"""
-        search_query = request.GET.get("query", None)
-        if search_query is None:
-            return Response(
-                {"message": "no search query specified"}, status=400
-            )
-
-        search_results = SearchForm().multi_search(search_query)
-        return Response(search_results)
-
-
-class NotificationView(ApiBaseView):
-    """resolves to /api/notification/
-    GET: returns a list of notifications
-    filter query to filter messages by group
-    """
-
-    valid_filters = ["download", "settings", "channel"]
-
-    def get(self, request):
-        """get all notifications"""
-        query = "message"
-        filter_by = request.GET.get("filter", None)
-        if filter_by in self.valid_filters:
-            query = f"{query}:{filter_by}"
-
-        return Response(RedisArchivist().list_items(query))

tubearchivist/appsettings/src/backup.py

@@ -10,10 +10,10 @@ import os
 import zipfile
 from datetime import datetime
 
-from home.src.es.connect import ElasticWrap, IndexPaginate
-from home.src.ta.config import AppConfig
-from home.src.ta.helper import get_mapping, ignore_filelist
-from home.src.ta.settings import EnvironmentSettings
+from appsettings.src.config import AppConfig
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap, IndexPaginate
+from common.src.helper import get_mapping, ignore_filelist
 from task.models import CustomPeriodicTask
 
 

tubearchivist/appsettings/src/config.py

@@ -9,8 +9,8 @@ from random import randint
 from time import sleep
 
 import requests
+from common.src.ta_redis import RedisArchivist
 from django.conf import settings
-from home.src.ta.ta_redis import RedisArchivist
 
 
 class AppConfig:

tubearchivist/appsettings/src/filesystem.py

@@ -5,9 +5,9 @@ Functionality:
 
 import os
 
-from home.src.es.connect import ElasticWrap, IndexPaginate
-from home.src.ta.helper import ignore_filelist
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap, IndexPaginate
+from common.src.helper import ignore_filelist
 from video.src.comments import CommentList
 from video.src.index import YoutubeVideo, index_new_video
 

tubearchivist/appsettings/src/index_setup.py

@@ -6,10 +6,10 @@ functionality:
 """
 
 from appsettings.src.backup import ElasticBackup
+from appsettings.src.config import AppConfig
 from appsettings.src.snapshot import ElasticSnapshot
-from home.src.es.connect import ElasticWrap
-from home.src.ta.config import AppConfig
-from home.src.ta.helper import get_mapping
+from common.src.es_connect import ElasticWrap
+from common.src.helper import get_mapping
 
 
 class ElasticIndex:

tubearchivist/appsettings/src/manual.py

@@ -11,10 +11,10 @@ import re
 import shutil
 import subprocess
 
+from appsettings.src.config import AppConfig
+from common.src.env_settings import EnvironmentSettings
+from common.src.helper import ignore_filelist
 from download.src.thumbnails import ThumbManager
-from home.src.ta.config import AppConfig
-from home.src.ta.helper import ignore_filelist
-from home.src.ta.settings import EnvironmentSettings
 from PIL import Image
 from video.src.comments import CommentList
 from video.src.index import YoutubeVideo

tubearchivist/appsettings/src/reindex.py

@@ -10,14 +10,14 @@ from datetime import datetime
 from time import sleep
 from typing import Callable, TypedDict
 
+from appsettings.src.config import AppConfig
 from channel.src.index import YoutubeChannel
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap, IndexPaginate
+from common.src.ta_redis import RedisQueue
 from download.src.subscriptions import ChannelSubscription
 from download.src.thumbnails import ThumbManager
 from download.src.yt_dlp_base import CookieHandler
-from home.src.es.connect import ElasticWrap, IndexPaginate
-from home.src.ta.config import AppConfig
-from home.src.ta.settings import EnvironmentSettings
-from home.src.ta.ta_redis import RedisQueue
 from playlist.src.index import YoutubePlaylist
 from task.models import CustomPeriodicTask
 from video.src.comments import Comments

tubearchivist/appsettings/src/snapshot.py

@@ -7,9 +7,9 @@ from datetime import datetime
 from time import sleep
 from zoneinfo import ZoneInfo
 
-from home.src.es.connect import ElasticWrap
-from home.src.ta.helper import get_mapping
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.helper import get_mapping
 
 
 class ElasticSnapshot:

tubearchivist/appsettings/views.py

@@ -1,11 +1,11 @@
 """all app settings API views"""
 
-from api.views import AdminOnly, ApiBaseView
 from appsettings.src.backup import ElasticBackup
+from appsettings.src.config import AppConfig
 from appsettings.src.snapshot import ElasticSnapshot
+from common.src.ta_redis import RedisArchivist
+from common.views_base import AdminOnly, ApiBaseView
 from download.src.yt_dlp_base import CookieHandler
-from home.src.ta.config import AppConfig
-from home.src.ta.ta_redis import RedisArchivist
 from rest_framework.response import Response
 from task.src.task_manager import TaskCommand
 from task.tasks import run_restore_backup

tubearchivist/channel/src/index.py

@@ -8,11 +8,11 @@ import json
 import os
 from datetime import datetime
 
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap, IndexPaginate
+from common.src.index_generic import YouTubeItem
 from download.src.thumbnails import ThumbManager
 from download.src.yt_dlp_base import YtWrap
-from home.src.es.connect import ElasticWrap, IndexPaginate
-from home.src.index.generic import YouTubeItem
-from home.src.ta.settings import EnvironmentSettings
 from playlist.src.index import YoutubePlaylist
 
 

tubearchivist/channel/views.py

@@ -1,9 +1,9 @@
 """all channel API views"""
 
-from api.views import AdminWriteOnly, ApiBaseView
 from channel.src.index import YoutubeChannel
+from common.src.urlparser import Parser
+from common.views_base import AdminWriteOnly, ApiBaseView
 from download.src.subscriptions import ChannelSubscription
-from home.src.ta.urlparser import Parser
 from rest_framework.response import Response
 from task.tasks import subscribe_to
 

tubearchivist/common/src/es_connect.py

@@ -11,7 +11,7 @@ from typing import Any
 
 import requests
 import urllib3
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
 
 
 class ElasticWrap:

tubearchivist/common/src/helper.py

@@ -13,8 +13,8 @@ from typing import Any
 from urllib.parse import urlparse
 
 import requests
-from home.src.es.connect import IndexPaginate
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import IndexPaginate
 
 
 def ignore_filelist(filelist: list[str]) -> list[str]:

tubearchivist/common/src/index_generic.py

@@ -5,9 +5,9 @@ functionality:
 
 import math
 
+from appsettings.src.config import AppConfig
+from common.src.es_connect import ElasticWrap
 from download.src.yt_dlp_base import YtWrap
-from home.src.es.connect import ElasticWrap
-from home.src.ta.config import AppConfig
 from user.src.user_config import UserConfig
 
 

tubearchivist/common/src/search_processor.py

@@ -6,9 +6,9 @@ Functionality:
 
 import urllib.parse
 
+from common.src.env_settings import EnvironmentSettings
+from common.src.helper import date_parser, get_duration_str
 from download.src.thumbnails import ThumbManager
-from home.src.ta.helper import date_parser, get_duration_str
-from home.src.ta.settings import EnvironmentSettings
 
 
 class SearchProcess:

tubearchivist/common/src/searching.py

@@ -6,8 +6,8 @@ Functionality:
 - calculate pagination values
 """
 
-from api.src.search_processor import SearchProcess
-from home.src.es.connect import ElasticWrap
+from common.src.es_connect import ElasticWrap
+from common.src.search_processor import SearchProcess
 
 
 class SearchForm:

tubearchivist/common/src/ta_redis.py

@@ -8,7 +8,7 @@ functionality:
 import json
 
 import redis
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
 
 
 class RedisBase:

tubearchivist/common/src/watched.py

@@ -5,8 +5,8 @@ functionality:
 
 from datetime import datetime
 
-from home.src.es.connect import ElasticWrap
-from home.src.ta.urlparser import Parser
+from common.src.es_connect import ElasticWrap
+from common.src.urlparser import Parser
 
 
 class WatchState:

tubearchivist/common/urls.py

@@ -1,6 +1,6 @@
 """all api urls"""
 
-from api import views
+from common import views
 from django.urls import path
 
 urlpatterns = [

tubearchivist/common/views.py

@@ -0,0 +1,115 @@
+"""all API views"""
+
+from appsettings.src.config import ReleaseVersion
+from appsettings.src.reindex import ReindexProgress
+from common.src.searching import SearchForm
+from common.src.ta_redis import RedisArchivist
+from common.src.watched import WatchState
+from common.views_base import AdminOnly, ApiBaseView
+from rest_framework.response import Response
+from task.tasks import check_reindex
+
+
+class PingView(ApiBaseView):
+    """resolves to /api/ping/
+    GET: test your connection
+    """
+
+    @staticmethod
+    def get(request):
+        """get pong"""
+        data = {
+            "response": "pong",
+            "user": request.user.id,
+            "version": ReleaseVersion().get_local_version(),
+        }
+        return Response(data)
+
+
+class RefreshView(ApiBaseView):
+    """resolves to /api/refresh/
+    GET: get refresh progress
+    POST: start a manual refresh task
+    """
+
+    permission_classes = [AdminOnly]
+
+    def get(self, request):
+        """handle get request"""
+        request_type = request.GET.get("type")
+        request_id = request.GET.get("id")
+
+        if request_id and not request_type:
+            return Response({"status": "Bad Request"}, status=400)
+
+        try:
+            progress = ReindexProgress(
+                request_type=request_type, request_id=request_id
+            ).get_progress()
+        except ValueError:
+            return Response({"status": "Bad Request"}, status=400)
+
+        return Response(progress)
+
+    def post(self, request):
+        """handle post request"""
+        data = request.data
+        extract_videos = bool(request.GET.get("extract_videos", False))
+        check_reindex.delay(data=data, extract_videos=extract_videos)
+
+        return Response(data)
+
+
+class WatchedView(ApiBaseView):
+    """resolves to /api/watched/
+    POST: change watched state of video, channel or playlist
+    """
+
+    def post(self, request):
+        """change watched state"""
+        youtube_id = request.data.get("id")
+        is_watched = request.data.get("is_watched")
+
+        if not youtube_id or is_watched is None:
+            message = {"message": "missing id or is_watched"}
+            return Response(message, status=400)
+
+        WatchState(youtube_id, is_watched).change()
+        return Response({"message": "success"}, status=200)
+
+
+class SearchView(ApiBaseView):
+    """resolves to /api/search/
+    GET: run a search with the string in the ?query parameter
+    """
+
+    @staticmethod
+    def get(request):
+        """handle get request
+        search through all indexes"""
+        search_query = request.GET.get("query", None)
+        if search_query is None:
+            return Response(
+                {"message": "no search query specified"}, status=400
+            )
+
+        search_results = SearchForm().multi_search(search_query)
+        return Response(search_results)
+
+
+class NotificationView(ApiBaseView):
+    """resolves to /api/notification/
+    GET: returns a list of notifications
+    filter query to filter messages by group
+    """
+
+    valid_filters = ["download", "settings", "channel"]
+
+    def get(self, request):
+        """get all notifications"""
+        query = "message"
+        filter_by = request.GET.get("filter", None)
+        if filter_by in self.valid_filters:
+            query = f"{query}:{filter_by}"
+
+        return Response(RedisArchivist().list_items(query))

tubearchivist/common/views_base.py

@@ -0,0 +1,98 @@
+"""base classes to inherit from"""
+
+from appsettings.src.config import AppConfig
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.index_generic import Pagination
+from common.src.search_processor import SearchProcess
+from rest_framework import permissions
+from rest_framework.authentication import (
+    SessionAuthentication,
+    TokenAuthentication,
+)
+from rest_framework.views import APIView
+
+
+def check_admin(user):
+    """check for admin permission for restricted views"""
+    return user.is_staff or user.groups.filter(name="admin").exists()
+
+
+class AdminOnly(permissions.BasePermission):
+    """allow only admin"""
+
+    def has_permission(self, request, view):
+        return check_admin(request.user)
+
+
+class AdminWriteOnly(permissions.BasePermission):
+    """allow only admin writes"""
+
+    def has_permission(self, request, view):
+        if request.method in permissions.SAFE_METHODS:
+            return permissions.IsAuthenticated().has_permission(request, view)
+
+        return check_admin(request.user)
+
+
+class ApiBaseView(APIView):
+    """base view to inherit from"""
+
+    authentication_classes = [SessionAuthentication, TokenAuthentication]
+    permission_classes = [permissions.IsAuthenticated]
+    search_base = ""
+    data = ""
+
+    def __init__(self):
+        super().__init__()
+        self.response = {
+            "data": False,
+            "config": {
+                "enable_cast": EnvironmentSettings.ENABLE_CAST,
+                "downloads": AppConfig().config["downloads"],
+            },
+        }
+        self.data = {"query": {"match_all": {}}}
+        self.status_code = False
+        self.context = False
+        self.pagination_handler = False
+
+    def get_document(self, document_id):
+        """get single document from es"""
+        path = f"{self.search_base}{document_id}"
+        response, status_code = ElasticWrap(path).get()
+        try:
+            self.response["data"] = SearchProcess(response).process()
+        except KeyError:
+            print(f"item not found: {document_id}")
+            self.response["data"] = False
+        self.status_code = status_code
+
+    def initiate_pagination(self, request):
+        """set initial pagination values"""
+        self.pagination_handler = Pagination(request)
+        self.data.update(
+            {
+                "size": self.pagination_handler.pagination["page_size"],
+                "from": self.pagination_handler.pagination["page_from"],
+            }
+        )
+
+    def get_document_list(self, request, pagination=True):
+        """get a list of results"""
+        if pagination:
+            self.initiate_pagination(request)
+
+        es_handler = ElasticWrap(self.search_base)
+        response, status_code = es_handler.get(data=self.data)
+        self.response["data"] = SearchProcess(response).process()
+        if self.response["data"]:
+            self.status_code = status_code
+        else:
+            self.status_code = 404
+
+        if pagination:
+            self.pagination_handler.validate(
+                response["hits"]["total"]["value"]
+            )
+            self.response["paginate"] = self.pagination_handler.pagination

tubearchivist/config/management/commands/ta_connection.py

@@ -6,10 +6,10 @@ Functionality:
 from time import sleep
 
 import requests
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.ta_redis import RedisArchivist
 from django.core.management.base import BaseCommand, CommandError
-from home.src.es.connect import ElasticWrap
-from home.src.ta.settings import EnvironmentSettings
-from home.src.ta.ta_redis import RedisArchivist
 
 TOPIC = """
 

tubearchivist/config/management/commands/ta_envcheck.py

@@ -9,8 +9,8 @@ Functionality:
 import os
 import re
 
+from common.src.env_settings import EnvironmentSettings
 from django.core.management.base import BaseCommand, CommandError
-from home.src.ta.settings import EnvironmentSettings
 from user.models import Account
 
 LOGO = """

tubearchivist/config/management/commands/ta_fix_channels.py

@@ -1,44 +0,0 @@
-"""
-channel fix for update from v0.4.7 to v0.4.8
-reindex channels with 0 subscriber count
-python manage.py ta_fix_channels
-"""
-
-from django.core.management.base import BaseCommand
-from home.src.es.connect import IndexPaginate
-from home.tasks import check_reindex
-
-
-class Command(BaseCommand):
-    """fix comment link"""
-
-    def handle(self, *args, **options):
-        """run command"""
-        self.stdout.write("reindex failed channels")
-        channels = self._get_channels()
-        if not channels:
-            self.stdout.write("did not find any failed channels")
-            return
-
-        self.stdout.write(f"add {len(channels)} channels(s) to queue")
-        to_reindex = {"channel": [i["channel_id"] for i in channels]}
-        check_reindex.delay(data=to_reindex)
-        self.stdout.write(self.style.SUCCESS("    ✓ task queued\n"))
-
-    def _get_channels(self):
-        """get failed channels"""
-        self.stdout.write("search for failed channels")
-        es_query = {
-            "query": {
-                "bool": {
-                    "must": [
-                        {"term": {"channel_subs": {"value": 0}}},
-                        {"term": {"channel_active": {"value": True}}},
-                    ]
-                },
-            },
-            "_source": ["channel_id"],
-        }
-        channels = IndexPaginate("ta_channel", es_query).get_results()
-
-        return channels

tubearchivist/config/management/commands/ta_fix_comment_link.py

@@ -1,76 +0,0 @@
-"""
-comment link fix for update from v0.4.7 to v0.4.8
-scan your videos and comments to fix comment_count field
-python manage.py ta_fix_comment_link
-"""
-
-from django.core.management.base import BaseCommand, CommandError
-from home.src.es.connect import ElasticWrap, IndexPaginate
-
-
-class Command(BaseCommand):
-    """fix comment link"""
-
-    def handle(self, *args, **options):
-        """run command"""
-        self.stdout.write("run comment link fix")
-        expected_count = self._get_comment_indexed()
-        all_videos = self._get_videos()
-
-        self.stdout.write(f"checking {len(all_videos)} video(s)")
-        videos_updated = []
-        for video in all_videos:
-            video_id = video["youtube_id"]
-            comment_count = expected_count.get(video_id)
-            if not comment_count:
-                continue
-
-            data = {"doc": {"comment_count": comment_count}}
-            path = f"ta_video/_update/{video_id}"
-            response, status_code = ElasticWrap(path).post(data=data)
-
-            if status_code != 200:
-                message = (
-                    "failed to add comment count to video"
-                    + f"response code: {status_code}"
-                    + response
-                )
-                raise CommandError(message)
-
-            videos_updated.append(video_id)
-
-        self.stdout.write(f"fixed {len(videos_updated)} video(s)")
-        self.stdout.write(self.style.SUCCESS("    ✓ task completed\n"))
-
-    def _get_comment_indexed(self):
-        """get comment count by index"""
-        self.stdout.write("get comments")
-        src = "params['_source']['comment_comments'].length"
-        data = {
-            "script_fields": {
-                "comments_length": {
-                    "script": {"source": src, "lang": "painless"}
-                }
-            }
-        }
-        all_comments = IndexPaginate(
-            "ta_comment", data=data, keep_source=True
-        ).get_results()
-
-        expected_count = {
-            i["_id"]: i["fields"]["comments_length"][0] for i in all_comments
-        }
-
-        return expected_count
-
-    def _get_videos(self):
-        """get videos without comment_count"""
-        self.stdout.write("get videos")
-        data = {
-            "query": {
-                "bool": {"must_not": [{"exists": {"field": "comment_count"}}]}
-            }
-        }
-        all_videos = IndexPaginate("ta_video", data).get_results()
-
-        return all_videos

tubearchivist/config/management/commands/ta_migpath.py

@@ -1,185 +0,0 @@
-"""
-filepath migration from v0.3.6 to v0.3.7
-not getting called at startup any more, to run manually if needed:
-python manage.py ta_migpath
-"""
-
-import json
-import os
-import shutil
-
-from django.core.management.base import BaseCommand
-from home.src.es.connect import ElasticWrap, IndexPaginate
-from home.src.ta.helper import ignore_filelist
-from home.src.ta.settings import EnvironmentSettings
-
-TOPIC = """
-
-########################
-# Filesystem Migration #
-########################
-
-"""
-
-
-class Command(BaseCommand):
-    """command framework"""
-
-    # pylint: disable=no-member
-
-    def handle(self, *args, **options):
-        """run commands"""
-        self.stdout.write(TOPIC)
-
-        handler = FolderMigration()
-        to_migrate = handler.get_to_migrate()
-        if not to_migrate:
-            self.stdout.write(
-                self.style.SUCCESS("    no channel migration needed\n")
-            )
-            return
-
-        self.stdout.write(self.style.SUCCESS("    migrating channels"))
-        total_channels = handler.create_folders(to_migrate)
-        self.stdout.write(
-            self.style.SUCCESS(f"    created {total_channels} channels")
-        )
-
-        self.stdout.write(
-            self.style.SUCCESS(f"    migrating {len(to_migrate)} videos")
-        )
-        handler.migrate_videos(to_migrate)
-        self.stdout.write(self.style.SUCCESS("    update videos in index"))
-        handler.send_bulk()
-
-        self.stdout.write(self.style.SUCCESS("    cleanup old folders"))
-        handler.delete_old()
-
-        self.stdout.write(self.style.SUCCESS("    ✓ migration completed\n"))
-
-
-class FolderMigration:
-    """migrate video archive folder"""
-
-    def __init__(self):
-        self.videos = EnvironmentSettings.MEDIA_DIR
-        self.bulk_list = []
-
-    def get_to_migrate(self):
-        """get videos to migrate"""
-        script = (
-            "doc['media_url'].value == "
-            + "doc['channel.channel_id'].value + '/'"
-            + " + doc['youtube_id'].value + '.mp4'"
-        )
-        data = {
-            "query": {"bool": {"must_not": [{"script": {"script": script}}]}},
-            "_source": [
-                "youtube_id",
-                "media_url",
-                "channel.channel_id",
-                "subtitles",
-            ],
-        }
-        response = IndexPaginate("ta_video", data).get_results()
-
-        return response
-
-    def create_folders(self, to_migrate):
-        """create required channel folders"""
-        host_uid = EnvironmentSettings.HOST_UID
-        host_gid = EnvironmentSettings.HOST_GID
-        all_channel_ids = {i["channel"]["channel_id"] for i in to_migrate}
-
-        for channel_id in all_channel_ids:
-            new_folder = os.path.join(self.videos, channel_id)
-            os.makedirs(new_folder, exist_ok=True)
-            if host_uid and host_gid:
-                os.chown(new_folder, host_uid, host_gid)
-
-        return len(all_channel_ids)
-
-    def migrate_videos(self, to_migrate):
-        """migrate all videos of channel"""
-        total = len(to_migrate)
-        for idx, video in enumerate(to_migrate):
-            new_media_url = self._move_video_file(video)
-            if not new_media_url:
-                continue
-
-            all_subtitles = self._move_subtitles(video)
-            action = {
-                "update": {"_id": video["youtube_id"], "_index": "ta_video"}
-            }
-            source = {"doc": {"media_url": new_media_url}}
-            if all_subtitles:
-                source["doc"].update({"subtitles": all_subtitles})
-
-            self.bulk_list.append(json.dumps(action))
-            self.bulk_list.append(json.dumps(source))
-            if idx % 1000 == 0:
-                print(f"processing migration [{idx}/{total}]")
-                self.send_bulk()
-
-    def _move_video_file(self, video):
-        """move video file to new location"""
-        old_path = os.path.join(self.videos, video["media_url"])
-        if not os.path.exists(old_path):
-            print(f"did not find expected video at {old_path}")
-            return False
-
-        new_media_url = os.path.join(
-            video["channel"]["channel_id"], video["youtube_id"] + ".mp4"
-        )
-        new_path = os.path.join(self.videos, new_media_url)
-        os.rename(old_path, new_path)
-
-        return new_media_url
-
-    def _move_subtitles(self, video):
-        """move subtitle files to new location"""
-        all_subtitles = video.get("subtitles")
-        if not all_subtitles:
-            return False
-
-        for subtitle in all_subtitles:
-            old_path = os.path.join(self.videos, subtitle["media_url"])
-            if not os.path.exists(old_path):
-                print(f"did not find expected subtitle at {old_path}")
-                continue
-
-            new_media_url = os.path.join(
-                video["channel"]["channel_id"],
-                f"{video.get('youtube_id')}.{subtitle.get('lang')}.vtt",
-            )
-            new_path = os.path.join(self.videos, new_media_url)
-            os.rename(old_path, new_path)
-            subtitle["media_url"] = new_media_url
-
-        return all_subtitles
-
-    def send_bulk(self):
-        """send bulk request to update index with new urls"""
-        if not self.bulk_list:
-            print("nothing to update")
-            return
-
-        self.bulk_list.append("\n")
-        path = "_bulk?refresh=true"
-        data = "\n".join(self.bulk_list)
-        response, status = ElasticWrap(path).post(data=data, ndjson=True)
-        if not status == 200:
-            print(response)
-
-        self.bulk_list = []
-
-    def delete_old(self):
-        """delete old empty folders"""
-        all_folders = ignore_filelist(os.listdir(self.videos))
-        for folder in all_folders:
-            folder_path = os.path.join(self.videos, folder)
-            if not os.path.isdir(folder_path):
-                continue
-
-            if not ignore_filelist(os.listdir(folder_path)):
-                shutil.rmtree(folder_path)

tubearchivist/config/management/commands/ta_startup.py

@@ -9,17 +9,17 @@ from datetime import datetime
 from random import randint
 from time import sleep
 
+from appsettings.src.config import AppConfig, ReleaseVersion
 from appsettings.src.index_setup import ElasitIndexWrap
 from appsettings.src.snapshot import ElasticSnapshot
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.helper import clear_dl_cache
+from common.src.ta_redis import RedisArchivist
 from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
 from django.utils import dateformat
 from django_celery_beat.models import CrontabSchedule, PeriodicTasks
-from home.src.es.connect import ElasticWrap
-from home.src.ta.config import AppConfig, ReleaseVersion
-from home.src.ta.helper import clear_dl_cache
-from home.src.ta.settings import EnvironmentSettings
-from home.src.ta.ta_redis import RedisArchivist
 from task.models import CustomPeriodicTask
 from task.src.config_schedule import ScheduleBuilder
 from task.src.notify import Notifications

tubearchivist/config/settings.py

@@ -15,10 +15,10 @@ from os import environ, path
 from pathlib import Path
 
 import ldap
+from common.src.env_settings import EnvironmentSettings
+from common.src.helper import ta_host_parser
 from corsheaders.defaults import default_headers
 from django_auth_ldap.config import LDAPSearch
-from home.src.ta.helper import ta_host_parser
-from home.src.ta.settings import EnvironmentSettings
 
 try:
     from dotenv import load_dotenv
@@ -61,7 +61,7 @@ INSTALLED_APPS = [
     "django.contrib.humanize",
     "rest_framework",
     "rest_framework.authtoken",
-    "api",
+    "common",
     "video",
     "channel",
     "playlist",
@@ -83,7 +83,7 @@ MIDDLEWARE = [
     "django.contrib.auth.middleware.AuthenticationMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
-    "home.src.ta.health.HealthCheckMiddleware",
+    "common.src.health.HealthCheckMiddleware",
 ]
 
 ROOT_URLCONF = "config.urls"
@@ -236,7 +236,7 @@ if bool(environ.get("TA_ENABLE_AUTH_PROXY")):
     )
     TA_AUTH_PROXY_LOGOUT_URL = environ.get("TA_AUTH_PROXY_LOGOUT_URL")
 
-    MIDDLEWARE.append("home.src.ta.auth.HttpRemoteUserMiddleware")
+    MIDDLEWARE.append("user.src.remote_user_auth.HttpRemoteUserMiddleware")
 
     AUTHENTICATION_BACKENDS = (
         "django.contrib.auth.backends.RemoteUserBackend",

tubearchivist/config/urls.py

@@ -19,7 +19,7 @@ from django.urls import include, path
 
 urlpatterns = [
     path("", include("home.urls")),
-    path("api/", include("api.urls")),
+    path("api/", include("common.urls")),
     path("api/video/", include("video.urls")),
     path("api/channel/", include("channel.urls")),
     path("api/playlist/", include("playlist.urls")),

tubearchivist/download/src/queue.py

@@ -7,12 +7,12 @@ Functionality:
 import json
 from datetime import datetime
 
+from appsettings.src.config import AppConfig
+from common.src.es_connect import ElasticWrap, IndexPaginate
+from common.src.helper import get_duration_str, is_shorts
 from download.src.subscriptions import ChannelSubscription
 from download.src.thumbnails import ThumbManager
 from download.src.yt_dlp_base import YtWrap
-from home.src.es.connect import ElasticWrap, IndexPaginate
-from home.src.ta.config import AppConfig
-from home.src.ta.helper import get_duration_str, is_shorts
 from playlist.src.index import YoutubePlaylist
 from video.src.constants import VideoTypeEnum
 

tubearchivist/download/src/subscriptions.py

@@ -4,13 +4,13 @@ Functionality:
 - handle playlist subscriptions
 """
 
+from appsettings.src.config import AppConfig
 from channel.src.index import YoutubeChannel
+from common.src.es_connect import IndexPaginate
+from common.src.helper import is_missing
+from common.src.urlparser import Parser
 from download.src.thumbnails import ThumbManager
 from download.src.yt_dlp_base import YtWrap
-from home.src.es.connect import IndexPaginate
-from home.src.ta.config import AppConfig
-from home.src.ta.helper import is_missing
-from home.src.ta.urlparser import Parser
 from playlist.src.index import YoutubePlaylist
 from video.src.constants import VideoTypeEnum
 from video.src.index import YoutubeVideo

tubearchivist/download/src/thumbnails.py

@@ -10,9 +10,9 @@ from io import BytesIO
 from time import sleep
 
 import requests
-from home.src.es.connect import ElasticWrap, IndexPaginate
-from home.src.ta.helper import is_missing
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap, IndexPaginate
+from common.src.helper import is_missing
 from mutagen.mp4 import MP4, MP4Cover
 from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError
 

tubearchivist/download/src/yt_dlp_base.py

@@ -10,8 +10,8 @@ from http import cookiejar
 from io import StringIO
 
 import yt_dlp
-from home.src.ta.settings import EnvironmentSettings
-from home.src.ta.ta_redis import RedisArchivist
+from common.src.env_settings import EnvironmentSettings
+from common.src.ta_redis import RedisArchivist
 
 
 class YtWrap:

tubearchivist/download/src/yt_dlp_handler.py

@@ -10,15 +10,15 @@ import os
 import shutil
 from datetime import datetime
 
+from appsettings.src.config import AppConfig
 from channel.src.index import YoutubeChannel
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap, IndexPaginate
+from common.src.helper import get_channel_overwrites, ignore_filelist
+from common.src.ta_redis import RedisQueue
 from download.src.queue import PendingList
 from download.src.subscriptions import PlaylistSubscription
 from download.src.yt_dlp_base import YtWrap
-from home.src.es.connect import ElasticWrap, IndexPaginate
-from home.src.ta.config import AppConfig
-from home.src.ta.helper import get_channel_overwrites, ignore_filelist
-from home.src.ta.settings import EnvironmentSettings
-from home.src.ta.ta_redis import RedisQueue
 from playlist.src.index import YoutubePlaylist
 from video.src.comments import CommentList
 from video.src.constants import VideoTypeEnum

tubearchivist/download/views.py

@@ -1,6 +1,6 @@
 """all download API views"""
 
-from api.views import AdminOnly, ApiBaseView
+from common.views_base import AdminOnly, ApiBaseView
 from download.src.queue import PendingInteract
 from rest_framework.response import Response
 from task.tasks import download_pending, extrac_dl

tubearchivist/home/celery.py

@@ -3,7 +3,7 @@
 import os
 
 from celery import Celery
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
 
 REDIS_HOST = EnvironmentSettings.REDIS_HOST
 REDIS_PORT = EnvironmentSettings.REDIS_PORT

tubearchivist/home/src/frontend/forms.py

@@ -4,10 +4,10 @@
 
 import os
 
+from common.src.helper import get_stylesheets
 from django import forms
 from django.contrib.auth.forms import AuthenticationForm
 from django.forms.widgets import PasswordInput, TextInput
-from home.src.ta.helper import get_stylesheets
 
 
 class CustomAuthForm(AuthenticationForm):

tubearchivist/home/views.py

@@ -9,12 +9,18 @@ import urllib.parse
 import uuid
 from time import sleep
 
-from api.src.search_processor import SearchProcess, process_aggs
-from api.views import check_admin
 from appsettings.src.backup import ElasticBackup
+from appsettings.src.config import AppConfig, ReleaseVersion
 from appsettings.src.reindex import ReindexProgress
 from appsettings.src.snapshot import ElasticSnapshot
 from channel.src.index import channel_overwrites
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.helper import check_stylesheet, time_parser
+from common.src.index_generic import Pagination
+from common.src.search_processor import SearchProcess, process_aggs
+from common.src.ta_redis import RedisArchivist
+from common.views_base import check_admin
 from django.conf import settings
 from django.contrib.auth import login
 from django.contrib.auth.decorators import user_passes_test
@@ -25,7 +31,6 @@ from django.utils.decorators import method_decorator
 from django.views import View
 from download.src.queue import PendingInteract
 from download.src.yt_dlp_base import CookieHandler
-from home.src.es.connect import ElasticWrap
 from home.src.frontend.forms import (
     AddToQueueForm,
     ApplicationSettingsForm,
@@ -41,11 +46,6 @@ from home.src.frontend.forms_schedule import (
     NotificationSettingsForm,
     SchedulerSettingsForm,
 )
-from home.src.index.generic import Pagination
-from home.src.ta.config import AppConfig, ReleaseVersion
-from home.src.ta.helper import check_stylesheet, time_parser
-from home.src.ta.settings import EnvironmentSettings
-from home.src.ta.ta_redis import RedisArchivist
 from playlist.src.index import YoutubePlaylist
 from rest_framework.authtoken.models import Token
 from task.models import CustomPeriodicTask

tubearchivist/playlist/src/index.py

@@ -8,9 +8,9 @@ import json
 from datetime import datetime
 
 from channel.src import index as channel
+from common.src.es_connect import ElasticWrap, IndexPaginate
+from common.src.index_generic import YouTubeItem
 from download.src.thumbnails import ThumbManager
-from home.src.es.connect import ElasticWrap, IndexPaginate
-from home.src.index.generic import YouTubeItem
 from video.src import index as ta_video
 
 

tubearchivist/playlist/views.py

@@ -1,6 +1,6 @@
 """all playlist API views"""
 
-from api.views import AdminWriteOnly, ApiBaseView
+from common.views_base import AdminWriteOnly, ApiBaseView
 from download.src.subscriptions import PlaylistSubscription
 from playlist.src.index import YoutubePlaylist
 from rest_framework import status

tubearchivist/stats/src/aggs.py

@@ -1,8 +1,8 @@
 """aggregations"""
 
-from home.src.es.connect import ElasticWrap
-from home.src.ta.helper import get_duration_str
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.helper import get_duration_str
 
 
 class AggBase:

tubearchivist/stats/views.py

@@ -1,6 +1,6 @@
 """all stats API views"""
 
-from api.views import ApiBaseView
+from common.views_base import ApiBaseView
 from rest_framework.response import Response
 from stats.src.aggs import (
     BiggestChannel,

tubearchivist/task/src/config_schedule.py

@@ -5,10 +5,10 @@ Functionality:
 
 from datetime import datetime
 
+from appsettings.src.config import AppConfig
+from common.src.env_settings import EnvironmentSettings
 from django.utils import dateformat
 from django_celery_beat.models import CrontabSchedule
-from home.src.ta.config import AppConfig
-from home.src.ta.settings import EnvironmentSettings
 from task.models import CustomPeriodicTask
 from task.src.task_config import TASK_CONFIG
 

tubearchivist/task/src/notify.py

@@ -1,7 +1,7 @@
 """send notifications using apprise"""
 
 import apprise
-from home.src.es.connect import ElasticWrap
+from common.src.es_connect import ElasticWrap
 from task.src.task_config import TASK_CONFIG
 from task.src.task_manager import TaskManager
 

tubearchivist/task/src/task_manager.py

@@ -4,8 +4,8 @@ functionality:
 - handle threads and locks
 """
 
+from common.src.ta_redis import RedisArchivist, TaskRedis
 from home.celery import app as celery_app
-from home.src.ta.ta_redis import RedisArchivist, TaskRedis
 from task.src.task_config import TASK_CONFIG
 
 

tubearchivist/task/tasks.py

@@ -7,6 +7,7 @@ Functionality:
 """
 
 from appsettings.src.backup import ElasticBackup
+from appsettings.src.config import ReleaseVersion
 from appsettings.src.filesystem import Scanner
 from appsettings.src.index_setup import ElasitIndexWrap
 from appsettings.src.manual import ImportFolderScanner
@@ -14,13 +15,12 @@ from appsettings.src.reindex import Reindex, ReindexManual, ReindexPopulate
 from celery import Task, shared_task
 from celery.exceptions import Retry
 from channel.src.index import YoutubeChannel
+from common.src.ta_redis import RedisArchivist
+from common.src.urlparser import Parser
 from download.src.queue import PendingList
 from download.src.subscriptions import SubscriptionHandler, SubscriptionScanner
 from download.src.thumbnails import ThumbFilesystem, ThumbValidator
 from download.src.yt_dlp_handler import VideoDownloader
-from home.src.ta.config import ReleaseVersion
-from home.src.ta.ta_redis import RedisArchivist
-from home.src.ta.urlparser import Parser
 from task.src.notify import Notifications
 from task.src.task_config import TASK_CONFIG
 from task.src.task_manager import TaskManager

tubearchivist/task/views.py

@@ -1,6 +1,6 @@
 """all task API views"""
 
-from api.views import AdminOnly, ApiBaseView
+from common.views_base import AdminOnly, ApiBaseView
 from rest_framework.response import Response
 from task.models import CustomPeriodicTask
 from task.src.notify import Notifications, get_all_notifications

tubearchivist/user/src/user_config.py

@@ -6,8 +6,8 @@ Functionality:
 
 from typing import TypedDict
 
-from home.src.es.connect import ElasticWrap
-from home.src.ta.helper import get_stylesheets
+from common.src.es_connect import ElasticWrap
+from common.src.helper import get_stylesheets
 
 
 class UserConfigType(TypedDict, total=False):

tubearchivist/user/views.py

@@ -1,6 +1,6 @@
 """all user api views"""
 
-from api.views import ApiBaseView
+from common.views import ApiBaseView
 from rest_framework.authtoken.models import Token
 from rest_framework.authtoken.views import ObtainAuthToken
 from rest_framework.response import Response

tubearchivist/video/src/comments.py

@@ -7,10 +7,10 @@ Functionality:
 
 from datetime import datetime
 
+from appsettings.src.config import AppConfig
+from common.src.es_connect import ElasticWrap
+from common.src.ta_redis import RedisQueue
 from download.src.yt_dlp_base import YtWrap
-from home.src.es.connect import ElasticWrap
-from home.src.ta.config import AppConfig
-from home.src.ta.ta_redis import RedisQueue
 
 
 class Comments:

tubearchivist/video/src/index.py

@@ -9,11 +9,11 @@ from datetime import datetime
 
 import requests
 from channel.src import index as ta_channel
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.helper import get_duration_sec, get_duration_str, randomizor
+from common.src.index_generic import YouTubeItem
 from django.conf import settings
-from home.src.es.connect import ElasticWrap
-from home.src.index.generic import YouTubeItem
-from home.src.ta.helper import get_duration_sec, get_duration_str, randomizor
-from home.src.ta.settings import EnvironmentSettings
 from playlist.src import index as ta_playlist
 from ryd_client import ryd_client
 from user.src.user_config import UserConfig

tubearchivist/video/src/subtitle.py

@@ -10,9 +10,9 @@ import os
 from datetime import datetime
 
 import requests
-from home.src.es.connect import ElasticWrap
-from home.src.ta.helper import requests_headers
-from home.src.ta.settings import EnvironmentSettings
+from common.src.env_settings import EnvironmentSettings
+from common.src.es_connect import ElasticWrap
+from common.src.helper import requests_headers
 
 
 class YoutubeSubtitle:

tubearchivist/video/views.py

@@ -1,7 +1,7 @@
 """all API views for video endpoints"""
 
-from api.views import AdminWriteOnly, ApiBaseView
-from home.src.ta.ta_redis import RedisArchivist
+from common.src.ta_redis import RedisArchivist
+from common.views_base import AdminWriteOnly, ApiBaseView
 from rest_framework.response import Response
 from video.src.index import YoutubeVideo