diff --git a/tubearchivist/README.md b/tubearchivist/README.md new file mode 100644 index 00000000..cb5bd2c2 --- /dev/null +++ b/tubearchivist/README.md @@ -0,0 +1,14 @@ +# Django Setup + +## Apps + +### appsettings +### channel +### common +### config +### download +### playlist +### stats +### task +### user +### video diff --git a/tubearchivist/api/README.md b/tubearchivist/api/README.md deleted file mode 100644 index 2815b6ec..00000000 --- a/tubearchivist/api/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# TubeArchivist API - -All API documentation has moved to [docs.tubearchivist.com](https://docs.tubearchivist.com/). diff --git a/tubearchivist/api/admin.py b/tubearchivist/api/admin.py deleted file mode 100644 index 4fd54902..00000000 --- a/tubearchivist/api/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin # noqa: F401 - -# Register your models here. diff --git a/tubearchivist/api/apps.py b/tubearchivist/api/apps.py deleted file mode 100644 index 9acf5fb4..00000000 --- a/tubearchivist/api/apps.py +++ /dev/null @@ -1,10 +0,0 @@ -"""apps file for api package""" - -from django.apps import AppConfig - - -class ApiConfig(AppConfig): - """app config""" - - default_auto_field = "django.db.models.BigAutoField" - name = "api" diff --git a/tubearchivist/api/models.py b/tubearchivist/api/models.py deleted file mode 100644 index b225e996..00000000 --- a/tubearchivist/api/models.py +++ /dev/null @@ -1,3 +0,0 @@ -"""api models""" - -# from django.db import models diff --git a/tubearchivist/api/serializers.py b/tubearchivist/api/serializers.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tubearchivist/api/tests.py b/tubearchivist/api/tests.py deleted file mode 100644 index e55d6890..00000000 --- a/tubearchivist/api/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase # noqa: F401 - -# Create your tests here. diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py deleted file mode 100644 index 01fc0050..00000000 --- a/tubearchivist/api/views.py +++ /dev/null @@ -1,209 +0,0 @@ -"""all API views""" - -from api.src.search_processor import SearchProcess -from appsettings.src.reindex import ReindexProgress -from home.src.es.connect import ElasticWrap -from home.src.frontend.searching import SearchForm -from home.src.frontend.watched import WatchState -from home.src.index.generic import Pagination -from home.src.ta.config import AppConfig, ReleaseVersion -from home.src.ta.settings import EnvironmentSettings -from home.src.ta.ta_redis import RedisArchivist -from rest_framework import permissions -from rest_framework.authentication import ( - SessionAuthentication, - TokenAuthentication, -) -from rest_framework.response import Response -from rest_framework.views import APIView -from task.tasks import check_reindex - - -def check_admin(user): - """check for admin permission for restricted views""" - return user.is_staff or user.groups.filter(name="admin").exists() - - -class AdminOnly(permissions.BasePermission): - """allow only admin""" - - def has_permission(self, request, view): - return check_admin(request.user) - - -class AdminWriteOnly(permissions.BasePermission): - """allow only admin writes""" - - def has_permission(self, request, view): - if request.method in permissions.SAFE_METHODS: - return permissions.IsAuthenticated().has_permission(request, view) - - return check_admin(request.user) - - -class ApiBaseView(APIView): - """base view to inherit from""" - - authentication_classes = [SessionAuthentication, TokenAuthentication] - permission_classes = [permissions.IsAuthenticated] - search_base = "" - data = "" - - def __init__(self): - super().__init__() - self.response = { - "data": False, - "config": { - "enable_cast": EnvironmentSettings.ENABLE_CAST, - "downloads": AppConfig().config["downloads"], - }, - } - self.data = {"query": {"match_all": {}}} - self.status_code = False - self.context = False - self.pagination_handler = False - - def get_document(self, document_id): - """get single document from es""" - path = f"{self.search_base}{document_id}" - response, status_code = ElasticWrap(path).get() - try: - self.response["data"] = SearchProcess(response).process() - except KeyError: - print(f"item not found: {document_id}") - self.response["data"] = False - self.status_code = status_code - - def initiate_pagination(self, request): - """set initial pagination values""" - self.pagination_handler = Pagination(request) - self.data.update( - { - "size": self.pagination_handler.pagination["page_size"], - "from": self.pagination_handler.pagination["page_from"], - } - ) - - def get_document_list(self, request, pagination=True): - """get a list of results""" - if pagination: - self.initiate_pagination(request) - - es_handler = ElasticWrap(self.search_base) - response, status_code = es_handler.get(data=self.data) - self.response["data"] = SearchProcess(response).process() - if self.response["data"]: - self.status_code = status_code - else: - self.status_code = 404 - - if pagination: - self.pagination_handler.validate( - response["hits"]["total"]["value"] - ) - self.response["paginate"] = self.pagination_handler.pagination - - -class PingView(ApiBaseView): - """resolves to /api/ping/ - GET: test your connection - """ - - @staticmethod - def get(request): - """get pong""" - data = { - "response": "pong", - "user": request.user.id, - "version": ReleaseVersion().get_local_version(), - } - return Response(data) - - -class RefreshView(ApiBaseView): - """resolves to /api/refresh/ - GET: get refresh progress - POST: start a manual refresh task - """ - - permission_classes = [AdminOnly] - - def get(self, request): - """handle get request""" - request_type = request.GET.get("type") - request_id = request.GET.get("id") - - if request_id and not request_type: - return Response({"status": "Bad Request"}, status=400) - - try: - progress = ReindexProgress( - request_type=request_type, request_id=request_id - ).get_progress() - except ValueError: - return Response({"status": "Bad Request"}, status=400) - - return Response(progress) - - def post(self, request): - """handle post request""" - data = request.data - extract_videos = bool(request.GET.get("extract_videos", False)) - check_reindex.delay(data=data, extract_videos=extract_videos) - - return Response(data) - - -class WatchedView(ApiBaseView): - """resolves to /api/watched/ - POST: change watched state of video, channel or playlist - """ - - def post(self, request): - """change watched state""" - youtube_id = request.data.get("id") - is_watched = request.data.get("is_watched") - - if not youtube_id or is_watched is None: - message = {"message": "missing id or is_watched"} - return Response(message, status=400) - - WatchState(youtube_id, is_watched).change() - return Response({"message": "success"}, status=200) - - -class SearchView(ApiBaseView): - """resolves to /api/search/ - GET: run a search with the string in the ?query parameter - """ - - @staticmethod - def get(request): - """handle get request - search through all indexes""" - search_query = request.GET.get("query", None) - if search_query is None: - return Response( - {"message": "no search query specified"}, status=400 - ) - - search_results = SearchForm().multi_search(search_query) - return Response(search_results) - - -class NotificationView(ApiBaseView): - """resolves to /api/notification/ - GET: returns a list of notifications - filter query to filter messages by group - """ - - valid_filters = ["download", "settings", "channel"] - - def get(self, request): - """get all notifications""" - query = "message" - filter_by = request.GET.get("filter", None) - if filter_by in self.valid_filters: - query = f"{query}:{filter_by}" - - return Response(RedisArchivist().list_items(query)) diff --git a/tubearchivist/appsettings/src/backup.py b/tubearchivist/appsettings/src/backup.py index c7c1b268..295b5345 100644 --- a/tubearchivist/appsettings/src/backup.py +++ b/tubearchivist/appsettings/src/backup.py @@ -10,10 +10,10 @@ import os import zipfile from datetime import datetime -from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.ta.config import AppConfig -from home.src.ta.helper import get_mapping, ignore_filelist -from home.src.ta.settings import EnvironmentSettings +from appsettings.src.config import AppConfig +from common.src.env_settings import EnvironmentSettings +from common.src.es_connect import ElasticWrap, IndexPaginate +from common.src.helper import get_mapping, ignore_filelist from task.models import CustomPeriodicTask diff --git a/tubearchivist/home/src/ta/config.py b/tubearchivist/appsettings/src/config.py similarity index 99% rename from tubearchivist/home/src/ta/config.py rename to tubearchivist/appsettings/src/config.py index af342b93..75948254 100644 --- a/tubearchivist/home/src/ta/config.py +++ b/tubearchivist/appsettings/src/config.py @@ -9,8 +9,8 @@ from random import randint from time import sleep import requests +from common.src.ta_redis import RedisArchivist from django.conf import settings -from home.src.ta.ta_redis import RedisArchivist class AppConfig: diff --git a/tubearchivist/appsettings/src/filesystem.py b/tubearchivist/appsettings/src/filesystem.py index d31fdbc5..64aad6aa 100644 --- a/tubearchivist/appsettings/src/filesystem.py +++ b/tubearchivist/appsettings/src/filesystem.py @@ -5,9 +5,9 @@ Functionality: import os -from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.ta.helper import ignore_filelist -from home.src.ta.settings import EnvironmentSettings +from common.src.env_settings import EnvironmentSettings +from common.src.es_connect import ElasticWrap, IndexPaginate +from common.src.helper import ignore_filelist from video.src.comments import CommentList from video.src.index import YoutubeVideo, index_new_video diff --git a/tubearchivist/appsettings/src/index_setup.py b/tubearchivist/appsettings/src/index_setup.py index 84484021..d22f07d3 100644 --- a/tubearchivist/appsettings/src/index_setup.py +++ b/tubearchivist/appsettings/src/index_setup.py @@ -6,10 +6,10 @@ functionality: """ from appsettings.src.backup import ElasticBackup +from appsettings.src.config import AppConfig from appsettings.src.snapshot import ElasticSnapshot -from home.src.es.connect import ElasticWrap -from home.src.ta.config import AppConfig -from home.src.ta.helper import get_mapping +from common.src.es_connect import ElasticWrap +from common.src.helper import get_mapping class ElasticIndex: diff --git a/tubearchivist/appsettings/src/manual.py b/tubearchivist/appsettings/src/manual.py index 6271bdf9..0a7015aa 100644 --- a/tubearchivist/appsettings/src/manual.py +++ b/tubearchivist/appsettings/src/manual.py @@ -11,10 +11,10 @@ import re import shutil import subprocess +from appsettings.src.config import AppConfig +from common.src.env_settings import EnvironmentSettings +from common.src.helper import ignore_filelist from download.src.thumbnails import ThumbManager -from home.src.ta.config import AppConfig -from home.src.ta.helper import ignore_filelist -from home.src.ta.settings import EnvironmentSettings from PIL import Image from video.src.comments import CommentList from video.src.index import YoutubeVideo diff --git a/tubearchivist/appsettings/src/reindex.py b/tubearchivist/appsettings/src/reindex.py index bdf58209..9af73d97 100644 --- a/tubearchivist/appsettings/src/reindex.py +++ b/tubearchivist/appsettings/src/reindex.py @@ -10,14 +10,14 @@ from datetime import datetime from time import sleep from typing import Callable, TypedDict +from appsettings.src.config import AppConfig from channel.src.index import YoutubeChannel +from common.src.env_settings import EnvironmentSettings +from common.src.es_connect import ElasticWrap, IndexPaginate +from common.src.ta_redis import RedisQueue from download.src.subscriptions import ChannelSubscription from download.src.thumbnails import ThumbManager from download.src.yt_dlp_base import CookieHandler -from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.ta.config import AppConfig -from home.src.ta.settings import EnvironmentSettings -from home.src.ta.ta_redis import RedisQueue from playlist.src.index import YoutubePlaylist from task.models import CustomPeriodicTask from video.src.comments import Comments diff --git a/tubearchivist/appsettings/src/snapshot.py b/tubearchivist/appsettings/src/snapshot.py index 5d946097..0b62f149 100644 --- a/tubearchivist/appsettings/src/snapshot.py +++ b/tubearchivist/appsettings/src/snapshot.py @@ -7,9 +7,9 @@ from datetime import datetime from time import sleep from zoneinfo import ZoneInfo -from home.src.es.connect import ElasticWrap -from home.src.ta.helper import get_mapping -from home.src.ta.settings import EnvironmentSettings +from common.src.env_settings import EnvironmentSettings +from common.src.es_connect import ElasticWrap +from common.src.helper import get_mapping class ElasticSnapshot: diff --git a/tubearchivist/appsettings/views.py b/tubearchivist/appsettings/views.py index c5480b61..77c7bbe9 100644 --- a/tubearchivist/appsettings/views.py +++ b/tubearchivist/appsettings/views.py @@ -1,11 +1,11 @@ """all app settings API views""" -from api.views import AdminOnly, ApiBaseView from appsettings.src.backup import ElasticBackup +from appsettings.src.config import AppConfig from appsettings.src.snapshot import ElasticSnapshot +from common.src.ta_redis import RedisArchivist +from common.views_base import AdminOnly, ApiBaseView from download.src.yt_dlp_base import CookieHandler -from home.src.ta.config import AppConfig -from home.src.ta.ta_redis import RedisArchivist from rest_framework.response import Response from task.src.task_manager import TaskCommand from task.tasks import run_restore_backup diff --git a/tubearchivist/channel/src/index.py b/tubearchivist/channel/src/index.py index f12e4790..b3b47db9 100644 --- a/tubearchivist/channel/src/index.py +++ b/tubearchivist/channel/src/index.py @@ -8,11 +8,11 @@ import json import os from datetime import datetime +from common.src.env_settings import EnvironmentSettings +from common.src.es_connect import ElasticWrap, IndexPaginate +from common.src.index_generic import YouTubeItem from download.src.thumbnails import ThumbManager from download.src.yt_dlp_base import YtWrap -from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.index.generic import YouTubeItem -from home.src.ta.settings import EnvironmentSettings from playlist.src.index import YoutubePlaylist diff --git a/tubearchivist/channel/views.py b/tubearchivist/channel/views.py index a0b82d24..601f78a0 100644 --- a/tubearchivist/channel/views.py +++ b/tubearchivist/channel/views.py @@ -1,9 +1,9 @@ """all channel API views""" -from api.views import AdminWriteOnly, ApiBaseView from channel.src.index import YoutubeChannel +from common.src.urlparser import Parser +from common.views_base import AdminWriteOnly, ApiBaseView from download.src.subscriptions import ChannelSubscription -from home.src.ta.urlparser import Parser from rest_framework.response import Response from task.tasks import subscribe_to diff --git a/tubearchivist/api/__init__.py b/tubearchivist/common/__init__.py similarity index 100% rename from tubearchivist/api/__init__.py rename to tubearchivist/common/__init__.py diff --git a/tubearchivist/api/migrations/__init__.py b/tubearchivist/common/migrations/__init__.py similarity index 100% rename from tubearchivist/api/migrations/__init__.py rename to tubearchivist/common/migrations/__init__.py diff --git a/tubearchivist/api/src/__init__.py b/tubearchivist/common/src/__init__.py similarity index 100% rename from tubearchivist/api/src/__init__.py rename to tubearchivist/common/src/__init__.py diff --git a/tubearchivist/home/src/ta/settings.py b/tubearchivist/common/src/env_settings.py similarity index 100% rename from tubearchivist/home/src/ta/settings.py rename to tubearchivist/common/src/env_settings.py diff --git a/tubearchivist/home/src/es/connect.py b/tubearchivist/common/src/es_connect.py similarity index 99% rename from tubearchivist/home/src/es/connect.py rename to tubearchivist/common/src/es_connect.py index 7f2fe3fc..f7a59e3a 100644 --- a/tubearchivist/home/src/es/connect.py +++ b/tubearchivist/common/src/es_connect.py @@ -11,7 +11,7 @@ from typing import Any import requests import urllib3 -from home.src.ta.settings import EnvironmentSettings +from common.src.env_settings import EnvironmentSettings class ElasticWrap: diff --git a/tubearchivist/home/src/ta/health.py b/tubearchivist/common/src/health.py similarity index 100% rename from tubearchivist/home/src/ta/health.py rename to tubearchivist/common/src/health.py diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/common/src/helper.py similarity index 98% rename from tubearchivist/home/src/ta/helper.py rename to tubearchivist/common/src/helper.py index c89246f4..255adcc1 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/common/src/helper.py @@ -13,8 +13,8 @@ from typing import Any from urllib.parse import urlparse import requests -from home.src.es.connect import IndexPaginate -from home.src.ta.settings import EnvironmentSettings +from common.src.env_settings import EnvironmentSettings +from common.src.es_connect import IndexPaginate def ignore_filelist(filelist: list[str]) -> list[str]: diff --git a/tubearchivist/home/src/index/generic.py b/tubearchivist/common/src/index_generic.py similarity index 98% rename from tubearchivist/home/src/index/generic.py rename to tubearchivist/common/src/index_generic.py index 0febec5e..3daeefe7 100644 --- a/tubearchivist/home/src/index/generic.py +++ b/tubearchivist/common/src/index_generic.py @@ -5,9 +5,9 @@ functionality: import math +from appsettings.src.config import AppConfig +from common.src.es_connect import ElasticWrap from download.src.yt_dlp_base import YtWrap -from home.src.es.connect import ElasticWrap -from home.src.ta.config import AppConfig from user.src.user_config import UserConfig diff --git a/tubearchivist/api/src/search_processor.py b/tubearchivist/common/src/search_processor.py similarity index 98% rename from tubearchivist/api/src/search_processor.py rename to tubearchivist/common/src/search_processor.py index 13015d78..8a4b98fb 100644 --- a/tubearchivist/api/src/search_processor.py +++ b/tubearchivist/common/src/search_processor.py @@ -6,9 +6,9 @@ Functionality: import urllib.parse +from common.src.env_settings import EnvironmentSettings +from common.src.helper import date_parser, get_duration_str from download.src.thumbnails import ThumbManager -from home.src.ta.helper import date_parser, get_duration_str -from home.src.ta.settings import EnvironmentSettings class SearchProcess: diff --git a/tubearchivist/home/src/frontend/searching.py b/tubearchivist/common/src/searching.py similarity index 99% rename from tubearchivist/home/src/frontend/searching.py rename to tubearchivist/common/src/searching.py index 932aa3b9..17c164b1 100644 --- a/tubearchivist/home/src/frontend/searching.py +++ b/tubearchivist/common/src/searching.py @@ -6,8 +6,8 @@ Functionality: - calculate pagination values """ -from api.src.search_processor import SearchProcess -from home.src.es.connect import ElasticWrap +from common.src.es_connect import ElasticWrap +from common.src.search_processor import SearchProcess class SearchForm: diff --git a/tubearchivist/home/src/ta/ta_redis.py b/tubearchivist/common/src/ta_redis.py similarity index 99% rename from tubearchivist/home/src/ta/ta_redis.py rename to tubearchivist/common/src/ta_redis.py index 1feb0232..fbb3a628 100644 --- a/tubearchivist/home/src/ta/ta_redis.py +++ b/tubearchivist/common/src/ta_redis.py @@ -8,7 +8,7 @@ functionality: import json import redis -from home.src.ta.settings import EnvironmentSettings +from common.src.env_settings import EnvironmentSettings class RedisBase: diff --git a/tubearchivist/home/src/ta/urlparser.py b/tubearchivist/common/src/urlparser.py similarity index 100% rename from tubearchivist/home/src/ta/urlparser.py rename to tubearchivist/common/src/urlparser.py diff --git a/tubearchivist/home/src/frontend/watched.py b/tubearchivist/common/src/watched.py similarity index 97% rename from tubearchivist/home/src/frontend/watched.py rename to tubearchivist/common/src/watched.py index ceb4870e..ad5a671a 100644 --- a/tubearchivist/home/src/frontend/watched.py +++ b/tubearchivist/common/src/watched.py @@ -5,8 +5,8 @@ functionality: from datetime import datetime -from home.src.es.connect import ElasticWrap -from home.src.ta.urlparser import Parser +from common.src.es_connect import ElasticWrap +from common.src.urlparser import Parser class WatchState: diff --git a/tubearchivist/api/urls.py b/tubearchivist/common/urls.py similarity index 95% rename from tubearchivist/api/urls.py rename to tubearchivist/common/urls.py index daba4004..e8a9fe94 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/common/urls.py @@ -1,6 +1,6 @@ """all api urls""" -from api import views +from common import views from django.urls import path urlpatterns = [ diff --git a/tubearchivist/common/views.py b/tubearchivist/common/views.py new file mode 100644 index 00000000..51fb0941 --- /dev/null +++ b/tubearchivist/common/views.py @@ -0,0 +1,115 @@ +"""all API views""" + +from appsettings.src.config import ReleaseVersion +from appsettings.src.reindex import ReindexProgress +from common.src.searching import SearchForm +from common.src.ta_redis import RedisArchivist +from common.src.watched import WatchState +from common.views_base import AdminOnly, ApiBaseView +from rest_framework.response import Response +from task.tasks import check_reindex + + +class PingView(ApiBaseView): + """resolves to /api/ping/ + GET: test your connection + """ + + @staticmethod + def get(request): + """get pong""" + data = { + "response": "pong", + "user": request.user.id, + "version": ReleaseVersion().get_local_version(), + } + return Response(data) + + +class RefreshView(ApiBaseView): + """resolves to /api/refresh/ + GET: get refresh progress + POST: start a manual refresh task + """ + + permission_classes = [AdminOnly] + + def get(self, request): + """handle get request""" + request_type = request.GET.get("type") + request_id = request.GET.get("id") + + if request_id and not request_type: + return Response({"status": "Bad Request"}, status=400) + + try: + progress = ReindexProgress( + request_type=request_type, request_id=request_id + ).get_progress() + except ValueError: + return Response({"status": "Bad Request"}, status=400) + + return Response(progress) + + def post(self, request): + """handle post request""" + data = request.data + extract_videos = bool(request.GET.get("extract_videos", False)) + check_reindex.delay(data=data, extract_videos=extract_videos) + + return Response(data) + + +class WatchedView(ApiBaseView): + """resolves to /api/watched/ + POST: change watched state of video, channel or playlist + """ + + def post(self, request): + """change watched state""" + youtube_id = request.data.get("id") + is_watched = request.data.get("is_watched") + + if not youtube_id or is_watched is None: + message = {"message": "missing id or is_watched"} + return Response(message, status=400) + + WatchState(youtube_id, is_watched).change() + return Response({"message": "success"}, status=200) + + +class SearchView(ApiBaseView): + """resolves to /api/search/ + GET: run a search with the string in the ?query parameter + """ + + @staticmethod + def get(request): + """handle get request + search through all indexes""" + search_query = request.GET.get("query", None) + if search_query is None: + return Response( + {"message": "no search query specified"}, status=400 + ) + + search_results = SearchForm().multi_search(search_query) + return Response(search_results) + + +class NotificationView(ApiBaseView): + """resolves to /api/notification/ + GET: returns a list of notifications + filter query to filter messages by group + """ + + valid_filters = ["download", "settings", "channel"] + + def get(self, request): + """get all notifications""" + query = "message" + filter_by = request.GET.get("filter", None) + if filter_by in self.valid_filters: + query = f"{query}:{filter_by}" + + return Response(RedisArchivist().list_items(query)) diff --git a/tubearchivist/common/views_base.py b/tubearchivist/common/views_base.py new file mode 100644 index 00000000..fe1b7965 --- /dev/null +++ b/tubearchivist/common/views_base.py @@ -0,0 +1,98 @@ +"""base classes to inherit from""" + +from appsettings.src.config import AppConfig +from common.src.env_settings import EnvironmentSettings +from common.src.es_connect import ElasticWrap +from common.src.index_generic import Pagination +from common.src.search_processor import SearchProcess +from rest_framework import permissions +from rest_framework.authentication import ( + SessionAuthentication, + TokenAuthentication, +) +from rest_framework.views import APIView + + +def check_admin(user): + """check for admin permission for restricted views""" + return user.is_staff or user.groups.filter(name="admin").exists() + + +class AdminOnly(permissions.BasePermission): + """allow only admin""" + + def has_permission(self, request, view): + return check_admin(request.user) + + +class AdminWriteOnly(permissions.BasePermission): + """allow only admin writes""" + + def has_permission(self, request, view): + if request.method in permissions.SAFE_METHODS: + return permissions.IsAuthenticated().has_permission(request, view) + + return check_admin(request.user) + + +class ApiBaseView(APIView): + """base view to inherit from""" + + authentication_classes = [SessionAuthentication, TokenAuthentication] + permission_classes = [permissions.IsAuthenticated] + search_base = "" + data = "" + + def __init__(self): + super().__init__() + self.response = { + "data": False, + "config": { + "enable_cast": EnvironmentSettings.ENABLE_CAST, + "downloads": AppConfig().config["downloads"], + }, + } + self.data = {"query": {"match_all": {}}} + self.status_code = False + self.context = False + self.pagination_handler = False + + def get_document(self, document_id): + """get single document from es""" + path = f"{self.search_base}{document_id}" + response, status_code = ElasticWrap(path).get() + try: + self.response["data"] = SearchProcess(response).process() + except KeyError: + print(f"item not found: {document_id}") + self.response["data"] = False + self.status_code = status_code + + def initiate_pagination(self, request): + """set initial pagination values""" + self.pagination_handler = Pagination(request) + self.data.update( + { + "size": self.pagination_handler.pagination["page_size"], + "from": self.pagination_handler.pagination["page_from"], + } + ) + + def get_document_list(self, request, pagination=True): + """get a list of results""" + if pagination: + self.initiate_pagination(request) + + es_handler = ElasticWrap(self.search_base) + response, status_code = es_handler.get(data=self.data) + self.response["data"] = SearchProcess(response).process() + if self.response["data"]: + self.status_code = status_code + else: + self.status_code = 404 + + if pagination: + self.pagination_handler.validate( + response["hits"]["total"]["value"] + ) + self.response["paginate"] = self.pagination_handler.pagination diff --git a/tubearchivist/config/management/commands/ta_connection.py b/tubearchivist/config/management/commands/ta_connection.py index 94b5e0dc..1b8158e6 100644 --- a/tubearchivist/config/management/commands/ta_connection.py +++ b/tubearchivist/config/management/commands/ta_connection.py @@ -6,10 +6,10 @@ Functionality: from time import sleep import requests +from common.src.env_settings import EnvironmentSettings +from common.src.es_connect import ElasticWrap +from common.src.ta_redis import RedisArchivist from django.core.management.base import BaseCommand, CommandError -from home.src.es.connect import ElasticWrap -from home.src.ta.settings import EnvironmentSettings -from home.src.ta.ta_redis import RedisArchivist TOPIC = """ diff --git a/tubearchivist/config/management/commands/ta_envcheck.py b/tubearchivist/config/management/commands/ta_envcheck.py index 1fe56afa..8c3afb4f 100644 --- a/tubearchivist/config/management/commands/ta_envcheck.py +++ b/tubearchivist/config/management/commands/ta_envcheck.py @@ -9,8 +9,8 @@ Functionality: import os import re +from common.src.env_settings import EnvironmentSettings from django.core.management.base import BaseCommand, CommandError -from home.src.ta.settings import EnvironmentSettings from user.models import Account LOGO = """ diff --git a/tubearchivist/config/management/commands/ta_fix_channels.py b/tubearchivist/config/management/commands/ta_fix_channels.py deleted file mode 100644 index d602e668..00000000 --- a/tubearchivist/config/management/commands/ta_fix_channels.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -channel fix for update from v0.4.7 to v0.4.8 -reindex channels with 0 subscriber count -python manage.py ta_fix_channels -""" - -from django.core.management.base import BaseCommand -from home.src.es.connect import IndexPaginate -from home.tasks import check_reindex - - -class Command(BaseCommand): - """fix comment link""" - - def handle(self, *args, **options): - """run command""" - self.stdout.write("reindex failed channels") - channels = self._get_channels() - if not channels: - self.stdout.write("did not find any failed channels") - return - - self.stdout.write(f"add {len(channels)} channels(s) to queue") - to_reindex = {"channel": [i["channel_id"] for i in channels]} - check_reindex.delay(data=to_reindex) - self.stdout.write(self.style.SUCCESS(" ✓ task queued\n")) - - def _get_channels(self): - """get failed channels""" - self.stdout.write("search for failed channels") - es_query = { - "query": { - "bool": { - "must": [ - {"term": {"channel_subs": {"value": 0}}}, - {"term": {"channel_active": {"value": True}}}, - ] - }, - }, - "_source": ["channel_id"], - } - channels = IndexPaginate("ta_channel", es_query).get_results() - - return channels diff --git a/tubearchivist/config/management/commands/ta_fix_comment_link.py b/tubearchivist/config/management/commands/ta_fix_comment_link.py deleted file mode 100644 index 55ce69d3..00000000 --- a/tubearchivist/config/management/commands/ta_fix_comment_link.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -comment link fix for update from v0.4.7 to v0.4.8 -scan your videos and comments to fix comment_count field -python manage.py ta_fix_comment_link -""" - -from django.core.management.base import BaseCommand, CommandError -from home.src.es.connect import ElasticWrap, IndexPaginate - - -class Command(BaseCommand): - """fix comment link""" - - def handle(self, *args, **options): - """run command""" - self.stdout.write("run comment link fix") - expected_count = self._get_comment_indexed() - all_videos = self._get_videos() - - self.stdout.write(f"checking {len(all_videos)} video(s)") - videos_updated = [] - for video in all_videos: - video_id = video["youtube_id"] - comment_count = expected_count.get(video_id) - if not comment_count: - continue - - data = {"doc": {"comment_count": comment_count}} - path = f"ta_video/_update/{video_id}" - response, status_code = ElasticWrap(path).post(data=data) - - if status_code != 200: - message = ( - "failed to add comment count to video" - + f"response code: {status_code}" - + response - ) - raise CommandError(message) - - videos_updated.append(video_id) - - self.stdout.write(f"fixed {len(videos_updated)} video(s)") - self.stdout.write(self.style.SUCCESS(" ✓ task completed\n")) - - def _get_comment_indexed(self): - """get comment count by index""" - self.stdout.write("get comments") - src = "params['_source']['comment_comments'].length" - data = { - "script_fields": { - "comments_length": { - "script": {"source": src, "lang": "painless"} - } - } - } - all_comments = IndexPaginate( - "ta_comment", data=data, keep_source=True - ).get_results() - - expected_count = { - i["_id"]: i["fields"]["comments_length"][0] for i in all_comments - } - - return expected_count - - def _get_videos(self): - """get videos without comment_count""" - self.stdout.write("get videos") - data = { - "query": { - "bool": {"must_not": [{"exists": {"field": "comment_count"}}]} - } - } - all_videos = IndexPaginate("ta_video", data).get_results() - - return all_videos diff --git a/tubearchivist/config/management/commands/ta_migpath.py b/tubearchivist/config/management/commands/ta_migpath.py deleted file mode 100644 index 28c9546a..00000000 --- a/tubearchivist/config/management/commands/ta_migpath.py +++ /dev/null @@ -1,185 +0,0 @@ -""" -filepath migration from v0.3.6 to v0.3.7 -not getting called at startup any more, to run manually if needed: -python manage.py ta_migpath -""" - -import json -import os -import shutil - -from django.core.management.base import BaseCommand -from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.ta.helper import ignore_filelist -from home.src.ta.settings import EnvironmentSettings - -TOPIC = """ - -######################## -# Filesystem Migration # -######################## - -""" - - -class Command(BaseCommand): - """command framework""" - - # pylint: disable=no-member - - def handle(self, *args, **options): - """run commands""" - self.stdout.write(TOPIC) - - handler = FolderMigration() - to_migrate = handler.get_to_migrate() - if not to_migrate: - self.stdout.write( - self.style.SUCCESS(" no channel migration needed\n") - ) - return - - self.stdout.write(self.style.SUCCESS(" migrating channels")) - total_channels = handler.create_folders(to_migrate) - self.stdout.write( - self.style.SUCCESS(f" created {total_channels} channels") - ) - - self.stdout.write( - self.style.SUCCESS(f" migrating {len(to_migrate)} videos") - ) - handler.migrate_videos(to_migrate) - self.stdout.write(self.style.SUCCESS(" update videos in index")) - handler.send_bulk() - - self.stdout.write(self.style.SUCCESS(" cleanup old folders")) - handler.delete_old() - - self.stdout.write(self.style.SUCCESS(" ✓ migration completed\n")) - - -class FolderMigration: - """migrate video archive folder""" - - def __init__(self): - self.videos = EnvironmentSettings.MEDIA_DIR - self.bulk_list = [] - - def get_to_migrate(self): - """get videos to migrate""" - script = ( - "doc['media_url'].value == " - + "doc['channel.channel_id'].value + '/'" - + " + doc['youtube_id'].value + '.mp4'" - ) - data = { - "query": {"bool": {"must_not": [{"script": {"script": script}}]}}, - "_source": [ - "youtube_id", - "media_url", - "channel.channel_id", - "subtitles", - ], - } - response = IndexPaginate("ta_video", data).get_results() - - return response - - def create_folders(self, to_migrate): - """create required channel folders""" - host_uid = EnvironmentSettings.HOST_UID - host_gid = EnvironmentSettings.HOST_GID - all_channel_ids = {i["channel"]["channel_id"] for i in to_migrate} - - for channel_id in all_channel_ids: - new_folder = os.path.join(self.videos, channel_id) - os.makedirs(new_folder, exist_ok=True) - if host_uid and host_gid: - os.chown(new_folder, host_uid, host_gid) - - return len(all_channel_ids) - - def migrate_videos(self, to_migrate): - """migrate all videos of channel""" - total = len(to_migrate) - for idx, video in enumerate(to_migrate): - new_media_url = self._move_video_file(video) - if not new_media_url: - continue - - all_subtitles = self._move_subtitles(video) - action = { - "update": {"_id": video["youtube_id"], "_index": "ta_video"} - } - source = {"doc": {"media_url": new_media_url}} - if all_subtitles: - source["doc"].update({"subtitles": all_subtitles}) - - self.bulk_list.append(json.dumps(action)) - self.bulk_list.append(json.dumps(source)) - if idx % 1000 == 0: - print(f"processing migration [{idx}/{total}]") - self.send_bulk() - - def _move_video_file(self, video): - """move video file to new location""" - old_path = os.path.join(self.videos, video["media_url"]) - if not os.path.exists(old_path): - print(f"did not find expected video at {old_path}") - return False - - new_media_url = os.path.join( - video["channel"]["channel_id"], video["youtube_id"] + ".mp4" - ) - new_path = os.path.join(self.videos, new_media_url) - os.rename(old_path, new_path) - - return new_media_url - - def _move_subtitles(self, video): - """move subtitle files to new location""" - all_subtitles = video.get("subtitles") - if not all_subtitles: - return False - - for subtitle in all_subtitles: - old_path = os.path.join(self.videos, subtitle["media_url"]) - if not os.path.exists(old_path): - print(f"did not find expected subtitle at {old_path}") - continue - - new_media_url = os.path.join( - video["channel"]["channel_id"], - f"{video.get('youtube_id')}.{subtitle.get('lang')}.vtt", - ) - new_path = os.path.join(self.videos, new_media_url) - os.rename(old_path, new_path) - subtitle["media_url"] = new_media_url - - return all_subtitles - - def send_bulk(self): - """send bulk request to update index with new urls""" - if not self.bulk_list: - print("nothing to update") - return - - self.bulk_list.append("\n") - path = "_bulk?refresh=true" - data = "\n".join(self.bulk_list) - response, status = ElasticWrap(path).post(data=data, ndjson=True) - if not status == 200: - print(response) - - self.bulk_list = [] - - def delete_old(self): - """delete old empty folders""" - all_folders = ignore_filelist(os.listdir(self.videos)) - for folder in all_folders: - folder_path = os.path.join(self.videos, folder) - if not os.path.isdir(folder_path): - continue - - if not ignore_filelist(os.listdir(folder_path)): - shutil.rmtree(folder_path) diff --git a/tubearchivist/config/management/commands/ta_startup.py b/tubearchivist/config/management/commands/ta_startup.py index 41cf4f94..f7e93796 100644 --- a/tubearchivist/config/management/commands/ta_startup.py +++ b/tubearchivist/config/management/commands/ta_startup.py @@ -9,17 +9,17 @@ from datetime import datetime from random import randint from time import sleep +from appsettings.src.config import AppConfig, ReleaseVersion from appsettings.src.index_setup import ElasitIndexWrap from appsettings.src.snapshot import ElasticSnapshot +from common.src.env_settings import EnvironmentSettings +from common.src.es_connect import ElasticWrap +from common.src.helper import clear_dl_cache +from common.src.ta_redis import RedisArchivist from django.conf import settings from django.core.management.base import BaseCommand, CommandError from django.utils import dateformat from django_celery_beat.models import CrontabSchedule, PeriodicTasks -from home.src.es.connect import ElasticWrap -from home.src.ta.config import AppConfig, ReleaseVersion -from home.src.ta.helper import clear_dl_cache -from home.src.ta.settings import EnvironmentSettings -from home.src.ta.ta_redis import RedisArchivist from task.models import CustomPeriodicTask from task.src.config_schedule import ScheduleBuilder from task.src.notify import Notifications diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index a8605744..09284cb5 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -15,10 +15,10 @@ from os import environ, path from pathlib import Path import ldap +from common.src.env_settings import EnvironmentSettings +from common.src.helper import ta_host_parser from corsheaders.defaults import default_headers from django_auth_ldap.config import LDAPSearch -from home.src.ta.helper import ta_host_parser -from home.src.ta.settings import EnvironmentSettings try: from dotenv import load_dotenv @@ -61,7 +61,7 @@ INSTALLED_APPS = [ "django.contrib.humanize", "rest_framework", "rest_framework.authtoken", - "api", + "common", "video", "channel", "playlist", @@ -83,7 +83,7 @@ MIDDLEWARE = [ "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", - "home.src.ta.health.HealthCheckMiddleware", + "common.src.health.HealthCheckMiddleware", ] ROOT_URLCONF = "config.urls" @@ -236,7 +236,7 @@ if bool(environ.get("TA_ENABLE_AUTH_PROXY")): ) TA_AUTH_PROXY_LOGOUT_URL = environ.get("TA_AUTH_PROXY_LOGOUT_URL") - MIDDLEWARE.append("home.src.ta.auth.HttpRemoteUserMiddleware") + MIDDLEWARE.append("user.src.remote_user_auth.HttpRemoteUserMiddleware") AUTHENTICATION_BACKENDS = ( "django.contrib.auth.backends.RemoteUserBackend", diff --git a/tubearchivist/config/urls.py b/tubearchivist/config/urls.py index 2aa26fc1..8477a794 100644 --- a/tubearchivist/config/urls.py +++ b/tubearchivist/config/urls.py @@ -19,7 +19,7 @@ from django.urls import include, path urlpatterns = [ path("", include("home.urls")), - path("api/", include("api.urls")), + path("api/", include("common.urls")), path("api/video/", include("video.urls")), path("api/channel/", include("channel.urls")), path("api/playlist/", include("playlist.urls")), diff --git a/tubearchivist/download/src/queue.py b/tubearchivist/download/src/queue.py index 449276a4..472da36f 100644 --- a/tubearchivist/download/src/queue.py +++ b/tubearchivist/download/src/queue.py @@ -7,12 +7,12 @@ Functionality: import json from datetime import datetime +from appsettings.src.config import AppConfig +from common.src.es_connect import ElasticWrap, IndexPaginate +from common.src.helper import get_duration_str, is_shorts from download.src.subscriptions import ChannelSubscription from download.src.thumbnails import ThumbManager from download.src.yt_dlp_base import YtWrap -from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.ta.config import AppConfig -from home.src.ta.helper import get_duration_str, is_shorts from playlist.src.index import YoutubePlaylist from video.src.constants import VideoTypeEnum diff --git a/tubearchivist/download/src/subscriptions.py b/tubearchivist/download/src/subscriptions.py index a7ce224d..833746ff 100644 --- a/tubearchivist/download/src/subscriptions.py +++ b/tubearchivist/download/src/subscriptions.py @@ -4,13 +4,13 @@ Functionality: - handle playlist subscriptions """ +from appsettings.src.config import AppConfig from channel.src.index import YoutubeChannel +from common.src.es_connect import IndexPaginate +from common.src.helper import is_missing +from common.src.urlparser import Parser from download.src.thumbnails import ThumbManager from download.src.yt_dlp_base import YtWrap -from home.src.es.connect import IndexPaginate -from home.src.ta.config import AppConfig -from home.src.ta.helper import is_missing -from home.src.ta.urlparser import Parser from playlist.src.index import YoutubePlaylist from video.src.constants import VideoTypeEnum from video.src.index import YoutubeVideo diff --git a/tubearchivist/download/src/thumbnails.py b/tubearchivist/download/src/thumbnails.py index cf4c485f..50af3887 100644 --- a/tubearchivist/download/src/thumbnails.py +++ b/tubearchivist/download/src/thumbnails.py @@ -10,9 +10,9 @@ from io import BytesIO from time import sleep import requests -from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.ta.helper import is_missing -from home.src.ta.settings import EnvironmentSettings +from common.src.env_settings import EnvironmentSettings +from common.src.es_connect import ElasticWrap, IndexPaginate +from common.src.helper import is_missing from mutagen.mp4 import MP4, MP4Cover from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError diff --git a/tubearchivist/download/src/yt_dlp_base.py b/tubearchivist/download/src/yt_dlp_base.py index d95a2972..08013ff4 100644 --- a/tubearchivist/download/src/yt_dlp_base.py +++ b/tubearchivist/download/src/yt_dlp_base.py @@ -10,8 +10,8 @@ from http import cookiejar from io import StringIO import yt_dlp -from home.src.ta.settings import EnvironmentSettings -from home.src.ta.ta_redis import RedisArchivist +from common.src.env_settings import EnvironmentSettings +from common.src.ta_redis import RedisArchivist class YtWrap: diff --git a/tubearchivist/download/src/yt_dlp_handler.py b/tubearchivist/download/src/yt_dlp_handler.py index 10db58ee..3731895b 100644 --- a/tubearchivist/download/src/yt_dlp_handler.py +++ b/tubearchivist/download/src/yt_dlp_handler.py @@ -10,15 +10,15 @@ import os import shutil from datetime import datetime +from appsettings.src.config import AppConfig from channel.src.index import YoutubeChannel +from common.src.env_settings import EnvironmentSettings +from common.src.es_connect import ElasticWrap, IndexPaginate +from common.src.helper import get_channel_overwrites, ignore_filelist +from common.src.ta_redis import RedisQueue from download.src.queue import PendingList from download.src.subscriptions import PlaylistSubscription from download.src.yt_dlp_base import YtWrap -from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.ta.config import AppConfig -from home.src.ta.helper import get_channel_overwrites, ignore_filelist -from home.src.ta.settings import EnvironmentSettings -from home.src.ta.ta_redis import RedisQueue from playlist.src.index import YoutubePlaylist from video.src.comments import CommentList from video.src.constants import VideoTypeEnum diff --git a/tubearchivist/download/views.py b/tubearchivist/download/views.py index 810559dc..5db809fa 100644 --- a/tubearchivist/download/views.py +++ b/tubearchivist/download/views.py @@ -1,6 +1,6 @@ """all download API views""" -from api.views import AdminOnly, ApiBaseView +from common.views_base import AdminOnly, ApiBaseView from download.src.queue import PendingInteract from rest_framework.response import Response from task.tasks import download_pending, extrac_dl diff --git a/tubearchivist/home/celery.py b/tubearchivist/home/celery.py index 1f3369f8..28cdbf72 100644 --- a/tubearchivist/home/celery.py +++ b/tubearchivist/home/celery.py @@ -3,7 +3,7 @@ import os from celery import Celery -from home.src.ta.settings import EnvironmentSettings +from common.src.env_settings import EnvironmentSettings REDIS_HOST = EnvironmentSettings.REDIS_HOST REDIS_PORT = EnvironmentSettings.REDIS_PORT diff --git a/tubearchivist/home/src/download/__init__.py b/tubearchivist/home/src/download/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tubearchivist/home/src/es/__init__.py b/tubearchivist/home/src/es/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tubearchivist/home/src/frontend/forms.py b/tubearchivist/home/src/frontend/forms.py index 5286a932..903f6a7f 100644 --- a/tubearchivist/home/src/frontend/forms.py +++ b/tubearchivist/home/src/frontend/forms.py @@ -4,10 +4,10 @@ import os +from common.src.helper import get_stylesheets from django import forms from django.contrib.auth.forms import AuthenticationForm from django.forms.widgets import PasswordInput, TextInput -from home.src.ta.helper import get_stylesheets class CustomAuthForm(AuthenticationForm): diff --git a/tubearchivist/home/src/index/__init__.py b/tubearchivist/home/src/index/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tubearchivist/home/src/ta/__init__.py b/tubearchivist/home/src/ta/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index a3509324..d032714c 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -9,12 +9,18 @@ import urllib.parse import uuid from time import sleep -from api.src.search_processor import SearchProcess, process_aggs -from api.views import check_admin from appsettings.src.backup import ElasticBackup +from appsettings.src.config import AppConfig, ReleaseVersion from appsettings.src.reindex import ReindexProgress from appsettings.src.snapshot import ElasticSnapshot from channel.src.index import channel_overwrites +from common.src.env_settings import EnvironmentSettings +from common.src.es_connect import ElasticWrap +from common.src.helper import check_stylesheet, time_parser +from common.src.index_generic import Pagination +from common.src.search_processor import SearchProcess, process_aggs +from common.src.ta_redis import RedisArchivist +from common.views_base import check_admin from django.conf import settings from django.contrib.auth import login from django.contrib.auth.decorators import user_passes_test @@ -25,7 +31,6 @@ from django.utils.decorators import method_decorator from django.views import View from download.src.queue import PendingInteract from download.src.yt_dlp_base import CookieHandler -from home.src.es.connect import ElasticWrap from home.src.frontend.forms import ( AddToQueueForm, ApplicationSettingsForm, @@ -41,11 +46,6 @@ from home.src.frontend.forms_schedule import ( NotificationSettingsForm, SchedulerSettingsForm, ) -from home.src.index.generic import Pagination -from home.src.ta.config import AppConfig, ReleaseVersion -from home.src.ta.helper import check_stylesheet, time_parser -from home.src.ta.settings import EnvironmentSettings -from home.src.ta.ta_redis import RedisArchivist from playlist.src.index import YoutubePlaylist from rest_framework.authtoken.models import Token from task.models import CustomPeriodicTask diff --git a/tubearchivist/playlist/src/index.py b/tubearchivist/playlist/src/index.py index d7c6cce2..26a42973 100644 --- a/tubearchivist/playlist/src/index.py +++ b/tubearchivist/playlist/src/index.py @@ -8,9 +8,9 @@ import json from datetime import datetime from channel.src import index as channel +from common.src.es_connect import ElasticWrap, IndexPaginate +from common.src.index_generic import YouTubeItem from download.src.thumbnails import ThumbManager -from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.index.generic import YouTubeItem from video.src import index as ta_video diff --git a/tubearchivist/playlist/views.py b/tubearchivist/playlist/views.py index a676f7d2..a712f9ab 100644 --- a/tubearchivist/playlist/views.py +++ b/tubearchivist/playlist/views.py @@ -1,6 +1,6 @@ """all playlist API views""" -from api.views import AdminWriteOnly, ApiBaseView +from common.views_base import AdminWriteOnly, ApiBaseView from download.src.subscriptions import PlaylistSubscription from playlist.src.index import YoutubePlaylist from rest_framework import status diff --git a/tubearchivist/stats/src/aggs.py b/tubearchivist/stats/src/aggs.py index e2c65dbd..81f7cbb8 100644 --- a/tubearchivist/stats/src/aggs.py +++ b/tubearchivist/stats/src/aggs.py @@ -1,8 +1,8 @@ """aggregations""" -from home.src.es.connect import ElasticWrap -from home.src.ta.helper import get_duration_str -from home.src.ta.settings import EnvironmentSettings +from common.src.env_settings import EnvironmentSettings +from common.src.es_connect import ElasticWrap +from common.src.helper import get_duration_str class AggBase: diff --git a/tubearchivist/stats/views.py b/tubearchivist/stats/views.py index 49af7baa..c5d83678 100644 --- a/tubearchivist/stats/views.py +++ b/tubearchivist/stats/views.py @@ -1,6 +1,6 @@ """all stats API views""" -from api.views import ApiBaseView +from common.views_base import ApiBaseView from rest_framework.response import Response from stats.src.aggs import ( BiggestChannel, diff --git a/tubearchivist/task/src/config_schedule.py b/tubearchivist/task/src/config_schedule.py index 03be9d0a..9c3adfcd 100644 --- a/tubearchivist/task/src/config_schedule.py +++ b/tubearchivist/task/src/config_schedule.py @@ -5,10 +5,10 @@ Functionality: from datetime import datetime +from appsettings.src.config import AppConfig +from common.src.env_settings import EnvironmentSettings from django.utils import dateformat from django_celery_beat.models import CrontabSchedule -from home.src.ta.config import AppConfig -from home.src.ta.settings import EnvironmentSettings from task.models import CustomPeriodicTask from task.src.task_config import TASK_CONFIG diff --git a/tubearchivist/task/src/notify.py b/tubearchivist/task/src/notify.py index ea995713..3a9b1f23 100644 --- a/tubearchivist/task/src/notify.py +++ b/tubearchivist/task/src/notify.py @@ -1,7 +1,7 @@ """send notifications using apprise""" import apprise -from home.src.es.connect import ElasticWrap +from common.src.es_connect import ElasticWrap from task.src.task_config import TASK_CONFIG from task.src.task_manager import TaskManager diff --git a/tubearchivist/task/src/task_manager.py b/tubearchivist/task/src/task_manager.py index 8684c823..a63f384a 100644 --- a/tubearchivist/task/src/task_manager.py +++ b/tubearchivist/task/src/task_manager.py @@ -4,8 +4,8 @@ functionality: - handle threads and locks """ +from common.src.ta_redis import RedisArchivist, TaskRedis from home.celery import app as celery_app -from home.src.ta.ta_redis import RedisArchivist, TaskRedis from task.src.task_config import TASK_CONFIG diff --git a/tubearchivist/task/tasks.py b/tubearchivist/task/tasks.py index 40cb2e04..ff3f0278 100644 --- a/tubearchivist/task/tasks.py +++ b/tubearchivist/task/tasks.py @@ -7,6 +7,7 @@ Functionality: """ from appsettings.src.backup import ElasticBackup +from appsettings.src.config import ReleaseVersion from appsettings.src.filesystem import Scanner from appsettings.src.index_setup import ElasitIndexWrap from appsettings.src.manual import ImportFolderScanner @@ -14,13 +15,12 @@ from appsettings.src.reindex import Reindex, ReindexManual, ReindexPopulate from celery import Task, shared_task from celery.exceptions import Retry from channel.src.index import YoutubeChannel +from common.src.ta_redis import RedisArchivist +from common.src.urlparser import Parser from download.src.queue import PendingList from download.src.subscriptions import SubscriptionHandler, SubscriptionScanner from download.src.thumbnails import ThumbFilesystem, ThumbValidator from download.src.yt_dlp_handler import VideoDownloader -from home.src.ta.config import ReleaseVersion -from home.src.ta.ta_redis import RedisArchivist -from home.src.ta.urlparser import Parser from task.src.notify import Notifications from task.src.task_config import TASK_CONFIG from task.src.task_manager import TaskManager diff --git a/tubearchivist/task/views.py b/tubearchivist/task/views.py index fc117ac0..9aeb6bdb 100644 --- a/tubearchivist/task/views.py +++ b/tubearchivist/task/views.py @@ -1,6 +1,6 @@ """all task API views""" -from api.views import AdminOnly, ApiBaseView +from common.views_base import AdminOnly, ApiBaseView from rest_framework.response import Response from task.models import CustomPeriodicTask from task.src.notify import Notifications, get_all_notifications diff --git a/tubearchivist/home/src/ta/auth.py b/tubearchivist/user/src/remote_user_auth.py similarity index 100% rename from tubearchivist/home/src/ta/auth.py rename to tubearchivist/user/src/remote_user_auth.py diff --git a/tubearchivist/user/src/user_config.py b/tubearchivist/user/src/user_config.py index 7181b3fe..39790ad7 100644 --- a/tubearchivist/user/src/user_config.py +++ b/tubearchivist/user/src/user_config.py @@ -6,8 +6,8 @@ Functionality: from typing import TypedDict -from home.src.es.connect import ElasticWrap -from home.src.ta.helper import get_stylesheets +from common.src.es_connect import ElasticWrap +from common.src.helper import get_stylesheets class UserConfigType(TypedDict, total=False): diff --git a/tubearchivist/user/views.py b/tubearchivist/user/views.py index 8fe58e81..4d0c3140 100644 --- a/tubearchivist/user/views.py +++ b/tubearchivist/user/views.py @@ -1,6 +1,6 @@ """all user api views""" -from api.views import ApiBaseView +from common.views import ApiBaseView from rest_framework.authtoken.models import Token from rest_framework.authtoken.views import ObtainAuthToken from rest_framework.response import Response diff --git a/tubearchivist/video/src/comments.py b/tubearchivist/video/src/comments.py index 6723e9af..109c0490 100644 --- a/tubearchivist/video/src/comments.py +++ b/tubearchivist/video/src/comments.py @@ -7,10 +7,10 @@ Functionality: from datetime import datetime +from appsettings.src.config import AppConfig +from common.src.es_connect import ElasticWrap +from common.src.ta_redis import RedisQueue from download.src.yt_dlp_base import YtWrap -from home.src.es.connect import ElasticWrap -from home.src.ta.config import AppConfig -from home.src.ta.ta_redis import RedisQueue class Comments: diff --git a/tubearchivist/video/src/index.py b/tubearchivist/video/src/index.py index cea69cce..ff90b142 100644 --- a/tubearchivist/video/src/index.py +++ b/tubearchivist/video/src/index.py @@ -9,11 +9,11 @@ from datetime import datetime import requests from channel.src import index as ta_channel +from common.src.env_settings import EnvironmentSettings +from common.src.es_connect import ElasticWrap +from common.src.helper import get_duration_sec, get_duration_str, randomizor +from common.src.index_generic import YouTubeItem from django.conf import settings -from home.src.es.connect import ElasticWrap -from home.src.index.generic import YouTubeItem -from home.src.ta.helper import get_duration_sec, get_duration_str, randomizor -from home.src.ta.settings import EnvironmentSettings from playlist.src import index as ta_playlist from ryd_client import ryd_client from user.src.user_config import UserConfig diff --git a/tubearchivist/video/src/subtitle.py b/tubearchivist/video/src/subtitle.py index 56973519..38df1783 100644 --- a/tubearchivist/video/src/subtitle.py +++ b/tubearchivist/video/src/subtitle.py @@ -10,9 +10,9 @@ import os from datetime import datetime import requests -from home.src.es.connect import ElasticWrap -from home.src.ta.helper import requests_headers -from home.src.ta.settings import EnvironmentSettings +from common.src.env_settings import EnvironmentSettings +from common.src.es_connect import ElasticWrap +from common.src.helper import requests_headers class YoutubeSubtitle: diff --git a/tubearchivist/video/views.py b/tubearchivist/video/views.py index 68725290..bff85579 100644 --- a/tubearchivist/video/views.py +++ b/tubearchivist/video/views.py @@ -1,7 +1,7 @@ """all API views for video endpoints""" -from api.views import AdminWriteOnly, ApiBaseView -from home.src.ta.ta_redis import RedisArchivist +from common.src.ta_redis import RedisArchivist +from common.views_base import AdminWriteOnly, ApiBaseView from rest_framework.response import Response from video.src.index import YoutubeVideo