From 528bc5f2efb49c609409c6c44819c8dfd10a5d59 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 18 Jul 2024 22:37:11 +0200 Subject: [PATCH] split to channel app --- tubearchivist/api/urls.py | 20 --- tubearchivist/api/views.py | 152 +---------------- tubearchivist/channel/__init__.py | 0 tubearchivist/channel/migrations/__init__.py | 0 tubearchivist/channel/src/__init__.py | 0 .../index/channel.py => channel/src/index.py} | 0 tubearchivist/channel/urls.py | 27 ++++ tubearchivist/channel/views.py | 153 ++++++++++++++++++ tubearchivist/config/settings.py | 1 + tubearchivist/config/urls.py | 1 + .../home/src/download/subscriptions.py | 2 +- .../home/src/download/yt_dlp_handler.py | 2 +- tubearchivist/home/src/index/playlist.py | 2 +- tubearchivist/home/src/index/reindex.py | 2 +- tubearchivist/home/tasks.py | 2 +- tubearchivist/home/views.py | 2 +- tubearchivist/video/src/index.py | 2 +- 17 files changed, 190 insertions(+), 178 deletions(-) create mode 100644 tubearchivist/channel/__init__.py create mode 100644 tubearchivist/channel/migrations/__init__.py create mode 100644 tubearchivist/channel/src/__init__.py rename tubearchivist/{home/src/index/channel.py => channel/src/index.py} (100%) create mode 100644 tubearchivist/channel/urls.py create mode 100644 tubearchivist/channel/views.py diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py index 4ef05e61..0fe2e635 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/api/urls.py @@ -6,26 +6,6 @@ from django.urls import path urlpatterns = [ path("ping/", views.PingView.as_view(), name="ping"), path("login/", views.LoginApiView.as_view(), name="api-login"), - path( - "channel/", - views.ChannelApiListView.as_view(), - name="api-channel-list", - ), - path( - "channel/search/", - views.ChannelApiSearchView.as_view(), - name="api-channel-search", - ), - path( - "channel//", - views.ChannelApiView.as_view(), - name="api-channel", - ), - path( - "channel//video/", - views.ChannelApiVideoView.as_view(), - name="api-channel-video", - ), path( "playlist/", views.PlaylistApiListView.as_view(), diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 2f205fe5..58c9daa7 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -12,17 +12,13 @@ from api.src.aggs import ( from api.src.search_processor import SearchProcess from home.models import CustomPeriodicTask from home.src.download.queue import PendingInteract -from home.src.download.subscriptions import ( - ChannelSubscription, - PlaylistSubscription, -) +from home.src.download.subscriptions import PlaylistSubscription from home.src.download.yt_dlp_base import CookieHandler from home.src.es.backup import ElasticBackup from home.src.es.connect import ElasticWrap from home.src.es.snapshot import ElasticSnapshot from home.src.frontend.searching import SearchForm from home.src.frontend.watched import WatchState -from home.src.index.channel import YoutubeChannel from home.src.index.generic import Pagination from home.src.index.playlist import YoutubePlaylist from home.src.index.reindex import ReindexProgress @@ -32,7 +28,6 @@ from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist from home.src.ta.task_config import TASK_CONFIG from home.src.ta.task_manager import TaskCommand, TaskManager -from home.src.ta.urlparser import Parser from home.src.ta.users import UserConfig from home.tasks import ( check_reindex, @@ -137,151 +132,6 @@ class ApiBaseView(APIView): self.response["paginate"] = self.pagination_handler.pagination -class ChannelApiView(ApiBaseView): - """resolves to /api/channel// - GET: returns metadata dict of channel - """ - - search_base = "ta_channel/_doc/" - permission_classes = [AdminWriteOnly] - - def get(self, request, channel_id): - # pylint: disable=unused-argument - """get request""" - self.get_document(channel_id) - return Response(self.response, status=self.status_code) - - def delete(self, request, channel_id): - # pylint: disable=unused-argument - """delete channel""" - message = {"channel": channel_id} - try: - YoutubeChannel(channel_id).delete_channel() - status_code = 200 - message.update({"state": "delete"}) - except FileNotFoundError: - status_code = 404 - message.update({"state": "not found"}) - - return Response(message, status=status_code) - - -class ChannelApiListView(ApiBaseView): - """resolves to /api/channel/ - GET: returns list of channels - POST: edit a list of channels - """ - - search_base = "ta_channel/_search/" - valid_filter = ["subscribed"] - permission_classes = [AdminWriteOnly] - - def get(self, request): - """get request""" - self.data.update( - {"sort": [{"channel_name.keyword": {"order": "asc"}}]} - ) - - query_filter = request.GET.get("filter", False) - must_list = [] - if query_filter: - if query_filter not in self.valid_filter: - message = f"invalid url query filter: {query_filter}" - print(message) - return Response({"message": message}, status=400) - - must_list.append({"term": {"channel_subscribed": {"value": True}}}) - - self.data["query"] = {"bool": {"must": must_list}} - self.get_document_list(request) - - return Response(self.response) - - def post(self, request): - """subscribe/unsubscribe to list of channels""" - data = request.data - try: - to_add = data["data"] - except KeyError: - message = "missing expected data key" - print(message) - return Response({"message": message}, status=400) - - pending = [] - for channel_item in to_add: - channel_id = channel_item["channel_id"] - if channel_item["channel_subscribed"]: - pending.append(channel_id) - else: - self._unsubscribe(channel_id) - - if pending: - url_str = " ".join(pending) - subscribe_to.delay(url_str, expected_type="channel") - - return Response(data) - - @staticmethod - def _unsubscribe(channel_id: str): - """unsubscribe""" - print(f"[{channel_id}] unsubscribe from channel") - ChannelSubscription().change_subscribe( - channel_id, channel_subscribed=False - ) - - -class ChannelApiSearchView(ApiBaseView): - """resolves to /api/channel/search/ - search for channel - """ - - search_base = "ta_channel/_doc/" - - def get(self, request): - """handle get request, search with s parameter""" - - query = request.GET.get("q") - if not query: - message = "missing expected q parameter" - return Response({"message": message, "data": False}, status=400) - - try: - parsed = Parser(query).parse()[0] - except (ValueError, IndexError, AttributeError): - message = f"channel not found: {query}" - return Response({"message": message, "data": False}, status=404) - - if not parsed["type"] == "channel": - message = "expected type channel" - return Response({"message": message, "data": False}, status=400) - - self.get_document(parsed["url"]) - - return Response(self.response, status=self.status_code) - - -class ChannelApiVideoView(ApiBaseView): - """resolves to /api/channel//video - GET: returns a list of videos of channel - """ - - search_base = "ta_video/_search/" - - def get(self, request, channel_id): - """handle get request""" - self.data.update( - { - "query": { - "term": {"channel.channel_id": {"value": channel_id}} - }, - "sort": [{"published": {"order": "desc"}}], - } - ) - self.get_document_list(request) - - return Response(self.response, status=self.status_code) - - class PlaylistApiListView(ApiBaseView): """resolves to /api/playlist/ GET: returns list of indexed playlists diff --git a/tubearchivist/channel/__init__.py b/tubearchivist/channel/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tubearchivist/channel/migrations/__init__.py b/tubearchivist/channel/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tubearchivist/channel/src/__init__.py b/tubearchivist/channel/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/channel/src/index.py similarity index 100% rename from tubearchivist/home/src/index/channel.py rename to tubearchivist/channel/src/index.py diff --git a/tubearchivist/channel/urls.py b/tubearchivist/channel/urls.py new file mode 100644 index 00000000..c00c2080 --- /dev/null +++ b/tubearchivist/channel/urls.py @@ -0,0 +1,27 @@ +"""all channel API urls""" + +from channel import views +from django.urls import path + +urlpatterns = [ + path( + "", + views.ChannelApiListView.as_view(), + name="api-channel-list", + ), + path( + "search/", + views.ChannelApiSearchView.as_view(), + name="api-channel-search", + ), + path( + "/", + views.ChannelApiView.as_view(), + name="api-channel", + ), + path( + "/video/", + views.ChannelApiVideoView.as_view(), + name="api-channel-video", + ), +] diff --git a/tubearchivist/channel/views.py b/tubearchivist/channel/views.py new file mode 100644 index 00000000..9f787c1c --- /dev/null +++ b/tubearchivist/channel/views.py @@ -0,0 +1,153 @@ +"""all channel API views""" + +from api.views import AdminWriteOnly, ApiBaseView +from channel.src.index import YoutubeChannel +from home.src.download.subscriptions import ChannelSubscription +from home.src.ta.urlparser import Parser +from home.tasks import subscribe_to +from rest_framework.response import Response + + +class ChannelApiView(ApiBaseView): + """resolves to /api/channel// + GET: returns metadata dict of channel + """ + + search_base = "ta_channel/_doc/" + permission_classes = [AdminWriteOnly] + + def get(self, request, channel_id): + # pylint: disable=unused-argument + """get request""" + self.get_document(channel_id) + return Response(self.response, status=self.status_code) + + def delete(self, request, channel_id): + # pylint: disable=unused-argument + """delete channel""" + message = {"channel": channel_id} + try: + YoutubeChannel(channel_id).delete_channel() + status_code = 200 + message.update({"state": "delete"}) + except FileNotFoundError: + status_code = 404 + message.update({"state": "not found"}) + + return Response(message, status=status_code) + + +class ChannelApiListView(ApiBaseView): + """resolves to /api/channel/ + GET: returns list of channels + POST: edit a list of channels + """ + + search_base = "ta_channel/_search/" + valid_filter = ["subscribed"] + permission_classes = [AdminWriteOnly] + + def get(self, request): + """get request""" + self.data.update( + {"sort": [{"channel_name.keyword": {"order": "asc"}}]} + ) + + query_filter = request.GET.get("filter", False) + must_list = [] + if query_filter: + if query_filter not in self.valid_filter: + message = f"invalid url query filter: {query_filter}" + print(message) + return Response({"message": message}, status=400) + + must_list.append({"term": {"channel_subscribed": {"value": True}}}) + + self.data["query"] = {"bool": {"must": must_list}} + self.get_document_list(request) + + return Response(self.response) + + def post(self, request): + """subscribe/unsubscribe to list of channels""" + data = request.data + try: + to_add = data["data"] + except KeyError: + message = "missing expected data key" + print(message) + return Response({"message": message}, status=400) + + pending = [] + for channel_item in to_add: + channel_id = channel_item["channel_id"] + if channel_item["channel_subscribed"]: + pending.append(channel_id) + else: + self._unsubscribe(channel_id) + + if pending: + url_str = " ".join(pending) + subscribe_to.delay(url_str, expected_type="channel") + + return Response(data) + + @staticmethod + def _unsubscribe(channel_id: str): + """unsubscribe""" + print(f"[{channel_id}] unsubscribe from channel") + ChannelSubscription().change_subscribe( + channel_id, channel_subscribed=False + ) + + +class ChannelApiSearchView(ApiBaseView): + """resolves to /api/channel/search/ + search for channel + """ + + search_base = "ta_channel/_doc/" + + def get(self, request): + """handle get request, search with s parameter""" + + query = request.GET.get("q") + if not query: + message = "missing expected q parameter" + return Response({"message": message, "data": False}, status=400) + + try: + parsed = Parser(query).parse()[0] + except (ValueError, IndexError, AttributeError): + message = f"channel not found: {query}" + return Response({"message": message, "data": False}, status=404) + + if not parsed["type"] == "channel": + message = "expected type channel" + return Response({"message": message, "data": False}, status=400) + + self.get_document(parsed["url"]) + + return Response(self.response, status=self.status_code) + + +class ChannelApiVideoView(ApiBaseView): + """resolves to /api/channel//video + GET: returns a list of videos of channel + """ + + search_base = "ta_video/_search/" + + def get(self, request, channel_id): + """handle get request""" + self.data.update( + { + "query": { + "term": {"channel.channel_id": {"value": channel_id}} + }, + "sort": [{"published": {"order": "desc"}}], + } + ) + self.get_document_list(request) + + return Response(self.response, status=self.status_code) diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 2a7c8edb..e2275a24 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -63,6 +63,7 @@ INSTALLED_APPS = [ "rest_framework.authtoken", "api", "video", + "channel", "config", ] diff --git a/tubearchivist/config/urls.py b/tubearchivist/config/urls.py index 146330fa..5d6bbd83 100644 --- a/tubearchivist/config/urls.py +++ b/tubearchivist/config/urls.py @@ -21,5 +21,6 @@ urlpatterns = [ path("", include("home.urls")), path("api/", include("api.urls")), path("api/video/", include("video.urls")), + path("api/channel/", include("channel.urls")), path("admin/", admin.site.urls), ] diff --git a/tubearchivist/home/src/download/subscriptions.py b/tubearchivist/home/src/download/subscriptions.py index 078a9e49..70b75556 100644 --- a/tubearchivist/home/src/download/subscriptions.py +++ b/tubearchivist/home/src/download/subscriptions.py @@ -4,10 +4,10 @@ Functionality: - handle playlist subscriptions """ +from channel.src.index import YoutubeChannel from home.src.download.thumbnails import ThumbManager from home.src.download.yt_dlp_base import YtWrap from home.src.es.connect import IndexPaginate -from home.src.index.channel import YoutubeChannel from home.src.index.playlist import YoutubePlaylist from home.src.ta.config import AppConfig from home.src.ta.helper import is_missing diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 9d2dc0a6..0b5e751f 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -10,11 +10,11 @@ import os import shutil from datetime import datetime +from channel.src.index import YoutubeChannel from home.src.download.queue import PendingList from home.src.download.subscriptions import PlaylistSubscription from home.src.download.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.index.channel import YoutubeChannel from home.src.index.playlist import YoutubePlaylist from home.src.ta.config import AppConfig from home.src.ta.helper import get_channel_overwrites, ignore_filelist diff --git a/tubearchivist/home/src/index/playlist.py b/tubearchivist/home/src/index/playlist.py index 65206f33..91806856 100644 --- a/tubearchivist/home/src/index/playlist.py +++ b/tubearchivist/home/src/index/playlist.py @@ -7,9 +7,9 @@ functionality: import json from datetime import datetime +from channel.src import index as channel from home.src.download.thumbnails import ThumbManager from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.index import channel from home.src.index.generic import YouTubeItem from video.src.index import YoutubeVideo diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index 527d44cc..9c2ebdf3 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -10,12 +10,12 @@ from datetime import datetime from time import sleep from typing import Callable, TypedDict +from channel.src.index import YoutubeChannel from home.models import CustomPeriodicTask from home.src.download.subscriptions import ChannelSubscription from home.src.download.thumbnails import ThumbManager from home.src.download.yt_dlp_base import CookieHandler from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.index.channel import YoutubeChannel from home.src.index.playlist import YoutubePlaylist from home.src.ta.config import AppConfig from home.src.ta.settings import EnvironmentSettings diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index c890fa64..0fa8e726 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -8,6 +8,7 @@ Functionality: from celery import Task, shared_task from celery.exceptions import Retry +from channel.src.index import YoutubeChannel from home.src.download.queue import PendingList from home.src.download.subscriptions import ( SubscriptionHandler, @@ -17,7 +18,6 @@ from home.src.download.thumbnails import ThumbFilesystem, ThumbValidator from home.src.download.yt_dlp_handler import VideoDownloader from home.src.es.backup import ElasticBackup from home.src.es.index_setup import ElasitIndexWrap -from home.src.index.channel import YoutubeChannel from home.src.index.filesystem import Scanner from home.src.index.manual import ImportFolderScanner from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index f9cbc4c8..b9a5466f 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -11,6 +11,7 @@ from time import sleep from api.src.search_processor import SearchProcess, process_aggs from api.views import check_admin +from channel.src.index import channel_overwrites from django.conf import settings from django.contrib.auth import login from django.contrib.auth.decorators import user_passes_test @@ -40,7 +41,6 @@ from home.src.frontend.forms_schedule import ( NotificationSettingsForm, SchedulerSettingsForm, ) -from home.src.index.channel import channel_overwrites from home.src.index.generic import Pagination from home.src.index.playlist import YoutubePlaylist from home.src.index.reindex import ReindexProgress diff --git a/tubearchivist/video/src/index.py b/tubearchivist/video/src/index.py index a02bc7f6..8b171fd5 100644 --- a/tubearchivist/video/src/index.py +++ b/tubearchivist/video/src/index.py @@ -8,9 +8,9 @@ import os from datetime import datetime import requests +from channel.src import index as ta_channel from django.conf import settings from home.src.es.connect import ElasticWrap -from home.src.index import channel as ta_channel from home.src.index import playlist as ta_playlist from home.src.index.generic import YouTubeItem from home.src.ta.helper import get_duration_sec, get_duration_str, randomizor