add appsettings serializers and docs

This commit is contained in:
Simon 2025-02-09 22:10:29 +07:00
parent df5a5cf449
commit 6723d6152f
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
6 changed files with 507 additions and 177 deletions

View File

@ -0,0 +1,132 @@
"""appsettings erializers"""
# pylint: disable=abstract-method
from common.serializers import ValidateUnknownFieldsMixin
from rest_framework import serializers
class BackupFileSerializer(serializers.Serializer):
"""serialize backup file"""
filename = serializers.CharField()
file_path = serializers.CharField()
file_size = serializers.IntegerField()
timestamp = serializers.CharField()
reason = serializers.CharField()
class AppConfigSubSerializer(
ValidateUnknownFieldsMixin, serializers.Serializer
):
"""serialize app config subscriptions"""
channel_size = serializers.IntegerField(required=False)
live_channel_size = serializers.IntegerField(required=False)
shorts_channel_size = serializers.IntegerField(required=False)
auto_start = serializers.BooleanField(required=False)
class AppConfigDownloadsSerializer(
ValidateUnknownFieldsMixin, serializers.Serializer
):
"""serialize app config downloads config"""
limit_speed = serializers.IntegerField(allow_null=True)
sleep_interval = serializers.IntegerField(allow_null=True)
autodelete_days = serializers.IntegerField(allow_null=True)
format = serializers.CharField(allow_null=True)
format_sort = serializers.CharField(allow_null=True)
add_metadata = serializers.BooleanField()
add_thumbnail = serializers.BooleanField()
subtitle = serializers.CharField(allow_null=True)
subtitle_source = serializers.ChoiceField(
choices=["auto", "user"], allow_null=True
)
subtitle_index = serializers.BooleanField()
comment_max = serializers.CharField(allow_null=True)
comment_sort = serializers.ChoiceField(
choices=["top", "new"], allow_null=True
)
cookie_import = serializers.BooleanField()
potoken = serializers.BooleanField()
throttledratelimit = serializers.IntegerField(allow_null=True)
extractor_lang = serializers.CharField(allow_null=True)
integrate_ryd = serializers.BooleanField()
integrate_sponsorblock = serializers.BooleanField()
class AppConfigAppSerializer(
ValidateUnknownFieldsMixin, serializers.Serializer
):
"""serialize app config"""
enable_snapshot = serializers.BooleanField()
class AppConfigSerializer(serializers.Serializer):
"""serialize appconfig"""
subscriptions = AppConfigSubSerializer(required=False)
downloads = AppConfigDownloadsSerializer(required=False)
application = AppConfigAppSerializer(required=False)
class CookieValidationSerializer(serializers.Serializer):
"""serialize cookie validation response"""
cookie_enabled = serializers.BooleanField()
status = serializers.BooleanField(required=False)
validated = serializers.IntegerField(required=False)
validated_str = serializers.CharField(required=False)
class CookieUpdateSerializer(serializers.Serializer):
"""serialize cookie to update"""
cookie = serializers.CharField()
class PoTokenSerializer(serializers.Serializer):
"""serialize PO token"""
potoken = serializers.CharField()
class SnapshotItemSerializer(serializers.Serializer):
"""serialize snapshot response"""
id = serializers.CharField()
state = serializers.CharField()
es_version = serializers.CharField()
start_date = serializers.CharField()
end_date = serializers.CharField()
end_stamp = serializers.IntegerField()
duration_s = serializers.IntegerField()
class SnapshotListSerializer(serializers.Serializer):
"""serialize snapshot list response"""
next_exec = serializers.IntegerField()
next_exec_str = serializers.CharField()
expire_after = serializers.CharField()
snapshots = SnapshotItemSerializer(many=True)
class SnapshotCreateResponseSerializer(serializers.Serializer):
"""serialize new snapshot creating response"""
snapshot_name = serializers.CharField()
class SnapshotRestoreResponseSerializer(serializers.Serializer):
"""serialize snapshot restore response"""
accepted = serializers.BooleanField()
class TokenResponseSerializer(serializers.Serializer):
"""serialize token response"""
token = serializers.CharField()

View File

@ -109,29 +109,24 @@ class AppConfig:
def update_config(self, data: dict) -> AppConfigType:
"""update single config value"""
new_config = self.config.copy()
for key, value in data.items():
key_map = key.split(".")
self._validate_key(key_map)
self.config[key_map[0]][key_map[1]] = value
if (
isinstance(value, dict)
and key in new_config
and isinstance(new_config[key], dict)
):
new_config[key].update(value)
else:
new_config[key] = value
response, status_code = ElasticWrap(self.ES_PATH).post(self.config)
response, status_code = ElasticWrap(self.ES_PATH).post(new_config)
if not status_code == 200:
print(response)
return self.config
self.config = new_config
def _update_config_dict(self, to_update) -> None:
"""none validated partial update for defaults sync"""
data = {"doc": to_update}
response, status_code = ElasticWrap(self.ES_UPDATE_PATH).post(data)
if not status_code == 200:
print(f"update failed: {response}, {status_code}")
def _validate_key(self, key_map: list[str]) -> None:
"""raise valueerror on invalid key"""
exists = key_map[1] in self.CONFIG_DEFAULTS.get(key_map[0], {}) # type: ignore # noqa: E501
if exists is None:
raise ValueError(f"trying to access invalid config key: {key_map}")
return new_config
def post_process_updated(self, data: dict) -> None:
"""apply hooks for some config keys"""
@ -163,7 +158,7 @@ class AppConfig:
for key, value in self.CONFIG_DEFAULTS.items():
if key not in self.config:
# complete new key
self._update_config_dict({key: value})
self.update_config({key: value})
updated.append(str({key: value}))
continue
@ -171,7 +166,7 @@ class AppConfig:
if sub_key not in self.config[key]:
# new partial key
to_update = {key: {sub_key: sub_value}}
self._update_config_dict(to_update)
self.update_config(to_update)
updated.append(str(to_update))
return updated

View File

@ -1,116 +1,35 @@
"""all app settings API views"""
from appsettings.serializers import (
AppConfigSerializer,
BackupFileSerializer,
CookieUpdateSerializer,
CookieValidationSerializer,
PoTokenSerializer,
SnapshotCreateResponseSerializer,
SnapshotItemSerializer,
SnapshotListSerializer,
SnapshotRestoreResponseSerializer,
TokenResponseSerializer,
)
from appsettings.src.backup import ElasticBackup
from appsettings.src.config import AppConfig
from appsettings.src.snapshot import ElasticSnapshot
from common.serializers import (
AsyncTaskResponseSerializer,
ErrorResponseSerializer,
)
from common.src.ta_redis import RedisArchivist
from common.views_base import AdminOnly, ApiBaseView
from django.conf import settings
from download.src.yt_dlp_base import CookieHandler, POTokenHandler
from drf_spectacular.utils import OpenApiResponse, extend_schema
from rest_framework.authtoken.models import Token
from rest_framework.response import Response
from task.src.task_manager import TaskCommand
from task.tasks import run_restore_backup
class AppConfigApiView(ApiBaseView):
"""resolves to /api/appsettings/config/
GET: return app settings
POST: update app settings
"""
permission_classes = [AdminOnly]
@staticmethod
def get(request):
"""get config"""
response = AppConfig().config
return Response(response)
@staticmethod
def post(request):
"""
update config values
data object where key is flatted CONFIG_DEFAULTS separated by '.', e.g.
{"subscriptions.channel_size": 5, "subscriptions.live_channel_size": 5}
"""
data = request.data
try:
config = AppConfig().update_config(data)
except ValueError as err:
return Response({"error": str(err)}, status=400)
return Response(config)
class SnapshotApiListView(ApiBaseView):
"""resolves to /api/appsettings/snapshot/
GET: returns snapshot config plus list of existing snapshots
POST: take snapshot now
"""
permission_classes = [AdminOnly]
@staticmethod
def get(request):
"""handle get request"""
# pylint: disable=unused-argument
snapshots = ElasticSnapshot().get_snapshot_stats()
return Response(snapshots)
@staticmethod
def post(request):
"""take snapshot now with post request"""
# pylint: disable=unused-argument
response = ElasticSnapshot().take_snapshot_now()
return Response(response)
class SnapshotApiView(ApiBaseView):
"""resolves to /api/appsettings/snapshot/<snapshot-id>/
GET: return a single snapshot
POST: restore snapshot
DELETE: delete a snapshot
"""
permission_classes = [AdminOnly]
@staticmethod
def get(request, snapshot_id):
"""handle get request"""
# pylint: disable=unused-argument
snapshot = ElasticSnapshot().get_single_snapshot(snapshot_id)
if not snapshot:
return Response({"message": "snapshot not found"}, status=404)
return Response(snapshot)
@staticmethod
def post(request, snapshot_id):
"""restore snapshot with post request"""
# pylint: disable=unused-argument
response = ElasticSnapshot().restore_all(snapshot_id)
if not response:
message = {"message": "failed to restore snapshot"}
return Response(message, status=400)
return Response(response)
@staticmethod
def delete(request, snapshot_id):
"""delete snapshot from index"""
# pylint: disable=unused-argument
response = ElasticSnapshot().delete_single_snapshot(snapshot_id)
if not response:
message = {"message": "failed to delete snapshot"}
return Response(message, status=400)
return Response(response)
class BackupApiListView(ApiBaseView):
"""resolves to /api/appsettings/backup/
GET: returns list of available zip backups
@ -121,22 +40,34 @@ class BackupApiListView(ApiBaseView):
task_name = "run_backup"
@staticmethod
@extend_schema(
responses={
200: OpenApiResponse(BackupFileSerializer(many=True)),
},
)
def get(request):
"""handle get request"""
"""get list of available backup files"""
# pylint: disable=unused-argument
backup_files = ElasticBackup().get_all_backup_files()
return Response(backup_files)
serializer = BackupFileSerializer(backup_files, many=True)
return Response(serializer.data)
@extend_schema(
responses={
200: OpenApiResponse(AsyncTaskResponseSerializer()),
},
)
def post(self, request):
"""handle post request"""
"""start new backup file task"""
# pylint: disable=unused-argument
response = TaskCommand().start(self.task_name)
message = {
"message": "backup task started",
"task_id": response["task_id"],
}
serializer = AsyncTaskResponseSerializer(message)
return Response(message)
return Response(serializer.data)
class BackupApiView(ApiBaseView):
@ -150,19 +81,42 @@ class BackupApiView(ApiBaseView):
task_name = "restore_backup"
@staticmethod
@extend_schema(
responses={
200: OpenApiResponse(BackupFileSerializer()),
404: OpenApiResponse(
ErrorResponseSerializer(), description="file not found"
),
}
)
def get(request, filename):
"""get single backup"""
# pylint: disable=unused-argument
backup_file = ElasticBackup().build_backup_file_data(filename)
if not backup_file:
message = {"message": "file not found"}
return Response(message, status=404)
error = ErrorResponseSerializer({"error": "file not found"})
return Response(error.data, status=404)
return Response(backup_file)
serializer = BackupFileSerializer(backup_file)
return Response(serializer.data)
@extend_schema(
responses={
200: OpenApiResponse(AsyncTaskResponseSerializer()),
404: OpenApiResponse(
ErrorResponseSerializer(), description="file not found"
),
}
)
def post(self, request, filename):
"""restore backup file"""
"""start new task to restore backup file"""
# pylint: disable=unused-argument
backup_file = ElasticBackup().build_backup_file_data(filename)
if not backup_file:
error = ErrorResponseSerializer({"error": "file not found"})
return Response(error.data, status=404)
task = run_restore_backup.delay(filename)
message = {
"message": "backup restore task started",
@ -172,17 +126,64 @@ class BackupApiView(ApiBaseView):
return Response(message)
@staticmethod
@extend_schema(
responses={
204: OpenApiResponse(description="file deleted"),
404: OpenApiResponse(
ErrorResponseSerializer(), description="file not found"
),
}
)
def delete(request, filename):
"""delete backup file"""
# pylint: disable=unused-argument
backup_file = ElasticBackup().delete_file(filename)
if not backup_file:
message = {"message": "file not found"}
return Response(message, status=404)
error = ErrorResponseSerializer({"error": "file not found"})
return Response(error.data, status=404)
message = {"message": f"file {filename} deleted"}
return Response(message)
return Response(status=204)
class AppConfigApiView(ApiBaseView):
"""resolves to /api/appsettings/config/
GET: return app settings
POST: update app settings
"""
permission_classes = [AdminOnly]
@staticmethod
@extend_schema(
responses={
200: OpenApiResponse(AppConfigSerializer()),
}
)
def get(request):
"""get app config"""
response = AppConfig().config
serializer = AppConfigSerializer(response)
return Response(serializer.data)
@staticmethod
@extend_schema(
request=AppConfigSerializer(),
responses={
200: OpenApiResponse(AppConfigSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="Bad request"
),
},
)
def post(request):
"""update config values, partial"""
serializer = AppConfigSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
validated_data = serializer.validated_data
updated_config = AppConfig().update_config(validated_data)
updated_serializer = AppConfigSerializer(updated_config)
return Response(updated_serializer.data)
class CookieView(ApiBaseView):
@ -195,54 +196,88 @@ class CookieView(ApiBaseView):
permission_classes = [AdminOnly]
@extend_schema(
responses={
200: OpenApiResponse(CookieValidationSerializer()),
}
)
def get(self, request):
"""handle get request"""
"""get cookie validation status"""
# pylint: disable=unused-argument
validation = self._get_cookie_validation()
serializer = CookieValidationSerializer(validation)
return Response(validation)
return Response(serializer.data)
@extend_schema(
responses={
200: OpenApiResponse(CookieValidationSerializer()),
}
)
def post(self, request):
"""handle cookie validation request"""
"""validate cookie"""
# pylint: disable=unused-argument
config = AppConfig().config
_ = CookieHandler(config).validate()
validation = self._get_cookie_validation()
serializer = CookieValidationSerializer(validation)
return Response(validation)
return Response(serializer.data)
@extend_schema(
request=CookieUpdateSerializer(),
responses={
200: OpenApiResponse(CookieValidationSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="Bad request"
),
},
)
def put(self, request):
"""handle put request"""
# pylint: disable=unused-argument
config = AppConfig().config
cookie = request.data.get("cookie")
serializer = CookieUpdateSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
validated_data = serializer.validated_data
cookie = validated_data.get("cookie")
if not cookie:
message = "missing cookie key in request data"
print(message)
return Response({"message": message}, status=400)
error = ErrorResponseSerializer({"error": message})
return Response(error.data, status=400)
if settings.DEBUG:
print(f"[cookie] preview:\n\n{cookie[:300]}")
config = AppConfig().config
handler = CookieHandler(config)
handler.set_cookie(cookie)
validated = handler.validate()
if not validated:
message = "[cookie]: import failed, not valid"
print(message)
error = ErrorResponseSerializer({"error": message})
handler.revoke()
print("[cookie]: import failed, not valid")
status = 400
else:
status = 200
return Response(error.data, status=400)
validation = self._get_cookie_validation()
return Response(validation, status=status)
serializer = CookieValidationSerializer(validation)
return Response(serializer.data)
@extend_schema(
responses={
204: OpenApiResponse(description="Cookie revoked"),
},
)
def delete(self, request):
"""delete the cookie"""
config = AppConfig().config
handler = CookieHandler(config)
handler.revoke()
return Response({"cookie_enabled": False})
return Response(status=204)
@staticmethod
def _get_cookie_validation():
@ -260,47 +295,199 @@ class POTokenView(ApiBaseView):
permission_classes = [AdminOnly]
@extend_schema(
responses={
200: OpenApiResponse(PoTokenSerializer()),
404: OpenApiResponse(
ErrorResponseSerializer(), description="PO token not found"
),
}
)
def get(self, request):
"""get token"""
"""get PO token"""
config = AppConfig().config
potoken = POTokenHandler(config).get()
return Response({"potoken": potoken})
if not potoken:
error = ErrorResponseSerializer({"error": "PO token not found"})
return Response(error.data, status=404)
serializer = PoTokenSerializer(data={"potoken": potoken})
serializer.is_valid(raise_exception=True)
return Response(serializer.data)
@extend_schema(
responses={
200: OpenApiResponse(PoTokenSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="Bad request"
),
}
)
def post(self, request):
"""post token"""
"""Update PO token"""
serializer = PoTokenSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
validated_data = serializer.validated_data
if not validated_data:
error = ErrorResponseSerializer(
{"error": "missing PO token key in request data"}
)
return Response(error.data, status=400)
config = AppConfig().config
new_token = request.data.get("potoken")
if not new_token:
message = "missing potoken key in request data"
print(message)
return Response({"message": message}, status=400)
new_token = validated_data["potoken"]
POTokenHandler(config).set_token(new_token)
return Response({"potoken": new_token})
return Response(serializer.data)
@extend_schema(
responses={
204: OpenApiResponse(description="PO token revoked"),
},
)
def delete(self, request):
"""delete token"""
"""delete PO token"""
config = AppConfig().config
POTokenHandler(config).revoke_token()
return Response({"potoken": None})
return Response(status=204)
class SnapshotApiListView(ApiBaseView):
"""resolves to /api/appsettings/snapshot/
GET: returns snapshot config plus list of existing snapshots
POST: take snapshot now
"""
permission_classes = [AdminOnly]
@staticmethod
@extend_schema(
responses={
200: OpenApiResponse(SnapshotListSerializer()),
}
)
def get(request):
"""get available snapshots with metadata"""
# pylint: disable=unused-argument
snapshots = ElasticSnapshot().get_snapshot_stats()
serializer = SnapshotListSerializer(snapshots)
return Response(serializer.data)
@staticmethod
@extend_schema(
responses={
200: OpenApiResponse(SnapshotCreateResponseSerializer()),
}
)
def post(request):
"""take snapshot now"""
# pylint: disable=unused-argument
response = ElasticSnapshot().take_snapshot_now()
serializer = SnapshotCreateResponseSerializer(response)
return Response(serializer.data)
class SnapshotApiView(ApiBaseView):
"""resolves to /api/appsettings/snapshot/<snapshot-id>/
GET: return a single snapshot
POST: restore snapshot
DELETE: delete a snapshot
"""
permission_classes = [AdminOnly]
@staticmethod
@extend_schema(
responses={
200: OpenApiResponse(SnapshotItemSerializer()),
404: OpenApiResponse(
ErrorResponseSerializer(), description="snapshot not found"
),
}
)
def get(request, snapshot_id):
"""handle get request"""
# pylint: disable=unused-argument
snapshot = ElasticSnapshot().get_single_snapshot(snapshot_id)
if not snapshot:
error = ErrorResponseSerializer({"error": "snapshot not found"})
return Response(error.data, status=404)
serializer = SnapshotItemSerializer(snapshot)
return Response(serializer.data)
@staticmethod
@extend_schema(
responses={
200: OpenApiResponse(SnapshotRestoreResponseSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="bad request"
),
}
)
def post(request, snapshot_id):
"""restore snapshot"""
# pylint: disable=unused-argument
response = ElasticSnapshot().restore_all(snapshot_id)
if not response:
error = ErrorResponseSerializer(
{"error": "failed to restore snapshot"}
)
return Response(error.data, status=400)
serializer = SnapshotRestoreResponseSerializer(response)
return Response(serializer.data)
@staticmethod
@extend_schema(
responses={
204: OpenApiResponse(description="delete snapshot from index"),
}
)
def delete(request, snapshot_id):
"""delete snapshot from index"""
# pylint: disable=unused-argument
response = ElasticSnapshot().delete_single_snapshot(snapshot_id)
if not response:
error = ErrorResponseSerializer(
{"error": "failed to delete snapshot"}
)
return Response(error.data, status=400)
return Response(status=204)
class TokenView(ApiBaseView):
"""resolves to /api/appsettings/token/
GET: get API token
DELETE: revoke the token
"""
permission_classes = [AdminOnly]
@staticmethod
@extend_schema(
responses={
200: OpenApiResponse(TokenResponseSerializer()),
}
)
def get(request):
"""get token"""
"""get your API token"""
token, _ = Token.objects.get_or_create(user=request.user)
return Response({"token": token.key})
serializer = TokenResponseSerializer({"token": token.key})
return Response(serializer.data)
@staticmethod
@extend_schema(
responses={
204: OpenApiResponse(description="delete token"),
}
)
def delete(request):
"""delete the token, new will get created automatically"""
"""delete your API token, new will get created on next get"""
print("revoke API token")
request.user.auth_token.delete()
return Response({"success": True})
return Response(status=204)

View File

@ -2,11 +2,13 @@
# pylint: disable=abstract-method
from common.serializers import PaginationSerializer
from common.serializers import PaginationSerializer, ValidateUnknownFieldsMixin
from rest_framework import serializers
class ChannelOverwriteSerializer(serializers.Serializer):
class ChannelOverwriteSerializer(
ValidateUnknownFieldsMixin, serializers.Serializer
):
"""serialize channel overwrites"""
download_format = serializers.CharField(required=False, allow_null=True)
@ -25,20 +27,6 @@ class ChannelOverwriteSerializer(serializers.Serializer):
required=False, allow_null=True
)
def to_internal_value(self, data):
"""Override this method to detect unknown fields."""
allowed_fields = set(self.fields.keys())
input_fields = set(data.keys())
unknown_fields = input_fields - allowed_fields
if unknown_fields:
raise serializers.ValidationError(
{"error": f"Unknown fields: {', '.join(unknown_fields)}"}
)
return super().to_internal_value(data)
class ChannelSerializer(serializers.Serializer):
"""serialize channel"""

View File

@ -5,6 +5,26 @@
from rest_framework import serializers
class ValidateUnknownFieldsMixin:
"""
Mixin to validate and reject unknown fields in a serializer.
"""
def to_internal_value(self, data):
"""check expected keys"""
allowed_fields = set(self.fields.keys())
input_fields = set(data.keys())
# Find unknown fields
unknown_fields = input_fields - allowed_fields
if unknown_fields:
raise serializers.ValidationError(
{"error": f"Unknown fields: {', '.join(unknown_fields)}"}
)
return super().to_internal_value(data)
class ErrorResponseSerializer(serializers.Serializer):
"""error message"""
@ -27,3 +47,11 @@ class PaginationSerializer(serializers.Serializer):
child=serializers.IntegerField(), allow_null=True
)
total_hits = serializers.IntegerField()
class AsyncTaskResponseSerializer(serializers.Serializer):
"""serialize new async task"""
message = serializers.CharField()
task_id = serializers.CharField()
filename = serializers.CharField(required=False)

View File

@ -134,7 +134,7 @@ class CookieHandler:
"""set cookie str and activate in config"""
cookie_clean = cookie.strip("\x00")
RedisArchivist().set_message("cookie", cookie_clean, save=True)
AppConfig().update_config({"downloads.cookie_import": True})
AppConfig().update_config({"downloads": {"cookie_import": True}})
self.config["downloads"]["cookie_import"] = True
print("[cookie]: activated and stored in Redis")
@ -143,7 +143,7 @@ class CookieHandler:
"""revoke cookie"""
RedisArchivist().del_message("cookie")
RedisArchivist().del_message("cookie:valid")
AppConfig().update_config({"downloads.cookie_import": False})
AppConfig().update_config({"downloads": {"cookie_import": False}})
print("[cookie]: revoked")
def validate(self):
@ -211,9 +211,9 @@ class POTokenHandler:
def set_token(self, new_token: str) -> None:
"""set new PO token"""
RedisArchivist().set_message(self.REDIS_KEY, new_token)
AppConfig().update_config({"downloads.potoken": True})
AppConfig().update_config({"downloads": {"potoken": True}})
def revoke_token(self) -> None:
"""revoke token"""
RedisArchivist().del_message(self.REDIS_KEY)
AppConfig().update_config({"downloads.potoken": False})
AppConfig().update_config({"downloads": {"potoken": False}})