From 6d377a1714e922c0bfccf635ff8d9dbc49d0daa7 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 26 Dec 2025 21:29:23 +0700 Subject: [PATCH] use embedded for manual import, add error control --- backend/appsettings/serializers.py | 7 ++ backend/appsettings/src/manual.py | 88 ++++++++++++++----- backend/appsettings/urls.py | 5 ++ backend/appsettings/views.py | 31 ++++++- backend/task/src/task_config.py | 2 +- backend/task/tasks.py | 6 +- frontend/src/api/actions/queueManualImport.ts | 9 ++ frontend/src/api/actions/updateTaskByName.ts | 7 +- frontend/src/pages/SettingsActions.tsx | 29 +++++- 9 files changed, 151 insertions(+), 33 deletions(-) create mode 100644 frontend/src/api/actions/queueManualImport.ts diff --git a/backend/appsettings/serializers.py b/backend/appsettings/serializers.py index 6da5ae1f..734b3131 100644 --- a/backend/appsettings/serializers.py +++ b/backend/appsettings/serializers.py @@ -107,6 +107,13 @@ class RescanFileSystemConfig(serializers.Serializer): prefer_local = serializers.BooleanField() +class ManualImportConfig(serializers.Serializer): + """serialize for manual import task""" + + ignore_error = serializers.BooleanField() + prefer_local = serializers.BooleanField() + + class SnapshotItemSerializer(serializers.Serializer): """serialize snapshot response""" diff --git a/backend/appsettings/src/manual.py b/backend/appsettings/src/manual.py index 3be2b52b..36c3c62b 100644 --- a/backend/appsettings/src/manual.py +++ b/backend/appsettings/src/manual.py @@ -18,6 +18,7 @@ from download.src.thumbnails import ThumbManager from PIL import Image from video.src.comments import Comments from video.src.index import YoutubeVideo +from video.src.meta_embed import IndexFromEmbed from yt_dlp.utils import ISO639Utils @@ -41,9 +42,16 @@ class ImportFolderScanner: "subtitle": [".vtt"], } - def __init__(self, task=False): + def __init__( + self, + task=False, + ignore_error: bool = False, + prefer_local: bool = False, + ): self.task = task self.to_import = False + self.ignore_error = ignore_error + self.prefer_local = prefer_local def scan(self): """scan and match media files""" @@ -142,11 +150,14 @@ class ImportFolderScanner: self._convert_thumb(current_video) self._get_subtitles(current_video) self._convert_video(current_video) - print(f"manual import: {current_video}") - ManualImport(current_video, config).run() - Comments(current_video["video_id"]).build_json(upload=True) - YoutubeVideo(current_video["video_id"]).embed_metadata() + print(f"manual import: {current_video}") + ManualImport( + current_video, + config, + ignore_error=self.ignore_error, + prefer_local=self.prefer_local, + ).run() def _notify(self, idx, current_video): """send notification back to task""" @@ -182,17 +193,27 @@ class ImportFolderScanner: expects filename ending in []. """ base_name, _ = os.path.splitext(file_name) + + # yt-dlp default like [youtubeid] id_search = re.search(r"\[([a-zA-Z0-9_-]{11})\]$", base_name) if id_search: youtube_id = id_search.group(1) return youtube_id + file_name_search = re.search(r"([a-zA-Z0-9_-]{11})$", base_name) + if file_name_search: + youtube_id = file_name_search.group(1) + return youtube_id + print(f"id extraction failed from filename: {file_name}") return False - def _extract_id_from_json(self, json_file): + def _extract_id_from_json(self, json_file: str | bool) -> str | None: """open json file and extract id""" + if not json_file or not isinstance(json_file, str): + return None + json_path = os.path.join(self.CACHE_DIR, "import", json_file) with open(json_path, "r", encoding="utf-8") as f: json_content = f.read() @@ -385,17 +406,49 @@ class ImportFolderScanner: class ManualImport: """import single identified video""" - def __init__(self, current_video, config): + def __init__( + self, current_video, config, ignore_error: bool, prefer_local: bool + ): self.current_video = current_video self.config = config + self.ignore_error: bool = ignore_error + self.prefer_local: bool = prefer_local def run(self): """run all""" - json_data = self.index_metadata() - self._move_to_archive(json_data) - self._cleanup(json_data) + json_data = None + if self.prefer_local: + # embedded first + json_data = IndexFromEmbed( + self.current_video["media"], + use_user_conf=False, + config=self.config, + ).run_index() + if json_data: + self._cleanup() + return - def index_metadata(self): + try: + json_data = self.index_metadata() + except ValueError as err: + json_data = IndexFromEmbed( + self.current_video["media"], + use_user_conf=False, + config=self.config, + ).run_index() + if not json_data and not self.ignore_error: + raise ValueError from err + + if not json_data: + return + + self._move_to_archive(json_data) + self._cleanup() + + Comments(self.current_video["video_id"]).build_json(upload=True) + YoutubeVideo(self.current_video["video_id"]).embed_metadata() + + def index_metadata(self) -> dict | None: """get metadata from yt or json""" video_id = self.current_video["video_id"] video = YoutubeVideo(video_id) @@ -408,6 +461,9 @@ class ManualImport: f"{video_id}: manual import failed, and no metadata found." ) print(message) + if self.ignore_error: + return None + raise ValueError(message) video.check_subtitles(subtitle_files=self.current_video["subtitle"]) @@ -460,7 +516,7 @@ class ManualImport: new_path = f"{base_name}.{lang}.vtt" shutil.move(old_path, new_path, copy_function=shutil.copyfile) - def _cleanup(self, json_data): + def _cleanup(self): """cleanup leftover files""" meta_data = self.current_video["metadata"] if meta_data and os.path.exists(meta_data): @@ -473,11 +529,3 @@ class ManualImport: for subtitle_file in self.current_video["subtitle"]: if os.path.exists(subtitle_file): os.remove(subtitle_file) - - channel_info = os.path.join( - EnvironmentSettings.CACHE_DIR, - "import", - f"{json_data['channel']['channel_id']}.info.json", - ) - if os.path.exists(channel_info): - os.remove(channel_info) diff --git a/backend/appsettings/urls.py b/backend/appsettings/urls.py index d4f96d4f..7a5c3632 100644 --- a/backend/appsettings/urls.py +++ b/backend/appsettings/urls.py @@ -49,6 +49,11 @@ urlpatterns = [ views.RescanFileSystem.as_view(), name="api-rescan-filesystem", ), + path( + "manual-import/", + views.ManualImportView.as_view(), + name="api-manual-import", + ), path( "membership/profile/", views_mb.MembershipProfileView.as_view(), diff --git a/backend/appsettings/views.py b/backend/appsettings/views.py index cb909219..83787b02 100644 --- a/backend/appsettings/views.py +++ b/backend/appsettings/views.py @@ -5,6 +5,7 @@ from appsettings.serializers import ( BackupFileSerializer, CookieUpdateSerializer, CookieValidationSerializer, + ManualImportConfig, PoTokenSerializer, RescanFileSystemConfig, SnapshotCreateResponseSerializer, @@ -395,9 +396,11 @@ class SnapshotApiListView(ApiBaseView): class RescanFileSystem(ApiBaseView): """resolves to /api/appsettings/rescan-filesystem/ - POST: start new task rescan filesystem task + POST: start new rescan filesystem task """ + permission_classes = [AdminOnly] + @staticmethod @extend_schema( request=RescanFileSystemConfig, @@ -417,6 +420,32 @@ class RescanFileSystem(ApiBaseView): return Response(serializer.data) +class ManualImportView(ApiBaseView): + """resolves to /api/appsettings/manual-import/ + POST: start new manual import task + """ + + permission_classes = [AdminOnly] + + @staticmethod + @extend_schema( + request=ManualImportConfig, + responses={ + 200: OpenApiResponse(AsyncTaskResponseSerializer()), + }, + ) + def post(request): + """manual import""" + data_serializer = ManualImportConfig(data=request.data) + data_serializer.is_valid(raise_exception=True) + validated_data = data_serializer.validated_data + + message = TaskCommand().start("manual_import", validated_data) + serializer = AsyncTaskResponseSerializer(message) + + return Response(serializer.data) + + class SnapshotApiView(ApiBaseView): """resolves to /api/appsettings/snapshot// GET: return a single snapshot diff --git a/backend/task/src/task_config.py b/backend/task/src/task_config.py index d2e3b02e..8d8e3a7e 100644 --- a/backend/task/src/task_config.py +++ b/backend/task/src/task_config.py @@ -48,7 +48,7 @@ CHECK_REINDEX: TaskItemConfig = { MANUAL_IMPORT: TaskItemConfig = { "title": "Manual video import", "group": "setting:import", - "api_start": True, + "api_start": False, "api_stop": False, } diff --git a/backend/task/tasks.py b/backend/task/tasks.py index 914e4036..8b31b8dc 100644 --- a/backend/task/tasks.py +++ b/backend/task/tasks.py @@ -216,7 +216,7 @@ def check_reindex(self, data=False, extract_videos=False): @shared_task(bind=True, name="manual_import", base=BaseTask) -def run_manual_import(self): +def manual_import(self, ignore_error, prefer_local): """called from settings page, to go through import folder""" manager = TaskManager() if manager.is_pending(self): @@ -225,7 +225,9 @@ def run_manual_import(self): return manager.init(self) - ImportFolderScanner(task=self).scan() + ImportFolderScanner( + task=self, ignore_error=ignore_error, prefer_local=prefer_local + ).scan() @shared_task(bind=True, name="run_backup", base=BaseTask) diff --git a/frontend/src/api/actions/queueManualImport.ts b/frontend/src/api/actions/queueManualImport.ts new file mode 100644 index 00000000..57663c85 --- /dev/null +++ b/frontend/src/api/actions/queueManualImport.ts @@ -0,0 +1,9 @@ +import APIClient from '../../functions/APIClient'; + +const queueManualImport = async (ignore_error: boolean, prefer_local: boolean) => { + return APIClient('/api/appsettings/manual-import/', { + method: 'POST', + body: { ignore_error, prefer_local }, + }); +}; +export default queueManualImport; diff --git a/frontend/src/api/actions/updateTaskByName.ts b/frontend/src/api/actions/updateTaskByName.ts index dabd492b..d4bfd104 100644 --- a/frontend/src/api/actions/updateTaskByName.ts +++ b/frontend/src/api/actions/updateTaskByName.ts @@ -1,11 +1,6 @@ import APIClient from '../../functions/APIClient'; -type TaskNamesType = - | 'download_pending' - | 'update_subscribed' - | 'manual_import' - | 'resync_thumbs' - | 'resync_metadata'; +type TaskNamesType = 'download_pending' | 'update_subscribed' | 'resync_thumbs' | 'resync_metadata'; const updateTaskByName = async (taskName: TaskNamesType) => { return APIClient(`/api/task/by-name/${taskName}/`, { diff --git a/frontend/src/pages/SettingsActions.tsx b/frontend/src/pages/SettingsActions.tsx index 9104dab7..5262e34b 100644 --- a/frontend/src/pages/SettingsActions.tsx +++ b/frontend/src/pages/SettingsActions.tsx @@ -9,6 +9,7 @@ import Button from '../components/Button'; import { ApiResponseType } from '../functions/APIClient'; import ToggleConfig from '../components/ToggleConfig'; import queueStartFilesystemRescan from '../api/actions/queueStartFilesystemRescan'; +import queueManualImport from '../api/actions/queueManualImport'; const SettingsActions = () => { const [deleteIgnored, setDeleteIgnored] = useState(false); @@ -21,6 +22,8 @@ const SettingsActions = () => { const [reScanningFileSystem, setReScanningFileSystem] = useState(false); const [rescanIgnoreErrors, setRescanIgnoreErrors] = useState(false); const [rescanPreferLocal, setRescanPreferLocal] = useState(false); + const [manualPreferLocal, setManualPreferLocal] = useState(false); + const [manualIgnoreErrors, setManualIgnoreErrors] = useState(false); const [backupListResponse, setBackupListResponse] = useState>(); @@ -73,22 +76,42 @@ const SettingsActions = () => {

Manual media files import.

Add files to the cache/import folder. Make - sure to follow the instructions in the Github{' '} + sure to follow the instructions in the{' '} - Wiki + Docs .

+
+
+

Prefer embedded metadata

+
+ setManualPreferLocal(!manualPreferLocal)} + /> +
+
+
+

Ignore missing metadata errors

+
+ setManualIgnoreErrors(!manualIgnoreErrors)} + /> +
{processingImports &&

Processing import

} {!processingImports && (