use embedded for manual import, add error control

backend/appsettings/serializers.py

@@ -107,6 +107,13 @@ class RescanFileSystemConfig(serializers.Serializer):
     prefer_local = serializers.BooleanField()
 
 
+class ManualImportConfig(serializers.Serializer):
+    """serialize for manual import task"""
+
+    ignore_error = serializers.BooleanField()
+    prefer_local = serializers.BooleanField()
+
+
 class SnapshotItemSerializer(serializers.Serializer):
     """serialize snapshot response"""
 

backend/appsettings/src/manual.py

@@ -18,6 +18,7 @@ from download.src.thumbnails import ThumbManager
 from PIL import Image
 from video.src.comments import Comments
 from video.src.index import YoutubeVideo
+from video.src.meta_embed import IndexFromEmbed
 from yt_dlp.utils import ISO639Utils
 
 
@@ -41,9 +42,16 @@ class ImportFolderScanner:
         "subtitle": [".vtt"],
     }
 
-    def __init__(self, task=False):
+    def __init__(
+        self,
+        task=False,
+        ignore_error: bool = False,
+        prefer_local: bool = False,
+    ):
         self.task = task
         self.to_import = False
+        self.ignore_error = ignore_error
+        self.prefer_local = prefer_local
 
     def scan(self):
         """scan and match media files"""
@@ -142,11 +150,14 @@ class ImportFolderScanner:
             self._convert_thumb(current_video)
             self._get_subtitles(current_video)
             self._convert_video(current_video)
-            print(f"manual import: {current_video}")
 
-            ManualImport(current_video, config).run()
+            print(f"manual import: {current_video}")
-            Comments(current_video["video_id"]).build_json(upload=True)
+            ManualImport(
-            YoutubeVideo(current_video["video_id"]).embed_metadata()
+                current_video,
+                config,
+                ignore_error=self.ignore_error,
+                prefer_local=self.prefer_local,
+            ).run()
 
     def _notify(self, idx, current_video):
         """send notification back to task"""
@@ -182,17 +193,27 @@ class ImportFolderScanner:
         expects filename ending in [<youtube_id>].<ext>
         """
         base_name, _ = os.path.splitext(file_name)
+
+        # yt-dlp default like [youtubeid]
         id_search = re.search(r"\[([a-zA-Z0-9_-]{11})\]$", base_name)
         if id_search:
             youtube_id = id_search.group(1)
             return youtube_id
 
+        file_name_search = re.search(r"([a-zA-Z0-9_-]{11})$", base_name)
+        if file_name_search:
+            youtube_id = file_name_search.group(1)
+            return youtube_id
+
         print(f"id extraction failed from filename: {file_name}")
 
         return False
 
-    def _extract_id_from_json(self, json_file):
+    def _extract_id_from_json(self, json_file: str | bool) -> str | None:
         """open json file and extract id"""
+        if not json_file or not isinstance(json_file, str):
+            return None
+
         json_path = os.path.join(self.CACHE_DIR, "import", json_file)
         with open(json_path, "r", encoding="utf-8") as f:
             json_content = f.read()
@@ -385,17 +406,49 @@ class ImportFolderScanner:
 class ManualImport:
     """import single identified video"""
 
-    def __init__(self, current_video, config):
+    def __init__(
+        self, current_video, config, ignore_error: bool, prefer_local: bool
+    ):
         self.current_video = current_video
         self.config = config
+        self.ignore_error: bool = ignore_error
+        self.prefer_local: bool = prefer_local
 
     def run(self):
         """run all"""
-        json_data = self.index_metadata()
+        json_data = None
-        self._move_to_archive(json_data)
+        if self.prefer_local:
-        self._cleanup(json_data)
+            # embedded first
+            json_data = IndexFromEmbed(
+                self.current_video["media"],
+                use_user_conf=False,
+                config=self.config,
+            ).run_index()
+            if json_data:
+                self._cleanup()
+                return
 
-    def index_metadata(self):
+        try:
+            json_data = self.index_metadata()
+        except ValueError as err:
+            json_data = IndexFromEmbed(
+                self.current_video["media"],
+                use_user_conf=False,
+                config=self.config,
+            ).run_index()
+            if not json_data and not self.ignore_error:
+                raise ValueError from err
+
+        if not json_data:
+            return
+
+        self._move_to_archive(json_data)
+        self._cleanup()
+
+        Comments(self.current_video["video_id"]).build_json(upload=True)
+        YoutubeVideo(self.current_video["video_id"]).embed_metadata()
+
+    def index_metadata(self) -> dict | None:
         """get metadata from yt or json"""
         video_id = self.current_video["video_id"]
         video = YoutubeVideo(video_id)
@@ -408,6 +461,9 @@ class ManualImport:
                 f"{video_id}: manual import failed, and no metadata found."
             )
             print(message)
+            if self.ignore_error:
+                return None
+
             raise ValueError(message)
 
         video.check_subtitles(subtitle_files=self.current_video["subtitle"])
@@ -460,7 +516,7 @@ class ManualImport:
             new_path = f"{base_name}.{lang}.vtt"
             shutil.move(old_path, new_path, copy_function=shutil.copyfile)
 
-    def _cleanup(self, json_data):
+    def _cleanup(self):
         """cleanup leftover files"""
         meta_data = self.current_video["metadata"]
         if meta_data and os.path.exists(meta_data):
@@ -473,11 +529,3 @@ class ManualImport:
         for subtitle_file in self.current_video["subtitle"]:
             if os.path.exists(subtitle_file):
                 os.remove(subtitle_file)
-
-        channel_info = os.path.join(
-            EnvironmentSettings.CACHE_DIR,
-            "import",
-            f"{json_data['channel']['channel_id']}.info.json",
-        )
-        if os.path.exists(channel_info):
-            os.remove(channel_info)

backend/appsettings/urls.py

@@ -49,6 +49,11 @@ urlpatterns = [
         views.RescanFileSystem.as_view(),
         name="api-rescan-filesystem",
     ),
+    path(
+        "manual-import/",
+        views.ManualImportView.as_view(),
+        name="api-manual-import",
+    ),
     path(
         "membership/profile/",
         views_mb.MembershipProfileView.as_view(),

backend/appsettings/views.py

@@ -5,6 +5,7 @@ from appsettings.serializers import (
     BackupFileSerializer,
     CookieUpdateSerializer,
     CookieValidationSerializer,
+    ManualImportConfig,
     PoTokenSerializer,
     RescanFileSystemConfig,
     SnapshotCreateResponseSerializer,
@@ -395,9 +396,11 @@ class SnapshotApiListView(ApiBaseView):
 
 class RescanFileSystem(ApiBaseView):
     """resolves to /api/appsettings/rescan-filesystem/
-    POST: start new task rescan filesystem task
+    POST: start new rescan filesystem task
     """
 
+    permission_classes = [AdminOnly]
+
     @staticmethod
     @extend_schema(
         request=RescanFileSystemConfig,
@@ -417,6 +420,32 @@ class RescanFileSystem(ApiBaseView):
         return Response(serializer.data)
 
 
+class ManualImportView(ApiBaseView):
+    """resolves to /api/appsettings/manual-import/
+    POST: start new manual import task
+    """
+
+    permission_classes = [AdminOnly]
+
+    @staticmethod
+    @extend_schema(
+        request=ManualImportConfig,
+        responses={
+            200: OpenApiResponse(AsyncTaskResponseSerializer()),
+        },
+    )
+    def post(request):
+        """manual import"""
+        data_serializer = ManualImportConfig(data=request.data)
+        data_serializer.is_valid(raise_exception=True)
+        validated_data = data_serializer.validated_data
+
+        message = TaskCommand().start("manual_import", validated_data)
+        serializer = AsyncTaskResponseSerializer(message)
+
+        return Response(serializer.data)
+
+
 class SnapshotApiView(ApiBaseView):
     """resolves to /api/appsettings/snapshot/<snapshot-id>/
     GET: return a single snapshot

backend/task/src/task_config.py

@@ -48,7 +48,7 @@ CHECK_REINDEX: TaskItemConfig = {
 MANUAL_IMPORT: TaskItemConfig = {
     "title": "Manual video import",
     "group": "setting:import",
-    "api_start": True,
+    "api_start": False,
     "api_stop": False,
 }
 

backend/task/tasks.py

@@ -216,7 +216,7 @@ def check_reindex(self, data=False, extract_videos=False):
 
 
 @shared_task(bind=True, name="manual_import", base=BaseTask)
-def run_manual_import(self):
+def manual_import(self, ignore_error, prefer_local):
     """called from settings page, to go through import folder"""
     manager = TaskManager()
     if manager.is_pending(self):
@@ -225,7 +225,9 @@ def run_manual_import(self):
         return
 
     manager.init(self)
-    ImportFolderScanner(task=self).scan()
+    ImportFolderScanner(
+        task=self, ignore_error=ignore_error, prefer_local=prefer_local
+    ).scan()
 
 
 @shared_task(bind=True, name="run_backup", base=BaseTask)

frontend/src/api/actions/queueManualImport.ts

@@ -0,0 +1,9 @@
+import APIClient from '../../functions/APIClient';
+
+const queueManualImport = async (ignore_error: boolean, prefer_local: boolean) => {
+  return APIClient('/api/appsettings/manual-import/', {
+    method: 'POST',
+    body: { ignore_error, prefer_local },
+  });
+};
+export default queueManualImport;

frontend/src/api/actions/updateTaskByName.ts

@@ -1,11 +1,6 @@
 import APIClient from '../../functions/APIClient';
 
-type TaskNamesType =
+type TaskNamesType = 'download_pending' | 'update_subscribed' | 'resync_thumbs' | 'resync_metadata';
-  | 'download_pending'
-  | 'update_subscribed'
-  | 'manual_import'
-  | 'resync_thumbs'
-  | 'resync_metadata';
 
 const updateTaskByName = async (taskName: TaskNamesType) => {
   return APIClient(`/api/task/by-name/${taskName}/`, {

frontend/src/pages/SettingsActions.tsx

@@ -9,6 +9,7 @@ import Button from '../components/Button';
 import { ApiResponseType } from '../functions/APIClient';
 import ToggleConfig from '../components/ToggleConfig';
 import queueStartFilesystemRescan from '../api/actions/queueStartFilesystemRescan';
+import queueManualImport from '../api/actions/queueManualImport';
 
 const SettingsActions = () => {
   const [deleteIgnored, setDeleteIgnored] = useState(false);
@@ -21,6 +22,8 @@ const SettingsActions = () => {
   const [reScanningFileSystem, setReScanningFileSystem] = useState(false);
   const [rescanIgnoreErrors, setRescanIgnoreErrors] = useState(false);
   const [rescanPreferLocal, setRescanPreferLocal] = useState(false);
+  const [manualPreferLocal, setManualPreferLocal] = useState(false);
+  const [manualIgnoreErrors, setManualIgnoreErrors] = useState(false);
 
   const [backupListResponse, setBackupListResponse] = useState<ApiResponseType<BackupListType>>();
 
@@ -73,22 +76,42 @@ const SettingsActions = () => {
           <h2>Manual media files import.</h2>
           <p>
             Add files to the <span className="settings-current">cache/import</span> folder. Make
-            sure to follow the instructions in the Github{' '}
+            sure to follow the instructions in the{' '}
             <a
               href="https://docs.tubearchivist.com/settings/actions/#manual-media-files-import"
               target="_blank"
             >
-              Wiki
+              Docs
             </a>
             .
           </p>
           <div id="manual-import">
+            <div className="settings-box-wrapper">
+              <div>
+                <p>Prefer embedded metadata</p>
+              </div>
+              <ToggleConfig
+                name="manual_prefer_local"
+                value={manualPreferLocal}
+                updateCallback={() => setManualPreferLocal(!manualPreferLocal)}
+              />
+            </div>
+            <div className="settings-box-wrapper">
+              <div>
+                <p>Ignore missing metadata errors</p>
+              </div>
+              <ToggleConfig
+                name="manual_ignore_error"
+                value={manualIgnoreErrors}
+                updateCallback={() => setManualIgnoreErrors(!manualIgnoreErrors)}
+              />
+            </div>
             {processingImports && <p>Processing import</p>}
             {!processingImports && (
               <Button
                 label="Start import"
                 onClick={async () => {
-                  await updateTaskByName('manual_import');
+                  await queueManualImport(manualIgnoreErrors, manualPreferLocal);
                   setProcessingImports(true);
                 }}
               />