diff --git a/backend/appsettings/serializers.py b/backend/appsettings/serializers.py index 70564465..6da5ae1f 100644 --- a/backend/appsettings/serializers.py +++ b/backend/appsettings/serializers.py @@ -100,6 +100,13 @@ class PoTokenSerializer(serializers.Serializer): potoken = serializers.CharField() +class RescanFileSystemConfig(serializers.Serializer): + """serialize rescan filesystem config""" + + ignore_error = serializers.BooleanField() + prefer_local = serializers.BooleanField() + + class SnapshotItemSerializer(serializers.Serializer): """serialize snapshot response""" diff --git a/backend/appsettings/src/filesystem.py b/backend/appsettings/src/filesystem.py index 0a609289..13324331 100644 --- a/backend/appsettings/src/filesystem.py +++ b/backend/appsettings/src/filesystem.py @@ -5,11 +5,13 @@ Functionality: import os +from appsettings.src.config import AppConfig from common.src.env_settings import EnvironmentSettings from common.src.es_connect import IndexPaginate -from common.src.helper import ignore_filelist +from common.src.helper import ignore_filelist, rand_sleep from video.src.comments import Comments from video.src.index import YoutubeVideo, index_new_video +from video.src.meta_embed import IndexFromEmbed class Scanner: @@ -17,19 +19,27 @@ class Scanner: VIDEOS: str = EnvironmentSettings.MEDIA_DIR - def __init__(self, task=False) -> None: + def __init__( + self, + task=False, + ignore_error: bool = False, + prefer_local: bool = False, + ) -> None: self.task = task - self.to_delete: set[str] = set() - self.to_index: set[str] = set() + self.ignore_error = ignore_error + self.prefer_local = prefer_local + self.config = None + self.to_delete: set[tuple[str, str]] = set() + self.to_index: set[tuple[str, str]] = set() def scan(self) -> None: """scan the filesystem""" - downloaded: set[str] = self._get_downloaded() - indexed: set[str] = self._get_indexed() + downloaded = self._get_downloaded() + indexed = self._get_indexed() self.to_index = downloaded - indexed self.to_delete = indexed - downloaded - def _get_downloaded(self) -> set[str]: + def _get_downloaded(self) -> set[tuple[str, str]]: """get downloaded ids""" if self.task: self.task.send_progress(["Scan your filesystem for videos."]) @@ -39,28 +49,40 @@ class Scanner: for channel in channels: folder = os.path.join(self.VIDEOS, channel) files = ignore_filelist(os.listdir(folder)) - downloaded.update({i.split(".")[0] for i in files}) + downloaded.update( + { + (i.split(".")[0], f"{channel}/{i}") + for i in files + if i.endswith(".mp4") + } + ) return downloaded - def _get_indexed(self) -> set: + def _get_indexed(self) -> set[tuple[str, str]]: """get all indexed ids""" if self.task: self.task.send_progress(["Get all videos indexed."]) - data = {"query": {"match_all": {}}, "_source": ["youtube_id"]} + data = { + "query": {"match_all": {}}, + "_source": ["youtube_id", "media_url"], + } response = IndexPaginate("ta_video", data).get_results() - return {i["youtube_id"] for i in response} + return {(i["youtube_id"], i["media_url"]) for i in response} def apply(self) -> None: """apply all changes""" + if not self.config: + self.config = AppConfig().config + self.delete() self.index() def delete(self) -> None: """delete videos from index""" if not self.to_delete: - print("nothing to delete") + print("[scanner] nothing to delete") return if self.task: @@ -68,25 +90,69 @@ class Scanner: [f"Remove {len(self.to_delete)} videos from index."] ) - for youtube_id in self.to_delete: + for youtube_id, _ in self.to_delete: YoutubeVideo(youtube_id).delete_media_file() def index(self) -> None: """index new""" if not self.to_index: - print("nothing to index") + print("[scanner] nothing to index") return total = len(self.to_index) - for idx, youtube_id in enumerate(self.to_index): - if self.task: - self.task.send_progress( - message_lines=[ - f"Index missing video {youtube_id}, {idx + 1}/{total}" - ], - progress=(idx + 1) / total, - ) - json_data = index_new_video(youtube_id) - if json_data: + for idx, (youtube_id, media_url) in enumerate(self.to_index): + self._notify(total, youtube_id, idx) + + file_path = os.path.join(self.VIDEOS, media_url) + if self.prefer_local: + # try index from embed + json_data = IndexFromEmbed( + file_path, use_user_conf=True, config=self.config + ).run_index() + if json_data: + continue + + try: + # try index from remote + json_data = index_new_video(youtube_id) Comments(youtube_id).build_json(upload=True) YoutubeVideo(youtube_id).embed_metadata() + rand_sleep(self.config) + except ValueError as err: + # fallback from index from embed + json_data = IndexFromEmbed( + file_path, use_user_conf=True, config=self.config + ).run_index() + if json_data: + continue + + if self.ignore_error: + self._notify_error(youtube_id) + rand_sleep(self.config) + continue + + raise ValueError from err + + def _notify(self, total, youtube_id, idx): + """send notification""" + if not self.task: + return + + self.task.send_progress( + message_lines=[ + f"Index missing video {youtube_id}, {idx + 1}/{total}" + ], + progress=(idx + 1) / total, + ) + + def _notify_error(self, youtube_id): + """notify error""" + if not self.task: + return + + message = f"[scanner] Failed to index {youtube_id}, no metadata" + print(f"[scanner] {message}") + self.task.send_progress( + message_lines=[message, "Continue..."], + level="error", + ) diff --git a/backend/appsettings/urls.py b/backend/appsettings/urls.py index 4dde4188..d4f96d4f 100644 --- a/backend/appsettings/urls.py +++ b/backend/appsettings/urls.py @@ -44,6 +44,11 @@ urlpatterns = [ views.TokenView.as_view(), name="api-token", ), + path( + "rescan-filesystem/", + views.RescanFileSystem.as_view(), + name="api-rescan-filesystem", + ), path( "membership/profile/", views_mb.MembershipProfileView.as_view(), diff --git a/backend/appsettings/views.py b/backend/appsettings/views.py index edb87ca3..cb909219 100644 --- a/backend/appsettings/views.py +++ b/backend/appsettings/views.py @@ -6,6 +6,7 @@ from appsettings.serializers import ( CookieUpdateSerializer, CookieValidationSerializer, PoTokenSerializer, + RescanFileSystemConfig, SnapshotCreateResponseSerializer, SnapshotItemSerializer, SnapshotListSerializer, @@ -291,7 +292,11 @@ class CookieView(ApiBaseView): class POTokenView(ApiBaseView): - """handle PO token""" + """resolves to /api/appsettings/potoken/ + GET: get potoken + POST: update potoken + DELETE: revoke potoken + """ permission_classes = [AdminOnly] @@ -388,6 +393,30 @@ class SnapshotApiListView(ApiBaseView): return Response(serializer.data) +class RescanFileSystem(ApiBaseView): + """resolves to /api/appsettings/rescan-filesystem/ + POST: start new task rescan filesystem task + """ + + @staticmethod + @extend_schema( + request=RescanFileSystemConfig, + responses={ + 200: OpenApiResponse(AsyncTaskResponseSerializer()), + }, + ) + def post(request): + """start new task rescan filesystem task""" + data_serializer = RescanFileSystemConfig(data=request.data) + data_serializer.is_valid(raise_exception=True) + validated_data = data_serializer.validated_data + + message = TaskCommand().start("rescan_filesystem", validated_data) + serializer = AsyncTaskResponseSerializer(message) + + return Response(serializer.data) + + class SnapshotApiView(ApiBaseView): """resolves to /api/appsettings/snapshot// GET: return a single snapshot diff --git a/backend/task/src/task_config.py b/backend/task/src/task_config.py index 9fd494e9..d2e3b02e 100644 --- a/backend/task/src/task_config.py +++ b/backend/task/src/task_config.py @@ -69,7 +69,7 @@ RESTORE_BACKUP: TaskItemConfig = { RESCAN_FILESYSTEM: TaskItemConfig = { "title": "Rescan your Filesystem", "group": "setting:filesystemscan", - "api_start": True, + "api_start": False, "api_stop": False, } diff --git a/backend/task/src/task_manager.py b/backend/task/src/task_manager.py index ea48e5e8..81b612ec 100644 --- a/backend/task/src/task_manager.py +++ b/backend/task/src/task_manager.py @@ -84,9 +84,9 @@ class TaskManager: class TaskCommand: """run commands on task""" - def start(self, task_name): + def start(self, task_name, kwargs: dict | None = None): """start task by task_name, only pass task that don't take args""" - task = celery_app.tasks.get(task_name).delay() + task = celery_app.tasks.get(task_name).delay(**(kwargs or {})) message = { "task_id": task.id, "status": task.status, diff --git a/backend/task/tasks.py b/backend/task/tasks.py index 995e1c85..914e4036 100644 --- a/backend/task/tasks.py +++ b/backend/task/tasks.py @@ -260,7 +260,7 @@ def run_restore_backup(self, filename): @shared_task(bind=True, name="rescan_filesystem", base=BaseTask) -def rescan_filesystem(self): +def rescan_filesystem(self, ignore_error, prefer_local): """check the media folder for mismatches""" manager = TaskManager() if manager.is_pending(self): @@ -269,7 +269,9 @@ def rescan_filesystem(self): return manager.init(self) - handler = Scanner(task=self) + handler = Scanner( + task=self, ignore_error=ignore_error, prefer_local=prefer_local + ) handler.scan() handler.apply() thumbnail_check.delay() diff --git a/backend/video/serializers.py b/backend/video/serializers.py index bc90a1c9..5a5d7e3e 100644 --- a/backend/video/serializers.py +++ b/backend/video/serializers.py @@ -100,7 +100,9 @@ class VideoSerializer(serializers.Serializer): media_size = serializers.IntegerField() media_url = serializers.CharField() player = PlayerSerializer() - playlist = serializers.ListField(child=serializers.CharField()) + playlist = serializers.ListField( + child=serializers.CharField(), required=False + ) published = serializers.CharField() sponsorblock = SponsorBlockSerializer(allow_null=True, required=False) stats = StatsSerializer() diff --git a/backend/video/src/meta_embed.py b/backend/video/src/meta_embed.py index e3be281c..a568d04f 100644 --- a/backend/video/src/meta_embed.py +++ b/backend/video/src/meta_embed.py @@ -90,14 +90,14 @@ class IndexFromEmbed: self.use_user_conf = use_user_conf self.config = config - def run_index(self) -> None: + def run_index(self) -> None | dict: """run index""" if not self.config: self.config = AppConfig().config json_embed = self._get_embedded() if not json_embed: - return + return None channel_data_clean = self.index_channel(json_embed) video = self.index_video(json_embed, channel_data_clean) @@ -105,6 +105,8 @@ class IndexFromEmbed: self.index_subtitles(json_embed, video) self.index_comments(json_embed) + return video.json_data + def _get_embedded(self) -> dict | None: """get embedded metadata""" video_mutagen = MP4(self.file_path) diff --git a/frontend/src/api/actions/queueStartFilesystemRescan.ts b/frontend/src/api/actions/queueStartFilesystemRescan.ts new file mode 100644 index 00000000..e6eeab0b --- /dev/null +++ b/frontend/src/api/actions/queueStartFilesystemRescan.ts @@ -0,0 +1,9 @@ +import APIClient from '../../functions/APIClient'; + +const queueStartFilesystemRescan = async (ignore_error: boolean, prefer_local: boolean) => { + return APIClient('/api/appsettings/rescan-filesystem/', { + method: 'POST', + body: { ignore_error, prefer_local }, + }); +}; +export default queueStartFilesystemRescan; diff --git a/frontend/src/api/actions/updateTaskByName.ts b/frontend/src/api/actions/updateTaskByName.ts index 7053dca9..dabd492b 100644 --- a/frontend/src/api/actions/updateTaskByName.ts +++ b/frontend/src/api/actions/updateTaskByName.ts @@ -5,8 +5,7 @@ type TaskNamesType = | 'update_subscribed' | 'manual_import' | 'resync_thumbs' - | 'resync_metadata' - | 'rescan_filesystem'; + | 'resync_metadata'; const updateTaskByName = async (taskName: TaskNamesType) => { return APIClient(`/api/task/by-name/${taskName}/`, { diff --git a/frontend/src/pages/SettingsActions.tsx b/frontend/src/pages/SettingsActions.tsx index a226019d..9104dab7 100644 --- a/frontend/src/pages/SettingsActions.tsx +++ b/frontend/src/pages/SettingsActions.tsx @@ -7,6 +7,8 @@ import restoreBackup from '../api/actions/restoreBackup'; import Notifications from '../components/Notifications'; import Button from '../components/Button'; import { ApiResponseType } from '../functions/APIClient'; +import ToggleConfig from '../components/ToggleConfig'; +import queueStartFilesystemRescan from '../api/actions/queueStartFilesystemRescan'; const SettingsActions = () => { const [deleteIgnored, setDeleteIgnored] = useState(false); @@ -17,6 +19,8 @@ const SettingsActions = () => { const [backupStarted, setBackupStarted] = useState(false); const [isRestoringBackup, setIsRestoringBackup] = useState(false); const [reScanningFileSystem, setReScanningFileSystem] = useState(false); + const [rescanIgnoreErrors, setRescanIgnoreErrors] = useState(false); + const [rescanPreferLocal, setRescanPreferLocal] = useState(false); const [backupListResponse, setBackupListResponse] = useState>(); @@ -196,23 +200,42 @@ const SettingsActions = () => { deleted videos from the filesystem.

- Rescan your media folder looking for missing videos and clean up index. More info on the - Github{' '} + Rescan your media folder looking for missing videos and clean up index. More info on the{' '} - Wiki + Docs .

+
+
+

Prefer embedded metadata

+
+ setRescanPreferLocal(!rescanPreferLocal)} + /> +
+
+
+

Ignore missing metadata errors

+
+ setRescanIgnoreErrors(!rescanIgnoreErrors)} + /> +
{reScanningFileSystem &&

File system scan in progress

} {!reScanningFileSystem && (