RomanNum3ral
/
tubearchivist
mirror of https://github.com/tubearchivist/tubearchivist.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add rescan filesystem from embed, ignore errors options

This commit is contained in:
Simon 2025-12-26 17:25:40 +07:00
parent 7fa776a919
commit 33d5f2e381
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
12 changed files with 183 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
backend/appsettings/serializers.py
View File

@ -100,6 +100,13 @@ class PoTokenSerializer(serializers.Serializer):
potoken = serializers.CharField()
class RescanFileSystemConfig(serializers.Serializer):
"""serialize rescan filesystem config"""
ignore_error = serializers.BooleanField()
prefer_local = serializers.BooleanField()
class SnapshotItemSerializer(serializers.Serializer):
"""serialize snapshot response"""

114
backend/appsettings/src/filesystem.py
View File

@ -5,11 +5,13 @@ Functionality:
import os
from appsettings.src.config import AppConfig
from common.src.env_settings import EnvironmentSettings
from common.src.es_connect import IndexPaginate
from common.src.helper import ignore_filelist
from common.src.helper import ignore_filelist, rand_sleep
from video.src.comments import Comments
from video.src.index import YoutubeVideo, index_new_video
from video.src.meta_embed import IndexFromEmbed
class Scanner:
@ -17,19 +19,27 @@ class Scanner:
VIDEOS: str = EnvironmentSettings.MEDIA_DIR
def __init__(self, task=False) -> None:
def __init__(
self,
task=False,
ignore_error: bool = False,
prefer_local: bool = False,
) -> None:
self.task = task
self.to_delete: set[str] = set()
self.to_index: set[str] = set()
self.ignore_error = ignore_error
self.prefer_local = prefer_local
self.config = None
self.to_delete: set[tuple[str, str]] = set()
self.to_index: set[tuple[str, str]] = set()
def scan(self) -> None:
"""scan the filesystem"""
downloaded: set[str] = self._get_downloaded()
indexed: set[str] = self._get_indexed()
downloaded = self._get_downloaded()
indexed = self._get_indexed()
self.to_index = downloaded - indexed
self.to_delete = indexed - downloaded
def _get_downloaded(self) -> set[str]:
def _get_downloaded(self) -> set[tuple[str, str]]:
"""get downloaded ids"""
if self.task:
self.task.send_progress(["Scan your filesystem for videos."])
@ -39,28 +49,40 @@ class Scanner:
for channel in channels:
folder = os.path.join(self.VIDEOS, channel)
files = ignore_filelist(os.listdir(folder))
downloaded.update({i.split(".")[0] for i in files})
downloaded.update(
{
(i.split(".")[0], f"{channel}/{i}")
for i in files
if i.endswith(".mp4")
}
)
return downloaded
def _get_indexed(self) -> set:
def _get_indexed(self) -> set[tuple[str, str]]:
"""get all indexed ids"""
if self.task:
self.task.send_progress(["Get all videos indexed."])
data = {"query": {"match_all": {}}, "_source": ["youtube_id"]}
data = {
"query": {"match_all": {}},
"_source": ["youtube_id", "media_url"],
}
response = IndexPaginate("ta_video", data).get_results()
return {i["youtube_id"] for i in response}
return {(i["youtube_id"], i["media_url"]) for i in response}
def apply(self) -> None:
"""apply all changes"""
if not self.config:
self.config = AppConfig().config
self.delete()
self.index()
def delete(self) -> None:
"""delete videos from index"""
if not self.to_delete:
print("nothing to delete")
print("[scanner] nothing to delete")
return
if self.task:
@ -68,25 +90,69 @@ class Scanner:
[f"Remove {len(self.to_delete)} videos from index."]
)
for youtube_id in self.to_delete:
for youtube_id, _ in self.to_delete:
YoutubeVideo(youtube_id).delete_media_file()
def index(self) -> None:
"""index new"""
if not self.to_index:
print("nothing to index")
print("[scanner] nothing to index")
return
total = len(self.to_index)
for idx, youtube_id in enumerate(self.to_index):
if self.task:
self.task.send_progress(
message_lines=[
f"Index missing video {youtube_id}, {idx + 1}/{total}"
],
progress=(idx + 1) / total,
)
json_data = index_new_video(youtube_id)
if json_data:
for idx, (youtube_id, media_url) in enumerate(self.to_index):
self._notify(total, youtube_id, idx)
file_path = os.path.join(self.VIDEOS, media_url)
if self.prefer_local:
# try index from embed
json_data = IndexFromEmbed(
file_path, use_user_conf=True, config=self.config
).run_index()
if json_data:
continue
try:
# try index from remote
json_data = index_new_video(youtube_id)
Comments(youtube_id).build_json(upload=True)
YoutubeVideo(youtube_id).embed_metadata()
rand_sleep(self.config)
except ValueError as err:
# fallback from index from embed
json_data = IndexFromEmbed(
file_path, use_user_conf=True, config=self.config
).run_index()
if json_data:
continue
if self.ignore_error:
self._notify_error(youtube_id)
rand_sleep(self.config)
continue
raise ValueError from err
def _notify(self, total, youtube_id, idx):
"""send notification"""
if not self.task:
return
self.task.send_progress(
message_lines=[
f"Index missing video {youtube_id}, {idx + 1}/{total}"
],
progress=(idx + 1) / total,
)
def _notify_error(self, youtube_id):
"""notify error"""
if not self.task:
return
message = f"[scanner] Failed to index {youtube_id}, no metadata"
print(f"[scanner] {message}")
self.task.send_progress(
message_lines=[message, "Continue..."],
level="error",
)

5
backend/appsettings/urls.py
View File

@ -44,6 +44,11 @@ urlpatterns = [
views.TokenView.as_view(),
name="api-token",
),
path(
"rescan-filesystem/",
views.RescanFileSystem.as_view(),
name="api-rescan-filesystem",
),
path(
"membership/profile/",
views_mb.MembershipProfileView.as_view(),

31
backend/appsettings/views.py
View File

@ -6,6 +6,7 @@ from appsettings.serializers import (
CookieUpdateSerializer,
CookieValidationSerializer,
PoTokenSerializer,
RescanFileSystemConfig,
SnapshotCreateResponseSerializer,
SnapshotItemSerializer,
SnapshotListSerializer,
@ -291,7 +292,11 @@ class CookieView(ApiBaseView):
class POTokenView(ApiBaseView):
"""handle PO token"""
"""resolves to /api/appsettings/potoken/
GET: get potoken
POST: update potoken
DELETE: revoke potoken
"""
permission_classes = [AdminOnly]
@ -388,6 +393,30 @@ class SnapshotApiListView(ApiBaseView):
return Response(serializer.data)
class RescanFileSystem(ApiBaseView):
"""resolves to /api/appsettings/rescan-filesystem/
POST: start new task rescan filesystem task
"""
@staticmethod
@extend_schema(
request=RescanFileSystemConfig,
responses={
200: OpenApiResponse(AsyncTaskResponseSerializer()),
},
)
def post(request):
"""start new task rescan filesystem task"""
data_serializer = RescanFileSystemConfig(data=request.data)
data_serializer.is_valid(raise_exception=True)
validated_data = data_serializer.validated_data
message = TaskCommand().start("rescan_filesystem", validated_data)
serializer = AsyncTaskResponseSerializer(message)
return Response(serializer.data)
class SnapshotApiView(ApiBaseView):
"""resolves to /api/appsettings/snapshot/<snapshot-id>/
GET: return a single snapshot

2
backend/task/src/task_config.py
View File

@ -69,7 +69,7 @@ RESTORE_BACKUP: TaskItemConfig = {
RESCAN_FILESYSTEM: TaskItemConfig = {
"title": "Rescan your Filesystem",
"group": "setting:filesystemscan",
"api_start": True,
"api_start": False,
"api_stop": False,
}

4
backend/task/src/task_manager.py
View File

@ -84,9 +84,9 @@ class TaskManager:
class TaskCommand:
"""run commands on task"""
def start(self, task_name):
def start(self, task_name, kwargs: dict | None = None):
"""start task by task_name, only pass task that don't take args"""
task = celery_app.tasks.get(task_name).delay()
task = celery_app.tasks.get(task_name).delay(**(kwargs or {}))
message = {
"task_id": task.id,
"status": task.status,

6
backend/task/tasks.py
View File

@ -260,7 +260,7 @@ def run_restore_backup(self, filename):
@shared_task(bind=True, name="rescan_filesystem", base=BaseTask)
def rescan_filesystem(self):
def rescan_filesystem(self, ignore_error, prefer_local):
"""check the media folder for mismatches"""
manager = TaskManager()
if manager.is_pending(self):
@ -269,7 +269,9 @@ def rescan_filesystem(self):
return
manager.init(self)
handler = Scanner(task=self)
handler = Scanner(
task=self, ignore_error=ignore_error, prefer_local=prefer_local
)
handler.scan()
handler.apply()
thumbnail_check.delay()

4
backend/video/serializers.py
View File

@ -100,7 +100,9 @@ class VideoSerializer(serializers.Serializer):
media_size = serializers.IntegerField()
media_url = serializers.CharField()
player = PlayerSerializer()
playlist = serializers.ListField(child=serializers.CharField())
playlist = serializers.ListField(
child=serializers.CharField(), required=False
)
published = serializers.CharField()
sponsorblock = SponsorBlockSerializer(allow_null=True, required=False)
stats = StatsSerializer()

6
backend/video/src/meta_embed.py
View File

@ -90,14 +90,14 @@ class IndexFromEmbed:
self.use_user_conf = use_user_conf
self.config = config
def run_index(self) -> None:
def run_index(self) -> None | dict:
"""run index"""
if not self.config:
self.config = AppConfig().config
json_embed = self._get_embedded()
if not json_embed:
return
return None
channel_data_clean = self.index_channel(json_embed)
video = self.index_video(json_embed, channel_data_clean)
@ -105,6 +105,8 @@ class IndexFromEmbed:
self.index_subtitles(json_embed, video)
self.index_comments(json_embed)
return video.json_data
def _get_embedded(self) -> dict | None:
"""get embedded metadata"""
video_mutagen = MP4(self.file_path)

9
frontend/src/api/actions/queueStartFilesystemRescan.ts Normal file
View File

@ -0,0 +1,9 @@
import APIClient from '../../functions/APIClient';
const queueStartFilesystemRescan = async (ignore_error: boolean, prefer_local: boolean) => {
return APIClient('/api/appsettings/rescan-filesystem/', {
method: 'POST',
body: { ignore_error, prefer_local },
});
};
export default queueStartFilesystemRescan;

3
frontend/src/api/actions/updateTaskByName.ts
View File

@ -5,8 +5,7 @@ type TaskNamesType =
| 'update_subscribed'
| 'manual_import'
| 'resync_thumbs'
| 'resync_metadata'
| 'rescan_filesystem';
| 'resync_metadata';
const updateTaskByName = async (taskName: TaskNamesType) => {
return APIClient(`/api/task/by-name/${taskName}/`, {

31
frontend/src/pages/SettingsActions.tsx
View File

@ -7,6 +7,8 @@ import restoreBackup from '../api/actions/restoreBackup';
import Notifications from '../components/Notifications';
import Button from '../components/Button';
import { ApiResponseType } from '../functions/APIClient';
import ToggleConfig from '../components/ToggleConfig';
import queueStartFilesystemRescan from '../api/actions/queueStartFilesystemRescan';
const SettingsActions = () => {
const [deleteIgnored, setDeleteIgnored] = useState(false);
@ -17,6 +19,8 @@ const SettingsActions = () => {
const [backupStarted, setBackupStarted] = useState(false);
const [isRestoringBackup, setIsRestoringBackup] = useState(false);
const [reScanningFileSystem, setReScanningFileSystem] = useState(false);
const [rescanIgnoreErrors, setRescanIgnoreErrors] = useState(false);
const [rescanPreferLocal, setRescanPreferLocal] = useState(false);
const [backupListResponse, setBackupListResponse] = useState<ApiResponseType<BackupListType>>();
@ -196,23 +200,42 @@ const SettingsActions = () => {
deleted videos from the filesystem.
</p>
<p>
Rescan your media folder looking for missing videos and clean up index. More info on the
Github{' '}
Rescan your media folder looking for missing videos and clean up index. More info on the{' '}
<a
href="https://docs.tubearchivist.com/settings/actions/#rescan-filesystem"
target="_blank"
>
Wiki
Docs
</a>
.
</p>
<div id="fs-rescan">
<div className="settings-box-wrapper">
<div>
<p>Prefer embedded metadata</p>
</div>
<ToggleConfig
name="prefer_local"
value={rescanPreferLocal}
updateCallback={() => setRescanPreferLocal(!rescanPreferLocal)}
/>
</div>
<div className="settings-box-wrapper">
<div>
<p>Ignore missing metadata errors</p>
</div>
<ToggleConfig
name="ignore_error"
value={rescanIgnoreErrors}
updateCallback={() => setRescanIgnoreErrors(!rescanIgnoreErrors)}
/>
</div>
{reScanningFileSystem && <p>File system scan in progress</p>}
{!reScanningFileSystem && (
<Button
label="Rescan filesystem"
onClick={async () => {
await updateTaskByName('rescan_filesystem');
await queueStartFilesystemRescan(rescanIgnoreErrors, rescanPreferLocal);
setReScanningFileSystem(true);
}}
/>