RomanNum3ral
/
tubearchivist
mirror of https://github.com/tubearchivist/tubearchivist.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

use embedded for manual import, add error control

This commit is contained in:
Simon 2025-12-26 21:29:23 +07:00
parent 33d5f2e381
commit 6d377a1714
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
9 changed files with 151 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
backend/appsettings/serializers.py
View File

@ -107,6 +107,13 @@ class RescanFileSystemConfig(serializers.Serializer):
prefer_local = serializers.BooleanField()
class ManualImportConfig(serializers.Serializer):
"""serialize for manual import task"""
ignore_error = serializers.BooleanField()
prefer_local = serializers.BooleanField()
class SnapshotItemSerializer(serializers.Serializer):
"""serialize snapshot response"""

88
backend/appsettings/src/manual.py
View File

@ -18,6 +18,7 @@ from download.src.thumbnails import ThumbManager
from PIL import Image
from video.src.comments import Comments
from video.src.index import YoutubeVideo
from video.src.meta_embed import IndexFromEmbed
from yt_dlp.utils import ISO639Utils
@ -41,9 +42,16 @@ class ImportFolderScanner:
"subtitle": [".vtt"],
}
def __init__(self, task=False):
def __init__(
self,
task=False,
ignore_error: bool = False,
prefer_local: bool = False,
):
self.task = task
self.to_import = False
self.ignore_error = ignore_error
self.prefer_local = prefer_local
def scan(self):
"""scan and match media files"""
@ -142,11 +150,14 @@ class ImportFolderScanner:
self._convert_thumb(current_video)
self._get_subtitles(current_video)
self._convert_video(current_video)
print(f"manual import: {current_video}")
ManualImport(current_video, config).run()
Comments(current_video["video_id"]).build_json(upload=True)
YoutubeVideo(current_video["video_id"]).embed_metadata()
print(f"manual import: {current_video}")
ManualImport(
current_video,
config,
ignore_error=self.ignore_error,
prefer_local=self.prefer_local,
).run()
def _notify(self, idx, current_video):
"""send notification back to task"""
@ -182,17 +193,27 @@ class ImportFolderScanner:
expects filename ending in [<youtube_id>].<ext>
"""
base_name, _ = os.path.splitext(file_name)
# yt-dlp default like [youtubeid]
id_search = re.search(r"\[([a-zA-Z0-9_-]{11})\]$", base_name)
if id_search:
youtube_id = id_search.group(1)
return youtube_id
file_name_search = re.search(r"([a-zA-Z0-9_-]{11})$", base_name)
if file_name_search:
youtube_id = file_name_search.group(1)
return youtube_id
print(f"id extraction failed from filename: {file_name}")
return False
def _extract_id_from_json(self, json_file):
def _extract_id_from_json(self, json_file: str | bool) -> str | None:
"""open json file and extract id"""
if not json_file or not isinstance(json_file, str):
return None
json_path = os.path.join(self.CACHE_DIR, "import", json_file)
with open(json_path, "r", encoding="utf-8") as f:
json_content = f.read()
@ -385,17 +406,49 @@ class ImportFolderScanner:
class ManualImport:
"""import single identified video"""
def __init__(self, current_video, config):
def __init__(
self, current_video, config, ignore_error: bool, prefer_local: bool
):
self.current_video = current_video
self.config = config
self.ignore_error: bool = ignore_error
self.prefer_local: bool = prefer_local
def run(self):
"""run all"""
json_data = self.index_metadata()
self._move_to_archive(json_data)
self._cleanup(json_data)
json_data = None
if self.prefer_local:
# embedded first
json_data = IndexFromEmbed(
self.current_video["media"],
use_user_conf=False,
config=self.config,
).run_index()
if json_data:
self._cleanup()
return
def index_metadata(self):
try:
json_data = self.index_metadata()
except ValueError as err:
json_data = IndexFromEmbed(
self.current_video["media"],
use_user_conf=False,
config=self.config,
).run_index()
if not json_data and not self.ignore_error:
raise ValueError from err
if not json_data:
return
self._move_to_archive(json_data)
self._cleanup()
Comments(self.current_video["video_id"]).build_json(upload=True)
YoutubeVideo(self.current_video["video_id"]).embed_metadata()
def index_metadata(self) -> dict | None:
"""get metadata from yt or json"""
video_id = self.current_video["video_id"]
video = YoutubeVideo(video_id)
@ -408,6 +461,9 @@ class ManualImport:
f"{video_id}: manual import failed, and no metadata found."
)
print(message)
if self.ignore_error:
return None
raise ValueError(message)
video.check_subtitles(subtitle_files=self.current_video["subtitle"])
@ -460,7 +516,7 @@ class ManualImport:
new_path = f"{base_name}.{lang}.vtt"
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
def _cleanup(self, json_data):
def _cleanup(self):
"""cleanup leftover files"""
meta_data = self.current_video["metadata"]
if meta_data and os.path.exists(meta_data):
@ -473,11 +529,3 @@ class ManualImport:
for subtitle_file in self.current_video["subtitle"]:
if os.path.exists(subtitle_file):
os.remove(subtitle_file)
channel_info = os.path.join(
EnvironmentSettings.CACHE_DIR,
"import",
f"{json_data['channel']['channel_id']}.info.json",
)
if os.path.exists(channel_info):
os.remove(channel_info)

5
backend/appsettings/urls.py
View File

@ -49,6 +49,11 @@ urlpatterns = [
views.RescanFileSystem.as_view(),
name="api-rescan-filesystem",
),
path(
"manual-import/",
views.ManualImportView.as_view(),
name="api-manual-import",
),
path(
"membership/profile/",
views_mb.MembershipProfileView.as_view(),

31
backend/appsettings/views.py
View File

@ -5,6 +5,7 @@ from appsettings.serializers import (
BackupFileSerializer,
CookieUpdateSerializer,
CookieValidationSerializer,
ManualImportConfig,
PoTokenSerializer,
RescanFileSystemConfig,
SnapshotCreateResponseSerializer,
@ -395,9 +396,11 @@ class SnapshotApiListView(ApiBaseView):
class RescanFileSystem(ApiBaseView):
"""resolves to /api/appsettings/rescan-filesystem/
POST: start new task rescan filesystem task
POST: start new rescan filesystem task
"""
permission_classes = [AdminOnly]
@staticmethod
@extend_schema(
request=RescanFileSystemConfig,
@ -417,6 +420,32 @@ class RescanFileSystem(ApiBaseView):
return Response(serializer.data)
class ManualImportView(ApiBaseView):
"""resolves to /api/appsettings/manual-import/
POST: start new manual import task
"""
permission_classes = [AdminOnly]
@staticmethod
@extend_schema(
request=ManualImportConfig,
responses={
200: OpenApiResponse(AsyncTaskResponseSerializer()),
},
)
def post(request):
"""manual import"""
data_serializer = ManualImportConfig(data=request.data)
data_serializer.is_valid(raise_exception=True)
validated_data = data_serializer.validated_data
message = TaskCommand().start("manual_import", validated_data)
serializer = AsyncTaskResponseSerializer(message)
return Response(serializer.data)
class SnapshotApiView(ApiBaseView):
"""resolves to /api/appsettings/snapshot/<snapshot-id>/
GET: return a single snapshot

2
backend/task/src/task_config.py
View File

@ -48,7 +48,7 @@ CHECK_REINDEX: TaskItemConfig = {
MANUAL_IMPORT: TaskItemConfig = {
"title": "Manual video import",
"group": "setting:import",
"api_start": True,
"api_start": False,
"api_stop": False,
}

6
backend/task/tasks.py
View File

@ -216,7 +216,7 @@ def check_reindex(self, data=False, extract_videos=False):
@shared_task(bind=True, name="manual_import", base=BaseTask)
def run_manual_import(self):
def manual_import(self, ignore_error, prefer_local):
"""called from settings page, to go through import folder"""
manager = TaskManager()
if manager.is_pending(self):
@ -225,7 +225,9 @@ def run_manual_import(self):
return
manager.init(self)
ImportFolderScanner(task=self).scan()
ImportFolderScanner(
task=self, ignore_error=ignore_error, prefer_local=prefer_local
).scan()
@shared_task(bind=True, name="run_backup", base=BaseTask)

9
frontend/src/api/actions/queueManualImport.ts Normal file
View File

@ -0,0 +1,9 @@
import APIClient from '../../functions/APIClient';
const queueManualImport = async (ignore_error: boolean, prefer_local: boolean) => {
return APIClient('/api/appsettings/manual-import/', {
method: 'POST',
body: { ignore_error, prefer_local },
});
};
export default queueManualImport;

7
frontend/src/api/actions/updateTaskByName.ts
View File

@ -1,11 +1,6 @@
import APIClient from '../../functions/APIClient';
type TaskNamesType =
| 'download_pending'
| 'update_subscribed'
| 'manual_import'
| 'resync_thumbs'
| 'resync_metadata';
type TaskNamesType = 'download_pending' | 'update_subscribed' | 'resync_thumbs' | 'resync_metadata';
const updateTaskByName = async (taskName: TaskNamesType) => {
return APIClient(`/api/task/by-name/${taskName}/`, {

29
frontend/src/pages/SettingsActions.tsx
View File

@ -9,6 +9,7 @@ import Button from '../components/Button';
import { ApiResponseType } from '../functions/APIClient';
import ToggleConfig from '../components/ToggleConfig';
import queueStartFilesystemRescan from '../api/actions/queueStartFilesystemRescan';
import queueManualImport from '../api/actions/queueManualImport';
const SettingsActions = () => {
const [deleteIgnored, setDeleteIgnored] = useState(false);
@ -21,6 +22,8 @@ const SettingsActions = () => {
const [reScanningFileSystem, setReScanningFileSystem] = useState(false);
const [rescanIgnoreErrors, setRescanIgnoreErrors] = useState(false);
const [rescanPreferLocal, setRescanPreferLocal] = useState(false);
const [manualPreferLocal, setManualPreferLocal] = useState(false);
const [manualIgnoreErrors, setManualIgnoreErrors] = useState(false);
const [backupListResponse, setBackupListResponse] = useState<ApiResponseType<BackupListType>>();
@ -73,22 +76,42 @@ const SettingsActions = () => {
<h2>Manual media files import.</h2>
<p>
Add files to the <span className="settings-current">cache/import</span> folder. Make
sure to follow the instructions in the Github{' '}
sure to follow the instructions in the{' '}
<a
href="https://docs.tubearchivist.com/settings/actions/#manual-media-files-import"
target="_blank"
>
Wiki
Docs
</a>
.
</p>
<div id="manual-import">
<div className="settings-box-wrapper">
<div>
<p>Prefer embedded metadata</p>
</div>
<ToggleConfig
name="manual_prefer_local"
value={manualPreferLocal}
updateCallback={() => setManualPreferLocal(!manualPreferLocal)}
/>
</div>
<div className="settings-box-wrapper">
<div>
<p>Ignore missing metadata errors</p>
</div>
<ToggleConfig
name="manual_ignore_error"
value={manualIgnoreErrors}
updateCallback={() => setManualIgnoreErrors(!manualIgnoreErrors)}
/>
</div>
{processingImports && <p>Processing import</p>}
{!processingImports && (
<Button
label="Start import"
onClick={async () => {
await updateTaskByName('manual_import');
await queueManualImport(manualIgnoreErrors, manualPreferLocal);
setProcessingImports(true);
}}
/>