Implement restore from embed, #build
Changed: - Added restore from embedded metadata - Extend and error handling for embedding - Embed artwork - Add error handling for rescan filesystem and manual import
This commit is contained in:
Simon
commit
b97a990237
|
|
@ -100,6 +100,20 @@ class PoTokenSerializer(serializers.Serializer):
|
|||
potoken = serializers.CharField()
|
||||
|
||||
|
||||
class RescanFileSystemConfig(serializers.Serializer):
|
||||
"""serialize rescan filesystem config"""
|
||||
|
||||
ignore_error = serializers.BooleanField()
|
||||
prefer_local = serializers.BooleanField()
|
||||
|
||||
|
||||
class ManualImportConfig(serializers.Serializer):
|
||||
"""serialize for manual import task"""
|
||||
|
||||
ignore_error = serializers.BooleanField()
|
||||
prefer_local = serializers.BooleanField()
|
||||
|
||||
|
||||
class SnapshotItemSerializer(serializers.Serializer):
|
||||
"""serialize snapshot response"""
|
||||
|
||||
|
|
|
|||
|
|
@ -5,11 +5,13 @@ Functionality:
|
|||
|
||||
import os
|
||||
|
||||
from appsettings.src.config import AppConfig
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import IndexPaginate
|
||||
from common.src.helper import ignore_filelist
|
||||
from video.src.comments import CommentList
|
||||
from common.src.helper import ignore_filelist, rand_sleep
|
||||
from video.src.comments import Comments
|
||||
from video.src.index import YoutubeVideo, index_new_video
|
||||
from video.src.meta_embed import IndexFromEmbed
|
||||
|
||||
|
||||
class Scanner:
|
||||
|
|
@ -17,19 +19,27 @@ class Scanner:
|
|||
|
||||
VIDEOS: str = EnvironmentSettings.MEDIA_DIR
|
||||
|
||||
def __init__(self, task=False) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
task=False,
|
||||
ignore_error: bool = False,
|
||||
prefer_local: bool = False,
|
||||
) -> None:
|
||||
self.task = task
|
||||
self.to_delete: set[str] = set()
|
||||
self.to_index: set[str] = set()
|
||||
self.ignore_error = ignore_error
|
||||
self.prefer_local = prefer_local
|
||||
self.config = None
|
||||
self.to_delete: set[tuple[str, str]] = set()
|
||||
self.to_index: set[tuple[str, str]] = set()
|
||||
|
||||
def scan(self) -> None:
|
||||
"""scan the filesystem"""
|
||||
downloaded: set[str] = self._get_downloaded()
|
||||
indexed: set[str] = self._get_indexed()
|
||||
downloaded = self._get_downloaded()
|
||||
indexed = self._get_indexed()
|
||||
self.to_index = downloaded - indexed
|
||||
self.to_delete = indexed - downloaded
|
||||
|
||||
def _get_downloaded(self) -> set[str]:
|
||||
def _get_downloaded(self) -> set[tuple[str, str]]:
|
||||
"""get downloaded ids"""
|
||||
if self.task:
|
||||
self.task.send_progress(["Scan your filesystem for videos."])
|
||||
|
|
@ -39,28 +49,40 @@ class Scanner:
|
|||
for channel in channels:
|
||||
folder = os.path.join(self.VIDEOS, channel)
|
||||
files = ignore_filelist(os.listdir(folder))
|
||||
downloaded.update({i.split(".")[0] for i in files})
|
||||
downloaded.update(
|
||||
{
|
||||
(i.split(".")[0], f"{channel}/{i}")
|
||||
for i in files
|
||||
if i.endswith(".mp4")
|
||||
}
|
||||
)
|
||||
|
||||
return downloaded
|
||||
|
||||
def _get_indexed(self) -> set:
|
||||
def _get_indexed(self) -> set[tuple[str, str]]:
|
||||
"""get all indexed ids"""
|
||||
if self.task:
|
||||
self.task.send_progress(["Get all videos indexed."])
|
||||
|
||||
data = {"query": {"match_all": {}}, "_source": ["youtube_id"]}
|
||||
data = {
|
||||
"query": {"match_all": {}},
|
||||
"_source": ["youtube_id", "media_url"],
|
||||
}
|
||||
response = IndexPaginate("ta_video", data).get_results()
|
||||
return {i["youtube_id"] for i in response}
|
||||
return {(i["youtube_id"], i["media_url"]) for i in response}
|
||||
|
||||
def apply(self) -> None:
|
||||
"""apply all changes"""
|
||||
if not self.config:
|
||||
self.config = AppConfig().config
|
||||
|
||||
self.delete()
|
||||
self.index()
|
||||
|
||||
def delete(self) -> None:
|
||||
"""delete videos from index"""
|
||||
if not self.to_delete:
|
||||
print("nothing to delete")
|
||||
print("[scanner] nothing to delete")
|
||||
return
|
||||
|
||||
if self.task:
|
||||
|
|
@ -68,26 +90,69 @@ class Scanner:
|
|||
[f"Remove {len(self.to_delete)} videos from index."]
|
||||
)
|
||||
|
||||
for youtube_id in self.to_delete:
|
||||
for youtube_id, _ in self.to_delete:
|
||||
YoutubeVideo(youtube_id).delete_media_file()
|
||||
|
||||
def index(self) -> None:
|
||||
"""index new"""
|
||||
if not self.to_index:
|
||||
print("nothing to index")
|
||||
print("[scanner] nothing to index")
|
||||
return
|
||||
|
||||
total = len(self.to_index)
|
||||
for idx, youtube_id in enumerate(self.to_index):
|
||||
if self.task:
|
||||
self.task.send_progress(
|
||||
message_lines=[
|
||||
f"Index missing video {youtube_id}, {idx + 1}/{total}"
|
||||
],
|
||||
progress=(idx + 1) / total,
|
||||
)
|
||||
index_new_video(youtube_id)
|
||||
for idx, (youtube_id, media_url) in enumerate(self.to_index):
|
||||
self._notify(total, youtube_id, idx)
|
||||
|
||||
comment_list = CommentList(task=self.task)
|
||||
comment_list.add(video_ids=list(self.to_index))
|
||||
comment_list.index()
|
||||
file_path = os.path.join(self.VIDEOS, media_url)
|
||||
if self.prefer_local:
|
||||
# try index from embed
|
||||
json_data = IndexFromEmbed(
|
||||
file_path, use_user_conf=True, config=self.config
|
||||
).run_index()
|
||||
if json_data:
|
||||
continue
|
||||
|
||||
try:
|
||||
# try index from remote
|
||||
json_data = index_new_video(youtube_id)
|
||||
Comments(youtube_id).build_json(upload=True)
|
||||
YoutubeVideo(youtube_id).embed_metadata()
|
||||
rand_sleep(self.config)
|
||||
except ValueError as err:
|
||||
# fallback from index from embed
|
||||
json_data = IndexFromEmbed(
|
||||
file_path, use_user_conf=True, config=self.config
|
||||
).run_index()
|
||||
if json_data:
|
||||
continue
|
||||
|
||||
if self.ignore_error:
|
||||
self._notify_error(youtube_id)
|
||||
rand_sleep(self.config)
|
||||
continue
|
||||
|
||||
raise ValueError from err
|
||||
|
||||
def _notify(self, total, youtube_id, idx):
|
||||
"""send notification"""
|
||||
if not self.task:
|
||||
return
|
||||
|
||||
self.task.send_progress(
|
||||
message_lines=[
|
||||
f"Index missing video {youtube_id}, {idx + 1}/{total}"
|
||||
],
|
||||
progress=(idx + 1) / total,
|
||||
)
|
||||
|
||||
def _notify_error(self, youtube_id):
|
||||
"""notify error"""
|
||||
if not self.task:
|
||||
return
|
||||
|
||||
message = f"[scanner] Failed to index {youtube_id}, no metadata"
|
||||
print(f"[scanner] {message}")
|
||||
self.task.send_progress(
|
||||
message_lines=[message, "Continue..."],
|
||||
level="error",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -16,8 +16,9 @@ from common.src.env_settings import EnvironmentSettings
|
|||
from common.src.helper import ignore_filelist
|
||||
from download.src.thumbnails import ThumbManager
|
||||
from PIL import Image
|
||||
from video.src.comments import CommentList
|
||||
from video.src.comments import Comments
|
||||
from video.src.index import YoutubeVideo
|
||||
from video.src.meta_embed import IndexFromEmbed
|
||||
from yt_dlp.utils import ISO639Utils
|
||||
|
||||
|
||||
|
|
@ -41,9 +42,16 @@ class ImportFolderScanner:
|
|||
"subtitle": [".vtt"],
|
||||
}
|
||||
|
||||
def __init__(self, task=False):
|
||||
def __init__(
|
||||
self,
|
||||
task=False,
|
||||
ignore_error: bool = False,
|
||||
prefer_local: bool = False,
|
||||
):
|
||||
self.task = task
|
||||
self.to_import = False
|
||||
self.ignore_error = ignore_error
|
||||
self.prefer_local = prefer_local
|
||||
|
||||
def scan(self):
|
||||
"""scan and match media files"""
|
||||
|
|
@ -142,14 +150,14 @@ class ImportFolderScanner:
|
|||
self._convert_thumb(current_video)
|
||||
self._get_subtitles(current_video)
|
||||
self._convert_video(current_video)
|
||||
|
||||
print(f"manual import: {current_video}")
|
||||
|
||||
ManualImport(current_video, config).run()
|
||||
|
||||
video_ids = [i["video_id"] for i in self.to_import]
|
||||
comment_list = CommentList(task=self.task)
|
||||
comment_list.add(video_ids=video_ids)
|
||||
comment_list.index()
|
||||
ManualImport(
|
||||
current_video,
|
||||
config,
|
||||
ignore_error=self.ignore_error,
|
||||
prefer_local=self.prefer_local,
|
||||
).run()
|
||||
|
||||
def _notify(self, idx, current_video):
|
||||
"""send notification back to task"""
|
||||
|
|
@ -185,17 +193,27 @@ class ImportFolderScanner:
|
|||
expects filename ending in [<youtube_id>].<ext>
|
||||
"""
|
||||
base_name, _ = os.path.splitext(file_name)
|
||||
|
||||
# yt-dlp default like [youtubeid]
|
||||
id_search = re.search(r"\[([a-zA-Z0-9_-]{11})\]$", base_name)
|
||||
if id_search:
|
||||
youtube_id = id_search.group(1)
|
||||
return youtube_id
|
||||
|
||||
file_name_search = re.search(r"([a-zA-Z0-9_-]{11})$", base_name)
|
||||
if file_name_search:
|
||||
youtube_id = file_name_search.group(1)
|
||||
return youtube_id
|
||||
|
||||
print(f"id extraction failed from filename: {file_name}")
|
||||
|
||||
return False
|
||||
|
||||
def _extract_id_from_json(self, json_file):
|
||||
def _extract_id_from_json(self, json_file: str | bool) -> str | None:
|
||||
"""open json file and extract id"""
|
||||
if not json_file or not isinstance(json_file, str):
|
||||
return None
|
||||
|
||||
json_path = os.path.join(self.CACHE_DIR, "import", json_file)
|
||||
with open(json_path, "r", encoding="utf-8") as f:
|
||||
json_content = f.read()
|
||||
|
|
@ -388,17 +406,49 @@ class ImportFolderScanner:
|
|||
class ManualImport:
|
||||
"""import single identified video"""
|
||||
|
||||
def __init__(self, current_video, config):
|
||||
def __init__(
|
||||
self, current_video, config, ignore_error: bool, prefer_local: bool
|
||||
):
|
||||
self.current_video = current_video
|
||||
self.config = config
|
||||
self.ignore_error: bool = ignore_error
|
||||
self.prefer_local: bool = prefer_local
|
||||
|
||||
def run(self):
|
||||
"""run all"""
|
||||
json_data = self.index_metadata()
|
||||
self._move_to_archive(json_data)
|
||||
self._cleanup(json_data)
|
||||
json_data = None
|
||||
if self.prefer_local:
|
||||
# embedded first
|
||||
json_data = IndexFromEmbed(
|
||||
self.current_video["media"],
|
||||
use_user_conf=False,
|
||||
config=self.config,
|
||||
).run_index()
|
||||
if json_data:
|
||||
self._cleanup()
|
||||
return
|
||||
|
||||
def index_metadata(self):
|
||||
try:
|
||||
json_data = self.index_metadata()
|
||||
except ValueError as err:
|
||||
json_data = IndexFromEmbed(
|
||||
self.current_video["media"],
|
||||
use_user_conf=False,
|
||||
config=self.config,
|
||||
).run_index()
|
||||
if not json_data and not self.ignore_error:
|
||||
raise ValueError from err
|
||||
|
||||
if not json_data:
|
||||
return
|
||||
|
||||
self._move_to_archive(json_data)
|
||||
self._cleanup()
|
||||
|
||||
Comments(self.current_video["video_id"]).build_json(upload=True)
|
||||
YoutubeVideo(self.current_video["video_id"]).embed_metadata()
|
||||
|
||||
def index_metadata(self) -> dict | None:
|
||||
"""get metadata from yt or json"""
|
||||
video_id = self.current_video["video_id"]
|
||||
video = YoutubeVideo(video_id)
|
||||
|
|
@ -411,6 +461,9 @@ class ManualImport:
|
|||
f"{video_id}: manual import failed, and no metadata found."
|
||||
)
|
||||
print(message)
|
||||
if self.ignore_error:
|
||||
return None
|
||||
|
||||
raise ValueError(message)
|
||||
|
||||
video.check_subtitles(subtitle_files=self.current_video["subtitle"])
|
||||
|
|
@ -463,7 +516,7 @@ class ManualImport:
|
|||
new_path = f"{base_name}.{lang}.vtt"
|
||||
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
|
||||
|
||||
def _cleanup(self, json_data):
|
||||
def _cleanup(self):
|
||||
"""cleanup leftover files"""
|
||||
meta_data = self.current_video["metadata"]
|
||||
if meta_data and os.path.exists(meta_data):
|
||||
|
|
@ -476,11 +529,3 @@ class ManualImport:
|
|||
for subtitle_file in self.current_video["subtitle"]:
|
||||
if os.path.exists(subtitle_file):
|
||||
os.remove(subtitle_file)
|
||||
|
||||
channel_info = os.path.join(
|
||||
EnvironmentSettings.CACHE_DIR,
|
||||
"import",
|
||||
f"{json_data['channel']['channel_id']}.info.json",
|
||||
)
|
||||
if os.path.exists(channel_info):
|
||||
os.remove(channel_info)
|
||||
|
|
|
|||
|
|
@ -345,6 +345,7 @@ class Reindex(ReindexBase):
|
|||
thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
|
||||
|
||||
Comments(youtube_id, config=self.config).reindex_comments()
|
||||
video.get_from_es()
|
||||
video.embed_metadata()
|
||||
self.processed["videos"] += 1
|
||||
|
||||
|
|
|
|||
|
|
@ -44,6 +44,16 @@ urlpatterns = [
|
|||
views.TokenView.as_view(),
|
||||
name="api-token",
|
||||
),
|
||||
path(
|
||||
"rescan-filesystem/",
|
||||
views.RescanFileSystem.as_view(),
|
||||
name="api-rescan-filesystem",
|
||||
),
|
||||
path(
|
||||
"manual-import/",
|
||||
views.ManualImportView.as_view(),
|
||||
name="api-manual-import",
|
||||
),
|
||||
path(
|
||||
"membership/profile/",
|
||||
views_mb.MembershipProfileView.as_view(),
|
||||
|
|
|
|||
|
|
@ -5,7 +5,9 @@ from appsettings.serializers import (
|
|||
BackupFileSerializer,
|
||||
CookieUpdateSerializer,
|
||||
CookieValidationSerializer,
|
||||
ManualImportConfig,
|
||||
PoTokenSerializer,
|
||||
RescanFileSystemConfig,
|
||||
SnapshotCreateResponseSerializer,
|
||||
SnapshotItemSerializer,
|
||||
SnapshotListSerializer,
|
||||
|
|
@ -291,7 +293,11 @@ class CookieView(ApiBaseView):
|
|||
|
||||
|
||||
class POTokenView(ApiBaseView):
|
||||
"""handle PO token"""
|
||||
"""resolves to /api/appsettings/potoken/
|
||||
GET: get potoken
|
||||
POST: update potoken
|
||||
DELETE: revoke potoken
|
||||
"""
|
||||
|
||||
permission_classes = [AdminOnly]
|
||||
|
||||
|
|
@ -388,6 +394,58 @@ class SnapshotApiListView(ApiBaseView):
|
|||
return Response(serializer.data)
|
||||
|
||||
|
||||
class RescanFileSystem(ApiBaseView):
|
||||
"""resolves to /api/appsettings/rescan-filesystem/
|
||||
POST: start new rescan filesystem task
|
||||
"""
|
||||
|
||||
permission_classes = [AdminOnly]
|
||||
|
||||
@staticmethod
|
||||
@extend_schema(
|
||||
request=RescanFileSystemConfig,
|
||||
responses={
|
||||
200: OpenApiResponse(AsyncTaskResponseSerializer()),
|
||||
},
|
||||
)
|
||||
def post(request):
|
||||
"""start new task rescan filesystem task"""
|
||||
data_serializer = RescanFileSystemConfig(data=request.data)
|
||||
data_serializer.is_valid(raise_exception=True)
|
||||
validated_data = data_serializer.validated_data
|
||||
|
||||
message = TaskCommand().start("rescan_filesystem", validated_data)
|
||||
serializer = AsyncTaskResponseSerializer(message)
|
||||
|
||||
return Response(serializer.data)
|
||||
|
||||
|
||||
class ManualImportView(ApiBaseView):
|
||||
"""resolves to /api/appsettings/manual-import/
|
||||
POST: start new manual import task
|
||||
"""
|
||||
|
||||
permission_classes = [AdminOnly]
|
||||
|
||||
@staticmethod
|
||||
@extend_schema(
|
||||
request=ManualImportConfig,
|
||||
responses={
|
||||
200: OpenApiResponse(AsyncTaskResponseSerializer()),
|
||||
},
|
||||
)
|
||||
def post(request):
|
||||
"""manual import"""
|
||||
data_serializer = ManualImportConfig(data=request.data)
|
||||
data_serializer.is_valid(raise_exception=True)
|
||||
validated_data = data_serializer.validated_data
|
||||
|
||||
message = TaskCommand().start("manual_import", validated_data)
|
||||
serializer = AsyncTaskResponseSerializer(message)
|
||||
|
||||
return Response(serializer.data)
|
||||
|
||||
|
||||
class SnapshotApiView(ApiBaseView):
|
||||
"""resolves to /api/appsettings/snapshot/<snapshot-id>/
|
||||
GET: return a single snapshot
|
||||
|
|
|
|||
|
|
@ -110,6 +110,8 @@ def date_parser(timestamp: int | str | None) -> str | None:
|
|||
|
||||
if isinstance(timestamp, int):
|
||||
date_obj = datetime.fromtimestamp(timestamp, tz=timezone.utc)
|
||||
elif isinstance(timestamp, str) and timestamp.isdigit():
|
||||
date_obj = datetime.fromtimestamp(int(timestamp), tz=timezone.utc)
|
||||
elif isinstance(timestamp, str):
|
||||
date_obj = datetime.strptime(timestamp, "%Y-%m-%d")
|
||||
date_obj = date_obj.replace(tzinfo=timezone.utc)
|
||||
|
|
|
|||
|
|
@ -113,6 +113,7 @@ class SearchParser:
|
|||
"index": "ta_subtitle",
|
||||
"lang": [],
|
||||
"source": [],
|
||||
"channel": [],
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -345,6 +346,22 @@ class QueryBuilder:
|
|||
"""build query for fulltext search"""
|
||||
must_list = []
|
||||
|
||||
if (channel := self.query_map.get("channel")) is not None:
|
||||
must_list.append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": channel,
|
||||
"type": "bool_prefix",
|
||||
"fuzziness": self._get_fuzzy(),
|
||||
"operator": "and",
|
||||
"fields": [
|
||||
"subtitle_channel",
|
||||
"subtitle_channel.keyword",
|
||||
],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
if (term := self.query_map.get("term")) is not None:
|
||||
must_list.append(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -26,6 +26,13 @@ def test_date_parser_with_int():
|
|||
assert date_parser(timestamp) == expected_date
|
||||
|
||||
|
||||
def test_date_parser_with_digit():
|
||||
"""unix timestamp"""
|
||||
timestamp = "1621539600"
|
||||
expected_date = "2021-05-20T19:40:00+00:00"
|
||||
assert date_parser(timestamp) == expected_date
|
||||
|
||||
|
||||
def test_date_parser_with_str():
|
||||
"""iso timestamp"""
|
||||
date_str = "2021-05-21"
|
||||
|
|
|
|||
|
|
@ -10,13 +10,15 @@ from video.src.constants import VideoTypeEnum
|
|||
class DownloadItemSerializer(serializers.Serializer):
|
||||
"""serialize download item"""
|
||||
|
||||
auto_start = serializers.BooleanField()
|
||||
auto_start = serializers.BooleanField(required=False)
|
||||
channel_id = serializers.CharField()
|
||||
channel_indexed = serializers.BooleanField()
|
||||
channel_name = serializers.CharField()
|
||||
duration = serializers.CharField()
|
||||
published = serializers.CharField(allow_null=True)
|
||||
status = serializers.ChoiceField(choices=["pending", "ignore"])
|
||||
status = serializers.ChoiceField(
|
||||
choices=["pending", "ignore"], required=False
|
||||
)
|
||||
timestamp = serializers.IntegerField(allow_null=True)
|
||||
title = serializers.CharField()
|
||||
vid_thumb_url = serializers.CharField(allow_null=True)
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from common.src.helper import (
|
|||
rand_sleep,
|
||||
)
|
||||
from common.src.urlparser import ParsedURLType
|
||||
from download.serializers import DownloadItemSerializer
|
||||
from download.src.queue_interact import PendingInteract
|
||||
from download.src.thumbnails import ThumbManager
|
||||
from playlist.src.index import YoutubePlaylist
|
||||
|
|
@ -379,6 +380,12 @@ class PendingList(PendingIndex):
|
|||
"channel_id": video_data["channel_id"],
|
||||
"channel_indexed": video_data["channel_id"] in self.all_channels,
|
||||
}
|
||||
serializer = DownloadItemSerializer(data=to_add)
|
||||
is_valid = serializer.is_valid()
|
||||
if not is_valid:
|
||||
print(f"{youtube_id}: serializer failed: {serializer.errors}")
|
||||
self._notify_fail(403, youtube_id)
|
||||
return None
|
||||
|
||||
return to_add
|
||||
|
||||
|
|
@ -393,18 +400,24 @@ class PendingList(PendingIndex):
|
|||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_published(video_data) -> str | int | None:
|
||||
def _extract_published(video_data) -> int | None:
|
||||
"""build published date or timestamp"""
|
||||
timestamp = video_data.get("timestamp")
|
||||
if timestamp:
|
||||
if timestamp and isinstance(timestamp, int):
|
||||
return timestamp
|
||||
|
||||
upload_date = video_data.get("upload_date")
|
||||
if upload_date:
|
||||
upload_date_time = datetime.strptime(upload_date, "%Y%m%d")
|
||||
return upload_date_time.replace(
|
||||
tzinfo=ZoneInfo(EnvironmentSettings.TZ)
|
||||
).timestamp()
|
||||
try:
|
||||
upload_date_time = datetime.strptime(upload_date, "%Y%m%d")
|
||||
except ValueError:
|
||||
youtube_id = video_data["id"]
|
||||
print(f"{youtube_id}: published date extraction failed.")
|
||||
return None
|
||||
|
||||
tz = ZoneInfo(EnvironmentSettings.TZ)
|
||||
timestamp = int(upload_date_time.replace(tzinfo=tz).timestamp())
|
||||
return timestamp
|
||||
|
||||
return None
|
||||
|
||||
|
|
|
|||
|
|
@ -4,9 +4,7 @@ functionality:
|
|||
- check for missing thumbnails
|
||||
"""
|
||||
|
||||
import base64
|
||||
import os
|
||||
from io import BytesIO
|
||||
from time import sleep
|
||||
|
||||
import requests
|
||||
|
|
@ -14,7 +12,7 @@ from common.src.env_settings import EnvironmentSettings
|
|||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from common.src.helper import is_missing
|
||||
from mutagen.mp4 import MP4, MP4Cover
|
||||
from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError
|
||||
from PIL import Image, ImageFile, UnidentifiedImageError
|
||||
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = True
|
||||
|
||||
|
|
@ -23,6 +21,7 @@ class ThumbManagerBase:
|
|||
"""base class for thumbnail management"""
|
||||
|
||||
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||
MEDIA_DIR = EnvironmentSettings.MEDIA_DIR
|
||||
VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
|
||||
CHANNEL_DIR = os.path.join(CACHE_DIR, "channels")
|
||||
PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists")
|
||||
|
|
@ -217,6 +216,56 @@ class ThumbManager(ThumbManagerBase):
|
|||
img_raw = img_raw.resize((336, 189))
|
||||
img_raw.convert("RGB").save(thumb_path)
|
||||
|
||||
def embed_video_art(self, json_data: dict):
|
||||
"""embed video artwork"""
|
||||
file_path = os.path.join(self.MEDIA_DIR, json_data["media_url"])
|
||||
video = MP4(file_path)
|
||||
|
||||
thumb_path = self.vid_thumb_path(absolute=True)
|
||||
if os.path.exists(thumb_path):
|
||||
with open(thumb_path, "rb") as f:
|
||||
cover_data = f.read()
|
||||
|
||||
video["covr"] = [
|
||||
MP4Cover(cover_data, imageformat=MP4Cover.FORMAT_JPEG)
|
||||
]
|
||||
|
||||
channel_id = json_data["channel"]["channel_id"]
|
||||
banner_path = os.path.join(
|
||||
self.CHANNEL_DIR, f"{channel_id}_banner.jpg"
|
||||
)
|
||||
if os.path.exists(banner_path):
|
||||
with open(banner_path, "rb") as f:
|
||||
banner_data = f.read()
|
||||
|
||||
video["----:com.tubearchivist:channel_banner"] = [
|
||||
MP4Cover(banner_data, imageformat=MP4Cover.FORMAT_JPEG)
|
||||
]
|
||||
|
||||
channel_icon_path = os.path.join(
|
||||
self.CHANNEL_DIR, f"{channel_id}_thumb.jpg"
|
||||
)
|
||||
if os.path.exists(channel_icon_path):
|
||||
with open(channel_icon_path, "rb") as f:
|
||||
icon_data = f.read()
|
||||
|
||||
video["----:com.tubearchivist:channel_icon"] = [
|
||||
MP4Cover(icon_data, imageformat=MP4Cover.FORMAT_JPEG)
|
||||
]
|
||||
|
||||
channel_tv_path = os.path.join(
|
||||
self.CHANNEL_DIR, f"{channel_id}_tvart.jpg"
|
||||
)
|
||||
if os.path.exists(channel_tv_path):
|
||||
with open(channel_tv_path, "rb") as f:
|
||||
tv_data = f.read()
|
||||
|
||||
video["----:com.tubearchivist:channel_tv"] = [
|
||||
MP4Cover(tv_data, imageformat=MP4Cover.FORMAT_JPEG)
|
||||
]
|
||||
|
||||
video.save()
|
||||
|
||||
def delete_video_thumb(self):
|
||||
"""delete video thumbnail if exists"""
|
||||
thumb_path = self.vid_thumb_path()
|
||||
|
|
@ -228,10 +277,13 @@ class ThumbManager(ThumbManagerBase):
|
|||
"""delete all artwork of channel"""
|
||||
thumb = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_thumb.jpg")
|
||||
banner = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_banner.jpg")
|
||||
tv = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_tvart.jpg")
|
||||
if os.path.exists(thumb):
|
||||
os.remove(thumb)
|
||||
if os.path.exists(banner):
|
||||
os.remove(banner)
|
||||
if os.path.exists(tv):
|
||||
os.remove(tv)
|
||||
|
||||
def delete_playlist_thumb(self):
|
||||
"""delete playlist thumbnail"""
|
||||
|
|
@ -239,20 +291,6 @@ class ThumbManager(ThumbManagerBase):
|
|||
if os.path.exists(thumb_path):
|
||||
os.remove(thumb_path)
|
||||
|
||||
def get_vid_base64_blur(self):
|
||||
"""return base64 encoded placeholder"""
|
||||
file_path = os.path.join(self.CACHE_DIR, self.vid_thumb_path())
|
||||
img_raw = Image.open(file_path)
|
||||
img_raw.thumbnail((img_raw.width // 20, img_raw.height // 20))
|
||||
img_blur = img_raw.filter(ImageFilter.BLUR)
|
||||
buffer = BytesIO()
|
||||
img_blur.save(buffer, format="JPEG")
|
||||
img_data = buffer.getvalue()
|
||||
img_base64 = base64.b64encode(img_data).decode()
|
||||
data_url = f"data:image/jpg;base64,{img_base64}"
|
||||
|
||||
return data_url
|
||||
|
||||
|
||||
class ValidatorCallback:
|
||||
"""handle callback validate thumbnails page by page"""
|
||||
|
|
@ -458,7 +496,7 @@ class ThumbFilesystem:
|
|||
"""entry point"""
|
||||
data = {
|
||||
"query": {"match_all": {}},
|
||||
"_source": ["media_url", "youtube_id"],
|
||||
"_source": ["media_url", "youtube_id", "channel.channel_id"],
|
||||
}
|
||||
paginate = IndexPaginate(
|
||||
index_name=self.INDEX_NAME,
|
||||
|
|
@ -481,9 +519,7 @@ class ThumbFilesystem:
|
|||
class EmbedCallback:
|
||||
"""callback class to embed thumbnails"""
|
||||
|
||||
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||
MEDIA_DIR = EnvironmentSettings.MEDIA_DIR
|
||||
FORMAT = MP4Cover.FORMAT_JPEG
|
||||
|
||||
def __init__(self, source, index_name, counter=0):
|
||||
self.source = source
|
||||
|
|
@ -494,19 +530,4 @@ class EmbedCallback:
|
|||
"""run embed"""
|
||||
for video in self.source:
|
||||
video_id = video["_source"]["youtube_id"]
|
||||
media_url = os.path.join(
|
||||
self.MEDIA_DIR, video["_source"]["media_url"]
|
||||
)
|
||||
thumb_path = os.path.join(
|
||||
self.CACHE_DIR, ThumbManager(video_id).vid_thumb_path()
|
||||
)
|
||||
if os.path.exists(thumb_path):
|
||||
self.embed(media_url, thumb_path)
|
||||
|
||||
def embed(self, media_url, thumb_path):
|
||||
"""embed thumb in single media file"""
|
||||
video = MP4(media_url)
|
||||
with open(thumb_path, "rb") as f:
|
||||
video["covr"] = [MP4Cover(f.read(), imageformat=self.FORMAT)]
|
||||
|
||||
video.save()
|
||||
ThumbManager(video_id).embed_video_art(video["_source"])
|
||||
|
|
|
|||
|
|
@ -196,15 +196,6 @@ class VideoDownloader(DownloaderBase):
|
|||
}
|
||||
)
|
||||
|
||||
if self.config["downloads"]["add_thumbnail"]:
|
||||
postprocessors.append(
|
||||
{
|
||||
"key": "EmbedThumbnail",
|
||||
"already_have_thumbnail": True,
|
||||
}
|
||||
)
|
||||
self.obs["writethumbnail"] = True
|
||||
|
||||
self.obs["postprocessors"] = postprocessors
|
||||
|
||||
def _set_overwrites(self, obs: dict, channel_id: str) -> None:
|
||||
|
|
@ -326,6 +317,9 @@ class DownloadPostProcess(DownloaderBase):
|
|||
for channel_id, value in self.channel_overwrites.items():
|
||||
if "autodelete_days" in value:
|
||||
autodelete_days = value.get("autodelete_days")
|
||||
if autodelete_days is None:
|
||||
continue
|
||||
|
||||
print(f"{channel_id}: delete older than {autodelete_days}d")
|
||||
now_lte = str(self.now - autodelete_days * 24 * 60 * 60)
|
||||
must_list = [
|
||||
|
|
@ -488,7 +482,10 @@ class DownloadPostProcess(DownloaderBase):
|
|||
|
||||
def embed_metadata(self):
|
||||
"""embed metadata in media file"""
|
||||
if not self.config["downloads"].get("add_metadata"):
|
||||
meta = self.config["downloads"].get("add_metadata")
|
||||
thumb = self.config["downloads"].get("add_thumbnail")
|
||||
|
||||
if not meta and not thumb:
|
||||
return
|
||||
|
||||
queue = RedisQueue(self.VIDEO_QUEUE)
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
apprise==1.9.5
|
||||
celery==5.5.3
|
||||
apprise==1.9.6
|
||||
celery==5.6.0
|
||||
django-auth-ldap==5.2.0
|
||||
django-celery-beat==2.8.1
|
||||
django-cors-headers==4.9.0
|
||||
Django==5.2.8
|
||||
Django==5.2.9
|
||||
djangorestframework==3.16.1
|
||||
drf-spectacular==0.29.0
|
||||
drf-spectacular==0.28.0 # rc:ignore
|
||||
Pillow==12.0.0
|
||||
redis==7.0.1
|
||||
redis==7.1.0
|
||||
requests==2.32.5
|
||||
ryd-client==0.0.6
|
||||
uvicorn==0.38.0
|
||||
yt-dlp[default]==2025.11.12
|
||||
yt-dlp[default]==2025.12.8
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ CHECK_REINDEX: TaskItemConfig = {
|
|||
MANUAL_IMPORT: TaskItemConfig = {
|
||||
"title": "Manual video import",
|
||||
"group": "setting:import",
|
||||
"api_start": True,
|
||||
"api_start": False,
|
||||
"api_stop": False,
|
||||
}
|
||||
|
||||
|
|
@ -69,7 +69,7 @@ RESTORE_BACKUP: TaskItemConfig = {
|
|||
RESCAN_FILESYSTEM: TaskItemConfig = {
|
||||
"title": "Rescan your Filesystem",
|
||||
"group": "setting:filesystemscan",
|
||||
"api_start": True,
|
||||
"api_start": False,
|
||||
"api_stop": False,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -84,9 +84,9 @@ class TaskManager:
|
|||
class TaskCommand:
|
||||
"""run commands on task"""
|
||||
|
||||
def start(self, task_name):
|
||||
def start(self, task_name, kwargs: dict | None = None):
|
||||
"""start task by task_name, only pass task that don't take args"""
|
||||
task = celery_app.tasks.get(task_name).delay()
|
||||
task = celery_app.tasks.get(task_name).delay(**(kwargs or {}))
|
||||
message = {
|
||||
"task_id": task.id,
|
||||
"status": task.status,
|
||||
|
|
|
|||
|
|
@ -216,7 +216,7 @@ def check_reindex(self, data=False, extract_videos=False):
|
|||
|
||||
|
||||
@shared_task(bind=True, name="manual_import", base=BaseTask)
|
||||
def run_manual_import(self):
|
||||
def manual_import(self, ignore_error, prefer_local):
|
||||
"""called from settings page, to go through import folder"""
|
||||
manager = TaskManager()
|
||||
if manager.is_pending(self):
|
||||
|
|
@ -225,7 +225,9 @@ def run_manual_import(self):
|
|||
return
|
||||
|
||||
manager.init(self)
|
||||
ImportFolderScanner(task=self).scan()
|
||||
ImportFolderScanner(
|
||||
task=self, ignore_error=ignore_error, prefer_local=prefer_local
|
||||
).scan()
|
||||
|
||||
|
||||
@shared_task(bind=True, name="run_backup", base=BaseTask)
|
||||
|
|
@ -260,7 +262,7 @@ def run_restore_backup(self, filename):
|
|||
|
||||
|
||||
@shared_task(bind=True, name="rescan_filesystem", base=BaseTask)
|
||||
def rescan_filesystem(self):
|
||||
def rescan_filesystem(self, ignore_error, prefer_local):
|
||||
"""check the media folder for mismatches"""
|
||||
manager = TaskManager()
|
||||
if manager.is_pending(self):
|
||||
|
|
@ -269,7 +271,9 @@ def rescan_filesystem(self):
|
|||
return
|
||||
|
||||
manager.init(self)
|
||||
handler = Scanner(task=self)
|
||||
handler = Scanner(
|
||||
task=self, ignore_error=ignore_error, prefer_local=prefer_local
|
||||
)
|
||||
handler.scan()
|
||||
handler.apply()
|
||||
thumbnail_check.delay()
|
||||
|
|
|
|||
|
|
@ -60,6 +60,23 @@ class StreamItemSerializer(serializers.Serializer):
|
|||
height = serializers.IntegerField(required=False)
|
||||
|
||||
|
||||
class SubtitleFragmentSerializer(serializers.Serializer):
|
||||
"""serialize subtitle fragment"""
|
||||
|
||||
subtitle_index = serializers.IntegerField()
|
||||
subtitle_line = serializers.CharField()
|
||||
subtitle_start = serializers.CharField()
|
||||
subtitle_fragment_id = serializers.CharField()
|
||||
subtitle_end = serializers.CharField()
|
||||
youtube_id = serializers.CharField()
|
||||
title = serializers.CharField()
|
||||
subtitle_channel = serializers.CharField()
|
||||
subtitle_channel_id = serializers.CharField()
|
||||
subtitle_last_refresh = serializers.IntegerField()
|
||||
subtitle_lang = serializers.CharField()
|
||||
subtitle_source = serializers.ChoiceField(choices=["user", "auto"])
|
||||
|
||||
|
||||
class SubtitleItemSerializer(serializers.Serializer):
|
||||
"""serialize subtitle item"""
|
||||
|
||||
|
|
@ -76,16 +93,18 @@ class VideoSerializer(serializers.Serializer):
|
|||
|
||||
active = serializers.BooleanField()
|
||||
category = serializers.ListField(child=serializers.CharField())
|
||||
channel = ChannelSerializer()
|
||||
comment_count = serializers.IntegerField(allow_null=True)
|
||||
channel = ChannelSerializer(required=False)
|
||||
comment_count = serializers.IntegerField(allow_null=True, required=False)
|
||||
date_downloaded = serializers.IntegerField()
|
||||
description = serializers.CharField()
|
||||
media_size = serializers.IntegerField()
|
||||
media_url = serializers.CharField()
|
||||
player = PlayerSerializer()
|
||||
playlist = serializers.ListField(child=serializers.CharField())
|
||||
playlist = serializers.ListField(
|
||||
child=serializers.CharField(), required=False
|
||||
)
|
||||
published = serializers.CharField()
|
||||
sponsorblock = SponsorBlockSerializer(allow_null=True)
|
||||
sponsorblock = SponsorBlockSerializer(allow_null=True, required=False)
|
||||
stats = StatsSerializer()
|
||||
streams = StreamItemSerializer(many=True)
|
||||
subtitles = SubtitleItemSerializer(many=True)
|
||||
|
|
@ -153,7 +172,16 @@ class CommentItemSerializer(serializers.Serializer):
|
|||
comment_author_thumbnail = serializers.URLField()
|
||||
comment_author_is_uploader = serializers.BooleanField()
|
||||
comment_parent = serializers.CharField()
|
||||
comment_replies = CommentThreadItemSerializer(many=True)
|
||||
comment_replies = CommentThreadItemSerializer(many=True, required=False)
|
||||
|
||||
|
||||
class CommentsSerializer(serializers.Serializer):
|
||||
"""serialize comments as indexed"""
|
||||
|
||||
youtube_id = serializers.CharField()
|
||||
comment_last_refresh = serializers.IntegerField()
|
||||
comment_channel_id = serializers.CharField()
|
||||
comment_comments = CommentItemSerializer(many=True)
|
||||
|
||||
|
||||
class PlaylistNavMetaSerializer(serializers.Serializer):
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ class Comments:
|
|||
self.is_activated = False
|
||||
self.comments_format = False
|
||||
|
||||
def build_json(self):
|
||||
def build_json(self, upload: bool = False):
|
||||
"""build json document for es"""
|
||||
print(f"{self.youtube_id}: get comments")
|
||||
self.check_config()
|
||||
|
|
@ -44,6 +44,8 @@ class Comments:
|
|||
"comment_channel_id": channel_id,
|
||||
"comment_comments": self.comments_format,
|
||||
}
|
||||
if upload:
|
||||
self.upload_comments()
|
||||
|
||||
def check_config(self):
|
||||
"""read config if not attached"""
|
||||
|
|
@ -142,14 +144,13 @@ class Comments:
|
|||
|
||||
def upload_comments(self):
|
||||
"""upload comments to es"""
|
||||
if not self.is_activated:
|
||||
return
|
||||
|
||||
print(f"{self.youtube_id}: upload comments")
|
||||
_, _ = ElasticWrap(self.es_path).put(self.json_data)
|
||||
|
||||
vid_path = f"ta_video/_update/{self.youtube_id}"
|
||||
data = {"doc": {"comment_count": len(self.comments_format)}}
|
||||
data = {
|
||||
"doc": {"comment_count": len(self.json_data["comment_comments"])}
|
||||
}
|
||||
_, _ = ElasticWrap(vid_path).post(data=data)
|
||||
|
||||
def delete_comments(self):
|
||||
|
|
|
|||
|
|
@ -220,7 +220,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
|||
def _build_published(self):
|
||||
"""build published date or timestamp"""
|
||||
timestamp = self.youtube_meta.get("timestamp")
|
||||
if timestamp:
|
||||
if timestamp and isinstance(timestamp, int):
|
||||
return timestamp
|
||||
|
||||
upload_date = self.youtube_meta["upload_date"]
|
||||
|
|
@ -424,10 +424,6 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
|||
|
||||
def embed_metadata(self):
|
||||
"""embed metadata for video"""
|
||||
if not self.config["downloads"].get("add_metadata"):
|
||||
return
|
||||
|
||||
print(f"{self.youtube_id}: embed metadata")
|
||||
if not self.json_data:
|
||||
self.get_from_es()
|
||||
|
||||
|
|
@ -435,6 +431,15 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
|||
print(f"{self.youtube_id}: skip embed, video not indexed")
|
||||
return
|
||||
|
||||
if self.config["downloads"].get("add_metadata"):
|
||||
self._embed_text_data()
|
||||
|
||||
if self.config["downloads"].get("add_thumbnail"):
|
||||
self._embed_artwork()
|
||||
|
||||
def _embed_text_data(self):
|
||||
"""embed text metadata"""
|
||||
print(f"{self.youtube_id}: embed metadata")
|
||||
video_base = EnvironmentSettings.MEDIA_DIR
|
||||
media_url = self.json_data.get("media_url")
|
||||
file_path = os.path.join(video_base, media_url)
|
||||
|
|
@ -479,11 +484,17 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
|||
"comments": comments,
|
||||
"subtitles": subtitles,
|
||||
"playlists": playlists,
|
||||
"version": settings.TA_VERSION,
|
||||
}
|
||||
)
|
||||
|
||||
return to_embed
|
||||
|
||||
def _embed_artwork(self):
|
||||
"""embed artwork"""
|
||||
print(f"{self.youtube_id}: embed artwork")
|
||||
ThumbManager(self.youtube_id).embed_video_art(self.json_data)
|
||||
|
||||
|
||||
def index_new_video(youtube_id, video_type=VideoTypeEnum.VIDEOS):
|
||||
"""combined classes to create new video in index"""
|
||||
|
|
|
|||
|
|
@ -1,7 +1,28 @@
|
|||
"""bulk metadata embedding"""
|
||||
"""
|
||||
Functionality:
|
||||
- bulk metadata embedding
|
||||
- restore from embedded metadata
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from appsettings.src.config import AppConfig, AppConfigType
|
||||
from channel.serializers import ChannelSerializer
|
||||
from channel.src.index import YoutubeChannel
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from download.src.thumbnails import ThumbManager
|
||||
from mutagen.mp4 import MP4, MP4FreeForm
|
||||
from video.serializers import (
|
||||
CommentsSerializer,
|
||||
SubtitleFragmentSerializer,
|
||||
VideoSerializer,
|
||||
)
|
||||
from video.src.comments import Comments
|
||||
from video.src.index import YoutubeVideo
|
||||
from video.src.subtitle import SubtitleParser, YoutubeSubtitle
|
||||
|
||||
|
||||
class MetadataEmbed:
|
||||
|
|
@ -49,3 +70,302 @@ class MetadataEmbedCallback:
|
|||
for video in self.source:
|
||||
youtube_id = video["_source"]["youtube_id"]
|
||||
YoutubeVideo(youtube_id).embed_metadata()
|
||||
|
||||
|
||||
class IndexFromEmbed:
|
||||
"""restore from embedded metadata, potential untrusted"""
|
||||
|
||||
VIDEOS_BASE = EnvironmentSettings.MEDIA_DIR
|
||||
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||
HOST_UID = EnvironmentSettings.HOST_UID
|
||||
HOST_GID = EnvironmentSettings.HOST_GID
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
file_path: str,
|
||||
use_user_conf: bool = True,
|
||||
config: AppConfigType | None = None,
|
||||
):
|
||||
self.file_path = file_path
|
||||
self.use_user_conf = use_user_conf
|
||||
self.config = config
|
||||
|
||||
def run_index(self) -> None | dict:
|
||||
"""run index"""
|
||||
if not self.config:
|
||||
self.config = AppConfig().config
|
||||
|
||||
json_embed = self._get_embedded()
|
||||
if not json_embed:
|
||||
return None
|
||||
|
||||
channel_data_clean = self.index_channel(json_embed)
|
||||
video = self.index_video(json_embed, channel_data_clean)
|
||||
self.archive_video(video)
|
||||
self.index_subtitles(json_embed, video)
|
||||
self.index_comments(json_embed)
|
||||
|
||||
return video.json_data
|
||||
|
||||
def _get_embedded(self) -> dict | None:
|
||||
"""get embedded metadata"""
|
||||
video_mutagen = MP4(self.file_path)
|
||||
ta_data = video_mutagen.get("----:com.tubearchivist:ta")
|
||||
if not ta_data:
|
||||
return None
|
||||
|
||||
if not isinstance(ta_data, list):
|
||||
raise ValueError(f"[{self.file_path}] unexpected embedded data")
|
||||
|
||||
to_decode = ta_data[0]
|
||||
|
||||
if not isinstance(to_decode, MP4FreeForm):
|
||||
raise ValueError(f"[{self.file_path}] unexpected embedded data")
|
||||
|
||||
try:
|
||||
json_embed = json.loads(to_decode.decode())
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
err = f"[{self.file_path}] embedded decoding failed: {str(exc)}"
|
||||
raise ValueError(err) from exc
|
||||
|
||||
if not json_embed.get("video"):
|
||||
err = f"[{self.file_path}] embedded does not contain video key"
|
||||
raise ValueError(err)
|
||||
|
||||
return json_embed
|
||||
|
||||
def index_channel(self, json_embed):
|
||||
"""index channel"""
|
||||
channel_data = json_embed["video"].get("channel")
|
||||
if not channel_data:
|
||||
raise ValueError(f"[{self.file_path}] missing channel metadata")
|
||||
|
||||
serializer = ChannelSerializer(data=channel_data)
|
||||
is_valid = serializer.is_valid()
|
||||
if not is_valid:
|
||||
err = serializer.errors
|
||||
raise ValueError(
|
||||
f"[{self.file_path}] channel serializer failed: {err}"
|
||||
)
|
||||
|
||||
channel_data_clean = dict(serializer.data)
|
||||
if not self.use_user_conf:
|
||||
if "channel_overwrites" in channel_data_clean:
|
||||
channel_data_clean.pop("channel_overwrites")
|
||||
|
||||
channel_data_clean["channel_subscribed"] = False
|
||||
|
||||
channel = YoutubeChannel(youtube_id=channel_data_clean["channel_id"])
|
||||
channel.get_from_es()
|
||||
if not channel.json_data:
|
||||
channel.json_data = channel_data_clean
|
||||
channel.upload_to_es()
|
||||
|
||||
return channel.json_data
|
||||
|
||||
def index_video(self, json_embed, channel_data_clean):
|
||||
"""index video"""
|
||||
video_data = json_embed["video"]
|
||||
video_data.pop("channel")
|
||||
|
||||
serializer = VideoSerializer(data=video_data)
|
||||
is_valid = serializer.is_valid()
|
||||
if not is_valid:
|
||||
err = serializer.errors
|
||||
raise ValueError(
|
||||
f"[{self.file_path}] video serializer failed: {err}"
|
||||
)
|
||||
|
||||
video_data_clean = dict(serializer.data)
|
||||
if not self.use_user_conf:
|
||||
video_data_clean["player"]["watched"] = False
|
||||
|
||||
video_data_clean["channel"] = channel_data_clean
|
||||
video = YoutubeVideo(youtube_id=video_data_clean["youtube_id"])
|
||||
video.get_from_es()
|
||||
if not video.json_data:
|
||||
video.json_data = video_data_clean
|
||||
video.upload_to_es()
|
||||
|
||||
return video
|
||||
|
||||
def archive_video(self, video):
|
||||
"""archive video file"""
|
||||
channel_id = video.json_data["channel"]["channel_id"]
|
||||
folder = os.path.join(self.VIDEOS_BASE, channel_id)
|
||||
if not os.path.exists(folder):
|
||||
os.makedirs(folder)
|
||||
if self.HOST_UID and self.HOST_GID:
|
||||
os.chown(folder, self.HOST_UID, self.HOST_GID)
|
||||
|
||||
new_path = os.path.join(folder, f"{video.youtube_id}.mp4")
|
||||
if self.file_path == new_path:
|
||||
# already archived
|
||||
return
|
||||
|
||||
shutil.move(self.file_path, new_path, copy_function=shutil.copyfile)
|
||||
if self.HOST_UID and self.HOST_GID:
|
||||
os.chown(new_path, self.HOST_UID, self.HOST_GID)
|
||||
|
||||
def restore_artwork(self, video):
|
||||
"""restore artwork if needed"""
|
||||
video_mutagen = MP4(self.file_path)
|
||||
|
||||
thumb = ThumbManager(video.youtube_id).vid_thumb_path(absolute=True)
|
||||
self._restore_art_item(video_mutagen, "covr", thumb)
|
||||
|
||||
channel_id = video.json_data["channel"]["channel_id"]
|
||||
banner_path = os.path.join(
|
||||
self.CACHE_DIR, "channels", f"{channel_id}_banner.jpg"
|
||||
)
|
||||
self._restore_art_item(
|
||||
video_mutagen, "----:com.tubearchivist:channel_banner", banner_path
|
||||
)
|
||||
|
||||
channel_icon = os.path.join(
|
||||
self.CACHE_DIR, "channels", f"{channel_id}_thumb.jpg"
|
||||
)
|
||||
self._restore_art_item(
|
||||
video_mutagen, "----:com.tubearchivist:channel_icon", channel_icon
|
||||
)
|
||||
|
||||
tv_art_path = os.path.join(
|
||||
self.CACHE_DIR, "channels", f"{channel_id}_tvart.jpg"
|
||||
)
|
||||
self._restore_art_item(
|
||||
video_mutagen, "----:com.tubearchivist:channel_tv", tv_art_path
|
||||
)
|
||||
|
||||
def _restore_art_item(
|
||||
self, video_mutagen, mutagen_key: str, target_path: str
|
||||
) -> None:
|
||||
"""restore single art item"""
|
||||
if os.path.exists(target_path):
|
||||
# don't overwrite
|
||||
return
|
||||
|
||||
art_item = video_mutagen.get(mutagen_key)
|
||||
if not art_item and not isinstance(art_item, list):
|
||||
# is not embedded
|
||||
return
|
||||
|
||||
with open(target_path, "wb") as f:
|
||||
f.write(bytes(art_item[0]))
|
||||
|
||||
def index_subtitles(self, json_embed, video):
|
||||
"""index subtitles"""
|
||||
subtitle_data = json_embed.get("subtitles")
|
||||
if not subtitle_data:
|
||||
return
|
||||
|
||||
serializer = SubtitleFragmentSerializer(data=subtitle_data, many=True)
|
||||
is_valid = serializer.is_valid()
|
||||
if not is_valid:
|
||||
err = serializer.errors
|
||||
raise ValueError(
|
||||
f"[{self.file_path}] subtitle serializer failed: {err}"
|
||||
)
|
||||
|
||||
self._process_embedded_subs(video, subtitle_data=serializer.data)
|
||||
|
||||
def _process_embedded_subs(self, video, subtitle_data):
|
||||
"""process single embedded subtitle"""
|
||||
embedded_subs = {
|
||||
(i["subtitle_lang"], i["subtitle_source"]) for i in subtitle_data
|
||||
}
|
||||
subs = YoutubeSubtitle(video)
|
||||
response = subs.get_es_subtitles()
|
||||
indexed = {
|
||||
(i["subtitle_lang"], i["subtitle_source"]) for i in response
|
||||
}
|
||||
|
||||
for embedded_lang, embedded_source in embedded_subs:
|
||||
needs_processing = self._process_subtitle(
|
||||
indexed, embedded_lang, embedded_source
|
||||
)
|
||||
if not needs_processing:
|
||||
continue
|
||||
|
||||
segments = [
|
||||
i
|
||||
for i in subtitle_data
|
||||
if i["subtitle_lang"] == embedded_lang
|
||||
and i["subtitle_source"] == embedded_source
|
||||
]
|
||||
to_index = sorted(segments, key=lambda d: d["subtitle_index"])
|
||||
parser = SubtitleParser(
|
||||
subtitle_str="{}", lang=embedded_lang, source=embedded_source
|
||||
)
|
||||
|
||||
for segment in to_index:
|
||||
parser.all_cues.append(
|
||||
{
|
||||
"start": segment["subtitle_start"],
|
||||
"end": segment["subtitle_end"],
|
||||
"text": segment["subtitle_line"],
|
||||
"idx": segment["subtitle_index"],
|
||||
}
|
||||
)
|
||||
|
||||
subtitle_str = parser.get_subtitle_str()
|
||||
query_str = parser.create_bulk_import(to_index)
|
||||
subs.index_subtitle(query_str)
|
||||
|
||||
media_url = subs.get_media_url(lang=embedded_lang)
|
||||
dest_path = os.path.join(self.VIDEOS_BASE, media_url)
|
||||
subs.write_subtitle_file(dest_path, subtitle_str)
|
||||
|
||||
def _process_subtitle(
|
||||
self, indexed, embedded_lang, embedded_source
|
||||
) -> bool:
|
||||
"""check if subtitle should be processed"""
|
||||
for sub_indexed in indexed:
|
||||
if (
|
||||
sub_indexed.get("lang") == embedded_lang
|
||||
and sub_indexed.get("source") == embedded_source
|
||||
):
|
||||
# already indexed
|
||||
return False
|
||||
|
||||
if not self.use_user_conf:
|
||||
return True
|
||||
|
||||
if not self.config:
|
||||
return False
|
||||
|
||||
langs = self.config["downloads"]["subtitle"]
|
||||
source = self.config["downloads"]["subtitle_source"]
|
||||
if not langs or not source:
|
||||
return False
|
||||
|
||||
lang_codes = [i.strip() for i in langs.split(",")]
|
||||
if embedded_lang not in lang_codes:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def index_comments(self, json_embed):
|
||||
"""index comments"""
|
||||
comment_data = json_embed.get("comments")
|
||||
if not comment_data:
|
||||
return
|
||||
|
||||
serializer = CommentsSerializer(data=comment_data)
|
||||
is_valid = serializer.is_valid()
|
||||
if not is_valid:
|
||||
err = serializer.errors
|
||||
raise ValueError(
|
||||
f"[{self.file_path}] comments serializer failed: {err}"
|
||||
)
|
||||
|
||||
if self.use_user_conf:
|
||||
if self.config and not self.config["downloads"]["comment_max"]:
|
||||
return
|
||||
|
||||
comments = Comments(youtube_id=serializer.data["youtube_id"])
|
||||
existing = comments.get_es_comments()
|
||||
if existing:
|
||||
return
|
||||
|
||||
comments.json_data = dict(serializer.data)
|
||||
comments.upload_comments()
|
||||
|
|
|
|||
|
|
@ -10,14 +10,25 @@ import os
|
|||
import re
|
||||
from datetime import datetime
|
||||
from operator import itemgetter
|
||||
from typing import TypedDict
|
||||
|
||||
import requests
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from common.src.helper import rand_sleep, requests_headers
|
||||
from download.src.yt_dlp_base import CookieHandler
|
||||
from yt_dlp.utils import orderedSet_from_options
|
||||
|
||||
|
||||
class SubtitleCue(TypedDict):
|
||||
"""describe single subtitle queue"""
|
||||
|
||||
start: str
|
||||
end: str
|
||||
text: str
|
||||
idx: int
|
||||
|
||||
|
||||
class YoutubeSubtitle:
|
||||
"""handle video subtitle functionality"""
|
||||
|
||||
|
|
@ -54,7 +65,9 @@ class YoutubeSubtitle:
|
|||
)
|
||||
]
|
||||
except re.error as e:
|
||||
raise ValueError(f"wrong regex in subtitle config: {e.pattern}")
|
||||
raise ValueError(
|
||||
f"wrong regex in subtitle config: {e.pattern}"
|
||||
) from e
|
||||
|
||||
return relevant_subtitles
|
||||
|
||||
|
|
@ -74,8 +87,7 @@ class YoutubeSubtitle:
|
|||
# not supported yet
|
||||
continue
|
||||
|
||||
video_media_url = self.video.json_data["media_url"]
|
||||
media_url = video_media_url.replace(".mp4", f".{lang}.vtt")
|
||||
media_url = self.get_media_url(lang)
|
||||
if not all_formats:
|
||||
# no subtitles found
|
||||
continue
|
||||
|
|
@ -93,6 +105,12 @@ class YoutubeSubtitle:
|
|||
|
||||
return candidate_subtitles
|
||||
|
||||
def get_media_url(self, lang: str) -> str:
|
||||
"""get media url"""
|
||||
video_media_url = self.video.json_data["media_url"]
|
||||
media_url = video_media_url.replace(".mp4", f".{lang}.vtt")
|
||||
return media_url
|
||||
|
||||
def get_es_subtitles(self) -> list[dict]:
|
||||
"""get subtitles from elastic"""
|
||||
data = {
|
||||
|
|
@ -113,39 +131,66 @@ class YoutubeSubtitle:
|
|||
dest_path = os.path.join(videos_base, subtitle["media_url"])
|
||||
source = subtitle["source"]
|
||||
lang = subtitle.get("lang")
|
||||
response = requests.get(
|
||||
subtitle["url"], headers=requests_headers(), timeout=30
|
||||
)
|
||||
if not response.ok:
|
||||
subtitle_key = f"{self.video.youtube_id}-{lang}"
|
||||
print(f"{subtitle_key}: failed to download subtitle")
|
||||
print(response.text)
|
||||
rand_sleep(self.video.config)
|
||||
|
||||
response_text = self._make_request(subtitle["url"], lang)
|
||||
if not response_text:
|
||||
continue
|
||||
|
||||
if not response.text:
|
||||
print(f"{subtitle_key}: skip empty subtitle")
|
||||
rand_sleep(self.video.config)
|
||||
continue
|
||||
|
||||
parser = SubtitleParser(response.text, lang, source)
|
||||
parser = SubtitleParser(response_text, lang, source)
|
||||
parser.process()
|
||||
if not parser.all_cues:
|
||||
rand_sleep(self.video.config)
|
||||
continue
|
||||
|
||||
subtitle_str = parser.get_subtitle_str()
|
||||
self._write_subtitle_file(dest_path, subtitle_str)
|
||||
self.write_subtitle_file(dest_path, subtitle_str)
|
||||
if self.video.config["downloads"]["subtitle_index"]:
|
||||
query_str = parser.create_bulk_import(self.video, source)
|
||||
self._index_subtitle(query_str)
|
||||
documents = parser.create_documents(self.video, source)
|
||||
query_str = parser.create_bulk_import(documents)
|
||||
self.index_subtitle(query_str)
|
||||
|
||||
indexed.append(subtitle)
|
||||
rand_sleep(self.video.config)
|
||||
|
||||
return indexed
|
||||
|
||||
def _write_subtitle_file(self, dest_path, subtitle_str):
|
||||
def _make_request(self, url: str, lang: str | None) -> str | None:
|
||||
"""make the request"""
|
||||
request_kwargs: dict = {
|
||||
"timeout": 30,
|
||||
"headers": requests_headers(),
|
||||
}
|
||||
if self.video.config["downloads"].get("cookie_import"):
|
||||
cookie = CookieHandler(self.video.config).get()
|
||||
if cookie:
|
||||
cookies_txt = cookie.read()
|
||||
jar = requests.cookies.RequestsCookieJar()
|
||||
for line in cookies_txt.split("\n"):
|
||||
words = line.split()
|
||||
if (len(words) == 7) and (words[0] != "#"):
|
||||
jar.set(
|
||||
words[5], words[6], domain=words[0], path=words[2]
|
||||
)
|
||||
|
||||
request_kwargs["cookies"] = jar
|
||||
|
||||
response = requests.get(url, **request_kwargs)
|
||||
|
||||
if not response.ok:
|
||||
subtitle_key = f"{self.video.youtube_id}-{lang}"
|
||||
print(f"{subtitle_key}: failed to download subtitle")
|
||||
print(response.text)
|
||||
rand_sleep(self.video.config)
|
||||
return None
|
||||
|
||||
if not response.text:
|
||||
print(f"{subtitle_key}: skip empty subtitle")
|
||||
rand_sleep(self.video.config)
|
||||
return None
|
||||
|
||||
return response.text
|
||||
|
||||
def write_subtitle_file(self, dest_path, subtitle_str):
|
||||
"""write subtitle file to disk"""
|
||||
# create folder here for first video of channel
|
||||
os.makedirs(os.path.split(dest_path)[0], exist_ok=True)
|
||||
|
|
@ -158,7 +203,7 @@ class YoutubeSubtitle:
|
|||
os.chown(dest_path, host_uid, host_gid)
|
||||
|
||||
@staticmethod
|
||||
def _index_subtitle(query_str):
|
||||
def index_subtitle(query_str):
|
||||
"""send subtitle to es for indexing"""
|
||||
_, _ = ElasticWrap("_bulk").post(data=query_str, ndjson=True)
|
||||
|
||||
|
|
@ -190,15 +235,14 @@ class YoutubeSubtitle:
|
|||
class SubtitleParser:
|
||||
"""parse subtitle str from youtube"""
|
||||
|
||||
def __init__(self, subtitle_str, lang, source):
|
||||
def __init__(self, subtitle_str: str, lang: str, source: str) -> None:
|
||||
self.subtitle_raw = json.loads(subtitle_str)
|
||||
self.lang = lang
|
||||
self.source = source
|
||||
self.all_cues = False
|
||||
self.all_cues: list[SubtitleCue] = []
|
||||
|
||||
def process(self):
|
||||
def process(self) -> None:
|
||||
"""extract relevant que data"""
|
||||
self.all_cues = []
|
||||
all_events = self.subtitle_raw.get("events")
|
||||
|
||||
if not all_events:
|
||||
|
|
@ -213,12 +257,12 @@ class SubtitleParser:
|
|||
print(f"skipping subtitle event without content: {event}")
|
||||
continue
|
||||
|
||||
cue = {
|
||||
"start": self._ms_conv(event["tStartMs"]),
|
||||
"end": self._ms_conv(event["tStartMs"] + event["dDurationMs"]),
|
||||
"text": "".join([i.get("utf8") for i in event["segs"]]),
|
||||
"idx": idx + 1,
|
||||
}
|
||||
cue = SubtitleCue(
|
||||
start=self._ms_conv(event["tStartMs"]),
|
||||
end=self._ms_conv(event["tStartMs"] + event["dDurationMs"]),
|
||||
text="".join([i.get("utf8") for i in event["segs"]]),
|
||||
idx=idx + 1,
|
||||
)
|
||||
self.all_cues.append(cue)
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -272,9 +316,8 @@ class SubtitleParser:
|
|||
|
||||
return subtitle_str
|
||||
|
||||
def create_bulk_import(self, video, source):
|
||||
def create_bulk_import(self, documents):
|
||||
"""subtitle lines for es import"""
|
||||
documents = self._create_documents(video, source)
|
||||
bulk_list = []
|
||||
|
||||
for document in documents:
|
||||
|
|
@ -288,7 +331,7 @@ class SubtitleParser:
|
|||
|
||||
return query_str
|
||||
|
||||
def _create_documents(self, video, source):
|
||||
def create_documents(self, video, source):
|
||||
"""process documents"""
|
||||
documents = self._chunk_list(video.youtube_id)
|
||||
channel = video.json_data.get("channel")
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -11,27 +11,27 @@
|
|||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"dompurify": "^3.2.6",
|
||||
"react": "^19.1.1",
|
||||
"react-dom": "^19.1.1",
|
||||
"react-router-dom": "^7.8.0",
|
||||
"zustand": "^5.0.7"
|
||||
"dompurify": "^3.3.1",
|
||||
"react": "^19.2.1",
|
||||
"react-dom": "^19.2.1",
|
||||
"react-router-dom": "^7.10.1",
|
||||
"zustand": "^5.0.9"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^19.1.9",
|
||||
"@types/react-dom": "^19.1.7",
|
||||
"@types/react": "^19.2.7",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@typescript-eslint/eslint-plugin": "^8.39.0",
|
||||
"@typescript-eslint/parser": "^8.39.0",
|
||||
"@vitejs/plugin-react-swc": "^4.0.0",
|
||||
"eslint": "^9.33.0",
|
||||
"@vitejs/plugin-react-swc": "^4.2.2",
|
||||
"eslint": "^9.39.1",
|
||||
"eslint-config-prettier": "^10.1.8",
|
||||
"eslint-plugin-react-hooks": "^5.2.0",
|
||||
"eslint-plugin-react-refresh": "^0.4.20",
|
||||
"globals": "^16.3.0",
|
||||
"prettier": "3.6.2",
|
||||
"typescript": "^5.9.2",
|
||||
"typescript-eslint": "^8.39.0",
|
||||
"vite": ">=7.1.1",
|
||||
"vite-plugin-checker": "^0.10.2"
|
||||
"eslint-plugin-react-hooks": "^7.0.1",
|
||||
"eslint-plugin-react-refresh": "^0.4.24",
|
||||
"globals": "^16.5.0",
|
||||
"prettier": "3.7.4",
|
||||
"typescript": "^5.9.3",
|
||||
"typescript-eslint": "^8.49.0",
|
||||
"vite": ">=7.2.7",
|
||||
"vite-plugin-checker": "^0.12.0"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
import APIClient from '../../functions/APIClient';
|
||||
|
||||
const queueManualImport = async (ignore_error: boolean, prefer_local: boolean) => {
|
||||
return APIClient('/api/appsettings/manual-import/', {
|
||||
method: 'POST',
|
||||
body: { ignore_error, prefer_local },
|
||||
});
|
||||
};
|
||||
export default queueManualImport;
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
import APIClient from '../../functions/APIClient';
|
||||
|
||||
const queueStartFilesystemRescan = async (ignore_error: boolean, prefer_local: boolean) => {
|
||||
return APIClient('/api/appsettings/rescan-filesystem/', {
|
||||
method: 'POST',
|
||||
body: { ignore_error, prefer_local },
|
||||
});
|
||||
};
|
||||
export default queueStartFilesystemRescan;
|
||||
|
|
@ -1,12 +1,6 @@
|
|||
import APIClient from '../../functions/APIClient';
|
||||
|
||||
type TaskNamesType =
|
||||
| 'download_pending'
|
||||
| 'update_subscribed'
|
||||
| 'manual_import'
|
||||
| 'resync_thumbs'
|
||||
| 'resync_metadata'
|
||||
| 'rescan_filesystem';
|
||||
type TaskNamesType = 'download_pending' | 'update_subscribed' | 'resync_thumbs' | 'resync_metadata';
|
||||
|
||||
const updateTaskByName = async (taskName: TaskNamesType) => {
|
||||
return APIClient(`/api/task/by-name/${taskName}/`, {
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ const Filterbar = ({
|
|||
}
|
||||
|
||||
if (currentViewStyle === ViewStylesEnum.Table) {
|
||||
// eslint-disable-next-line react-hooks/set-state-in-effect
|
||||
setShowHidden(true);
|
||||
} else {
|
||||
setShowHidden(false);
|
||||
|
|
|
|||
|
|
@ -104,6 +104,10 @@ const SearchExampleQueries = () => {
|
|||
auto-generated subtitles only, or <i>user</i> to search through user-uploaded
|
||||
subtitles only
|
||||
</li>
|
||||
<li>
|
||||
<span>channel:</span> — limit subtitle search to a specific channel name (for
|
||||
example: <code>full:javascript channel:corey schafer</code>)
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
|
|
|
|||
|
|
@ -146,6 +146,7 @@ const VideoPlayer = ({
|
|||
}
|
||||
if (setSeekToTimestamp) setSeekToTimestamp(undefined);
|
||||
window.scroll(0, 0);
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [seekToTimestamp]);
|
||||
|
||||
const [searchParams] = useSearchParams();
|
||||
|
|
@ -389,6 +390,7 @@ const VideoPlayer = ({
|
|||
} else if (!theaterModePressed) {
|
||||
setTheaterModeKeyPressed(false);
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [theaterModePressed, isTheaterMode, theaterModeKeyPressed]);
|
||||
|
||||
useEffect(() => {
|
||||
|
|
@ -401,6 +403,7 @@ const VideoPlayer = ({
|
|||
|
||||
infoDialog('Normal mode');
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [escapePressed, isTheaterMode]);
|
||||
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -108,7 +108,6 @@ const Playlist = () => {
|
|||
}
|
||||
setRefresh(false);
|
||||
})();
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [
|
||||
playlistId,
|
||||
userConfig.hide_watched_playlist,
|
||||
|
|
|
|||
|
|
@ -65,6 +65,13 @@ const Search = () => {
|
|||
const gridView = isGridView ? `boxed-${gridItems}` : '';
|
||||
const gridViewGrid = isGridView ? `grid-${gridItems}` : '';
|
||||
|
||||
const fetchResults = async (searchQuery: string) => {
|
||||
const searchResults = await loadSearch(searchQuery);
|
||||
|
||||
setSearchResults(searchResults);
|
||||
setRefresh(false);
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
const handler = setTimeout(() => {
|
||||
setDebouncedSearchTerm(searchTerm);
|
||||
|
|
@ -77,19 +84,13 @@ const Search = () => {
|
|||
|
||||
useEffect(() => {
|
||||
if (debouncedSearchTerm.trim() !== '') {
|
||||
// eslint-disable-next-line react-hooks/set-state-in-effect
|
||||
fetchResults(debouncedSearchTerm);
|
||||
} else {
|
||||
setSearchResults(EmptySearchResponse);
|
||||
}
|
||||
}, [debouncedSearchTerm, refresh, videoId]);
|
||||
|
||||
const fetchResults = async (searchQuery: string) => {
|
||||
const searchResults = await loadSearch(searchQuery);
|
||||
|
||||
setSearchResults(searchResults);
|
||||
setRefresh(false);
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<title>TubeArchivist</title>
|
||||
|
|
|
|||
|
|
@ -7,6 +7,9 @@ import restoreBackup from '../api/actions/restoreBackup';
|
|||
import Notifications from '../components/Notifications';
|
||||
import Button from '../components/Button';
|
||||
import { ApiResponseType } from '../functions/APIClient';
|
||||
import ToggleConfig from '../components/ToggleConfig';
|
||||
import queueStartFilesystemRescan from '../api/actions/queueStartFilesystemRescan';
|
||||
import queueManualImport from '../api/actions/queueManualImport';
|
||||
|
||||
const SettingsActions = () => {
|
||||
const [deleteIgnored, setDeleteIgnored] = useState(false);
|
||||
|
|
@ -17,6 +20,10 @@ const SettingsActions = () => {
|
|||
const [backupStarted, setBackupStarted] = useState(false);
|
||||
const [isRestoringBackup, setIsRestoringBackup] = useState(false);
|
||||
const [reScanningFileSystem, setReScanningFileSystem] = useState(false);
|
||||
const [rescanIgnoreErrors, setRescanIgnoreErrors] = useState(false);
|
||||
const [rescanPreferLocal, setRescanPreferLocal] = useState(false);
|
||||
const [manualPreferLocal, setManualPreferLocal] = useState(false);
|
||||
const [manualIgnoreErrors, setManualIgnoreErrors] = useState(false);
|
||||
|
||||
const [backupListResponse, setBackupListResponse] = useState<ApiResponseType<BackupListType>>();
|
||||
|
||||
|
|
@ -69,22 +76,42 @@ const SettingsActions = () => {
|
|||
<h2>Manual media files import.</h2>
|
||||
<p>
|
||||
Add files to the <span className="settings-current">cache/import</span> folder. Make
|
||||
sure to follow the instructions in the Github{' '}
|
||||
sure to follow the instructions in the{' '}
|
||||
<a
|
||||
href="https://docs.tubearchivist.com/settings/actions/#manual-media-files-import"
|
||||
target="_blank"
|
||||
>
|
||||
Wiki
|
||||
Docs
|
||||
</a>
|
||||
.
|
||||
</p>
|
||||
<div id="manual-import">
|
||||
<div className="settings-box-wrapper">
|
||||
<div>
|
||||
<p>Prefer embedded metadata</p>
|
||||
</div>
|
||||
<ToggleConfig
|
||||
name="manual_prefer_local"
|
||||
value={manualPreferLocal}
|
||||
updateCallback={() => setManualPreferLocal(!manualPreferLocal)}
|
||||
/>
|
||||
</div>
|
||||
<div className="settings-box-wrapper">
|
||||
<div>
|
||||
<p>Ignore missing metadata errors</p>
|
||||
</div>
|
||||
<ToggleConfig
|
||||
name="manual_ignore_error"
|
||||
value={manualIgnoreErrors}
|
||||
updateCallback={() => setManualIgnoreErrors(!manualIgnoreErrors)}
|
||||
/>
|
||||
</div>
|
||||
{processingImports && <p>Processing import</p>}
|
||||
{!processingImports && (
|
||||
<Button
|
||||
label="Start import"
|
||||
onClick={async () => {
|
||||
await updateTaskByName('manual_import');
|
||||
await queueManualImport(manualIgnoreErrors, manualPreferLocal);
|
||||
setProcessingImports(true);
|
||||
}}
|
||||
/>
|
||||
|
|
@ -196,23 +223,42 @@ const SettingsActions = () => {
|
|||
deleted videos from the filesystem.
|
||||
</p>
|
||||
<p>
|
||||
Rescan your media folder looking for missing videos and clean up index. More info on the
|
||||
Github{' '}
|
||||
Rescan your media folder looking for missing videos and clean up index. More info on the{' '}
|
||||
<a
|
||||
href="https://docs.tubearchivist.com/settings/actions/#rescan-filesystem"
|
||||
target="_blank"
|
||||
>
|
||||
Wiki
|
||||
Docs
|
||||
</a>
|
||||
.
|
||||
</p>
|
||||
<div id="fs-rescan">
|
||||
<div className="settings-box-wrapper">
|
||||
<div>
|
||||
<p>Prefer embedded metadata</p>
|
||||
</div>
|
||||
<ToggleConfig
|
||||
name="prefer_local"
|
||||
value={rescanPreferLocal}
|
||||
updateCallback={() => setRescanPreferLocal(!rescanPreferLocal)}
|
||||
/>
|
||||
</div>
|
||||
<div className="settings-box-wrapper">
|
||||
<div>
|
||||
<p>Ignore missing metadata errors</p>
|
||||
</div>
|
||||
<ToggleConfig
|
||||
name="ignore_error"
|
||||
value={rescanIgnoreErrors}
|
||||
updateCallback={() => setRescanIgnoreErrors(!rescanIgnoreErrors)}
|
||||
/>
|
||||
</div>
|
||||
{reScanningFileSystem && <p>File system scan in progress</p>}
|
||||
{!reScanningFileSystem && (
|
||||
<Button
|
||||
label="Rescan filesystem"
|
||||
onClick={async () => {
|
||||
await updateTaskByName('rescan_filesystem');
|
||||
await queueStartFilesystemRescan(rescanIgnoreErrors, rescanPreferLocal);
|
||||
setReScanningFileSystem(true);
|
||||
}}
|
||||
/>
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ const SettingsApplication = () => {
|
|||
const { userConfig } = useUserConfigStore();
|
||||
const [response, setResponse] = useState<SettingsApplicationReponses>();
|
||||
const [refresh, setRefresh] = useState(false);
|
||||
const [visibleSnapshotCount, setVisibleSnapshotCount] = useState(10);
|
||||
|
||||
const snapshots = response?.snapshots;
|
||||
const appSettingsConfig = response?.appSettingsConfig;
|
||||
|
|
@ -182,11 +183,13 @@ const SettingsApplication = () => {
|
|||
};
|
||||
|
||||
useEffect(() => {
|
||||
// eslint-disable-next-line react-hooks/set-state-in-effect
|
||||
fetchData();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (refresh) {
|
||||
// eslint-disable-next-line react-hooks/set-state-in-effect
|
||||
fetchData();
|
||||
setRefresh(false);
|
||||
}
|
||||
|
|
@ -974,10 +977,9 @@ const SettingsApplication = () => {
|
|||
</p>
|
||||
<br />
|
||||
{restoringSnapshot && <p>Snapshot restore started</p>}
|
||||
{!restoringSnapshot &&
|
||||
snapshots.snapshots &&
|
||||
snapshots.snapshots.map(snapshot => {
|
||||
return (
|
||||
{!restoringSnapshot && snapshots.snapshots && (
|
||||
<>
|
||||
{snapshots.snapshots?.slice(0, visibleSnapshotCount).map(snapshot => (
|
||||
<p key={snapshot.id}>
|
||||
<Button
|
||||
label="Restore"
|
||||
|
|
@ -992,8 +994,17 @@ const SettingsApplication = () => {
|
|||
<span className="settings-current">{snapshot.duration_s}s</span> to
|
||||
create. State: <i>{snapshot.state}</i>
|
||||
</p>
|
||||
);
|
||||
})}
|
||||
))}
|
||||
{visibleSnapshotCount < snapshots.snapshots.length && (
|
||||
<Button
|
||||
label="Load More"
|
||||
onClick={() => {
|
||||
setVisibleSnapshotCount(visibleSnapshotCount + 10);
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -103,6 +103,7 @@ const SettingsScheduling = () => {
|
|||
}, [refresh]);
|
||||
|
||||
useEffect(() => {
|
||||
// eslint-disable-next-line react-hooks/set-state-in-effect
|
||||
setRefresh(true);
|
||||
}, []);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
-r backend/requirements.txt
|
||||
ipython==9.7.0
|
||||
pre-commit==4.4.0
|
||||
ipython==9.8.0
|
||||
pre-commit==4.5.0
|
||||
pylint-django==2.6.1
|
||||
pylint==3.3.9
|
||||
pytest-django==4.11.1
|
||||
pytest==9.0.1
|
||||
pytest==9.0.2
|
||||
python-dotenv==1.2.1
|
||||
requirementscheck==0.1.0
|
||||
types-requests==2.32.4.20250913
|
||||
|
|
|
|||
Loading…
Reference in New Issue