Add Deno, #build
Changed: - Add deno runtime to container - Add full metadata embedding - Fix backend refresh timestamp offset
This commit is contained in:
Simon
commit
e0d493ae6e
|
|
@ -42,6 +42,8 @@ ARG INSTALL_DEBUG
|
|||
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
COPY --from=denoland/deno:bin /deno /usr/local/bin/deno
|
||||
|
||||
# copy build requirements
|
||||
COPY --from=builder /root/.local /root/.local
|
||||
ENV PATH=/root/.local/bin:$PATH
|
||||
|
|
|
|||
|
|
@ -345,6 +345,7 @@ class Reindex(ReindexBase):
|
|||
thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
|
||||
|
||||
Comments(youtube_id, config=self.config).reindex_comments()
|
||||
video.embed_metadata()
|
||||
self.processed["videos"] += 1
|
||||
|
||||
def _reindex_single_channel(self, channel_id: str) -> None:
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
"""middleware"""
|
||||
|
||||
from django.conf import settings
|
||||
from django.utils.deprecation import MiddlewareMixin
|
||||
|
||||
|
||||
class StartTimeMiddleware(MiddlewareMixin):
|
||||
"""add a start time header"""
|
||||
class StartTimeMiddleware:
|
||||
"""set start time header"""
|
||||
|
||||
def __init__(self, get_response):
|
||||
self.get_response = get_response
|
||||
|
||||
def __call__(self, request):
|
||||
response = self.get_response(request)
|
||||
|
|
|
|||
|
|
@ -228,7 +228,7 @@ CORS_EXPOSE_HEADERS = ["X-Start-Timestamp"]
|
|||
# TA application settings
|
||||
TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
|
||||
TA_VERSION = "v0.5.8-unstable"
|
||||
TA_START = str(int(datetime.now().timestamp()))
|
||||
TA_START = str(int(datetime.now().timestamp() // 10 * 10)) # round it
|
||||
|
||||
# API
|
||||
REST_FRAMEWORK = {
|
||||
|
|
|
|||
|
|
@ -462,8 +462,8 @@ class PendingList(PendingIndex):
|
|||
response, status_code = ElasticWrap("_bulk").post(
|
||||
query_str, ndjson=True
|
||||
)
|
||||
print(response)
|
||||
if status_code != 200:
|
||||
if status_code not in [200, 201]:
|
||||
print(response)
|
||||
self._notify_fail(status_code)
|
||||
elif response.get("errors", False):
|
||||
failed_video_ids = []
|
||||
|
|
|
|||
|
|
@ -188,22 +188,11 @@ class VideoDownloader(DownloaderBase):
|
|||
postprocessors = []
|
||||
|
||||
if self.config["downloads"]["add_metadata"]:
|
||||
# full metadata is added in DownloadPostProcess
|
||||
postprocessors.append(
|
||||
{
|
||||
"key": "FFmpegMetadata",
|
||||
"add_chapters": True,
|
||||
"add_metadata": True,
|
||||
}
|
||||
)
|
||||
postprocessors.append(
|
||||
{
|
||||
"key": "MetadataFromField",
|
||||
"formats": [
|
||||
"%(title)s:%(meta_title)s",
|
||||
"%(uploader)s:%(meta_artist)s",
|
||||
":(?P<album>)",
|
||||
],
|
||||
"when": "pre_process",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -303,6 +292,9 @@ class DownloadPostProcess(DownloaderBase):
|
|||
self.refresh_playlist()
|
||||
self.match_videos()
|
||||
self.get_comments()
|
||||
self.embed_metadata()
|
||||
|
||||
RedisQueue(self.VIDEO_QUEUE).clear()
|
||||
|
||||
def auto_delete_all(self):
|
||||
"""handle auto delete"""
|
||||
|
|
@ -488,6 +480,26 @@ class DownloadPostProcess(DownloaderBase):
|
|||
video_queue = RedisQueue(self.VIDEO_QUEUE)
|
||||
comment_list = CommentList(task=self.task)
|
||||
comment_list.add(video_ids=video_queue.get_all())
|
||||
|
||||
video_queue.clear()
|
||||
comment_list.index()
|
||||
|
||||
def embed_metadata(self):
|
||||
"""embed metadata in media file"""
|
||||
if not self.config["downloads"].get("add_metadata"):
|
||||
return
|
||||
|
||||
queue = RedisQueue(self.VIDEO_QUEUE)
|
||||
total = queue.max_score()
|
||||
video_ids = queue.get_all()
|
||||
|
||||
for idx, youtube_id in enumerate(video_ids):
|
||||
YoutubeVideo(youtube_id).embed_metadata()
|
||||
|
||||
if not self.task:
|
||||
continue
|
||||
|
||||
message = [
|
||||
"Post Processing Videos.",
|
||||
f"Embed metadata: - {idx}/{total}",
|
||||
]
|
||||
progress = idx / total
|
||||
self.task.send_progress(message, progress=progress)
|
||||
|
|
|
|||
|
|
@ -12,4 +12,4 @@ requests==2.32.5
|
|||
ryd-client==0.0.6
|
||||
uvicorn==0.38.0
|
||||
whitenoise==6.11.0
|
||||
yt-dlp[default]==2025.10.22
|
||||
yt-dlp[default] @ git+https://github.com/yt-dlp/yt-dlp@6224a3898821965a7d6a2cb9cc2de40a0fd6e6bc
|
||||
|
|
|
|||
|
|
@ -87,6 +87,13 @@ RESYNC_THUMBS: TaskItemConfig = {
|
|||
"api_stop": False,
|
||||
}
|
||||
|
||||
RESYNC_METADATA: TaskItemConfig = {
|
||||
"title": "Sync Metadata to Media Files",
|
||||
"group": "setting:thumbnailsync",
|
||||
"api_start": True,
|
||||
"api_stop": False,
|
||||
}
|
||||
|
||||
INDEX_PLAYLISTS: TaskItemConfig = {
|
||||
"title": "Index Channel Playlist",
|
||||
"group": "channel:indexplaylist",
|
||||
|
|
@ -119,6 +126,7 @@ TASK_CONFIG: dict[str, TaskItemConfig] = {
|
|||
"rescan_filesystem": RESCAN_FILESYSTEM,
|
||||
"thumbnail_check": THUMBNAIL_CHECK,
|
||||
"resync_thumbs": RESYNC_THUMBS,
|
||||
"resync_metadata": RESYNC_METADATA,
|
||||
"index_playlists": INDEX_PLAYLISTS,
|
||||
"subscribe_to": SUBSCRIBE_TO,
|
||||
"version_check": VERSION_CHECK,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from download.src.yt_dlp_handler import VideoDownloader
|
|||
from task.src.notify import Notifications
|
||||
from task.src.task_config import TASK_CONFIG
|
||||
from task.src.task_manager import TaskManager
|
||||
from video.src.meta_embed import MetadataEmbed
|
||||
|
||||
|
||||
class BaseTask(Task):
|
||||
|
|
@ -302,6 +303,19 @@ def re_sync_thumbs(self):
|
|||
ThumbFilesystem(task=self).embed()
|
||||
|
||||
|
||||
@shared_task(bind=True, name="resync_metadata", base=BaseTask)
|
||||
def re_sync_metadata(self):
|
||||
"""resync metadata to media files"""
|
||||
manager = TaskManager()
|
||||
if manager.is_pending(self):
|
||||
print(f"[task][{self.name}] metadata re-embed is already running")
|
||||
self.send_progress(["Metadata re-embed is already running."])
|
||||
return
|
||||
|
||||
manager.init(self)
|
||||
MetadataEmbed(task=self).embed()
|
||||
|
||||
|
||||
@shared_task(bind=True, name="subscribe_to", base=BaseTask)
|
||||
def subscribe_to(self, url_str: str, expected_type: str | bool = False):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ functionality:
|
|||
- index and update in es
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
|
|
@ -14,6 +15,7 @@ from common.src.helper import get_duration_sec, get_duration_str, randomizor
|
|||
from common.src.index_generic import YouTubeItem
|
||||
from django.conf import settings
|
||||
from download.src.thumbnails import ThumbManager
|
||||
from mutagen.mp4 import MP4
|
||||
from playlist.src import index as ta_playlist
|
||||
from ryd_client import ryd_client
|
||||
from user.src.user_config import UserConfig
|
||||
|
|
@ -420,6 +422,68 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
|||
|
||||
return subtitles
|
||||
|
||||
def embed_metadata(self):
|
||||
"""embed metadata for video"""
|
||||
if not self.config["downloads"].get("add_metadata"):
|
||||
return
|
||||
|
||||
print(f"{self.youtube_id}: embed metadata")
|
||||
if not self.json_data:
|
||||
self.get_from_es()
|
||||
|
||||
if not self.json_data:
|
||||
print(f"{self.youtube_id}: skip embed, video not indexed")
|
||||
return
|
||||
|
||||
video_base = EnvironmentSettings.MEDIA_DIR
|
||||
media_url = self.json_data.get("media_url")
|
||||
file_path = os.path.join(video_base, media_url)
|
||||
if not os.path.exists(file_path):
|
||||
print(f"{self.youtube_id}: skip embed, file not found")
|
||||
return
|
||||
|
||||
title = self.json_data["title"]
|
||||
artist = self.json_data["channel"]["channel_name"]
|
||||
description = self.json_data["description"]
|
||||
to_embed = self._get_to_embed()
|
||||
|
||||
video = MP4(file_path)
|
||||
video["\xa9nam"] = [title] # title
|
||||
video["\xa9ART"] = [artist] # artist
|
||||
video["desc"] = [description] # description
|
||||
video["ldes"] = [description] # synopsis
|
||||
video["----:com.tubearchivist:ta"] = [to_embed.encode("utf-8")]
|
||||
video.save()
|
||||
|
||||
def _get_to_embed(self) -> str:
|
||||
"""get metadata json str to embed"""
|
||||
comments = None
|
||||
if self.json_data.get("comment_count"):
|
||||
comments = Comments(self.youtube_id).get_es_comments()
|
||||
|
||||
subtitles = None
|
||||
if self.json_data.get("subtitles"):
|
||||
subtitles = YoutubeSubtitle(video=self).get_es_subtitles()
|
||||
|
||||
playlists = None
|
||||
if self.json_data.get("playlist"):
|
||||
playlists = []
|
||||
for playlist_id in self.json_data["playlist"]:
|
||||
playlist = ta_playlist.YoutubePlaylist(playlist_id)
|
||||
playlist.get_from_es()
|
||||
playlists.append(playlist.json_data)
|
||||
|
||||
to_embed = json.dumps(
|
||||
{
|
||||
"video": self.json_data,
|
||||
"comments": comments,
|
||||
"subtitles": subtitles,
|
||||
"playlists": playlists,
|
||||
}
|
||||
)
|
||||
|
||||
return to_embed
|
||||
|
||||
|
||||
def index_new_video(youtube_id, video_type=VideoTypeEnum.VIDEOS):
|
||||
"""combined classes to create new video in index"""
|
||||
|
|
|
|||
|
|
@ -0,0 +1,51 @@
|
|||
"""bulk metadata embedding"""
|
||||
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from video.src.index import YoutubeVideo
|
||||
|
||||
|
||||
class MetadataEmbed:
|
||||
"""sync metadata to videos in bulk"""
|
||||
|
||||
INDEX_NAME = "ta_video"
|
||||
|
||||
def __init__(self, task=False):
|
||||
self.task = task
|
||||
|
||||
def embed(self):
|
||||
"""entry point"""
|
||||
data = {
|
||||
"query": {"match_all": {}},
|
||||
"_source": ["youtube_id"],
|
||||
}
|
||||
paginate = IndexPaginate(
|
||||
index_name=self.INDEX_NAME,
|
||||
data=data,
|
||||
size=200,
|
||||
callback=MetadataEmbedCallback,
|
||||
task=self.task,
|
||||
total=self._get_total(),
|
||||
)
|
||||
_ = paginate.get_results()
|
||||
|
||||
def _get_total(self):
|
||||
"""get total documents in index"""
|
||||
path = f"{self.INDEX_NAME}/_count"
|
||||
response, _ = ElasticWrap(path).get()
|
||||
|
||||
return response.get("count")
|
||||
|
||||
|
||||
class MetadataEmbedCallback:
|
||||
"""callback for metadata embed"""
|
||||
|
||||
def __init__(self, source, index_name, counter=0):
|
||||
self.source = source
|
||||
self.index_name = index_name
|
||||
self.counter = counter
|
||||
|
||||
def run(self):
|
||||
"""run embed"""
|
||||
for video in self.source:
|
||||
youtube_id = video["_source"]["youtube_id"]
|
||||
YoutubeVideo(youtube_id).embed_metadata()
|
||||
|
|
@ -13,7 +13,7 @@ from operator import itemgetter
|
|||
|
||||
import requests
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from common.src.helper import rand_sleep, requests_headers
|
||||
from yt_dlp.utils import orderedSet_from_options
|
||||
|
||||
|
|
@ -93,6 +93,14 @@ class YoutubeSubtitle:
|
|||
|
||||
return candidate_subtitles
|
||||
|
||||
def get_es_subtitles(self) -> list[dict]:
|
||||
"""get subtitles from elastic"""
|
||||
data = {
|
||||
"query": {"term": {"youtube_id": {"value": self.video.youtube_id}}}
|
||||
}
|
||||
response = IndexPaginate("ta_subtitle", data).get_results()
|
||||
return response
|
||||
|
||||
def download_subtitles(self, relevant_subtitles):
|
||||
"""download subtitle files to archive"""
|
||||
subtitle_list = ", ".join(map(itemgetter("lang"), relevant_subtitles))
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ type TaskNamesType =
|
|||
| 'update_subscribed'
|
||||
| 'manual_import'
|
||||
| 'resync_thumbs'
|
||||
| 'resync_metadata'
|
||||
| 'rescan_filesystem';
|
||||
|
||||
const updateTaskByName = async (taskName: TaskNamesType) => {
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ const SettingsActions = () => {
|
|||
const [deletePending, setDeletePending] = useState(false);
|
||||
const [processingImports, setProcessingImports] = useState(false);
|
||||
const [reEmbed, setReEmbed] = useState(false);
|
||||
const [reSyncMeta, setReSyncMeta] = useState(false);
|
||||
const [backupStarted, setBackupStarted] = useState(false);
|
||||
const [isRestoringBackup, setIsRestoringBackup] = useState(false);
|
||||
const [reScanningFileSystem, setReScanningFileSystem] = useState(false);
|
||||
|
|
@ -44,6 +45,7 @@ const SettingsActions = () => {
|
|||
deletePending ||
|
||||
processingImports ||
|
||||
reEmbed ||
|
||||
reSyncMeta ||
|
||||
backupStarted ||
|
||||
isRestoringBackup ||
|
||||
reScanningFileSystem
|
||||
|
|
@ -53,6 +55,7 @@ const SettingsActions = () => {
|
|||
setDeletePending(false);
|
||||
setProcessingImports(false);
|
||||
setReEmbed(false);
|
||||
setReSyncMeta(false);
|
||||
setBackupStarted(false);
|
||||
setIsRestoringBackup(false);
|
||||
setReScanningFileSystem(false);
|
||||
|
|
@ -104,6 +107,22 @@ const SettingsActions = () => {
|
|||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="settings-group">
|
||||
<h2>Embed metadata into media file</h2>
|
||||
<p>Embed metadata into media files as mp4 tags.</p>
|
||||
<div id="re-meta">
|
||||
{reSyncMeta && <p>Processing video metadata</p>}
|
||||
{!reSyncMeta && (
|
||||
<Button
|
||||
label="Start process"
|
||||
onClick={async () => {
|
||||
await updateTaskByName('resync_metadata');
|
||||
setReSyncMeta(true);
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="settings-group">
|
||||
<h2>ZIP file index backup</h2>
|
||||
<p>
|
||||
|
|
|
|||
Loading…
Reference in New Issue