Merge branch 'develop' into pr_test_1082
This commit is contained in:
Simon
commit
95525fc9f0
|
|
@ -18,7 +18,7 @@ Welcome, and thanks for showing interest in improving Tube Archivist!
|
|||
## Beta Testing
|
||||
Be the first to help test new features/improvements and provide feedback! Regular `:unstable` builds are available for early access. These are for the tinkerers and the brave. Ideally, use a testing environment first, before upgrading your main installation.
|
||||
|
||||
There is always something that can get missed during development. Look at the commit messages tagged with `#build` - these are the unstable builds and give a quick overview of what has changed.
|
||||
There is always something that can get missed during development. Look at the commit messages tagged with [`#build`](https://github.com/search?q=repo%3Atubearchivist%2Ftubearchivist+%22%23build%22&type=commits&s=committer-date&o=desc) - these are the unstable builds and give a quick overview of what has changed.
|
||||
|
||||
- Test the features mentioned, play around, try to break it.
|
||||
- Test the update path by installing the `:latest` release first, then upgrade to `:unstable` to check for any errors.
|
||||
|
|
@ -41,6 +41,7 @@ Please read this carefully before opening any [issue](https://github.com/tubearc
|
|||
- Don't open an issue for something that's already on the [roadmap](https://github.com/tubearchivist/tubearchivist#roadmap), this needs your help to implement it, not another issue.
|
||||
- Don't open an issue for something that's a [known limitation](https://github.com/tubearchivist/tubearchivist#known-limitations). These are *known* by definition and don't need another reminder. Some limitations may be solved in the future, maybe by you?
|
||||
- Don't overwrite the *issue template*, they are there for a reason. Overwriting that shows that you don't really care about this project. It shows that you have a misunderstanding how open source collaboration works and just want to push your ideas through. Overwriting the template may result in a ban.
|
||||
- Don't redirect people trying to help to other platforms. E.g. in this context, imgur or other image hosting platforms are not needed, you can add an image directly in the issue as an attachments. Same goes for log files.
|
||||
|
||||
### Bug Report
|
||||
Bug reports are highly welcome! This project has improved a lot due to your help by providing feedback when something doesn't work as expected. The developers can't possibly cover all edge cases in an ever changing environment like YouTube and yt-dlp.
|
||||
|
|
|
|||
|
|
@ -42,6 +42,8 @@ ARG INSTALL_DEBUG
|
|||
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
COPY --from=denoland/deno:bin /deno /usr/local/bin/deno
|
||||
|
||||
# copy build requirements
|
||||
COPY --from=builder /root/.local /root/.local
|
||||
ENV PATH=/root/.local/bin:$PATH
|
||||
|
|
@ -54,6 +56,7 @@ COPY --from=ffmpeg-builder ./ffprobe/ffprobe /usr/bin/ffprobe
|
|||
RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recommends \
|
||||
nginx \
|
||||
atomicparsley \
|
||||
tini \
|
||||
curl && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# install debug tools for testing environment
|
||||
|
|
@ -88,4 +91,4 @@ EXPOSE 8000
|
|||
|
||||
RUN chmod +x ./run.sh
|
||||
|
||||
CMD ["./run.sh"]
|
||||
CMD ["/bin/tini", "--", "./run.sh"]
|
||||
|
|
|
|||
|
|
@ -345,6 +345,7 @@ class Reindex(ReindexBase):
|
|||
thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
|
||||
|
||||
Comments(youtube_id, config=self.config).reindex_comments()
|
||||
video.embed_metadata()
|
||||
self.processed["videos"] += 1
|
||||
|
||||
def _reindex_single_channel(self, channel_id: str) -> None:
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ class Command(BaseCommand):
|
|||
self._create_default_schedules()
|
||||
self._update_schedule_tz()
|
||||
self._init_app_config()
|
||||
self._set_ta_startup_time()
|
||||
self._mig_fix_download_channel_indexed()
|
||||
self._mig_add_default_playlist_sort()
|
||||
self._mig_set_channel_tabs()
|
||||
|
|
@ -259,6 +260,17 @@ class Command(BaseCommand):
|
|||
self.style.SUCCESS(f" Status code: {status_code}")
|
||||
)
|
||||
|
||||
def _set_ta_startup_time(self) -> None:
|
||||
"""set startup time to trigger frontend refresh, threadsafe"""
|
||||
self.stdout.write("[11] Set startup timestamp")
|
||||
message = str(int(datetime.now().timestamp() // 10 * 10))
|
||||
RedisArchivist().set_message(
|
||||
"STARTTIMESTAMP", message=message, save=True
|
||||
)
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(f" ✓ set timestamp to {message}.")
|
||||
)
|
||||
|
||||
def _mig_fix_download_channel_indexed(self) -> None:
|
||||
"""migrate from v0.5.2 to 0.5.3, fix missing channel_indexed"""
|
||||
self.stdout.write("[MIGRATION] fix incorrect video channel tags types")
|
||||
|
|
|
|||
|
|
@ -11,12 +11,12 @@ https://docs.djangoproject.com/en/3.2/ref/settings/
|
|||
"""
|
||||
|
||||
import hashlib
|
||||
from datetime import datetime
|
||||
from os import environ, path
|
||||
from pathlib import Path
|
||||
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.helper import ta_host_parser
|
||||
from common.src.ta_redis import RedisArchivist
|
||||
from corsheaders.defaults import default_headers
|
||||
|
||||
try:
|
||||
|
|
@ -221,8 +221,12 @@ CORS_EXPOSE_HEADERS = ["X-Start-Timestamp"]
|
|||
|
||||
# TA application settings
|
||||
TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
|
||||
TA_VERSION = "v0.5.8-unstable"
|
||||
TA_START = str(int(datetime.now().timestamp()))
|
||||
TA_VERSION = "v0.5.8"
|
||||
try:
|
||||
TA_START = RedisArchivist().get_message_str("STARTTIMESTAMP")
|
||||
except ValueError:
|
||||
# fails in unittests bootstrap
|
||||
pass
|
||||
|
||||
# API
|
||||
REST_FRAMEWORK = {
|
||||
|
|
@ -234,6 +238,7 @@ SPECTACULAR_SETTINGS = {
|
|||
"DESCRIPTION": "API documentation for Tube Archivist backend.",
|
||||
"VERSION": TA_VERSION,
|
||||
"SERVE_INCLUDE_SCHEMA": False,
|
||||
"SERVE_PERMISSIONS": ["rest_framework.permissions.IsAuthenticated"],
|
||||
}
|
||||
|
||||
# Logging configuration
|
||||
|
|
|
|||
|
|
@ -462,8 +462,8 @@ class PendingList(PendingIndex):
|
|||
response, status_code = ElasticWrap("_bulk").post(
|
||||
query_str, ndjson=True
|
||||
)
|
||||
print(response)
|
||||
if status_code != 200:
|
||||
if status_code not in [200, 201]:
|
||||
print(response)
|
||||
self._notify_fail(status_code)
|
||||
elif response.get("errors", False):
|
||||
failed_video_ids = []
|
||||
|
|
|
|||
|
|
@ -188,22 +188,11 @@ class VideoDownloader(DownloaderBase):
|
|||
postprocessors = []
|
||||
|
||||
if self.config["downloads"]["add_metadata"]:
|
||||
# full metadata is added in DownloadPostProcess
|
||||
postprocessors.append(
|
||||
{
|
||||
"key": "FFmpegMetadata",
|
||||
"add_chapters": True,
|
||||
"add_metadata": True,
|
||||
}
|
||||
)
|
||||
postprocessors.append(
|
||||
{
|
||||
"key": "MetadataFromField",
|
||||
"formats": [
|
||||
"%(title)s:%(meta_title)s",
|
||||
"%(uploader)s:%(meta_artist)s",
|
||||
":(?P<album>)",
|
||||
],
|
||||
"when": "pre_process",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -303,6 +292,9 @@ class DownloadPostProcess(DownloaderBase):
|
|||
self.refresh_playlist()
|
||||
self.match_videos()
|
||||
self.get_comments()
|
||||
self.embed_metadata()
|
||||
|
||||
RedisQueue(self.VIDEO_QUEUE).clear()
|
||||
|
||||
def auto_delete_all(self):
|
||||
"""handle auto delete"""
|
||||
|
|
@ -389,6 +381,9 @@ class DownloadPostProcess(DownloaderBase):
|
|||
try:
|
||||
playlist = YoutubePlaylist(playlist_id)
|
||||
playlist.update_playlist(skip_on_empty=True)
|
||||
if not playlist.json_data:
|
||||
raise ValueError("no json data extracted for playlist")
|
||||
|
||||
except ValueError as err:
|
||||
message = [
|
||||
f"{playlist_id}: skip failed playlist import",
|
||||
|
|
@ -488,6 +483,26 @@ class DownloadPostProcess(DownloaderBase):
|
|||
video_queue = RedisQueue(self.VIDEO_QUEUE)
|
||||
comment_list = CommentList(task=self.task)
|
||||
comment_list.add(video_ids=video_queue.get_all())
|
||||
|
||||
video_queue.clear()
|
||||
comment_list.index()
|
||||
|
||||
def embed_metadata(self):
|
||||
"""embed metadata in media file"""
|
||||
if not self.config["downloads"].get("add_metadata"):
|
||||
return
|
||||
|
||||
queue = RedisQueue(self.VIDEO_QUEUE)
|
||||
total = queue.max_score()
|
||||
video_ids = queue.get_all()
|
||||
|
||||
for idx, youtube_id in enumerate(video_ids):
|
||||
YoutubeVideo(youtube_id).embed_metadata()
|
||||
|
||||
if not self.task:
|
||||
continue
|
||||
|
||||
message = [
|
||||
"Post Processing Videos.",
|
||||
f"Embed metadata: - {idx}/{total}",
|
||||
]
|
||||
progress = idx / total
|
||||
self.task.send_progress(message, progress=progress)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ celery==5.5.3
|
|||
django-auth-ldap==5.2.0
|
||||
django-celery-beat==2.8.1
|
||||
django-cors-headers==4.9.0
|
||||
Django==5.2.7
|
||||
Django==5.2.8
|
||||
djangorestframework==3.16.1
|
||||
drf-spectacular==0.28.0
|
||||
Pillow==12.0.0
|
||||
|
|
@ -11,4 +11,4 @@ redis==7.0.0
|
|||
requests==2.32.5
|
||||
ryd-client==0.0.6
|
||||
uvicorn==0.38.0
|
||||
yt-dlp[default]==2025.10.22
|
||||
yt-dlp[default] @ git+https://github.com/yt-dlp/yt-dlp@6224a3898821965a7d6a2cb9cc2de40a0fd6e6bc
|
||||
|
|
|
|||
|
|
@ -87,6 +87,13 @@ RESYNC_THUMBS: TaskItemConfig = {
|
|||
"api_stop": False,
|
||||
}
|
||||
|
||||
RESYNC_METADATA: TaskItemConfig = {
|
||||
"title": "Sync Metadata to Media Files",
|
||||
"group": "setting:thumbnailsync",
|
||||
"api_start": True,
|
||||
"api_stop": False,
|
||||
}
|
||||
|
||||
INDEX_PLAYLISTS: TaskItemConfig = {
|
||||
"title": "Index Channel Playlist",
|
||||
"group": "channel:indexplaylist",
|
||||
|
|
@ -119,6 +126,7 @@ TASK_CONFIG: dict[str, TaskItemConfig] = {
|
|||
"rescan_filesystem": RESCAN_FILESYSTEM,
|
||||
"thumbnail_check": THUMBNAIL_CHECK,
|
||||
"resync_thumbs": RESYNC_THUMBS,
|
||||
"resync_metadata": RESYNC_METADATA,
|
||||
"index_playlists": INDEX_PLAYLISTS,
|
||||
"subscribe_to": SUBSCRIBE_TO,
|
||||
"version_check": VERSION_CHECK,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from download.src.yt_dlp_handler import VideoDownloader
|
|||
from task.src.notify import Notifications
|
||||
from task.src.task_config import TASK_CONFIG
|
||||
from task.src.task_manager import TaskManager
|
||||
from video.src.meta_embed import MetadataEmbed
|
||||
|
||||
|
||||
class BaseTask(Task):
|
||||
|
|
@ -302,6 +303,19 @@ def re_sync_thumbs(self):
|
|||
ThumbFilesystem(task=self).embed()
|
||||
|
||||
|
||||
@shared_task(bind=True, name="resync_metadata", base=BaseTask)
|
||||
def re_sync_metadata(self):
|
||||
"""resync metadata to media files"""
|
||||
manager = TaskManager()
|
||||
if manager.is_pending(self):
|
||||
print(f"[task][{self.name}] metadata re-embed is already running")
|
||||
self.send_progress(["Metadata re-embed is already running."])
|
||||
return
|
||||
|
||||
manager.init(self)
|
||||
MetadataEmbed(task=self).embed()
|
||||
|
||||
|
||||
@shared_task(bind=True, name="subscribe_to", base=BaseTask)
|
||||
def subscribe_to(self, url_str: str, expected_type: str | bool = False):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ functionality:
|
|||
- index and update in es
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
|
|
@ -14,6 +15,7 @@ from common.src.helper import get_duration_sec, get_duration_str, randomizor
|
|||
from common.src.index_generic import YouTubeItem
|
||||
from django.conf import settings
|
||||
from download.src.thumbnails import ThumbManager
|
||||
from mutagen.mp4 import MP4
|
||||
from playlist.src import index as ta_playlist
|
||||
from ryd_client import ryd_client
|
||||
from user.src.user_config import UserConfig
|
||||
|
|
@ -420,6 +422,68 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
|||
|
||||
return subtitles
|
||||
|
||||
def embed_metadata(self):
|
||||
"""embed metadata for video"""
|
||||
if not self.config["downloads"].get("add_metadata"):
|
||||
return
|
||||
|
||||
print(f"{self.youtube_id}: embed metadata")
|
||||
if not self.json_data:
|
||||
self.get_from_es()
|
||||
|
||||
if not self.json_data:
|
||||
print(f"{self.youtube_id}: skip embed, video not indexed")
|
||||
return
|
||||
|
||||
video_base = EnvironmentSettings.MEDIA_DIR
|
||||
media_url = self.json_data.get("media_url")
|
||||
file_path = os.path.join(video_base, media_url)
|
||||
if not os.path.exists(file_path):
|
||||
print(f"{self.youtube_id}: skip embed, file not found")
|
||||
return
|
||||
|
||||
title = self.json_data["title"]
|
||||
artist = self.json_data["channel"]["channel_name"]
|
||||
description = self.json_data["description"]
|
||||
to_embed = self._get_to_embed()
|
||||
|
||||
video = MP4(file_path)
|
||||
video["\xa9nam"] = [title] # title
|
||||
video["\xa9ART"] = [artist] # artist
|
||||
video["desc"] = [description] # description
|
||||
video["ldes"] = [description] # synopsis
|
||||
video["----:com.tubearchivist:ta"] = [to_embed.encode("utf-8")]
|
||||
video.save()
|
||||
|
||||
def _get_to_embed(self) -> str:
|
||||
"""get metadata json str to embed"""
|
||||
comments = None
|
||||
if self.json_data.get("comment_count"):
|
||||
comments = Comments(self.youtube_id).get_es_comments()
|
||||
|
||||
subtitles = None
|
||||
if self.json_data.get("subtitles"):
|
||||
subtitles = YoutubeSubtitle(video=self).get_es_subtitles()
|
||||
|
||||
playlists = None
|
||||
if self.json_data.get("playlist"):
|
||||
playlists = []
|
||||
for playlist_id in self.json_data["playlist"]:
|
||||
playlist = ta_playlist.YoutubePlaylist(playlist_id)
|
||||
playlist.get_from_es()
|
||||
playlists.append(playlist.json_data)
|
||||
|
||||
to_embed = json.dumps(
|
||||
{
|
||||
"video": self.json_data,
|
||||
"comments": comments,
|
||||
"subtitles": subtitles,
|
||||
"playlists": playlists,
|
||||
}
|
||||
)
|
||||
|
||||
return to_embed
|
||||
|
||||
|
||||
def index_new_video(youtube_id, video_type=VideoTypeEnum.VIDEOS):
|
||||
"""combined classes to create new video in index"""
|
||||
|
|
|
|||
|
|
@ -0,0 +1,51 @@
|
|||
"""bulk metadata embedding"""
|
||||
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from video.src.index import YoutubeVideo
|
||||
|
||||
|
||||
class MetadataEmbed:
|
||||
"""sync metadata to videos in bulk"""
|
||||
|
||||
INDEX_NAME = "ta_video"
|
||||
|
||||
def __init__(self, task=False):
|
||||
self.task = task
|
||||
|
||||
def embed(self):
|
||||
"""entry point"""
|
||||
data = {
|
||||
"query": {"match_all": {}},
|
||||
"_source": ["youtube_id"],
|
||||
}
|
||||
paginate = IndexPaginate(
|
||||
index_name=self.INDEX_NAME,
|
||||
data=data,
|
||||
size=200,
|
||||
callback=MetadataEmbedCallback,
|
||||
task=self.task,
|
||||
total=self._get_total(),
|
||||
)
|
||||
_ = paginate.get_results()
|
||||
|
||||
def _get_total(self):
|
||||
"""get total documents in index"""
|
||||
path = f"{self.INDEX_NAME}/_count"
|
||||
response, _ = ElasticWrap(path).get()
|
||||
|
||||
return response.get("count")
|
||||
|
||||
|
||||
class MetadataEmbedCallback:
|
||||
"""callback for metadata embed"""
|
||||
|
||||
def __init__(self, source, index_name, counter=0):
|
||||
self.source = source
|
||||
self.index_name = index_name
|
||||
self.counter = counter
|
||||
|
||||
def run(self):
|
||||
"""run embed"""
|
||||
for video in self.source:
|
||||
youtube_id = video["_source"]["youtube_id"]
|
||||
YoutubeVideo(youtube_id).embed_metadata()
|
||||
|
|
@ -13,7 +13,7 @@ from operator import itemgetter
|
|||
|
||||
import requests
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from common.src.helper import rand_sleep, requests_headers
|
||||
from yt_dlp.utils import orderedSet_from_options
|
||||
|
||||
|
|
@ -93,6 +93,14 @@ class YoutubeSubtitle:
|
|||
|
||||
return candidate_subtitles
|
||||
|
||||
def get_es_subtitles(self) -> list[dict]:
|
||||
"""get subtitles from elastic"""
|
||||
data = {
|
||||
"query": {"term": {"youtube_id": {"value": self.video.youtube_id}}}
|
||||
}
|
||||
response = IndexPaginate("ta_subtitle", data).get_results()
|
||||
return response
|
||||
|
||||
def download_subtitles(self, relevant_subtitles):
|
||||
"""download subtitle files to archive"""
|
||||
subtitle_list = ", ".join(map(itemgetter("lang"), relevant_subtitles))
|
||||
|
|
|
|||
|
|
@ -150,6 +150,9 @@ function sync_docker {
|
|||
git tag -a "$VERSION" -m "new release version $VERSION"
|
||||
git push origin "$VERSION"
|
||||
|
||||
# update API docs
|
||||
python backend/manage.py spectacular --file ../docs/mkdocs/docs/api/schema.yaml
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -5,6 +5,7 @@ type TaskNamesType =
|
|||
| 'update_subscribed'
|
||||
| 'manual_import'
|
||||
| 'resync_thumbs'
|
||||
| 'resync_metadata'
|
||||
| 'rescan_filesystem';
|
||||
|
||||
const updateTaskByName = async (taskName: TaskNamesType) => {
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ const SettingsActions = () => {
|
|||
const [deletePending, setDeletePending] = useState(false);
|
||||
const [processingImports, setProcessingImports] = useState(false);
|
||||
const [reEmbed, setReEmbed] = useState(false);
|
||||
const [reSyncMeta, setReSyncMeta] = useState(false);
|
||||
const [backupStarted, setBackupStarted] = useState(false);
|
||||
const [isRestoringBackup, setIsRestoringBackup] = useState(false);
|
||||
const [reScanningFileSystem, setReScanningFileSystem] = useState(false);
|
||||
|
|
@ -44,6 +45,7 @@ const SettingsActions = () => {
|
|||
deletePending ||
|
||||
processingImports ||
|
||||
reEmbed ||
|
||||
reSyncMeta ||
|
||||
backupStarted ||
|
||||
isRestoringBackup ||
|
||||
reScanningFileSystem
|
||||
|
|
@ -53,6 +55,7 @@ const SettingsActions = () => {
|
|||
setDeletePending(false);
|
||||
setProcessingImports(false);
|
||||
setReEmbed(false);
|
||||
setReSyncMeta(false);
|
||||
setBackupStarted(false);
|
||||
setIsRestoringBackup(false);
|
||||
setReScanningFileSystem(false);
|
||||
|
|
@ -104,6 +107,22 @@ const SettingsActions = () => {
|
|||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="settings-group">
|
||||
<h2>Embed metadata into media file</h2>
|
||||
<p>Embed metadata into media files as mp4 tags.</p>
|
||||
<div id="re-meta">
|
||||
{reSyncMeta && <p>Processing video metadata</p>}
|
||||
{!reSyncMeta && (
|
||||
<Button
|
||||
label="Start process"
|
||||
onClick={async () => {
|
||||
await updateTaskByName('resync_metadata');
|
||||
setReSyncMeta(true);
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="settings-group">
|
||||
<h2>ZIP file index backup</h2>
|
||||
<p>
|
||||
|
|
|
|||
Loading…
Reference in New Issue