diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py
index d7a61e22..0431842a 100644
--- a/tubearchivist/home/src/download/thumbnails.py
+++ b/tubearchivist/home/src/download/thumbnails.py
@@ -81,7 +81,7 @@ class ThumbManagerBase:
"banner": os.path.join(
app_root, "static/img/default-channel-banner.jpg"
),
- "art": os.path.join(
+ "tvart": os.path.join(
app_root, "static/img/default-channel-art.jpg"
),
}
@@ -270,6 +270,7 @@ class ValidatorCallback:
urls = (
channel["_source"]["channel_thumb_url"],
channel["_source"]["channel_banner_url"],
+ channel["_source"]["channel_tvart_url"],
)
handler = ThumbManager(channel["_source"]["channel_id"])
handler.download_channel_art(urls, skip_existing=True)
diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py
index 5db44bfd..fe31c03f 100644
--- a/tubearchivist/home/src/download/yt_dlp_handler.py
+++ b/tubearchivist/home/src/download/yt_dlp_handler.py
@@ -395,8 +395,8 @@ class VideoDownloader:
new_folder = os.path.join(videos, channel_name)
if not os.path.exists(new_folder):
os.makedirs(new_folder)
- if host_uid and host_gid:
- os.chown(new_folder, host_uid, host_gid)
+ if host_uid and host_gid:
+ os.chown(new_folder, host_uid, host_gid)
# find real filename
cache_dir = self.config["application"]["cache_dir"]
all_cached = ignore_filelist(os.listdir(cache_dir + "/download/"))
diff --git a/tubearchivist/home/src/es/connect.py b/tubearchivist/home/src/es/connect.py
index fd6e5ab0..5af1838d 100644
--- a/tubearchivist/home/src/es/connect.py
+++ b/tubearchivist/home/src/es/connect.py
@@ -166,7 +166,7 @@ class IndexPaginate:
def _notify(self, processed):
"""send notification on task"""
total = self.kwargs.get("total")
- progress = (processed + 1) / total
+ progress = processed / total
index_clean = self.index_name.lstrip("ta_").title()
message = [f"Processing {index_clean}s {processed}/{total}"]
self.kwargs.get("task").send_progress(message, progress=progress)
diff --git a/tubearchivist/home/src/index/manual.py b/tubearchivist/home/src/index/manual.py
index a4fa2e40..66b6bca2 100644
--- a/tubearchivist/home/src/index/manual.py
+++ b/tubearchivist/home/src/index/manual.py
@@ -12,6 +12,7 @@ import shutil
import subprocess
from home.src.download.thumbnails import ThumbManager
+from home.src.index.comments import CommentList
from home.src.index.video import YoutubeVideo
from home.src.ta.config import AppConfig
from home.src.ta.helper import ignore_filelist
@@ -141,6 +142,9 @@ class ImportFolderScanner:
ManualImport(current_video, self.CONFIG).run()
+ video_ids = [i["video_id"] for i in self.to_import]
+ CommentList(video_ids, task=self.task).index()
+
def _notify(self, idx, current_video):
"""send notification back to task"""
filename = os.path.split(current_video["media"])[-1]
@@ -427,15 +431,22 @@ class ManualImport:
def _move_to_archive(self, json_data):
"""move identified media file to archive"""
videos = self.config["application"]["videos"]
+ host_uid = self.config["application"]["HOST_UID"]
+ host_gid = self.config["application"]["HOST_GID"]
channel, file = os.path.split(json_data["media_url"])
channel_folder = os.path.join(videos, channel)
if not os.path.exists(channel_folder):
os.makedirs(channel_folder)
+ if host_uid and host_gid:
+ os.chown(channel_folder, host_uid, host_gid)
+
old_path = self.current_video["media"]
new_path = os.path.join(channel_folder, file)
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
+ if host_uid and host_gid:
+ os.chown(new_path, host_uid, host_gid)
base_name, _ = os.path.splitext(new_path)
for old_path in self.current_video["subtitle"]:
diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py
index baeadfa8..b5ae83cd 100644
--- a/tubearchivist/home/src/index/reindex.py
+++ b/tubearchivist/home/src/index/reindex.py
@@ -49,6 +49,7 @@ class ReindexBase:
}
MULTIPLY = 1.2
+ DAYS3 = 60 * 60 * 24 * 3
def __init__(self):
self.config = AppConfig().config
@@ -62,13 +63,34 @@ class ReindexBase:
RedisQueue(queue_name=reindex_config["queue_name"]).add_list(all_ids)
-class ReindexOutdated(ReindexBase):
- """add outdated documents to reindex queue"""
+class ReindexPopulate(ReindexBase):
+ """add outdated and recent documents to reindex queue"""
def __init__(self):
super().__init__()
self.interval = self.config["scheduler"]["check_reindex_days"]
+ def add_recent(self):
+ """add recent videos to refresh"""
+ gte = datetime.fromtimestamp(self.now - self.DAYS3).date().isoformat()
+ must_list = [
+ {"term": {"active": {"value": True}}},
+ {"range": {"published": {"gte": gte}}},
+ ]
+ data = {
+ "size": 10000,
+ "query": {"bool": {"must": must_list}},
+ "sort": [{"published": {"order": "desc"}}],
+ }
+ response, _ = ElasticWrap("ta_video/_search").get(data=data)
+ hits = response["hits"]["hits"]
+ if not hits:
+ return
+
+ all_ids = [i["_source"]["youtube_id"] for i in hits]
+ reindex_config = self.REINDEX_CONFIG.get("video")
+ self.populate(all_ids, reindex_config)
+
def add_outdated(self):
"""add outdated documents"""
for reindex_config in self.REINDEX_CONFIG.values():
diff --git a/tubearchivist/home/src/index/subtitle.py b/tubearchivist/home/src/index/subtitle.py
index 7c56c4d0..e289fbe8 100644
--- a/tubearchivist/home/src/index/subtitle.py
+++ b/tubearchivist/home/src/index/subtitle.py
@@ -115,7 +115,7 @@ class YoutubeSubtitle:
source = subtitle["source"]
lang = subtitle.get("lang")
response = requests.get(
- subtitle["url"], headers=requests_headers()
+ subtitle["url"], headers=requests_headers(), timeout=30
)
if not response.ok:
print(f"{self.video.youtube_id}: failed to download subtitle")
@@ -137,14 +137,18 @@ class YoutubeSubtitle:
return indexed
- @staticmethod
- def _write_subtitle_file(dest_path, subtitle_str):
+ def _write_subtitle_file(self, dest_path, subtitle_str):
"""write subtitle file to disk"""
# create folder here for first video of channel
os.makedirs(os.path.split(dest_path)[0], exist_ok=True)
with open(dest_path, "w", encoding="utf-8") as subfile:
subfile.write(subtitle_str)
+ host_uid = self.video.config["application"]["HOST_UID"]
+ host_gid = self.video.config["application"]["HOST_GID"]
+ if host_uid and host_gid:
+ os.chown(dest_path, host_uid, host_gid)
+
@staticmethod
def _index_subtitle(query_str):
"""send subtitle to es for indexing"""
diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py
index f2c5341e..a86c1dc1 100644
--- a/tubearchivist/home/tasks.py
+++ b/tubearchivist/home/tasks.py
@@ -21,7 +21,7 @@ from home.src.es.index_setup import ElasitIndexWrap
from home.src.index.channel import YoutubeChannel
from home.src.index.filesystem import Filesystem
from home.src.index.manual import ImportFolderScanner
-from home.src.index.reindex import Reindex, ReindexManual, ReindexOutdated
+from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate
from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
from home.src.ta.ta_redis import RedisArchivist
from home.src.ta.task_manager import TaskManager
@@ -220,9 +220,12 @@ def check_reindex(self, data=False, extract_videos=False):
manager.init(self)
if not data:
# started from scheduler
+ populate = ReindexPopulate()
print(f"[task][{self.name}] reindex outdated documents")
+ self.send_progress("Add recent documents to the reindex Queue.")
+ populate.add_recent()
self.send_progress("Add outdated documents to the reindex Queue.")
- ReindexOutdated().add_outdated()
+ populate.add_outdated()
Reindex(task=self).reindex_all()
diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html
index 9539f58d..fc85d0f0 100644
--- a/tubearchivist/home/templates/home/channel_id.html
+++ b/tubearchivist/home/templates/home/channel_id.html
@@ -22,7 +22,7 @@
Downloads
{% endif %}
-