RomanNum3ral
/
tubearchivist
mirror of https://github.com/tubearchivist/tubearchivist.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Track channel tabs, #build 

Changed:
- Added channel_tabs to channel index
- Limit subscription refresh to available tabs
- Fix custom playlists
- Fix empty add to queue response
This commit is contained in:
Simon 2025-07-12 22:53:51 +07:00
parent b5a79c4885 3fd061e838
commit 4baaabe3f3
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
18 changed files with 177 additions and 313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
backend/appsettings/index_mapping.json
View File

@ -62,6 +62,9 @@
}
}
},
"channel_tabs": {
"type": "keyword"
},
"channel_overwrites": {
"properties": {
"download_format": {
@ -107,10 +110,6 @@
"type": "text",
"index": false
},
"vid_thumb_base64": {
"type": "text",
"index": false
},
"date_downloaded": {
"type": "date",
"format": "epoch_second"
@ -165,6 +164,9 @@
}
}
},
"channel_tabs": {
"type": "keyword"
},
"channel_overwrites": {
"properties": {
"download_format": {

4
backend/channel/serializers.py
View File

@ -4,6 +4,7 @@
from common.serializers import PaginationSerializer, ValidateUnknownFieldsMixin
from rest_framework import serializers
from video.src.constants import VideoTypeEnum
class ChannelOverwriteSerializer(
@ -45,6 +46,9 @@ class ChannelSerializer(serializers.Serializer):
channel_tags = serializers.ListField(
child=serializers.CharField(), required=False
)
channel_tabs = serializers.ListField(
child=serializers.ChoiceField(VideoTypeEnum.values_known())
)
channel_views = serializers.IntegerField()
_index = serializers.CharField(required=False)
_score = serializers.IntegerField(required=False)

28
backend/channel/src/index.py
View File

@ -7,12 +7,14 @@ functionality:
import os
from datetime import datetime
from channel.src.remote_query import get_last_channel_videos
from common.src.env_settings import EnvironmentSettings
from common.src.es_connect import ElasticWrap, IndexPaginate
from common.src.helper import rand_sleep
from common.src.index_generic import YouTubeItem
from download.src.thumbnails import ThumbManager
from download.src.yt_dlp_base import YtWrap
from video.src.constants import VideoTypeEnum
class YoutubeChannel(YouTubeItem):
@ -68,6 +70,7 @@ class YoutubeChannel(YouTubeItem):
"channel_thumb_url": self._get_thumb_art(),
"channel_tvart_url": self._get_tv_art(),
"channel_views": self.youtube_meta.get("view_count") or 0,
"channel_tabs": self.get_channel_tabs(),
}
def _get_thumb_art(self):
@ -103,6 +106,31 @@ class YoutubeChannel(YouTubeItem):
return False
def get_channel_tabs(self) -> list[str]:
"""get channel tabs"""
tabs = VideoTypeEnum.values_known()
config_cp = self.config.copy()
config_cp["subscriptions"] = {
"channel_size": 1,
"live_channel_size": 1,
"shorts_channel_size": 1,
}
tabs = []
for query_filter in VideoTypeEnum:
if query_filter == VideoTypeEnum.UNKNOWN:
continue
videos = get_last_channel_videos(
channel_id=self.youtube_id,
config=config_cp,
limit=True,
query_filter=query_filter,
)
if videos:
tabs.append(query_filter.value)
return tabs
def _video_fallback(self, fallback):
"""use video metadata as fallback"""
print(f"{self.youtube_id}: fallback to video metadata")

22
backend/channel/src/remote_query.py
View File

@ -12,7 +12,9 @@ class VideoQueryBuilder:
self.channel_overwrites = channel_overwrites or {}
def build_queries(
self, video_type: VideoTypeEnum | None, limit: bool = True
self,
video_type: VideoTypeEnum | list[VideoTypeEnum] | None,
limit: bool = True,
) -> list[tuple[VideoTypeEnum, int | None]]:
"""Build queries for all or specific video type."""
query_methods = {
@ -22,13 +24,21 @@ class VideoQueryBuilder:
}
if video_type:
# build query for specific type
query_method = query_methods.get(video_type)
if query_method:
# build query for specific type/s
if not isinstance(video_type, list):
video_type = [video_type]
queries = []
for video_type_item in video_type:
query_method = query_methods.get(video_type_item)
if not query_method:
continue
query = query_method(limit)
if query[1] != 0:
return [query]
return []
queries.append(query)
return queries
# Build and return queries for all video types
queries = []

1
backend/common/src/search_processor.py
View File

@ -160,6 +160,7 @@ class SearchProcess:
"playlist_last_refresh": playlist_last_refresh,
}
)
print(playlist_dict)
return dict(sorted(playlist_dict.items()))

76
backend/config/management/commands/ta_config_backup.py
View File

@ -1,76 +0,0 @@
"""backup config for sqlite reset and restore"""
import json
from pathlib import Path
from django.contrib.auth import get_user_model
from django.core.management.base import BaseCommand
from home.models import CustomPeriodicTask
from home.src.ta.settings import EnvironmentSettings
from rest_framework.authtoken.models import Token
User = get_user_model()
class Command(BaseCommand):
"""export"""
help = "Exports all users and their auth tokens to a JSON file"
FILE = Path(EnvironmentSettings.CACHE_DIR) / "backup" / "migration.json"
def handle(self, *args, **kwargs):
"""entry point"""
data = {
"user_data": self.get_users(),
"schedule_data": self.get_schedules(),
}
with open(self.FILE, "w", encoding="utf-8") as json_file:
json_file.write(json.dumps(data))
def get_users(self):
"""get users"""
users = User.objects.all()
user_data = []
for user in users:
user_info = {
"username": user.name,
"is_staff": user.is_staff,
"is_superuser": user.is_superuser,
"password": user.password,
"tokens": [],
}
try:
token = Token.objects.get(user=user)
user_info["tokens"] = [token.key]
except Token.DoesNotExist:
user_info["tokens"] = []
user_data.append(user_info)
return user_data
def get_schedules(self):
"""get schedules"""
all_schedules = CustomPeriodicTask.objects.all()
schedule_data = []
for schedule in all_schedules:
schedule_info = {
"name": schedule.name,
"crontab": {
"minute": schedule.crontab.minute,
"hour": schedule.crontab.hour,
"day_of_week": schedule.crontab.day_of_week,
},
}
schedule_data.append(schedule_info)
return schedule_data

89
backend/config/management/commands/ta_config_restore.py
View File

@ -1,89 +0,0 @@
"""restore config from backup"""
import json
from pathlib import Path
from common.src.env_settings import EnvironmentSettings
from django.core.management.base import BaseCommand
from django_celery_beat.models import CrontabSchedule
from rest_framework.authtoken.models import Token
from task.models import CustomPeriodicTask
from task.src.task_config import TASK_CONFIG
from user.models import Account
class Command(BaseCommand):
"""export"""
help = "Exports all users and their auth tokens to a JSON file"
FILE = Path(EnvironmentSettings.CACHE_DIR) / "backup" / "migration.json"
def handle(self, *args, **options):
"""handle"""
self.stdout.write("restore users and schedules")
data = self.get_config()
self.restore_users(data["user_data"])
self.restore_schedules(data["schedule_data"])
self.stdout.write(
self.style.SUCCESS(
" ✓ restore completed. Please restart the container."
)
)
def get_config(self) -> dict:
"""get config from backup"""
with open(self.FILE, "r", encoding="utf-8") as json_file:
data = json.loads(json_file.read())
self.stdout.write(
self.style.SUCCESS(f" ✓ json file found: {self.FILE}")
)
return data
def restore_users(self, user_data: list[dict]) -> None:
"""restore users from config"""
self.stdout.write("delete existing users")
Account.objects.all().delete()
self.stdout.write("recreate users")
for user_info in user_data:
user = Account.objects.create(
name=user_info["username"],
is_staff=user_info["is_staff"],
is_superuser=user_info["is_superuser"],
password=user_info["password"],
)
for token in user_info["tokens"]:
Token.objects.create(user=user, key=token)
self.stdout.write(
self.style.SUCCESS(
f" ✓ recreated user with name: {user_info['username']}"
)
)
def restore_schedules(self, schedule_data: list[dict]) -> None:
"""restore schedules"""
self.stdout.write("delete existing schedules")
CustomPeriodicTask.objects.all().delete()
self.stdout.write("recreate schedules")
for schedule in schedule_data:
task_name = schedule["name"]
description = TASK_CONFIG[task_name].get("title")
crontab, _ = CrontabSchedule.objects.get_or_create(
minute=schedule["crontab"]["minute"],
hour=schedule["crontab"]["hour"],
day_of_week=schedule["crontab"]["day_of_week"],
timezone=EnvironmentSettings.TZ,
)
task = CustomPeriodicTask.objects.create(
name=task_name,
task=task_name,
description=description,
crontab=crontab,
)
self.stdout.write(
self.style.SUCCESS(f" ✓ recreated schedule: {task}")
)

36
backend/config/management/commands/ta_index_channel_tags.py Normal file
View File

@ -0,0 +1,36 @@
"""
migration for 0.5.4 to 0.5.5
index channel_tags for subscribed channels
"""
import time
from channel.src.index import YoutubeChannel
from common.src.helper import get_channels
from django.core.management.base import BaseCommand
class Command(BaseCommand):
"""command"""
def handle(self, *args, **kwargs):
"""handle task"""
self.stdout.write("channel tags initial index")
channels = get_channels(subscribed_only=True, source=["channel_id"])
for es_channel in channels:
channel = YoutubeChannel(es_channel["channel_id"])
channel.get_from_es()
channel_name = channel.json_data["channel_name"]
channel_tabs = channel.get_channel_tabs()
channel.json_data["channel_tabs"] = channel_tabs
channel.upload_to_es()
channel.sync_to_videos()
self.stdout.write(
self.style.SUCCESS(
f" ✓ updated '{channel_name}' tabs: {channel_tabs}"
)
)
time.sleep(5)

137
backend/config/management/commands/ta_startup.py
View File

@ -19,11 +19,11 @@ from common.src.ta_redis import RedisArchivist
from django.core.management.base import BaseCommand, CommandError
from django.utils import dateformat
from django_celery_beat.models import CrontabSchedule, PeriodicTasks
from redis.exceptions import ResponseError
from task.models import CustomPeriodicTask
from task.src.config_schedule import ScheduleBuilder
from task.src.task_manager import TaskManager
from task.tasks import version_check
from video.src.constants import VideoTypeEnum
TOPIC = """
@ -49,14 +49,12 @@ class Command(BaseCommand):
self._version_check()
self._index_setup()
self._snapshot_check()
self._mig_app_settings()
self._create_default_schedules()
self._update_schedule_tz()
self._init_app_config()
self._mig_channel_tags()
self._mig_video_channel_tags()
self._mig_fix_download_channel_indexed()
self._mig_add_default_playlist_sort()
self._mig_set_channel_tabs()
def _make_folders(self):
"""make expected cache folders"""
@ -159,39 +157,6 @@ class Command(BaseCommand):
self.stdout.write("[7] setup snapshots")
ElasticSnapshot().setup()
def _mig_app_settings(self) -> None:
"""update from v0.4.13 to v0.5.0, migrate application settings"""
self.stdout.write("[MIGRATION] move appconfig to ES")
try:
config = RedisArchivist().get_message("config")
except ResponseError:
self.stdout.write(
self.style.SUCCESS(" Redis does not support JSON decoding")
)
return
if not config or config == {"status": False}:
self.stdout.write(
self.style.SUCCESS(" no config values to migrate")
)
return
path = "ta_config/_doc/appsettings"
response, status_code = ElasticWrap(path).post(config)
if status_code in [200, 201]:
self.stdout.write(
self.style.SUCCESS(" ✓ migrated appconfig to ES")
)
RedisArchivist().del_message("config", save=True)
return
message = " 🗙 failed to migrate app config"
self.stdout.write(self.style.ERROR(message))
self.stdout.write(response)
sleep(60)
raise CommandError(message)
def _create_default_schedules(self) -> None:
"""create default schedules for new installations"""
self.stdout.write("[8] create initial schedules")
@ -293,70 +258,6 @@ class Command(BaseCommand):
self.style.SUCCESS(f" Status code: {status_code}")
)
def _mig_channel_tags(self) -> None:
"""update from v0.4.13 to v0.5.0, migrate incorrect data types"""
self.stdout.write("[MIGRATION] fix incorrect channel tags types")
path = "ta_channel/_update_by_query"
data = {
"query": {"match": {"channel_tags": False}},
"script": {
"source": "ctx._source.channel_tags = []",
"lang": "painless",
},
}
response, status_code = ElasticWrap(path).post(data)
if status_code in [200, 201]:
updated = response.get("updated")
if updated:
self.stdout.write(
self.style.SUCCESS(f" ✓ fixed {updated} channel tags")
)
else:
self.stdout.write(
self.style.SUCCESS(" no channel tags needed fixing")
)
return
message = " 🗙 failed to fix channel tags"
self.stdout.write(self.style.ERROR(message))
self.stdout.write(response)
sleep(60)
raise CommandError(message)
def _mig_video_channel_tags(self) -> None:
"""update from v0.4.13 to v0.5.0, migrate incorrect data types"""
self.stdout.write("[MIGRATION] fix incorrect video channel tags types")
path = "ta_video/_update_by_query"
data = {
"query": {"match": {"channel.channel_tags": False}},
"script": {
"source": "ctx._source.channel.channel_tags = []",
"lang": "painless",
},
}
response, status_code = ElasticWrap(path).post(data)
if status_code in [200, 201]:
updated = response.get("updated")
if updated:
self.stdout.write(
self.style.SUCCESS(
f" ✓ fixed {updated} video channel tags"
)
)
else:
self.stdout.write(
self.style.SUCCESS(
" no video channel tags needed fixing"
)
)
return
message = " 🗙 failed to fix video channel tags"
self.stdout.write(self.style.ERROR(message))
self.stdout.write(response)
sleep(60)
raise CommandError(message)
def _mig_fix_download_channel_indexed(self) -> None:
"""migrate from v0.5.2 to 0.5.3, fix missing channel_indexed"""
self.stdout.write("[MIGRATION] fix incorrect video channel tags types")
@ -424,3 +325,37 @@ class Command(BaseCommand):
self.stdout.write(response)
sleep(60)
raise CommandError(message)
def _mig_set_channel_tabs(self) -> None:
"""migrate from 0.5.4 to 0.5.5 set initial channel tabs"""
self.stdout.write("[MIGRATION] set default channel_tabs")
path = "ta_channel/_update_by_query"
tabs = VideoTypeEnum.values_known()
data = {
"query": {
"bool": {"must_not": [{"exists": {"field": "channel_tabs"}}]}
},
"script": {
"source": f"ctx._source.channel_tabs = {tabs}",
"lang": "painless",
},
}
response, status_code = ElasticWrap(path).post(data)
if status_code in [200, 201]:
updated = response.get("updated")
if updated:
self.stdout.write(
self.style.SUCCESS(f" ✓ updated {updated} channels")
)
else:
self.stdout.write(
self.style.SUCCESS(" no channels need updating")
)
return
message = " 🗙 failed to set default channel_tabs"
self.stdout.write(self.style.ERROR(message))
self.stdout.write(response)
sleep(60)
raise CommandError(message)

11
backend/download/src/queue.py
View File

@ -165,7 +165,7 @@ class PendingList(PendingIndex):
else:
raise ValueError(f"invalid url_type: {entry}")
def _add_video(self, url, vid_type):
def _add_video(self, url, vid_type) -> dict | None:
"""add video to list"""
if self.auto_start and url in set(
i["youtube_id"] for i in self.all_pending
@ -175,10 +175,11 @@ class PendingList(PendingIndex):
if url in self.missing_videos or url in self.to_skip:
print(f"{url}: skipped adding already indexed video to download.")
else:
to_add = self._parse_video(url, vid_type)
if to_add:
self.missing_videos.append(to_add)
return None
to_add = self._parse_video(url, vid_type)
if to_add:
self.missing_videos.append(to_add)
return to_add

10
backend/download/src/subscriptions.py
View File

@ -28,7 +28,8 @@ class ChannelSubscription:
self.task.send_progress(["Looking up channels."])
all_channels = get_channels(
subscribed_only=True, source=["channel_id", "channel_overwrites"]
subscribed_only=True,
source=["channel_id", "channel_overwrites", "channel_tabs"],
)
if not all_channels:
return 0
@ -36,10 +37,15 @@ class ChannelSubscription:
all_channel_urls: list[ParsedURLType] = []
for channel in all_channels:
channel_tabs = channel["channel_tabs"]
if not channel_tabs:
continue
enum = [getattr(VideoTypeEnum, i.upper()) for i in channel_tabs]
queries = VideoQueryBuilder(
config=self.config,
channel_overwrites=channel.get("channel_overwrites", {}),
).build_queries(video_type=None)
).build_queries(video_type=enum)
for query in queries:
all_channel_urls.append(

2
backend/playlist/src/index.py
View File

@ -325,6 +325,7 @@ class YoutubePlaylist(YouTubeItem):
self.delete_metadata()
def create(self, name):
"""create custom playlist"""
self.json_data = {
"playlist_id": self.youtube_id,
"playlist_active": False,
@ -337,6 +338,7 @@ class YoutubePlaylist(YouTubeItem):
"playlist_description": False,
"playlist_thumbnail": False,
"playlist_subscribed": False,
"playlist_sort_order": "top",
}
self.upload_to_es()
self.get_playlist_art()

2
backend/playlist/src/query_building.py
View File

@ -45,7 +45,7 @@ class QueryBuilder:
type_parsed = getattr(PlaylistTypesEnum, playlist_type.upper()).value
return {"match": {"playlist_type.keyword": type_parsed}}
return {"match": {"playlist_type": type_parsed}}
def parse_sort(self) -> dict:
"""return sort"""

2
backend/playlist/tests/test_src/test_query_building.py
View File

@ -27,4 +27,4 @@ def test_parse_type():
qb.parse_type("invalid")
result = qb.parse_type("custom")
assert result == {"match": {"playlist_type.keyword": "custom"}}
assert result == {"match": {"playlist_type": "custom"}}

6
backend/playlist/views.py
View File

@ -239,6 +239,12 @@ class PlaylistApiView(ApiBaseView):
error = ErrorResponseSerializer({"error": "playlist not found"})
return Response(error.data, status=404)
if self.response["playlist_type"] == "custom":
error = ErrorResponseSerializer(
{"error": f"playlist with ID {playlist_id} is custom"}
)
return Response(error.data, status=400)
subscribed = validated_data.get("playlist_subscribed")
sort_order = validated_data.get("playlist_sort_order")

3
backend/video/src/index.py
View File

@ -174,15 +174,12 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
# extract
self.channel_id = self.youtube_meta["channel_id"]
last_refresh = int(datetime.now().timestamp())
# base64_blur = ThumbManager().get_base64_blur(self.youtube_id)
base64_blur = False
# build json_data basics
self.json_data = {
"title": self.youtube_meta["title"],
"description": self.youtube_meta.get("description", ""),
"category": self.youtube_meta.get("categories", []),
"vid_thumb_url": self.youtube_meta["thumbnail"],
"vid_thumb_base64": base64_blur,
"tags": self.youtube_meta.get("tags", []),
"published": self._build_published(),
"vid_last_refresh": last_refresh,

1
frontend/src/pages/Home.tsx
View File

@ -74,7 +74,6 @@ export type VideoType = {
tags: string[];
title: string;
vid_last_refresh: string;
vid_thumb_base64: boolean;
vid_thumb_url: string;
vid_type: string;
youtube_id: string;

50
frontend/src/pages/Playlist.tsx
View File

@ -288,31 +288,33 @@ const Playlist = () => {
/>
</div>
)}
<div className="toggle">
<span>Switch sort order:</span>
<div className="toggleBox">
<input
id="playlist_sort_order"
type="checkbox"
checked={playlist.playlist_sort_order === 'bottom'}
onChange={async () => {
const newSortOrder =
playlist.playlist_sort_order === 'top' ? 'bottom' : 'top';
await updatePlaylistSortOrder(playlist.playlist_id, newSortOrder);
setRefresh(true);
}}
/>
{playlist.playlist_sort_order === 'bottom' ? (
<label htmlFor="" className="onbtn">
On
</label>
) : (
<label htmlFor="" className="ofbtn">
Off
</label>
)}
{playlist.playlist_type !== 'custom' && (
<div className="toggle">
<span>Switch sort order:</span>
<div className="toggleBox">
<input
id="playlist_sort_order"
type="checkbox"
checked={playlist.playlist_sort_order === 'bottom'}
onChange={async () => {
const newSortOrder =
playlist.playlist_sort_order === 'top' ? 'bottom' : 'top';
await updatePlaylistSortOrder(playlist.playlist_id, newSortOrder);
setRefresh(true);
}}
/>
{playlist.playlist_sort_order === 'bottom' ? (
<label htmlFor="" className="onbtn">
On
</label>
) : (
<label htmlFor="" className="ofbtn">
Off
</label>
)}
</div>
</div>
</div>
)}
</div>
</div>
</div>