RomanNum3ral
/
tubearchivist
mirror of https://github.com/tubearchivist/tubearchivist.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

serialize and validate channel data

This commit is contained in:
Simon 2026-01-03 15:44:08 +07:00
parent 9d4ecdf9b7
commit 6e346efcb3
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
8 changed files with 203 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

146
backend/appsettings/index_mapping.json
View File

@ -12,6 +12,28 @@
"channel_id": {
"type": "keyword"
},
"channel_active": {
"type": "boolean"
},
"channel_banner_url": {
"type": "keyword",
"index": false
},
"channel_thumb_url": {
"type": "keyword",
"index": false
},
"channel_tvart_url": {
"type": "keyword",
"index": false
},
"channel_description": {
"type": "text"
},
"channel_last_refresh": {
"type": "date",
"format": "epoch_second"
},
"channel_name": {
"type": "text",
"analyzer": "english",
@ -28,38 +50,6 @@
}
}
},
"channel_banner_url": {
"type": "keyword",
"index": false
},
"channel_tvart_url": {
"type": "keyword",
"index": false
},
"channel_thumb_url": {
"type": "keyword",
"index": false
},
"channel_description": {
"type": "text"
},
"channel_last_refresh": {
"type": "date",
"format": "epoch_second"
},
"channel_tags": {
"type": "text",
"analyzer": "english",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"channel_tabs": {
"type": "keyword"
},
"channel_overwrites": {
"properties": {
"download_format": {
@ -84,6 +74,25 @@
"type": "long"
}
}
},
"channel_subs": {
"type": "long"
},
"channel_subscribed": {
"type": "boolean"
},
"channel_tags": {
"type": "text",
"analyzer": "english",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"channel_tabs": {
"type": "keyword"
}
},
"expected_set": {
@ -118,6 +127,28 @@
"channel_id": {
"type": "keyword"
},
"channel_active": {
"type": "boolean"
},
"channel_banner_url": {
"type": "keyword",
"index": false
},
"channel_thumb_url": {
"type": "keyword",
"index": false
},
"channel_tvart_url": {
"type": "keyword",
"index": false
},
"channel_description": {
"type": "text"
},
"channel_last_refresh": {
"type": "date",
"format": "epoch_second"
},
"channel_name": {
"type": "text",
"analyzer": "english",
@ -134,38 +165,6 @@
}
}
},
"channel_banner_url": {
"type": "keyword",
"index": false
},
"channel_tvart_url": {
"type": "keyword",
"index": false
},
"channel_thumb_url": {
"type": "keyword",
"index": false
},
"channel_description": {
"type": "text"
},
"channel_last_refresh": {
"type": "date",
"format": "epoch_second"
},
"channel_tags": {
"type": "text",
"analyzer": "english",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"channel_tabs": {
"type": "keyword"
},
"channel_overwrites": {
"properties": {
"download_format": {
@ -190,6 +189,25 @@
"type": "long"
}
}
},
"channel_subs": {
"type": "long"
},
"channel_subscribed": {
"type": "boolean"
},
"channel_tags": {
"type": "text",
"analyzer": "english",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"channel_tabs": {
"type": "keyword"
}
}
},

11
backend/channel/serializers.py
View File

@ -34,10 +34,12 @@ class ChannelSerializer(serializers.Serializer):
channel_id = serializers.CharField()
channel_active = serializers.BooleanField()
channel_banner_url = serializers.CharField()
channel_thumb_url = serializers.CharField()
channel_tvart_url = serializers.CharField()
channel_description = serializers.CharField()
channel_banner_url = serializers.CharField(allow_null=True, required=False)
channel_thumb_url = serializers.CharField(allow_null=True, required=False)
channel_tvart_url = serializers.CharField(allow_null=True, required=False)
channel_description = serializers.CharField(
allow_null=True, required=False
)
channel_last_refresh = serializers.CharField()
channel_name = serializers.CharField()
channel_overwrites = ChannelOverwriteSerializer(required=False)
@ -49,7 +51,6 @@ class ChannelSerializer(serializers.Serializer):
channel_tabs = serializers.ListField(
child=serializers.ChoiceField(VideoTypeEnum.values_known())
)
channel_views = serializers.IntegerField()
_index = serializers.CharField(required=False)
_score = serializers.IntegerField(required=False)

88
backend/channel/src/index.py
View File

@ -57,54 +57,56 @@ class YoutubeChannel(YouTubeItem):
"""extract relevant fields"""
self.youtube_meta["thumbnails"].reverse()
channel_name = self.youtube_meta["uploader"] or self.youtube_meta["id"]
description = self.youtube_meta.get("description") or None
self.json_data = {
"channel_active": True,
"channel_description": self.youtube_meta.get("description", ""),
"channel_description": description,
"channel_id": self.youtube_id,
"channel_last_refresh": int(datetime.now().timestamp()),
"channel_name": channel_name,
"channel_subs": self.youtube_meta.get("channel_follower_count", 0),
"channel_subs": self.youtube_meta.get("channel_follower_count")
or 0,
"channel_subscribed": False,
"channel_tags": self.youtube_meta.get("tags", []),
"channel_banner_url": self._get_banner_art(),
"channel_thumb_url": self._get_thumb_art(),
"channel_tvart_url": self._get_tv_art(),
"channel_views": self.youtube_meta.get("view_count") or 0,
"channel_tabs": self.get_channel_tabs(),
}
def _get_thumb_art(self):
self._get_thumb_art()
self._get_tv_art()
self._get_banner_art()
def _get_thumb_art(self) -> None:
"""extract thumb art"""
for i in self.youtube_meta["thumbnails"]:
if not i.get("width"):
continue
if i.get("width") == i.get("height"):
return i["url"]
self.json_data["channel_thumb_url"] = i["url"]
return
return False
def _get_tv_art(self):
def _get_tv_art(self) -> None:
"""extract tv artwork"""
for i in self.youtube_meta["thumbnails"]:
if i.get("id") == "banner_uncropped":
return i["url"]
self.json_data["channel_tvart_url"] = i["url"]
return
for i in self.youtube_meta["thumbnails"]:
if not i.get("width"):
continue
if i["width"] // i["height"] < 2 and not i["width"] == i["height"]:
return i["url"]
self.json_data["channel_tvart_url"] = i["url"]
return
return False
return
def _get_banner_art(self):
def _get_banner_art(self) -> None:
"""extract banner artwork"""
for i in self.youtube_meta["thumbnails"]:
if not i.get("width"):
continue
if i["width"] // i["height"] > 5:
return i["url"]
return False
self.json_data["channel_banner_url"] = i["url"]
return
def get_channel_tabs(self) -> list[str]:
"""get channel tabs"""
@ -132,51 +134,39 @@ class YoutubeChannel(YouTubeItem):
self.json_data = {
"channel_active": False,
"channel_last_refresh": int(datetime.now().timestamp()),
"channel_subs": fallback.get("channel_follower_count", 0),
"channel_subs": fallback.get("channel_follower_count") or 0,
"channel_name": fallback["uploader"],
"channel_banner_url": False,
"channel_tvart_url": False,
"channel_id": self.youtube_id,
"channel_subscribed": False,
"channel_tags": [],
"channel_description": "",
"channel_thumb_url": False,
"channel_views": 0,
}
def get_channel_art(self):
"""download channel art for new channels"""
urls = (
self.json_data["channel_thumb_url"],
self.json_data["channel_banner_url"],
self.json_data["channel_tvart_url"],
self.json_data.get("channel_thumb_url"),
self.json_data.get("channel_banner_url"),
self.json_data.get("channel_tvart_url"),
)
ThumbManager(self.youtube_id, item_type="channel").download(urls)
def sync_to_videos(self):
"""sync new channel_dict to all videos of channel"""
# add ingest pipeline
processors = []
for field, value in self.json_data.items():
if value is None:
line = {
"script": {
"lang": "painless",
"source": f"ctx['{field}'] = null;",
}
}
else:
line = {"set": {"field": "channel." + field, "value": value}}
processors.append(line)
data = {"description": self.youtube_id, "processors": processors}
ingest_path = f"_ingest/pipeline/{self.youtube_id}"
_, _ = ElasticWrap(ingest_path).put(data)
# apply pipeline
data = {"query": {"match": {"channel.channel_id": self.youtube_id}}}
update_path = f"ta_video/_update_by_query?pipeline={self.youtube_id}"
_, _ = ElasticWrap(update_path).post(data)
data = {
"query": {
"term": {"channel.channel_id": {"value": self.youtube_id}},
},
"script": {
"lang": "painless",
"params": {"channel": self.json_data},
"source": "ctx._source.channel = params.channel",
},
}
update_path = "ta_video/_update_by_query"
response, status_code = ElasticWrap(update_path).post(data)
if status_code not in [200, 201]:
print(f"sync to videos failed with status code {status_code}")
print(response)
def change_subscribe(self, new_subscribe_state: bool):
"""change subscribe status"""

19
backend/common/src/search_processor.py
View File

@ -89,12 +89,25 @@ class SearchProcess:
channel_dict.update(
{
"channel_last_refresh": date_str,
"channel_banner_url": f"{art_base}_banner.jpg",
"channel_thumb_url": f"{art_base}_thumb.jpg",
"channel_tvart_url": f"{art_base}_tvart.jpg",
"channel_description": channel_dict.get("channel_description"),
}
)
if channel_dict.get("channel_banner_url"):
channel_dict["channel_banner_url"] = f"{art_base}_banner.jpg"
else:
channel_dict["channel_banner_url"] = None
if channel_dict.get("channel_thumb_url"):
channel_dict["channel_thumb_url"] = f"{art_base}_thumb.jpg"
else:
channel_dict["channel_thumb_url"] = None
if channel_dict.get("channel_tvart_url"):
channel_dict["channel_tvart_url"] = f"{art_base}_tvart.jpg"
else:
channel_dict["channel_tvart_url"] = None
return dict(sorted(channel_dict.items()))
def _process_video(self, video_dict):

58
backend/config/management/commands/ta_startup.py
View File

@ -12,9 +12,10 @@ from time import sleep
from appsettings.src.config import AppConfig, ReleaseVersion
from appsettings.src.index_setup import ElasticIndexWrap
from appsettings.src.snapshot import ElasticSnapshot
from channel.src.index import YoutubeChannel
from common.src.env_settings import EnvironmentSettings
from common.src.es_connect import ElasticWrap
from common.src.helper import clear_dl_cache
from common.src.helper import clear_dl_cache, get_channels
from common.src.ta_redis import RedisArchivist
from django.core.management.base import BaseCommand, CommandError
from django.utils import dateformat
@ -56,6 +57,8 @@ class Command(BaseCommand):
self._mig_set_video_channel_tabs()
self._mig_fix_playlist_description()
self._mig_fix_missing_stats()
self._mig_fix_channel_art_types()
self._mig_fix_channel_description()
def _make_folders(self):
"""make expected cache folders"""
@ -353,6 +356,59 @@ class Command(BaseCommand):
},
)
def _mig_fix_channel_art_types(self) -> None:
"""migrate from 0.5.8 to 0.5.9, fix channel artwork types"""
fields = [
"channel_banner_url",
"channel_thumb_url",
"channel_tvart_url",
]
for field in fields:
self._run_migration(
index_name="ta_channel",
desc=f"fix missing data type for field {field}",
query={"term": {field: {"value": False}}},
script={
"source": f"ctx._source.remove('{field}')",
"lang": "painless",
},
)
self._run_migration(
index_name="ta_video",
desc=f"fix missing data type for field channel.{field}",
query={"term": {f"channel.{field}": {"value": False}}},
script={
"source": f"ctx._source.remove('channel.{field}')",
"lang": "painless",
},
)
def _mig_fix_channel_description(self) -> None:
"""migrate from 0.5.8 to 0.5.9, fix channel desc null value"""
desc = "fix channel description null value"
self.stdout.write(f"[MIGRATION] run {desc}")
channels = get_channels(
subscribed_only=False, source=["channel_description", "channel_id"]
)
counter = 0
for channel_response in channels:
if not channel_response.get("channel_description") == "":
continue
channel = YoutubeChannel(youtube_id=channel_response["channel_id"])
channel.get_from_es()
channel.json_data.pop("channel_description")
channel.upload_to_es()
channel.sync_to_videos()
counter += 1
if counter:
suc_msg = f" ✓ updated {counter} channels with videos"
self.stdout.write(self.style.SUCCESS(suc_msg))
else:
noop_msg = " no items needed updating"
self.stdout.write(self.style.SUCCESS(noop_msg))
def _run_migration(
self, index_name: str, desc: str, query: dict, script: dict
):

6
backend/download/src/thumbnails.py
View File

@ -324,9 +324,9 @@ class ValidatorCallback:
"""check if all channel artwork is there"""
for channel in self.source:
urls = (
channel["_source"]["channel_thumb_url"],
channel["_source"]["channel_banner_url"],
channel["_source"].get("channel_tvart_url", False),
channel["_source"].get("channel_thumb_url"),
channel["_source"].get("channel_banner_url"),
channel["_source"].get("channel_tvart_url"),
)
handler = ThumbManager(channel["_source"]["channel_id"])
handler.download_channel_art(urls, skip_existing=True)

5
frontend/src/pages/ChannelAbout.tsx
View File

@ -8,7 +8,6 @@ import Routes from '../configuration/routes/RouteList';
import queueReindex, { ReindexType, ReindexTypeEnum } from '../api/actions/queueReindex';
import formatDate from '../functions/formatDates';
import PaginationDummy from '../components/PaginationDummy';
import FormattedNumber from '../components/FormattedNumber';
import Button from '../components/Button';
import updateChannelOverwrites from '../api/actions/updateChannelOverwrite';
import useIsAdmin from '../functions/useIsAdmin';
@ -145,10 +144,6 @@ const ChannelAbout = () => {
<div className="info-box-item">
<div>
{channel.channel_views > 0 && (
<FormattedNumber text="Channel views:" number={channel.channel_views} />
)}
{isAdmin && (
<>
<div className="button-box">

1
frontend/src/pages/Channels.tsx
View File

@ -41,7 +41,6 @@ export type ChannelType = {
channel_tags?: string[];
channel_thumb_url: string;
channel_tvart_url: string;
channel_views: number;
};
const Channels = () => {