RomanNum3ral
/
tubearchivist
mirror of https://github.com/tubearchivist/tubearchivist.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

serializer improvements for video, mapping definitions

This commit is contained in:
Simon 2026-01-04 14:21:56 +07:00
parent 56cfe50316
commit 2ec9f9e78b
7 changed files with 215 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

268
backend/appsettings/index_mapping.json
View File

@ -110,17 +110,17 @@
{
"index_name": "video",
"expected_map": {
"vid_thumb_url": {
"type": "text",
"index": false
"active": {
"type": "boolean"
},
"vid_thumb_base64": {
"category": {
"type": "text",
"index": false
},
"date_downloaded": {
"type": "date",
"format": "epoch_second"
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"channel": {
"properties": {
@ -211,84 +211,22 @@
}
}
},
"comment_count": {
"type": "long"
},
"date_downloaded": {
"type": "date",
"format": "epoch_second"
},
"description": {
"type": "text"
},
"media_url": {
"type": "keyword",
"index": false
},
"media_size": {
"type": "long"
},
"tags": {
"type": "text",
"analyzer": "english",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"analyzer": "english",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower"
},
"search_as_you_type": {
"type": "search_as_you_type",
"doc_values": false,
"max_shingle_size": 3
}
}
},
"vid_last_refresh": {
"type": "date",
"format": "epoch_second"
},
"youtube_id": {
"type": "keyword"
},
"vid_type": {
"type": "keyword"
},
"published": {
"type": "date",
"format": "epoch_second||strict_date_optional_time"
},
"playlist": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower"
}
}
},
"comment_count": {
"type": "long"
},
"stats": {
"properties": {
"average_rating": {
"type": "float"
},
"dislike_count": {
"type": "long"
},
"like_count": {
"type": "long"
},
"view_count": {
"type": "long"
}
}
"media_url": {
"type": "keyword",
"index": false
},
"player": {
"properties": {
@ -308,68 +246,32 @@
}
}
},
"subtitles": {
"properties": {
"ext": {
"playlist": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"index": false
},
"lang": {
"type": "keyword",
"index": false
},
"media_url": {
"type": "keyword",
"index": false
},
"name": {
"type": "keyword"
},
"source": {
"type": "keyword"
},
"url": {
"type": "keyword",
"index": false
"ignore_above": 256,
"normalizer": "to_lower"
}
}
},
"streams": {
"properties": {
"type": {
"type": "keyword",
"index": false
},
"index": {
"type": "short",
"index": false
},
"codec": {
"type": "text"
},
"width": {
"type": "short"
},
"height": {
"type": "short"
},
"bitrate": {
"type": "integer"
}
}
"published": {
"type": "date",
"format": "epoch_second||strict_date_optional_time"
},
"sponsorblock": {
"properties": {
"last_refresh": {
"type": "date",
"format": "epoch_second"
},
"has_unlocked": {
"type": "boolean"
},
"is_enabled": {
"type": "boolean"
},
"last_refresh": {
"type": "date",
"format": "epoch_second"
},
"segments": {
"properties": {
"UUID": {
@ -405,6 +307,112 @@
}
}
}
},
"stats": {
"properties": {
"average_rating": {
"type": "float"
},
"dislike_count": {
"type": "long"
},
"like_count": {
"type": "long"
},
"view_count": {
"type": "long"
}
}
},
"streams": {
"properties": {
"bitrate": {
"type": "integer"
},
"codec": {
"type": "text"
},
"height": {
"type": "short"
},
"index": {
"type": "short",
"index": false
},
"type": {
"type": "keyword",
"index": false
},
"width": {
"type": "short"
}
}
},
"subtitles": {
"properties": {
"ext": {
"type": "keyword",
"index": false
},
"lang": {
"type": "keyword",
"index": false
},
"media_url": {
"type": "keyword",
"index": false
},
"name": {
"type": "keyword"
},
"source": {
"type": "keyword"
},
"url": {
"type": "keyword",
"index": false
}
}
},
"tags": {
"type": "text",
"analyzer": "english",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"analyzer": "english",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower"
},
"search_as_you_type": {
"type": "search_as_you_type",
"doc_values": false,
"max_shingle_size": 3
}
}
},
"vid_last_refresh": {
"type": "date",
"format": "epoch_second"
},
"vid_thumb_url": {
"type": "text",
"index": false
},
"vid_type": {
"type": "keyword"
},
"youtube_id": {
"type": "keyword"
}
},
"expected_set": {

1
backend/common/src/search_processor.py
View File

@ -138,6 +138,7 @@ class SearchProcess:
"vid_last_refresh": vid_last_refresh,
"published": published,
"vid_thumb_url": f"{cache_root}/{vid_thumb_url}",
"description": video_dict.get("description"),
}
)

31
backend/config/management/commands/ta_startup.py
View File

@ -14,7 +14,7 @@ from appsettings.src.index_setup import ElasticIndexWrap
from appsettings.src.snapshot import ElasticSnapshot
from channel.src.index import YoutubeChannel
from common.src.env_settings import EnvironmentSettings
from common.src.es_connect import ElasticWrap
from common.src.es_connect import ElasticWrap, IndexPaginate
from common.src.helper import clear_dl_cache, get_channels
from common.src.ta_redis import RedisArchivist
from django.core.management.base import BaseCommand, CommandError
@ -25,6 +25,7 @@ from task.src.config_schedule import ScheduleBuilder
from task.src.task_manager import TaskManager
from task.tasks import version_check
from video.src.constants import VideoTypeEnum
from video.src.index import YoutubeVideo
TOPIC = """
@ -59,6 +60,7 @@ class Command(BaseCommand):
self._mig_fix_missing_stats()
self._mig_fix_channel_art_types()
self._mig_fix_channel_description()
self._mig_fix_video_description()
def _make_folders(self):
"""make expected cache folders"""
@ -409,6 +411,33 @@ class Command(BaseCommand):
noop_msg = " no items needed updating"
self.stdout.write(self.style.SUCCESS(noop_msg))
def _mig_fix_video_description(self) -> None:
"""migrate from 0.5.8 to 0.5.9, fix video desc null value"""
desc = "fix video description null value"
self.stdout.write(f"[MIGRATION] run {desc}")
data = {"_source": ["youtube_id", "description"]}
videos = IndexPaginate("ta_video", data=data).get_results()
counter = 0
for video_response in videos:
if not video_response.get("description") == "":
continue
video = YoutubeVideo(youtube_id=video_response["youtube_id"])
video.get_from_es()
video.json_data.pop("description")
video.upload_to_es()
counter += 1
if counter:
suc_msg = f" ✓ updated {counter} videos"
self.stdout.write(self.style.SUCCESS(suc_msg))
else:
noop_msg = " no items needed updating"
self.stdout.write(self.style.SUCCESS(noop_msg))
def _run_migration(
self, index_name: str, desc: str, query: dict, script: dict
):

17
backend/video/serializers.py
View File

@ -16,6 +16,7 @@ class PlayerSerializer(serializers.Serializer):
watched_date = serializers.IntegerField(required=False)
duration = serializers.IntegerField()
duration_str = serializers.CharField()
progress = serializers.FloatField(required=False)
position = serializers.FloatField(required=False)
@ -53,12 +54,12 @@ class StatsSerializer(serializers.Serializer):
class StreamItemSerializer(serializers.Serializer):
"""serialize stream item"""
index = serializers.IntegerField()
codec = serializers.CharField()
bitrate = serializers.IntegerField()
codec = serializers.CharField()
height = serializers.IntegerField(required=False)
index = serializers.IntegerField()
type = serializers.ChoiceField(choices=["video", "audio"])
width = serializers.IntegerField(required=False)
height = serializers.IntegerField(required=False)
class SubtitleFragmentSerializer(serializers.Serializer):
@ -81,12 +82,12 @@ class SubtitleFragmentSerializer(serializers.Serializer):
class SubtitleItemSerializer(serializers.Serializer):
"""serialize subtitle item"""
ext = serializers.ChoiceField(choices=["json3"])
name = serializers.CharField()
source = serializers.ChoiceField(choices=["user", "auto"])
ext = serializers.ChoiceField(choices=["json3", "vtt"])
lang = serializers.CharField()
media_url = serializers.CharField()
url = serializers.URLField()
name = serializers.CharField()
source = serializers.ChoiceField(choices=["user", "auto"])
url = serializers.URLField(allow_null=True)
class VideoSerializer(serializers.Serializer):
@ -97,7 +98,7 @@ class VideoSerializer(serializers.Serializer):
channel = ChannelSerializer(required=False)
comment_count = serializers.IntegerField(allow_null=True, required=False)
date_downloaded = serializers.IntegerField()
description = serializers.CharField()
description = serializers.CharField(allow_null=True)
media_size = serializers.IntegerField()
media_url = serializers.CharField()
player = PlayerSerializer()

31
backend/video/src/index.py
View File

@ -198,26 +198,25 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
def process_youtube_meta(self):
"""extract relevant fields from youtube"""
self._validate_id()
# extract
self.channel_id = self.youtube_meta["channel_id"]
last_refresh = int(datetime.now().timestamp())
# build json_data basics
self.json_data = {
"title": self.youtube_meta["title"],
"description": self.youtube_meta.get("description", ""),
"category": self.youtube_meta.get("categories", []),
"vid_thumb_url": self.youtube_meta["thumbnail"],
"tags": self.youtube_meta.get("tags", []),
"published": self._build_published(),
"vid_last_refresh": last_refresh,
"date_downloaded": last_refresh,
"youtube_id": self.youtube_id,
# Using .value to make json encodable
"vid_type": self.video_type.value,
"active": True,
"category": self.youtube_meta.get("categories", []),
"date_downloaded": last_refresh,
"published": self._build_published(),
"tags": self.youtube_meta.get("tags", []),
"title": self.youtube_meta["title"],
"vid_last_refresh": last_refresh,
"vid_thumb_url": self.youtube_meta["thumbnail"],
"vid_type": self.video_type.value,
"youtube_id": self.youtube_id,
}
def _build_published(self):
if description := self.youtube_meta.get("description"):
self.json_data["description"] = description
def _build_published(self) -> int | str:
"""build published date or timestamp"""
timestamp = self.youtube_meta.get("timestamp")
if timestamp and isinstance(timestamp, int):
@ -288,9 +287,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
self.json_data.update(
{
"player": {
"watched": False,
"duration": duration,
"duration_str": get_duration_str(duration),
"watched": False,
}
}
)
@ -412,7 +411,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
subtitle_media_url = f"{base_name}.{lang}.vtt"
to_add = {
"ext": "vtt",
"url": False,
"url": None,
"name": lang,
"lang": lang,
"source": "file",

26
backend/video/src/media_streams.py
View File

@ -12,7 +12,7 @@ class MediaStreamExtractor:
self.media_path = media_path
self.metadata = []
def extract_metadata(self):
def extract_metadata(self) -> list[dict]:
"""entry point to extract metadata"""
cmd = [
@ -38,17 +38,15 @@ class MediaStreamExtractor:
return self.metadata
def process_stream(self, stream):
def process_stream(self, stream) -> None:
"""parse stream to metadata"""
codec_type = stream.get("codec_type")
if codec_type == "video":
self._extract_video_metadata(stream)
elif codec_type == "audio":
self._extract_audio_metadata(stream)
else:
return
def _extract_video_metadata(self, stream):
def _extract_video_metadata(self, stream) -> None:
"""parse video metadata"""
if "bit_rate" not in stream:
# is probably thumbnail
@ -56,26 +54,26 @@ class MediaStreamExtractor:
self.metadata.append(
{
"type": "video",
"index": stream["index"],
"bitrate": int(stream.get("bit_rate", 0)),
"codec": stream["codec_name"],
"width": stream["width"],
"height": stream["height"],
"bitrate": int(stream["bit_rate"]),
"index": stream["index"],
"type": "video",
"width": stream["width"],
}
)
def _extract_audio_metadata(self, stream):
def _extract_audio_metadata(self, stream) -> None:
"""extract audio metadata"""
self.metadata.append(
{
"type": "audio",
"index": stream["index"],
"codec": stream.get("codec_name", "undefined"),
"bitrate": int(stream.get("bit_rate", 0)),
"codec": stream.get("codec_name", "undefined"),
"index": stream["index"],
"type": "audio",
}
)
def get_file_size(self):
def get_file_size(self) -> int:
"""get filesize in bytes"""
return stat(self.media_path).st_size

11
backend/video/src/subtitle.py
View File

@ -149,7 +149,16 @@ class YoutubeSubtitle:
query_str = parser.create_bulk_import(documents)
self.index_subtitle(query_str)
indexed.append(subtitle)
indexed.append(
{
"ext": "json3",
"name": subtitle["name"],
"source": subtitle["source"],
"lang": subtitle["lang"],
"media_url": subtitle["media_url"],
"url": subtitle["url"],
}
)
rand_sleep(self.video.config)
return indexed