Revamped index setup, #build
Changed: - Changed index setup and migrations handling - Use index aliaes on mapping update - Detect what needs a mapping refresh - Fix several inconsitent indexing issues and serialization - Add PO token provider URL - Add temporary cookie persistence
This commit is contained in:
Simon
commit
8b3db0e78f
10
Dockerfile
10
Dockerfile
|
|
@ -1,11 +1,11 @@
|
|||
# multi stage to build tube archivist
|
||||
# build python wheel, download and extract ffmpeg, copy into final image
|
||||
|
||||
FROM node:lts-alpine AS npm-builder
|
||||
FROM node:22.12.0-alpine AS npm-builder
|
||||
COPY frontend/package.json frontend/package-lock.json /
|
||||
RUN npm i
|
||||
|
||||
FROM node:lts-alpine AS node-builder
|
||||
FROM node:22.12.0-alpine AS node-builder
|
||||
|
||||
# RUN npm config set registry https://registry.npmjs.org/
|
||||
|
||||
|
|
@ -18,7 +18,7 @@ RUN npm run build:deploy
|
|||
WORKDIR /
|
||||
|
||||
# First stage to build python wheel
|
||||
FROM python:3.11.13-slim-bookworm AS builder
|
||||
FROM python:3.13.11-slim-trixie AS builder
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential gcc libldap2-dev libsasl2-dev libssl-dev git
|
||||
|
|
@ -28,7 +28,7 @@ COPY ./backend/requirements.txt /requirements.txt
|
|||
RUN pip install --user -r requirements.txt
|
||||
|
||||
# build ffmpeg
|
||||
FROM python:3.11.13-slim-bookworm AS ffmpeg-builder
|
||||
FROM python:3.13.11-slim-trixie AS ffmpeg-builder
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
|
||||
|
|
@ -36,7 +36,7 @@ COPY docker_assets/ffmpeg_download.py ffmpeg_download.py
|
|||
RUN python ffmpeg_download.py $TARGETPLATFORM
|
||||
|
||||
# build final image
|
||||
FROM python:3.11.13-slim-bookworm AS tubearchivist
|
||||
FROM python:3.13.11-slim-trixie AS tubearchivist
|
||||
|
||||
ARG INSTALL_DEBUG
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,28 @@
|
|||
"channel_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"channel_active": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"channel_banner_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_thumb_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_tvart_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_description": {
|
||||
"type": "text"
|
||||
},
|
||||
"channel_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"channel_name": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
|
|
@ -28,38 +50,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"channel_banner_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_tvart_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_thumb_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_description": {
|
||||
"type": "text"
|
||||
},
|
||||
"channel_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"channel_tags": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"channel_tabs": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"channel_overwrites": {
|
||||
"properties": {
|
||||
"download_format": {
|
||||
|
|
@ -84,6 +74,25 @@
|
|||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"channel_subs": {
|
||||
"type": "long"
|
||||
},
|
||||
"channel_subscribed": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"channel_tags": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"channel_tabs": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"expected_set": {
|
||||
|
|
@ -101,23 +110,45 @@
|
|||
{
|
||||
"index_name": "video",
|
||||
"expected_map": {
|
||||
"vid_thumb_url": {
|
||||
"type": "text",
|
||||
"index": false
|
||||
"active": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"vid_thumb_base64": {
|
||||
"category": {
|
||||
"type": "text",
|
||||
"index": false
|
||||
},
|
||||
"date_downloaded": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"channel": {
|
||||
"properties": {
|
||||
"channel_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"channel_active": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"channel_banner_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_thumb_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_tvart_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_description": {
|
||||
"type": "text"
|
||||
},
|
||||
"channel_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"channel_name": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
|
|
@ -134,38 +165,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"channel_banner_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_tvart_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_thumb_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"channel_description": {
|
||||
"type": "text"
|
||||
},
|
||||
"channel_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"channel_tags": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"channel_tabs": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"channel_overwrites": {
|
||||
"properties": {
|
||||
"download_format": {
|
||||
|
|
@ -190,87 +189,44 @@
|
|||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": {
|
||||
"type": "text"
|
||||
},
|
||||
"media_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"media_size": {
|
||||
"type": "long"
|
||||
},
|
||||
"tags": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"title": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
},
|
||||
"search_as_you_type": {
|
||||
"type": "search_as_you_type",
|
||||
"doc_values": false,
|
||||
"max_shingle_size": 3
|
||||
}
|
||||
}
|
||||
},
|
||||
"vid_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"youtube_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"vid_type": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"published": {
|
||||
"type": "date",
|
||||
"format": "epoch_second||strict_date_optional_time"
|
||||
},
|
||||
"playlist": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
"channel_subs": {
|
||||
"type": "long"
|
||||
},
|
||||
"channel_subscribed": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"channel_tags": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"channel_tabs": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"comment_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"stats": {
|
||||
"properties": {
|
||||
"average_rating": {
|
||||
"type": "float"
|
||||
},
|
||||
"dislike_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"like_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"view_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
"date_downloaded": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"description": {
|
||||
"type": "text"
|
||||
},
|
||||
"media_size": {
|
||||
"type": "long"
|
||||
},
|
||||
"media_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"player": {
|
||||
"properties": {
|
||||
|
|
@ -290,68 +246,32 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"subtitles": {
|
||||
"properties": {
|
||||
"ext": {
|
||||
"playlist": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"lang": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"media_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"name": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"source": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
}
|
||||
}
|
||||
},
|
||||
"streams": {
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"index": {
|
||||
"type": "short",
|
||||
"index": false
|
||||
},
|
||||
"codec": {
|
||||
"type": "text"
|
||||
},
|
||||
"width": {
|
||||
"type": "short"
|
||||
},
|
||||
"height": {
|
||||
"type": "short"
|
||||
},
|
||||
"bitrate": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
"published": {
|
||||
"type": "date",
|
||||
"format": "epoch_second||strict_date_optional_time"
|
||||
},
|
||||
"sponsorblock": {
|
||||
"properties": {
|
||||
"last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"has_unlocked": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"is_enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"segments": {
|
||||
"properties": {
|
||||
"UUID": {
|
||||
|
|
@ -387,6 +307,112 @@
|
|||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"stats": {
|
||||
"properties": {
|
||||
"average_rating": {
|
||||
"type": "float"
|
||||
},
|
||||
"dislike_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"like_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"view_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"streams": {
|
||||
"properties": {
|
||||
"bitrate": {
|
||||
"type": "integer"
|
||||
},
|
||||
"codec": {
|
||||
"type": "text"
|
||||
},
|
||||
"height": {
|
||||
"type": "short"
|
||||
},
|
||||
"index": {
|
||||
"type": "short",
|
||||
"index": false
|
||||
},
|
||||
"type": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"width": {
|
||||
"type": "short"
|
||||
}
|
||||
}
|
||||
},
|
||||
"subtitles": {
|
||||
"properties": {
|
||||
"ext": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"lang": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"media_url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
},
|
||||
"name": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"source": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"url": {
|
||||
"type": "keyword",
|
||||
"index": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"tags": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"title": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
},
|
||||
"search_as_you_type": {
|
||||
"type": "search_as_you_type",
|
||||
"doc_values": false,
|
||||
"max_shingle_size": 3
|
||||
}
|
||||
}
|
||||
},
|
||||
"vid_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"vid_thumb_url": {
|
||||
"type": "text",
|
||||
"index": false
|
||||
},
|
||||
"vid_type": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"youtube_id": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"expected_set": {
|
||||
|
|
@ -404,13 +430,15 @@
|
|||
{
|
||||
"index_name": "download",
|
||||
"expected_map": {
|
||||
"timestamp": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
"auto_start": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"channel_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"channel_indexed": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"channel_name": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
|
|
@ -421,9 +449,23 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"duration": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"message": {
|
||||
"type": "text"
|
||||
},
|
||||
"published": {
|
||||
"type": "date",
|
||||
"format": "epoch_second||strict_date_optional_time"
|
||||
},
|
||||
"status": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"timestamp": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"title": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
|
|
@ -434,24 +476,14 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"published": {
|
||||
"type": "date",
|
||||
"format": "epoch_second||strict_date_optional_time"
|
||||
},
|
||||
"vid_thumb_url": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"youtube_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"vid_type": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"auto_start": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"message": {
|
||||
"type": "text"
|
||||
"youtube_id": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"expected_set": {
|
||||
|
|
@ -469,37 +501,9 @@
|
|||
{
|
||||
"index_name": "playlist",
|
||||
"expected_map": {
|
||||
"playlist_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"playlist_description": {
|
||||
"type": "text"
|
||||
},
|
||||
"playlist_subscribed": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"playlist_type": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"playlist_active": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"playlist_name": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
},
|
||||
"search_as_you_type": {
|
||||
"type": "search_as_you_type",
|
||||
"doc_values": false,
|
||||
"max_shingle_size": 3
|
||||
}
|
||||
}
|
||||
},
|
||||
"playlist_channel": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
|
|
@ -513,15 +517,8 @@
|
|||
"playlist_channel_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"playlist_thumbnail": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"playlist_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"playlist_sort_order": {
|
||||
"type": "keyword"
|
||||
"playlist_description": {
|
||||
"type": "text"
|
||||
},
|
||||
"playlist_entries": {
|
||||
"properties": {
|
||||
|
|
@ -557,6 +554,41 @@
|
|||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"playlist_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"playlist_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"playlist_name": {
|
||||
"type": "text",
|
||||
"analyzer": "english",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
},
|
||||
"search_as_you_type": {
|
||||
"type": "search_as_you_type",
|
||||
"doc_values": false,
|
||||
"max_shingle_size": 3
|
||||
}
|
||||
}
|
||||
},
|
||||
"playlist_sort_order": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"playlist_subscribed": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"playlist_thumbnail": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"playlist_type": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"expected_set": {
|
||||
|
|
@ -574,22 +606,6 @@
|
|||
{
|
||||
"index_name": "subtitle",
|
||||
"expected_map": {
|
||||
"youtube_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"title": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
}
|
||||
}
|
||||
},
|
||||
"subtitle_fragment_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"subtitle_channel": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
|
|
@ -603,15 +619,11 @@
|
|||
"subtitle_channel_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"subtitle_start": {
|
||||
"type": "text"
|
||||
},
|
||||
"subtitle_end": {
|
||||
"type": "text"
|
||||
},
|
||||
"subtitle_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
"subtitle_fragment_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"subtitle_index": {
|
||||
"type": "long"
|
||||
|
|
@ -619,12 +631,32 @@
|
|||
"subtitle_lang": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"subtitle_source": {
|
||||
"type": "keyword"
|
||||
"subtitle_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"subtitle_line": {
|
||||
"type": "text",
|
||||
"analyzer": "english"
|
||||
},
|
||||
"subtitle_source": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"subtitle_start": {
|
||||
"type": "text"
|
||||
},
|
||||
"title": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256,
|
||||
"normalizer": "to_lower"
|
||||
}
|
||||
}
|
||||
},
|
||||
"youtube_id": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"expected_set": {
|
||||
|
|
@ -642,37 +674,11 @@
|
|||
{
|
||||
"index_name": "comment",
|
||||
"expected_map": {
|
||||
"youtube_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"comment_channel_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_comments": {
|
||||
"properties": {
|
||||
"comment_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_text": {
|
||||
"type": "text"
|
||||
},
|
||||
"comment_timestamp": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"comment_time_text": {
|
||||
"type": "text"
|
||||
},
|
||||
"comment_likecount": {
|
||||
"type": "long"
|
||||
},
|
||||
"comment_is_favorited": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"comment_author": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
|
|
@ -686,16 +692,42 @@
|
|||
"comment_author_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_author_thumbnail": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_author_is_uploader": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"comment_author_thumbnail": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_is_favorited": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"comment_likecount": {
|
||||
"type": "long"
|
||||
},
|
||||
"comment_parent": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_text": {
|
||||
"type": "text"
|
||||
},
|
||||
"comment_time_text": {
|
||||
"type": "text"
|
||||
},
|
||||
"comment_timestamp": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
}
|
||||
}
|
||||
},
|
||||
"comment_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"youtube_id": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"expected_set": {
|
||||
|
|
|
|||
|
|
@ -44,7 +44,6 @@ class AppConfigDownloadsSerializer(
|
|||
format = serializers.CharField(allow_null=True)
|
||||
format_sort = serializers.CharField(allow_null=True)
|
||||
add_metadata = serializers.BooleanField()
|
||||
add_thumbnail = serializers.BooleanField()
|
||||
subtitle = serializers.CharField(allow_null=True)
|
||||
subtitle_source = serializers.ChoiceField(
|
||||
choices=["auto", "user"], allow_null=True
|
||||
|
|
@ -55,6 +54,7 @@ class AppConfigDownloadsSerializer(
|
|||
choices=["top", "new"], allow_null=True
|
||||
)
|
||||
cookie_import = serializers.BooleanField()
|
||||
pot_provider_url = serializers.CharField(allow_null=True)
|
||||
potoken = serializers.BooleanField()
|
||||
throttledratelimit = serializers.IntegerField(allow_null=True)
|
||||
extractor_lang = serializers.CharField(allow_null=True)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ Functionality:
|
|||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
|
||||
|
|
@ -19,7 +20,10 @@ from task.models import CustomPeriodicTask
|
|||
class ElasticBackup:
|
||||
"""dump index to nd-json files for later bulk import"""
|
||||
|
||||
INDEX_SPLIT = ["comment"]
|
||||
INDEX_SIZE_CONF = {
|
||||
"comment": 200,
|
||||
"subtitle": 10000,
|
||||
}
|
||||
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||
BACKUP_DIR = os.path.join(CACHE_DIR, "backup")
|
||||
|
||||
|
|
@ -62,8 +66,8 @@ class ElasticBackup:
|
|||
"total": self._get_total(index_name),
|
||||
}
|
||||
|
||||
if index_name in self.INDEX_SPLIT:
|
||||
paginate_kwargs.update({"size": 200})
|
||||
if size_overwrite := self.INDEX_SIZE_CONF.get(index_name):
|
||||
paginate_kwargs.update({"size": size_overwrite})
|
||||
|
||||
paginate = IndexPaginate(f"ta_{index_name}", **paginate_kwargs)
|
||||
_ = paginate.get_results()
|
||||
|
|
@ -98,8 +102,7 @@ class ElasticBackup:
|
|||
|
||||
def post_bulk_restore(self, file_name):
|
||||
"""send bulk to es"""
|
||||
file_path = os.path.join(self.CACHE_DIR, file_name)
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
with open(file_name, "r", encoding="utf-8") as f:
|
||||
data = f.read()
|
||||
|
||||
if not data.strip():
|
||||
|
|
@ -156,6 +159,7 @@ class ElasticBackup:
|
|||
call reset from ElasticIndexWrap first to start blank
|
||||
"""
|
||||
zip_content = self._unpack_zip_backup(filename)
|
||||
zip_content.sort()
|
||||
self._restore_json_files(zip_content)
|
||||
|
||||
def _unpack_zip_backup(self, filename):
|
||||
|
|
@ -252,7 +256,7 @@ class BackupCallback:
|
|||
|
||||
for document in self.source:
|
||||
document_id = document["_id"]
|
||||
es_index = document["_index"]
|
||||
es_index = re.sub(r"_v\d+$", "", document["_index"]) # remove _v
|
||||
action = {"index": {"_index": es_index, "_id": document_id}}
|
||||
source = document["_source"]
|
||||
bulk_list.append(json.dumps(action))
|
||||
|
|
|
|||
|
|
@ -35,13 +35,13 @@ class DownloadsConfigType(TypedDict):
|
|||
format: str | None
|
||||
format_sort: str | None
|
||||
add_metadata: bool
|
||||
add_thumbnail: bool
|
||||
subtitle: str | None
|
||||
subtitle_source: Literal["user", "auto"] | None
|
||||
subtitle_index: bool
|
||||
comment_max: str | None
|
||||
comment_sort: Literal["top", "new"] | None
|
||||
cookie_import: bool
|
||||
pot_provider_url: str | None
|
||||
potoken: bool
|
||||
throttledratelimit: int | None
|
||||
extractor_lang: str | None
|
||||
|
|
@ -85,13 +85,13 @@ class AppConfig:
|
|||
"format": None,
|
||||
"format_sort": None,
|
||||
"add_metadata": False,
|
||||
"add_thumbnail": False,
|
||||
"subtitle": None,
|
||||
"subtitle_source": None,
|
||||
"subtitle_index": False,
|
||||
"comment_max": None,
|
||||
"comment_sort": "top",
|
||||
"cookie_import": False,
|
||||
"pot_provider_url": None,
|
||||
"potoken": False,
|
||||
"throttledratelimit": None,
|
||||
"extractor_lang": None,
|
||||
|
|
|
|||
|
|
@ -5,83 +5,109 @@ functionality:
|
|||
- backup and restore metadata
|
||||
"""
|
||||
|
||||
from enum import Enum, auto
|
||||
|
||||
from appsettings.src.backup import ElasticBackup
|
||||
from appsettings.src.config import AppConfig
|
||||
from appsettings.src.snapshot import ElasticSnapshot
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from common.src.helper import get_mapping
|
||||
from deepdiff import DeepDiff
|
||||
from django.conf import settings
|
||||
|
||||
|
||||
class MappingAction(Enum):
|
||||
"""index action options"""
|
||||
|
||||
NOOP = auto()
|
||||
PUT_MAPPING = auto()
|
||||
REINDEX = auto()
|
||||
|
||||
|
||||
class ElasticIndex:
|
||||
"""interact with a single index"""
|
||||
|
||||
REINDEX_KEYS = {
|
||||
"type",
|
||||
"analyzer",
|
||||
"search_analyzer",
|
||||
"normalizer",
|
||||
"index",
|
||||
"doc_values",
|
||||
"norms",
|
||||
"ignore_above",
|
||||
"enabled",
|
||||
"format",
|
||||
}
|
||||
|
||||
def __init__(self, index_name, expected_map=False, expected_set=False):
|
||||
self.index_name = index_name
|
||||
self.expected_map = expected_map
|
||||
self.expected_set = expected_set
|
||||
self.exists, self.details = self.index_exists()
|
||||
|
||||
@property
|
||||
def index_namespace(self) -> str:
|
||||
"""namespaced index"""
|
||||
return f"ta_{self.index_name}"
|
||||
|
||||
def index_exists(self):
|
||||
"""check if index already exists and return mapping if it does"""
|
||||
response, status_code = ElasticWrap(f"ta_{self.index_name}").get()
|
||||
response, status_code = ElasticWrap(self.index_namespace).get()
|
||||
exists = status_code == 200
|
||||
details = response.get(f"ta_{self.index_name}", False)
|
||||
if not exists:
|
||||
return False, False
|
||||
|
||||
index_key = f"{self.index_namespace}"
|
||||
current_version = self.get_current_version()
|
||||
if current_version:
|
||||
index_key += f"_v{current_version}"
|
||||
|
||||
details = response.get(index_key, False)
|
||||
|
||||
return exists, details
|
||||
|
||||
def validate(self):
|
||||
def get_current_version(self) -> None | int:
|
||||
"""get current version from aliases of index"""
|
||||
response, _ = ElasticWrap(f"{self.index_namespace}/_alias").get()
|
||||
if not response:
|
||||
raise ValueError("failed to fetch aliases: ", response)
|
||||
|
||||
alias_name = list(response.keys())
|
||||
if not alias_name:
|
||||
return None
|
||||
|
||||
version_str = alias_name[0].lstrip(f"{self.index_namespace}_v")
|
||||
if not version_str:
|
||||
# is initial version
|
||||
return None
|
||||
|
||||
if not version_str.isdigit():
|
||||
raise ValueError("unexpected version_str: ", version_str)
|
||||
|
||||
return int(version_str)
|
||||
|
||||
def validate(self) -> tuple[MappingAction, set[str]]:
|
||||
"""
|
||||
check if all expected mappings and settings match
|
||||
returns True when rebuild is needed
|
||||
"""
|
||||
|
||||
if self.expected_map or self.expected_map == {}:
|
||||
rebuild = self.validate_mappings()
|
||||
if rebuild:
|
||||
return rebuild
|
||||
mapping_diff = self._get_mapping_diff()
|
||||
removed_fields = self._get_fields_to_delete(diff=mapping_diff)
|
||||
|
||||
if self.expected_set:
|
||||
rebuild = self.validate_settings()
|
||||
if rebuild:
|
||||
return rebuild
|
||||
settings_diff = self._validate_settings()
|
||||
if settings_diff:
|
||||
# treat settings diff as full reindex
|
||||
return MappingAction.REINDEX, removed_fields
|
||||
|
||||
return False
|
||||
if self.expected_map or self.expected_map == {}:
|
||||
action = self._classify_mapping_diff(diff=mapping_diff)
|
||||
return action, removed_fields
|
||||
|
||||
def validate_mappings(self):
|
||||
"""check if all mappings are as expected"""
|
||||
now_map = self.details["mappings"].get("properties", {})
|
||||
return MappingAction.NOOP, removed_fields
|
||||
|
||||
for key, value in self.expected_map.items():
|
||||
# nested
|
||||
if list(value.keys()) == ["properties"]:
|
||||
for key_n, value_n in value["properties"].items():
|
||||
if key not in now_map:
|
||||
print(f"detected mapping change: {key_n}, {value_n}")
|
||||
return True
|
||||
if key_n not in now_map[key]["properties"].keys():
|
||||
print(f"detected mapping change: {key_n}, {value_n}")
|
||||
return True
|
||||
if not value_n == now_map[key]["properties"][key_n]:
|
||||
print(f"detected mapping change: {key_n}, {value_n}")
|
||||
return True
|
||||
|
||||
continue
|
||||
|
||||
# not nested
|
||||
if key not in now_map.keys():
|
||||
print(f"detected mapping change: {key}, {value}")
|
||||
return True
|
||||
if not value == now_map[key]:
|
||||
print(f"detected mapping change: {key}, {value}")
|
||||
return True
|
||||
|
||||
# simple doc store
|
||||
if self.expected_map == {} and now_map != self.expected_map:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def validate_settings(self):
|
||||
def _validate_settings(self):
|
||||
"""check if all settings are as expected"""
|
||||
|
||||
now_set = self.details["settings"]["index"]
|
||||
|
|
@ -97,44 +123,94 @@ class ElasticIndex:
|
|||
|
||||
return False
|
||||
|
||||
def rebuild_index(self):
|
||||
def _get_mapping_diff(self) -> DeepDiff:
|
||||
"""check if all mappings are as expected"""
|
||||
now_map = self.details.get("mappings", {}).get("properties", {})
|
||||
diff = DeepDiff(
|
||||
now_map,
|
||||
self.expected_map,
|
||||
ignore_order=True,
|
||||
report_repetition=True,
|
||||
view="tree",
|
||||
)
|
||||
if diff:
|
||||
print(f"[{self.index_namespace}] detected mapping change")
|
||||
if settings.DEBUG:
|
||||
print(f"[{self.index_namespace}] mapping change: {diff}")
|
||||
|
||||
return diff
|
||||
|
||||
def _classify_mapping_diff(self, diff: DeepDiff) -> MappingAction:
|
||||
"""use diff to detect what to do"""
|
||||
if not diff:
|
||||
return MappingAction.NOOP
|
||||
|
||||
if diff.get("type_changes"):
|
||||
# always incompatible, needs reindex
|
||||
return MappingAction.REINDEX
|
||||
|
||||
added = diff.get("dictionary_item_added", [])
|
||||
has_additions = bool(added)
|
||||
|
||||
for item in diff.get("values_changed", []):
|
||||
path = item.path(output_format="list")
|
||||
if not path:
|
||||
continue
|
||||
|
||||
if path[-1] in self.REINDEX_KEYS:
|
||||
return MappingAction.REINDEX
|
||||
|
||||
# compatible addition
|
||||
has_additions = True
|
||||
|
||||
if has_additions:
|
||||
return MappingAction.PUT_MAPPING
|
||||
|
||||
return MappingAction.NOOP
|
||||
|
||||
def _get_fields_to_delete(self, diff: DeepDiff) -> set[str]:
|
||||
"""fields to remove during next reindex"""
|
||||
removed_fields = set()
|
||||
for item in diff.get("dictionary_item_removed", []):
|
||||
value = item.t1 or {}
|
||||
is_field_definition = "type" in value or "properties" in value
|
||||
if not is_field_definition:
|
||||
continue
|
||||
|
||||
path = item.path(output_format="list")
|
||||
removed_fields.add(".".join(path))
|
||||
|
||||
return removed_fields
|
||||
|
||||
def rebuild_index(self, removed_fields: set[str]):
|
||||
"""rebuild with new mapping"""
|
||||
print(f"applying new mappings to index ta_{self.index_name}...")
|
||||
self.create_blank(for_backup=True)
|
||||
self.reindex("backup")
|
||||
self.delete_index(backup=False)
|
||||
self.create_blank()
|
||||
self.reindex("restore")
|
||||
self.delete_index()
|
||||
print(f"[{self.index_namespace}] applying new mappings to index")
|
||||
current_version = self.get_current_version()
|
||||
|
||||
def reindex(self, method):
|
||||
"""create on elastic search"""
|
||||
if method == "backup":
|
||||
source = f"ta_{self.index_name}"
|
||||
destination = f"ta_{self.index_name}_backup"
|
||||
elif method == "restore":
|
||||
source = f"ta_{self.index_name}_backup"
|
||||
destination = f"ta_{self.index_name}"
|
||||
else:
|
||||
raise ValueError("invalid method, expected 'backup' or 'restore'")
|
||||
new_version = current_version + 1 if current_version else 2
|
||||
|
||||
data = {"source": {"index": source}, "dest": {"index": destination}}
|
||||
_, _ = ElasticWrap("_reindex?refresh=true").post(data=data)
|
||||
self.create_blank(new_version=new_version)
|
||||
self.reindex(new_version=new_version, removed_fields=removed_fields)
|
||||
self.delete_index(by_version=current_version)
|
||||
self.create_alias(new_version=new_version, old_version=current_version)
|
||||
|
||||
def delete_index(self, backup=True):
|
||||
def delete_index(self, by_version: int | None):
|
||||
"""delete index passed as argument"""
|
||||
path = f"ta_{self.index_name}"
|
||||
if backup:
|
||||
path = path + "_backup"
|
||||
path = self.index_namespace
|
||||
if by_version is not None:
|
||||
path += f"_v{by_version}"
|
||||
|
||||
_, _ = ElasticWrap(path).delete()
|
||||
print(f"[{path}] delete index")
|
||||
response, status_code = ElasticWrap(path).delete()
|
||||
if status_code not in [200, 201]:
|
||||
print(f"{status_code}: {response}")
|
||||
raise ValueError("index delete failed")
|
||||
|
||||
def create_blank(self, for_backup=False):
|
||||
"""apply new mapping and settings for blank new index"""
|
||||
print(f"create new blank index with name ta_{self.index_name}...")
|
||||
path = f"ta_{self.index_name}"
|
||||
if for_backup:
|
||||
path = f"{path}_backup"
|
||||
def create_blank(self, new_version: int | None = None):
|
||||
"""create blank"""
|
||||
path = self.index_namespace
|
||||
if new_version is not None:
|
||||
path += f"_v{new_version}"
|
||||
|
||||
data = {}
|
||||
if self.expected_set:
|
||||
|
|
@ -145,7 +221,98 @@ class ElasticIndex:
|
|||
# no indexing for config
|
||||
data["mappings"]["dynamic"] = False
|
||||
|
||||
_, _ = ElasticWrap(path).put(data)
|
||||
print(f"[{path}] create new blank index")
|
||||
if settings.DEBUG:
|
||||
print(f"[{path}] creat new blank index with data: {data}")
|
||||
|
||||
response, status_code = ElasticWrap(path).put(data)
|
||||
if status_code not in [200, 201]:
|
||||
print(f"{status_code}: {response}")
|
||||
raise ValueError(f"create blank index {path} failed")
|
||||
|
||||
def reindex(self, new_version: int, removed_fields: set[str]):
|
||||
"""reindex to versioned new index after creating"""
|
||||
source = self.index_namespace
|
||||
dest = f"{self.index_namespace}_v{new_version}"
|
||||
data: dict = {"source": {"index": source}, "dest": {"index": dest}}
|
||||
|
||||
if removed_fields:
|
||||
script = "\n".join(
|
||||
f"ctx._source.remove('{i}');" for i in removed_fields
|
||||
)
|
||||
data["script"] = {"lang": "painless", "source": script}
|
||||
|
||||
msg = f"[{self.index_namespace}] reindex from {source} to {dest}"
|
||||
if removed_fields:
|
||||
msg += f", remove unexpected fields: {removed_fields}"
|
||||
|
||||
print(msg)
|
||||
|
||||
if settings.DEBUG:
|
||||
print(f"send data: {data}")
|
||||
|
||||
path = "_reindex?refresh=true"
|
||||
response, status_code = ElasticWrap(path).post(data=data)
|
||||
if status_code not in [200, 201]:
|
||||
print(f"{status_code}: {response}")
|
||||
raise ValueError("reindex failed failed")
|
||||
|
||||
def create_alias(self, new_version: int, old_version: int | None = None):
|
||||
"""create aliast for moved index"""
|
||||
index_new = f"{self.index_namespace}_v{new_version}"
|
||||
index_old = None
|
||||
|
||||
data: dict = {
|
||||
"actions": [
|
||||
{
|
||||
"add": {
|
||||
"index": index_new,
|
||||
"alias": self.index_namespace,
|
||||
"is_write_index": True,
|
||||
},
|
||||
},
|
||||
]
|
||||
}
|
||||
if old_version:
|
||||
index_old = f"{self.index_namespace}_v{old_version}"
|
||||
data["actions"].append(
|
||||
{
|
||||
"remove": {
|
||||
"index": index_old,
|
||||
"alias": self.index_namespace,
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
message = f"create new alias {index_new}"
|
||||
if index_old:
|
||||
message += f", remove old alias {index_old}"
|
||||
|
||||
print(f"[{self.index_namespace}] {message}")
|
||||
if settings.DEBUG:
|
||||
print(f"create alias with data: {data}")
|
||||
|
||||
response, status_code = ElasticWrap("_alias").put(data=data)
|
||||
if status_code not in [200, 201]:
|
||||
print(f"{status_code}: {response}")
|
||||
raise ValueError("alias update failed")
|
||||
|
||||
def mapping_update(self):
|
||||
"""simple mapping update only, use migrations for defaults"""
|
||||
current_version = self.get_current_version()
|
||||
path = self.index_namespace
|
||||
if current_version is not None:
|
||||
path += f"_v{current_version}"
|
||||
|
||||
data = {"properties": self.expected_map}
|
||||
print(f"[{path}] update mapping")
|
||||
if settings.DEBUG:
|
||||
print(f"[{path}] update mapping with data: {data}")
|
||||
|
||||
response, status_code = ElasticWrap(f"{path}/_mapping").put(data)
|
||||
if status_code not in [200, 201]:
|
||||
print(f"{status_code}: {response}")
|
||||
raise ValueError(f"create blank index {path} failed")
|
||||
|
||||
|
||||
class ElasticIndexWrap:
|
||||
|
|
@ -164,14 +331,22 @@ class ElasticIndexWrap:
|
|||
handler.create_blank()
|
||||
continue
|
||||
|
||||
rebuild = handler.validate()
|
||||
if rebuild:
|
||||
action, removed_fields = handler.validate()
|
||||
if action == MappingAction.REINDEX:
|
||||
self._check_backup()
|
||||
handler.rebuild_index()
|
||||
handler.rebuild_index(removed_fields)
|
||||
continue
|
||||
|
||||
# else all good
|
||||
print(f"ta_{index_name} index is created and up to date...")
|
||||
if action == MappingAction.PUT_MAPPING:
|
||||
handler.mapping_update()
|
||||
|
||||
if removed_fields:
|
||||
print(
|
||||
f"[ta_{index_name}] skip removing unexpected fields:"
|
||||
+ f" {removed_fields}"
|
||||
)
|
||||
else:
|
||||
print(f"[ta_{index_name}] index status is as expected.")
|
||||
|
||||
def reset(self):
|
||||
"""reset all indexes to blank"""
|
||||
|
|
@ -180,11 +355,15 @@ class ElasticIndexWrap:
|
|||
|
||||
def delete_all(self):
|
||||
"""delete all indexes"""
|
||||
print("reset elastic index")
|
||||
for index in self.index_config:
|
||||
index_name, _, _ = self._config_split(index)
|
||||
print(f"[ta_{index_name}] reset elastic index")
|
||||
handler = ElasticIndex(index_name)
|
||||
handler.delete_index(backup=False)
|
||||
if not handler.exists:
|
||||
continue
|
||||
|
||||
current_version = handler.get_current_version()
|
||||
handler.delete_index(by_version=current_version)
|
||||
|
||||
def create_all_blank(self):
|
||||
"""create all blank indexes"""
|
||||
|
|
|
|||
|
|
@ -261,10 +261,15 @@ class ElasticSnapshot:
|
|||
def restore_all(self, snapshot_name):
|
||||
"""restore snapshot by name"""
|
||||
for index in self.all_indices:
|
||||
_, _ = ElasticWrap(index).delete()
|
||||
response, status_code = ElasticWrap(index).get()
|
||||
if status_code == 404:
|
||||
continue
|
||||
|
||||
index_alias = list(response.keys())[0]
|
||||
_, _ = ElasticWrap(index_alias).delete()
|
||||
|
||||
path = f"_snapshot/{self.REPO}/{snapshot_name}/_restore"
|
||||
data = {"indices": "*"}
|
||||
data = {"indices": "*,-.*"}
|
||||
response, statuscode = ElasticWrap(path).post(data=data)
|
||||
if statuscode == 200:
|
||||
print(f"snapshot: executing now: {response}")
|
||||
|
|
|
|||
|
|
@ -34,10 +34,12 @@ class ChannelSerializer(serializers.Serializer):
|
|||
|
||||
channel_id = serializers.CharField()
|
||||
channel_active = serializers.BooleanField()
|
||||
channel_banner_url = serializers.CharField()
|
||||
channel_thumb_url = serializers.CharField()
|
||||
channel_tvart_url = serializers.CharField()
|
||||
channel_description = serializers.CharField()
|
||||
channel_banner_url = serializers.CharField(allow_null=True, required=False)
|
||||
channel_thumb_url = serializers.CharField(allow_null=True, required=False)
|
||||
channel_tvart_url = serializers.CharField(allow_null=True, required=False)
|
||||
channel_description = serializers.CharField(
|
||||
allow_null=True, required=False
|
||||
)
|
||||
channel_last_refresh = serializers.CharField()
|
||||
channel_name = serializers.CharField()
|
||||
channel_overwrites = ChannelOverwriteSerializer(required=False)
|
||||
|
|
@ -49,7 +51,6 @@ class ChannelSerializer(serializers.Serializer):
|
|||
channel_tabs = serializers.ListField(
|
||||
child=serializers.ChoiceField(VideoTypeEnum.values_known())
|
||||
)
|
||||
channel_views = serializers.IntegerField()
|
||||
_index = serializers.CharField(required=False)
|
||||
_score = serializers.IntegerField(required=False)
|
||||
|
||||
|
|
|
|||
|
|
@ -57,54 +57,56 @@ class YoutubeChannel(YouTubeItem):
|
|||
"""extract relevant fields"""
|
||||
self.youtube_meta["thumbnails"].reverse()
|
||||
channel_name = self.youtube_meta["uploader"] or self.youtube_meta["id"]
|
||||
description = self.youtube_meta.get("description") or None
|
||||
self.json_data = {
|
||||
"channel_active": True,
|
||||
"channel_description": self.youtube_meta.get("description", ""),
|
||||
"channel_description": description,
|
||||
"channel_id": self.youtube_id,
|
||||
"channel_last_refresh": int(datetime.now().timestamp()),
|
||||
"channel_name": channel_name,
|
||||
"channel_subs": self.youtube_meta.get("channel_follower_count", 0),
|
||||
"channel_subs": self.youtube_meta.get("channel_follower_count")
|
||||
or 0,
|
||||
"channel_subscribed": False,
|
||||
"channel_tags": self.youtube_meta.get("tags", []),
|
||||
"channel_banner_url": self._get_banner_art(),
|
||||
"channel_thumb_url": self._get_thumb_art(),
|
||||
"channel_tvart_url": self._get_tv_art(),
|
||||
"channel_views": self.youtube_meta.get("view_count") or 0,
|
||||
"channel_tabs": self.get_channel_tabs(),
|
||||
}
|
||||
|
||||
def _get_thumb_art(self):
|
||||
self._get_thumb_art()
|
||||
self._get_tv_art()
|
||||
self._get_banner_art()
|
||||
|
||||
def _get_thumb_art(self) -> None:
|
||||
"""extract thumb art"""
|
||||
for i in self.youtube_meta["thumbnails"]:
|
||||
if not i.get("width"):
|
||||
continue
|
||||
if i.get("width") == i.get("height"):
|
||||
return i["url"]
|
||||
self.json_data["channel_thumb_url"] = i["url"]
|
||||
return
|
||||
|
||||
return False
|
||||
|
||||
def _get_tv_art(self):
|
||||
def _get_tv_art(self) -> None:
|
||||
"""extract tv artwork"""
|
||||
for i in self.youtube_meta["thumbnails"]:
|
||||
if i.get("id") == "banner_uncropped":
|
||||
return i["url"]
|
||||
self.json_data["channel_tvart_url"] = i["url"]
|
||||
return
|
||||
for i in self.youtube_meta["thumbnails"]:
|
||||
if not i.get("width"):
|
||||
continue
|
||||
if i["width"] // i["height"] < 2 and not i["width"] == i["height"]:
|
||||
return i["url"]
|
||||
self.json_data["channel_tvart_url"] = i["url"]
|
||||
return
|
||||
|
||||
return False
|
||||
return
|
||||
|
||||
def _get_banner_art(self):
|
||||
def _get_banner_art(self) -> None:
|
||||
"""extract banner artwork"""
|
||||
for i in self.youtube_meta["thumbnails"]:
|
||||
if not i.get("width"):
|
||||
continue
|
||||
if i["width"] // i["height"] > 5:
|
||||
return i["url"]
|
||||
|
||||
return False
|
||||
self.json_data["channel_banner_url"] = i["url"]
|
||||
return
|
||||
|
||||
def get_channel_tabs(self) -> list[str]:
|
||||
"""get channel tabs"""
|
||||
|
|
@ -132,51 +134,39 @@ class YoutubeChannel(YouTubeItem):
|
|||
self.json_data = {
|
||||
"channel_active": False,
|
||||
"channel_last_refresh": int(datetime.now().timestamp()),
|
||||
"channel_subs": fallback.get("channel_follower_count", 0),
|
||||
"channel_subs": fallback.get("channel_follower_count") or 0,
|
||||
"channel_name": fallback["uploader"],
|
||||
"channel_banner_url": False,
|
||||
"channel_tvart_url": False,
|
||||
"channel_id": self.youtube_id,
|
||||
"channel_subscribed": False,
|
||||
"channel_tags": [],
|
||||
"channel_description": "",
|
||||
"channel_thumb_url": False,
|
||||
"channel_views": 0,
|
||||
}
|
||||
|
||||
def get_channel_art(self):
|
||||
"""download channel art for new channels"""
|
||||
urls = (
|
||||
self.json_data["channel_thumb_url"],
|
||||
self.json_data["channel_banner_url"],
|
||||
self.json_data["channel_tvart_url"],
|
||||
self.json_data.get("channel_thumb_url"),
|
||||
self.json_data.get("channel_banner_url"),
|
||||
self.json_data.get("channel_tvart_url"),
|
||||
)
|
||||
ThumbManager(self.youtube_id, item_type="channel").download(urls)
|
||||
|
||||
def sync_to_videos(self):
|
||||
"""sync new channel_dict to all videos of channel"""
|
||||
# add ingest pipeline
|
||||
processors = []
|
||||
for field, value in self.json_data.items():
|
||||
if value is None:
|
||||
line = {
|
||||
"script": {
|
||||
"lang": "painless",
|
||||
"source": f"ctx['{field}'] = null;",
|
||||
}
|
||||
}
|
||||
else:
|
||||
line = {"set": {"field": "channel." + field, "value": value}}
|
||||
|
||||
processors.append(line)
|
||||
|
||||
data = {"description": self.youtube_id, "processors": processors}
|
||||
ingest_path = f"_ingest/pipeline/{self.youtube_id}"
|
||||
_, _ = ElasticWrap(ingest_path).put(data)
|
||||
# apply pipeline
|
||||
data = {"query": {"match": {"channel.channel_id": self.youtube_id}}}
|
||||
update_path = f"ta_video/_update_by_query?pipeline={self.youtube_id}"
|
||||
_, _ = ElasticWrap(update_path).post(data)
|
||||
data = {
|
||||
"query": {
|
||||
"term": {"channel.channel_id": {"value": self.youtube_id}},
|
||||
},
|
||||
"script": {
|
||||
"lang": "painless",
|
||||
"params": {"channel": self.json_data},
|
||||
"source": "ctx._source.channel = params.channel",
|
||||
},
|
||||
}
|
||||
update_path = "ta_video/_update_by_query"
|
||||
response, status_code = ElasticWrap(update_path).post(data)
|
||||
if status_code not in [200, 201]:
|
||||
print(f"sync to videos failed with status code {status_code}")
|
||||
print(response)
|
||||
|
||||
def change_subscribe(self, new_subscribe_state: bool):
|
||||
"""change subscribe status"""
|
||||
|
|
|
|||
|
|
@ -168,7 +168,7 @@ class IndexPaginate:
|
|||
|
||||
def get_pit(self):
|
||||
"""get pit for index"""
|
||||
path = f"{self.index_name}/_pit?keep_alive=10m"
|
||||
path = f"{self.index_name}/_pit?keep_alive=15m"
|
||||
response, _ = ElasticWrap(path).post()
|
||||
self.pit_id = response["id"]
|
||||
|
||||
|
|
@ -184,7 +184,7 @@ class IndexPaginate:
|
|||
self.data.update({"sort": [{"_doc": {"order": "desc"}}]})
|
||||
|
||||
self.data["size"] = self.kwargs.get("size") or self.DEFAULT_SIZE
|
||||
self.data["pit"] = {"id": self.pit_id, "keep_alive": "10m"}
|
||||
self.data["pit"] = {"id": self.pit_id, "keep_alive": "15m"}
|
||||
|
||||
def run_loop(self):
|
||||
"""loop through results until last hit"""
|
||||
|
|
|
|||
|
|
@ -56,17 +56,17 @@ class SearchProcess:
|
|||
"""detect which type of data to process"""
|
||||
index = result["_index"]
|
||||
processed = False
|
||||
if index == "ta_video":
|
||||
if index.startswith("ta_video"):
|
||||
processed = self._process_video(result["_source"])
|
||||
if index == "ta_channel":
|
||||
if index.startswith("ta_channel"):
|
||||
processed = self._process_channel(result["_source"])
|
||||
if index == "ta_playlist":
|
||||
if index.startswith("ta_playlist"):
|
||||
processed = self._process_playlist(result["_source"])
|
||||
if index == "ta_download":
|
||||
if index.startswith("ta_download"):
|
||||
processed = self._process_download(result["_source"])
|
||||
if index == "ta_comment":
|
||||
if index.startswith("ta_comment"):
|
||||
processed = self._process_comment(result["_source"])
|
||||
if index == "ta_subtitle":
|
||||
if index.startswith("ta_subtitle"):
|
||||
processed = self._process_subtitle(result)
|
||||
|
||||
if isinstance(processed, dict):
|
||||
|
|
@ -89,12 +89,25 @@ class SearchProcess:
|
|||
channel_dict.update(
|
||||
{
|
||||
"channel_last_refresh": date_str,
|
||||
"channel_banner_url": f"{art_base}_banner.jpg",
|
||||
"channel_thumb_url": f"{art_base}_thumb.jpg",
|
||||
"channel_tvart_url": f"{art_base}_tvart.jpg",
|
||||
"channel_description": channel_dict.get("channel_description"),
|
||||
}
|
||||
)
|
||||
|
||||
if channel_dict.get("channel_banner_url"):
|
||||
channel_dict["channel_banner_url"] = f"{art_base}_banner.jpg"
|
||||
else:
|
||||
channel_dict["channel_banner_url"] = None
|
||||
|
||||
if channel_dict.get("channel_thumb_url"):
|
||||
channel_dict["channel_thumb_url"] = f"{art_base}_thumb.jpg"
|
||||
else:
|
||||
channel_dict["channel_thumb_url"] = None
|
||||
|
||||
if channel_dict.get("channel_tvart_url"):
|
||||
channel_dict["channel_tvart_url"] = f"{art_base}_tvart.jpg"
|
||||
else:
|
||||
channel_dict["channel_tvart_url"] = None
|
||||
|
||||
return dict(sorted(channel_dict.items()))
|
||||
|
||||
def _process_video(self, video_dict):
|
||||
|
|
@ -125,6 +138,7 @@ class SearchProcess:
|
|||
"vid_last_refresh": vid_last_refresh,
|
||||
"published": published,
|
||||
"vid_thumb_url": f"{cache_root}/{vid_thumb_url}",
|
||||
"description": video_dict.get("description"),
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -154,10 +168,12 @@ class SearchProcess:
|
|||
)
|
||||
cache_root = EnvironmentSettings().get_cache_root()
|
||||
playlist_thumbnail = f"{cache_root}/playlists/{playlist_id}.jpg"
|
||||
description = playlist_dict.get("playlist_description")
|
||||
playlist_dict.update(
|
||||
{
|
||||
"playlist_thumbnail": playlist_thumbnail,
|
||||
"playlist_last_refresh": playlist_last_refresh,
|
||||
"playlist_description": description,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -12,9 +12,10 @@ from time import sleep
|
|||
from appsettings.src.config import AppConfig, ReleaseVersion
|
||||
from appsettings.src.index_setup import ElasticIndexWrap
|
||||
from appsettings.src.snapshot import ElasticSnapshot
|
||||
from channel.src.index import YoutubeChannel
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from common.src.helper import clear_dl_cache
|
||||
from common.src.es_connect import ElasticWrap, IndexPaginate
|
||||
from common.src.helper import clear_dl_cache, get_channels
|
||||
from common.src.ta_redis import RedisArchivist
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.utils import dateformat
|
||||
|
|
@ -24,6 +25,7 @@ from task.src.config_schedule import ScheduleBuilder
|
|||
from task.src.task_manager import TaskManager
|
||||
from task.tasks import version_check
|
||||
from video.src.constants import VideoTypeEnum
|
||||
from video.src.index import YoutubeVideo
|
||||
|
||||
TOPIC = """
|
||||
|
||||
|
|
@ -55,6 +57,10 @@ class Command(BaseCommand):
|
|||
self._mig_set_channel_tabs()
|
||||
self._mig_set_video_channel_tabs()
|
||||
self._mig_fix_playlist_description()
|
||||
self._mig_fix_missing_stats()
|
||||
self._mig_fix_channel_art_types()
|
||||
self._mig_fix_channel_description()
|
||||
self._mig_fix_video_description()
|
||||
|
||||
def _make_folders(self):
|
||||
"""make expected cache folders"""
|
||||
|
|
@ -66,6 +72,7 @@ class Command(BaseCommand):
|
|||
"import",
|
||||
"playlists",
|
||||
"videos",
|
||||
"ytdlp",
|
||||
]
|
||||
cache_dir = EnvironmentSettings.CACHE_DIR
|
||||
for folder in folders:
|
||||
|
|
@ -271,134 +278,189 @@ class Command(BaseCommand):
|
|||
|
||||
def _mig_add_default_playlist_sort(self) -> None:
|
||||
"""migrate from 0.5.4 to 0.5.5 set default playlist sortorder"""
|
||||
self.stdout.write("[MIGRATION] set default playlist sort order")
|
||||
path = "ta_playlist/_update_by_query"
|
||||
data = {
|
||||
"query": {
|
||||
self._run_migration(
|
||||
index_name="ta_playlist",
|
||||
desc="set default playlist sort order",
|
||||
query={
|
||||
"bool": {
|
||||
"must_not": [{"exists": {"field": "playlist_sort_order"}}]
|
||||
}
|
||||
},
|
||||
"script": {
|
||||
script={
|
||||
"source": "ctx._source.playlist_sort_order = 'top'",
|
||||
"lang": "painless",
|
||||
},
|
||||
}
|
||||
response, status_code = ElasticWrap(path).post(data)
|
||||
if status_code in [200, 201]:
|
||||
updated = response.get("updated")
|
||||
if updated:
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(f" ✓ updated {updated} playlists")
|
||||
)
|
||||
else:
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(" no playlists need updating")
|
||||
)
|
||||
return
|
||||
|
||||
message = " 🗙 failed to set default playlist sort order"
|
||||
self.stdout.write(self.style.ERROR(message))
|
||||
self.stdout.write(response)
|
||||
sleep(60)
|
||||
raise CommandError(message)
|
||||
)
|
||||
|
||||
def _mig_set_channel_tabs(self) -> None:
|
||||
"""migrate from 0.5.4 to 0.5.5 set initial channel tabs"""
|
||||
self.stdout.write("[MIGRATION] set default channel_tabs")
|
||||
|
||||
path = "ta_channel/_update_by_query"
|
||||
tabs = VideoTypeEnum.values_known()
|
||||
data = {
|
||||
"query": {
|
||||
self._run_migration(
|
||||
index_name="ta_channel",
|
||||
desc="set default channel_tabs in channel index",
|
||||
query={
|
||||
"bool": {"must_not": [{"exists": {"field": "channel_tabs"}}]}
|
||||
},
|
||||
"script": {
|
||||
script={
|
||||
"source": f"ctx._source.channel_tabs = {tabs}",
|
||||
"lang": "painless",
|
||||
},
|
||||
}
|
||||
response, status_code = ElasticWrap(path).post(data)
|
||||
if status_code in [200, 201]:
|
||||
updated = response.get("updated")
|
||||
if updated:
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(f" ✓ updated {updated} channels")
|
||||
)
|
||||
else:
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(" no channels need updating")
|
||||
)
|
||||
return
|
||||
|
||||
message = " 🗙 failed to set default channel_tabs"
|
||||
self.stdout.write(self.style.ERROR(message))
|
||||
self.stdout.write(response)
|
||||
sleep(60)
|
||||
raise CommandError(message)
|
||||
)
|
||||
|
||||
def _mig_set_video_channel_tabs(self) -> None:
|
||||
"""migrate from 0.5.4 to 0.5.5 set initial video channel tabs"""
|
||||
self.stdout.write("[MIGRATION] set default channel_tabs for videos")
|
||||
|
||||
path = "ta_video/_update_by_query"
|
||||
tabs = VideoTypeEnum.values_known()
|
||||
data = {
|
||||
"query": {
|
||||
self._run_migration(
|
||||
index_name="ta_video",
|
||||
desc="set default channel_tabs for videos",
|
||||
query={
|
||||
"bool": {
|
||||
"must_not": [{"exists": {"field": "channel.channel_tabs"}}]
|
||||
}
|
||||
},
|
||||
"script": {
|
||||
script={
|
||||
"source": f"ctx._source.channel.channel_tabs = {tabs}",
|
||||
"lang": "painless",
|
||||
},
|
||||
}
|
||||
response, status_code = ElasticWrap(path).post(data)
|
||||
if status_code in [200, 201]:
|
||||
updated = response.get("updated")
|
||||
if updated:
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(f" ✓ updated {updated} videos")
|
||||
)
|
||||
else:
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(" no videos need updating")
|
||||
)
|
||||
return
|
||||
|
||||
message = " 🗙 failed to set default channel_tabs"
|
||||
self.stdout.write(self.style.ERROR(message))
|
||||
self.stdout.write(response)
|
||||
sleep(60)
|
||||
raise CommandError(message)
|
||||
)
|
||||
|
||||
def _mig_fix_playlist_description(self) -> None:
|
||||
"""migrate from 0.5.8 to 0.5.9 fix playlist desc null data type"""
|
||||
self.stdout.write("[MIGRATION] fix playlist description data type")
|
||||
|
||||
path = "ta_playlist/_update_by_query"
|
||||
data = {
|
||||
"query": {"term": {"playlist_description": {"value": False}}},
|
||||
"script": {
|
||||
"source": "ctx._source.playlist_description = null",
|
||||
self._run_migration(
|
||||
index_name="ta_playlist",
|
||||
desc="fix playlist description data type",
|
||||
query={"term": {"playlist_description": {"value": False}}},
|
||||
script={
|
||||
"source": "ctx._source.remove('playlist_description')",
|
||||
"lang": "painless",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
def _mig_fix_missing_stats(self) -> None:
|
||||
"""migrate from 0.5.8 to 0.5.9, fix missing stats values"""
|
||||
fields = [
|
||||
"like_count",
|
||||
"average_rating",
|
||||
"view_count",
|
||||
"dislike_count",
|
||||
]
|
||||
for field in fields:
|
||||
self._run_migration(
|
||||
index_name="ta_video",
|
||||
desc=f"fix missing stats field {field}",
|
||||
query={
|
||||
"bool": {
|
||||
"must_not": [{"exists": {"field": f"stats.{field}"}}]
|
||||
}
|
||||
},
|
||||
script={
|
||||
"source": f"ctx._source.stats.{field} = 0",
|
||||
"lang": "painless",
|
||||
},
|
||||
)
|
||||
|
||||
def _mig_fix_channel_art_types(self) -> None:
|
||||
"""migrate from 0.5.8 to 0.5.9, fix channel artwork types"""
|
||||
fields = [
|
||||
"channel_banner_url",
|
||||
"channel_thumb_url",
|
||||
"channel_tvart_url",
|
||||
]
|
||||
for field in fields:
|
||||
self._run_migration(
|
||||
index_name="ta_channel",
|
||||
desc=f"fix missing data type for field {field}",
|
||||
query={"term": {field: {"value": False}}},
|
||||
script={
|
||||
"source": f"ctx._source.remove('{field}')",
|
||||
"lang": "painless",
|
||||
},
|
||||
)
|
||||
self._run_migration(
|
||||
index_name="ta_video",
|
||||
desc=f"fix missing data type for field channel.{field}",
|
||||
query={"term": {f"channel.{field}": {"value": False}}},
|
||||
script={
|
||||
"source": f"ctx._source.remove('channel.{field}')",
|
||||
"lang": "painless",
|
||||
},
|
||||
)
|
||||
|
||||
def _mig_fix_channel_description(self) -> None:
|
||||
"""migrate from 0.5.8 to 0.5.9, fix channel desc null value"""
|
||||
desc = "fix channel description null value"
|
||||
self.stdout.write(f"[MIGRATION] run {desc}")
|
||||
channels = get_channels(
|
||||
subscribed_only=False, source=["channel_description", "channel_id"]
|
||||
)
|
||||
counter = 0
|
||||
for channel_response in channels:
|
||||
if not channel_response.get("channel_description") == "":
|
||||
continue
|
||||
|
||||
channel = YoutubeChannel(youtube_id=channel_response["channel_id"])
|
||||
channel.get_from_es()
|
||||
channel.json_data.pop("channel_description")
|
||||
channel.upload_to_es()
|
||||
channel.sync_to_videos()
|
||||
counter += 1
|
||||
|
||||
if counter:
|
||||
suc_msg = f" ✓ updated {counter} channels with videos"
|
||||
self.stdout.write(self.style.SUCCESS(suc_msg))
|
||||
else:
|
||||
noop_msg = " no items needed updating"
|
||||
self.stdout.write(self.style.SUCCESS(noop_msg))
|
||||
|
||||
def _mig_fix_video_description(self) -> None:
|
||||
"""migrate from 0.5.8 to 0.5.9, fix video desc null value"""
|
||||
desc = "fix video description null value"
|
||||
self.stdout.write(f"[MIGRATION] run {desc}")
|
||||
|
||||
data = {"_source": ["youtube_id", "description"]}
|
||||
videos = IndexPaginate("ta_video", data=data).get_results()
|
||||
|
||||
counter = 0
|
||||
for video_response in videos:
|
||||
if not video_response.get("description") == "":
|
||||
continue
|
||||
|
||||
video = YoutubeVideo(youtube_id=video_response["youtube_id"])
|
||||
video.get_from_es()
|
||||
video.json_data.pop("description")
|
||||
video.upload_to_es()
|
||||
|
||||
counter += 1
|
||||
|
||||
if counter:
|
||||
suc_msg = f" ✓ updated {counter} videos"
|
||||
self.stdout.write(self.style.SUCCESS(suc_msg))
|
||||
else:
|
||||
noop_msg = " no items needed updating"
|
||||
self.stdout.write(self.style.SUCCESS(noop_msg))
|
||||
|
||||
def _run_migration(
|
||||
self, index_name: str, desc: str, query: dict, script: dict
|
||||
):
|
||||
"""run migration"""
|
||||
self.stdout.write(f"[MIGRATION] run {desc}")
|
||||
path = f"{index_name}/_update_by_query?wait_for_completion=true"
|
||||
data = {"query": query, "script": script}
|
||||
response, status_code = ElasticWrap(path).post(data)
|
||||
if status_code in [200, 201]:
|
||||
updated = response.get("updated")
|
||||
if updated:
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(f" ✓ updated {updated} playlists")
|
||||
)
|
||||
suc_msg = f" ✓ updated {updated} docs in {index_name}"
|
||||
self.stdout.write(self.style.SUCCESS(suc_msg))
|
||||
|
||||
# ensure index consistency
|
||||
ElasticWrap(f"{index_name}/_refresh").post()
|
||||
else:
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(" no playlists need updating")
|
||||
)
|
||||
noop_msg = f" no items in {index_name} need updating"
|
||||
self.stdout.write(self.style.SUCCESS(noop_msg))
|
||||
return
|
||||
|
||||
message = " 🗙 failed to fix playlist description null data type"
|
||||
message = f" 🗙 failed to run {desc} on index {index_name}"
|
||||
self.stdout.write(self.style.ERROR(message))
|
||||
self.stdout.write(response)
|
||||
sleep(60)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ class DownloadItemSerializer(serializers.Serializer):
|
|||
channel_indexed = serializers.BooleanField()
|
||||
channel_name = serializers.CharField()
|
||||
duration = serializers.CharField()
|
||||
message = serializers.CharField(required=False)
|
||||
published = serializers.CharField(allow_null=True)
|
||||
status = serializers.ChoiceField(
|
||||
choices=["pending", "ignore"], required=False
|
||||
|
|
@ -24,7 +25,6 @@ class DownloadItemSerializer(serializers.Serializer):
|
|||
vid_thumb_url = serializers.CharField(allow_null=True)
|
||||
vid_type = serializers.ChoiceField(choices=VideoTypeEnum.values())
|
||||
youtube_id = serializers.CharField()
|
||||
message = serializers.CharField(required=False)
|
||||
_index = serializers.CharField(required=False)
|
||||
_score = serializers.IntegerField(required=False)
|
||||
|
||||
|
|
|
|||
|
|
@ -369,16 +369,16 @@ class PendingList(PendingIndex):
|
|||
return None
|
||||
|
||||
to_add = {
|
||||
"youtube_id": video_data["id"],
|
||||
"title": video_data["title"],
|
||||
"vid_thumb_url": self._extract_thumb(video_data),
|
||||
"channel_id": video_data["channel_id"],
|
||||
"channel_indexed": video_data["channel_id"] in self.all_channels,
|
||||
"channel_name": video_data["channel"],
|
||||
"duration": get_duration_str(video_data.get("duration", 0)),
|
||||
"published": self._extract_published(video_data),
|
||||
"timestamp": int(datetime.now().timestamp()),
|
||||
"title": video_data["title"],
|
||||
"vid_thumb_url": self._extract_thumb(video_data),
|
||||
"vid_type": self._extract_vid_type(video_data),
|
||||
"channel_name": video_data["channel"],
|
||||
"channel_id": video_data["channel_id"],
|
||||
"channel_indexed": video_data["channel_id"] in self.all_channels,
|
||||
"youtube_id": video_data["id"],
|
||||
}
|
||||
serializer = DownloadItemSerializer(data=to_add)
|
||||
is_valid = serializer.is_valid()
|
||||
|
|
@ -406,6 +406,9 @@ class PendingList(PendingIndex):
|
|||
if timestamp and isinstance(timestamp, int):
|
||||
return timestamp
|
||||
|
||||
if timestamp and isinstance(timestamp, float):
|
||||
return int(timestamp)
|
||||
|
||||
upload_date = video_data.get("upload_date")
|
||||
if upload_date:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -324,9 +324,9 @@ class ValidatorCallback:
|
|||
"""check if all channel artwork is there"""
|
||||
for channel in self.source:
|
||||
urls = (
|
||||
channel["_source"]["channel_thumb_url"],
|
||||
channel["_source"]["channel_banner_url"],
|
||||
channel["_source"].get("channel_tvart_url", False),
|
||||
channel["_source"].get("channel_thumb_url"),
|
||||
channel["_source"].get("channel_banner_url"),
|
||||
channel["_source"].get("channel_tvart_url"),
|
||||
)
|
||||
handler = ThumbManager(channel["_source"]["channel_id"])
|
||||
handler.download_channel_art(urls, skip_existing=True)
|
||||
|
|
|
|||
|
|
@ -7,9 +7,11 @@ functionality:
|
|||
from datetime import datetime
|
||||
from http import cookiejar
|
||||
from io import StringIO
|
||||
from os import path
|
||||
|
||||
import yt_dlp
|
||||
from appsettings.src.config import AppConfig
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.ta_redis import RedisArchivist
|
||||
from django.conf import settings
|
||||
|
||||
|
|
@ -23,6 +25,9 @@ class YtWrap:
|
|||
"socket_timeout": 10,
|
||||
"extractor_retries": 3,
|
||||
"retries": 10,
|
||||
"cachedir": path.abspath(
|
||||
path.join(EnvironmentSettings.CACHE_DIR, "ytdlp")
|
||||
),
|
||||
}
|
||||
|
||||
def __init__(self, obs_request, config=False):
|
||||
|
|
@ -37,6 +42,7 @@ class YtWrap:
|
|||
if self.config:
|
||||
self._add_cookie()
|
||||
self._add_potoken()
|
||||
self._add_potoken_url()
|
||||
|
||||
if getattr(settings, "DEBUG", False):
|
||||
del self.obs["quiet"]
|
||||
|
|
@ -46,7 +52,10 @@ class YtWrap:
|
|||
"""add cookie if enabled"""
|
||||
if self.config["downloads"]["cookie_import"]:
|
||||
cookie_io = CookieHandler(self.config).get()
|
||||
self.obs["cookiefile"] = cookie_io
|
||||
else:
|
||||
cookie_io = CookieHandler(self.config).get("cookie_temp")
|
||||
|
||||
self.obs["cookiefile"] = cookie_io
|
||||
|
||||
def _add_potoken(self):
|
||||
"""add potoken if enabled"""
|
||||
|
|
@ -63,6 +72,21 @@ class YtWrap:
|
|||
}
|
||||
)
|
||||
|
||||
def _add_potoken_url(self):
|
||||
"""add bgutils token url"""
|
||||
if pot_provider_url := self.config["downloads"].get(
|
||||
"pot_provider_url"
|
||||
):
|
||||
self.obs.update(
|
||||
{
|
||||
"extractor_args": {
|
||||
"youtubepot-bgutilhttp": {
|
||||
"base_url": [pot_provider_url]
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
def download(self, url):
|
||||
"""make download request"""
|
||||
self.obs.update({"check_formats": "selected"})
|
||||
|
|
@ -111,26 +135,40 @@ class YtWrap:
|
|||
def _validate_cookie(self):
|
||||
"""check cookie and write it back for next use"""
|
||||
if not self.obs.get("cookiefile"):
|
||||
# empty in tests
|
||||
return
|
||||
|
||||
new_cookie = self.obs["cookiefile"].read()
|
||||
old_cookie = RedisArchivist().get_message_str("cookie")
|
||||
self.obs["cookiefile"].seek(0)
|
||||
new_cookie = self.obs["cookiefile"].read().strip("\x00")
|
||||
|
||||
if self.config["downloads"]["cookie_import"]:
|
||||
cookie_key = "cookie"
|
||||
expire = False
|
||||
else:
|
||||
cookie_key = "cookie_temp"
|
||||
expire = 60 * 30 # 30 min
|
||||
|
||||
old_cookie = RedisArchivist().get_message_str(cookie_key)
|
||||
if new_cookie and old_cookie != new_cookie:
|
||||
print("refreshed stored cookie")
|
||||
RedisArchivist().set_message("cookie", new_cookie, save=True)
|
||||
print(f"refreshed stored {cookie_key}")
|
||||
RedisArchivist().set_message(
|
||||
cookie_key, new_cookie, expire=expire, save=True
|
||||
)
|
||||
|
||||
|
||||
class CookieHandler:
|
||||
"""handle youtube cookie for yt-dlp"""
|
||||
|
||||
COOKIE_EMPTY = "# Netscape HTTP Cookie File\n"
|
||||
|
||||
def __init__(self, config):
|
||||
self.cookie_io = False
|
||||
self.config = config
|
||||
|
||||
def get(self):
|
||||
def get(self, message_str: str = "cookie"):
|
||||
"""get cookie io stream"""
|
||||
cookie = RedisArchivist().get_message_str("cookie")
|
||||
self.cookie_io = StringIO(cookie)
|
||||
cookie = RedisArchivist().get_message_str(message_str)
|
||||
self.cookie_io = StringIO(cookie or self.COOKIE_EMPTY)
|
||||
return self.cookie_io
|
||||
|
||||
def set_cookie(self, cookie):
|
||||
|
|
|
|||
|
|
@ -482,10 +482,7 @@ class DownloadPostProcess(DownloaderBase):
|
|||
|
||||
def embed_metadata(self):
|
||||
"""embed metadata in media file"""
|
||||
meta = self.config["downloads"].get("add_metadata")
|
||||
thumb = self.config["downloads"].get("add_thumbnail")
|
||||
|
||||
if not meta and not thumb:
|
||||
if not self.config["downloads"].get("add_metadata"):
|
||||
return
|
||||
|
||||
queue = RedisQueue(self.VIDEO_QUEUE)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,12 @@ from datetime import datetime, timezone
|
|||
from download.src.queue import PendingList
|
||||
|
||||
|
||||
def test_returns_scientific_timestamp_if_present():
|
||||
video_data = {"timestamp": 1.5135732e9}
|
||||
result = PendingList._extract_published(video_data)
|
||||
assert result == 1513573200
|
||||
|
||||
|
||||
def test_returns_timestamp_if_present():
|
||||
video_data = {"timestamp": 1508457600}
|
||||
result = PendingList._extract_published(video_data)
|
||||
|
|
|
|||
|
|
@ -81,10 +81,13 @@ class YoutubePlaylist(YouTubeItem):
|
|||
"playlist_channel": self.youtube_meta["channel"],
|
||||
"playlist_channel_id": self.youtube_meta["channel_id"],
|
||||
"playlist_thumbnail": playlist_thumbnail,
|
||||
"playlist_description": self.youtube_meta["description"] or None,
|
||||
"playlist_last_refresh": int(datetime.now().timestamp()),
|
||||
"playlist_type": "regular",
|
||||
}
|
||||
if self.youtube_meta.get("description"):
|
||||
self.json_data["playlist_description"] = self.youtube_meta[
|
||||
"description"
|
||||
]
|
||||
|
||||
def _ensure_channel(self):
|
||||
"""make sure channel is indexed"""
|
||||
|
|
|
|||
|
|
@ -1,14 +1,16 @@
|
|||
apprise==1.9.6
|
||||
celery==5.6.0
|
||||
django-auth-ldap==5.2.0
|
||||
apprise==1.9.7
|
||||
bgutil-ytdlp-pot-provider==1.2.2
|
||||
celery==5.6.2
|
||||
deepdiff==8.6.1
|
||||
django-auth-ldap==5.3.0
|
||||
django-celery-beat==2.8.1
|
||||
django-cors-headers==4.9.0
|
||||
Django==5.2.9
|
||||
Django==5.2.10
|
||||
djangorestframework==3.16.1
|
||||
drf-spectacular==0.28.0 # rc:ignore
|
||||
Pillow==12.0.0
|
||||
Pillow==12.1.0
|
||||
redis==7.1.0
|
||||
requests==2.32.5
|
||||
ryd-client==0.0.6
|
||||
uvicorn==0.38.0
|
||||
yt-dlp[default] @ git+https://github.com/yt-dlp/yt-dlp@468aa6a9b431194949ede9eaad8f33e314a288d6
|
||||
uvicorn==0.40.0
|
||||
yt-dlp[default] @ git+https://github.com/yt-dlp/yt-dlp@23b846506378a6a9c9a0958382d37f943f7cfa51
|
||||
|
|
|
|||
|
|
@ -39,7 +39,9 @@ class Migration(migrations.Migration):
|
|||
"is_superuser",
|
||||
models.BooleanField(
|
||||
default=False,
|
||||
help_text="Designates that this user has all permissions without explicitly assigning them.",
|
||||
help_text=(
|
||||
"Designates that this user has all permissions without explicitly assigning them."
|
||||
),
|
||||
verbose_name="superuser status",
|
||||
),
|
||||
),
|
||||
|
|
@ -49,7 +51,9 @@ class Migration(migrations.Migration):
|
|||
"groups",
|
||||
models.ManyToManyField(
|
||||
blank=True,
|
||||
help_text="The groups this user belongs to. A user will get all permissions granted to each of their groups.",
|
||||
help_text=(
|
||||
"The groups this user belongs to. A user will get all permissions granted to each of their groups."
|
||||
),
|
||||
related_name="user_set",
|
||||
related_query_name="user",
|
||||
to="auth.group",
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ class PlayerSerializer(serializers.Serializer):
|
|||
watched_date = serializers.IntegerField(required=False)
|
||||
duration = serializers.IntegerField()
|
||||
duration_str = serializers.CharField()
|
||||
|
||||
progress = serializers.FloatField(required=False)
|
||||
position = serializers.FloatField(required=False)
|
||||
|
||||
|
|
@ -44,49 +45,49 @@ class SponsorBlockSerializer(serializers.Serializer):
|
|||
class StatsSerializer(serializers.Serializer):
|
||||
"""serialize stats"""
|
||||
|
||||
like_count = serializers.IntegerField(required=False)
|
||||
average_rating = serializers.FloatField(required=False)
|
||||
view_count = serializers.IntegerField(required=False)
|
||||
dislike_count = serializers.IntegerField(required=False)
|
||||
like_count = serializers.IntegerField()
|
||||
average_rating = serializers.FloatField()
|
||||
view_count = serializers.IntegerField()
|
||||
dislike_count = serializers.IntegerField()
|
||||
|
||||
|
||||
class StreamItemSerializer(serializers.Serializer):
|
||||
"""serialize stream item"""
|
||||
|
||||
index = serializers.IntegerField()
|
||||
codec = serializers.CharField()
|
||||
bitrate = serializers.IntegerField()
|
||||
codec = serializers.CharField()
|
||||
height = serializers.IntegerField(required=False)
|
||||
index = serializers.IntegerField()
|
||||
type = serializers.ChoiceField(choices=["video", "audio"])
|
||||
width = serializers.IntegerField(required=False)
|
||||
height = serializers.IntegerField(required=False)
|
||||
|
||||
|
||||
class SubtitleFragmentSerializer(serializers.Serializer):
|
||||
"""serialize subtitle fragment"""
|
||||
|
||||
subtitle_index = serializers.IntegerField()
|
||||
subtitle_line = serializers.CharField()
|
||||
subtitle_start = serializers.CharField()
|
||||
subtitle_fragment_id = serializers.CharField()
|
||||
subtitle_end = serializers.CharField()
|
||||
youtube_id = serializers.CharField()
|
||||
title = serializers.CharField()
|
||||
subtitle_channel = serializers.CharField()
|
||||
subtitle_channel_id = serializers.CharField()
|
||||
subtitle_last_refresh = serializers.IntegerField()
|
||||
subtitle_end = serializers.CharField()
|
||||
subtitle_fragment_id = serializers.CharField()
|
||||
subtitle_index = serializers.IntegerField()
|
||||
subtitle_lang = serializers.CharField()
|
||||
subtitle_last_refresh = serializers.IntegerField()
|
||||
subtitle_line = serializers.CharField()
|
||||
subtitle_source = serializers.ChoiceField(choices=["user", "auto"])
|
||||
subtitle_start = serializers.CharField()
|
||||
title = serializers.CharField()
|
||||
youtube_id = serializers.CharField()
|
||||
|
||||
|
||||
class SubtitleItemSerializer(serializers.Serializer):
|
||||
"""serialize subtitle item"""
|
||||
|
||||
ext = serializers.ChoiceField(choices=["json3"])
|
||||
name = serializers.CharField()
|
||||
source = serializers.ChoiceField(choices=["user", "auto"])
|
||||
ext = serializers.ChoiceField(choices=["json3", "vtt"])
|
||||
lang = serializers.CharField()
|
||||
media_url = serializers.CharField()
|
||||
url = serializers.URLField()
|
||||
name = serializers.CharField()
|
||||
source = serializers.ChoiceField(choices=["user", "auto"])
|
||||
url = serializers.URLField(allow_null=True)
|
||||
|
||||
|
||||
class VideoSerializer(serializers.Serializer):
|
||||
|
|
@ -97,7 +98,7 @@ class VideoSerializer(serializers.Serializer):
|
|||
channel = ChannelSerializer(required=False)
|
||||
comment_count = serializers.IntegerField(allow_null=True, required=False)
|
||||
date_downloaded = serializers.IntegerField()
|
||||
description = serializers.CharField()
|
||||
description = serializers.CharField(allow_null=True)
|
||||
media_size = serializers.IntegerField()
|
||||
media_url = serializers.CharField()
|
||||
player = PlayerSerializer()
|
||||
|
|
@ -146,17 +147,18 @@ class VideoListQuerySerializer(serializers.Serializer):
|
|||
class CommentItemSerializer(serializers.Serializer):
|
||||
"""serialize comment item"""
|
||||
|
||||
comment_id = serializers.CharField()
|
||||
comment_text = serializers.CharField()
|
||||
comment_timestamp = serializers.IntegerField()
|
||||
comment_time_text = serializers.CharField()
|
||||
comment_likecount = serializers.IntegerField()
|
||||
comment_is_favorited = serializers.BooleanField()
|
||||
comment_author = serializers.CharField()
|
||||
comment_author_id = serializers.CharField()
|
||||
comment_author_thumbnail = serializers.URLField()
|
||||
comment_author_is_uploader = serializers.BooleanField()
|
||||
comment_author_thumbnail = serializers.URLField()
|
||||
comment_id = serializers.CharField()
|
||||
comment_is_favorited = serializers.BooleanField()
|
||||
comment_likecount = serializers.IntegerField()
|
||||
comment_parent = serializers.CharField()
|
||||
comment_text = serializers.CharField()
|
||||
comment_time_text = serializers.CharField()
|
||||
comment_timestamp = serializers.IntegerField()
|
||||
|
||||
comment_replies = serializers.SerializerMethodField()
|
||||
|
||||
@extend_schema_field(serializers.ListField())
|
||||
|
|
@ -170,10 +172,10 @@ class CommentItemSerializer(serializers.Serializer):
|
|||
class CommentsSerializer(serializers.Serializer):
|
||||
"""serialize comments as indexed"""
|
||||
|
||||
youtube_id = serializers.CharField()
|
||||
comment_last_refresh = serializers.IntegerField()
|
||||
comment_channel_id = serializers.CharField()
|
||||
comment_comments = CommentItemSerializer(many=True)
|
||||
comment_last_refresh = serializers.IntegerField()
|
||||
youtube_id = serializers.CharField()
|
||||
|
||||
|
||||
class PlaylistNavMetaSerializer(serializers.Serializer):
|
||||
|
|
|
|||
|
|
@ -39,10 +39,10 @@ class Comments:
|
|||
self.format_comments(comments_raw)
|
||||
|
||||
self.json_data = {
|
||||
"youtube_id": self.youtube_id,
|
||||
"comment_last_refresh": int(datetime.now().timestamp()),
|
||||
"comment_channel_id": channel_id,
|
||||
"comment_comments": self.comments_format,
|
||||
"comment_last_refresh": int(datetime.now().timestamp()),
|
||||
"youtube_id": self.youtube_id,
|
||||
}
|
||||
if upload:
|
||||
self.upload_comments()
|
||||
|
|
@ -124,20 +124,20 @@ class Comments:
|
|||
if not comment.get("author"):
|
||||
comment["author"] = comment.get("author_id", "Unknown")
|
||||
|
||||
is_uploader = comment.get("author_is_uploader", False)
|
||||
|
||||
cleaned_comment = {
|
||||
"comment_id": comment["id"],
|
||||
"comment_text": comment["text"].replace("\xa0", ""),
|
||||
"comment_timestamp": comment["timestamp"],
|
||||
"comment_time_text": time_text,
|
||||
"comment_likecount": comment.get("like_count", None),
|
||||
"comment_is_favorited": comment.get("is_favorited", False),
|
||||
"comment_author": comment["author"],
|
||||
"comment_author_id": comment["author_id"],
|
||||
"comment_author_is_uploader": is_uploader,
|
||||
"comment_author_thumbnail": comment["author_thumbnail"],
|
||||
"comment_author_is_uploader": comment.get(
|
||||
"author_is_uploader", False
|
||||
),
|
||||
"comment_id": comment["id"],
|
||||
"comment_is_favorited": comment.get("is_favorited", False),
|
||||
"comment_likecount": comment.get("like_count", None),
|
||||
"comment_parent": comment["parent"],
|
||||
"comment_text": comment["text"].replace("\xa0", ""),
|
||||
"comment_time_text": time_text,
|
||||
"comment_timestamp": comment["timestamp"],
|
||||
}
|
||||
|
||||
return cleaned_comment
|
||||
|
|
|
|||
|
|
@ -198,31 +198,33 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
|||
def process_youtube_meta(self):
|
||||
"""extract relevant fields from youtube"""
|
||||
self._validate_id()
|
||||
# extract
|
||||
self.channel_id = self.youtube_meta["channel_id"]
|
||||
last_refresh = int(datetime.now().timestamp())
|
||||
# build json_data basics
|
||||
self.json_data = {
|
||||
"title": self.youtube_meta["title"],
|
||||
"description": self.youtube_meta.get("description", ""),
|
||||
"category": self.youtube_meta.get("categories", []),
|
||||
"vid_thumb_url": self.youtube_meta["thumbnail"],
|
||||
"tags": self.youtube_meta.get("tags", []),
|
||||
"published": self._build_published(),
|
||||
"vid_last_refresh": last_refresh,
|
||||
"date_downloaded": last_refresh,
|
||||
"youtube_id": self.youtube_id,
|
||||
# Using .value to make json encodable
|
||||
"vid_type": self.video_type.value,
|
||||
"active": True,
|
||||
"category": self.youtube_meta.get("categories", []),
|
||||
"date_downloaded": last_refresh,
|
||||
"published": self._build_published(),
|
||||
"tags": self.youtube_meta.get("tags", []),
|
||||
"title": self.youtube_meta["title"],
|
||||
"vid_last_refresh": last_refresh,
|
||||
"vid_thumb_url": self.youtube_meta["thumbnail"],
|
||||
"vid_type": self.video_type.value,
|
||||
"youtube_id": self.youtube_id,
|
||||
}
|
||||
|
||||
def _build_published(self):
|
||||
if description := self.youtube_meta.get("description"):
|
||||
self.json_data["description"] = description
|
||||
|
||||
def _build_published(self) -> int | str:
|
||||
"""build published date or timestamp"""
|
||||
timestamp = self.youtube_meta.get("timestamp")
|
||||
if timestamp and isinstance(timestamp, int):
|
||||
return timestamp
|
||||
|
||||
if timestamp and isinstance(timestamp, float):
|
||||
return int(timestamp)
|
||||
|
||||
upload_date = self.youtube_meta["upload_date"]
|
||||
if not upload_date:
|
||||
raise ValueError(
|
||||
|
|
@ -255,10 +257,10 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
|||
def _add_stats(self):
|
||||
"""add stats dicst to json_data"""
|
||||
stats = {
|
||||
"view_count": self.youtube_meta.get("view_count", 0),
|
||||
"like_count": self.youtube_meta.get("like_count", 0),
|
||||
"dislike_count": self.youtube_meta.get("dislike_count", 0),
|
||||
"average_rating": self.youtube_meta.get("average_rating", 0),
|
||||
"view_count": self.youtube_meta.get("view_count") or 0,
|
||||
"like_count": self.youtube_meta.get("like_count") or 0,
|
||||
"dislike_count": self.youtube_meta.get("dislike_count") or 0,
|
||||
"average_rating": self.youtube_meta.get("average_rating") or 0,
|
||||
}
|
||||
self.json_data.update({"stats": stats})
|
||||
|
||||
|
|
@ -288,9 +290,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
|||
self.json_data.update(
|
||||
{
|
||||
"player": {
|
||||
"watched": False,
|
||||
"duration": duration,
|
||||
"duration_str": get_duration_str(duration),
|
||||
"watched": False,
|
||||
}
|
||||
}
|
||||
)
|
||||
|
|
@ -379,8 +381,8 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
|||
return
|
||||
|
||||
dislikes = {
|
||||
"dislike_count": result.get("dislikes", 0),
|
||||
"average_rating": result.get("rating", 0),
|
||||
"dislike_count": result.get("dislikes") or 0,
|
||||
"average_rating": result.get("rating") or 0,
|
||||
}
|
||||
self.json_data["stats"].update(dislikes)
|
||||
|
||||
|
|
@ -412,7 +414,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
|||
subtitle_media_url = f"{base_name}.{lang}.vtt"
|
||||
to_add = {
|
||||
"ext": "vtt",
|
||||
"url": False,
|
||||
"url": None,
|
||||
"name": lang,
|
||||
"lang": lang,
|
||||
"source": "file",
|
||||
|
|
@ -433,8 +435,6 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
|||
|
||||
if self.config["downloads"].get("add_metadata"):
|
||||
self._embed_text_data()
|
||||
|
||||
if self.config["downloads"].get("add_thumbnail"):
|
||||
self._embed_artwork()
|
||||
|
||||
def _embed_text_data(self):
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ class MediaStreamExtractor:
|
|||
self.media_path = media_path
|
||||
self.metadata = []
|
||||
|
||||
def extract_metadata(self):
|
||||
def extract_metadata(self) -> list[dict]:
|
||||
"""entry point to extract metadata"""
|
||||
|
||||
cmd = [
|
||||
|
|
@ -38,17 +38,15 @@ class MediaStreamExtractor:
|
|||
|
||||
return self.metadata
|
||||
|
||||
def process_stream(self, stream):
|
||||
def process_stream(self, stream) -> None:
|
||||
"""parse stream to metadata"""
|
||||
codec_type = stream.get("codec_type")
|
||||
if codec_type == "video":
|
||||
self._extract_video_metadata(stream)
|
||||
elif codec_type == "audio":
|
||||
self._extract_audio_metadata(stream)
|
||||
else:
|
||||
return
|
||||
|
||||
def _extract_video_metadata(self, stream):
|
||||
def _extract_video_metadata(self, stream) -> None:
|
||||
"""parse video metadata"""
|
||||
if "bit_rate" not in stream:
|
||||
# is probably thumbnail
|
||||
|
|
@ -56,26 +54,26 @@ class MediaStreamExtractor:
|
|||
|
||||
self.metadata.append(
|
||||
{
|
||||
"type": "video",
|
||||
"index": stream["index"],
|
||||
"bitrate": int(stream.get("bit_rate", 0)),
|
||||
"codec": stream["codec_name"],
|
||||
"width": stream["width"],
|
||||
"height": stream["height"],
|
||||
"bitrate": int(stream["bit_rate"]),
|
||||
"index": stream["index"],
|
||||
"type": "video",
|
||||
"width": stream["width"],
|
||||
}
|
||||
)
|
||||
|
||||
def _extract_audio_metadata(self, stream):
|
||||
def _extract_audio_metadata(self, stream) -> None:
|
||||
"""extract audio metadata"""
|
||||
self.metadata.append(
|
||||
{
|
||||
"type": "audio",
|
||||
"index": stream["index"],
|
||||
"codec": stream.get("codec_name", "undefined"),
|
||||
"bitrate": int(stream.get("bit_rate", 0)),
|
||||
"codec": stream.get("codec_name", "undefined"),
|
||||
"index": stream["index"],
|
||||
"type": "audio",
|
||||
}
|
||||
)
|
||||
|
||||
def get_file_size(self):
|
||||
def get_file_size(self) -> int:
|
||||
"""get filesize in bytes"""
|
||||
return stat(self.media_path).st_size
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ class MetadataEmbed:
|
|||
paginate = IndexPaginate(
|
||||
index_name=self.INDEX_NAME,
|
||||
data=data,
|
||||
size=200,
|
||||
size=100,
|
||||
callback=MetadataEmbedCallback,
|
||||
task=self.task,
|
||||
total=self._get_total(),
|
||||
|
|
|
|||
|
|
@ -149,7 +149,16 @@ class YoutubeSubtitle:
|
|||
query_str = parser.create_bulk_import(documents)
|
||||
self.index_subtitle(query_str)
|
||||
|
||||
indexed.append(subtitle)
|
||||
indexed.append(
|
||||
{
|
||||
"ext": "json3",
|
||||
"name": subtitle["name"],
|
||||
"source": subtitle["source"],
|
||||
"lang": subtitle["lang"],
|
||||
"media_url": subtitle["media_url"],
|
||||
"url": subtitle["url"],
|
||||
}
|
||||
)
|
||||
rand_sleep(self.video.config)
|
||||
|
||||
return indexed
|
||||
|
|
@ -336,13 +345,13 @@ class SubtitleParser:
|
|||
documents = self._chunk_list(video.youtube_id)
|
||||
channel = video.json_data.get("channel")
|
||||
meta_dict = {
|
||||
"youtube_id": video.youtube_id,
|
||||
"title": video.json_data.get("title"),
|
||||
"subtitle_channel": channel.get("channel_name"),
|
||||
"subtitle_channel_id": channel.get("channel_id"),
|
||||
"subtitle_last_refresh": int(datetime.now().timestamp()),
|
||||
"subtitle_lang": self.lang,
|
||||
"subtitle_last_refresh": int(datetime.now().timestamp()),
|
||||
"subtitle_source": source,
|
||||
"title": video.json_data.get("title"),
|
||||
"youtube_id": video.youtube_id,
|
||||
}
|
||||
|
||||
_ = [i.update(meta_dict) for i in documents]
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
22.12.0
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -12,26 +12,26 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"dompurify": "^3.3.1",
|
||||
"react": "^19.2.1",
|
||||
"react-dom": "^19.2.1",
|
||||
"react-router-dom": "^7.10.1",
|
||||
"react": "^19.2.3",
|
||||
"react-dom": "^19.2.3",
|
||||
"react-router-dom": "^7.11.0",
|
||||
"zustand": "^5.0.9"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^19.2.7",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@typescript-eslint/eslint-plugin": "^8.39.0",
|
||||
"@typescript-eslint/parser": "^8.39.0",
|
||||
"@typescript-eslint/eslint-plugin": "^8.51.0",
|
||||
"@typescript-eslint/parser": "^8.51.0",
|
||||
"@vitejs/plugin-react-swc": "^4.2.2",
|
||||
"eslint": "^9.39.1",
|
||||
"eslint": "^9.39.2",
|
||||
"eslint-config-prettier": "^10.1.8",
|
||||
"eslint-plugin-react-hooks": "^7.0.1",
|
||||
"eslint-plugin-react-refresh": "^0.4.24",
|
||||
"globals": "^16.5.0",
|
||||
"eslint-plugin-react-refresh": "^0.4.26",
|
||||
"globals": "^17.0.0",
|
||||
"prettier": "3.7.4",
|
||||
"typescript": "^5.9.3",
|
||||
"typescript-eslint": "^8.49.0",
|
||||
"vite": ">=7.2.7",
|
||||
"typescript-eslint": "^8.51.0",
|
||||
"vite": ">=7.3.0",
|
||||
"vite-plugin-checker": "^0.12.0"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,13 +16,13 @@ export type AppSettingsConfigType = {
|
|||
format: string | null;
|
||||
format_sort: string | null;
|
||||
add_metadata: boolean;
|
||||
add_thumbnail: boolean;
|
||||
subtitle: string | null;
|
||||
subtitle_source: string | null;
|
||||
subtitle_index: boolean;
|
||||
comment_max: string | null;
|
||||
comment_sort: string;
|
||||
cookie_import: boolean;
|
||||
pot_provider_url: string | null;
|
||||
potoken: boolean;
|
||||
throttledratelimit: number | null;
|
||||
extractor_lang: string | null;
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import Routes from '../configuration/routes/RouteList';
|
|||
import queueReindex, { ReindexType, ReindexTypeEnum } from '../api/actions/queueReindex';
|
||||
import formatDate from '../functions/formatDates';
|
||||
import PaginationDummy from '../components/PaginationDummy';
|
||||
import FormattedNumber from '../components/FormattedNumber';
|
||||
import Button from '../components/Button';
|
||||
import updateChannelOverwrites from '../api/actions/updateChannelOverwrite';
|
||||
import useIsAdmin from '../functions/useIsAdmin';
|
||||
|
|
@ -145,10 +144,6 @@ const ChannelAbout = () => {
|
|||
|
||||
<div className="info-box-item">
|
||||
<div>
|
||||
{channel.channel_views > 0 && (
|
||||
<FormattedNumber text="Channel views:" number={channel.channel_views} />
|
||||
)}
|
||||
|
||||
{isAdmin && (
|
||||
<>
|
||||
<div className="button-box">
|
||||
|
|
|
|||
|
|
@ -41,7 +41,6 @@ export type ChannelType = {
|
|||
channel_tags?: string[];
|
||||
channel_thumb_url: string;
|
||||
channel_tvart_url: string;
|
||||
channel_views: number;
|
||||
};
|
||||
|
||||
const Channels = () => {
|
||||
|
|
|
|||
|
|
@ -89,7 +89,6 @@ export type DownloadsType = {
|
|||
format: boolean;
|
||||
format_sort: boolean;
|
||||
add_metadata: boolean;
|
||||
add_thumbnail: boolean;
|
||||
subtitle: boolean;
|
||||
subtitle_source: boolean;
|
||||
subtitle_index: boolean;
|
||||
|
|
|
|||
|
|
@ -58,7 +58,6 @@ const SettingsApplication = () => {
|
|||
const [downloadsFormatSort, setDownloadsFormatSort] = useState<string | null>(null);
|
||||
const [downloadsExtractorLang, setDownloadsExtractorLang] = useState<string | null>(null);
|
||||
const [embedMetadata, setEmbedMetadata] = useState(false);
|
||||
const [embedThumbnail, setEmbedThumbnail] = useState(false);
|
||||
|
||||
// Subtitles
|
||||
const [subtitleLang, setSubtitleLang] = useState<string | null>(null);
|
||||
|
|
@ -74,6 +73,7 @@ const SettingsApplication = () => {
|
|||
const [showCookieForm, setShowCookieForm] = useState<boolean>(false);
|
||||
const [poTokenFormData, setPoTokenFormData] = useState<string>('web+');
|
||||
const [showPoTokenForm, setShowPoTokenForm] = useState<boolean>(false);
|
||||
const [potProviderUrl, setPotProviderUrl] = useState<string | null>(null);
|
||||
|
||||
// Integrations
|
||||
const [showApiToken, setShowApiToken] = useState(false);
|
||||
|
|
@ -116,7 +116,6 @@ const SettingsApplication = () => {
|
|||
setDownloadsFormatSort(appSettingsConfigData?.downloads.format_sort || null);
|
||||
setDownloadsExtractorLang(appSettingsConfigData?.downloads.extractor_lang || null);
|
||||
setEmbedMetadata(appSettingsConfigData?.downloads.add_metadata || false);
|
||||
setEmbedThumbnail(appSettingsConfigData?.downloads.add_thumbnail || false);
|
||||
|
||||
// Subtitles
|
||||
setSubtitleLang(appSettingsConfigData?.downloads.subtitle || null);
|
||||
|
|
@ -127,6 +126,9 @@ const SettingsApplication = () => {
|
|||
setCommentsMax(appSettingsConfigData?.downloads.comment_max || null);
|
||||
setCommentsSort(appSettingsConfigData?.downloads.comment_sort || '');
|
||||
|
||||
// Cookie
|
||||
setPotProviderUrl(appSettingsConfigData?.downloads.pot_provider_url || null);
|
||||
|
||||
// Integrations
|
||||
setDownloadDislikes(appSettingsConfigData?.downloads.integrate_ryd || false);
|
||||
setEnableSponsorBlock(appSettingsConfigData?.downloads.integrate_sponsorblock || false);
|
||||
|
|
@ -476,9 +478,8 @@ const SettingsApplication = () => {
|
|||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li>Embedding metadata adds additional metadata directly to the mp4 file.</li>
|
||||
<li>
|
||||
Embedding the thumbnail embeds the video thumbnail as a cover.jpg to the mp4
|
||||
Embedding metadata adds additional metadata and thumbnails directly to the mp4
|
||||
file.
|
||||
</li>
|
||||
</ul>
|
||||
|
|
@ -533,16 +534,6 @@ const SettingsApplication = () => {
|
|||
updateCallback={handleUpdateConfig}
|
||||
/>
|
||||
</div>
|
||||
<div className="settings-box-wrapper">
|
||||
<div>
|
||||
<p>Embed Thumbnail</p>
|
||||
</div>
|
||||
<ToggleConfig
|
||||
name="downloads.add_thumbnail"
|
||||
value={embedThumbnail}
|
||||
updateCallback={handleUpdateConfig}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="info-box-item">
|
||||
<h2 id="subtitles">Subtitles</h2>
|
||||
|
|
@ -714,6 +705,16 @@ const SettingsApplication = () => {
|
|||
PO guide
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
The PO Token Provider URL running external to tubearchivist. Make sure to
|
||||
review{' '}
|
||||
<a
|
||||
target="_blank"
|
||||
href="https://docs.tubearchivist.com/settings/application/#po-token-provider-url"
|
||||
>
|
||||
User Guide
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
|
|
@ -813,6 +814,19 @@ const SettingsApplication = () => {
|
|||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="settings-box-wrapper">
|
||||
<div>
|
||||
<p>PO Token Provider URL</p>
|
||||
</div>
|
||||
<InputConfig
|
||||
type="text"
|
||||
name="downloads.pot_provider_url"
|
||||
value={potProviderUrl}
|
||||
setValue={setPotProviderUrl}
|
||||
oldValue={appSettingsConfig.downloads.pot_provider_url}
|
||||
updateCallback={handleUpdateConfig}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="info-box-item">
|
||||
<h2 id="sntegrations">Integrations</h2>
|
||||
|
|
|
|||
|
|
@ -23,13 +23,13 @@ export const useAppSettingsStore = create<AppSettingsState>(set => ({
|
|||
format: null,
|
||||
format_sort: null,
|
||||
add_metadata: false,
|
||||
add_thumbnail: false,
|
||||
subtitle: null,
|
||||
subtitle_source: null,
|
||||
subtitle_index: false,
|
||||
comment_max: null,
|
||||
comment_sort: 'asc',
|
||||
cookie_import: false,
|
||||
pot_provider_url: null,
|
||||
potoken: false,
|
||||
throttledratelimit: null,
|
||||
extractor_lang: null,
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
-r backend/requirements.txt
|
||||
ipython==9.8.0
|
||||
pre-commit==4.5.0
|
||||
pylint-django==2.6.1
|
||||
pylint==3.3.9
|
||||
ipython==9.9.0
|
||||
pre-commit==4.5.1
|
||||
pylint-django==2.7.0
|
||||
pylint==4.0.4
|
||||
pytest-django==4.11.1
|
||||
pytest==9.0.2
|
||||
python-dotenv==1.2.1
|
||||
requirementscheck==0.1.0
|
||||
types-requests==2.32.4.20250913
|
||||
types-requests==2.32.4.20260107
|
||||
|
|
|
|||
Loading…
Reference in New Issue