From cadbdc791f5e9f917cf01702c24e25a6001cdc18 Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 26 Aug 2025 09:24:20 +0700 Subject: [PATCH 1/6] validate expected keys for adding to queue --- backend/download/src/queue.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/backend/download/src/queue.py b/backend/download/src/queue.py index bdadaa95..b70f178a 100644 --- a/backend/download/src/queue.py +++ b/backend/download/src/queue.py @@ -324,6 +324,19 @@ class PendingList(PendingIndex): ) return None + expected_keys = {"id", "title", "channel", "channel_id"} + if not set(video.youtube_meta.keys()).issuperset(expected_keys): + print(f"{url}: video metadata extraction incomplete, skipping") + if self.task: + self.task.send_progress( + message_lines=[ + "Video extraction failed.", + "Metadata extraction incomplete.", + ], + level="error", + ) + return None + video.youtube_meta["vid_type"] = vid_type to_add = self._parse_entry( youtube_id=url, From 2d19d85b75056d45f35f8ec628948d2d47b16b12 Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 30 Aug 2025 09:20:32 +0700 Subject: [PATCH 2/6] add debug var to template --- .github/ISSUE_TEMPLATE/BUG-REPORT.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/BUG-REPORT.yml b/.github/ISSUE_TEMPLATE/BUG-REPORT.yml index 0a1ee32d..bd1d0ec1 100644 --- a/.github/ISSUE_TEMPLATE/BUG-REPORT.yml +++ b/.github/ISSUE_TEMPLATE/BUG-REPORT.yml @@ -40,7 +40,7 @@ body: id: logs attributes: label: Relevant log output - description: Please copy and paste any relevant Docker logs. This will be automatically formatted into code, so no need for backticks. + description: Please copy and paste any relevant Docker logs. Make sure the logs are created with [DJANGO_DEBUG](https://docs.tubearchivist.com/installation/env-vars/#django_debug) enabled. This will be automatically formatted into code, so no need for backticks. render: shell validations: required: true From 4adfea6e5c4f85775d8111bb8bd1084a7e615e9d Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 3 Sep 2025 15:06:30 +0700 Subject: [PATCH 3/6] refac channel query building, better handle overwrites, #1047, #1048 --- backend/appsettings/src/reindex.py | 15 +- backend/channel/src/index.py | 7 +- backend/channel/src/remote_query.py | 182 +++++++++--------- .../tests/test_src/test_remote_query.py | 33 ++-- backend/download/src/queue.py | 2 +- backend/download/src/subscriptions.py | 8 +- backend/download/views.py | 1 + backend/video/src/constants.py | 5 + 8 files changed, 120 insertions(+), 133 deletions(-) diff --git a/backend/appsettings/src/reindex.py b/backend/appsettings/src/reindex.py index 20f82425..b8569c08 100644 --- a/backend/appsettings/src/reindex.py +++ b/backend/appsettings/src/reindex.py @@ -500,11 +500,14 @@ class ChannelFullScan: self.config = config self.to_update = False - def scan(self): + def scan(self) -> None: """match local with remote""" print(f"{self.channel_id}: start full scan") all_local_videos = self._get_all_local() - all_remote_videos = self._get_all_remote() + all_remote_videos = get_last_channel_videos( + self.channel_id, self.config, limit=None + ) + self.to_update = [] for video in all_local_videos: video_id = video["youtube_id"] @@ -526,14 +529,6 @@ class ChannelFullScan: self.update() - def _get_all_remote(self): - """get all channel videos""" - all_remote_videos = get_last_channel_videos( - self.channel_id, self.config, limit=False - ) - - return all_remote_videos - def _get_all_local(self): """get all local indexed channel_videos""" channel = YoutubeChannel(self.channel_id) diff --git a/backend/channel/src/index.py b/backend/channel/src/index.py index ab14b820..af240693 100644 --- a/backend/channel/src/index.py +++ b/backend/channel/src/index.py @@ -110,11 +110,6 @@ class YoutubeChannel(YouTubeItem): """get channel tabs""" tabs = VideoTypeEnum.values_known() config_cp = self.config.copy() - config_cp["subscriptions"] = { - "channel_size": 1, - "live_channel_size": 1, - "shorts_channel_size": 1, - } tabs = [] for query_filter in VideoTypeEnum: if query_filter == VideoTypeEnum.UNKNOWN: @@ -123,7 +118,7 @@ class YoutubeChannel(YouTubeItem): videos = get_last_channel_videos( channel_id=self.youtube_id, config=config_cp, - limit=True, + limit=1, query_filter=query_filter, ) if videos: diff --git a/backend/channel/src/remote_query.py b/backend/channel/src/remote_query.py index 52104fce..2a49a0b7 100644 --- a/backend/channel/src/remote_query.py +++ b/backend/channel/src/remote_query.py @@ -1,128 +1,130 @@ """build queries for video extraction from channel subscriptions""" +from appsettings.src.config import AppConfigType from download.src.yt_dlp_base import YtWrap from video.src.constants import VideoTypeEnum class VideoQueryBuilder: - """Build queries for yt-dlp.""" + """ + Build queries for yt-dlp. + limit: + - None: no limit + - bool: limit lookup from overwrite or config if True + - int: limit as int direct + """ - def __init__(self, config: dict, channel_overwrites: dict | None = None): + MAPPING = { + VideoTypeEnum.VIDEOS: { + "config_key": "channel_size", + "overwrite_key": "subscriptions_channel_size", + }, + VideoTypeEnum.SHORTS: { + "config_key": "shorts_channel_size", + "overwrite_key": "subscriptions_shorts_channel_size", + }, + VideoTypeEnum.STREAMS: { + "config_key": "live_channel_size", + "overwrite_key": "subscriptions_live_channel_size", + }, + } + + def __init__( + self, + config: AppConfigType, + channel_overwrites: dict | None = None, + limit: None | bool | int = True, + ): self.config = config self.channel_overwrites = channel_overwrites or {} + self.limit = limit def build_queries( self, - video_type: VideoTypeEnum | list[VideoTypeEnum] | None, - limit: bool = True, + vid_types: list[VideoTypeEnum] = VideoTypeEnum.known(), ) -> list[tuple[VideoTypeEnum, int | None]]: - """Build queries for all or specific video type.""" - query_methods = { - VideoTypeEnum.VIDEOS: self.videos_query, - VideoTypeEnum.STREAMS: self.streams_query, - VideoTypeEnum.SHORTS: self.shorts_query, - } + """build queries""" + queries: list[tuple[VideoTypeEnum, int | None]] = [] + for vid_type in vid_types: + if vid_type not in self.MAPPING: + continue - if video_type and video_type != VideoTypeEnum.UNKNOWN: - # build query for specific type/s - if not isinstance(video_type, list): - video_type = [video_type] - - queries = [] - for video_type_item in video_type: - query_method = query_methods.get(video_type_item) - if not query_method: - continue - - query = query_method(limit) - if query[1] != 0: - queries.append(query) - - return queries - - # Build and return queries for all video types - queries = [] - for build_query in query_methods.values(): - query = build_query(limit) - if query[1] != 0: + query = self.build_query_type(vid_type) + if query: queries.append(query) return queries - def videos_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]: - """Build query for videos.""" - return self._build_generic_query( - video_type=VideoTypeEnum.VIDEOS, - overwrite_key="subscriptions_channel_size", - config_key="channel_size", - limit=limit, - ) - - def streams_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]: - """Build query for streams.""" - return self._build_generic_query( - video_type=VideoTypeEnum.STREAMS, - overwrite_key="subscriptions_live_channel_size", - config_key="live_channel_size", - limit=limit, - ) - - def shorts_query(self, limit: bool) -> tuple[VideoTypeEnum, int | None]: - """Build query for shorts.""" - return self._build_generic_query( - video_type=VideoTypeEnum.SHORTS, - overwrite_key="subscriptions_shorts_channel_size", - config_key="shorts_channel_size", - limit=limit, - ) - - def _build_generic_query( + def build_query_type( self, - video_type: VideoTypeEnum, - overwrite_key: str, - config_key: str, - limit: bool, - ) -> tuple[VideoTypeEnum, int | None]: - """Generic query for video page scraping.""" - app_config_size = self.config["subscriptions"].get(config_key) - if not limit or app_config_size is None: - # treat None as unlimited - return (video_type, None) + vid_type: VideoTypeEnum, + ) -> tuple[VideoTypeEnum, int | None] | None: + """build query for vid_type""" + if self.limit is None: + return (vid_type, None) - if ( - overwrite_key in self.channel_overwrites - and self.channel_overwrites[overwrite_key] is not None - ): - overwrite = self.channel_overwrites[overwrite_key] - return (video_type, overwrite) + if isinstance(self.limit, bool): + if self.limit is False: + return (vid_type, None) - if app_config_size: - return (video_type, app_config_size) + overwrite_key = self.MAPPING[vid_type]["overwrite_key"] + overwrite = self.channel_overwrites.get(overwrite_key) + if overwrite == 0: + return None - return (video_type, 0) + if overwrite: + return (vid_type, overwrite) + + config_key = self.MAPPING[vid_type]["config_key"] + app_config = self.config["subscriptions"].get(config_key) + if app_config == 0: + return None + + if app_config: + return (vid_type, app_config) # type: ignore + + return (vid_type, None) + + if isinstance(self.limit, int): + return (vid_type, self.limit) + + return (vid_type, None) def get_last_channel_videos( - channel_id, - config, - limit=None, - query_filter=None, - channel_overwrites=None, -): + channel_id: str, + config: AppConfigType, + limit: None | bool | int = None, + query_filter: VideoTypeEnum | list[VideoTypeEnum] | None = None, +) -> list[dict]: """get a list of last videos from channel""" - query_handler = VideoQueryBuilder(config, channel_overwrites) - queries = query_handler.build_queries(query_filter) - last_videos = [] + + builder = VideoQueryBuilder(config, limit=limit) + + queries = [] + if query_filter is None: + queries = builder.build_queries() + elif isinstance(query_filter, list): + queries = builder.build_queries(vid_types=query_filter) + else: + query = builder.build_query_type(vid_type=query_filter) + if query: + queries.append(query) + + last_videos: list[dict] = [] + + if not query: + return last_videos for vid_type_enum, limit_amount in queries: - obs = { + obs: dict[str, bool | str] = { "skip_download": True, "extract_flat": True, } vid_type = vid_type_enum.value if limit is not None: - obs.update({"playlist_items": f":{limit_amount}:1"}) + obs["playlist_items"] = f":{limit_amount}:1" url = f"https://www.youtube.com/channel/{channel_id}/{vid_type}" channel_query, _ = YtWrap(obs, config).extract(url) diff --git a/backend/channel/tests/test_src/test_remote_query.py b/backend/channel/tests/test_src/test_remote_query.py index 470b50d9..a680b2fb 100644 --- a/backend/channel/tests/test_src/test_remote_query.py +++ b/backend/channel/tests/test_src/test_remote_query.py @@ -40,7 +40,7 @@ def overwrites(): def test_build_all_queries_with_limit(default_config, empty_overwrites): """default, empty overwrite""" builder = VideoQueryBuilder(default_config, empty_overwrites) - result = builder.build_queries(None, limit=True) + result = builder.build_queries() expected = [ (VideoTypeEnum.VIDEOS, 5), (VideoTypeEnum.STREAMS, 3), @@ -51,8 +51,8 @@ def test_build_all_queries_with_limit(default_config, empty_overwrites): def test_build_all_queries_without_limit(default_config, empty_overwrites): """limit disabled""" - builder = VideoQueryBuilder(default_config, empty_overwrites) - result = builder.build_queries(None, limit=False) + builder = VideoQueryBuilder(default_config, empty_overwrites, limit=False) + result = builder.build_queries() expected = [ (VideoTypeEnum.VIDEOS, None), (VideoTypeEnum.STREAMS, None), @@ -64,8 +64,8 @@ def test_build_all_queries_without_limit(default_config, empty_overwrites): def test_build_specific_query(default_config, empty_overwrites): """single vid_type""" builder = VideoQueryBuilder(default_config, empty_overwrites) - result = builder.build_queries(VideoTypeEnum.VIDEOS) - assert result == [(VideoTypeEnum.VIDEOS, 5)] + result = builder.build_query_type(VideoTypeEnum.VIDEOS) + assert result == (VideoTypeEnum.VIDEOS, 5) def test_build_multiple_queries(default_config, empty_overwrites): @@ -77,21 +77,10 @@ def test_build_multiple_queries(default_config, empty_overwrites): assert result == [(VideoTypeEnum.VIDEOS, 5), (VideoTypeEnum.SHORTS, 2)] -def test_build_unknown_queries(default_config, empty_overwrites): - """vid_type unknown""" - builder = VideoQueryBuilder(default_config, empty_overwrites) - result = builder.build_queries(VideoTypeEnum.UNKNOWN) - assert result == [ - (VideoTypeEnum.VIDEOS, 5), - (VideoTypeEnum.STREAMS, 3), - (VideoTypeEnum.SHORTS, 2), - ] - - def test_overwrite_applied(default_config, overwrites): """with overwrite from channel config""" builder = VideoQueryBuilder(default_config, overwrites) - result = builder.build_queries(None, limit=True) + result = builder.build_queries() expected = [ (VideoTypeEnum.VIDEOS, 10), # Overwritten # STREAMS is overwritten to 0, should be excluded @@ -102,16 +91,16 @@ def test_overwrite_applied(default_config, overwrites): def test_no_limit_ignores_config_and_overwrites(default_config, overwrites): """no limit single vid_type""" - builder = VideoQueryBuilder(default_config, overwrites) - result = builder.build_queries([VideoTypeEnum.STREAMS], limit=False) + builder = VideoQueryBuilder(default_config, overwrites, limit=False) + result = builder.build_queries([VideoTypeEnum.STREAMS]) assert result == [(VideoTypeEnum.STREAMS, None)] def test_zero_query_not_included(default_config): """overwrite to zero to disable""" overwrites = {"subscriptions_live_channel_size": 0} - builder = VideoQueryBuilder(default_config, overwrites) - result = builder.build_queries([VideoTypeEnum.STREAMS], limit=True) + builder = VideoQueryBuilder(default_config, overwrites, limit=True) + result = builder.build_queries([VideoTypeEnum.STREAMS]) assert not result # Should be skipped due to 0 @@ -124,5 +113,5 @@ def test_invalid_video_type_is_ignored(default_config): INVALID = "invalid" - result = builder.build_queries([FakeEnum.INVALID], limit=True) + result = builder.build_queries([FakeEnum.INVALID]) assert not result diff --git a/backend/download/src/queue.py b/backend/download/src/queue.py index b70f178a..b81e68e4 100644 --- a/backend/download/src/queue.py +++ b/backend/download/src/queue.py @@ -191,7 +191,7 @@ class PendingList(PendingIndex): return to_add - def _parse_channel(self, entry): + def _parse_channel(self, entry) -> None: """parse channel""" url = entry["url"] vid_type = entry["vid_type"] diff --git a/backend/download/src/subscriptions.py b/backend/download/src/subscriptions.py index d18f5399..1b194188 100644 --- a/backend/download/src/subscriptions.py +++ b/backend/download/src/subscriptions.py @@ -63,15 +63,15 @@ class ChannelSubscription: queries = VideoQueryBuilder( config=self.config, channel_overwrites=channel.get("channel_overwrites", {}), - ).build_queries(video_type=enums) + ).build_queries(vid_types=enums) - for query in queries: + for vid_type, limit in queries: all_channel_urls.append( ParsedURLType( type="channel", url=channel["channel_id"], - vid_type=query[0], - limit=query[1], + vid_type=vid_type, + limit=limit, ) ) diff --git a/backend/download/views.py b/backend/download/views.py index 7f34716e..7599f05c 100644 --- a/backend/download/views.py +++ b/backend/download/views.py @@ -124,6 +124,7 @@ class DownloadApiListView(ApiBaseView): pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"] url_str = " ".join(pending) + print(f"url_str: {url_str}") task = extrac_dl.delay( url_str, auto_start=auto_start, flat=flat, force=force ) diff --git a/backend/video/src/constants.py b/backend/video/src/constants.py index 382636d4..fbc25b5e 100644 --- a/backend/video/src/constants.py +++ b/backend/video/src/constants.py @@ -24,6 +24,11 @@ class VideoTypeEnum(enum.Enum): """values known""" return [i.value for i in cls if i.value != "unknown"] + @classmethod + def known(cls): + """known members""" + return [i for i in cls if i.value != "unknown"] + class SortEnum(enum.Enum): """all sort by options""" From 1e94715442f3d232d1a0a13fe5cba8ff9f3bf031 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 3 Sep 2025 15:20:13 +0700 Subject: [PATCH 4/6] disable inline play for similar videos, #1044 --- frontend/src/components/VideoList.tsx | 3 +++ frontend/src/components/VideoListItem.tsx | 27 +++++++++++++++-------- frontend/src/pages/Video.tsx | 1 + 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/frontend/src/components/VideoList.tsx b/frontend/src/components/VideoList.tsx index 28737a1c..6af5e6e6 100644 --- a/frontend/src/components/VideoList.tsx +++ b/frontend/src/components/VideoList.tsx @@ -9,6 +9,7 @@ type VideoListProps = { viewStyle: ViewStylesType; playlistId?: string; showReorderButton?: boolean; + allowInlinePlay?: boolean; refreshVideoList: (refresh: boolean) => void; }; @@ -17,6 +18,7 @@ const VideoList = ({ viewStyle, playlistId, showReorderButton = false, + allowInlinePlay = true, refreshVideoList, }: VideoListProps) => { if (!videoList) { @@ -40,6 +42,7 @@ const VideoList = ({ viewStyle={viewStyle} playlistId={playlistId} showReorderButton={showReorderButton} + allowInlinePlay={allowInlinePlay} refreshVideoList={refreshVideoList} /> ); diff --git a/frontend/src/components/VideoListItem.tsx b/frontend/src/components/VideoListItem.tsx index e034b732..d5e7be7f 100644 --- a/frontend/src/components/VideoListItem.tsx +++ b/frontend/src/components/VideoListItem.tsx @@ -1,4 +1,4 @@ -import { Link, useSearchParams } from 'react-router-dom'; +import { Link, useNavigate, useSearchParams } from 'react-router-dom'; import Routes from '../configuration/routes/RouteList'; import { VideoType } from '../pages/Home'; import iconPlay from '/img/icon-play.svg'; @@ -21,6 +21,7 @@ type VideoListItemProps = { viewStyle: ViewStylesType; playlistId?: string; showReorderButton?: boolean; + allowInlinePlay?: boolean; refreshVideoList: (refresh: boolean) => void; }; @@ -29,8 +30,10 @@ const VideoListItem = ({ viewStyle, playlistId, showReorderButton = false, + allowInlinePlay = true, refreshVideoList, }: VideoListItemProps) => { + const navigate = useNavigate(); const [, setSearchParams] = useSearchParams(); const [showReorderMenu, setShowReorderMenu] = useState(false); @@ -58,11 +61,15 @@ const VideoListItem = ({ )} { - setSearchParams(params => { - const newParams = new URLSearchParams(params); - newParams.set('videoId', video.youtube_id); - return newParams; - }); + if (allowInlinePlay) { + setSearchParams(params => { + const newParams = new URLSearchParams(params); + newParams.set('videoId', video.youtube_id); + return newParams; + }); + } else { + navigate(Routes.Video(video.youtube_id)); + } }} >
@@ -86,9 +93,11 @@ const VideoListItem = ({ >
)} -
- play-icon -
+ {allowInlinePlay && ( +
+ play-icon +
+ )}
diff --git a/frontend/src/pages/Video.tsx b/frontend/src/pages/Video.tsx index 93c211b0..abd1e8ef 100644 --- a/frontend/src/pages/Video.tsx +++ b/frontend/src/pages/Video.tsx @@ -595,6 +595,7 @@ const Video = () => { videoList={similarVideosResponseData} viewStyle={ViewStylesEnum.Grid as ViewStylesType} refreshVideoList={setRefreshVideoList} + allowInlinePlay={false} />
From 94745c7f8444183c51e17164851c5c45fe6e6c45 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 3 Sep 2025 15:42:42 +0700 Subject: [PATCH 5/6] bump dependencies --- backend/requirements.txt | 2 +- requirements-dev.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index 31a3a214..af5d2cab 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -12,4 +12,4 @@ requests==2.32.5 ryd-client==0.0.6 uvicorn==0.35.0 whitenoise==6.9.0 -yt-dlp[default]==2025.8.22 +yt-dlp[default]==2025.8.27 diff --git a/requirements-dev.txt b/requirements-dev.txt index 0983c774..f2b7786f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ -r backend/requirements.txt -ipython==9.4.0 +ipython==9.5.0 pre-commit==4.3.0 pylint-django==2.6.1 pylint==3.3.8 From 7bae974aa69ddf3d64d2802bacfa06621ec27bd3 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 3 Sep 2025 16:11:28 +0700 Subject: [PATCH 6/6] ignore playlist error in post processing and channel indexing, #1032 --- backend/channel/src/index.py | 16 ++++++++++++---- backend/download/src/yt_dlp_handler.py | 15 +++++++++++++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/backend/channel/src/index.py b/backend/channel/src/index.py index af240693..37f2e625 100644 --- a/backend/channel/src/index.py +++ b/backend/channel/src/index.py @@ -241,13 +241,21 @@ class YoutubeChannel(YouTubeItem): ] self.task.send_progress(message, progress=(idx + 1) / total) - @staticmethod - def _index_single_playlist(playlist): + def _index_single_playlist(self, playlist): """add single playlist if needed""" from playlist.src.index import YoutubePlaylist - playlist = YoutubePlaylist(playlist[0]) - playlist.update_playlist(skip_on_empty=True) + try: + playlist = YoutubePlaylist(playlist[0]) + playlist.update_playlist(skip_on_empty=True) + except ValueError as err: + message = [ + f"{self.youtube_id}: skip failed playlist import", + str(err), + ] + print(message) + if self.task: + self.task.send_progress(message) def get_channel_videos(self): """get all videos from channel""" diff --git a/backend/download/src/yt_dlp_handler.py b/backend/download/src/yt_dlp_handler.py index 096207b1..2221390d 100644 --- a/backend/download/src/yt_dlp_handler.py +++ b/backend/download/src/yt_dlp_handler.py @@ -386,8 +386,19 @@ class DownloadPostProcess(DownloaderBase): if not playlist_id or not idx or not total: break - playlist = YoutubePlaylist(playlist_id) - playlist.update_playlist(skip_on_empty=True) + try: + playlist = YoutubePlaylist(playlist_id) + playlist.update_playlist(skip_on_empty=True) + except ValueError as err: + message = [ + f"{playlist_id}: skip failed playlist import", + str(err), + ] + print(message) + if self.task: + self.task.send_progress(message) + + continue if not self.task: continue