From e62a4e0fcf83b5f39308432669fdc6c0e74ed783 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 8 Jun 2025 10:04:45 +0700 Subject: [PATCH 01/55] fix none existing timestamp key in info json --- backend/video/src/index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/video/src/index.py b/backend/video/src/index.py index 01b8435b..6f324eb3 100644 --- a/backend/video/src/index.py +++ b/backend/video/src/index.py @@ -195,7 +195,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): def _build_published(self): """build published date or timestamp""" - timestamp = self.youtube_meta["timestamp"] + timestamp = self.youtube_meta.get("timestamp") if timestamp: return timestamp From f8c5efb87abdedf422a3c466e955d108b3fe4db3 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 8 Jun 2025 11:34:05 +0700 Subject: [PATCH 02/55] fix type hints --- backend/config/settings.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/config/settings.py b/backend/config/settings.py index b2d9318f..93755321 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -242,7 +242,9 @@ if bool(environ.get("TA_ENABLE_AUTH_PROXY")): # Configure Authentication Backend Combinations _login_auth_mode = (environ.get("TA_LOGIN_AUTH_MODE") or "single").casefold() if _login_auth_mode == "local": - AUTHENTICATION_BACKENDS = ("django.contrib.auth.backends.ModelBackend",) + AUTHENTICATION_BACKENDS: tuple = ( + "django.contrib.auth.backends.ModelBackend", + ) elif _login_auth_mode == "ldap": AUTHENTICATION_BACKENDS = ("django_auth_ldap.backend.LDAPBackend",) elif _login_auth_mode == "forwardauth": From ffe9444295a32d9ca4fe0cad1532add9cbde79e8 Mon Sep 17 00:00:00 2001 From: Baku <83136747+IAmABakuAMA@users.noreply.github.com> Date: Sun, 8 Jun 2025 15:13:59 +1000 Subject: [PATCH 03/55] Update CONTRIBUTING.md (#987) * Update CONTRIBUTING.md Reworded the Beta Testing section for clarity and flow. Also fixed typos. * Update CONTRIBUTING.md Fixed additional typo --- CONTRIBUTING.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c2f62d79..2a4425a9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,17 +16,17 @@ Welcome, and thanks for showing interest in improving Tube Archivist! --- ## Beta Testing -Be the first to help test new features and improvements and provide feedback! There are regular `:unstable` builds for easy access. That's for the tinkerers and the breave. Ideally use a testing environment first, before a release be the first to install it on your main system. +Be the first to help test new features/improvements and provide feedback! Regular `:unstable` builds are available for early access. These are for the tinkerers and the brave. Ideally, use a testing environment first, before upgrading your main installation. -There is always something that can get missed during development. Look at the commit messages tagged with `#build`, these are the unstable builds and give a quick overview what has changed. +There is always something that can get missed during development. Look at the commit messages tagged with `#build` - these are the unstable builds and give a quick overview of what has changed. - Test the features mentioned, play around, try to break it. -- Test the update path by installing the `:latest` release first, the upgrade to `:unstable` to check for any errors. +- Test the update path by installing the `:latest` release first, then upgrade to `:unstable` to check for any errors. - Test the unstable build on a fresh install. -Then provide feedback, if there is a problem but also if there is no problem. Reach out on [Discord](https://tubearchivist.com/discord) in the `#beta-testing` channel with your findings. +Then provide feedback - even if you don't encounter any issues! You can do this in the `#beta-testing` channel on the [Discord](https://tubearchivist.com/discord) Discord server. -This will help with a smooth update for the regular release. Plus you get to test things out early! +This helps ensure a smooth update for the stable release. Plus you get to test things out early! ## How to open an issue Please read this carefully before opening any [issue](https://github.com/tubearchivist/tubearchivist/issues) on GitHub. From 2c2129179aef78e5a411faf1251df91412d034ff Mon Sep 17 00:00:00 2001 From: James Kerrane Date: Sat, 7 Jun 2025 23:14:28 -0600 Subject: [PATCH 04/55] Remove obsolete "version" attribute (#988) The attribute `version` is obsolete, and docker compose recommends removing it to avoid potential confusion, so this removes the attribute. --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a906ce7b..5a5bc2f3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.5' - services: tubearchivist: container_name: tubearchivist From 22b53e682060aa47e87a192d9b5af2eafda0da52 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 8 Jun 2025 12:16:48 +0700 Subject: [PATCH 05/55] remove unstable tag --- backend/config/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/config/settings.py b/backend/config/settings.py index 93755321..b92e8ae5 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -316,7 +316,7 @@ CORS_ALLOW_HEADERS = list(default_headers) + [ # TA application settings TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist" -TA_VERSION = "v0.5.3-unstable" +TA_VERSION = "v0.5.3" # API REST_FRAMEWORK = { From 178c25f2f06723cad15cc3666f6530e61f177eb7 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 8 Jun 2025 19:52:55 +0700 Subject: [PATCH 06/55] really no more feature requests please --- .github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml b/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml index fbdc7998..f84b1ab8 100644 --- a/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml +++ b/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml @@ -10,3 +10,5 @@ body: options: - label: I understand that this issue will be closed without comment. required: true + - label: I will resist the temptation and I will not submit this issue. If I submit this, I understand I might get blocked from this repo. + required: true From aa943567dede7398c054253f9e2c3057a2f8eeec Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 10 Jun 2025 08:37:48 +0700 Subject: [PATCH 07/55] bump bump yt-dlp --- backend/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index fa21380e..97f24a6c 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -12,4 +12,4 @@ requests==2.32.3 ryd-client==0.0.6 uvicorn==0.34.3 whitenoise==6.9.0 -yt-dlp[default]==2025.5.22 +yt-dlp[default]==2025.6.9 From dfcd46efbf80f28d29d23167a536a8c3604c0ca0 Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 10 Jun 2025 08:38:58 +0700 Subject: [PATCH 08/55] add unstable tag --- backend/config/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/config/settings.py b/backend/config/settings.py index b92e8ae5..938cc225 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -316,7 +316,7 @@ CORS_ALLOW_HEADERS = list(default_headers) + [ # TA application settings TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist" -TA_VERSION = "v0.5.3" +TA_VERSION = "v0.5.4-unstable" # API REST_FRAMEWORK = { From 3631fa0b2c002b28c4c0b1fccc8e78c8eeb925f2 Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 10 Jun 2025 08:47:57 +0700 Subject: [PATCH 09/55] bump yt-dlp, #build --- backend/config/settings.py | 2 +- backend/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/config/settings.py b/backend/config/settings.py index b92e8ae5..938cc225 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -316,7 +316,7 @@ CORS_ALLOW_HEADERS = list(default_headers) + [ # TA application settings TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist" -TA_VERSION = "v0.5.3" +TA_VERSION = "v0.5.4-unstable" # API REST_FRAMEWORK = { diff --git a/backend/requirements.txt b/backend/requirements.txt index fa21380e..97f24a6c 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -12,4 +12,4 @@ requests==2.32.3 ryd-client==0.0.6 uvicorn==0.34.3 whitenoise==6.9.0 -yt-dlp[default]==2025.5.22 +yt-dlp[default]==2025.6.9 From 12fc7e1663aae6952063ca69d99b145029596820 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 11 Jun 2025 08:34:22 +0700 Subject: [PATCH 10/55] bump requirements --- backend/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index 97f24a6c..9751c2cb 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -8,7 +8,7 @@ djangorestframework==3.16.0 drf-spectacular==0.28.0 Pillow==11.2.1 redis==6.2.0 -requests==2.32.3 +requests==2.32.4 ryd-client==0.0.6 uvicorn==0.34.3 whitenoise==6.9.0 From 21f0a09f5f9f94681d1a0a2c25c271f652a92161 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 11 Jun 2025 08:34:56 +0700 Subject: [PATCH 11/55] bump TA_VERSION --- backend/config/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/config/settings.py b/backend/config/settings.py index 938cc225..39d2186e 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -316,7 +316,7 @@ CORS_ALLOW_HEADERS = list(default_headers) + [ # TA application settings TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist" -TA_VERSION = "v0.5.4-unstable" +TA_VERSION = "v0.5.4" # API REST_FRAMEWORK = { From ff94c324b3db6ce8afb15c4038d19bffe6a28fed Mon Sep 17 00:00:00 2001 From: MerlinScheurer Date: Fri, 13 Jun 2025 20:10:57 +0200 Subject: [PATCH 12/55] Refac extract loading indicator into its own component --- frontend/src/components/InputConfig.tsx | 9 ++------- frontend/src/components/LoadingIndicator.tsx | 9 +++++++++ frontend/src/components/WatchedCheckBox.tsx | 5 ++--- frontend/src/pages/Login.tsx | 6 ++---- 4 files changed, 15 insertions(+), 14 deletions(-) create mode 100644 frontend/src/components/LoadingIndicator.tsx diff --git a/frontend/src/components/InputConfig.tsx b/frontend/src/components/InputConfig.tsx index a378a5fb..cf1bfa13 100644 --- a/frontend/src/components/InputConfig.tsx +++ b/frontend/src/components/InputConfig.tsx @@ -1,4 +1,5 @@ import { useState } from 'react'; +import LoadingIndicator from './LoadingIndicator'; type InputTextProps = { type: 'text' | 'number'; @@ -51,13 +52,7 @@ const InputConfig = ({ type, name, value, setValue, oldValue, updateCallback }: )} {oldValue !== null && } - {loading && ( - <> -
-
-
- - )} + {loading && } {success && ✅}
diff --git a/frontend/src/components/LoadingIndicator.tsx b/frontend/src/components/LoadingIndicator.tsx new file mode 100644 index 00000000..0eaeaeb4 --- /dev/null +++ b/frontend/src/components/LoadingIndicator.tsx @@ -0,0 +1,9 @@ +const LoadingIndicator = () => { + return ( +
+
+
+ ); +}; + +export default LoadingIndicator; diff --git a/frontend/src/components/WatchedCheckBox.tsx b/frontend/src/components/WatchedCheckBox.tsx index 490bb3c3..0585ec1a 100644 --- a/frontend/src/components/WatchedCheckBox.tsx +++ b/frontend/src/components/WatchedCheckBox.tsx @@ -1,6 +1,7 @@ import iconUnseen from '/img/icon-unseen.svg'; import iconSeen from '/img/icon-seen.svg'; import { useEffect, useState } from 'react'; +import LoadingIndicator from './LoadingIndicator'; type WatchedCheckBoxProps = { watched: boolean; @@ -32,9 +33,7 @@ const WatchedCheckBox = ({ watched, onClick, onDone }: WatchedCheckBoxProps) => <> {loading && ( <> -
-
-
+ )} {!loading && watched && ( diff --git a/frontend/src/pages/Login.tsx b/frontend/src/pages/Login.tsx index da8a0385..d07c392e 100644 --- a/frontend/src/pages/Login.tsx +++ b/frontend/src/pages/Login.tsx @@ -5,6 +5,7 @@ import Colours from '../configuration/colours/Colours'; import Button from '../components/Button'; import signIn from '../api/actions/signIn'; import loadAuth from '../api/loader/loadAuth'; +import LoadingIndicator from '../components/LoadingIndicator'; const Login = () => { const navigate = useNavigate(); @@ -134,10 +135,7 @@ const Login = () => { {waitingForBackend && ( <>

- Waiting for backend{' '} -

-
-
+ Waiting for backend

)} From bb4e5ecb503e22ae1633806899c4c4d93c5d550f Mon Sep 17 00:00:00 2001 From: Craig Alexander Date: Sun, 15 Jun 2025 05:19:14 -0400 Subject: [PATCH 13/55] Fix not found message showing as API is loading (#995) --- .gitignore | 2 + frontend/src/pages/ChannelVideo.tsx | 173 +++++++++++++--------------- 2 files changed, 85 insertions(+), 90 deletions(-) diff --git a/.gitignore b/.gitignore index 812a03ba..474bdc25 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ backend/.env # JavaScript stuff node_modules + +.editorconfig diff --git a/frontend/src/pages/ChannelVideo.tsx b/frontend/src/pages/ChannelVideo.tsx index e5a123c5..981c9aae 100644 --- a/frontend/src/pages/ChannelVideo.tsx +++ b/frontend/src/pages/ChannelVideo.tsx @@ -97,104 +97,97 @@ const ChannelVideo = ({ videoType }: ChannelVideoProps) => { videoId, ]); - if (!channel) { - return ( -
-
-

Channel {channelId} not found!

-
- ); - } - return ( - <> - {`TA | Channel: ${channel.channel_name}`} - -
-
- -
- {videoAggs && ( - <> -

- {videoAggs.total_items.value} videos |{' '} - {videoAggs.total_duration.value_str} playback{' '} - | Total size{' '} - {humanFileSize(videoAggs.total_size.value, useSiUnits)} -

-
-
- - )} + setRefresh(true); + }} + /> +
+ + )} +
-
-
- -
- - - -
-
- {!hasVideos && ( - <> -

No videos found...

-

- Try going to the downloads page to start the scan - and download tasks. -

- - )} - - +
+
-
- {pagination && ( -
- + + + +
+
+ {!hasVideos && ( + <> +

No videos found...

+

+ Try going to the downloads page to start the + scan and download tasks. +

+ + )} + + +
- )} - + {pagination && ( +
+ +
+ )} + + ) ); }; From 990cb9aaecdc4c59628f09f1ddf5a347929f75ee Mon Sep 17 00:00:00 2001 From: Craig Alexander Date: Fri, 20 Jun 2025 12:33:43 -0400 Subject: [PATCH 14/55] Move npm install into its own docker stage (#999) --- Dockerfile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index b8260679..25eec9ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,18 @@ # multi stage to build tube archivist # build python wheel, download and extract ffmpeg, copy into final image +FROM node:lts-alpine AS npm-builder +COPY frontend/package.json frontend/package-lock.json / +RUN npm i + FROM node:lts-alpine AS node-builder # RUN npm config set registry https://registry.npmjs.org/ +COPY --from=npm-builder ./node_modules /frontend/node_modules COPY ./frontend /frontend - WORKDIR /frontend -RUN npm i + RUN npm run build:deploy WORKDIR / @@ -54,9 +58,9 @@ RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recomm # install debug tools for testing environment RUN if [ "$INSTALL_DEBUG" ] ; then \ - apt-get -y update && apt-get -y install --no-install-recommends \ - vim htop bmon net-tools iputils-ping procps lsof \ - && pip install --user ipython pytest pytest-django \ + apt-get -y update && apt-get -y install --no-install-recommends \ + vim htop bmon net-tools iputils-ping procps lsof \ + && pip install --user ipython pytest pytest-django \ ; fi # make folders From b14309daebf3a3c69f4de615b069ff090185de8f Mon Sep 17 00:00:00 2001 From: MerlinScheurer Date: Sun, 22 Jun 2025 11:41:00 +0200 Subject: [PATCH 15/55] Fix do not send referrer when opening youtube, sponsorblock or returndislite links --- frontend/src/components/DownloadListItem.tsx | 6 ++++- frontend/src/components/VideoPlayer.tsx | 24 ++++++++++++++++---- frontend/src/pages/ChannelAbout.tsx | 8 +++++-- frontend/src/pages/Playlist.tsx | 1 + frontend/src/pages/SettingsApplication.tsx | 20 ++++++++++++---- frontend/src/pages/Video.tsx | 6 ++++- 6 files changed, 53 insertions(+), 12 deletions(-) diff --git a/frontend/src/components/DownloadListItem.tsx b/frontend/src/components/DownloadListItem.tsx index bb0a82e3..ff0273dd 100644 --- a/frontend/src/components/DownloadListItem.tsx +++ b/frontend/src/components/DownloadListItem.tsx @@ -47,7 +47,11 @@ const DownloadListItem = ({ download, setRefresh }: DownloadListItemProps) => { {!download.channel_indexed && {download.channel_name}} - +

{download.title}
diff --git a/frontend/src/components/VideoPlayer.tsx b/frontend/src/components/VideoPlayer.tsx index 7684e24a..001c5e7c 100644 --- a/frontend/src/components/VideoPlayer.tsx +++ b/frontend/src/components/VideoPlayer.tsx @@ -467,11 +467,19 @@ const VideoPlayer = ({

This video doesn't have any sponsor segments added. To add a segment go to{' '} - this video on YouTube + + this video on YouTube + {' '} and add a segment using the{' '} - SponsorBlock + + SponsorBlock + {' '} extension.

@@ -480,11 +488,19 @@ const VideoPlayer = ({

This video has unlocked sponsor segments. Go to{' '} - this video on YouTube + + this video on YouTube + {' '} and vote on the segments using the{' '} - SponsorBlock + + SponsorBlock + {' '} extension.

diff --git a/frontend/src/pages/ChannelAbout.tsx b/frontend/src/pages/ChannelAbout.tsx index 1ad1cf46..54898b92 100644 --- a/frontend/src/pages/ChannelAbout.tsx +++ b/frontend/src/pages/ChannelAbout.tsx @@ -130,7 +130,11 @@ const ChannelAbout = () => { {channel.channel_active && (

Youtube:{' '} - + Active

@@ -316,7 +320,7 @@ const ChannelAbout = () => {

Overwrite{' '} - + SponsorBlock

diff --git a/frontend/src/pages/Playlist.tsx b/frontend/src/pages/Playlist.tsx index 37657f64..f8affa20 100644 --- a/frontend/src/pages/Playlist.tsx +++ b/frontend/src/pages/Playlist.tsx @@ -180,6 +180,7 @@ const Playlist = () => { Active diff --git a/frontend/src/pages/SettingsApplication.tsx b/frontend/src/pages/SettingsApplication.tsx index f8e1a39b..594ef29c 100644 --- a/frontend/src/pages/SettingsApplication.tsx +++ b/frontend/src/pages/SettingsApplication.tsx @@ -779,7 +779,11 @@ const SettingsApplication = () => {
  • Make sure to contribute to this excellent project.
  • More details{' '} - + here . @@ -794,7 +798,11 @@ const SettingsApplication = () => {
  • Make sure to contribute to this excellent project.
  • More details{' '} - + here . @@ -831,7 +839,11 @@ const SettingsApplication = () => {

    Enable{' '} - + returnyoutubedislike

    @@ -846,7 +858,7 @@ const SettingsApplication = () => {

    Enable{' '} - + Sponsorblock

    diff --git a/frontend/src/pages/Video.tsx b/frontend/src/pages/Video.tsx index 785a6e23..68929f38 100644 --- a/frontend/src/pages/Video.tsx +++ b/frontend/src/pages/Video.tsx @@ -279,7 +279,11 @@ const Video = () => { {video.active && (

    Youtube:{' '} - + Active

    From d19190bf6a0228a7ed18e6ca692b8541be69aee9 Mon Sep 17 00:00:00 2001 From: MerlinScheurer Date: Sun, 22 Jun 2025 12:10:29 +0200 Subject: [PATCH 16/55] Add theater mode to normal video player --- frontend/src/components/VideoPlayer.tsx | 57 ++++++++++++++++++++++++- frontend/src/style.css | 30 +++++++++++++ 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/VideoPlayer.tsx b/frontend/src/components/VideoPlayer.tsx index 001c5e7c..47a19977 100644 --- a/frontend/src/components/VideoPlayer.tsx +++ b/frontend/src/components/VideoPlayer.tsx @@ -126,6 +126,7 @@ const VideoPlayer = ({ const [searchParams] = useSearchParams(); const searchParamVideoProgress = searchParams.get('t'); + const theaterModeFromStorage = localStorage.getItem('theaterMode') === 'true'; const volumeFromStorage = Number(localStorage.getItem('playerVolume') ?? 1); const playBackSpeedFromStorage = Number(localStorage.getItem('playerSpeed') || 1); const playBackSpeedIndex = @@ -140,6 +141,8 @@ const VideoPlayer = ({ const [showHelpDialog, setShowHelpDialog] = useState(false); const [showInfoDialog, setShowInfoDialog] = useState(false); const [infoDialogContent, setInfoDialogContent] = useState(''); + const [isTheaterMode, setIsTheaterMode] = useState(theaterModeFromStorage); + const [theaterModeKeyPressed, setTheaterModeKeyPressed] = useState(false); const questionmarkPressed = useKeyPress('?'); const mutePressed = useKeyPress('m'); @@ -151,6 +154,8 @@ const VideoPlayer = ({ const arrowRightPressed = useKeyPress('ArrowRight'); const arrowLeftPressed = useKeyPress('ArrowLeft'); const pPausedPressed = useKeyPress('p'); + const theaterModePressed = useKeyPress('t'); + const escapePressed = useKeyPress('Escape'); const videoId = video.youtube_id; const videoUrl = video.media_url; @@ -345,10 +350,46 @@ const VideoPlayer = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [questionmarkPressed]); + useEffect(() => { + if (embed) { + return; + } + + if (theaterModePressed && !theaterModeKeyPressed) { + setTheaterModeKeyPressed(true); + + const newTheaterMode = !isTheaterMode; + setIsTheaterMode(newTheaterMode); + + localStorage.setItem('theaterMode', newTheaterMode.toString()); + + infoDialog(newTheaterMode ? 'Theater mode' : 'Normal mode'); + } else if (!theaterModePressed) { + setTheaterModeKeyPressed(false); + } + }, [theaterModePressed, isTheaterMode, theaterModeKeyPressed]); + + useEffect(() => { + if (embed) { + return; + } + + if (escapePressed && isTheaterMode) { + setIsTheaterMode(false); + + localStorage.setItem('theaterMode', 'false'); + + infoDialog('Normal mode'); + } + }, [escapePressed, isTheaterMode]); + return ( <> -
    -
    +
    +
    + {channelAggsList && channelAggsList.length > 1 && ( { {channel_filter_name} )} + {vidTypeFilterFromUrl && ( + <> + {' - by type '} + {vidTypeFilterFromUrl} + + )} + {showQueueActions && ( +
    +

    Bulk actions

    +

    + Applied filtered by status '{showIgnoredFilter}' + {channelFilterFromUrl && ( + + {' '} + and by channel: '{channel_filter_name}' + + )} + {vidTypeFilterFromUrl && ( + + {' '} + and by type: '{vidTypeFilterFromUrl}' + + )} +

    +
    + {showDeleteConfirm ? ( + <> + + + + ) : ( + + )} +
    +
    + )}
    diff --git a/frontend/src/pages/SettingsActions.tsx b/frontend/src/pages/SettingsActions.tsx index 577c0262..8d7b44cf 100644 --- a/frontend/src/pages/SettingsActions.tsx +++ b/frontend/src/pages/SettingsActions.tsx @@ -1,7 +1,6 @@ import { useEffect, useState } from 'react'; import loadBackupList, { BackupListType } from '../api/loader/loadBackupList'; import SettingsNavigation from '../components/SettingsNavigation'; -import deleteDownloadQueueByFilter from '../api/actions/deleteDownloadQueueByFilter'; import updateTaskByName from '../api/actions/updateTaskByName'; import queueBackup from '../api/actions/queueBackup'; import restoreBackup from '../api/actions/restoreBackup'; @@ -63,32 +62,6 @@ const SettingsActions = () => {

    Actions

    -
    -

    Delete download queue

    -

    Delete your pending or previously ignored videos from your download queue.

    - {deleteIgnored &&

    Deleting download queue: ignored

    } - {!deleteIgnored && ( -

    Manual media files import.

    From 3947653595f20b821ab744e4f632c8db244ed13d Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 6 Jul 2025 20:14:56 +0700 Subject: [PATCH 28/55] add bulk status update in queue --- backend/download/serializers.py | 16 +++++++++ backend/download/src/queue.py | 24 +++++++++++++ backend/download/views.py | 34 +++++++++++++++++++ .../actions/updateDownloadQueueByFilter.ts | 23 +++++++++++++ frontend/src/functions/APIClient.ts | 2 +- frontend/src/pages/Download.tsx | 27 ++++++++++++++- 6 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 frontend/src/api/actions/updateDownloadQueueByFilter.ts diff --git a/backend/download/serializers.py b/backend/download/serializers.py index 5ab3eca8..4b818fb5 100644 --- a/backend/download/serializers.py +++ b/backend/download/serializers.py @@ -78,6 +78,22 @@ class AddToDownloadQuerySerializer(serializers.Serializer): autostart = serializers.BooleanField(required=False) +class BulkUpdateDowloadQuerySerializer(serializers.Serializer): + """serialize bulk update query""" + + filter = serializers.ChoiceField(choices=["pending", "ignore", "priority"]) + channel = serializers.CharField(required=False) + vid_type = serializers.ChoiceField( + choices=VideoTypeEnum.values_known(), required=False + ) + + +class BulkUpdateDowloadDataSerializer(serializers.Serializer): + """serialize data""" + + status = serializers.ChoiceField(choices=["pending", "ignore", "priority"]) + + class DownloadQueueItemUpdateSerializer(serializers.Serializer): """update single download queue item""" diff --git a/backend/download/src/queue.py b/backend/download/src/queue.py index 3b21fc68..000dae57 100644 --- a/backend/download/src/queue.py +++ b/backend/download/src/queue.py @@ -114,6 +114,30 @@ class PendingInteract: path = "ta_download/_delete_by_query?refresh=true" _, _ = ElasticWrap(path).post(data=data) + def update_bulk( + self, channel_id: str | None, vid_type: str | None, new_status: str + ): + """update status in bulk""" + must_list = [{"term": {"status": {"value": self.status}}}] + if channel_id: + must_list.append({"term": {"channel_id": {"value": channel_id}}}) + + if vid_type: + must_list.append({"term": {"vid_type": {"value": vid_type}}}) + + data = { + "query": {"bool": {"must": must_list}}, + "script": { + "source": f"ctx._source.status = '{new_status}'", + "lang": "painless", + }, + } + print(data) + path = "ta_download/_update_by_query?refresh=true" + response, status_code = ElasticWrap(path).post(data) + print(status_code) + print(response) + def update_status(self): """update status of pending item""" if self.status == "priority": diff --git a/backend/download/views.py b/backend/download/views.py index b3cec1a9..0d66faca 100644 --- a/backend/download/views.py +++ b/backend/download/views.py @@ -8,6 +8,8 @@ from common.views_base import AdminOnly, ApiBaseView from download.serializers import ( AddToDownloadListSerializer, AddToDownloadQuerySerializer, + BulkUpdateDowloadDataSerializer, + BulkUpdateDowloadQuerySerializer, DownloadAggsSerializer, DownloadItemSerializer, DownloadListQuerySerializer, @@ -120,6 +122,38 @@ class DownloadApiListView(ApiBaseView): return Response(response_serializer.data) + @staticmethod + @extend_schema( + request=BulkUpdateDowloadDataSerializer(), + parameters=[BulkUpdateDowloadQuerySerializer()], + responses={204: OpenApiResponse(description="Status updated")}, + ) + def patch(request): + """bulk update status""" + data_serializer = BulkUpdateDowloadDataSerializer(data=request.data) + data_serializer.is_valid(raise_exception=True) + validated_data = data_serializer.validated_data + + new_status = validated_data["status"] + + query_serializer = BulkUpdateDowloadQuerySerializer( + data=request.query_params + ) + query_serializer.is_valid(raise_exception=True) + validated_query = query_serializer.validated_data + status_filter = validated_query.get("filter") + channel = validated_query.get("channel") + vid_type = validated_query.get("vid_type") + + PendingInteract(status=status_filter).update_bulk( + channel_id=channel, vid_type=vid_type, new_status=new_status + ) + + if new_status == "priority": + download_pending.delay(auto_only=True) + + return Response(status=204) + @extend_schema( parameters=[DownloadListQueueDeleteQuerySerializer()], responses={ diff --git a/frontend/src/api/actions/updateDownloadQueueByFilter.ts b/frontend/src/api/actions/updateDownloadQueueByFilter.ts new file mode 100644 index 00000000..1410360e --- /dev/null +++ b/frontend/src/api/actions/updateDownloadQueueByFilter.ts @@ -0,0 +1,23 @@ +import APIClient from '../../functions/APIClient'; + +type FilterType = 'ignore' | 'pending'; +export type DownloadQueueStatus = 'ignore' | 'pending' | 'priority'; + +const updateDownloadQueueByFilter = async ( + filter: FilterType, + channel: string | null, + vid_type: string | null, + status: DownloadQueueStatus, +) => { + const searchParams = new URLSearchParams(); + if (filter) searchParams.append('filter', filter); + if (channel) searchParams.append('channel', channel); + if (vid_type) searchParams.append('vid_type', vid_type); + + return APIClient(`/api/download/?${searchParams.toString()}`, { + method: 'PATCH', + body: { status: status }, + }); +}; + +export default updateDownloadQueueByFilter; diff --git a/frontend/src/functions/APIClient.ts b/frontend/src/functions/APIClient.ts index c0656a63..5b464f3d 100644 --- a/frontend/src/functions/APIClient.ts +++ b/frontend/src/functions/APIClient.ts @@ -6,7 +6,7 @@ import getCookie from './getCookie'; import Routes from '../configuration/routes/RouteList'; export interface ApiClientOptions extends Omit { - method?: 'GET' | 'POST' | 'PUT' | 'DELETE'; + method?: 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE'; body?: Record | string; } diff --git a/frontend/src/pages/Download.tsx b/frontend/src/pages/Download.tsx index a4a4388b..3cd98d0b 100644 --- a/frontend/src/pages/Download.tsx +++ b/frontend/src/pages/Download.tsx @@ -22,6 +22,9 @@ import { useUserConfigStore } from '../stores/UserConfigStore'; import updateUserConfig, { UserConfigType } from '../api/actions/updateUserConfig'; import { ApiResponseType } from '../functions/APIClient'; import deleteDownloadQueueByFilter from '../api/actions/deleteDownloadQueueByFilter'; +import updateDownloadQueueByFilter, { + DownloadQueueStatus, +} from '../api/actions/updateDownloadQueueByFilter'; type Download = { auto_start: boolean; @@ -141,6 +144,16 @@ const Download = () => { })(); }, [lastVideoCount, showIgnored]); + const handleBulkStatusUpdate = async (status: DownloadQueueStatus) => { + await updateDownloadQueueByFilter( + showIgnoredFilter, + channelFilterFromUrl, + vidTypeFilterFromUrl, + status, + ); + setRefresh(true); + }; + return ( <> TA | Downloads @@ -397,6 +410,18 @@ const Download = () => { )}

    +
    + {showIgnored ? ( +
    + +
    + ) : ( +
    + + +
    + )} +
    {showDeleteConfirm ? ( <> @@ -416,7 +441,7 @@ const Download = () => { ) : ( - + )}
    From f87309dbd12bdc420ea3e5c5dd2d9ff8d6df28dd Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 6 Jul 2025 20:19:40 +0700 Subject: [PATCH 29/55] change reset queue filtering on status filter change --- frontend/src/pages/Download.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/pages/Download.tsx b/frontend/src/pages/Download.tsx index 3cd98d0b..5c6cdfc7 100644 --- a/frontend/src/pages/Download.tsx +++ b/frontend/src/pages/Download.tsx @@ -257,7 +257,7 @@ const Download = () => { id="showIgnored" onChange={() => { handleUserConfigUpdate({ show_ignored_only: !showIgnored }); - const newParams = new URLSearchParams(searchParams.toString()); + const newParams = new URLSearchParams(); newParams.set('ignored', String(!showIgnored)); setSearchParams(newParams); setRefresh(true); From 74c708baa2ea9374cfdb03e00527e9a3881e0bd7 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 6 Jul 2025 20:23:39 +0700 Subject: [PATCH 30/55] fix notification url delete --- frontend/src/api/actions/deleteAppriseNotificationUrl.ts | 4 ++-- frontend/src/pages/SettingsScheduling.tsx | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/frontend/src/api/actions/deleteAppriseNotificationUrl.ts b/frontend/src/api/actions/deleteAppriseNotificationUrl.ts index 5018b79f..33153c49 100644 --- a/frontend/src/api/actions/deleteAppriseNotificationUrl.ts +++ b/frontend/src/api/actions/deleteAppriseNotificationUrl.ts @@ -6,10 +6,10 @@ type AppriseTaskNameType = | 'download_pending' | 'check_reindex'; -const deleteAppriseNotificationUrl = async (taskName: AppriseTaskNameType) => { +const deleteAppriseNotificationUrl = async (taskName: AppriseTaskNameType, url: string) => { return APIClient('/api/task/notification/', { method: 'DELETE', - body: { task_name: taskName }, + body: { task_name: taskName, url: url }, }); }; diff --git a/frontend/src/pages/SettingsScheduling.tsx b/frontend/src/pages/SettingsScheduling.tsx index b2d064b5..7ee95012 100644 --- a/frontend/src/pages/SettingsScheduling.tsx +++ b/frontend/src/pages/SettingsScheduling.tsx @@ -519,7 +519,10 @@ const SettingsScheduling = () => { className="danger-button" label="Delete" onClick={async () => { - await deleteAppriseNotificationUrl(key as AppriseTaskNameType); + await deleteAppriseNotificationUrl( + key as AppriseTaskNameType, + url, + ); setRefresh(true); }} From 66f37a92d3a643bc766947d8776c75c6e5868a23 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 6 Jul 2025 20:55:04 +0700 Subject: [PATCH 31/55] fix priority bulk download --- backend/download/src/queue.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/backend/download/src/queue.py b/backend/download/src/queue.py index 000dae57..aa214ac9 100644 --- a/backend/download/src/queue.py +++ b/backend/download/src/queue.py @@ -125,18 +125,22 @@ class PendingInteract: if vid_type: must_list.append({"term": {"vid_type": {"value": vid_type}}}) + if new_status == "priority": + source = """ + ctx._source.status = 'pending'; + ctx._source.auto_start = true; + ctx._source.message = null; + """ + else: + source = f"ctx._source.status = '{new_status}'" + data = { "query": {"bool": {"must": must_list}}, - "script": { - "source": f"ctx._source.status = '{new_status}'", - "lang": "painless", - }, + "script": {"source": source, "lang": "painless"}, } - print(data) + path = "ta_download/_update_by_query?refresh=true" - response, status_code = ElasticWrap(path).post(data) - print(status_code) - print(response) + _, _ = ElasticWrap(path).post(data) def update_status(self): """update status of pending item""" From b6d38e9319fabefe83c9df76fe543367add3ee0b Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 6 Jul 2025 21:59:26 +0700 Subject: [PATCH 32/55] return complete vid_entry dict from scan --- backend/appsettings/src/reindex.py | 6 ++++-- backend/download/src/queue.py | 4 ++-- backend/download/src/subscriptions.py | 13 ++++++------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/backend/appsettings/src/reindex.py b/backend/appsettings/src/reindex.py index 56fd339e..712eac72 100644 --- a/backend/appsettings/src/reindex.py +++ b/backend/appsettings/src/reindex.py @@ -510,12 +510,14 @@ class ChannelFullScan: self.to_update = [] for video in all_local_videos: video_id = video["youtube_id"] - remote_match = [i for i in all_remote_videos if i[0] == video_id] + remote_match = [ + i for i in all_remote_videos if i["id"] == video_id + ] if not remote_match: print(f"{video_id}: no remote match found") continue - expected_type = remote_match[0][-1] + expected_type = remote_match[0]["vid_type"] if video["vid_type"] != expected_type: self.to_update.append( { diff --git a/backend/download/src/queue.py b/backend/download/src/queue.py index aa214ac9..3bd202a1 100644 --- a/backend/download/src/queue.py +++ b/backend/download/src/queue.py @@ -258,8 +258,8 @@ class PendingList(PendingIndex): video_results = ChannelSubscription().get_last_youtube_videos( url, limit=False, query_filter=vid_type ) - for video_id, _, vid_type in video_results: - self._add_video(video_id, vid_type) + for video_entry in video_results: + self._add_video(video_entry["id"], video_entry["vid_type"]) def _parse_playlist(self, url): """add all videos of playlist to list""" diff --git a/backend/download/src/subscriptions.py b/backend/download/src/subscriptions.py index abc0a4eb..524202e5 100644 --- a/backend/download/src/subscriptions.py +++ b/backend/download/src/subscriptions.py @@ -65,12 +65,9 @@ class ChannelSubscription: if not channel_query: continue - last_videos.extend( - [ - (i["id"], i["title"], vid_type) - for i in channel_query["entries"] - ] - ) + for entry in channel_query["entries"]: + entry["vid_type"] = vid_type + last_videos.append(entry) return last_videos @@ -93,7 +90,9 @@ class ChannelSubscription: if last_videos: ids_to_add = is_missing([i[0] for i in last_videos]) - for video_id, _, vid_type in last_videos: + for video_entry in last_videos: + video_id = video_entry["id"] + vid_type = video_entry["vid_type"] if video_id in ids_to_add: missing_videos.append((video_id, vid_type)) From 15ec8b5ab645e98f00927d42a3c1c74e21815bd4 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 9 Jul 2025 16:16:01 +0700 Subject: [PATCH 33/55] refac pending list, implement flat add --- backend/common/src/es_connect.py | 2 +- backend/common/src/helper.py | 8 +- backend/download/serializers.py | 5 +- backend/download/src/queue.py | 359 ++++++++++++++++++------------- backend/download/views.py | 5 +- backend/task/tasks.py | 14 +- 6 files changed, 234 insertions(+), 159 deletions(-) diff --git a/backend/common/src/es_connect.py b/backend/common/src/es_connect.py index f7a59e3a..b2d11635 100644 --- a/backend/common/src/es_connect.py +++ b/backend/common/src/es_connect.py @@ -56,7 +56,7 @@ class ElasticWrap: return response.json(), response.status_code def post( - self, data: bool | dict = False, ndjson: bool = False + self, data: bool | dict | str = False, ndjson: bool = False ) -> tuple[dict, int]: """post data to es""" diff --git a/backend/common/src/helper.py b/backend/common/src/helper.py index fcf75dfe..7b1d43be 100644 --- a/backend/common/src/helper.py +++ b/backend/common/src/helper.py @@ -103,8 +103,11 @@ def requests_headers() -> dict[str, str]: return {"User-Agent": template} -def date_parser(timestamp: int | str) -> str: +def date_parser(timestamp: int | str | None) -> str | None: """return formatted date string""" + if timestamp is None: + return None + if isinstance(timestamp, int): date_obj = datetime.fromtimestamp(timestamp, tz=timezone.utc) elif isinstance(timestamp, str): @@ -184,11 +187,12 @@ def get_duration_sec(file_path: str) -> int: return duration_sec -def get_duration_str(seconds: int) -> str: +def get_duration_str(seconds: int | float) -> str: """Return a human-readable duration string from seconds.""" if not seconds: return "NA" + seconds = int(seconds) units = [("y", 31536000), ("d", 86400), ("h", 3600), ("m", 60), ("s", 1)] duration_parts = [] diff --git a/backend/download/serializers.py b/backend/download/serializers.py index 4b818fb5..631541df 100644 --- a/backend/download/serializers.py +++ b/backend/download/serializers.py @@ -15,9 +15,9 @@ class DownloadItemSerializer(serializers.Serializer): channel_indexed = serializers.BooleanField() channel_name = serializers.CharField() duration = serializers.CharField() - published = serializers.CharField() + published = serializers.CharField(allow_null=True) status = serializers.ChoiceField(choices=["pending", "ignore"]) - timestamp = serializers.IntegerField() + timestamp = serializers.IntegerField(allow_null=True) title = serializers.CharField() vid_thumb_url = serializers.CharField() vid_type = serializers.ChoiceField(choices=VideoTypeEnum.values()) @@ -76,6 +76,7 @@ class AddToDownloadQuerySerializer(serializers.Serializer): """add to queue query serializer""" autostart = serializers.BooleanField(required=False) + flat = serializers.BooleanField(required=False) class BulkUpdateDowloadQuerySerializer(serializers.Serializer): diff --git a/backend/download/src/queue.py b/backend/download/src/queue.py index 3bd202a1..8eb0a391 100644 --- a/backend/download/src/queue.py +++ b/backend/download/src/queue.py @@ -4,16 +4,18 @@ Functionality: - linked with ta_dowload index """ +import json from datetime import datetime from appsettings.src.config import AppConfig +from channel.src.index import YoutubeChannel from common.src.es_connect import ElasticWrap, IndexPaginate from common.src.helper import get_duration_str, is_shorts, rand_sleep from download.src.subscriptions import ChannelSubscription from download.src.thumbnails import ThumbManager -from download.src.yt_dlp_base import YtWrap from playlist.src.index import YoutubePlaylist from video.src.constants import VideoTypeEnum +from video.src.index import YoutubeVideo class PendingIndex: @@ -195,22 +197,27 @@ class PendingList(PendingIndex): "check_formats": None, } - def __init__(self, youtube_ids=False, task=False): + def __init__( + self, youtube_ids=False, task=False, auto_start=False, flat=False + ): super().__init__() self.config = AppConfig().config self.youtube_ids = youtube_ids self.task = task + self.auto_start = auto_start + self.flat = flat self.to_skip = False - self.missing_videos = False - - def parse_url_list(self, auto_start=False): - """extract youtube ids from list""" self.missing_videos = [] + + def parse_url_list(self): + """extract youtube ids from list""" self.get_download() self.get_indexed() + self.get_channels() total = len(self.youtube_ids) for idx, entry in enumerate(self.youtube_ids): - self._process_entry(entry, auto_start=auto_start) + self._process_entry(entry) + rand_sleep(self.config) if not self.task: continue @@ -219,102 +226,214 @@ class PendingList(PendingIndex): progress=(idx + 1) / total, ) - def _process_entry(self, entry, auto_start=False): + def _process_entry(self, entry: dict): """process single entry from url list""" - vid_type = self._get_vid_type(entry) if entry["type"] == "video": - self._add_video(entry["url"], vid_type, auto_start=auto_start) + self._add_video(entry["url"], entry["vid_type"]) elif entry["type"] == "channel": - self._parse_channel(entry["url"], vid_type) + self._parse_channel(entry["url"], entry["vid_type"]) elif entry["type"] == "playlist": self._parse_playlist(entry["url"]) else: raise ValueError(f"invalid url_type: {entry}") - @staticmethod - def _get_vid_type(entry): - """add vid type enum if available""" - vid_type_str = entry.get("vid_type") - if not vid_type_str: - return VideoTypeEnum.UNKNOWN - - return VideoTypeEnum(vid_type_str) - - def _add_video(self, url, vid_type, auto_start=False): + def _add_video(self, url, vid_type): """add video to list""" - if auto_start and url in set( + if self.auto_start and url in set( i["youtube_id"] for i in self.all_pending ): PendingInteract(youtube_id=url, status="priority").update_status() return - if url not in self.missing_videos and url not in self.to_skip: - self.missing_videos.append((url, vid_type)) - else: + if url in self.missing_videos or url in self.to_skip: print(f"{url}: skipped adding already indexed video to download.") + else: + to_add = self._parse_video(url, vid_type) + if to_add: + self.missing_videos.append(to_add) def _parse_channel(self, url, vid_type): - """add all videos of channel to list""" + """parse channel""" video_results = ChannelSubscription().get_last_youtube_videos( url, limit=False, query_filter=vid_type ) - for video_entry in video_results: - self._add_video(video_entry["id"], video_entry["vid_type"]) + channel_handler = YoutubeChannel(url) + channel_handler.build_json(upload=False) - def _parse_playlist(self, url): - """add all videos of playlist to list""" - playlist = YoutubePlaylist(url) - is_active = playlist.update_playlist() - if not is_active: - message = f"{playlist.youtube_id}: failed to extract metadata" - print(message) - raise ValueError(message) - - entries = playlist.json_data["playlist_entries"] - to_add = [i["youtube_id"] for i in entries if not i["downloaded"]] - if not to_add: - return - - for video_id in to_add: - # match vid_type later - self._add_video(video_id, VideoTypeEnum.UNKNOWN) - - def add_to_pending(self, status="pending", auto_start=False): - """add missing videos to pending list""" - self.get_channels() - - total = len(self.missing_videos) - videos_added = [] - for idx, (youtube_id, vid_type) in enumerate(self.missing_videos): - if self.task and self.task.is_stopped(): - break - - print(f"{youtube_id}: [{idx + 1}/{total}]: add to queue") - self._notify_add(idx, total) - video_details = self.get_youtube_details(youtube_id, vid_type) - if not video_details: - rand_sleep(self.config) + for video_data in video_results: + video_id = video_data["id"] + if video_id in self.to_skip: continue - video_details.update( + if self.flat: + if not video_data.get("channel"): + channel_name = channel_handler.json_data["channel_name"] + video_data["channel"] = channel_name + + if not video_data.get("channel_id"): + channel_id = channel_handler.json_data["channel_id"] + video_data["channel_id"] = channel_id + + to_add = self._parse_entry( + youtube_id=video_id, video_data=video_data + ) + else: + to_add = self._parse_video(video_id, vid_type) + + if to_add: + self.missing_videos.append(to_add) + + def _parse_playlist(self, url): + """fast parse playlist""" + playlist = YoutubePlaylist(url) + playlist.get_from_youtube() + video_results = playlist.youtube_meta["entries"] + + for video_data in video_results: + video_id = video_data["id"] + if video_id in self.to_skip: + continue + + if self.flat: + to_add = self._parse_entry(video_id, video_data) + else: + to_add = self._parse_video(video_id, vid_type=None) + + if to_add: + self.missing_videos.append(to_add) + + def _parse_video(self, url, vid_type): + """parse video""" + video = YoutubeVideo(youtube_id=url) + video.get_from_youtube() + + video_data = video.youtube_meta + video_data["vid_type"] = vid_type + to_add = self._parse_entry(youtube_id=url, video_data=video_data) + + return to_add + + def _parse_entry(self, youtube_id: str, video_data: dict) -> dict | None: + """parse entry""" + if video_data.get("id") != youtube_id: + # skip premium videos with different id or redirects + print(f"{youtube_id}: skipping redirect, id not matching") + return None + + if video_data.get("live_status") in ["is_upcoming", "is_live"]: + print(f"{youtube_id}: skip is_upcoming or is_live") + return None + + to_add = { + "youtube_id": video_data["id"], + "title": video_data["title"], + "vid_thumb_url": self.__extract_thumb(video_data), + "duration": get_duration_str(video_data.get("duration", 0)), + "published": self.__extract_published(video_data), + "timestamp": int(datetime.now().timestamp()), + "vid_type": self.__extract_vid_type(video_data), + "channel_name": video_data["channel"], + "channel_id": video_data["channel_id"], + "channel_indexed": video_data["channel_id"] in self.all_channels, + } + thumb_url = to_add["vid_thumb_url"] + ThumbManager(to_add["youtube_id"]).download_video_thumb(thumb_url) + + return to_add + + def __extract_thumb(self, video_data) -> str | None: + """extract thumb""" + if "thumbnail" in video_data: + return video_data["thumbnail"] + + if "thumbnails" in video_data: + return video_data["thumbnails"][-1]["url"] + + return None + + def __extract_published(self, video_data) -> str | int | None: + """build published date or timestamp""" + timestamp = video_data.get("timestamp") + if timestamp: + return timestamp + + upload_date = video_data.get("upload_date") + if not upload_date: + return None + + upload_date_time = datetime.strptime(upload_date, "%Y%m%d") + published = upload_date_time.strftime("%Y-%m-%d") + + return published + + def __extract_vid_type(self, video_data) -> str: + """build vid type""" + if "vid_type" in video_data: + return video_data["vid_type"] + + if video_data.get("live_status") == "was_live": + return VideoTypeEnum.STREAMS.value + + if video_data.get("width", 0) > video_data.get("height", 0): + return VideoTypeEnum.VIDEOS.value + + duration = video_data.get("duration") + if duration and isinstance(duration, int): + if duration > 3 * 60: + return VideoTypeEnum.VIDEOS.value + + if is_shorts(video_data["id"]): + return VideoTypeEnum.SHORTS.value + + return VideoTypeEnum.VIDEOS.value + + def add_to_pending(self, status="pending") -> int: + """add missing videos to pending list""" + + total = len(self.missing_videos) + + if not self.missing_videos: + self._notify_empty() + return 0 + + self._notify_start(total) + bulk_list = [] + for video_entry in self.missing_videos: + video_entry.update( { "status": status, - "auto_start": auto_start, + "auto_start": self.auto_start, } ) + video_id = video_entry["youtube_id"] + action = {"index": {"_index": "ta_download", "_id": video_id}} + bulk_list.append(json.dumps(action)) + bulk_list.append(json.dumps(video_entry)) - url = video_details["vid_thumb_url"] - ThumbManager(youtube_id).download_video_thumb(url) - es_url = f"ta_download/_doc/{youtube_id}" - _, _ = ElasticWrap(es_url).put(video_details) - videos_added.append(youtube_id) + # add last newline + bulk_list.append("\n") + query_str = "\n".join(bulk_list) + _, status_code = ElasticWrap("_bulk").post(query_str, ndjson=True) + if status_code != 200: + self._notify_fail(status_code) + else: + self._notify_done(total) - if idx != total: - rand_sleep(self.config) + return len(self.missing_videos) - return videos_added + def _notify_empty(self): + """notify nothing to add""" + if not self.task: + return - def _notify_add(self, idx, total): + self.task.send_progress( + message_lines=[ + "Extractinc videos completed.", + "No new videos found to add.", + ] + ) + + def _notify_start(self, total): """send notification for adding videos to download queue""" if not self.task: return @@ -322,82 +441,30 @@ class PendingList(PendingIndex): self.task.send_progress( message_lines=[ "Adding new videos to download queue.", - f"Extracting items {idx + 1}/{total}", - ], - progress=(idx + 1) / total, + f"Bulk adding {total} videos", + ] ) - def get_youtube_details(self, youtube_id, vid_type=VideoTypeEnum.VIDEOS): - """get details from youtubedl for single pending video""" - vid = YtWrap(self.yt_obs, self.config).extract(youtube_id) - if not vid: - return False + def _notify_done(self, total): + """send done notification""" + if not self.task: + return - if vid.get("id") != youtube_id: - # skip premium videos with different id - print(f"{youtube_id}: skipping premium video, id not matching") - return False - # stop if video is streaming live now - if vid["live_status"] in ["is_upcoming", "is_live"]: - print(f"{youtube_id}: skip is_upcoming or is_live") - return False + self.task.send_progress( + message_lines=[ + "Adding new videos to the queue completed.", + f"Added {total} videos.", + ] + ) - if vid["live_status"] == "was_live": - vid_type = VideoTypeEnum.STREAMS - else: - if self._check_shorts(vid): - vid_type = VideoTypeEnum.SHORTS - else: - vid_type = VideoTypeEnum.VIDEOS + def _notify_fail(self, status_code): + """failed to add""" + if not self.task: + return - if not vid.get("channel"): - print(f"{youtube_id}: skip video not part of channel") - return False - - return self._parse_youtube_details(vid, vid_type) - - @staticmethod - def _check_shorts(vid): - """check if vid is shorts video""" - if vid["width"] > vid["height"]: - return False - - duration = vid.get("duration") - if duration and isinstance(duration, int): - if duration > 3 * 60: - return False - - return is_shorts(vid["id"]) - - def _parse_youtube_details(self, vid, vid_type=VideoTypeEnum.VIDEOS): - """parse response""" - vid_id = vid.get("id") - - # build dict - youtube_details = { - "youtube_id": vid_id, - "channel_name": vid["channel"], - "vid_thumb_url": vid["thumbnail"], - "title": vid["title"], - "channel_id": vid["channel_id"], - "duration": get_duration_str(vid["duration"]), - "published": self._build_published(vid), - "timestamp": int(datetime.now().timestamp()), - "vid_type": vid_type.value, - "channel_indexed": vid["channel_id"] in self.all_channels, - } - - return youtube_details - - @staticmethod - def _build_published(vid): - """build published date or timestamp""" - timestamp = vid["timestamp"] - if timestamp: - return timestamp - - upload_date = vid["upload_date"] - upload_date_time = datetime.strptime(upload_date, "%Y%m%d") - published = upload_date_time.strftime("%Y-%m-%d") - - return published + self.task.send_progress( + message_lines=[ + "Adding extracted videos failed.", + f"Status code: {status_code}", + ] + ) diff --git a/backend/download/views.py b/backend/download/views.py index 0d66faca..80ed95b5 100644 --- a/backend/download/views.py +++ b/backend/download/views.py @@ -107,12 +107,13 @@ class DownloadApiListView(ApiBaseView): validated_query = query_serializer.validated_data auto_start = validated_query.get("autostart") - print(f"auto_start: {auto_start}") + flat = validated_query.get("flat", False) + print(f"auto_start: {auto_start}, flat: {flat}") to_add = validated_data["data"] pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"] url_str = " ".join(pending) - task = extrac_dl.delay(url_str, auto_start=auto_start) + task = extrac_dl.delay(url_str, auto_start=auto_start, flat=flat) message = { "message": "add to queue task started", diff --git a/backend/task/tasks.py b/backend/task/tasks.py index d879cb9e..8583e916 100644 --- a/backend/task/tasks.py +++ b/backend/task/tasks.py @@ -147,7 +147,9 @@ def download_pending(self, auto_only=False): @shared_task(name="extract_download", bind=True, base=BaseTask) -def extrac_dl(self, youtube_ids, auto_start=False, status="pending"): +def extrac_dl( + self, youtube_ids, auto_start=False, flat=False, status="pending" +): """parse list passed and add to pending""" TaskManager().init(self) if isinstance(youtube_ids, str): @@ -155,17 +157,17 @@ def extrac_dl(self, youtube_ids, auto_start=False, status="pending"): else: to_add = youtube_ids - pending_handler = PendingList(youtube_ids=to_add, task=self) - pending_handler.parse_url_list(auto_start=auto_start) - videos_added = pending_handler.add_to_pending( - status=status, auto_start=auto_start + pending_handler = PendingList( + youtube_ids=to_add, task=self, auto_start=auto_start, flat=flat ) + pending_handler.parse_url_list() + videos_added = pending_handler.add_to_pending(status=status) if auto_start: download_pending.delay(auto_only=True) if videos_added: - return f"added {len(videos_added)} Videos to Queue" + return f"added {videos_added} Videos to Queue" return None From 97bc6f225f78578546b0b31bc5bd0b6eb222cc3e Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 9 Jul 2025 17:09:45 +0700 Subject: [PATCH 34/55] add fast add, use toggles --- .../src/api/actions/updateDownloadQueue.ts | 12 ++-- frontend/src/components/DownloadListItem.tsx | 5 +- frontend/src/pages/Download.tsx | 59 ++++++++++++++----- 3 files changed, 54 insertions(+), 22 deletions(-) diff --git a/frontend/src/api/actions/updateDownloadQueue.ts b/frontend/src/api/actions/updateDownloadQueue.ts index 071707df..beb14120 100644 --- a/frontend/src/api/actions/updateDownloadQueue.ts +++ b/frontend/src/api/actions/updateDownloadQueue.ts @@ -1,6 +1,6 @@ import APIClient from '../../functions/APIClient'; -const updateDownloadQueue = async (youtubeIdStrings: string, autostart: boolean) => { +const updateDownloadQueue = async (youtubeIdStrings: string, autostart: boolean, flat: boolean) => { const urls = []; const containsMultiple = youtubeIdStrings.includes('\n'); @@ -16,12 +16,12 @@ const updateDownloadQueue = async (youtubeIdStrings: string, autostart: boolean) urls.push({ youtube_id: youtubeIdStrings, status: 'pending' }); } - let params = ''; - if (autostart) { - params = '?autostart=true'; - } + const searchParams = new URLSearchParams(); + if (autostart) searchParams.append('autostart', 'true'); + if (flat) searchParams.append('flat', 'true'); + const endpoint = `/api/download/${searchParams.toString() ? `?${searchParams.toString()}` : ''}`; - return APIClient(`/api/download/${params}`, { + return APIClient(endpoint, { method: 'POST', body: { data: [...urls] }, }); diff --git a/frontend/src/components/DownloadListItem.tsx b/frontend/src/components/DownloadListItem.tsx index ff0273dd..cd540c87 100644 --- a/frontend/src/components/DownloadListItem.tsx +++ b/frontend/src/components/DownloadListItem.tsx @@ -57,8 +57,9 @@ const DownloadListItem = ({ download, setRefresh }: DownloadListItemProps) => {

    - Published: {formatDate(download.published)} | Duration: {download.duration} |{' '} - {download.youtube_id} + {download.published && Published: {formatDate(download.published)} | } + Duration: {download.duration} | + {download.youtube_id}

    {download.message &&

    {download.message}

    } diff --git a/frontend/src/pages/Download.tsx b/frontend/src/pages/Download.tsx index 5c6cdfc7..c141648e 100644 --- a/frontend/src/pages/Download.tsx +++ b/frontend/src/pages/Download.tsx @@ -33,9 +33,9 @@ type Download = { channel_name: string; duration: string; message?: string; - published: string; + published: string | null; status: string; - timestamp: number; + timestamp: number | null; title: string; vid_thumb_url: string; vid_type: string; @@ -61,6 +61,8 @@ const Download = () => { const [refresh, setRefresh] = useState(false); const [showHiddenForm, setShowHiddenForm] = useState(false); + const [addAsAutoStart, setAddAsAutoStart] = useState(false); + const [addAsFlat, setAddAsFlat] = useState(false); const [showQueueActions, setShowQueueActions] = useState(false); const [showDeleteConfirm, setShowDeleteConfirm] = useState(false); const [downloadPending, setDownloadPending] = useState(false); @@ -215,6 +217,46 @@ const Download = () => { {showHiddenForm && (
    +
    +
    + setAddAsFlat(!addAsFlat)} + /> + {addAsFlat ? ( + + ) : ( + + )} +
    + Fast add +
    +
    +
    + setAddAsAutoStart(!addAsAutoStart)} + /> + {addAsAutoStart ? ( + + ) : ( + + )} +
    + Auto Download +