RomanNum3ral
/
tubearchivist
mirror of https://github.com/tubearchivist/tubearchivist.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Split up audio language downloads into separate methods, remove use_effect(), use lint formatting & fix documentation in code for feature.

This commit is contained in:
Dakota Gravitt 2026-03-09 14:12:12 -05:00
parent 79b1f9e499
commit 3470611a3e
11 changed files with 195 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/appsettings/serializers.py
View File

@ -59,7 +59,7 @@ class AppConfigDownloadsSerializer(
extractor_lang = serializers.CharField(allow_null=True)
integrate_ryd = serializers.BooleanField()
integrate_sponsorblock = serializers.BooleanField()
audio_multistream = serializers.BooleanField()
audio_multistreams = serializers.BooleanField()
audio_languages = serializers.CharField(allow_null=True)

4
backend/appsettings/src/config.py
View File

@ -46,7 +46,7 @@ class DownloadsConfigType(TypedDict):
extractor_lang: str | None
integrate_ryd: bool
integrate_sponsorblock: bool
audio_multistream: bool
audio_multistreams: bool
audio_languages: str | None
@ -97,7 +97,7 @@ class AppConfig:
"extractor_lang": None,
"integrate_ryd": False,
"integrate_sponsorblock": False,
"audio_multistream": False,
"audio_multistreams": False,
"audio_languages": None,
},
"application": {

2
backend/channel/serializers.py
View File

@ -13,7 +13,7 @@ class ChannelOverwriteSerializer(
"""serialize channel overwrites"""
download_format = serializers.CharField(required=False, allow_null=True)
audio_multistream = serializers.BooleanField(
audio_multistreams = serializers.BooleanField(
required=False, allow_null=True
)
audio_languages = serializers.CharField(required=False, allow_null=True)

2
backend/channel/src/index.py
View File

@ -266,7 +266,7 @@ class YoutubeChannel(YouTubeItem):
"""set per channel overwrites"""
valid_keys = [
"download_format",
"audio_multistream",
"audio_multistreams",
"audio_languages",
"autodelete_days",
"index_playlists",

241
backend/download/src/yt_dlp_handler.py
View File

@ -177,7 +177,7 @@ class VideoDownloader(DownloaderBase):
format_sort = self.config["downloads"]["format_sort"]
format_sort_list = [i.strip() for i in format_sort.split(",")]
self.obs["format_sort"] = format_sort_list
if self.config["downloads"].get("audio_multistream"):
if self.config["downloads"].get("audio_multistreams"):
self.obs["audio_multistreams"] = True
if self.config["downloads"]["limit_speed"]:
self.obs["ratelimit"] = (
@ -295,10 +295,16 @@ class VideoDownloader(DownloaderBase):
"noplaylist": True,
}
print(f"[audio_languages] downloading HLS fallback {lang}: {format_id}")
print(
f"[audio_languages] downloading HLS fallback "
f"{lang}: {format_id}"
)
success, _ = YtWrap(hls_obs, self.config).download(youtube_id)
if not success:
print(f"[audio_languages] failed HLS fallback download for: {lang}")
print(
f"[audio_languages] failed HLS fallback "
f"download for: {lang}"
)
return None
# yt-dlp chooses extension by source/container, so discover by prefix.
@ -361,7 +367,9 @@ class VideoDownloader(DownloaderBase):
"csv=p=0",
path,
]
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
result = subprocess.run(
cmd, capture_output=True, text=True, check=False
)
if result.returncode != 0:
return 0
@ -391,13 +399,15 @@ class VideoDownloader(DownloaderBase):
for input_idx in range(1, len(audio_tracks) + 1):
cmd += ["-map", f"{input_idx}:a:0"]
# Label appended tracks so ffprobe/UI can show meaningful language info.
# We offset by existing main-file audio stream count, then annotate each
# Label appended tracks so ffprobe/UI can show meaningful
# language info. We offset by existing main-file audio stream count,
# appended stream with both language and title metadata.
existing_audio_count = VideoDownloader._count_audio_streams(main_path)
for idx, (lang, _) in enumerate(audio_tracks):
audio_stream_idx = existing_audio_count + idx
language_code = VideoDownloader._normalize_language_code(lang)
language_code = VideoDownloader._normalize_language_code(
lang
)
language_title = VideoDownloader._language_title(lang)
cmd += [
f"-metadata:s:a:{audio_stream_idx}",
@ -426,7 +436,7 @@ class VideoDownloader(DownloaderBase):
def _build_main_format(
formats: list[dict], dash_lang_formats: dict
) -> str | None:
"""build DASH-only format string: bestvideo + DASH audio per language"""
"""build DASH-only format string for requested audio tracks"""
video_only = [
f
for f in formats
@ -475,8 +485,8 @@ class VideoDownloader(DownloaderBase):
overwrites = self.channel_overwrites.get(channel_id)
if overwrites and overwrites.get("download_format"):
obs["format"] = overwrites.get("download_format")
if overwrites and overwrites.get("audio_multistream") is not None:
obs["audio_multistreams"] = overwrites.get("audio_multistream")
if overwrites and overwrites.get("audio_multistreams") is not None:
obs["audio_multistreams"] = overwrites.get("audio_multistreams")
# Keep output container mp4-only regardless of channel/global values.
obs["merge_output_format"] = "mp4"
@ -485,8 +495,8 @@ class VideoDownloader(DownloaderBase):
def _get_audio_languages(self, channel_id: str) -> list[str] | None:
"""get audio languages from config or channel overwrites.
Returns None if audio_multistream is not effectively enabled — audio_languages
alone must not force multi-track downloads (Option A / strict mode).
Returns None if audio_multistreams is not effectively enabled.
audio_languages alone must not force multi-track downloads.
"""
if not self._is_audio_multistream_enabled(channel_id):
return None
@ -506,11 +516,14 @@ class VideoDownloader(DownloaderBase):
]
def _is_audio_multistream_enabled(self, channel_id: str) -> bool:
"""return True if audio_multistream is enabled globally or via channel overwrite"""
"""return True when audio_multistreams is enabled"""
overwrites = self.channel_overwrites.get(channel_id, {})
if "audio_multistream" in overwrites and overwrites["audio_multistream"] is not None:
return bool(overwrites["audio_multistream"])
return bool(self.config["downloads"].get("audio_multistream"))
if (
"audio_multistreams" in overwrites
and overwrites["audio_multistreams"] is not None
):
return bool(overwrites["audio_multistreams"])
return bool(self.config["downloads"].get("audio_multistreams"))
@staticmethod
def _discover_audio_languages(formats: list[dict]) -> list[str]:
@ -526,92 +539,138 @@ class VideoDownloader(DownloaderBase):
langs.append(lang)
return langs
def _resolve_requested_audio_languages(
self, youtube_id: str, channel_id: str
) -> tuple[list[str] | None, list[dict] | None]:
"""get configured or auto-discovered audio languages"""
languages = self._get_audio_languages(channel_id)
if languages:
return languages, None
if not self._is_audio_multistream_enabled(channel_id):
return None, None
print(
f"{youtube_id}: audio_multistreams enabled, "
"auto-discovering languages"
)
formats = self._get_formats(youtube_id)
if not formats:
return None, formats
languages = self._discover_audio_languages(formats)
print(f"{youtube_id}: discovered audio languages: {languages}")
return languages, formats
def _set_hls_base_download(
self, obs: dict, youtube_id: str, hls_formats: dict[str, str]
) -> dict[str, str]:
"""use the first HLS language as the base download"""
first_lang, first_fmt = next(iter(hls_formats.items()))
obs["format"] = first_fmt
print(
f"{youtube_id}: no DASH main format, using HLS "
f"{first_lang}:{first_fmt} as base"
)
return {
lang: fmt
for lang, fmt in hls_formats.items()
if lang != first_lang
}
def _select_audio_language_formats(
self,
obs: dict,
youtube_id: str,
languages: list[str] | None,
formats: list[dict] | None,
) -> dict[str, str]:
"""update yt-dlp options for selected audio languages"""
if not languages:
return {}
print(f"{youtube_id}: applying audio languages {languages}")
available_formats = formats or self._get_formats(youtube_id)
if not available_formats:
return {}
dash_fmt = self._resolve_dash_audio_formats(
available_formats, languages
)
hls_fmt = self._resolve_hls_audio_formats(
available_formats, languages, dash_fmt
)
main_fmt = self._build_main_format(available_formats, dash_fmt)
if main_fmt:
selected_audio_count = len(dash_fmt) + len(hls_fmt)
obs["format"] = main_fmt
if selected_audio_count > 1:
obs["audio_multistreams"] = True
print(f"{youtube_id}: main format: {main_fmt}")
return hls_fmt
if not hls_fmt:
return {}
return self._set_hls_base_download(obs, youtube_id, hls_fmt)
@staticmethod
def _cleanup_audio_tracks(
audio_tracks: list[tuple[str, str]]
) -> None:
"""remove temporary fallback track files"""
for _, track_path in audio_tracks:
try:
os.remove(track_path)
except FileNotFoundError:
pass
def _merge_hls_audio_tracks(
self,
youtube_id: str,
dl_cache: str,
hls_formats_to_merge: dict[str, str],
) -> None:
"""download HLS fallback tracks and merge them into the MP4"""
if not hls_formats_to_merge:
return
# Post-step for HLS-only languages: download temporary fallback files
# and append their audio streams to the MP4 generated above.
main_path = os.path.join(dl_cache, f"{youtube_id}.mp4")
audio_tracks: list[tuple[str, str]] = []
try:
for lang, fmt_id in hls_formats_to_merge.items():
track_path = self._download_hls_audio(youtube_id, fmt_id, lang)
if track_path:
audio_tracks.append((lang, track_path))
if audio_tracks:
self._merge_additional_audio_tracks(main_path, audio_tracks)
finally:
self._cleanup_audio_tracks(audio_tracks)
def _dl_single_vid(self, youtube_id: str, channel_id: str) -> bool:
"""download single video with optional multi-language audio selection"""
"""download one video with optional multi-language audio"""
obs = self.obs.copy()
self._set_overwrites(obs, channel_id)
dl_cache = os.path.join(self.CACHE_DIR, "download")
languages = self._get_audio_languages(channel_id)
hls_formats_to_merge: dict[str, str] = {}
if not languages and self._is_audio_multistream_enabled(channel_id):
# audio_multistream is on but no explicit language list – auto-discover
print(f"{youtube_id}: audio_multistream enabled, auto-discovering languages")
formats = self._get_formats(youtube_id)
if formats:
languages = self._discover_audio_languages(formats)
print(f"{youtube_id}: discovered audio languages: {languages}")
else:
formats = None # will be fetched below if needed
if languages:
print(f"{youtube_id}: applying audio languages {languages}")
if formats is None:
formats = self._get_formats(youtube_id)
if formats:
# Build yt-dlp format string using DASH audio tracks only.
# If no DASH tracks are found, fall back to the normal format selection.
dash_fmt = self._resolve_dash_audio_formats(formats, languages)
hls_fmt = self._resolve_hls_audio_formats(
formats, languages, dash_fmt
)
selected_audio_count = len(dash_fmt) + len(hls_fmt)
# DASH-first: use bestvideo + selected DASH audios as main file.
main_fmt = self._build_main_format(formats, dash_fmt)
if main_fmt:
obs["format"] = main_fmt
# If we selected more than one total audio track (DASH+HLS),
# keep yt-dlp multistream flag enabled for the main pass.
if selected_audio_count > 1:
obs["audio_multistreams"] = True
print(f"{youtube_id}: main format: {main_fmt}")
# Any languages that only exist as HLS are merged afterward.
hls_formats_to_merge = hls_fmt
elif hls_fmt:
# No DASH audio matched. Use first HLS language as the main
# download to ensure at least one requested language lands
# in the base file; merge remaining HLS tracks afterward.
first_lang, first_fmt = next(iter(hls_fmt.items()))
obs["format"] = first_fmt
hls_formats_to_merge = {
lang: fmt
for lang, fmt in hls_fmt.items()
if lang != first_lang
}
print(
f"{youtube_id}: no DASH main format, using HLS "
f"{first_lang}:{first_fmt} as base"
)
languages, formats = self._resolve_requested_audio_languages(
youtube_id, channel_id
)
hls_formats_to_merge = self._select_audio_language_formats(
obs, youtube_id, languages, formats
)
success, message = YtWrap(obs, self.config).download(youtube_id)
if not success:
self._handle_error(youtube_id, message)
return False
if hls_formats_to_merge:
# Post-step for HLS-only languages: download temporary fallback files
# and append their audio streams to the MP4 generated above.
main_path = os.path.join(dl_cache, f"{youtube_id}.mp4")
audio_tracks: list[tuple[str, str]] = []
try:
for lang, fmt_id in hls_formats_to_merge.items():
track_path = self._download_hls_audio(
youtube_id, fmt_id, lang
)
if track_path:
audio_tracks.append((lang, track_path))
if audio_tracks:
self._merge_additional_audio_tracks(main_path, audio_tracks)
finally:
for _, track_path in audio_tracks:
try:
os.remove(track_path)
except FileNotFoundError:
pass
self._merge_hls_audio_tracks(
youtube_id, dl_cache, hls_formats_to_merge
)
if self.obs["writethumbnail"]:
# webp files don't get cleaned up automatically

2
frontend/src/api/loader/loadAppsettingsConfig.ts
View File

@ -27,7 +27,7 @@ export type AppSettingsConfigType = {
extractor_lang: string | null;
integrate_ryd: boolean;
integrate_sponsorblock: boolean;
audio_multistream: boolean;
audio_multistreams: boolean;
audio_languages: string | null;
};
application: {

39
frontend/src/components/AudioLanguageSelector.tsx
View File

@ -1,4 +1,4 @@
import { useEffect, useMemo, useState } from 'react';
import { useMemo, useState } from 'react';
type AudioLanguageSelectorProps = {
name: string;
@ -31,6 +31,15 @@ const normalizeCsv = (value: string | null): string[] => {
.filter(Boolean);
};
const splitLanguages = (value: string | null, commonCodes: Set<string>) => {
const parsed = normalizeCsv(value);
return {
known: parsed.filter(item => commonCodes.has(item)),
custom: parsed.filter(item => !commonCodes.has(item)),
};
};
const AudioLanguageSelector = ({
name,
value,
@ -38,17 +47,9 @@ const AudioLanguageSelector = ({
updateCallback,
}: AudioLanguageSelectorProps) => {
const commonCodes = useMemo(() => new Set(COMMON_AUDIO_LANGUAGES.map(item => item.code)), []);
const [selectedCommon, setSelectedCommon] = useState<string[]>([]);
const [customLanguages, setCustomLanguages] = useState('');
useEffect(() => {
const parsed = normalizeCsv(value);
const known = parsed.filter(item => commonCodes.has(item));
const custom = parsed.filter(item => !commonCodes.has(item));
setSelectedCommon(known);
setCustomLanguages(custom.join(', '));
}, [value, commonCodes]);
const initialSelection = useMemo(() => splitLanguages(value, commonCodes), [value, commonCodes]);
const [selectedCommon, setSelectedCommon] = useState<string[]>(initialSelection.known);
const [customLanguages, setCustomLanguages] = useState(initialSelection.custom.join(', '));
const mergedValue = useMemo(() => {
const custom = normalizeCsv(customLanguages);
@ -58,7 +59,9 @@ const AudioLanguageSelector = ({
return (
<div>
<p className="settings-help-text">Select common languages and/or add custom tags. No selection will download all languages.</p>
<p className="settings-help-text">
Select common languages and/or add custom tags. No selection will download all languages.
</p>
<select
multiple
value={selectedCommon}
@ -87,11 +90,9 @@ const AudioLanguageSelector = ({
<button onClick={() => updateCallback(name, mergedValue)}>Update</button>
<button
onClick={() => {
const parsed = normalizeCsv(oldValue);
const known = parsed.filter(item => commonCodes.has(item));
const custom = parsed.filter(item => !commonCodes.has(item));
setSelectedCommon(known);
setCustomLanguages(custom.join(', '));
const resetSelection = splitLanguages(oldValue, commonCodes);
setSelectedCommon(resetSelection.known);
setCustomLanguages(resetSelection.custom.join(', '));
}}
>
Cancel
@ -104,4 +105,4 @@ const AudioLanguageSelector = ({
);
};
export default AudioLanguageSelector;
export default AudioLanguageSelector;

15
frontend/src/pages/ChannelAbout.tsx
View File

@ -63,7 +63,7 @@ const ChannelAbout = () => {
const { data: channelResponseData } = channelResponse ?? {};
const channel = channelResponseData;
const globalAudioMultistream = appSettingsConfig.downloads.audio_multistream;
const globalAudioMultistream = appSettingsConfig.downloads.audio_multistreams;
const effectiveAudioMultistream = audioMultistreams ?? globalAudioMultistream;
useEffect(() => {
@ -75,7 +75,7 @@ const ChannelAbout = () => {
setChannelResponse(channelResponse);
setDownloadFormat(channelResponseData?.channel_overwrites?.download_format ?? null);
setAudioMultistreams(
channelResponseData?.channel_overwrites?.audio_multistream ?? null,
channelResponseData?.channel_overwrites?.audio_multistreams ?? null,
);
setAudioMultistreamsWarning(null);
setAudioLanguages(channelResponseData?.channel_overwrites?.audio_languages ?? null);
@ -120,7 +120,7 @@ const ChannelAbout = () => {
}
setAudioMultistreamsWarning(null);
if (configKey === 'audio_multistream') {
if (configKey === 'audio_multistreams') {
setAudioMultistreams(configValue === null ? null : Boolean(configValue));
}
setRefresh(true);
@ -326,7 +326,7 @@ const ChannelAbout = () => {
<p>Enable multistream audio</p>
</div>
<ToggleConfig
name="audio_multistream"
name="audio_multistreams"
value={effectiveAudioMultistream}
helperText={
audioMultistreams === null
@ -335,7 +335,7 @@ const ChannelAbout = () => {
}
updateCallback={handleAudioUpdateConfig}
resetCallback={() => {
handleAudioUpdateConfig('audio_multistream', null);
handleAudioUpdateConfig('audio_multistreams', null);
setAudioMultistreams(null);
}}
/>
@ -349,6 +349,11 @@ const ChannelAbout = () => {
<p>Audio Languages</p>
</div>
<AudioLanguageSelector
key={[
'audio_languages',
audioLanguages ?? '',
channel.channel_overwrites?.audio_languages ?? '',
].join(':')}
name="audio_languages"
value={audioLanguages}
oldValue={channel.channel_overwrites?.audio_languages ?? null}

2
frontend/src/pages/Channels.tsx
View File

@ -20,7 +20,7 @@ import { ViewStylesEnum, ViewStylesType } from '../configuration/constants/ViewS
type ChannelOverwritesType = {
download_format: string | null;
audio_multistream: boolean | null;
audio_multistreams: boolean | null;
audio_languages: string | null;
autodelete_days: number | null;
index_playlists: boolean | null;

9
frontend/src/pages/SettingsApplication.tsx
View File

@ -116,7 +116,7 @@ const SettingsApplication = () => {
setDownloadsFormatSort(appSettingsConfigData?.downloads.format_sort || null);
setDownloadsExtractorLang(appSettingsConfigData?.downloads.extractor_lang || null);
setEmbedMetadata(appSettingsConfigData?.downloads.add_metadata || false);
setAudioMultistreams(appSettingsConfigData?.downloads.audio_multistream || false);
setAudioMultistreams(appSettingsConfigData?.downloads.audio_multistreams || false);
setAudioLanguages(appSettingsConfigData?.downloads.audio_languages || null);
setAudioMultistreamsWarning(null);
@ -550,7 +550,7 @@ const SettingsApplication = () => {
<p>Enable multistream audio</p>
</div>
<ToggleConfig
name="downloads.audio_multistream"
name="downloads.audio_multistreams"
value={audioMultistreams}
helperText={'Enable to include multiple audio languages when available.'}
updateCallback={handleAudioUpdateConfig}
@ -565,6 +565,11 @@ const SettingsApplication = () => {
<p>Audio languages</p>
</div>
<AudioLanguageSelector
key={[
'downloads.audio_languages',
audioLanguages ?? '',
appSettingsConfig.downloads.audio_languages ?? '',
].join(':')}
name="downloads.audio_languages"
value={audioLanguages}
oldValue={appSettingsConfig.downloads.audio_languages}

2
frontend/src/stores/AppSettingsStore.ts
View File

@ -34,7 +34,7 @@ export const useAppSettingsStore = create<AppSettingsState>(set => ({
extractor_lang: null,
integrate_ryd: false,
integrate_sponsorblock: false,
audio_multistream: false,
audio_multistreams: false,
audio_languages: null,
},
application: {