diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d9311fd8..ea60d380 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -134,6 +134,7 @@ The documentation available at [docs.tubearchivist.com](https://docs.tubearchivi This codebase is set up to be developed natively outside of docker as well as in a docker container. Developing outside of a docker container can be convenient, as IDE and hot reload usually works out of the box. But testing inside of a container is still essential, as there are subtle differences, especially when working with the filesystem and networking between containers. Note: +- This project doesn't look for contributions to this to cover additional dev setup environments from new contributors. If you are a regular contributor and you see ways to improve this, please reach out on Discord first. - Subtitles currently fail to load with `DJANGO_DEBUG=True`, that is due to incorrect `Content-Type` error set by Django's static file implementation. That's only if you run the Django dev server, Nginx sets the correct headers in the container. ### Native Instruction diff --git a/backend/appsettings/serializers_mb.py b/backend/appsettings/serializers_mb.py index 18bbba4f..330fc96d 100644 --- a/backend/appsettings/serializers_mb.py +++ b/backend/appsettings/serializers_mb.py @@ -29,3 +29,4 @@ class MembershipProfileSerializer(serializers.Serializer): sponsor_tier = SponsortierSerializer() subscription_count = serializers.IntegerField() subscription_is_max = serializers.BooleanField() + is_connected = serializers.BooleanField() diff --git a/backend/appsettings/src/backup.py b/backend/appsettings/src/backup.py index 702acd99..c4569196 100644 --- a/backend/appsettings/src/backup.py +++ b/backend/appsettings/src/backup.py @@ -21,7 +21,7 @@ class ElasticBackup: """dump index to nd-json files for later bulk import""" INDEX_SIZE_CONF = { - "comment": 200, + "comment": 100, "subtitle": 10000, } CACHE_DIR = EnvironmentSettings.CACHE_DIR @@ -64,6 +64,7 @@ class ElasticBackup: "callback": BackupCallback, "task": self.task, "total": self._get_total(index_name), + "timeout": 30, } if size_overwrite := self.INDEX_SIZE_CONF.get(index_name): diff --git a/backend/common/src/es_connect.py b/backend/common/src/es_connect.py index ecd46661..6638ffc8 100644 --- a/backend/common/src/es_connect.py +++ b/backend/common/src/es_connect.py @@ -148,6 +148,7 @@ class IndexPaginate: - callback: obj, Class implementing run method callback for every loop - task: task object to send notification - total: int, total items in index for progress message + - timeout: int, overwrite timeout in get request """ DEFAULT_SIZE = 500 @@ -191,7 +192,11 @@ class IndexPaginate: all_results = [] counter = 0 while True: - response, _ = ElasticWrap("_search").get(data=self.data) + get_kwargs = {"data": self.data} + if timeout_overwrite := self.kwargs.get("timeout"): + get_kwargs.update({"timeout": timeout_overwrite}) + + response, _ = ElasticWrap("_search").get(**get_kwargs) all_hits = response["hits"]["hits"] if not all_hits: break diff --git a/backend/common/src/helper.py b/backend/common/src/helper.py index 0bb43aca..c6f0f06c 100644 --- a/backend/common/src/helper.py +++ b/backend/common/src/helper.py @@ -103,13 +103,15 @@ def requests_headers() -> dict[str, str]: return {"User-Agent": template} -def date_parser(timestamp: int | str | None) -> str | None: +def date_parser(timestamp: int | float | str | None) -> str | None: """return formatted date string""" if timestamp is None: return None if isinstance(timestamp, int): date_obj = datetime.fromtimestamp(timestamp, tz=timezone.utc) + elif isinstance(timestamp, float): + date_obj = datetime.fromtimestamp(int(timestamp), tz=timezone.utc) elif isinstance(timestamp, str) and timestamp.isdigit(): date_obj = datetime.fromtimestamp(int(timestamp), tz=timezone.utc) elif isinstance(timestamp, str): diff --git a/backend/common/tests/test_src/test_helper.py b/backend/common/tests/test_src/test_helper.py index 73578a76..acc81e46 100644 --- a/backend/common/tests/test_src/test_helper.py +++ b/backend/common/tests/test_src/test_helper.py @@ -33,6 +33,13 @@ def test_date_parser_with_digit(): assert date_parser(timestamp) == expected_date +def test_date_parser_with_float(): + """iso timestamp""" + date_float = 1766210400.0 + expected_date = "2025-12-20T06:00:00+00:00" + assert date_parser(date_float) == expected_date + + def test_date_parser_with_str(): """iso timestamp""" date_str = "2021-05-21" diff --git a/backend/config/management/commands/ta_startup.py b/backend/config/management/commands/ta_startup.py index dc5e8aa2..57b1691e 100644 --- a/backend/config/management/commands/ta_startup.py +++ b/backend/config/management/commands/ta_startup.py @@ -376,14 +376,15 @@ class Command(BaseCommand): "lang": "painless", }, ) + source = f""" + if (ctx._source.containsKey('channel')) + {{ctx._source.channel.remove('{field}');}} + """ self._run_migration( index_name="ta_video", desc=f"fix missing data type for field channel.{field}", query={"term": {f"channel.{field}": {"value": False}}}, - script={ - "source": f"ctx._source.remove('channel.{field}')", - "lang": "painless", - }, + script={"source": source, "lang": "painless"}, ) def _mig_fix_channel_description(self) -> None: diff --git a/backend/download/src/yt_dlp_base.py b/backend/download/src/yt_dlp_base.py index 212401d2..383d9166 100644 --- a/backend/download/src/yt_dlp_base.py +++ b/backend/download/src/yt_dlp_base.py @@ -89,6 +89,17 @@ class YtWrap: } }, ) + return + + # https://github.com/Brainicism/bgutil-ytdlp-pot-provider/pull/185 + deep_merge( + self.obs, + { + "extractor_args": { + "youtubepot-bgutilhttp": {"disable": ["True"]} + } + }, + ) def download(self, url): """make download request""" diff --git a/backend/requirements.txt b/backend/requirements.txt index 6408e29f..5bde3ac0 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,5 +1,5 @@ apprise==1.9.7 -bgutil-ytdlp-pot-provider==1.2.2 +bgutil-ytdlp-pot-provider @ git+https://github.com/bbilly1/bgutil-ytdlp-pot-provider@c3f7bf4c99f8eb62015cb1c4142324f3d681cf07#subdirectory=plugin celery==5.6.2 deepdiff==8.6.1 django-auth-ldap==5.3.0 @@ -13,4 +13,4 @@ redis==7.1.0 requests==2.32.5 ryd-client==0.0.6 uvicorn==0.40.0 -yt-dlp[default] @ git+https://github.com/yt-dlp/yt-dlp@23b846506378a6a9c9a0958382d37f943f7cfa51 +yt-dlp[default]==2026.1.29 diff --git a/frontend/src/components/MembershipAppsettings.tsx b/frontend/src/components/MembershipAppsettings.tsx index 9474c689..d2899da0 100644 --- a/frontend/src/components/MembershipAppsettings.tsx +++ b/frontend/src/components/MembershipAppsettings.tsx @@ -24,6 +24,7 @@ type ProfileResponseType = { sponsor_tier: SponsorTierType; subscription_count: number; subscription_is_max: boolean; + is_connected: boolean; }; export default function MembershipAppsettings({ show_help_text }: { show_help_text: boolean }) { @@ -136,6 +137,13 @@ export default function MembershipAppsettings({ show_help_text }: { show_help_te .
  • Click on validate to verify everything is working.
    • +
  • + Setup the{' '} + + Client Container + + . +
  • If you are subscribed to less channels than your sponsor tier allows, you can directly sync all your subscriptions here. @@ -206,6 +214,19 @@ export default function MembershipAppsettings({ show_help_text }: { show_help_te
    Subscriptions: {profileResponse.subscription_count}/ {profileResponse.sponsor_tier.max_subs} +
    + Socket:{' '} + {profileResponse.is_connected ? ( + 'Established' + ) : ( + <> + Not established. Make sure the{' '} + + Client Container + {' '} + is connected. + + )}

    )}