From 28f7aec2af51a4c87e76ebf856f948dd071932e3 Mon Sep 17 00:00:00 2001 From: codefiles <11915375+codefiles@users.noreply.github.com> Date: Mon, 26 Jan 2026 19:23:01 -0500 Subject: [PATCH] Refactor mirror_list_handler to use DI (#4170) --- archinstall/lib/global_menu.py | 17 +- archinstall/lib/installer.py | 4 +- archinstall/lib/mirrors.py | 320 +++++++++++++++--------------- archinstall/lib/models/mirrors.py | 23 ++- archinstall/scripts/guided.py | 27 ++- 5 files changed, 214 insertions(+), 177 deletions(-) diff --git a/archinstall/lib/global_menu.py b/archinstall/lib/global_menu.py index 40e114f8..70a5e35d 100644 --- a/archinstall/lib/global_menu.py +++ b/archinstall/lib/global_menu.py @@ -25,7 +25,7 @@ from .interactions.network_menu import ask_to_configure_network from .interactions.system_conf import ask_for_swap, select_kernel from .locale.locale_menu import LocaleMenu from .menu.abstract_menu import CONFIG_KEY, AbstractMenu -from .mirrors import MirrorMenu +from .mirrors import MirrorListHandler, MirrorMenu from .models.bootloader import Bootloader, BootloaderConfiguration from .models.locale import LocaleConfiguration from .models.mirrors import MirrorConfiguration @@ -38,8 +38,14 @@ from .translationhandler import Language, tr, translation_handler class GlobalMenu(AbstractMenu[None]): - def __init__(self, arch_config: ArchConfig, title: str | None = None) -> None: + def __init__( + self, + arch_config: ArchConfig, + mirror_list_handler: MirrorListHandler | None = None, + title: str | None = None, + ) -> None: self._arch_config = arch_config + self._mirror_list_handler = mirror_list_handler menu_options = self._get_menu_options() self._item_group = MenuItemGroup( @@ -550,7 +556,12 @@ class GlobalMenu(AbstractMenu[None]): return packages def _mirror_configuration(self, preset: MirrorConfiguration | None = None) -> MirrorConfiguration | None: - mirror_configuration = MirrorMenu(preset=preset).run() + if self._mirror_list_handler is not None: + mirror_list_handler = self._mirror_list_handler + else: + mirror_list_handler = MirrorListHandler() + + mirror_configuration = MirrorMenu(mirror_list_handler, preset=preset).run() if mirror_configuration and mirror_configuration.optional_repositories: # reset the package list cache in case the repository selection has changed diff --git a/archinstall/lib/installer.py b/archinstall/lib/installer.py index 1b96c155..90d8b9fb 100644 --- a/archinstall/lib/installer.py +++ b/archinstall/lib/installer.py @@ -42,6 +42,7 @@ from .general import SysCommand, run from .hardware import SysInfo from .locale.utils import verify_keyboard_layout, verify_x11_keyboard_layout from .luks import Luks2 +from .mirrors import MirrorListHandler from .models.bootloader import Bootloader from .models.locale import LocaleConfiguration from .models.mirrors import MirrorConfiguration @@ -528,6 +529,7 @@ class Installer: def set_mirrors( self, + mirror_list_handler: MirrorListHandler, mirror_config: MirrorConfiguration, on_target: bool = False, ) -> None: @@ -558,7 +560,7 @@ class Installer: with open(pacman_config, 'a') as fp: fp.write(repositories_config) - regions_config = mirror_config.regions_config(speed_sort=True) + regions_config = mirror_config.regions_config(mirror_list_handler, speed_sort=True) if regions_config: debug(f'Mirrorlist:\n{regions_config}') mirrorlist_config.write_text(regions_config) diff --git a/archinstall/lib/mirrors.py b/archinstall/lib/mirrors.py index 72c2f81a..d683da16 100644 --- a/archinstall/lib/mirrors.py +++ b/archinstall/lib/mirrors.py @@ -207,9 +207,165 @@ class CustomMirrorServersList(ListManager[CustomServer]): return None +class MirrorListHandler: + def __init__( + self, + local_mirrorlist: Path = Path('/etc/pacman.d/mirrorlist'), + offline: bool = False, + verbose: bool = False, + ) -> None: + self._local_mirrorlist = local_mirrorlist + self._status_mappings: dict[str, list[MirrorStatusEntryV3]] | None = None + self._fetched_remote: bool = False + self.offline = offline + self.verbose = verbose + + def _mappings(self) -> dict[str, list[MirrorStatusEntryV3]]: + if self._status_mappings is None: + self.load_mirrors() + + assert self._status_mappings is not None + return self._status_mappings + + def get_mirror_regions(self) -> list[MirrorRegion]: + available_mirrors = [] + mappings = self._mappings() + + for region_name, status_entry in mappings.items(): + urls = [entry.server_url for entry in status_entry] + region = MirrorRegion(region_name, urls) + available_mirrors.append(region) + + return available_mirrors + + def load_mirrors(self) -> None: + if self.offline: + self._fetched_remote = False + self.load_local_mirrors() + else: + self._fetched_remote = self.load_remote_mirrors() + debug(f'load mirrors: {self._fetched_remote}') + if not self._fetched_remote: + self.load_local_mirrors() + + def load_remote_mirrors(self) -> bool: + url = 'https://archlinux.org/mirrors/status/json/' + attempts = 3 + + for attempt_nr in range(attempts): + try: + mirrorlist = fetch_data_from_url(url) + self._status_mappings = self._parse_remote_mirror_list(mirrorlist) + return True + except Exception as e: + debug(f'Error while fetching mirror list: {e}') + time.sleep(attempt_nr + 1) + + debug('Unable to fetch mirror list remotely, falling back to local mirror list') + return False + + def load_local_mirrors(self) -> None: + with self._local_mirrorlist.open('r') as fp: + mirrorlist = fp.read() + self._status_mappings = self._parse_locale_mirrors(mirrorlist) + + def get_status_by_region(self, region: str, speed_sort: bool) -> list[MirrorStatusEntryV3]: + mappings = self._mappings() + region_list = mappings[region] + + # Only sort if we have remote mirror data with score/speed info + # Local mirrors lack this data and can be modified manually before-hand + # Or reflector potentially ran already + if self._fetched_remote and speed_sort: + info('Sorting your selected mirror list based on the speed between you and the individual mirrors (this might take a while)') + # Sort by speed descending (higher is better in bitrate form core.db download) + return sorted(region_list, key=lambda mirror: -mirror.speed) + # just return as-is without sorting? + return region_list + + def _parse_remote_mirror_list(self, mirrorlist: str) -> dict[str, list[MirrorStatusEntryV3]]: + context = {'verbose': self.verbose} + mirror_status = MirrorStatusListV3.model_validate_json(mirrorlist, context=context) + + sorting_placeholder: dict[str, list[MirrorStatusEntryV3]] = {} + + for mirror in mirror_status.urls: + # We filter out mirrors that have bad criteria values + if any( + [ + mirror.active is False, # Disabled by mirror-list admins + mirror.last_sync is None, # Has not synced recently + # mirror.score (error rate) over time reported from backend: + # https://github.com/archlinux/archweb/blob/31333d3516c91db9a2f2d12260bd61656c011fd1/mirrors/utils.py#L111C22-L111C66 + (mirror.score is None or mirror.score >= 100), + ] + ): + continue + + if mirror.country == '': + # TODO: This should be removed once RFC!29 is merged and completed + # Until then, there are mirrors which lacks data in the backend + # and there is no way of knowing where they're located. + # So we have to assume world-wide + mirror.country = 'Worldwide' + + if mirror.url.startswith('http'): + sorting_placeholder.setdefault(mirror.country, []).append(mirror) + + sorted_by_regions: dict[str, list[MirrorStatusEntryV3]] = dict( + {region: unsorted_mirrors for region, unsorted_mirrors in sorted(sorting_placeholder.items(), key=lambda item: item[0])} + ) + + return sorted_by_regions + + def _parse_locale_mirrors(self, mirrorlist: str) -> dict[str, list[MirrorStatusEntryV3]]: + lines = mirrorlist.splitlines() + + # remove empty lines + # lines = [line for line in lines if line] + + mirror_list: dict[str, list[MirrorStatusEntryV3]] = {} + + current_region = '' + + for line in lines: + line = line.strip() + + if line.startswith('## '): + current_region = line.replace('## ', '').strip() + mirror_list.setdefault(current_region, []) + + if line.startswith('Server = '): + if not current_region: + current_region = 'Local' + mirror_list.setdefault(current_region, []) + + url = line.removeprefix('Server = ') + + mirror_entry = MirrorStatusEntryV3( + url=url.removesuffix('$repo/os/$arch'), + protocol=urllib.parse.urlparse(url).scheme, + active=True, + country=current_region or 'Worldwide', + # The following values are normally populated by + # archlinux.org mirror-list endpoint, and can't be known + # from just the local mirror-list file. + country_code='WW', + isos=True, + ipv4=True, + ipv6=True, + details='Locally defined mirror', + ) + + mirror_list[current_region].append(mirror_entry) + + return mirror_list + + class MirrorMenu(AbstractSubMenu[MirrorConfiguration]): def __init__( self, + mirror_list_handler: MirrorListHandler, preset: MirrorConfiguration | None = None, ): if preset: @@ -217,6 +373,8 @@ class MirrorMenu(AbstractSubMenu[MirrorConfiguration]): else: self._mirror_config = MirrorConfiguration() + self._mirror_list_handler = mirror_list_handler + menu_options = self._define_menu_options() self._item_group = MenuItemGroup(menu_options, checkmarks=True) @@ -230,7 +388,7 @@ class MirrorMenu(AbstractSubMenu[MirrorConfiguration]): return [ MenuItem( text=tr('Select regions'), - action=select_mirror_regions, + action=lambda x: select_mirror_regions(self._mirror_list_handler, x), value=self._mirror_config.mirror_regions, preview_action=self._prev_regions, key='mirror_regions', @@ -300,7 +458,10 @@ class MirrorMenu(AbstractSubMenu[MirrorConfiguration]): return super().run() -def select_mirror_regions(preset: list[MirrorRegion]) -> list[MirrorRegion]: +def select_mirror_regions( + mirror_list_handler: MirrorListHandler, + preset: list[MirrorRegion], +) -> list[MirrorRegion]: Loading[None]( header=tr('Loading mirror regions...'), data_callback=mirror_list_handler.load_mirrors, @@ -388,158 +549,3 @@ def select_optional_repositories(preset: list[Repository]) -> list[Repository]: return [] case ResultType.Selection: return result.get_values() - - -class MirrorListHandler: - def __init__( - self, - local_mirrorlist: Path = Path('/etc/pacman.d/mirrorlist'), - ) -> None: - self._local_mirrorlist = local_mirrorlist - self._status_mappings: dict[str, list[MirrorStatusEntryV3]] | None = None - self._fetched_remote: bool = False - - def _mappings(self) -> dict[str, list[MirrorStatusEntryV3]]: - if self._status_mappings is None: - self.load_mirrors() - - assert self._status_mappings is not None - return self._status_mappings - - def get_mirror_regions(self) -> list[MirrorRegion]: - available_mirrors = [] - mappings = self._mappings() - - for region_name, status_entry in mappings.items(): - urls = [entry.server_url for entry in status_entry] - region = MirrorRegion(region_name, urls) - available_mirrors.append(region) - - return available_mirrors - - def load_mirrors(self) -> None: - from .args import arch_config_handler - - if arch_config_handler.args.offline: - self._fetched_remote = False - self.load_local_mirrors() - else: - self._fetched_remote = self.load_remote_mirrors() - debug(f'load mirrors: {self._fetched_remote}') - if not self._fetched_remote: - self.load_local_mirrors() - - def load_remote_mirrors(self) -> bool: - url = 'https://archlinux.org/mirrors/status/json/' - attempts = 3 - - for attempt_nr in range(attempts): - try: - mirrorlist = fetch_data_from_url(url) - self._status_mappings = self._parse_remote_mirror_list(mirrorlist) - return True - except Exception as e: - debug(f'Error while fetching mirror list: {e}') - time.sleep(attempt_nr + 1) - - debug('Unable to fetch mirror list remotely, falling back to local mirror list') - return False - - def load_local_mirrors(self) -> None: - with self._local_mirrorlist.open('r') as fp: - mirrorlist = fp.read() - self._status_mappings = self._parse_locale_mirrors(mirrorlist) - - def get_status_by_region(self, region: str, speed_sort: bool) -> list[MirrorStatusEntryV3]: - mappings = self._mappings() - region_list = mappings[region] - - # Only sort if we have remote mirror data with score/speed info - # Local mirrors lack this data and can be modified manually before-hand - # Or reflector potentially ran already - if self._fetched_remote and speed_sort: - info('Sorting your selected mirror list based on the speed between you and the individual mirrors (this might take a while)') - # Sort by speed descending (higher is better in bitrate form core.db download) - return sorted(region_list, key=lambda mirror: -mirror.speed) - # just return as-is without sorting? - return region_list - - def _parse_remote_mirror_list(self, mirrorlist: str) -> dict[str, list[MirrorStatusEntryV3]]: - mirror_status = MirrorStatusListV3.model_validate_json(mirrorlist) - - sorting_placeholder: dict[str, list[MirrorStatusEntryV3]] = {} - - for mirror in mirror_status.urls: - # We filter out mirrors that have bad criteria values - if any( - [ - mirror.active is False, # Disabled by mirror-list admins - mirror.last_sync is None, # Has not synced recently - # mirror.score (error rate) over time reported from backend: - # https://github.com/archlinux/archweb/blob/31333d3516c91db9a2f2d12260bd61656c011fd1/mirrors/utils.py#L111C22-L111C66 - (mirror.score is None or mirror.score >= 100), - ] - ): - continue - - if mirror.country == '': - # TODO: This should be removed once RFC!29 is merged and completed - # Until then, there are mirrors which lacks data in the backend - # and there is no way of knowing where they're located. - # So we have to assume world-wide - mirror.country = 'Worldwide' - - if mirror.url.startswith('http'): - sorting_placeholder.setdefault(mirror.country, []).append(mirror) - - sorted_by_regions: dict[str, list[MirrorStatusEntryV3]] = dict( - {region: unsorted_mirrors for region, unsorted_mirrors in sorted(sorting_placeholder.items(), key=lambda item: item[0])} - ) - - return sorted_by_regions - - def _parse_locale_mirrors(self, mirrorlist: str) -> dict[str, list[MirrorStatusEntryV3]]: - lines = mirrorlist.splitlines() - - # remove empty lines - # lines = [line for line in lines if line] - - mirror_list: dict[str, list[MirrorStatusEntryV3]] = {} - - current_region = '' - - for line in lines: - line = line.strip() - - if line.startswith('## '): - current_region = line.replace('## ', '').strip() - mirror_list.setdefault(current_region, []) - - if line.startswith('Server = '): - if not current_region: - current_region = 'Local' - mirror_list.setdefault(current_region, []) - - url = line.removeprefix('Server = ') - - mirror_entry = MirrorStatusEntryV3( - url=url.removesuffix('$repo/os/$arch'), - protocol=urllib.parse.urlparse(url).scheme, - active=True, - country=current_region or 'Worldwide', - # The following values are normally populated by - # archlinux.org mirror-list endpoint, and can't be known - # from just the local mirror-list file. - country_code='WW', - isos=True, - ipv4=True, - ipv6=True, - details='Locally defined mirror', - ) - - mirror_list[current_region].append(mirror_entry) - - return mirror_list - - -mirror_list_handler = MirrorListHandler() diff --git a/archinstall/lib/models/mirrors.py b/archinstall/lib/models/mirrors.py index 7768c50b..4cd09acd 100644 --- a/archinstall/lib/models/mirrors.py +++ b/archinstall/lib/models/mirrors.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datetime import http.client import urllib.error @@ -5,14 +7,17 @@ import urllib.parse import urllib.request from dataclasses import dataclass, field from enum import Enum -from typing import Any, Self, TypedDict, override +from typing import TYPE_CHECKING, Any, Self, TypedDict, override -from pydantic import BaseModel, field_validator, model_validator +from pydantic import BaseModel, ValidationInfo, field_validator, model_validator from ..models.packages import Repository from ..networking import DownloadTimer, ping from ..output import debug +if TYPE_CHECKING: + from ..mirrors import MirrorListHandler + class MirrorStatusEntryV3(BaseModel): url: str @@ -104,13 +109,11 @@ class MirrorStatusEntryV3(BaseModel): return value @model_validator(mode='after') - def debug_output(self) -> Self: - from ..args import arch_config_handler - + def debug_output(self, info: ValidationInfo) -> Self: self._hostname, *port = urllib.parse.urlparse(self.url).netloc.split(':', 1) self._port = int(port[0]) if port and len(port) >= 1 else None - if arch_config_handler.args.verbose: + if (ctx := info.context) and ctx.get('verbose'): debug(f'Loaded mirror {self._hostname}' + (f' with current score of {self.score}' if self.score else '')) return self @@ -271,9 +274,11 @@ class MirrorConfiguration: return config.strip() - def regions_config(self, speed_sort: bool = True) -> str: - from ..mirrors import mirror_list_handler - + def regions_config( + self, + mirror_list_handler: MirrorListHandler, + speed_sort: bool = True, + ) -> str: config = '' for mirror_region in self.mirror_regions: diff --git a/archinstall/scripts/guided.py b/archinstall/scripts/guided.py index df60c8fa..ffc80f52 100644 --- a/archinstall/scripts/guided.py +++ b/archinstall/scripts/guided.py @@ -12,6 +12,7 @@ from archinstall.lib.global_menu import GlobalMenu from archinstall.lib.hardware import SysInfo from archinstall.lib.installer import Installer, accessibility_tools_in_use, run_custom_user_commands from archinstall.lib.interactions.general_conf import PostInstallationAction, ask_post_installation +from archinstall.lib.mirrors import MirrorListHandler from archinstall.lib.models import Bootloader from archinstall.lib.models.device import ( DiskLayoutType, @@ -24,7 +25,7 @@ from archinstall.lib.profile.profiles_handler import profile_handler from archinstall.lib.translationhandler import tr -def ask_user_questions() -> None: +def ask_user_questions(mirror_list_handler: MirrorListHandler) -> None: """ First, we'll ask the user for a bunch of user input. Not until we're satisfied with what we want to install @@ -38,7 +39,11 @@ def ask_user_questions() -> None: text = tr('New version available') + f': {upgrade}' title_text += f' ({text})' - global_menu = GlobalMenu(arch_config_handler.config, title=title_text) + global_menu = GlobalMenu( + arch_config_handler.config, + mirror_list_handler, + title=title_text, + ) if not arch_config_handler.args.advanced: global_menu.set_enabled('parallel_downloads', False) @@ -46,7 +51,10 @@ def ask_user_questions() -> None: global_menu.run() -def perform_installation(mountpoint: Path) -> None: +def perform_installation( + mountpoint: Path, + mirror_list_handler: MirrorListHandler, +) -> None: """ Performs the installation steps on a block device. Only requirement is that the block devices are @@ -84,7 +92,7 @@ def perform_installation(mountpoint: Path) -> None: installation.generate_key_files() if mirror_config := config.mirror_config: - installation.set_mirrors(mirror_config, on_target=False) + installation.set_mirrors(mirror_list_handler, mirror_config, on_target=False) installation.minimal_installation( optional_repositories=optional_repositories, @@ -94,7 +102,7 @@ def perform_installation(mountpoint: Path) -> None: ) if mirror_config := config.mirror_config: - installation.set_mirrors(mirror_config, on_target=True) + installation.set_mirrors(mirror_list_handler, mirror_config, on_target=True) if config.swap and config.swap.enabled: installation.setup_swap('zram', algo=config.swap.algorithm) @@ -183,8 +191,13 @@ def perform_installation(mountpoint: Path) -> None: def guided() -> None: + mirror_list_handler = MirrorListHandler( + offline=arch_config_handler.args.offline, + verbose=arch_config_handler.args.verbose, + ) + if not arch_config_handler.args.silent: - ask_user_questions() + ask_user_questions(mirror_list_handler) config = ConfigurationOutput(arch_config_handler.config) config.write_debug() @@ -206,7 +219,7 @@ def guided() -> None: fs_handler = FilesystemHandler(arch_config_handler.config.disk_config) fs_handler.perform_filesystem_operations() - perform_installation(arch_config_handler.args.mountpoint) + perform_installation(arch_config_handler.args.mountpoint, mirror_list_handler) guided()