Refactor mirror_list_handler to use DI (#4170)

This commit is contained in:
codefiles 2026-01-26 19:23:01 -05:00 committed by GitHub
parent a025c79ac3
commit 28f7aec2af
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 214 additions and 177 deletions

View File

@ -25,7 +25,7 @@ from .interactions.network_menu import ask_to_configure_network
from .interactions.system_conf import ask_for_swap, select_kernel
from .locale.locale_menu import LocaleMenu
from .menu.abstract_menu import CONFIG_KEY, AbstractMenu
from .mirrors import MirrorMenu
from .mirrors import MirrorListHandler, MirrorMenu
from .models.bootloader import Bootloader, BootloaderConfiguration
from .models.locale import LocaleConfiguration
from .models.mirrors import MirrorConfiguration
@ -38,8 +38,14 @@ from .translationhandler import Language, tr, translation_handler
class GlobalMenu(AbstractMenu[None]):
def __init__(self, arch_config: ArchConfig, title: str | None = None) -> None:
def __init__(
self,
arch_config: ArchConfig,
mirror_list_handler: MirrorListHandler | None = None,
title: str | None = None,
) -> None:
self._arch_config = arch_config
self._mirror_list_handler = mirror_list_handler
menu_options = self._get_menu_options()
self._item_group = MenuItemGroup(
@ -550,7 +556,12 @@ class GlobalMenu(AbstractMenu[None]):
return packages
def _mirror_configuration(self, preset: MirrorConfiguration | None = None) -> MirrorConfiguration | None:
mirror_configuration = MirrorMenu(preset=preset).run()
if self._mirror_list_handler is not None:
mirror_list_handler = self._mirror_list_handler
else:
mirror_list_handler = MirrorListHandler()
mirror_configuration = MirrorMenu(mirror_list_handler, preset=preset).run()
if mirror_configuration and mirror_configuration.optional_repositories:
# reset the package list cache in case the repository selection has changed

View File

@ -42,6 +42,7 @@ from .general import SysCommand, run
from .hardware import SysInfo
from .locale.utils import verify_keyboard_layout, verify_x11_keyboard_layout
from .luks import Luks2
from .mirrors import MirrorListHandler
from .models.bootloader import Bootloader
from .models.locale import LocaleConfiguration
from .models.mirrors import MirrorConfiguration
@ -528,6 +529,7 @@ class Installer:
def set_mirrors(
self,
mirror_list_handler: MirrorListHandler,
mirror_config: MirrorConfiguration,
on_target: bool = False,
) -> None:
@ -558,7 +560,7 @@ class Installer:
with open(pacman_config, 'a') as fp:
fp.write(repositories_config)
regions_config = mirror_config.regions_config(speed_sort=True)
regions_config = mirror_config.regions_config(mirror_list_handler, speed_sort=True)
if regions_config:
debug(f'Mirrorlist:\n{regions_config}')
mirrorlist_config.write_text(regions_config)

View File

@ -207,9 +207,165 @@ class CustomMirrorServersList(ListManager[CustomServer]):
return None
class MirrorListHandler:
def __init__(
self,
local_mirrorlist: Path = Path('/etc/pacman.d/mirrorlist'),
offline: bool = False,
verbose: bool = False,
) -> None:
self._local_mirrorlist = local_mirrorlist
self._status_mappings: dict[str, list[MirrorStatusEntryV3]] | None = None
self._fetched_remote: bool = False
self.offline = offline
self.verbose = verbose
def _mappings(self) -> dict[str, list[MirrorStatusEntryV3]]:
if self._status_mappings is None:
self.load_mirrors()
assert self._status_mappings is not None
return self._status_mappings
def get_mirror_regions(self) -> list[MirrorRegion]:
available_mirrors = []
mappings = self._mappings()
for region_name, status_entry in mappings.items():
urls = [entry.server_url for entry in status_entry]
region = MirrorRegion(region_name, urls)
available_mirrors.append(region)
return available_mirrors
def load_mirrors(self) -> None:
if self.offline:
self._fetched_remote = False
self.load_local_mirrors()
else:
self._fetched_remote = self.load_remote_mirrors()
debug(f'load mirrors: {self._fetched_remote}')
if not self._fetched_remote:
self.load_local_mirrors()
def load_remote_mirrors(self) -> bool:
url = 'https://archlinux.org/mirrors/status/json/'
attempts = 3
for attempt_nr in range(attempts):
try:
mirrorlist = fetch_data_from_url(url)
self._status_mappings = self._parse_remote_mirror_list(mirrorlist)
return True
except Exception as e:
debug(f'Error while fetching mirror list: {e}')
time.sleep(attempt_nr + 1)
debug('Unable to fetch mirror list remotely, falling back to local mirror list')
return False
def load_local_mirrors(self) -> None:
with self._local_mirrorlist.open('r') as fp:
mirrorlist = fp.read()
self._status_mappings = self._parse_locale_mirrors(mirrorlist)
def get_status_by_region(self, region: str, speed_sort: bool) -> list[MirrorStatusEntryV3]:
mappings = self._mappings()
region_list = mappings[region]
# Only sort if we have remote mirror data with score/speed info
# Local mirrors lack this data and can be modified manually before-hand
# Or reflector potentially ran already
if self._fetched_remote and speed_sort:
info('Sorting your selected mirror list based on the speed between you and the individual mirrors (this might take a while)')
# Sort by speed descending (higher is better in bitrate form core.db download)
return sorted(region_list, key=lambda mirror: -mirror.speed)
# just return as-is without sorting?
return region_list
def _parse_remote_mirror_list(self, mirrorlist: str) -> dict[str, list[MirrorStatusEntryV3]]:
context = {'verbose': self.verbose}
mirror_status = MirrorStatusListV3.model_validate_json(mirrorlist, context=context)
sorting_placeholder: dict[str, list[MirrorStatusEntryV3]] = {}
for mirror in mirror_status.urls:
# We filter out mirrors that have bad criteria values
if any(
[
mirror.active is False, # Disabled by mirror-list admins
mirror.last_sync is None, # Has not synced recently
# mirror.score (error rate) over time reported from backend:
# https://github.com/archlinux/archweb/blob/31333d3516c91db9a2f2d12260bd61656c011fd1/mirrors/utils.py#L111C22-L111C66
(mirror.score is None or mirror.score >= 100),
]
):
continue
if mirror.country == '':
# TODO: This should be removed once RFC!29 is merged and completed
# Until then, there are mirrors which lacks data in the backend
# and there is no way of knowing where they're located.
# So we have to assume world-wide
mirror.country = 'Worldwide'
if mirror.url.startswith('http'):
sorting_placeholder.setdefault(mirror.country, []).append(mirror)
sorted_by_regions: dict[str, list[MirrorStatusEntryV3]] = dict(
{region: unsorted_mirrors for region, unsorted_mirrors in sorted(sorting_placeholder.items(), key=lambda item: item[0])}
)
return sorted_by_regions
def _parse_locale_mirrors(self, mirrorlist: str) -> dict[str, list[MirrorStatusEntryV3]]:
lines = mirrorlist.splitlines()
# remove empty lines
# lines = [line for line in lines if line]
mirror_list: dict[str, list[MirrorStatusEntryV3]] = {}
current_region = ''
for line in lines:
line = line.strip()
if line.startswith('## '):
current_region = line.replace('## ', '').strip()
mirror_list.setdefault(current_region, [])
if line.startswith('Server = '):
if not current_region:
current_region = 'Local'
mirror_list.setdefault(current_region, [])
url = line.removeprefix('Server = ')
mirror_entry = MirrorStatusEntryV3(
url=url.removesuffix('$repo/os/$arch'),
protocol=urllib.parse.urlparse(url).scheme,
active=True,
country=current_region or 'Worldwide',
# The following values are normally populated by
# archlinux.org mirror-list endpoint, and can't be known
# from just the local mirror-list file.
country_code='WW',
isos=True,
ipv4=True,
ipv6=True,
details='Locally defined mirror',
)
mirror_list[current_region].append(mirror_entry)
return mirror_list
class MirrorMenu(AbstractSubMenu[MirrorConfiguration]):
def __init__(
self,
mirror_list_handler: MirrorListHandler,
preset: MirrorConfiguration | None = None,
):
if preset:
@ -217,6 +373,8 @@ class MirrorMenu(AbstractSubMenu[MirrorConfiguration]):
else:
self._mirror_config = MirrorConfiguration()
self._mirror_list_handler = mirror_list_handler
menu_options = self._define_menu_options()
self._item_group = MenuItemGroup(menu_options, checkmarks=True)
@ -230,7 +388,7 @@ class MirrorMenu(AbstractSubMenu[MirrorConfiguration]):
return [
MenuItem(
text=tr('Select regions'),
action=select_mirror_regions,
action=lambda x: select_mirror_regions(self._mirror_list_handler, x),
value=self._mirror_config.mirror_regions,
preview_action=self._prev_regions,
key='mirror_regions',
@ -300,7 +458,10 @@ class MirrorMenu(AbstractSubMenu[MirrorConfiguration]):
return super().run()
def select_mirror_regions(preset: list[MirrorRegion]) -> list[MirrorRegion]:
def select_mirror_regions(
mirror_list_handler: MirrorListHandler,
preset: list[MirrorRegion],
) -> list[MirrorRegion]:
Loading[None](
header=tr('Loading mirror regions...'),
data_callback=mirror_list_handler.load_mirrors,
@ -388,158 +549,3 @@ def select_optional_repositories(preset: list[Repository]) -> list[Repository]:
return []
case ResultType.Selection:
return result.get_values()
class MirrorListHandler:
def __init__(
self,
local_mirrorlist: Path = Path('/etc/pacman.d/mirrorlist'),
) -> None:
self._local_mirrorlist = local_mirrorlist
self._status_mappings: dict[str, list[MirrorStatusEntryV3]] | None = None
self._fetched_remote: bool = False
def _mappings(self) -> dict[str, list[MirrorStatusEntryV3]]:
if self._status_mappings is None:
self.load_mirrors()
assert self._status_mappings is not None
return self._status_mappings
def get_mirror_regions(self) -> list[MirrorRegion]:
available_mirrors = []
mappings = self._mappings()
for region_name, status_entry in mappings.items():
urls = [entry.server_url for entry in status_entry]
region = MirrorRegion(region_name, urls)
available_mirrors.append(region)
return available_mirrors
def load_mirrors(self) -> None:
from .args import arch_config_handler
if arch_config_handler.args.offline:
self._fetched_remote = False
self.load_local_mirrors()
else:
self._fetched_remote = self.load_remote_mirrors()
debug(f'load mirrors: {self._fetched_remote}')
if not self._fetched_remote:
self.load_local_mirrors()
def load_remote_mirrors(self) -> bool:
url = 'https://archlinux.org/mirrors/status/json/'
attempts = 3
for attempt_nr in range(attempts):
try:
mirrorlist = fetch_data_from_url(url)
self._status_mappings = self._parse_remote_mirror_list(mirrorlist)
return True
except Exception as e:
debug(f'Error while fetching mirror list: {e}')
time.sleep(attempt_nr + 1)
debug('Unable to fetch mirror list remotely, falling back to local mirror list')
return False
def load_local_mirrors(self) -> None:
with self._local_mirrorlist.open('r') as fp:
mirrorlist = fp.read()
self._status_mappings = self._parse_locale_mirrors(mirrorlist)
def get_status_by_region(self, region: str, speed_sort: bool) -> list[MirrorStatusEntryV3]:
mappings = self._mappings()
region_list = mappings[region]
# Only sort if we have remote mirror data with score/speed info
# Local mirrors lack this data and can be modified manually before-hand
# Or reflector potentially ran already
if self._fetched_remote and speed_sort:
info('Sorting your selected mirror list based on the speed between you and the individual mirrors (this might take a while)')
# Sort by speed descending (higher is better in bitrate form core.db download)
return sorted(region_list, key=lambda mirror: -mirror.speed)
# just return as-is without sorting?
return region_list
def _parse_remote_mirror_list(self, mirrorlist: str) -> dict[str, list[MirrorStatusEntryV3]]:
mirror_status = MirrorStatusListV3.model_validate_json(mirrorlist)
sorting_placeholder: dict[str, list[MirrorStatusEntryV3]] = {}
for mirror in mirror_status.urls:
# We filter out mirrors that have bad criteria values
if any(
[
mirror.active is False, # Disabled by mirror-list admins
mirror.last_sync is None, # Has not synced recently
# mirror.score (error rate) over time reported from backend:
# https://github.com/archlinux/archweb/blob/31333d3516c91db9a2f2d12260bd61656c011fd1/mirrors/utils.py#L111C22-L111C66
(mirror.score is None or mirror.score >= 100),
]
):
continue
if mirror.country == '':
# TODO: This should be removed once RFC!29 is merged and completed
# Until then, there are mirrors which lacks data in the backend
# and there is no way of knowing where they're located.
# So we have to assume world-wide
mirror.country = 'Worldwide'
if mirror.url.startswith('http'):
sorting_placeholder.setdefault(mirror.country, []).append(mirror)
sorted_by_regions: dict[str, list[MirrorStatusEntryV3]] = dict(
{region: unsorted_mirrors for region, unsorted_mirrors in sorted(sorting_placeholder.items(), key=lambda item: item[0])}
)
return sorted_by_regions
def _parse_locale_mirrors(self, mirrorlist: str) -> dict[str, list[MirrorStatusEntryV3]]:
lines = mirrorlist.splitlines()
# remove empty lines
# lines = [line for line in lines if line]
mirror_list: dict[str, list[MirrorStatusEntryV3]] = {}
current_region = ''
for line in lines:
line = line.strip()
if line.startswith('## '):
current_region = line.replace('## ', '').strip()
mirror_list.setdefault(current_region, [])
if line.startswith('Server = '):
if not current_region:
current_region = 'Local'
mirror_list.setdefault(current_region, [])
url = line.removeprefix('Server = ')
mirror_entry = MirrorStatusEntryV3(
url=url.removesuffix('$repo/os/$arch'),
protocol=urllib.parse.urlparse(url).scheme,
active=True,
country=current_region or 'Worldwide',
# The following values are normally populated by
# archlinux.org mirror-list endpoint, and can't be known
# from just the local mirror-list file.
country_code='WW',
isos=True,
ipv4=True,
ipv6=True,
details='Locally defined mirror',
)
mirror_list[current_region].append(mirror_entry)
return mirror_list
mirror_list_handler = MirrorListHandler()

View File

@ -1,3 +1,5 @@
from __future__ import annotations
import datetime
import http.client
import urllib.error
@ -5,14 +7,17 @@ import urllib.parse
import urllib.request
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Self, TypedDict, override
from typing import TYPE_CHECKING, Any, Self, TypedDict, override
from pydantic import BaseModel, field_validator, model_validator
from pydantic import BaseModel, ValidationInfo, field_validator, model_validator
from ..models.packages import Repository
from ..networking import DownloadTimer, ping
from ..output import debug
if TYPE_CHECKING:
from ..mirrors import MirrorListHandler
class MirrorStatusEntryV3(BaseModel):
url: str
@ -104,13 +109,11 @@ class MirrorStatusEntryV3(BaseModel):
return value
@model_validator(mode='after')
def debug_output(self) -> Self:
from ..args import arch_config_handler
def debug_output(self, info: ValidationInfo) -> Self:
self._hostname, *port = urllib.parse.urlparse(self.url).netloc.split(':', 1)
self._port = int(port[0]) if port and len(port) >= 1 else None
if arch_config_handler.args.verbose:
if (ctx := info.context) and ctx.get('verbose'):
debug(f'Loaded mirror {self._hostname}' + (f' with current score of {self.score}' if self.score else ''))
return self
@ -271,9 +274,11 @@ class MirrorConfiguration:
return config.strip()
def regions_config(self, speed_sort: bool = True) -> str:
from ..mirrors import mirror_list_handler
def regions_config(
self,
mirror_list_handler: MirrorListHandler,
speed_sort: bool = True,
) -> str:
config = ''
for mirror_region in self.mirror_regions:

View File

@ -12,6 +12,7 @@ from archinstall.lib.global_menu import GlobalMenu
from archinstall.lib.hardware import SysInfo
from archinstall.lib.installer import Installer, accessibility_tools_in_use, run_custom_user_commands
from archinstall.lib.interactions.general_conf import PostInstallationAction, ask_post_installation
from archinstall.lib.mirrors import MirrorListHandler
from archinstall.lib.models import Bootloader
from archinstall.lib.models.device import (
DiskLayoutType,
@ -24,7 +25,7 @@ from archinstall.lib.profile.profiles_handler import profile_handler
from archinstall.lib.translationhandler import tr
def ask_user_questions() -> None:
def ask_user_questions(mirror_list_handler: MirrorListHandler) -> None:
"""
First, we'll ask the user for a bunch of user input.
Not until we're satisfied with what we want to install
@ -38,7 +39,11 @@ def ask_user_questions() -> None:
text = tr('New version available') + f': {upgrade}'
title_text += f' ({text})'
global_menu = GlobalMenu(arch_config_handler.config, title=title_text)
global_menu = GlobalMenu(
arch_config_handler.config,
mirror_list_handler,
title=title_text,
)
if not arch_config_handler.args.advanced:
global_menu.set_enabled('parallel_downloads', False)
@ -46,7 +51,10 @@ def ask_user_questions() -> None:
global_menu.run()
def perform_installation(mountpoint: Path) -> None:
def perform_installation(
mountpoint: Path,
mirror_list_handler: MirrorListHandler,
) -> None:
"""
Performs the installation steps on a block device.
Only requirement is that the block devices are
@ -84,7 +92,7 @@ def perform_installation(mountpoint: Path) -> None:
installation.generate_key_files()
if mirror_config := config.mirror_config:
installation.set_mirrors(mirror_config, on_target=False)
installation.set_mirrors(mirror_list_handler, mirror_config, on_target=False)
installation.minimal_installation(
optional_repositories=optional_repositories,
@ -94,7 +102,7 @@ def perform_installation(mountpoint: Path) -> None:
)
if mirror_config := config.mirror_config:
installation.set_mirrors(mirror_config, on_target=True)
installation.set_mirrors(mirror_list_handler, mirror_config, on_target=True)
if config.swap and config.swap.enabled:
installation.setup_swap('zram', algo=config.swap.algorithm)
@ -183,8 +191,13 @@ def perform_installation(mountpoint: Path) -> None:
def guided() -> None:
mirror_list_handler = MirrorListHandler(
offline=arch_config_handler.args.offline,
verbose=arch_config_handler.args.verbose,
)
if not arch_config_handler.args.silent:
ask_user_questions()
ask_user_questions(mirror_list_handler)
config = ConfigurationOutput(arch_config_handler.config)
config.write_debug()
@ -206,7 +219,7 @@ def guided() -> None:
fs_handler = FilesystemHandler(arch_config_handler.config.disk_config)
fs_handler.perform_filesystem_operations()
perform_installation(arch_config_handler.args.mountpoint)
perform_installation(arch_config_handler.args.mountpoint, mirror_list_handler)
guided()