diff --git a/archinstall/lib/mirrors.py b/archinstall/lib/mirrors.py index 7eab1946..ca92f3f3 100644 --- a/archinstall/lib/mirrors.py +++ b/archinstall/lib/mirrors.py @@ -1,6 +1,6 @@ import time import json -import pathlib +from pathlib import Path from dataclasses import dataclass, field from enum import Enum from typing import Dict, Any, List, Optional, TYPE_CHECKING @@ -257,29 +257,51 @@ def select_mirror_regions(preset_values: Dict[str, List[str]] = {}) -> Dict[str, else: preselected = list(preset_values.keys()) - mirrors = list_mirrors() + remote_mirrors = list_mirrors_from_remote() + mirrors: Dict[str, list[str]] = {} - choice = Menu( - _('Select one of the regions to download packages from'), - list(mirrors.keys()), - preset_values=preselected, - multi=True, - allow_reset=True - ).run() + if remote_mirrors: + choice = Menu( + _('Select one of the regions to download packages from'), + list(remote_mirrors.keys()), + preset_values=preselected, + multi=True, + allow_reset=True + ).run() - match choice.type_: - case MenuSelectionType.Reset: - return {} - case MenuSelectionType.Skip: - return preset_values - case MenuSelectionType.Selection: - return { - selected: [ - f"{mirror.url}$repo/os/$arch" for mirror in sort_mirrors_by_performance(mirrors[selected]) - ] for selected in choice.multi_value - } + match choice.type_: + case MenuSelectionType.Reset: + return {} + case MenuSelectionType.Skip: + return preset_values + case MenuSelectionType.Selection: + for region in choice.multi_value: + mirrors.setdefault(region, []) + for mirror in _sort_mirrors_by_performance(remote_mirrors[region]): + mirrors[region].append(mirror.server_url) + return mirrors + else: + local_mirrors = list_mirrors_from_local() - return {} + choice = Menu( + _('Select one of the regions to download packages from'), + list(local_mirrors.keys()), + preset_values=preselected, + multi=True, + allow_reset=True + ).run() + + match choice.type_: + case MenuSelectionType.Reset: + return {} + case MenuSelectionType.Skip: + return preset_values + case MenuSelectionType.Selection: + for region in choice.multi_value: + mirrors[region] = local_mirrors[region] + return mirrors + + return mirrors def select_custom_mirror(prompt: str = '', preset: List[CustomMirror] = []) -> list[CustomMirror]: @@ -287,11 +309,35 @@ def select_custom_mirror(prompt: str = '', preset: List[CustomMirror] = []) -> l return custom_mirrors -def sort_mirrors_by_performance(mirror_list: List[MirrorStatusEntryV3]) -> List[MirrorStatusEntryV3]: +def list_mirrors_from_remote() -> Optional[Dict[str, List[MirrorStatusEntryV3]]]: + if not storage['arguments']['offline']: + url = "https://archlinux.org/mirrors/status/json/" + attempts = 3 + + for attempt_nr in range(attempts): + try: + mirrorlist = fetch_data_from_url(url) + return _parse_remote_mirror_list(mirrorlist) + except Exception as e: + debug(f'Error while fetching mirror list: {e}') + time.sleep(attempt_nr + 1) + + debug('Unable to fetch mirror list remotely, falling back to local mirror list') + + return None + + +def list_mirrors_from_local() -> Dict[str, list[str]]: + with Path('/etc/pacman.d/mirrorlist').open('r') as fp: + mirrorlist = fp.read() + return _parse_locale_mirrors(mirrorlist) + + +def _sort_mirrors_by_performance(mirror_list: List[MirrorStatusEntryV3]) -> List[MirrorStatusEntryV3]: return sorted(mirror_list, key=lambda mirror: (mirror.score, mirror.speed)) -def _parse_mirror_list(mirrorlist: str) -> Dict[str, List[MirrorStatusEntryV3]]: +def _parse_remote_mirror_list(mirrorlist: str) -> Dict[str, List[MirrorStatusEntryV3]]: mirror_status = MirrorStatusListV3(**json.loads(mirrorlist)) sorting_placeholder: Dict[str, List[MirrorStatusEntryV3]] = {} @@ -324,23 +370,27 @@ def _parse_mirror_list(mirrorlist: str) -> Dict[str, List[MirrorStatusEntryV3]]: return sorted_by_regions -def list_mirrors() -> Dict[str, List[MirrorStatusEntryV3]]: - if not storage['arguments']['offline']: - url = "https://archlinux.org/mirrors/status/json/" - attempts = 3 +def _parse_locale_mirrors(mirrorlist: str) -> Dict[str, List[str]]: + lines = mirrorlist.splitlines() - for attempt_nr in range(attempts): - try: - mirrorlist = fetch_data_from_url(url) - return _parse_mirror_list(mirrorlist) - except Exception as e: - debug(f'Error while fetching mirror list: {e}') - time.sleep(attempt_nr + 1) + # remove empty lines + lines = [line for line in lines if line] - debug('Unable to fetch mirror list remotely, falling back to local mirror list') + mirror_list: Dict[str, List[str]] = {} - # we'll use the local mirror list if the offline flag is set - # or if fetching the mirror list remotely failed - with pathlib.Path('/etc/pacman.d/mirrorlist').open('r') as fp: - mirrorlist = fp.read() - return _parse_mirror_list(mirrorlist) + current_region = '' + for idx, line in enumerate(lines): + line = line.strip() + + if line.lower().startswith('server'): + if not current_region: + for i in range(idx - 1, 0, -1): + if lines[i].startswith('##'): + current_region = lines[i].replace('#', '').strip() + mirror_list.setdefault(current_region, []) + break + + url = line.removeprefix('Server = ') + mirror_list[current_region].append(url) + + return mirror_list diff --git a/archinstall/lib/models/mirrors.py b/archinstall/lib/models/mirrors.py index 304d5371..dc95644d 100644 --- a/archinstall/lib/models/mirrors.py +++ b/archinstall/lib/models/mirrors.py @@ -10,7 +10,7 @@ from typing import ( ) from ..networking import ping, DownloadTimer -from ..output import info, debug +from ..output import debug class MirrorStatusEntryV3(pydantic.BaseModel): @@ -35,6 +35,10 @@ class MirrorStatusEntryV3(pydantic.BaseModel): _port: int | None = None _speedtest_retries: int | None = None + @property + def server_url(self) -> str: + return f'{self.url}$repo/os/$arch' + @property def speed(self) -> float: if self._speed is None: @@ -45,7 +49,7 @@ class MirrorStatusEntryV3(pydantic.BaseModel): _retry = 0 while _retry < self._speedtest_retries and self._speed is None: - info(f"Checking download speed of {self._hostname}[{self.score}] by fetching: {self.url}core/os/x86_64/core.db") + debug(f"Checking download speed of {self._hostname}[{self.score}] by fetching: {self.url}core/os/x86_64/core.db") req = urllib.request.Request(url=f"{self.url}core/os/x86_64/core.db") try: @@ -81,7 +85,7 @@ class MirrorStatusEntryV3(pydantic.BaseModel): We do this because some hosts blocks ICMP so we'll have to rely on .speed() instead which is slower. """ if self._latency is None: - info(f"Checking latency for {self.url}") + debug(f"Checking latency for {self.url}") self._latency = ping(self._hostname, timeout=2) debug(f" latency: {self._latency}")