From 697ccd1ac51488fabd381c7aa08fa1a434e6c918 Mon Sep 17 00:00:00 2001 From: Daniel Girtler Date: Wed, 5 Mar 2025 22:19:50 +1100 Subject: [PATCH] Fix 2379 - Mirror and region definitions (#3223) * Fix 2379 - Mirror and repository settings * Fix alignment --- archinstall/lib/args.py | 15 +- archinstall/lib/global_menu.py | 43 +-- archinstall/lib/installer.py | 55 +-- archinstall/lib/interactions/__init__.py | 1 - archinstall/lib/interactions/general_conf.py | 33 +- archinstall/lib/mirrors.py | 342 +++++++++++++++++-- archinstall/lib/models/__init__.py | 4 +- archinstall/lib/models/mirrors.py | 235 ++++--------- archinstall/lib/pacman/__init__.py | 1 - archinstall/lib/pacman/config.py | 27 +- archinstall/lib/pacman/repo.py | 6 - archinstall/scripts/guided.py | 7 +- archinstall/tui/curses_menu.py | 7 +- docs/installing/guided.rst | 321 ++++++++--------- examples/config-sample.json | 271 ++++++++------- examples/interactive_installation.py | 7 +- tests/conftest.py | 5 + tests/data/test_config.json | 327 +++++++++--------- tests/data/test_config_mirror_backwards.json | 18 + tests/test_args.py | 50 ++- tests/test_configuration_output.py | 8 +- tests/test_mirrorlist.py | 2 +- 22 files changed, 1027 insertions(+), 758 deletions(-) delete mode 100644 archinstall/lib/pacman/repo.py create mode 100644 tests/data/test_config_mirror_backwards.json diff --git a/archinstall/lib/args.py b/archinstall/lib/args.py index 559b535d..bc9d02b3 100644 --- a/archinstall/lib/args.py +++ b/archinstall/lib/args.py @@ -14,6 +14,7 @@ from pydantic.dataclasses import dataclass as p_dataclass from archinstall.lib.mirrors import MirrorConfiguration from archinstall.lib.models import AudioConfiguration, Bootloader, NetworkConfiguration, User from archinstall.lib.models.device_model import DiskEncryption, DiskLayoutConfiguration +from archinstall.lib.models.gen import Repository from archinstall.lib.models.locale import LocaleConfiguration from archinstall.lib.models.profile_model import ProfileConfiguration from archinstall.lib.output import error, warn @@ -61,7 +62,6 @@ class ArchConfig: parallel_downloads: int = 0 swap: bool = True timezone: str = 'UTC' - additional_repositories: list[str] = field(default_factory=list) services: list[str] = field(default_factory=list) custom_commands: list[str] = field(default_factory=list) @@ -92,7 +92,6 @@ class ArchConfig: 'parallel_downloads': self.parallel_downloads, 'swap': self.swap, 'timezone': self.timezone, - 'additional-repositories': self.additional_repositories, 'services': self.services, 'custom_commands': self.custom_commands, 'bootloader': self.bootloader.json(), @@ -135,7 +134,14 @@ class ArchConfig: arch_config.profile_config = ProfileConfiguration.parse_arg(profile_config) if mirror_config := args_config.get('mirror_config', None): - arch_config.mirror_config = MirrorConfiguration.parse_args(mirror_config) + backwards_compatible_repo = [] + if additional_repositories := args_config.get('additional-repositories', []): + backwards_compatible_repo = [Repository(r) for r in additional_repositories] + + arch_config.mirror_config = MirrorConfiguration.parse_args( + mirror_config, + backwards_compatible_repo + ) if net_config := args_config.get('network_config', None): arch_config.network_config = NetworkConfiguration.parse_arg(net_config) @@ -181,9 +187,6 @@ class ArchConfig: if timezone := args_config.get('timezone', 'UTC'): arch_config.timezone = timezone - if additional_repositories := args_config.get('additional-repositories', []): - arch_config.additional_repositories = additional_repositories - if services := args_config.get('services', []): arch_config.services = services diff --git a/archinstall/lib/global_menu.py b/archinstall/lib/global_menu.py index a91f6ce3..d4de9f35 100644 --- a/archinstall/lib/global_menu.py +++ b/archinstall/lib/global_menu.py @@ -23,7 +23,6 @@ from .interactions import ( ask_hostname, ask_ntp, ask_to_configure_network, - select_additional_repositories, select_kernel, ) from .locale.locale_menu import LocaleMenu @@ -75,7 +74,7 @@ class GlobalMenu(AbstractMenu): key='locale_config' ), MenuItem( - text=str(_('Mirrors')), + text=str(_('Mirrors and repositories')), action=self._mirror_configuration, preview_action=self._prev_mirror_config, key='mirror_config' @@ -177,13 +176,6 @@ class GlobalMenu(AbstractMenu): preview_action=self._prev_additional_pkgs, key='packages' ), - MenuItem( - text=str(_('Optional repositories')), - action=select_additional_repositories, - value=[], - preview_action=self._prev_additional_repos, - key='additional_repositories' - ), MenuItem( text=str(_('Timezone')), action=ask_for_a_timezone, @@ -324,12 +316,6 @@ class GlobalMenu(AbstractMenu): return output return None - def _prev_additional_repos(self, item: MenuItem) -> str | None: - if item.value: - repos = ', '.join(item.value) - return f'{_("Additional repositories")}: {repos}' - return None - def _prev_tz(self, item: MenuItem) -> str | None: if item.value: return f'{_("Timezone")}: {item.value}' @@ -546,10 +532,27 @@ class GlobalMenu(AbstractMenu): mirror_config: MirrorConfiguration = item.value output = '' - if mirror_config.regions: - output += '{}: {}\n\n'.format(str(_('Mirror regions')), mirror_config.regions) - if mirror_config.custom_mirrors: - table = FormattedOutput.as_table(mirror_config.custom_mirrors) - output += '{}\n{}'.format(str(_('Custom mirrors')), table) + if mirror_config.mirror_regions: + title = str(_('Selected mirror regions')) + divider = '-' * len(title) + regions = mirror_config.region_names + output += f'{title}\n{divider}\n{regions}\n\n' + + if mirror_config.custom_servers: + title = str(_('Custom servers')) + divider = '-' * len(title) + servers = mirror_config.custom_server_urls + output += f'{title}\n{divider}\n{servers}\n\n' + + if mirror_config.optional_repositories: + title = str(_('Optional repositories')) + divider = '-' * len(title) + repos = ', '.join([r.value for r in mirror_config.optional_repositories]) + output += f'{title}\n{divider}\n{repos}\n\n' + + if mirror_config.custom_repositories: + title = str(_('Custom repositories')) + table = FormattedOutput.as_table(mirror_config.custom_repositories) + output += f'{title}:\n\n{table}' return output.strip() diff --git a/archinstall/lib/installer.py b/archinstall/lib/installer.py index 17a1bc4e..21cb22c5 100644 --- a/archinstall/lib/installer.py +++ b/archinstall/lib/installer.py @@ -25,6 +25,7 @@ from archinstall.lib.models.device_model import ( SubvolumeModification, Unit, ) +from archinstall.lib.models.gen import Repository from archinstall.tui.curses_menu import Tui from . import pacman @@ -482,7 +483,11 @@ class Installer: def post_install_check(self, *args: str, **kwargs: str) -> list[str]: return [step for step, flag in self.helper_flags.items() if flag is False] - def set_mirrors(self, mirror_config: MirrorConfiguration, on_target: bool = False) -> None: + def set_mirrors( + self, + mirror_config: MirrorConfiguration, + on_target: bool = False + ) -> None: """ Set the mirror configuration for the installation. @@ -492,28 +497,35 @@ class Installer: :on_target: Whether to set the mirrors on the target system or the live system. :param on_target: bool """ - debug('Setting mirrors') + debug('Setting mirrors on ' + ('target' if on_target else 'live system')) for plugin in plugins.values(): if hasattr(plugin, 'on_mirrors'): if result := plugin.on_mirrors(mirror_config): mirror_config = result - mirrorlist_config = mirror_config.mirrorlist_config(speed_sort=True) - pacman_config = mirror_config.pacman_config() - root = self.target if on_target else Path('/') + mirrorlist_config = (root / 'etc/pacman.d/mirrorlist') + pacman_config = root / 'etc/pacman.conf' - if pacman_config: - debug(f'Pacman config: {pacman_config}') + repositories_config = mirror_config.repositories_config() + if repositories_config: + debug(f'Pacman config: {repositories_config}') - with open(root / 'etc/pacman.conf', 'a') as fp: - fp.write(pacman_config) + with open(pacman_config, 'a') as fp: + fp.write(repositories_config) - if mirrorlist_config: - debug(f'Mirrorlist: {mirrorlist_config}') + regions_config = mirror_config.regions_config(speed_sort=True) + if regions_config: + debug(f'Mirrorlist:\n{regions_config}') + mirrorlist_config.write_text(regions_config) - (root / 'etc/pacman.d/mirrorlist').write_text(mirrorlist_config) + custom_servers = mirror_config.custom_servers_config() + if custom_servers: + debug(f'Custom servers:\n{custom_servers}') + + content = mirrorlist_config.read_text() + mirrorlist_config.write_text(f'{custom_servers}\n\n{content}') def genfstab(self, flags: str = '-pU') -> None: fstab_path = self.target / "etc" / "fstab" @@ -791,8 +803,7 @@ class Installer: def minimal_installation( self, - testing: bool = False, - multilib: bool = False, + optional_repositories: list[Repository] = [], mkinitcpio: bool = True, hostname: str | None = None, locale_config: LocaleConfiguration | None = LocaleConfiguration.default() @@ -827,21 +838,11 @@ class Installer: else: debug('Archinstall will not install any ucode.') - # Determine whether to enable multilib/testing repositories before running pacstrap if testing flag is set. + debug(f'Optional respotories: {optional_repositories}') + # This action takes place on the host system as pacstrap copies over package repository lists. pacman_conf = pacman.Config(self.target) - if multilib: - info("The multilib flag is set. This system will be installed with the multilib repository enabled.") - pacman_conf.enable(pacman.Repo.MULTILIB) - else: - info("The multilib flag is not set. This system will be installed without multilib repositories enabled.") - - if testing: - info("The testing flag is set. This system will be installed with testing repositories enabled.") - pacman_conf.enable(pacman.Repo.TESTING) - else: - info("The testing flag is not set. This system will be installed without testing repositories enabled.") - + pacman_conf.enable(optional_repositories) pacman_conf.apply() self.pacman.strap(self._base_packages) diff --git a/archinstall/lib/interactions/__init__.py b/archinstall/lib/interactions/__init__.py index bd269d8c..7c951bf9 100644 --- a/archinstall/lib/interactions/__init__.py +++ b/archinstall/lib/interactions/__init__.py @@ -13,7 +13,6 @@ from .general_conf import ( ask_for_audio_selection, ask_hostname, ask_ntp, - select_additional_repositories, select_archinstall_language, ) from .manage_users_conf import UserList, ask_for_additional_users diff --git a/archinstall/lib/interactions/general_conf.py b/archinstall/lib/interactions/general_conf.py index 7629b2c3..e54dd809 100644 --- a/archinstall/lib/interactions/general_conf.py +++ b/archinstall/lib/interactions/general_conf.py @@ -168,7 +168,7 @@ def ask_additional_packages_to_install( preset: list[str] = [], repositories: set[Repository] = set() ) -> list[str]: - Tui.print('Loading packages...', clear_screen=True) + Tui.print(str(_('Loading packages...')), clear_screen=True) repositories |= {Repository.Core, Repository.Extra} packages = list_available_packages(tuple(repositories)) @@ -266,37 +266,6 @@ def add_number_of_parallel_downloads(preset: int | None = None) -> int | None: return downloads -def select_additional_repositories(preset: list[str]) -> list[str]: - """ - Allows the user to select additional repositories (multilib, and testing) if desired. - - :return: The string as a selected repository - :rtype: string - """ - - repositories = ["multilib", "testing"] - items = [MenuItem(r, value=r) for r in repositories] - group = MenuItemGroup(items, sort_items=True) - group.set_selected_by_value(preset) - - result = SelectMenu( - group, - alignment=Alignment.CENTER, - frame=FrameProperties.min('Additional repositories'), - allow_reset=True, - allow_skip=True, - multi=True - ).run() - - match result.type_: - case ResultType.Skip: - return preset - case ResultType.Reset: - return [] - case ResultType.Selection: - return result.get_values() - - def ask_chroot() -> bool: prompt = str(_('Would you like to chroot into the newly created installation and perform post-installation configuration?')) + '\n' group = MenuItemGroup.yes_no() diff --git a/archinstall/lib/mirrors.py b/archinstall/lib/mirrors.py index 0a40fc46..75836efe 100644 --- a/archinstall/lib/mirrors.py +++ b/archinstall/lib/mirrors.py @@ -1,10 +1,24 @@ +import time +import urllib +from pathlib import Path from typing import TYPE_CHECKING, override -from archinstall.tui import Alignment, EditMenu, FrameProperties, MenuItem, MenuItemGroup, ResultType, SelectMenu +from archinstall.tui import Alignment, EditMenu, FrameProperties, MenuItem, MenuItemGroup, ResultType, SelectMenu, Tui from .menu import AbstractSubMenu, ListManager -from .models.mirrors import CustomMirror, MirrorConfiguration, MirrorRegion, SignCheck, SignOption, mirror_list_handler -from .output import FormattedOutput +from .models.gen import Repository +from .models.mirrors import ( + CustomRepository, + CustomServer, + MirrorConfiguration, + MirrorRegion, + MirrorStatusEntryV3, + MirrorStatusListV3, + SignCheck, + SignOption, +) +from .networking import fetch_data_from_url +from .output import FormattedOutput, debug if TYPE_CHECKING: from collections.abc import Callable @@ -14,50 +28,50 @@ if TYPE_CHECKING: _: Callable[[str], DeferredTranslation] -class CustomMirrorList(ListManager): - def __init__(self, custom_mirrors: list[CustomMirror]): +class CustomMirrorRepositoriesList(ListManager): + def __init__(self, custom_repositories: list[CustomRepository]): self._actions = [ - str(_('Add a custom mirror')), - str(_('Change custom mirror')), - str(_('Delete custom mirror')) + str(_('Add a custom repository')), + str(_('Change custom repository')), + str(_('Delete custom repository')) ] super().__init__( - custom_mirrors, + custom_repositories, [self._actions[0]], self._actions[1:], '' ) @override - def selected_action_display(self, selection: CustomMirror) -> str: + def selected_action_display(self, selection: CustomRepository) -> str: return selection.name @override def handle_action( self, action: str, - entry: CustomMirror | None, - data: list[CustomMirror] - ) -> list[CustomMirror]: + entry: CustomRepository | None, + data: list[CustomRepository] + ) -> list[CustomRepository]: if action == self._actions[0]: # add - new_mirror = self._add_custom_mirror() - if new_mirror is not None: - data = [d for d in data if d.name != new_mirror.name] - data += [new_mirror] - elif action == self._actions[1] and entry: # modify mirror - new_mirror = self._add_custom_mirror(entry) - if new_mirror is not None: + new_repo = self._add_custom_repository() + if new_repo is not None: + data = [d for d in data if d.name != new_repo.name] + data += [new_repo] + elif action == self._actions[1] and entry: # modify repo + new_repo = self._add_custom_repository(entry) + if new_repo is not None: data = [d for d in data if d.name != entry.name] - data += [new_mirror] + data += [new_repo] elif action == self._actions[2] and entry: # delete data = [d for d in data if d != entry] return data - def _add_custom_mirror(self, preset: CustomMirror | None = None) -> CustomMirror | None: + def _add_custom_repository(self, preset: CustomRepository | None = None) -> CustomRepository | None: edit_result = EditMenu( - str(_('Mirror name')), + str(_('Repository name')), alignment=Alignment.CENTER, allow_skip=True, default_text=preset.name if preset else None @@ -133,7 +147,66 @@ class CustomMirrorList(ListManager): case _: raise ValueError('Unhandled return type') - return CustomMirror(name, url, sign_check, sign_opt) + return CustomRepository(name, url, sign_check, sign_opt) + + +class CustomMirrorServersList(ListManager): + def __init__(self, custom_servers: list[CustomServer]): + self._actions = [ + str(_('Add a custom server')), + str(_('Change custom server')), + str(_('Delete custom server')) + ] + + super().__init__( + custom_servers, + [self._actions[0]], + self._actions[1:], + '' + ) + + @override + def selected_action_display(self, selection: CustomServer) -> str: + return selection.url + + @override + def handle_action( + self, + action: str, + entry: CustomServer | None, + data: list[CustomServer] + ) -> list[CustomServer]: + if action == self._actions[0]: # add + new_server = self._add_custom_server() + if new_server is not None: + data = [d for d in data if d.url != new_server.url] + data += [new_server] + elif action == self._actions[1] and entry: # modify repo + new_server = self._add_custom_server(entry) + if new_server is not None: + data = [d for d in data if d.url != entry.url] + data += [new_server] + elif action == self._actions[2] and entry: # delete + data = [d for d in data if d != entry] + + return data + + def _add_custom_server(self, preset: CustomServer | None = None) -> CustomServer | None: + edit_result = EditMenu( + str(_('Server url')), + alignment=Alignment.CENTER, + allow_skip=True, + default_text=preset.url if preset else None + ).input() + + match edit_result.type_: + case ResultType.Selection: + uri = edit_result.text() + return CustomServer(uri) + case ResultType.Skip: + return preset + + return None class MirrorMenu(AbstractSubMenu): @@ -158,18 +231,32 @@ class MirrorMenu(AbstractSubMenu): def _define_menu_options(self) -> list[MenuItem]: return [ MenuItem( - text=str(_('Mirror region')), + text=str(_('Select regions')), action=select_mirror_regions, value=self._mirror_config.mirror_regions, preview_action=self._prev_regions, key='mirror_regions' ), MenuItem( - text=str(_('Custom mirrors')), + text=str(_('Add custom servers')), + action=add_custom_mirror_servers, + value=self._mirror_config.custom_servers, + preview_action=self._prev_custom_servers, + key='custom_servers' + ), + MenuItem( + text=str(_('Optional repositories')), + action=select_optional_repositories, + value=[], + preview_action=self._prev_additional_repos, + key='optional_repositories' + ), + MenuItem( + text=str(_('Add custom repository')), action=select_custom_mirror, - value=self._mirror_config.custom_mirrors, + value=self._mirror_config.custom_repositories, preview_action=self._prev_custom_mirror, - key='custom_mirrors' + key='custom_repositories' ) ] @@ -187,14 +274,29 @@ class MirrorMenu(AbstractSubMenu): return output + def _prev_additional_repos(self, item: MenuItem) -> str | None: + if item.value: + repositories: list[Repository] = item.value + repos = ', '.join([repo.value for repo in repositories]) + return f'{_("Additional repositories")}: {repos}' + return None + def _prev_custom_mirror(self, item: MenuItem) -> str | None: if not item.value: return None - custom_mirrors: list[CustomMirror] = item.value + custom_mirrors: list[CustomRepository] = item.value output = FormattedOutput.as_table(custom_mirrors) return output.strip() + def _prev_custom_servers(self, item: MenuItem) -> str | None: + if not item.value: + return None + + custom_servers: list[CustomServer] = item.value + output = '\n'.join([server.url for server in custom_servers]) + return output.strip() + @override def run(self) -> MirrorConfiguration: super().run() @@ -202,6 +304,8 @@ class MirrorMenu(AbstractSubMenu): def select_mirror_regions(preset: list[MirrorRegion]) -> list[MirrorRegion]: + Tui.print(str(_('Loading mirror regions...')), clear_screen=True) + mirror_list_handler.load_mirrors() available_regions = mirror_list_handler.get_mirror_regions() @@ -234,6 +338,182 @@ def select_mirror_regions(preset: list[MirrorRegion]) -> list[MirrorRegion]: return selected_mirrors -def select_custom_mirror(preset: list[CustomMirror] = []): - custom_mirrors = CustomMirrorList(preset).run() +def add_custom_mirror_servers(preset: list[CustomServer] = []): + custom_mirrors = CustomMirrorServersList(preset).run() return custom_mirrors + + +def select_custom_mirror(preset: list[CustomRepository] = []): + custom_mirrors = CustomMirrorRepositoriesList(preset).run() + return custom_mirrors + + +def select_optional_repositories(preset: list[Repository]) -> list[Repository]: + """ + Allows the user to select additional repositories (multilib, and testing) if desired. + + :return: The string as a selected repository + :rtype: Repository + """ + + repositories = [Repository.Multilib, Repository.Testing] + items = [MenuItem(r.value, value=r) for r in repositories] + group = MenuItemGroup(items, sort_items=True) + group.set_selected_by_value(preset) + + result = SelectMenu( + group, + alignment=Alignment.CENTER, + frame=FrameProperties.min('Additional repositories'), + allow_reset=True, + allow_skip=True, + multi=True + ).run() + + match result.type_: + case ResultType.Skip: + return preset + case ResultType.Reset: + return [] + case ResultType.Selection: + return result.get_values() + + +class MirrorListHandler: + def __init__( + self, + local_mirrorlist: Path = Path('/etc/pacman.d/mirrorlist'), + ) -> None: + self._local_mirrorlist = local_mirrorlist + self._status_mappings: dict[str, list[MirrorStatusEntryV3]] | None = None + + def _mappings(self) -> dict[str, list[MirrorStatusEntryV3]]: + if self._status_mappings is None: + self.load_mirrors() + + assert self._status_mappings is not None + return self._status_mappings + + def get_mirror_regions(self) -> list[MirrorRegion]: + available_mirrors = [] + mappings = self._mappings() + + for region_name, status_entry in mappings.items(): + urls = [entry.server_url for entry in status_entry] + region = MirrorRegion(region_name, urls) + available_mirrors.append(region) + + return available_mirrors + + def load_mirrors(self) -> None: + from .args import arch_config_handler + if arch_config_handler.args.offline: + self.load_local_mirrors() + else: + if not self.load_remote_mirrors(): + self.load_local_mirrors() + + def load_remote_mirrors(self) -> bool: + url = "https://archlinux.org/mirrors/status/json/" + attempts = 3 + + for attempt_nr in range(attempts): + try: + mirrorlist = fetch_data_from_url(url) + self._status_mappings = self._parse_remote_mirror_list(mirrorlist) + return True + except Exception as e: + debug(f'Error while fetching mirror list: {e}') + time.sleep(attempt_nr + 1) + + debug('Unable to fetch mirror list remotely, falling back to local mirror list') + return False + + def load_local_mirrors(self) -> None: + with self._local_mirrorlist.open('r') as fp: + mirrorlist = fp.read() + self._status_mappings = self._parse_locale_mirrors(mirrorlist) + + def get_status_by_region(self, region: str, speed_sort: bool) -> list[MirrorStatusEntryV3]: + mappings = self._mappings() + region_list = mappings[region] + return sorted(region_list, key=lambda mirror: (mirror.score, mirror.speed)) + + def _parse_remote_mirror_list(self, mirrorlist: str) -> dict[str, list[MirrorStatusEntryV3]] | None: + mirror_status = MirrorStatusListV3.model_validate_json(mirrorlist) + + sorting_placeholder: dict[str, list[MirrorStatusEntryV3]] = {} + + for mirror in mirror_status.urls: + # We filter out mirrors that have bad criteria values + if any([ + mirror.active is False, # Disabled by mirror-list admins + mirror.last_sync is None, # Has not synced recently + # mirror.score (error rate) over time reported from backend: + # https://github.com/archlinux/archweb/blob/31333d3516c91db9a2f2d12260bd61656c011fd1/mirrors/utils.py#L111C22-L111C66 + (mirror.score is None or mirror.score >= 100), + ]): + continue + + if mirror.country == "": + # TODO: This should be removed once RFC!29 is merged and completed + # Until then, there are mirrors which lacks data in the backend + # and there is no way of knowing where they're located. + # So we have to assume world-wide + mirror.country = "Worldwide" + + if mirror.url.startswith('http'): + sorting_placeholder.setdefault(mirror.country, []).append(mirror) + + sorted_by_regions: dict[str, list[MirrorStatusEntryV3]] = dict({ + region: unsorted_mirrors + for region, unsorted_mirrors in sorted(sorting_placeholder.items(), key=lambda item: item[0]) + }) + + return sorted_by_regions + + def _parse_locale_mirrors(self, mirrorlist: str) -> dict[str, list[MirrorStatusEntryV3]] | None: + lines = mirrorlist.splitlines() + + # remove empty lines + # lines = [line for line in lines if line] + + mirror_list: dict[str, list[MirrorStatusEntryV3]] = {} + + current_region = '' + + for line in lines: + line = line.strip() + + if line.startswith('## '): + current_region = line.replace('## ', '').strip() + mirror_list.setdefault(current_region, []) + + if line.startswith('Server = '): + if not current_region: + current_region = 'Local' + mirror_list.setdefault(current_region, []) + + url = line.removeprefix('Server = ') + + mirror_entry = MirrorStatusEntryV3( + url=url.removesuffix('$repo/os/$arch'), + protocol=urllib.parse.urlparse(url).scheme, + active=True, + country=current_region or 'Worldwide', + # The following values are normally populated by + # archlinux.org mirror-list endpoint, and can't be known + # from just the local mirror-list file. + country_code='WW', + isos=True, + ipv4=True, + ipv6=True, + details='Locally defined mirror', + ) + + mirror_list[current_region].append(mirror_entry) + + return mirror_list + + +mirror_list_handler = MirrorListHandler() diff --git a/archinstall/lib/models/__init__.py b/archinstall/lib/models/__init__.py index fd21d452..887592e7 100644 --- a/archinstall/lib/models/__init__.py +++ b/archinstall/lib/models/__init__.py @@ -27,9 +27,9 @@ from .device_model import ( Unit, _DeviceInfo, ) -from .gen import LocalPackage, PackageSearch, PackageSearchResult +from .gen import LocalPackage, PackageSearch, PackageSearchResult, Repository from .locale import LocaleConfiguration -from .mirrors import CustomMirror, MirrorConfiguration, MirrorRegion +from .mirrors import CustomRepository, MirrorConfiguration, MirrorRegion from .network_configuration import NetworkConfiguration, Nic, NicType from .profile_model import ProfileConfiguration from .users import PasswordStrength, User diff --git a/archinstall/lib/models/mirrors.py b/archinstall/lib/models/mirrors.py index e0368acc..10211414 100644 --- a/archinstall/lib/models/mirrors.py +++ b/archinstall/lib/models/mirrors.py @@ -1,17 +1,16 @@ import datetime import http.client -import time import urllib.error import urllib.parse import urllib.request from dataclasses import dataclass, field from enum import Enum -from pathlib import Path from typing import TYPE_CHECKING, Any, override from pydantic import BaseModel, field_validator, model_validator -from ..networking import DownloadTimer, fetch_data_from_url, ping +from ..models.gen import Repository +from ..networking import DownloadTimer, ping from ..output import debug if TYPE_CHECKING: @@ -153,146 +152,6 @@ class MirrorRegion: return self.name == other.name -class MirrorListHandler: - def __init__( - self, - local_mirrorlist: Path = Path('/etc/pacman.d/mirrorlist'), - ) -> None: - self._local_mirrorlist = local_mirrorlist - self._status_mappings: dict[str, list[MirrorStatusEntryV3]] | None = None - - def _mappings(self) -> dict[str, list[MirrorStatusEntryV3]]: - if self._status_mappings is None: - self.load_mirrors() - - assert self._status_mappings is not None - return self._status_mappings - - def get_mirror_regions(self) -> list[MirrorRegion]: - available_mirrors = [] - mappings = self._mappings() - - for region_name, status_entry in mappings.items(): - urls = [entry.server_url for entry in status_entry] - region = MirrorRegion(region_name, urls) - available_mirrors.append(region) - - return available_mirrors - - def load_mirrors(self) -> None: - from ..args import arch_config_handler - if arch_config_handler.args.offline: - self.load_local_mirrors() - else: - if not self.load_remote_mirrors(): - self.load_local_mirrors() - - def load_remote_mirrors(self) -> bool: - url = "https://archlinux.org/mirrors/status/json/" - attempts = 3 - - for attempt_nr in range(attempts): - try: - mirrorlist = fetch_data_from_url(url) - self._status_mappings = self._parse_remote_mirror_list(mirrorlist) - return True - except Exception as e: - debug(f'Error while fetching mirror list: {e}') - time.sleep(attempt_nr + 1) - - debug('Unable to fetch mirror list remotely, falling back to local mirror list') - return False - - def load_local_mirrors(self) -> None: - with self._local_mirrorlist.open('r') as fp: - mirrorlist = fp.read() - self._status_mappings = self._parse_locale_mirrors(mirrorlist) - - def get_status_by_region(self, region: str, speed_sort: bool) -> list[MirrorStatusEntryV3]: - mappings = self._mappings() - region_list = mappings[region] - return sorted(region_list, key=lambda mirror: (mirror.score, mirror.speed)) - - def _parse_remote_mirror_list(self, mirrorlist: str) -> dict[str, list[MirrorStatusEntryV3]] | None: - mirror_status = MirrorStatusListV3.model_validate_json(mirrorlist) - - sorting_placeholder: dict[str, list[MirrorStatusEntryV3]] = {} - - for mirror in mirror_status.urls: - # We filter out mirrors that have bad criteria values - if any([ - mirror.active is False, # Disabled by mirror-list admins - mirror.last_sync is None, # Has not synced recently - # mirror.score (error rate) over time reported from backend: - # https://github.com/archlinux/archweb/blob/31333d3516c91db9a2f2d12260bd61656c011fd1/mirrors/utils.py#L111C22-L111C66 - (mirror.score is None or mirror.score >= 100), - ]): - continue - - if mirror.country == "": - # TODO: This should be removed once RFC!29 is merged and completed - # Until then, there are mirrors which lacks data in the backend - # and there is no way of knowing where they're located. - # So we have to assume world-wide - mirror.country = "Worldwide" - - if mirror.url.startswith('http'): - sorting_placeholder.setdefault(mirror.country, []).append(mirror) - - sorted_by_regions: dict[str, list[MirrorStatusEntryV3]] = dict({ - region: unsorted_mirrors - for region, unsorted_mirrors in sorted(sorting_placeholder.items(), key=lambda item: item[0]) - }) - - return sorted_by_regions - - def _parse_locale_mirrors(self, mirrorlist: str) -> dict[str, list[MirrorStatusEntryV3]] | None: - lines = mirrorlist.splitlines() - - # remove empty lines - # lines = [line for line in lines if line] - - mirror_list: dict[str, list[MirrorStatusEntryV3]] = {} - - current_region = '' - - for line in lines: - line = line.strip() - - if line.startswith('## '): - current_region = line.replace('## ', '').strip() - mirror_list.setdefault(current_region, []) - - if line.startswith('Server = '): - if not current_region: - current_region = 'Local' - mirror_list.setdefault(current_region, []) - - url = line.removeprefix('Server = ') - - mirror_entry = MirrorStatusEntryV3( - url=url.removesuffix('$repo/os/$arch'), - protocol=urllib.parse.urlparse(url).scheme, - active=True, - country=current_region or 'Worldwide', - # The following values are normally populated by - # archlinux.org mirror-list endpoint, and can't be known - # from just the local mirror-list file. - country_code='WW', - isos=True, - ipv4=True, - ipv6=True, - details='Locally defined mirror', - ) - - mirror_list[current_region].append(mirror_entry) - - return mirror_list - - -mirror_list_handler = MirrorListHandler() - - class SignCheck(Enum): Never = 'Never' Optional = 'Optional' @@ -305,7 +164,7 @@ class SignOption(Enum): @dataclass -class CustomMirror: +class CustomRepository: name: str url: str sign_check: SignCheck @@ -328,11 +187,11 @@ class CustomMirror: } @classmethod - def parse_args(cls, args: list[dict[str, str]]) -> list['CustomMirror']: + def parse_args(cls, args: list[dict[str, str]]) -> list['CustomRepository']: configs = [] for arg in args: configs.append( - CustomMirror( + CustomRepository( arg['name'], arg['url'], SignCheck(arg['sign_check']), @@ -343,14 +202,41 @@ class CustomMirror: return configs +@dataclass +class CustomServer: + url: str + + def table_data(self) -> dict[str, str]: + return {'Url': self.url} + + def json(self) -> dict[str, str]: + return {'url': self.url} + + @classmethod + def parse_args(cls, args: list[dict[str, str]]) -> list['CustomServer']: + configs = [] + for arg in args: + configs.append( + CustomServer(arg['url']) + ) + + return configs + + @dataclass class MirrorConfiguration: mirror_regions: list[MirrorRegion] = field(default_factory=list) - custom_mirrors: list[CustomMirror] = field(default_factory=list) + custom_servers: list[CustomServer] = field(default_factory=list) + optional_repositories: list[Repository] = field(default_factory=list) + custom_repositories: list[CustomRepository] = field(default_factory=list) @property - def regions(self) -> str: - return ', '.join([m.name for m in self.mirror_regions]) + def region_names(self) -> str: + return '\n'.join([m.name for m in self.mirror_regions]) + + @property + def custom_server_urls(self) -> str: + return '\n'.join([s.url for s in self.custom_servers]) def json(self) -> dict[str, Any]: regions = {} @@ -359,10 +245,22 @@ class MirrorConfiguration: return { 'mirror_regions': regions, - 'custom_mirrors': [c.json() for c in self.custom_mirrors] + 'custom_servers': self.custom_servers, + 'optional_repositories': [r.value for r in self.optional_repositories], + 'custom_repositories': [c.json() for c in self.custom_repositories] } - def mirrorlist_config(self, speed_sort: bool = True) -> str: + def custom_servers_config(self) -> str: + config = '## Custom Servers\n' + + for server in self.custom_servers: + config += f'Server = {server.url}\n' + + return config.strip() + + def regions_config(self, speed_sort: bool = True) -> str: + from ..mirrors import mirror_list_handler + config = '' for mirror_region in self.mirror_regions: @@ -376,23 +274,24 @@ class MirrorConfiguration: for status in sorted_stati: config += f'Server = {status.server_url}\n' - for cm in self.custom_mirrors: - config += f'\n\n## {cm.name}\nServer = {cm.url}\n' - return config - def pacman_config(self) -> str: + def repositories_config(self) -> str: config = '' - for mirror in self.custom_mirrors: - config += f'\n\n[{mirror.name}]\n' - config += f'SigLevel = {mirror.sign_check.value} {mirror.sign_option.value}\n' - config += f'Server = {mirror.url}\n' + for repo in self.custom_repositories: + config += f'\n\n[{repo.name}]\n' + config += f'SigLevel = {repo.sign_check.value} {repo.sign_option.value}\n' + config += f'Server = {repo.url}\n' return config @classmethod - def parse_args(cls, args: dict[str, Any]) -> 'MirrorConfiguration': + def parse_args( + cls, + args: dict[str, Any], + backwards_compatible_repo: list[Repository] = [] + ) -> 'MirrorConfiguration': config = MirrorConfiguration() mirror_regions = args.get('mirror_regions', []) @@ -400,7 +299,21 @@ class MirrorConfiguration: for region, urls in mirror_regions.items(): config.mirror_regions.append(MirrorRegion(region, urls)) + if args.get('custom_servers', []): + config.custom_servers = CustomServer.parse_args(args['custom_servers']) + + # backwards compatibility with the new custom_repository if 'custom_mirrors' in args: - config.custom_mirrors = CustomMirror.parse_args(args['custom_mirrors']) + config.custom_repositories = CustomRepository.parse_args(args['custom_mirrors']) + if 'custom_repositories' in args: + config.custom_repositories = CustomRepository.parse_args(args['custom_repositories']) + + if 'optional_repositories' in args: + config.optional_repositories = [Repository(r) for r in args['optional_repositories']] + + if backwards_compatible_repo: + for r in backwards_compatible_repo: + if r not in config.optional_repositories: + config.optional_repositories.append(r) return config diff --git a/archinstall/lib/pacman/__init__.py b/archinstall/lib/pacman/__init__.py index e767ce97..32839a71 100644 --- a/archinstall/lib/pacman/__init__.py +++ b/archinstall/lib/pacman/__init__.py @@ -10,7 +10,6 @@ from ..general import SysCommand from ..output import error, info, warn from ..plugins import plugins from .config import Config -from .repo import Repo if TYPE_CHECKING: from archinstall.lib.translationhandler import DeferredTranslation diff --git a/archinstall/lib/pacman/config.py b/archinstall/lib/pacman/config.py index 7dbb2e17..a90e7f9e 100644 --- a/archinstall/lib/pacman/config.py +++ b/archinstall/lib/pacman/config.py @@ -2,33 +2,36 @@ import re from pathlib import Path from shutil import copy2 -from .repo import Repo +from ..models.gen import Repository class Config: def __init__(self, target: Path): self.path = Path("/etc") / "pacman.conf" self.chroot_path = target / "etc" / "pacman.conf" - self.repos: list[Repo] = [] + self._repositories: list[Repository] = [] - def enable(self, repo: Repo) -> None: - self.repos.append(repo) + def enable(self, repo: Repository | list[Repository]) -> None: + if not isinstance(repo, list): + repo = [repo] + + self._repositories += repo def apply(self) -> None: - if not self.repos: + if not self._repositories: return - if Repo.TESTING in self.repos: - if Repo.MULTILIB in self.repos: - repos_pattern = f'({Repo.MULTILIB}|.+-{Repo.TESTING})' + if Repository.Testing in self._repositories: + if Repository.Multilib in self._repositories: + repos_pattern = f'({Repository.Multilib.value}|.+-{Repository.Testing.value})' else: - repos_pattern = f'(?!{Repo.MULTILIB}).+-{Repo.TESTING}' + repos_pattern = f'(?!{Repository.Multilib.value}).+-{Repository.Testing.value}' else: - repos_pattern = Repo.MULTILIB + repos_pattern = Repository.Multilib.value pattern = re.compile(rf"^#\s*\[{repos_pattern}\]$") - lines = iter(self.path.read_text().splitlines(keepends=True)) + with open(self.path, 'w') as f: for line in lines: if pattern.match(line): @@ -39,5 +42,5 @@ class Config: f.write(line) def persist(self) -> None: - if self.repos: + if self._repositories: copy2(self.path, self.chroot_path) diff --git a/archinstall/lib/pacman/repo.py b/archinstall/lib/pacman/repo.py deleted file mode 100644 index 4ba34946..00000000 --- a/archinstall/lib/pacman/repo.py +++ /dev/null @@ -1,6 +0,0 @@ -from enum import StrEnum, auto - - -class Repo(StrEnum): - MULTILIB = auto() - TESTING = auto() diff --git a/archinstall/scripts/guided.py b/archinstall/scripts/guided.py index c2e0f61f..70b8c605 100644 --- a/archinstall/scripts/guided.py +++ b/archinstall/scripts/guided.py @@ -51,13 +51,11 @@ def perform_installation(mountpoint: Path) -> None: error("No disk configuration provided") return - # Retrieve list of additional repositories and set boolean values appropriately disk_config: DiskLayoutConfiguration = config.disk_config - enable_testing = 'testing' in config.additional_repositories - enable_multilib = 'multilib' in config.additional_repositories run_mkinitcpio = not config.uki locale_config = config.locale_config disk_encryption = config.disk_encryption + optional_repositories = config.mirror_config.optional_repositories if config.mirror_config else [] with Installer( mountpoint, @@ -80,8 +78,7 @@ def perform_installation(mountpoint: Path) -> None: installation.set_mirrors(mirror_config, on_target=False) installation.minimal_installation( - testing=enable_testing, - multilib=enable_multilib, + optional_repositories=optional_repositories, mkinitcpio=run_mkinitcpio, hostname=arch_config_handler.config.hostname, locale_config=locale_config diff --git a/archinstall/tui/curses_menu.py b/archinstall/tui/curses_menu.py index 7d07e620..044d184a 100644 --- a/archinstall/tui/curses_menu.py +++ b/archinstall/tui/curses_menu.py @@ -750,12 +750,13 @@ class SelectMenu(AbstractCurses): return 0 lines = header.split('\n') if header else [] - table_header = [line for line in lines if '|' in line] + table_header = [line for line in lines if '-' in line] + longest_header = len(table_header[0]) if table_header else 0 longest_entry = self._item_group.get_max_width() - delta = abs(longest_header - longest_entry) - offset = delta + 3 # 3 because it seems to align it... + delta = longest_header - longest_entry + offset = delta + 2 # 2 because it seems to align it... return offset diff --git a/docs/installing/guided.rst b/docs/installing/guided.rst index 49be3da1..9861add4 100644 --- a/docs/installing/guided.rst +++ b/docs/installing/guided.rst @@ -53,165 +53,168 @@ The contents of :code:`https://domain.lan/config.json`: .. code-block:: json - { - "__separator__": null, - "additional-repositories": [], - "archinstall-language": "English", - "audio_config": null, - "bootloader": "Systemd-boot", - "config_version": "2.6.0", - "debug": false, - "disk_config": { - "config_type": "manual_partitioning", - "device_modifications": [ - { - "device": "/dev/sda", - "partitions": [ - { - "btrfs": [], - "flags": [ - "boot" - ], - "fs_type": "fat32", - "length": { - "sector_size": null, - "total_size": null, - "unit": "B", - "value": 99982592 - }, - "mount_options": [], - "mountpoint": "/boot", - "obj_id": "369f31a8-2781-4d6b-96e7-75680552b7c9", - "start": { - "sector_size": { - "sector_size": null, - "total_size": null, - "unit": "B", - "value": 512 - }, - "total_size": null, - "unit": "sectors", - "value": 34 - }, - "status": "create", - "type": "primary" - }, - { - "btrfs": [], - "flags": [], - "fs_type": "fat32", - "length": { - "sector_size": null, - "total_size": null, - "unit": "B", - "value": 100000000 - }, - "mount_options": [], - "mountpoint": "/efi", - "obj_id": "13cf2c96-8b0f-4ade-abaa-c530be589aad", - "start": { - "sector_size": { - "sector_size": null, - "total_size": null, - "unit": "B", - "value": 512 - }, - "total_size": { - "sector_size": null, - "total_size": null, - "unit": "B", - "value": 16106127360 - }, - "unit": "MB", - "value": 100 - }, - "status": "create", - "type": "primary" - }, - { - "btrfs": [], - "flags": [], - "fs_type": "ext4", - "length": { - "sector_size": null, - "total_size": null, - "unit": "B", - "value": 15805127360 - }, - "mount_options": [], - "mountpoint": "/", - "obj_id": "3e75d045-21a4-429d-897e-8ec19a006e8b", - "start": { - "sector_size": { - "sector_size": null, - "total_size": null, - "unit": "B", - "value": 512 - }, - "total_size": { - "sector_size": null, - "total_size": null, - "unit": "B", - "value": 16106127360 - }, - "unit": "MB", - "value": 301 - }, - "status": "create", - "type": "primary" - } - ], - "wipe": false - } - ] - }, - "disk_encryption": { - "encryption_type": "luks", - "partitions": [ - "3e75d045-21a4-429d-897e-8ec19a006e8b" - ] - }, - "hostname": "archlinux", - "kernels": [ - "linux" - ], - "locale_config": { - "kb_layout": "us", - "sys_enc": "UTF-8", - "sys_lang": "en_US" - }, - "mirror_config": { - "custom_mirrors": [], - "mirror_regions": { - "Sweden": [ - "https://mirror.osbeck.com/archlinux/$repo/os/$arch", - "https://mirror.bahnhof.net/pub/archlinux/$repo/os/$arch", - "https://ftp.myrveln.se/pub/linux/archlinux/$repo/os/$arch", - "https://ftp.lysator.liu.se/pub/archlinux/$repo/os/$arch", - "https://ftp.ludd.ltu.se/mirrors/archlinux/$repo/os/$arch", - "https://ftp.acc.umu.se/mirror/archlinux/$repo/os/$arch", - "http://mirror.bahnhof.net/pub/archlinux/$repo/os/$arch", - "http://ftpmirror.infania.net/mirror/archlinux/$repo/os/$arch", - "http://ftp.myrveln.se/pub/linux/archlinux/$repo/os/$arch", - "http://ftp.lysator.liu.se/pub/archlinux/$repo/os/$arch", - "http://ftp.acc.umu.se/mirror/archlinux/$repo/os/$arch" - ] - } - }, - "network_config": {}, - "no_pkg_lookups": false, - "ntp": true, - "offline": false, - "packages": [], - "parallel downloads": 0, - "profile_config": null, - "save_config": null, - "script": "guided", - "silent": false, - "swap": true, - "timezone": "UTC", - "version": "2.6.0" - } +{ + "additional-repositories": [], + "archinstall-language": "English", + "audio_config": null, + "bootloader": "Systemd-boot", + "debug": false, + "disk_config": { + "config_type": "manual_partitioning", + "device_modifications": [ + { + "device": "/dev/sda", + "partitions": [ + { + "btrfs": [], + "flags": [ + "boot" + ], + "fs_type": "fat32", + "length": { + "sector_size": null, + "total_size": null, + "unit": "B", + "value": 99982592 + }, + "mount_options": [], + "mountpoint": "/boot", + "obj_id": "369f31a8-2781-4d6b-96e7-75680552b7c9", + "start": { + "sector_size": { + "sector_size": null, + "total_size": null, + "unit": "B", + "value": 512 + }, + "total_size": null, + "unit": "sectors", + "value": 34 + }, + "status": "create", + "type": "primary" + }, + { + "btrfs": [], + "flags": [], + "fs_type": "fat32", + "length": { + "sector_size": null, + "total_size": null, + "unit": "B", + "value": 100000000 + }, + "mount_options": [], + "mountpoint": "/efi", + "obj_id": "13cf2c96-8b0f-4ade-abaa-c530be589aad", + "start": { + "sector_size": { + "sector_size": null, + "total_size": null, + "unit": "B", + "value": 512 + }, + "total_size": { + "sector_size": null, + "total_size": null, + "unit": "B", + "value": 16106127360 + }, + "unit": "MB", + "value": 100 + }, + "status": "create", + "type": "primary" + }, + { + "btrfs": [], + "flags": [], + "fs_type": "ext4", + "length": { + "sector_size": null, + "total_size": null, + "unit": "B", + "value": 15805127360 + }, + "mount_options": [], + "mountpoint": "/", + "obj_id": "3e75d045-21a4-429d-897e-8ec19a006e8b", + "start": { + "sector_size": { + "sector_size": null, + "total_size": null, + "unit": "B", + "value": 512 + }, + "total_size": { + "sector_size": null, + "total_size": null, + "unit": "B", + "value": 16106127360 + }, + "unit": "MB", + "value": 301 + }, + "status": "create", + "type": "primary" + } + ], + "wipe": false + } + ] + }, + "disk_encryption": { + "encryption_type": "luks", + "partitions": [ + "3e75d045-21a4-429d-897e-8ec19a006e8b" + ] + }, + "hostname": "archlinux", + "kernels": [ + "linux" + ], + "locale_config": { + "kb_layout": "us", + "sys_enc": "UTF-8", + "sys_lang": "en_US" + }, + "mirror_config": { + "custom_servers": [ + { + "url": "https://mymirror.com/$repo/os/$arch" + } + ], + "mirror_regions": { + "Australia": [ + "http://archlinux.mirror.digitalpacific.com.au/$repo/os/$arch" + ] + }, + "optional_repositories": [ + "testing" + ], + "custom_repositories": [ + { + "name": "myrepo", + "url": "https://myrepo.com/$repo/os/$arch", + "sign_check": "Required", + "sign_option": "TrustAll" + } + ] + }, + "network_config": {}, + "no_pkg_lookups": false, + "ntp": true, + "offline": false, + "packages": [], + "parallel downloads": 0, + "profile_config": null, + "save_config": null, + "script": "guided", + "silent": false, + "swap": true, + "timezone": "UTC", + "version": "2.6.0" +} ``--config`` options ^^^^^^^^^^^^^^^^^^^^ diff --git a/examples/config-sample.json b/examples/config-sample.json index 6a4f2416..40a349c9 100644 --- a/examples/config-sample.json +++ b/examples/config-sample.json @@ -1,133 +1,148 @@ { - "__separator__": null, - "config_version": "2.8.6", - "additional-repositories": [], - "archinstall-language": "English", - "audio_config": {"audio": "pipewire"}, - "bootloader": "Systemd-boot", - "debug": false, - "disk_config": { - "config_type": "default_layout", - "device_modifications": [ - { - "device": "/dev/sda", - "partitions": [ - { - "btrfs": [], - "flags": [ - "boot" - ], - "fs_type": "fat32", - "size": { - "sector_size": null, - "unit": "MiB", - "value": 512 - }, - "mount_options": [], - "mountpoint": "/boot", - "obj_id": "2c3fa2d5-2c79-4fab-86ec-22d0ea1543c0", - "start": { - "sector_size": null, - "unit": "MiB", - "value": 1 - }, - "status": "create", - "type": "primary" - }, - { - "btrfs": [], - "flags": [], - "fs_type": "ext4", - "size": { - "sector_size": null, - "unit": "GiB", - "value": 20 - }, - "mount_options": [], - "mountpoint": "/", - "obj_id": "3e7018a0-363b-4d05-ab83-8e82d13db208", - "start": { - "sector_size": null, - "unit": "MiB", - "value": 513 - }, - "status": "create", - "type": "primary" - }, - { - "btrfs": [], - "flags": [], - "fs_type": "ext4", - "size": { - "sector_size": null, - "unit": "Percent", - "value": 100 - }, - "mount_options": [], - "mountpoint": "/home", - "obj_id": "ce58b139-f041-4a06-94da-1f8bad775d3f", - "start": { - "sector_size": null, - "unit": "GiB", - "value": 20 - }, - "status": "create", - "type": "primary" - } - ], - "wipe": true - } - ] - }, - "hostname": "archlinux", - "kernels": [ - "linux" - ], - "locale_config": { - "kb_layout": "us", - "sys_enc": "UTF-8", - "sys_lang": "en_US" - }, - "mirror_config": { - "mirror-regions": { - "Australia": [ - "http://archlinux.mirror.digitalpacific.com.au/$repo/os/$arch" - ] + "archinstall-language": "English", + "audio_config": { + "audio": "pipewire" + }, + "bootloader": "Systemd-boot", + "debug": false, + "disk_config": { + "config_type": "default_layout", + "device_modifications": [ + { + "device": "/dev/sda", + "partitions": [ + { + "btrfs": [], + "flags": [ + "boot" + ], + "fs_type": "fat32", + "size": { + "sector_size": null, + "unit": "MiB", + "value": 512 + }, + "mount_options": [], + "mountpoint": "/boot", + "obj_id": "2c3fa2d5-2c79-4fab-86ec-22d0ea1543c0", + "start": { + "sector_size": null, + "unit": "MiB", + "value": 1 + }, + "status": "create", + "type": "primary" + }, + { + "btrfs": [], + "flags": [], + "fs_type": "ext4", + "size": { + "sector_size": null, + "unit": "GiB", + "value": 20 + }, + "mount_options": [], + "mountpoint": "/", + "obj_id": "3e7018a0-363b-4d05-ab83-8e82d13db208", + "start": { + "sector_size": null, + "unit": "MiB", + "value": 513 + }, + "status": "create", + "type": "primary" + }, + { + "btrfs": [], + "flags": [], + "fs_type": "ext4", + "size": { + "sector_size": null, + "unit": "Percent", + "value": 100 + }, + "mount_options": [], + "mountpoint": "/home", + "obj_id": "ce58b139-f041-4a06-94da-1f8bad775d3f", + "start": { + "sector_size": null, + "unit": "GiB", + "value": 20 + }, + "status": "create", + "type": "primary" + } + ], + "wipe": true } - }, - "network_config": { - "type": "manual", - "nics": [ - { - "iface": "eno1", - "ip": "192.168.1.15/24", - "dhcp": true, - "gateway": "192.168.1.1", - "dns": [ - "192.168.1.1", - "9.9.9.9" - ] - } + ] + }, + "hostname": "archlinux", + "kernels": [ + "linux" + ], + "locale_config": { + "kb_layout": "us", + "sys_enc": "UTF-8", + "sys_lang": "en_US" + }, + "mirror_config": { + "custom_servers": [ + { + "url": "https://mymirror.com/$repo/os/$arch" + } + ], + "mirror_regions": { + "Australia": [ + "http://archlinux.mirror.digitalpacific.com.au/$repo/os/$arch" ] }, - "no_pkg_lookups": false, - "ntp": true, - "offline": false, - "packages": [], - "parallel downloads": 0, - "profile_config": { - "gfx_driver": "All open-source (default)", - "greeter": "sddm", - "profile": { - "details": [ - "KDE Plasma" - ], - "main": "Desktop" - } - }, - "script": "guided", - "silent": false, - "swap": true, - "timezone": "UTC", - "version": "2.8.6" + "optional_repositories": [ + "testing" + ], + "custom_repositories": [ + { + "name": "myrepo", + "url": "https://myrepo.com/$repo/os/$arch", + "sign_check": "Required", + "sign_option": "TrustAll" + } + ] + }, + "network_config": { + "type": "manual", + "nics": [ + { + "iface": "eno1", + "ip": "192.168.1.15/24", + "dhcp": true, + "gateway": "192.168.1.1", + "dns": [ + "192.168.1.1", + "9.9.9.9" + ] + } + ] + }, + "no_pkg_lookups": false, + "ntp": true, + "offline": false, + "packages": [], + "parallel downloads": 0, + "profile_config": { + "gfx_driver": "All open-source (default)", + "greeter": "sddm", + "profile": { + "details": [ + "KDE Plasma" + ], + "main": "Desktop" + } + }, + "script": "guided", + "silent": false, + "swap": true, + "timezone": "UTC", + "version": "2.8.6" } diff --git a/examples/interactive_installation.py b/examples/interactive_installation.py index 47409be5..90e90548 100644 --- a/examples/interactive_installation.py +++ b/examples/interactive_installation.py @@ -50,13 +50,11 @@ def perform_installation(mountpoint: Path) -> None: error("No disk configuration provided") return - # Retrieve list of additional repositories and set boolean values appropriately disk_config: DiskLayoutConfiguration = config.disk_config - enable_testing = 'testing' in config.additional_repositories - enable_multilib = 'multilib' in config.additional_repositories run_mkinitcpio = not config.uki locale_config = config.locale_config disk_encryption = config.disk_encryption + optional_repositories = config.mirror_config.optional_repositories if config.mirror_config else [] with Installer( mountpoint, @@ -79,8 +77,7 @@ def perform_installation(mountpoint: Path) -> None: installation.set_mirrors(mirror_config, on_target=False) installation.minimal_installation( - testing=enable_testing, - multilib=enable_multilib, + optional_repositories=optional_repositories, mkinitcpio=run_mkinitcpio, hostname=arch_config_handler.config.hostname, locale_config=locale_config diff --git a/tests/conftest.py b/tests/conftest.py index 66c7eae6..bbdf8a7b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,11 @@ def creds_fixture() -> Path: return Path(__file__).parent / 'data' / 'test_creds.json' +@pytest.fixture(scope='session') +def mirror_backwards_config() -> Path: + return Path(__file__).parent / 'data' / 'test_config_mirror_backwards.json' + + @pytest.fixture(scope='session') def mirrorlist_no_country_fixture() -> Path: return Path(__file__).parent / 'data' / 'mirrorlists' / 'test_no_country' diff --git a/tests/data/test_config.json b/tests/data/test_config.json index 4e50417b..cdfd51b8 100644 --- a/tests/data/test_config.json +++ b/tests/data/test_config.json @@ -1,161 +1,182 @@ { - "additional-repositories": ["testing"], - "archinstall-language": "English", - "audio_config": {"audio": "pipewire"}, - "bootloader": "Systemd-boot", - "services": ["service_1", "service_2"], - "disk_config": { - "config_type": "default_layout", - "device_modifications": [ - { - "device": "/dev/sda", - "partitions": [ - { - "btrfs": [], - "dev_path": null, - "flags": [ - "boot" - ], - "fs_type": "fat32", - "size": { - "sector_size": { - "unit": "B", - "value": 512 - }, - "unit": "MiB", - "value": 512 - }, - "mount_options": [], - "mountpoint": "/boot", - "obj_id": "2c3fa2d5-2c79-4fab-86ec-22d0ea1543c0", - "start": { - "sector_size": { - "unit": "B", - "value": 512 - }, - "unit": "MiB", - "value": 1 - }, - "status": "create", - "type": "primary" - }, - { - "btrfs": [], - "dev_path": null, - "flags": [], - "fs_type": "ext4", - "size": { - "sector_size": { - "unit": "B", - "value": 512 - }, - "unit": "GiB", - "value": 32 - }, - "mount_options": [], - "mountpoint": "/", - "obj_id": "3e7018a0-363b-4d05-ab83-8e82d13db208", - "start": { - "sector_size": { - "unit": "B", - "value": 512 - }, - "unit": "MiB", - "value": 513 - }, - "status": "create", - "type": "primary" - }, - { - "btrfs": [], - "dev_path": null, - "flags": [], - "fs_type": "ext4", - "size": { - "sector_size": { - "unit": "B", - "value": 512 - }, - "unit": "GiB", - "value": 32 - }, - "mount_options": [], - "mountpoint": "/home", - "obj_id": "ce58b139-f041-4a06-94da-1f8bad775d3f", - "start": { - "sector_size": { - "unit": "B", - "value": 512 - }, - "unit": "MiB", - "value": 33281 - }, - "status": "create", - "type": "primary" - } - ], - "wipe": true - } - ] - }, - "hostname": "archy", - "kernels": [ - "linux-zen" - ], - "locale_config": { - "kb_layout": "us", - "sys_enc": "UTF-8", - "sys_lang": "en_US" - }, - "mirror_config": { - "custom_mirrors": [], - "mirror_regions": { - "Australia": [ - "http://archlinux.mirror.digitalpacific.com.au/$repo/os/$arch" - ] + "archinstall-language": "English", + "audio_config": { + "audio": "pipewire" + }, + "bootloader": "Systemd-boot", + "services": [ + "service_1", + "service_2" + ], + "disk_config": { + "config_type": "default_layout", + "device_modifications": [ + { + "device": "/dev/sda", + "partitions": [ + { + "btrfs": [], + "dev_path": null, + "flags": [ + "boot" + ], + "fs_type": "fat32", + "size": { + "sector_size": { + "unit": "B", + "value": 512 + }, + "unit": "MiB", + "value": 512 + }, + "mount_options": [], + "mountpoint": "/boot", + "obj_id": "2c3fa2d5-2c79-4fab-86ec-22d0ea1543c0", + "start": { + "sector_size": { + "unit": "B", + "value": 512 + }, + "unit": "MiB", + "value": 1 + }, + "status": "create", + "type": "primary" + }, + { + "btrfs": [], + "dev_path": null, + "flags": [], + "fs_type": "ext4", + "size": { + "sector_size": { + "unit": "B", + "value": 512 + }, + "unit": "GiB", + "value": 32 + }, + "mount_options": [], + "mountpoint": "/", + "obj_id": "3e7018a0-363b-4d05-ab83-8e82d13db208", + "start": { + "sector_size": { + "unit": "B", + "value": 512 + }, + "unit": "MiB", + "value": 513 + }, + "status": "create", + "type": "primary" + }, + { + "btrfs": [], + "dev_path": null, + "flags": [], + "fs_type": "ext4", + "size": { + "sector_size": { + "unit": "B", + "value": 512 + }, + "unit": "GiB", + "value": 32 + }, + "mount_options": [], + "mountpoint": "/home", + "obj_id": "ce58b139-f041-4a06-94da-1f8bad775d3f", + "start": { + "sector_size": { + "unit": "B", + "value": 512 + }, + "unit": "MiB", + "value": 33281 + }, + "status": "create", + "type": "primary" + } + ], + "wipe": true } - }, - "network_config": { - "type": "manual", - "nics": [ + ] + }, + "hostname": "archy", + "kernels": [ + "linux-zen" + ], + "locale_config": { + "kb_layout": "us", + "sys_enc": "UTF-8", + "sys_lang": "en_US" + }, + "mirror_config": { + "custom_servers": [ { - "iface": "eno1", - "ip": "192.168.1.15/24", - "dhcp": true, - "gateway": "192.168.1.1", - "dns": [ - "192.168.1.1", - "9.9.9.9" - ] + "url": "https://mymirror.com/$repo/os/$arch" } + ], + "mirror_regions": { + "Australia": [ + "http://archlinux.mirror.digitalpacific.com.au/$repo/os/$arch" ] }, - "ntp": true, - "packages": ["firefox"], - "parallel_downloads": 66, - "profile_config": { - "gfx_driver": "All open-source", - "greeter": "lightdm-gtk-greeter", - "profile": { - "custom_settings": { - "Hyprland": { - "seat_access": "polkit" - }, - "Sway": { - "seat_access": "seatd" - } - }, - "details": [ - "Sway", - "Hyprland" - ], - "main": "Desktop" - } - }, - "custom_commands": [ - "echo 'Hello, World!'" + "optional_repositories": [ + "testing" ], - "swap": false, - "timezone": "UTC", - "version": "3.0.2" + "custom_repositories": [ + { + "name": "myrepo", + "url": "https://myrepo.com/$repo/os/$arch", + "sign_check": "Required", + "sign_option": "TrustAll" + } + ] + }, + "network_config": { + "type": "manual", + "nics": [ + { + "iface": "eno1", + "ip": "192.168.1.15/24", + "dhcp": true, + "gateway": "192.168.1.1", + "dns": [ + "192.168.1.1", + "9.9.9.9" + ] + } + ] + }, + "ntp": true, + "packages": [ + "firefox" + ], + "parallel_downloads": 66, + "profile_config": { + "gfx_driver": "All open-source", + "greeter": "lightdm-gtk-greeter", + "profile": { + "custom_settings": { + "Hyprland": { + "seat_access": "polkit" + }, + "Sway": { + "seat_access": "seatd" + } + }, + "details": [ + "Sway", + "Hyprland" + ], + "main": "Desktop" + } + }, + "custom_commands": [ + "echo 'Hello, World!'" + ], + "swap": false, + "timezone": "UTC", + "version": "3.0.2" } diff --git a/tests/data/test_config_mirror_backwards.json b/tests/data/test_config_mirror_backwards.json new file mode 100644 index 00000000..ee82a974 --- /dev/null +++ b/tests/data/test_config_mirror_backwards.json @@ -0,0 +1,18 @@ +{ + "additional-repositories": ["testing"], + "mirror_config": { + "custom_mirrors": [ + { + "name": "my_mirror", + "sign_check": "Optional", + "sign_option": "TrustedOnly", + "url": "example.com" + } + ], + "mirror_regions": { + "Australia": [ + "http://archlinux.mirror.digitalpacific.com.au/$repo/os/$arch" + ] + } + } +} diff --git a/tests/test_args.py b/tests/test_args.py index 8fbf4bba..8f84b4b6 100644 --- a/tests/test_args.py +++ b/tests/test_args.py @@ -5,9 +5,10 @@ from pytest import MonkeyPatch from archinstall.default_profiles.profile import GreeterType from archinstall.lib.args import ArchConfig, ArchConfigHandler, Arguments from archinstall.lib.hardware import GfxDriver -from archinstall.lib.mirrors import MirrorConfiguration, MirrorRegion -from archinstall.lib.models import Audio, AudioConfiguration, Bootloader, DiskLayoutConfiguration, DiskLayoutType, NetworkConfiguration, User +from archinstall.lib.mirrors import CustomRepository, MirrorConfiguration, MirrorRegion, SignCheck, SignOption +from archinstall.lib.models import Audio, AudioConfiguration, Bootloader, DiskLayoutConfiguration, DiskLayoutType, NetworkConfiguration, Repository, User from archinstall.lib.models.locale import LocaleConfiguration +from archinstall.lib.models.mirrors import CustomServer from archinstall.lib.models.network_configuration import Nic, NicType from archinstall.lib.models.profile_model import ProfileConfiguration from archinstall.lib.profile.profiles_handler import profile_handler @@ -149,7 +150,16 @@ def test_config_file_parsing( urls=['http://archlinux.mirror.digitalpacific.com.au/$repo/os/$arch'] ) ], - custom_mirrors=[] + custom_servers=[CustomServer('https://mymirror.com/$repo/os/$arch')], + optional_repositories=[Repository.Testing], + custom_repositories=[ + CustomRepository( + name='myrepo', + url='https://myrepo.com/$repo/os/$arch', + sign_check=SignCheck.Required, + sign_option=SignOption.TrustAll + ) + ] ), network_config=NetworkConfiguration( type=NicType.MANUAL, @@ -176,10 +186,42 @@ def test_config_file_parsing( parallel_downloads=66, swap=False, timezone='UTC', - additional_repositories=["testing"], users=[User(username='user_name', password='user_pwd', sudo=True)], disk_encryption=None, services=['service_1', 'service_2'], root_password='super_pwd', custom_commands=["echo 'Hello, World!'"] ) + + +def test_mirror_backwards_config_file_parsing( + monkeypatch: MonkeyPatch, + mirror_backwards_config: Path, +) -> None: + monkeypatch.setattr('sys.argv', [ + 'archinstall', + '--config', + str(mirror_backwards_config), + ]) + + handler = ArchConfigHandler() + arch_config = handler.config + + assert arch_config.mirror_config == MirrorConfiguration( + mirror_regions=[ + MirrorRegion( + name='Australia', + urls=['http://archlinux.mirror.digitalpacific.com.au/$repo/os/$arch'] + ) + ], + custom_servers=[], + optional_repositories=[Repository.Testing], + custom_repositories=[ + CustomRepository( + name='my_mirror', + url='example.com', + sign_check=SignCheck.Optional, + sign_option=SignOption.TrustedOnly + ) + ] + ) diff --git a/tests/test_configuration_output.py b/tests/test_configuration_output.py index 7733410e..d4ca5467 100644 --- a/tests/test_configuration_output.py +++ b/tests/test_configuration_output.py @@ -37,7 +37,13 @@ def test_user_config_roundtrip( result['disk_config']['config_type'] = expected['disk_config']['config_type'] result['disk_config']['device_modifications'] = expected['disk_config']['device_modifications'] - assert sorted(result.items()) == sorted(expected.items()) + assert json.dumps( + result['mirror_config'], + sort_keys=True + ) == json.dumps( + expected['mirror_config'], + sort_keys=True + ) def test_creds_roundtrip( diff --git a/tests/test_mirrorlist.py b/tests/test_mirrorlist.py index b7ddfa13..196e0ac9 100644 --- a/tests/test_mirrorlist.py +++ b/tests/test_mirrorlist.py @@ -1,6 +1,6 @@ from pathlib import Path -from archinstall.lib.models.mirrors import MirrorListHandler +from archinstall.lib.mirrors import MirrorListHandler def test_mirrorlist_no_country(mirrorlist_no_country_fixture: Path) -> None: