From b5f50889eb5fa544ff11199626329df5679dac8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Werner=20Ll=C3=A1cer?= Date: Mon, 24 Jan 2022 12:36:23 +0100 Subject: [PATCH 01/10] get rid of an exit() (#899) --- archinstall/lib/menu/selection_menu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archinstall/lib/menu/selection_menu.py b/archinstall/lib/menu/selection_menu.py index 648fed94..8128fefc 100644 --- a/archinstall/lib/menu/selection_menu.py +++ b/archinstall/lib/menu/selection_menu.py @@ -242,7 +242,7 @@ class GlobalMenu: if key not in archinstall.arguments: archinstall.arguments[key] = sel._current_selection self._post_processing() - exit() + def _process_selection(self, selection): # find the selected option in our option list option = [[k, v] for k, v in self._menu_options.items() if v.text.strip() == selection] From 26f9b681a45842d9a1ffd90e8b1397048b3c3645 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Tue, 25 Jan 2022 13:39:27 +0100 Subject: [PATCH 02/10] Removed last lsblk to grab PARTUUID (#901) Co-authored-by: Anton Hvornum --- archinstall/lib/disk/filesystem.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/archinstall/lib/disk/filesystem.py b/archinstall/lib/disk/filesystem.py index 3b09ec6c..3aa09b15 100644 --- a/archinstall/lib/disk/filesystem.py +++ b/archinstall/lib/disk/filesystem.py @@ -45,12 +45,14 @@ class Filesystem: self.partprobe() time.sleep(5) - # TODO: Convert to blkid (or something similar, but blkid doesn't support traversing to list sub-PARTUUIDs based on blockdevice path?) - output = json.loads(SysCommand(f"lsblk --json -o+PARTUUID {self.blockdevice.device}").decode('UTF-8')) + # We'll use unreliable lbslk to grab children under the /dev/ + output = json.loads(SysCommand(f"lsblk --json {self.blockdevice.device}").decode('UTF-8')) for device in output['blockdevices']: for index, partition in enumerate(device['children']): - if (partuuid := partition.get('partuuid', None)) and partuuid.lower() == uuid: + # But we'll use blkid to reliably grab the PARTUUID for that child device (partition) + partition_uuid = SysCommand(f"blkid -s PARTUUID -o value /dev/{partition.get('name')}").decode().strip() + if partition_uuid.lower() == uuid.lower(): return index time.sleep(storage['DISK_TIMEOUTS']) From eb712a9ca557f594d891b5b6942692b6c98de62c Mon Sep 17 00:00:00 2001 From: Victor Gavro Date: Tue, 25 Jan 2022 15:45:28 +0200 Subject: [PATCH 03/10] FIX: disk layout: partprobe should be called and checked only for target device (#896) * disk layout: partprobe should be called and checked only for target device * disk layout: partprobe: removed unnecessary bash subprocess --- archinstall/lib/disk/filesystem.py | 2 +- archinstall/lib/disk/partition.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/archinstall/lib/disk/filesystem.py b/archinstall/lib/disk/filesystem.py index 3aa09b15..edae5378 100644 --- a/archinstall/lib/disk/filesystem.py +++ b/archinstall/lib/disk/filesystem.py @@ -141,7 +141,7 @@ class Filesystem: return partition def partprobe(self) -> bool: - return SysCommand(f'bash -c "partprobe"').exit_code == 0 + return SysCommand(f'partprobe {self.blockdevice.device}').exit_code == 0 def raw_parted(self, string: str) -> SysCommand: if (cmd_handle := SysCommand(f'/usr/bin/parted -s {string}')).exit_code != 0: diff --git a/archinstall/lib/disk/partition.py b/archinstall/lib/disk/partition.py index 3117a42c..b3db143a 100644 --- a/archinstall/lib/disk/partition.py +++ b/archinstall/lib/disk/partition.py @@ -225,7 +225,7 @@ class Partition: return bind_name def partprobe(self) -> bool: - if SysCommand(f'bash -c "partprobe"').exit_code == 0: + if SysCommand(f'partprobe {self.block_device.device}').exit_code == 0: time.sleep(1) return True return False From 5406f1ed4541f46b28435434be04e82ee4378f1b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 25 Jan 2022 09:31:38 -0500 Subject: [PATCH 04/10] Make text for getting filesystems more grammatically correct (#903) * Make text for getting filesystems more grammatically correct Use 'Retry attempt {count} of 10.' instead of Waiting for the {count} time". It's more grammatically correct since we're not doing '1st', '2nd', etc. * Maybe mention waiting still --- archinstall/lib/disk/filesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archinstall/lib/disk/filesystem.py b/archinstall/lib/disk/filesystem.py index edae5378..a4fe2cff 100644 --- a/archinstall/lib/disk/filesystem.py +++ b/archinstall/lib/disk/filesystem.py @@ -200,7 +200,7 @@ class Filesystem: raise err else: count += 1 - log(f"Could not get UUID for partition. Waiting for the {count} time",level=logging.DEBUG) + log(f"Could not get UUID for partition. Waiting before retry attempt {count} of 10 ...",level=logging.DEBUG) time.sleep(float(storage['arguments'].get('disk-sleep', 0.2))) else: log("Add partition is exiting due to excessive wait time",level=logging.INFO) From 1aa738691e261b3dfaf5195ec7636617a283d47a Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Tue, 25 Jan 2022 16:09:34 +0100 Subject: [PATCH 05/10] Fixing the 'device_instance' being None in some partition places (#902) * Added a new return value from add_partition. Also added an exception to make sure `add_partition` can't continue silently * Added a log of debugging to add_partition * Removed a blank line (flake8) * Misconfigured variable * Added some more debugging information to partprobe * FIX: disk layout: partprobe should be called and checked only for target device (#896) * disk layout: partprobe should be called and checked only for target device * disk layout: partprobe: removed unnecessary bash subprocess * Properly defined BlockDevice() on Partition() creation. Also made sure mount-checks got some rrro handling and non-block devices should no longer attempt to return a size Co-authored-by: Anton Hvornum Co-authored-by: Victor Gavro --- .gitignore | 1 + archinstall/lib/disk/blockdevice.py | 2 +- archinstall/lib/disk/filesystem.py | 46 +++++++++++++++++++++-------- archinstall/lib/disk/helpers.py | 20 +++++++++++-- archinstall/lib/disk/partition.py | 14 +++++---- 5 files changed, 63 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index d2f817c4..ef051974 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ venv .idea/** **/install.log .DS_Store +**/cmd_history.txt diff --git a/archinstall/lib/disk/blockdevice.py b/archinstall/lib/disk/blockdevice.py index 5288f92b..5ffa06a8 100644 --- a/archinstall/lib/disk/blockdevice.py +++ b/archinstall/lib/disk/blockdevice.py @@ -241,7 +241,7 @@ class BlockDevice: count = 0 while count < 5: for partition_uuid, partition in self.partitions.items(): - if partition.uuid == uuid: + if partition.uuid.lower() == uuid.lower(): return partition else: log(f"uuid {uuid} not found. Waiting for {count +1} time",level=logging.DEBUG) diff --git a/archinstall/lib/disk/filesystem.py b/archinstall/lib/disk/filesystem.py index a4fe2cff..b5dcdf85 100644 --- a/archinstall/lib/disk/filesystem.py +++ b/archinstall/lib/disk/filesystem.py @@ -89,12 +89,16 @@ class Filesystem: print("Re-using partition_instance:", partition_instance) partition['device_instance'] = partition_instance else: - raise ValueError(f"{self}.load_layout() doesn't know how to continue without a new partition definition or a UUID ({partition.get('PARTUUID')}) on the device ({self.blockdevice.get_partition(uuid=partition_uuid)}).") + raise ValueError(f"{self}.load_layout() doesn't know how to continue without a new partition definition or a UUID ({partition.get('PARTUUID')}) on the device ({self.blockdevice.get_partition(uuid=partition.get('PARTUUID'))}).") if partition.get('filesystem', {}).get('format', False): + # needed for backward compatibility with the introduction of the new "format_options" format_options = partition.get('options',[]) + partition.get('filesystem',{}).get('format_options',[]) if partition.get('encrypted', False): + if not partition['device_instance']: + raise DiskError(f"Internal error caused us to loose the partition. Please report this issue upstream!") + if not partition.get('!password'): if not storage['arguments'].get('!encryption-password'): if storage['arguments'] == 'silent': @@ -109,6 +113,7 @@ class Filesystem: loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(partition['mountpoint']).name}loop" else: loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(partition['device_instance'].path).name}" + partition['device_instance'].encrypt(password=partition['!password']) # Immediately unlock the encrypted device to format the inner volume with luks2(partition['device_instance'], loopdev, partition['!password'], auto_unmount=True) as unlocked_device: @@ -129,6 +134,9 @@ class Filesystem: unlocked_device.format(partition['filesystem']['format'], options=format_options) elif partition.get('format', False): + if not partition['device_instance']: + raise DiskError(f"Internal error caused us to loose the partition. Please report this issue upstream!") + partition['device_instance'].format(partition['filesystem']['format'], options=format_options) if partition.get('boot', False): @@ -141,7 +149,13 @@ class Filesystem: return partition def partprobe(self) -> bool: - return SysCommand(f'partprobe {self.blockdevice.device}').exit_code == 0 + result = SysCommand(f'partprobe {self.blockdevice.device}') + + if result.exit_code != 0: + log(f"Could not execute partprobe: {result!r}", level=logging.ERROR, fg="red") + raise DiskError(f"Could not run partprobe: {result!r}") + + return True def raw_parted(self, string: str) -> SysCommand: if (cmd_handle := SysCommand(f'/usr/bin/parted -s {string}')).exit_code != 0: @@ -157,9 +171,7 @@ class Filesystem: :type string: str """ if (parted_handle := self.raw_parted(string)).exit_code == 0: - if self.partprobe(): - return True - return False + return self.partprobe() else: raise DiskError(f"Parted failed to add a partition: {parted_handle}") @@ -167,7 +179,7 @@ class Filesystem: # TODO: Implement this with declarative profiles instead. raise ValueError("Installation().use_entire_disk() has to be re-worked.") - def add_partition(self, partition_type :str, start :str, end :str, partition_format :Optional[str] = None) -> None: + def add_partition(self, partition_type :str, start :str, end :str, partition_format :Optional[str] = None) -> Partition: log(f'Adding partition to {self.blockdevice}, {start}->{end}', level=logging.INFO) previous_partition_uuids = {partition.uuid for partition in self.blockdevice.partitions.values()} @@ -181,31 +193,41 @@ class Filesystem: else: parted_string = f'{self.blockdevice.device} mkpart {partition_type} {start} {end}' + log(f"Adding partition using the following parted command: {parted_string}", level=logging.DEBUG) + if self.parted(parted_string): count = 0 while count < 10: new_uuid = None new_uuid_set = (previous_partition_uuids ^ {partition.uuid for partition in self.blockdevice.partitions.values()}) + if len(new_uuid_set) > 0: new_uuid = new_uuid_set.pop() + if new_uuid: try: return self.blockdevice.get_partition(new_uuid) except Exception as err: - print('Blockdevice:', self.blockdevice) - print('Partitions:', self.blockdevice.partitions) - print('Partition set:', new_uuid_set) - print('New UUID:', [new_uuid]) - print('get_partition():', self.blockdevice.get_partition) + log(f'Blockdevice: {self.blockdevice}', level=logging.ERROR, fg="red") + log(f'Partitions: {self.blockdevice.partitions}', level=logging.ERROR, fg="red") + log(f'Partition set: {new_uuid_set}', level=logging.ERROR, fg="red") + log(f'New UUID: {[new_uuid]}', level=logging.ERROR, fg="red") + log(f'get_partition(): {self.blockdevice.get_partition}', level=logging.ERROR, fg="red") raise err else: count += 1 log(f"Could not get UUID for partition. Waiting before retry attempt {count} of 10 ...",level=logging.DEBUG) time.sleep(float(storage['arguments'].get('disk-sleep', 0.2))) else: - log("Add partition is exiting due to excessive wait time",level=logging.INFO) + log("Add partition is exiting due to excessive wait time", level=logging.ERROR, fg="red") raise DiskError(f"New partition never showed up after adding new partition on {self}.") + # TODO: This should never be able to happen + log(f"Could not find the new PARTUUID after adding the partition.", level=logging.ERROR, fg="red") + log(f"Previous partitions: {previous_partition_uuids}", level=logging.ERROR, fg="red") + log(f"New partitions: {(previous_partition_uuids ^ {partition.uuid for partition in self.blockdevice.partitions.values()})}", level=logging.ERROR, fg="red") + raise DiskError(f"Could not add partition using: {parted_string}") + def set_name(self, partition: int, name: str) -> bool: return self.parted(f'{self.blockdevice.device} name {partition + 1} "{name}"') == 0 diff --git a/archinstall/lib/disk/helpers.py b/archinstall/lib/disk/helpers.py index e9f6bc10..26f701d2 100644 --- a/archinstall/lib/disk/helpers.py +++ b/archinstall/lib/disk/helpers.py @@ -190,10 +190,26 @@ def get_partitions_in_use(mountpoint :str) -> List[Partition]: output = json.loads(output) for target in output.get('filesystems', []): - mounts.append(Partition(target['source'], None, filesystem=target.get('fstype', None), mountpoint=target['target'])) + # We need to create a BlockDevice() instead of 'None' here when creaiting Partition() + # Otherwise subsequent calls to .size etc will fail due to BlockDevice being None. + + # So first, we create the partition without a BlockDevice and carefully only use it to get .real_device + # Note: doing print(partition) here will break because the above mentioned issue. + partition = Partition(target['source'], None, filesystem=target.get('fstype', None), mountpoint=target['target']) + partition = Partition(target['source'], partition.real_device, filesystem=target.get('fstype', None), mountpoint=target['target']) + + # Once we have the real device (for instance /dev/nvme0n1p5) we can find the parent block device using + # (lsblk pkname lists both the partition and blockdevice, BD being the last entry) + result = SysCommand(f'lsblk -no pkname {partition.real_device}').decode().rstrip('\r\n').split('\r\n')[-1] + block_device = BlockDevice(f"/dev/{result}") + + # Once we figured the block device out, we can properly create the partition object + partition = Partition(target['source'], block_device, filesystem=target.get('fstype', None), mountpoint=target['target']) + + mounts.append(partition) for child in target.get('children', []): - mounts.append(Partition(child['source'], None, filesystem=child.get('fstype', None), mountpoint=child['target'])) + mounts.append(Partition(child['source'], block_device, filesystem=child.get('fstype', None), mountpoint=child['target'])) return mounts diff --git a/archinstall/lib/disk/partition.py b/archinstall/lib/disk/partition.py index b3db143a..afd34f20 100644 --- a/archinstall/lib/disk/partition.py +++ b/archinstall/lib/disk/partition.py @@ -46,11 +46,12 @@ class Partition: except DiskError: mount_information = {} - if self.mountpoint != mount_information.get('target', None) and mountpoint: - raise DiskError(f"{self} was given a mountpoint but the actual mountpoint differs: {mount_information.get('target', None)}") + if mount_information.get('target', None): + if self.mountpoint != mount_information.get('target', None) and mountpoint: + raise DiskError(f"{self} was given a mountpoint but the actual mountpoint differs: {mount_information.get('target', None)}") - if target := mount_information.get('target', None): - self.mountpoint = target + if target := mount_information.get('target', None): + self.mountpoint = target if not self.filesystem and autodetect_filesystem: if fstype := mount_information.get('fstype', get_filesystem_type(path)): @@ -130,6 +131,9 @@ class Partition: for device in lsblk['blockdevices']: return convert_size_to_gb(device['size']) + elif handle.exit_code == 8192: + # Device is not a block device + return None time.sleep(storage['DISK_TIMEOUTS']) @@ -225,7 +229,7 @@ class Partition: return bind_name def partprobe(self) -> bool: - if SysCommand(f'partprobe {self.block_device.device}').exit_code == 0: + if self.block_device and SysCommand(f'partprobe {self.block_device.device}').exit_code == 0: time.sleep(1) return True return False From e629729d555987778262ff2db7e5a78726b2ea0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Werner=20Ll=C3=A1cer?= Date: Tue, 25 Jan 2022 18:12:45 +0100 Subject: [PATCH 06/10] Correct definition of btrfs standard layout (#906) --- archinstall/lib/disk/user_guides.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archinstall/lib/disk/user_guides.py b/archinstall/lib/disk/user_guides.py index 49ee10e4..4ba22b4f 100644 --- a/archinstall/lib/disk/user_guides.py +++ b/archinstall/lib/disk/user_guides.py @@ -66,7 +66,7 @@ def suggest_single_disk_layout(block_device :BlockDevice, "start" : "206MiB", "encrypted" : False, "format" : True, - "mountpoint" : "/", + "mountpoint" : "/" if not using_subvolumes else None, "filesystem" : { "format" : default_filesystem } From 90e320dd110ee99537e9c55b158fed6edaa53b78 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 26 Jan 2022 01:53:38 -0500 Subject: [PATCH 07/10] Allow mirror reachability check to be skipped (#910) * Allow network check to be skipped * Technically it is a mirror check - not a check for general connectivity --- examples/guided.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/guided.py b/examples/guided.py index 723fa629..9092d8f6 100644 --- a/examples/guided.py +++ b/examples/guided.py @@ -304,7 +304,7 @@ def perform_installation(mountpoint): archinstall.log(f"Disk states after installing: {archinstall.disk_layouts()}", level=logging.DEBUG) -if not archinstall.check_mirror_reachable(): +if not (archinstall.check_mirror_reachable() or archinstall.arguments.get('skip-mirror-check', False)): log_file = os.path.join(archinstall.storage.get('LOG_PATH', None), archinstall.storage.get('LOG_FILE', None)) archinstall.log(f"Arch Linux mirrors are not reachable. Please check your internet connection and the log file '{log_file}'.", level=logging.INFO, fg="red") exit(1) From f0886f3a7f44df4e9bf212df8f54c8984de03d0a Mon Sep 17 00:00:00 2001 From: OneLongneck <95297344+OneLongneck@users.noreply.github.com> Date: Wed, 26 Jan 2022 09:38:01 +0100 Subject: [PATCH 08/10] Update installer.py (#1) (#793) Added "--removable" after "--bootloader-id=GRUB" on Line 669, because it would throw an input/output error without it on my laptop --- archinstall/lib/installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archinstall/lib/installer.py b/archinstall/lib/installer.py index 438f2119..6a580ac0 100644 --- a/archinstall/lib/installer.py +++ b/archinstall/lib/installer.py @@ -767,7 +767,7 @@ class Installer: log(f"GRUB uses {boot_partition.path} as the boot partition.", level=logging.INFO) if has_uefi(): self.pacstrap('efibootmgr') # TODO: Do we need? Yes, but remove from minimal_installation() instead? - if not (handle := SysCommand(f'/usr/bin/arch-chroot {self.target} grub-install --target=x86_64-efi --efi-directory=/boot --bootloader-id=GRUB')).exit_code == 0: + if not (handle := SysCommand(f'/usr/bin/arch-chroot {self.target} grub-install --target=x86_64-efi --efi-directory=/boot --bootloader-id=GRUB --removable')).exit_code == 0: raise DiskError(f"Could not install GRUB to {self.target}/boot: {handle}") else: if not (handle := SysCommand(f'/usr/bin/arch-chroot {self.target} grub-install --target=i386-pc --recheck {boot_partition.parent}')).exit_code == 0: From fe9fb988c9f46fe39058cba48c92fcda5f3a0b2d Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Thu, 27 Jan 2022 12:01:06 -0500 Subject: [PATCH 09/10] Move from the basic pipewire media session manager to wireplumber (#914) --- profiles/applications/pipewire.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/profiles/applications/pipewire.py b/profiles/applications/pipewire.py index 38554000..b6e79347 100644 --- a/profiles/applications/pipewire.py +++ b/profiles/applications/pipewire.py @@ -3,7 +3,7 @@ import logging # Define the package list in order for lib to source # which packages will be installed by this profile -__packages__ = ["pipewire", "pipewire-alsa", "pipewire-jack", "pipewire-media-session", "pipewire-pulse", "gst-plugin-pipewire", "libpulse"] +__packages__ = ["pipewire", "pipewire-alsa", "pipewire-jack", "pipewire-pulse", "gst-plugin-pipewire", "libpulse", "wireplumber"] archinstall.log('Installing pipewire', level=logging.INFO) archinstall.storage['installation_session'].add_additional_packages(__packages__) @@ -11,4 +11,4 @@ archinstall.storage['installation_session'].add_additional_packages(__packages__ @archinstall.plugin def on_user_created(installation :archinstall.Installer, user :str): archinstall.log(f"Enabling pipewire-pulse for {user}", level=logging.INFO) - installation.chroot('systemctl enable --user pipewire-pulse.service', run_as=user) \ No newline at end of file + installation.chroot('systemctl enable --user pipewire-pulse.service', run_as=user) From 275a960e6cb7163f4f37c4d16a4a62f599046613 Mon Sep 17 00:00:00 2001 From: Sebas-h Date: Thu, 27 Jan 2022 21:41:13 +0100 Subject: [PATCH 10/10] Fix trailing comma in schema.json (#915) (Tried to [generate docs](https://github.com/coveooss/json-schema-for-humans) and noticed the comma preventing that) --- schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema.json b/schema.json index 98a25abd..9731e5f2 100644 --- a/schema.json +++ b/schema.json @@ -147,7 +147,7 @@ "required": [ "bootloader", "kernels", - "mirror-region", + "mirror-region" ], "anyOf": [ {