Btrfs II (#838)

* Btrfs with encrypted partitions.
We have changed installer.mount_ordered_layout into a series of loops
* open the encrypted devices
* manage btrfs subvolumes
* mount whatever
* create kyefiles for encrypted volumes
We have simplified the btrfs subvolume manager

We merged the locale branch as it is needed here

* We allow only the creation of keyfiles if the partition does not contain the root mount point.
Also, adapt examples/only_hd to the new __init__.py
Also, assorted flake8 warnings

* Cleanup code

* Naming schema for encrypted volumes
revert global locale association (provisional)

* We introduce the option of defining mount options in the partition dictionary.
It has forced us to define two new entries in this dictionary:
* format_options (formerly options) for mkfs options and
* mount_options for mount -o ones.
The different meaning of compress between partition and subvolumes is treated

* Function lib/disk/btrfs.py mount_subvolume marked as deprecated
Code cleanup.

* format_options now filesystem.options

* format_options now filesystem.format_options
mount_options  nof filesystem.mount_options

* flake8 uncovered a slip in the code

archinstall/lib/disk/btrfs.py

@@ -11,7 +11,6 @@ from .helpers import get_mount_info
 from ..exceptions import DiskError
 from ..general import SysCommand
 from ..output import log
-from .partition import Partition
 
 
 def mount_subvolume(installation :Installer, subvolume_location :Union[pathlib.Path, str], force=False) -> bool:
@@ -21,8 +20,11 @@ def mount_subvolume(installation :Installer, subvolume_location :Union[pathlib.P
 	@installation: archinstall.Installer instance
 	@subvolume_location: a localized string or path inside the installation / or /boot for instance without specifying /mnt/boot
 	@force: overrides the check for weither or not the subvolume mountpoint is empty or not
-	"""
 
+	This function is DEPRECATED. you can get the same result creating a partition dict like any other partition, and using the standard mount procedure.
+	Only change partition['device_instance'].path with the apropriate bind name: real_partition_path[/subvolume_name]
+	"""
+	log("function btrfs.mount_subvolume DEPRECATED. See code for alternatives",fg="yellow",level=logging.WARNING)
 	installation_mountpoint = installation.target
 	if type(installation_mountpoint) == str:
 		installation_mountpoint = pathlib.Path(installation_mountpoint)
@@ -80,27 +82,38 @@ def create_subvolume(installation :Installer, subvolume_location :Union[pathlib.
 	if (cmd := SysCommand(f"btrfs subvolume create {target}")).exit_code != 0:
 		raise DiskError(f"Could not create a subvolume at {target}: {cmd}")
 
+def _has_option(option :str,options :list) -> bool:
+	""" auxiliary routine to check if an option is present in a list.
+	we check if the string appears in one of the options, 'cause it can appear in severl forms (option, option=val,...)
+	"""
+	if not options:
+		return False
+	for item in options:
+		if option in item:
+			return True
+	return False
+
 def manage_btrfs_subvolumes(installation :Installer,
-	partition :Dict[str, str],
+	partition :Dict[str, str],) -> list:
-	mountpoints :Dict[str, str],
+	from copy import deepcopy
-	subvolumes :Dict[str, str],
-	unlocked_device :Dict[str, str] = None
-) -> None:
 	""" we do the magic with subvolumes in a centralized place
 	parameters:
 	* the installation object
 	* the partition dictionary entry which represents the physical partition
-	* mountpoinst, the dictionary which contains all the partititon to be mounted
+	returns
-	* subvolumes is the dictionary with the names of the subvolumes and its location
+	* mountpoinst, the list which contains all the "new" partititon to be mounted
+
 	We expect the partition has been mounted as / , and it to be unmounted after the processing
 	Then we create all the subvolumes inside btrfs as demand
 	We clone then, both the partition dictionary and the object inside it and adapt it to the subvolume needs
-	Then we add it them to the mountpoints dictionary to be processed as "normal" partitions
+	Then we return a list of "new" partitions to be processed as "normal" partitions
 	# TODO For encrypted devices we need some special processing prior to it
 	"""
 	# We process each of the pairs <subvolume name: mount point | None | mount info dict>
 	# th mount info dict has an entry for the path of the mountpoint (named 'mountpoint') and 'options' which is a list
 	# of mount options (or similar used by brtfs)
+	mountpoints = []
+	subvolumes = partition['btrfs']['subvolumes']
 	for name, right_hand in subvolumes.items():
 		try:
 			# we normalize the subvolume name (getting rid of slash at the start if exists. In our implemenation has no semantic load - every subvolume is created from the top of the hierarchy- and simplifies its further use
@@ -108,7 +121,7 @@ def manage_btrfs_subvolumes(installation :Installer,
 				name = name[1:]
 			# renormalize the right hand.
 			location = None
-			mount_options = []
+			subvol_options = []
 			# no contents, so it is not to be mounted
 			if not right_hand:
 				location = None
@@ -118,38 +131,37 @@ def manage_btrfs_subvolumes(installation :Installer,
 			# a dict. two elements 'mountpoint' (obvious) and and a mount options list ¿?
 			elif isinstance(right_hand,dict):
 				location = right_hand.get('mountpoint',None)
-				mount_options = right_hand.get('options',[])
+				subvol_options = right_hand.get('options',[])
 			# we create the subvolume
 			create_subvolume(installation,name)
 			# Make the nodatacow processing now
 			# It will be the main cause of creation of subvolumes which are not to be mounted
 			# it is not an options which can be established by subvolume (but for whole file systems), and can be
 			# set up via a simple attribute change in a directory (if empty). And here the directories are brand new
-			if 'nodatacow' in mount_options:
+			if 'nodatacow' in subvol_options:
 				if (cmd := SysCommand(f"chattr +C {installation.target}/{name}")).exit_code != 0:
 					raise DiskError(f"Could not set  nodatacow attribute at {installation.target}/{name}: {cmd}")
 				# entry is deleted so nodatacow doesn't propagate to the mount options
-				del mount_options[mount_options.index('nodatacow')]
+				del subvol_options[subvol_options.index('nodatacow')]
 			# Make the compress processing now
 			# it is not an options which can be established by subvolume (but for whole file systems), and can be
 			# set up via a simple attribute change in a directory (if empty). And here the directories are brand new
 			# in this way only zstd compression is activaded
 			# TODO WARNING it is not clear if it should be a standard feature, so it might need to be deactivated
-			if 'compress' in mount_options:
+			if 'compress' in subvol_options:
-				if (cmd := SysCommand(f"chattr +c {installation.target}/{name}")).exit_code != 0:
+				if not _has_option('compress',partition.get('filesystem',{}).get('mount_options',[])):
-					raise DiskError(f"Could not set compress attribute at {installation.target}/{name}: {cmd}")
+					if (cmd := SysCommand(f"chattr +c {installation.target}/{name}")).exit_code != 0:
-				# entry is deleted so nodatacow doesn't propagate to the mount options
+						raise DiskError(f"Could not set compress attribute at {installation.target}/{name}: {cmd}")
-				del mount_options[mount_options.index('compress')]
+				# entry is deleted so compress doesn't propagate to the mount options
+				del subvol_options[subvol_options.index('compress')]
 			# END compress processing.
 			# we do not mount if THE basic partition will be mounted or if we exclude explicitly this subvolume
 			if not partition['mountpoint'] and location is not None:
 				# we begin to create a fake partition entry. First we copy the original -the one that corresponds to
-				# the primary partition
+				# the primary partition. We make a deepcopy to avoid altering the original content in any case
-				fake_partition = partition.copy()
+				fake_partition = deepcopy(partition)
 				# we start to modify entries in the "fake partition" to match the needs of the subvolumes
-				#
 				# to avoid any chance of entering in a loop (not expected) we delete the list of subvolumes in the copy
-				# and reset the encryption parameters
 				del fake_partition['btrfs']
 				fake_partition['encrypted'] = False
 				fake_partition['generate-encryption-key-file'] = False
@@ -157,22 +169,16 @@ def manage_btrfs_subvolumes(installation :Installer,
 				fake_partition['mountpoint'] = location
 				# we load the name in an attribute called subvolume, but i think it is not needed anymore, 'cause the mount logic uses a different path.
 				fake_partition['subvolume'] = name
-				# here we add the mount options
+				# here we add the special mount options for the subvolume, if any.
-				fake_partition['options'] = mount_options
+				# if the original partition['options'] is not a list might give trouble
-				# Here comes the most exotic part. The dictionary attribute 'device_instance' contains an instance of Partition. This instance will be queried along the mount process at the installer.
+				if fake_partition.get('filesystem',{}).get('mount_options',[]):
-				# We instanciate a new object with following attributes coming / adapted from the instance which was in the primary partition entry (the one we are coping - partition['device_instance']
+					fake_partition['filesystem']['mount_options'].extend(subvol_options)
-				# * path, which will be expanded with the subvolume name to use the bind mount syntax the system uses for naming mounted subvolumes
-				# * size. When the OS queries all the subvolumes share the same size as the full partititon
-				# * uuid. All the subvolumes on a partition share the same uuid
-				if not unlocked_device:
-					fake_partition['device_instance'] = Partition(f"{partition['device_instance'].path}[/{name}]",partition['device_instance'].size,partition['device_instance'].uuid)
 				else:
-					# for subvolumes IN an encrypted partition we make our device instance from unlocked device instead of the raw partition.
+					fake_partition['filesystem']['mount_options'] = subvol_options
-					# This time we make a copy (we should to the same above TODO) and alter the path by hand
+				# Here comes the most exotic part. The dictionary attribute 'device_instance' contains an instance of Partition. This instance will be queried along the mount process at the installer.
-					from copy import copy
+				# As the rest will query there the path of the "partition" to be mounted, we feed it with the bind name needed to mount subvolumes
-					# KIDS DONT'T DO THIS AT HOME
+				# As we made a deepcopy we have a fresh instance of this object we can manipulate problemless
-					fake_partition['device_instance'] = copy(unlocked_device)
+				fake_partition['device_instance'].path = f"{partition['device_instance'].path}[/{name}]"
-					fake_partition['device_instance'].path = f"{unlocked_device.path}[/{name}]"
 				# we reset this attribute, which holds where the partition is actually mounted. Remember, the physical partition is mounted at this moment and therefore has the value '/'.
 				# If i don't reset it, process will abort as "already mounted' .
 				# TODO It works for this purpose, but the fact that this bevahiour can happed, should make think twice
@@ -180,9 +186,7 @@ def manage_btrfs_subvolumes(installation :Installer,
 				#
 				# Well, now that this "fake partition" is ready, we add it to the list of the ones which are to be mounted,
 				# as "normal" ones
-				mountpoints[fake_partition['mountpoint']] = fake_partition
+				mountpoints.append(fake_partition)
 		except Exception as e:
 			raise e
-	# if the physical partition has been selected to be mounted, we include it at the list. Remmeber, all the above treatement won't happen except the creation of the subvolume
+	return mountpoints
-	if partition['mountpoint']:
-		mountpoints[partition['mountpoint']] = partition

archinstall/lib/disk/filesystem.py

@@ -90,6 +90,8 @@ class Filesystem:
 				raise ValueError(f"{self}.load_layout() doesn't know how to continue without a new partition definition or a UUID ({partition.get('PARTUUID')}) on the device ({self.blockdevice.get_partition(uuid=partition_uuid)}).")
 
 			if partition.get('filesystem', {}).get('format', False):
+				# needed for backward compatibility with the introduction of the new "format_options"
+				format_options = partition.get('options',[]) + partition.get('filesystem',{}).get('format_options',[])
 				if partition.get('encrypted', False):
 					if not partition.get('!password'):
 						if not storage['arguments'].get('!encryption-password'):
@@ -100,15 +102,12 @@ class Filesystem:
 							storage['arguments']['!encryption-password'] = get_password(f"Enter a encryption password for {partition['device_instance']}")
 
 						partition['!password'] = storage['arguments']['!encryption-password']
-					# to be able to generate an unique name in case the partition will not be mounted
+
 					if partition.get('mountpoint',None):
-						ppath = partition['mountpoint']
+						loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(partition['mountpoint']).name}loop"
 					else:
-						ppath = partition['device_instance'].path
+						loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(partition['device_instance'].path).name}"
-					loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(ppath).name}loop"
-
 					partition['device_instance'].encrypt(password=partition['!password'])
-
 					# Immediately unlock the encrypted device to format the inner volume
 					with luks2(partition['device_instance'], loopdev, partition['!password'], auto_unmount=True) as unlocked_device:
 						if not partition.get('format'):
@@ -126,9 +125,9 @@ class Filesystem:
 											continue
 										break
 
-						unlocked_device.format(partition['filesystem']['format'], options=partition.get('options', []))
+						unlocked_device.format(partition['filesystem']['format'], options=format_options)
 				elif partition.get('format', False):
-					partition['device_instance'].format(partition['filesystem']['format'], options=partition.get('options', []))
+					partition['device_instance'].format(partition['filesystem']['format'], options=format_options)
 
 			if partition.get('boot', False):
 				log(f"Marking partition {partition['device_instance']} as bootable.")

archinstall/lib/installer.py

@@ -180,80 +180,101 @@ class Installer:
 
 		return True
 
+	def _create_keyfile(self,luks_handle , partition :dict, password :str):
+		""" roiutine to create keyfiles, so it can be moved elsewere
+		"""
+		if partition.get('generate-encryption-key-file'):
+			if not (cryptkey_dir := pathlib.Path(f"{self.target}/etc/cryptsetup-keys.d")).exists():
+				cryptkey_dir.mkdir(parents=True)
+			# Once we store the key as ../xyzloop.key systemd-cryptsetup can automatically load this key
+			# if we name the device to "xyzloop".
+			if partition.get('mountpoint',None):
+				encryption_key_path = f"/etc/cryptsetup-keys.d/{pathlib.Path(partition['mountpoint']).name}loop.key"
+			else:
+				encryption_key_path = f"/etc/cryptsetup-keys.d/{pathlib.Path(partition['device_instance'].path).name}.key"
+			with open(f"{self.target}{encryption_key_path}", "w") as keyfile:
+				keyfile.write(generate_password(length=512))
+
+			os.chmod(f"{self.target}{encryption_key_path}", 0o400)
+
+			luks_handle.add_key(pathlib.Path(f"{self.target}{encryption_key_path}"), password=password)
+			luks_handle.crypttab(self, encryption_key_path, options=["luks", "key-slot=1"])
+
+	def _has_root(self, partition :dict) -> bool:
+		"""
+		Determine if an encrypted partition contains root in it
+		"""
+		if partition.get("mountpoint") is None:
+			if (sub_list := partition.get("btrfs",{}).get('subvolumes',{})):
+				for mountpoint in [sub_list[subvolume] if isinstance(sub_list[subvolume],str) else sub_list[subvolume].get("mountpoint") for subvolume in sub_list]:
+					if mountpoint == '/':
+						return True
+				return False
+			else:
+				return False
+		elif partition.get("mountpoint") == '/':
+			return True
+		else:
+			return False
+
 	def mount_ordered_layout(self, layouts: Dict[str, Any]) -> None:
 		from .luks import luks2
-
+		# set the partitions as a list not part of a tree (which we don't need anymore (i think)
-		mountpoints = {}
+		list_part = []
+		list_luks_handles = []
 		for blockdevice in layouts:
-			for partition in layouts[blockdevice]['partitions']:
+			list_part.extend(layouts[blockdevice]['partitions'])
-				if (subvolumes := partition.get('btrfs', {}).get('subvolumes', {})):
+
-					if partition.get('encrypted',False):
+		# we manage the encrypted partititons
-						if partition.get('mountpoint',None):
+		for partition in [entry for entry in list_part if entry.get('encrypted',False)]:
-							ppath = partition['mountpoint']
+			# open the luks device and all associate stuff
-						else:
+			if not (password := partition.get('!password', None)):
-							ppath = partition['device_instance'].path
+				raise RequirementError(f"Missing partition {partition['device_instance'].path} encryption password in layout: {partition}")
-						loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(ppath).name}loop"
+			# i change a bit the naming conventions for the loop device
-						# Immediately unlock the encrypted device to format the inner volume
-						with luks2(partition['device_instance'], loopdev, partition['!password'], auto_unmount=False) as unlocked_device:
-							unlocked_device.mount(f"{self.target}/")
-							try:
-								manage_btrfs_subvolumes(self,partition,mountpoints,subvolumes,unlocked_device)
-							except Exception as e:
-								# every exception unmounts the physical volume. Otherwise we let the system in an unstable state
-								unlocked_device.unmount()
-								raise e
-							unlocked_device.unmount()
-						# TODO generate key
-					else:
-						self.mount(partition['device_instance'],"/")
-						try:
-							manage_btrfs_subvolumes(self,partition,mountpoints,subvolumes)
-						except Exception as e:
-							# every exception unmounts the physical volume. Otherwise we let the system in an unstable state
-							partition['device_instance'].unmount()
-							raise e
-						partition['device_instance'].unmount()
-				else:
-					mountpoints[partition['mountpoint']] = partition
-		for mountpoint in sorted([mnt_dest for mnt_dest in mountpoints.keys() if mnt_dest is not None]):
-			partition = mountpoints[mountpoint]
-			if partition.get('encrypted', False) and not partition.get('subvolume',None):
 				loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(partition['mountpoint']).name}loop"
-				if not (password := partition.get('!password', None)):
-					raise RequirementError(f"Missing mountpoint {mountpoint} encryption password in layout: {partition}")
-
-				with (luks_handle := luks2(partition['device_instance'], loopdev, password, auto_unmount=False)) as unlocked_device:
-					if partition.get('generate-encryption-key-file'):
-						if not (cryptkey_dir := pathlib.Path(f"{self.target}/etc/cryptsetup-keys.d")).exists():
-							cryptkey_dir.mkdir(parents=True)
-
-						# Once we store the key as ../xyzloop.key systemd-cryptsetup can automatically load this key
-						# if we name the device to "xyzloop".
-						encryption_key_path = f"/etc/cryptsetup-keys.d/{pathlib.Path(partition['mountpoint']).name}loop.key"
-						with open(f"{self.target}{encryption_key_path}", "w") as keyfile:
-							keyfile.write(generate_password(length=512))
-
-						os.chmod(f"{self.target}{encryption_key_path}", 0o400)
-
-						luks_handle.add_key(pathlib.Path(f"{self.target}{encryption_key_path}"), password=password)
-						luks_handle.crypttab(self, encryption_key_path, options=["luks", "key-slot=1"])
-
-					log(f"Mounting {mountpoint} to {self.target}{mountpoint} using {unlocked_device}", level=logging.INFO)
-					unlocked_device.mount(f"{self.target}{mountpoint}")
-
 			else:
-				log(f"Mounting {mountpoint} to {self.target}{mountpoint} using {partition['device_instance']}", level=logging.INFO)
+				loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(partition['device_instance'].path).name}"
-				if partition.get('options',[]):
+			# note that we DON'T auto_unmount (i.e. close the encrypted device so it can be used
-					mount_options = ','.join(partition['options'])
+			with (luks_handle := luks2(partition['device_instance'], loopdev, password, auto_unmount=False)) as unlocked_device:
-					partition['device_instance'].mount(f"{self.target}{mountpoint}",options=mount_options)
+				if partition.get('generate-encryption-key-file',False) and not self._has_root(partition):
-				else:
+					list_luks_handles.append([luks_handle,partition,password])
-					partition['device_instance'].mount(f"{self.target}{mountpoint}")
+				# this way all the requesrs will be to the dm_crypt device and not to the physical partition
+				partition['device_instance'] = unlocked_device
+
+		# we manage the btrfs partitions
+		for partition in [entry for entry in list_part if entry.get('btrfs', {}).get('subvolumes', {})]:
+			self.mount(partition['device_instance'],"/")
+			try:
+				new_mountpoints = manage_btrfs_subvolumes(self,partition)
+			except Exception as e:
+				# every exception unmounts the physical volume. Otherwise we let the system in an unstable state
+				partition['device_instance'].unmount()
+				raise e
+			partition['device_instance'].unmount()
+			if new_mountpoints:
+				list_part.extend(new_mountpoints)
+
+		# we mount. We need to sort by mountpoint to get a good working order
+		for partition in sorted([entry for entry in list_part if entry.get('mountpoint',False)],key=lambda part: part['mountpoint']):
+			mountpoint = partition['mountpoint']
+			log(f"Mounting {mountpoint} to {self.target}{mountpoint} using {partition['device_instance']}", level=logging.INFO)
+			if partition.get('filesystem',{}).get('mount_options',[]):
+				mount_options = ','.join(partition['filesystem']['mount_options'])
+				partition['device_instance'].mount(f"{self.target}{mountpoint}",options=mount_options)
+			else:
+				partition['device_instance'].mount(f"{self.target}{mountpoint}")
 			time.sleep(1)
 			try:
 				get_mount_info(f"{self.target}{mountpoint}", traverse=False)
 			except DiskError:
 				raise DiskError(f"Target {self.target}{mountpoint} never got mounted properly (unable to get mount information using findmnt).")
 
+		# once everything is mounted, we generate the key files in the correct place
+		for handle in list_luks_handles:
+			ppath = handle[1]['device_instance'].path
+			log(f"creating key-file for {ppath}",level=logging.INFO)
+			self._create_keyfile(handle[0],handle[1],handle[2])
+
 	def mount(self, partition :Partition, mountpoint :str, create_mountpoint :bool = True) -> None:
 		if create_mountpoint and not os.path.isdir(f'{self.target}{mountpoint}'):
 			os.makedirs(f'{self.target}{mountpoint}')
@@ -692,6 +713,7 @@ class Installer:
 					base_path,bind_path = split_bind_name(str(root_partition.path))
 					if bind_path is not None: # and root_fs_type == 'btrfs':
 						options_entry = f"rootflags=subvol={bind_path} " + options_entry
+
 					if real_device := self.detect_encryption(root_partition):
 						# TODO: We need to detect if the encrypted device is a whole disk encryption,
 						#       or simply a partition encryption. Right now we assume it's a partition (and we always have)