Rework btrfs handling (#1234)

* Restructuring btrfs.py into lib/btrfs/*.py

* Reworking how BTRFS subvolumes get represented, and worked with. Subvolumes are now their own entity which can be used to access it's information, parents or mount location.

* Added BtrfsSubvolume.partition and other stuff.

* Reworking the way luks2().unlock and .format() returns device instances. They should now return BTRFSSubvolume where appropriate.

* Fixed a missing import

* Fixed an issue where mkfs.btrfs wouldn't trigger due to busy disk.

* Fixing subvol mounting without creating a fake instance.

* Added creation of mountpint for btrfs subvolume

* Fixed root detection

* Re-worked mounting into a queue system using frozen mounting calls using lambda

* Removed old mount_subvolume() function

* Removed get_subvolumes_from_findmnt()

* Fixed Partition().subvolumes iteration

* Adding .root to BtrfsSubvolume

* Fixed issue in SysCommandWorker where log output would break and crash execution due to cmd being a string vs list

* Changed return-value from MapperDev.mountpoint to pathlib.Path

archinstall/lib/disk/__init__.py

@@ -4,4 +4,4 @@ from .blockdevice import BlockDevice
 from .filesystem import Filesystem, MBR, GPT
 from .partition import *
 from .user_guides import *
-from .validators import *
+from .validators import *

archinstall/lib/disk/btrfs/__init__.py

@@ -4,44 +4,25 @@ import glob
 import logging
 import re
 from typing import Union, Dict, TYPE_CHECKING, Any, Iterator
-from dataclasses import dataclass
 
 # https://stackoverflow.com/a/39757388/929999
 if TYPE_CHECKING:
-	from ..installer import Installer
+	from ...installer import Installer
-from .helpers import get_mount_info
-from ..exceptions import DiskError
-from ..general import SysCommand
-from ..output import log
-from ..exceptions import SysCallError
 
-@dataclass
+from .btrfs_helpers import (
-class BtrfsSubvolume:
+	subvolume_info_from_path as subvolume_info_from_path,
-	target :str
+	find_parent_subvolume as find_parent_subvolume,
-	source :str
+	setup_subvolumes as setup_subvolumes,
-	fstype :str
+	mount_subvolume as mount_subvolume
-	name :str
+)
-	options :str
+from .btrfssubvolume import BtrfsSubvolume as BtrfsSubvolume
-	root :bool = False
+from .btrfspartition import BTRFSPartition as BTRFSPartition
 
-def get_subvolumes_from_findmnt(struct :Dict[str, Any], index=0) -> Iterator[BtrfsSubvolume]:
+from ..helpers import get_mount_info
-	if '[' in struct['source']:
+from ...exceptions import DiskError, Deprecated
-		subvolume = re.findall(r'\[.*?\]', struct['source'])[0][1:-1]
+from ...general import SysCommand
-		struct['source'] = struct['source'].replace(f"[{subvolume}]", "")
+from ...output import log
-		yield BtrfsSubvolume(
+from ...exceptions import SysCallError
-			target=struct['target'],
-			source=struct['source'],
-			fstype=struct['fstype'],
-			name=subvolume,
-			options=struct['options'],
-			root=index == 0
-		)
-		index += 1
-
-		for child in struct.get('children', []):
-			for item in get_subvolumes_from_findmnt(child, index=index):
-				yield item
-				index += 1
 
 def get_subvolume_info(path :pathlib.Path) -> Dict[str, Any]:
 	try:
@@ -57,42 +38,6 @@ def get_subvolume_info(path :pathlib.Path) -> Dict[str, Any]:
 
 	return result
 
-def mount_subvolume(installation :Installer, subvolume_location :Union[pathlib.Path, str], force=False) -> bool:
-	"""
-	This function uses mount to mount a subvolume on a given device, at a given location with a given subvolume name.
-
-	@installation: archinstall.Installer instance
-	@subvolume_location: a localized string or path inside the installation / or /boot for instance without specifying /mnt/boot
-	@force: overrides the check for weither or not the subvolume mountpoint is empty or not
-
-	This function is DEPRECATED. you can get the same result creating a partition dict like any other partition, and using the standard mount procedure.
-	Only change partition['device_instance'].path with the apropriate bind name: real_partition_path[/subvolume_name]
-	"""
-	log("[Deprecated] function btrfs.mount_subvolume is deprecated. See code for alternatives",fg="yellow",level=logging.WARNING)
-	installation_mountpoint = installation.target
-	if type(installation_mountpoint) == str:
-		installation_mountpoint = pathlib.Path(installation_mountpoint)
-	# Set up the required physical structure
-	if type(subvolume_location) == str:
-		subvolume_location = pathlib.Path(subvolume_location)
-
-	target = installation_mountpoint / subvolume_location.relative_to(subvolume_location.anchor)
-
-	if not target.exists():
-		target.mkdir(parents=True)
-
-	if glob.glob(str(target / '*')) and force is False:
-		raise DiskError(f"Cannot mount subvolume to {target} because it contains data (non-empty folder target)")
-
-	log(f"Mounting {target} as a subvolume", level=logging.INFO)
-	# Mount the logical volume to the physical structure
-	mount_information, mountpoint_device_real_path = get_mount_info(target, traverse=True, return_real_path=True)
-	if mountpoint_device_real_path == str(target):
-		log(f"Unmounting non-subvolume {mount_information['source']} previously mounted at {target}")
-		SysCommand(f"umount {mount_information['source']}")
-
-	return SysCommand(f"mount {mount_information['source']} {target} -o subvol=@{subvolume_location}").exit_code == 0
-
 def create_subvolume(installation :Installer, subvolume_location :Union[pathlib.Path, str]) -> bool:
 	"""
 	This function uses btrfs to create a subvolume.
@@ -132,13 +77,18 @@ def _has_option(option :str,options :list) -> bool:
 	"""
 	if not options:
 		return False
+
 	for item in options:
 		if option in item:
 			return True
+
 	return False
 
 def manage_btrfs_subvolumes(installation :Installer,
 	partition :Dict[str, str],) -> list:
+
+	raise Deprecated("Use setup_subvolumes() instead.")
+
 	from copy import deepcopy
 	""" we do the magic with subvolumes in a centralized place
 	parameters:

archinstall/lib/disk/btrfs/btrfs_helpers.py

@@ -0,0 +1,132 @@
+import pathlib
+import logging
+from typing import Optional
+
+from ...exceptions import SysCallError, DiskError
+from ...general import SysCommand
+from ...output import log
+from ..helpers import get_mount_info
+from .btrfssubvolume import BtrfsSubvolume
+
+
+def mount_subvolume(installation, device, name, subvolume_information):
+	# we normalize the subvolume name (getting rid of slash at the start if exists. In our implemenation has no semantic load.
+	# Every subvolume is created from the top of the hierarchy- and simplifies its further use
+	name = name.lstrip('/')
+
+	# renormalize the right hand.
+	mountpoint = subvolume_information.get('mountpoint', None)
+	if not mountpoint:
+		return None
+
+	if type(mountpoint) == str:
+		mountpoint = pathlib.Path(mountpoint)
+
+	installation_target = installation.target
+	if type(installation_target) == str:
+		installation_target = pathlib.Path(installation_target)
+
+	mountpoint = installation_target / mountpoint.relative_to(mountpoint.anchor)
+	mountpoint.mkdir(parents=True, exist_ok=True)
+
+	mount_options = subvolume_information.get('options', [])
+	if not any('subvol=' in x for x in mount_options):
+		mount_options += [f'subvol={name}']
+
+	log(f"Mounting subvolume {name} on {device} to {mountpoint}", level=logging.INFO, fg="gray")
+	SysCommand(f"mount {device.path} {mountpoint} -o {','.join(mount_options)}")
+
+
+def setup_subvolumes(installation, partition_dict):
+	"""
+	Taken from: ..user_guides.py
+
+	partition['btrfs'] = {
+		"subvolumes" : {
+			"@":           "/",
+			"@home":       "/home",
+			"@log":        "/var/log",
+			"@pkg":        "/var/cache/pacman/pkg",
+			"@.snapshots": "/.snapshots"
+		}
+	}
+	"""
+	log(f"Setting up subvolumes: {partition_dict['btrfs']['subvolumes']}", level=logging.INFO, fg="gray")
+	for name, right_hand in partition_dict['btrfs']['subvolumes'].items():
+		# we normalize the subvolume name (getting rid of slash at the start if exists. In our implemenation has no semantic load.
+		# Every subvolume is created from the top of the hierarchy- and simplifies its further use
+		name = name.lstrip('/')
+
+		# renormalize the right hand.
+		# mountpoint = None
+		subvol_options = []
+
+		match right_hand:
+			# case str(): # backwards-compatability
+			# 	mountpoint = right_hand
+			case dict():
+				# mountpoint = right_hand.get('mountpoint', None)
+				subvol_options = right_hand.get('options', [])
+
+		# We create the subvolume using the BTRFSPartition instance.
+		# That way we ensure not only easy access, but also accurate mount locations etc.
+		partition_dict['device_instance'].create_subvolume(name, installation=installation)
+
+		# Make the nodatacow processing now
+		# It will be the main cause of creation of subvolumes which are not to be mounted
+		# it is not an options which can be established by subvolume (but for whole file systems), and can be
+		# set up via a simple attribute change in a directory (if empty). And here the directories are brand new
+		if 'nodatacow' in subvol_options:
+			if (cmd := SysCommand(f"chattr +C {installation.target}/{name}")).exit_code != 0:
+				raise DiskError(f"Could not set  nodatacow attribute at {installation.target}/{name}: {cmd}")
+			# entry is deleted so nodatacow doesn't propagate to the mount options
+			del subvol_options[subvol_options.index('nodatacow')]
+		# Make the compress processing now
+		# it is not an options which can be established by subvolume (but for whole file systems), and can be
+		# set up via a simple attribute change in a directory (if empty). And here the directories are brand new
+		# in this way only zstd compression is activaded
+		# TODO WARNING it is not clear if it should be a standard feature, so it might need to be deactivated
+
+		if 'compress' in subvol_options:
+			if not any(['compress' in filesystem_option for filesystem_option in partition_dict.get('filesystem', {}).get('mount_options', [])]):
+				if (cmd := SysCommand(f"chattr +c {installation.target}/{name}")).exit_code != 0:
+					raise DiskError(f"Could not set compress attribute at {installation.target}/{name}: {cmd}")
+			# entry is deleted so compress doesn't propagate to the mount options
+			del subvol_options[subvol_options.index('compress')]
+
+def subvolume_info_from_path(path :pathlib.Path) -> Optional[BtrfsSubvolume]:
+	try:
+		subvolume_name = None
+		result = {}
+		for index, line in enumerate(SysCommand(f"btrfs subvolume show {path}")):
+			if index == 0:
+				subvolume_name = line.strip().decode('UTF-8')
+				continue
+
+			if b':' in line:
+				key, value = line.strip().decode('UTF-8').split(':', 1)
+
+				# A bit of a hack, until I figure out how @dataclass
+				# allows for hooking in a pre-processor to do this we have to do it here:
+				result[key.lower().replace(' ', '_').replace('(s)', 's')] = value.strip()
+
+		return BtrfsSubvolume(**{'full_path' : path, 'name' : subvolume_name, **result})
+
+	except SysCallError:
+		pass
+
+	return None
+
+def find_parent_subvolume(path :pathlib.Path, filters=[]):
+	# A root path cannot have a parent
+	if str(path) == '/':
+		return None
+
+	if found_mount := get_mount_info(str(path.parent), traverse=True, ignore=filters):
+		if not (subvolume := subvolume_info_from_path(found_mount['target'])):
+			if found_mount['target'] == '/':
+				return None
+
+			return find_parent_subvolume(path.parent, traverse=True, filters=[*filters, found_mount['target']])
+
+		return subvolume

archinstall/lib/disk/btrfs/btrfspartition.py

@@ -0,0 +1,116 @@
+import glob
+import pathlib
+import logging
+from typing import Optional, TYPE_CHECKING
+
+from ...exceptions import DiskError
+from ...storage import storage
+from ...output import log
+from ...general import SysCommand
+from ..partition import Partition
+from ..helpers import findmnt
+from .btrfs_helpers import (
+	subvolume_info_from_path
+)
+
+if TYPE_CHECKING:
+	from ...installer import Installer
+	from .btrfssubvolume import BtrfsSubvolume
+
+class BTRFSPartition(Partition):
+	def __init__(self, *args, **kwargs):
+		Partition.__init__(self, *args, **kwargs)
+
+	def __repr__(self, *args :str, **kwargs :str) -> str:
+		mount_repr = ''
+		if self.mountpoint:
+			mount_repr = f", mounted={self.mountpoint}"
+		elif self.target_mountpoint:
+			mount_repr = f", rel_mountpoint={self.target_mountpoint}"
+
+		if self._encrypted:
+			return f'BTRFSPartition(path={self.path}, size={self.size}, PARTUUID={self._safe_uuid}, parent={self.real_device}, fs={self.filesystem}{mount_repr})'
+		else:
+			return f'BTRFSPartition(path={self.path}, size={self.size}, PARTUUID={self._safe_uuid}, fs={self.filesystem}{mount_repr})'
+
+	@property
+	def subvolumes(self):
+		for filesystem in findmnt(pathlib.Path(self.path), recurse=True).get('filesystems', []):
+			if '[' in filesystem.get('source', ''):
+				yield subvolume_info_from_path(filesystem['target'])
+
+			def iterate_children(struct):
+				for child in struct.get('children', []):
+					if '[' in child.get('source', ''):
+						yield subvolume_info_from_path(child['target'])
+
+					for sub_child in iterate_children(child):
+						yield sub_child
+
+			for child in iterate_children(filesystem):
+				yield child
+
+	def create_subvolume(self, subvolume :pathlib.Path, installation :Optional['Installer'] = None) -> 'BtrfsSubvolume':
+		"""
+		Subvolumes have to be created within a mountpoint.
+		This means we need to get the current installation target.
+		After we get it, we need to verify it is a btrfs subvolume filesystem.
+		Finally, the destination must be empty.
+		"""
+
+		# Allow users to override the installation session
+		if not installation:
+			installation = storage.get('installation_session')
+
+		# Determain if the path given, is an absolute path or a releative path.
+		# We do this by checking if the path contains a known mountpoint.
+		if str(subvolume)[0] == '/':
+			if filesystems := findmnt(subvolume, traverse=True).get('filesystems'):
+				if (target := filesystems[0].get('target')) and target != '/' and str(subvolume).startswith(target):
+					# Path starts with a known mountpoint which isn't /
+					# Which means it's an absolut path to a mounted location.
+					pass
+				else:
+					# Since it's not an absolute position with a known start.
+					# We omit the anchor ('/' basically) and make sure it's appendable
+					# to the installation.target later
+					subvolume = subvolume.relative_to(subvolume.anchor)
+		# else: We don't need to do anything about relative paths, they should be appendable to installation.target as-is.
+
+		# If the subvolume is not absolute, then we do two checks:
+		#  1. Check if the partition itself is mounted somewhere, and use that as a root
+		#  2. Use an active Installer().target as the root, assuming it's filesystem is btrfs
+		# If both above fail, we need to warn the user that such setup is not supported.
+		if str(subvolume)[0] != '/':
+			if self.mountpoint is None and installation is None:
+				raise DiskError("When creating a subvolume on BTRFSPartition()'s, you need to either initiate a archinstall.Installer() or give absolute paths when creating the subvoulme.")
+			elif self.mountpoint:
+				subvolume = self.mountpoint / subvolume
+			elif installation:
+				ongoing_installation_destination = installation.target
+				if type(ongoing_installation_destination) == str:
+					ongoing_installation_destination = pathlib.Path(ongoing_installation_destination)
+
+				subvolume = ongoing_installation_destination / subvolume
+
+		subvolume.parent.mkdir(parents=True, exist_ok=True)
+
+		# <!--
+		# We perform one more check from the given absolute position.
+		# And we traverse backwards in order to locate any if possible subvolumes above
+		# our new btrfs subvolume. This is because it needs to be mounted under it to properly
+		# function.
+		# if btrfs_parent := find_parent_subvolume(subvolume):
+		# 	print('Found parent:', btrfs_parent)
+		# -->
+
+		log(f'Attempting to create subvolume at {subvolume}', level=logging.DEBUG, fg="grey")
+
+		if glob.glob(str(subvolume / '*')):
+			raise DiskError(f"Cannot create subvolume at {subvolume} because it contains data (non-empty folder target is not supported by BTRFS)")
+		elif subvolinfo := subvolume_info_from_path(subvolume):
+			raise DiskError(f"Destination {subvolume} is already a subvolume: {subvolinfo}")
+
+		SysCommand(f"btrfs subvolume create {subvolume}")
+
+		return subvolume_info_from_path(subvolume)

archinstall/lib/disk/btrfs/btrfssubvolume.py

@@ -0,0 +1,191 @@
+import pathlib
+import datetime
+import logging
+import string
+import random
+import shutil
+from dataclasses import dataclass
+from typing import Optional, List# , TYPE_CHECKING
+from functools import cached_property
+
+# if TYPE_CHECKING:
+# 	from ..blockdevice import BlockDevice
+
+from ...exceptions import DiskError
+from ...general import SysCommand
+from ...output import log
+from ...storage import storage
+
+@dataclass
+class BtrfsSubvolume:
+	full_path :pathlib.Path
+	name :str
+	uuid :str
+	parent_uuid :str
+	creation_time :datetime.datetime
+	subvolume_id :int
+	generation :int
+	gen_at_creation :int
+	parent_id :int
+	top_level_id :int
+	send_transid :int
+	send_time :datetime.datetime
+	receive_transid :int
+	received_uuid :Optional[str] = None
+	flags :Optional[str] = None
+	receive_time :Optional[datetime.datetime] = None
+	snapshots :Optional[List] = None
+
+	def __post_init__(self):
+		self.full_path = pathlib.Path(self.full_path)
+
+		# Convert "-" entries to `None`
+		if self.parent_uuid == "-":
+			self.parent_uuid = None
+		if self.received_uuid == "-":
+			self.received_uuid = None
+		if self.flags == "-":
+			self.flags = None
+		if self.receive_time == "-":
+			self.receive_time = None
+		if self.snapshots == "":
+			self.snapshots = []
+
+		# Convert timestamps into datetime workable objects (and preserve timezone by using ISO formats)
+		self.creation_time = datetime.datetime.fromisoformat(self.convert_to_ISO_format(self.creation_time))
+		self.send_time = datetime.datetime.fromisoformat(self.convert_to_ISO_format(self.send_time))
+		if self.receive_time:
+			self.receive_time = datetime.datetime.fromisoformat(self.convert_to_ISO_format(self.receive_time))
+
+	@property
+	def parent_subvolume(self):
+		from .btrfs_helpers import find_parent_subvolume
+
+		return find_parent_subvolume(self.full_path)
+
+	@property
+	def root(self) -> bool:
+		from .btrfs_helpers import subvolume_info_from_path
+
+		# TODO: Make this function traverse storage['MOUNT_POINT'] and find the first
+		# occurance of a mountpoint that is a btrfs volume instead of lazy assume / is a subvolume.
+		# It would also be nice if it could use findmnt(self.full_path) and traverse backwards
+		# finding the last occurance of a subvolume which 'self' belongs to.
+		if volume := subvolume_info_from_path(storage['MOUNT_POINT']):
+			return self.full_path == volume.full_path
+
+		return False
+
+	@cached_property
+	def partition(self):
+		from ..helpers import findmnt, get_parent_of_partition, all_blockdevices
+		from ..partition import Partition
+		from ..blockdevice import BlockDevice
+		from ..mapperdev import MapperDev
+		from .btrfspartition import BTRFSPartition
+		from .btrfs_helpers import subvolume_info_from_path
+
+		try:
+			# If the subvolume is mounted, it's pretty trivial to lookup the partition (parent) device.
+			if filesystem := findmnt(self.full_path).get('filesystems', []):
+				if source := filesystem[0].get('source', None):
+					# Strip away subvolume definitions from findmnt
+					if '[' in source:
+						source = source[:source.find('[')]
+
+					if filesystem[0].get('fstype', '') == 'btrfs':
+						return BTRFSPartition(source, BlockDevice(get_parent_of_partition(pathlib.Path(source))))
+					elif filesystem[0].get('source', '').startswith('/dev/mapper'):
+						return MapperDev(source)
+					else:
+						return Partition(source, BlockDevice(get_parent_of_partition(pathlib.Path(source))))
+		except DiskError:
+			# Subvolume has never been mounted, we have no reliable way of finding where it is.
+			# But we have the UUID of the partition, and can begin looking for it by mounting
+			# all blockdevices that we can reliably support.. This is taxing tho and won't cover all devices.
+
+			log(f"Looking up {self}, this might take time.", fg="orange", level=logging.WARNING)
+			for blockdevice, instance in all_blockdevices(mappers=True, partitions=True, error=True).items():
+				if type(instance) in (Partition, MapperDev):
+					we_mounted_it = False
+					detection_mountpoint = instance.mountpoint
+					if not detection_mountpoint:
+						if type(instance) == Partition and instance.encrypted:
+							# TODO: Perhaps support unlocking encrypted volumes?
+							# This will cause a lot of potential user interactions tho.
+							log(f"Ignoring {blockdevice} because it's encrypted.", fg="gray", level=logging.DEBUG)
+							continue
+
+						detection_mountpoint = pathlib.Path(f"/tmp/{''.join([random.choice(string.ascii_letters) for x in range(20)])}")
+						detection_mountpoint.mkdir(parents=True, exist_ok=True)
+
+						instance.mount(str(detection_mountpoint))
+						we_mounted_it = True
+
+					if (filesystem := findmnt(detection_mountpoint)) and (filesystem := filesystem.get('filesystems', [])):
+						if subvolume := subvolume_info_from_path(filesystem[0]['target']):
+							if subvolume.uuid == self.uuid:
+								# The top level subvolume matched of ourselves,
+								# which means the instance we're iterating has the subvol we're looking for.
+								log(f"Found the subvolume on device {instance}", level=logging.DEBUG, fg="gray")
+								return instance
+
+						def iterate_children(struct):
+							for child in struct.get('children', []):
+								if '[' in child.get('source', ''):
+									yield subvolume_info_from_path(child['target'])
+
+								for sub_child in iterate_children(child):
+									yield sub_child
+
+						for child in iterate_children(filesystem[0]):
+							if child.uuid == self.uuid:
+								# We found a child within the instance that has the subvol we're looking for.
+								log(f"Found the subvolume on device {instance}", level=logging.DEBUG, fg="gray")
+								return instance
+
+					if we_mounted_it:
+						instance.unmount()
+						shutil.rmtree(detection_mountpoint)
+
+	@cached_property
+	def mount_options(self) -> Optional[List[str]]:
+		from ..helpers import findmnt
+
+		if filesystem := findmnt(self.full_path).get('filesystems', []):
+			return filesystem[0].get('options').split(',')
+
+	def convert_to_ISO_format(self, time_string):
+		time_string_almost_done = time_string.replace(' ', 'T', 1).replace(' ', '')
+		iso_string = f"{time_string_almost_done[:-2]}:{time_string_almost_done[-2:]}"
+		return iso_string
+
+	def mount(self, mountpoint :pathlib.Path, options=None, include_previously_known_options=True):
+		from ..helpers import findmnt
+
+		try:
+			if mnt_info := findmnt(pathlib.Path(mountpoint), traverse=False):
+				log(f"Unmounting {mountpoint} as it was already mounted using {mnt_info}")
+				SysCommand(f"umount {mountpoint}")
+		except DiskError:
+			# No previously mounted device at the mountpoint
+			pass
+
+		if not options:
+			options = []
+
+		try:
+			if include_previously_known_options and (cached_options := self.mount_options):
+				options += cached_options
+		except DiskError:
+			pass
+
+		if not any('subvol=' in x for x in options):
+			options += f'subvol={self.name}'
+
+		SysCommand(f"mount {self.partition.path} {mountpoint} -o {','.join(options)}")
+		log(f"{self} has successfully been mounted to {mountpoint}", level=logging.INFO, fg="gray")
+
+	def unmount(self, recurse :bool = True):
+		SysCommand(f"umount {'-R' if recurse else ''} {self.full_path}")
+		log(f"Successfully unmounted {self}", level=logging.INFO, fg="gray")

archinstall/lib/disk/filesystem.py

@@ -65,6 +65,7 @@ class Filesystem:
 
 	def load_layout(self, layout :Dict[str, Any]) -> None:
 		from ..luks import luks2
+		from .btrfs import BTRFSPartition
 
 		# If the layout tells us to wipe the drive, we do so
 		if layout.get('wipe', False):
@@ -142,12 +143,24 @@ class Filesystem:
 										break
 
 						unlocked_device.format(partition['filesystem']['format'], options=format_options)
+
 				elif partition.get('wipe', False):
 					if not partition['device_instance']:
 						raise DiskError(f"Internal error caused us to loose the partition. Please report this issue upstream!")
 
 					partition['device_instance'].format(partition['filesystem']['format'], options=format_options)
 
+					if partition['filesystem']['format'] == 'btrfs':
+						# We upgrade the device instance to a BTRFSPartition if we format it as such.
+						# This is so that we can gain access to more features than otherwise available in Partition()
+						partition['device_instance'] = BTRFSPartition(
+							partition['device_instance'].path,
+							block_device=partition['device_instance'].block_device,
+							encrypted=False,
+							filesystem='btrfs',
+							autodetect_filesystem=False
+						)
+
 			if partition.get('boot', False):
 				log(f"Marking partition {partition['device_instance']} as bootable.")
 				self.set(self.partuuid_to_index(partition['device_instance'].part_uuid), 'boot on')

archinstall/lib/disk/helpers.py

@@ -291,11 +291,37 @@ def find_mountpoint(device_path :str) -> Dict[str, Any]:
 	except SysCallError:
 		return {}
 
-def get_mount_info(path :Union[pathlib.Path, str], traverse :bool = False, return_real_path :bool = False) -> Dict[str, Any]:
+def findmnt(path :pathlib.Path, traverse :bool = False, ignore :List = [], recurse :bool = True) -> Dict[str, Any]:
+	for traversal in list(map(str, [str(path)] + list(path.parents))):
+		if traversal in ignore:
+			continue
+
+		try:
+			log(f"Getting mount information for device path {traversal}", level=logging.DEBUG)
+			if (output := SysCommand(f"/usr/bin/findmnt --json {'--submounts' if recurse else ''} {traversal}").decode('UTF-8')):
+				return json.loads(output)
+
+		except SysCallError as error:
+			log(f"Could not get mount information on {path} but continuing and ignoring: {error}", level=logging.INFO, fg="gray")
+			pass
+
+		if not traverse:
+			break
+
+	raise DiskError(f"Could not get mount information for path {path}")
+
+
+def get_mount_info(path :Union[pathlib.Path, str], traverse :bool = False, return_real_path :bool = False, ignore :List = []) -> Dict[str, Any]:
+	import traceback
+
+	log(f"Deprecated: archinstall.get_mount_info(). Use archinstall.findmnt() instead, which does not do any automatic parsing. Please change at:\n{''.join(traceback.format_stack())}")
 	device_path, bind_path = split_bind_name(path)
 	output = {}
 
 	for traversal in list(map(str, [str(device_path)] + list(pathlib.Path(str(device_path)).parents))):
+		if traversal in ignore:
+			continue
+
 		try:
 			log(f"Getting mount information for device path {traversal}", level=logging.DEBUG)
 			if (output := SysCommand(f'/usr/bin/findmnt --json {traversal}').decode('UTF-8')):
@@ -385,9 +411,8 @@ def get_partitions_in_use(mountpoint :str) -> List[Partition]:
 
 
 def get_filesystem_type(path :str) -> Optional[str]:
-	device_name, bind_name = split_bind_name(path)
 	try:
-		return SysCommand(f"blkid -o value -s TYPE {device_name}").decode('UTF-8').strip()
+		return SysCommand(f"blkid -o value -s TYPE {path}").decode('UTF-8').strip()
 	except SysCallError:
 		return None
 

archinstall/lib/disk/mapperdev.py

@@ -51,11 +51,11 @@ class MapperDev:
 		raise ValueError(f"Could not convert {self.mappername} to a real dm-crypt device")
 
 	@property
-	def mountpoint(self) -> Optional[str]:
+	def mountpoint(self) -> Optional[pathlib.Path]:
 		try:
 			data = json.loads(SysCommand(f"findmnt --json -R {self.path}").decode())
 			for filesystem in data['filesystems']:
-				return filesystem.get('target')
+				return pathlib.Path(filesystem.get('target'))
 
 		except SysCallError as error:
 			# Not mounted anywhere most likely
@@ -76,8 +76,8 @@ class MapperDev:
 
 	@property
 	def subvolumes(self) -> Iterator['BtrfsSubvolume']:
-		from .btrfs import get_subvolumes_from_findmnt
+		from .btrfs import subvolume_info_from_path
 		
 		for mountpoint in self.mount_information:
-			for result in get_subvolumes_from_findmnt(mountpoint):
+			if subvolume := subvolume_info_from_path(mountpoint):
-				yield result
+				yield subvolume

archinstall/lib/disk/partition.py

@@ -13,7 +13,8 @@ from ..storage import storage
 from ..exceptions import DiskError, SysCallError, UnknownFilesystemFormat
 from ..output import log
 from ..general import SysCommand
-from .btrfs import get_subvolumes_from_findmnt, BtrfsSubvolume
+from .btrfs.btrfs_helpers import subvolume_info_from_path
+from .btrfs.btrfssubvolume import BtrfsSubvolume
 
 class Partition:
 	def __init__(self,
@@ -96,7 +97,7 @@ class Partition:
 		try:
 			data = json.loads(SysCommand(f"findmnt --json -R {self.path}").decode())
 			for filesystem in data['filesystems']:
-				return filesystem.get('target')
+				return pathlib.Path(filesystem.get('target'))
 
 		except SysCallError as error:
 			# Not mounted anywhere most likely
@@ -304,9 +305,26 @@ class Partition:
 
 	@property
 	def subvolumes(self) -> Iterator[BtrfsSubvolume]:
+		from .helpers import findmnt
+		
+		def iterate_children_recursively(information):
+			for child in information.get('children', []):
+				if target := child.get('target'):
+					if subvolume := subvolume_info_from_path(pathlib.Path(target)):
+						yield subvolume
+
+					if child.get('children'):
+						for subchild in iterate_children_recursively(child):
+							yield subchild
+
 		for mountpoint in self.mount_information:
-			for result in get_subvolumes_from_findmnt(mountpoint):
+			if result := findmnt(pathlib.Path(mountpoint['target'])):
-				yield result
+				for filesystem in result.get('filesystems', []):
+					if subvolume := subvolume_info_from_path(pathlib.Path(mountpoint['target'])):
+						yield subvolume
+
+					for child in iterate_children_recursively(filesystem):
+						yield child
 
 	def partprobe(self) -> bool:
 		try:
@@ -357,7 +375,7 @@ class Partition:
 		handle = luks2(self, None, None)
 		return handle.encrypt(self, *args, **kwargs)
 
-	def format(self, filesystem :Optional[str] = None, path :Optional[str] = None, log_formatting :bool = True, options :List[str] = []) -> bool:
+	def format(self, filesystem :Optional[str] = None, path :Optional[str] = None, log_formatting :bool = True, options :List[str] = [], retry :bool = True) -> bool:
 		"""
 		Format can be given an overriding path, for instance /dev/null to test
 		the formatting functionality and in essence the support for the given filesystem.
@@ -379,63 +397,71 @@ class Partition:
 		if log_formatting:
 			log(f'Formatting {path} -> {filesystem}', level=logging.INFO)
 
-		if filesystem == 'btrfs':
+		try:
-			options = ['-f'] + options
+			if filesystem == 'btrfs':
+				options = ['-f'] + options
 
-			if 'UUID:' not in (mkfs := SysCommand(f"/usr/bin/mkfs.btrfs {' '.join(options)} {path}").decode('UTF-8')):
+				if 'UUID:' not in (mkfs := SysCommand(f"/usr/bin/mkfs.btrfs {' '.join(options)} {path}").decode('UTF-8')):
-				raise DiskError(f'Could not format {path} with {filesystem} because: {mkfs}')
+					raise DiskError(f'Could not format {path} with {filesystem} because: {mkfs}')
-			self.filesystem = filesystem
+				self.filesystem = filesystem
 
-		elif filesystem == 'vfat':
+			elif filesystem == 'vfat':
-			options = ['-F32'] + options
+				options = ['-F32'] + options
 
-			if (handle := SysCommand(f"/usr/bin/mkfs.vfat {' '.join(options)} {path}")).exit_code != 0:
+				if (handle := SysCommand(f"/usr/bin/mkfs.vfat {' '.join(options)} {path}")).exit_code != 0:
-				raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}")
+					raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}")
-			self.filesystem = filesystem
+				self.filesystem = filesystem
 
-		elif filesystem == 'ext4':
+			elif filesystem == 'ext4':
-			options = ['-F'] + options
+				options = ['-F'] + options
 
-			if (handle := SysCommand(f"/usr/bin/mkfs.ext4 {' '.join(options)} {path}")).exit_code != 0:
+				if (handle := SysCommand(f"/usr/bin/mkfs.ext4 {' '.join(options)} {path}")).exit_code != 0:
-				raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}")
+					raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}")
-			self.filesystem = filesystem
+				self.filesystem = filesystem
 
-		elif filesystem == 'ext2':
+			elif filesystem == 'ext2':
-			options = ['-F'] + options
+				options = ['-F'] + options
 
-			if (handle := SysCommand(f"/usr/bin/mkfs.ext2 {' '.join(options)} {path}")).exit_code != 0:
+				if (handle := SysCommand(f"/usr/bin/mkfs.ext2 {' '.join(options)} {path}")).exit_code != 0:
-				raise DiskError(f'Could not format {path} with {filesystem} because: {b"".join(handle)}')
+					raise DiskError(f'Could not format {path} with {filesystem} because: {b"".join(handle)}')
-			self.filesystem = 'ext2'
+				self.filesystem = 'ext2'
 
-		elif filesystem == 'xfs':
+			elif filesystem == 'xfs':
-			options = ['-f'] + options
+				options = ['-f'] + options
 
-			if (handle := SysCommand(f"/usr/bin/mkfs.xfs {' '.join(options)} {path}")).exit_code != 0:
+				if (handle := SysCommand(f"/usr/bin/mkfs.xfs {' '.join(options)} {path}")).exit_code != 0:
-				raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}")
+					raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}")
-			self.filesystem = filesystem
+				self.filesystem = filesystem
 
-		elif filesystem == 'f2fs':
+			elif filesystem == 'f2fs':
-			options = ['-f'] + options
+				options = ['-f'] + options
 
-			if (handle := SysCommand(f"/usr/bin/mkfs.f2fs {' '.join(options)} {path}")).exit_code != 0:
+				if (handle := SysCommand(f"/usr/bin/mkfs.f2fs {' '.join(options)} {path}")).exit_code != 0:
-				raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}")
+					raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}")
-			self.filesystem = filesystem
+				self.filesystem = filesystem
 
-		elif filesystem == 'ntfs3':
+			elif filesystem == 'ntfs3':
-			options = ['-f'] + options
+				options = ['-f'] + options
 
-			if (handle := SysCommand(f"/usr/bin/mkfs.ntfs -Q {' '.join(options)} {path}")).exit_code != 0:
+				if (handle := SysCommand(f"/usr/bin/mkfs.ntfs -Q {' '.join(options)} {path}")).exit_code != 0:
-				raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}")
+					raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}")
-			self.filesystem = filesystem
+				self.filesystem = filesystem
 
-		elif filesystem == 'crypto_LUKS':
+			elif filesystem == 'crypto_LUKS':
-			# 	from ..luks import luks2
+				# 	from ..luks import luks2
-			# 	encrypted_partition = luks2(self, None, None)
+				# 	encrypted_partition = luks2(self, None, None)
-			# 	encrypted_partition.format(path)
+				# 	encrypted_partition.format(path)
-			self.filesystem = filesystem
+				self.filesystem = filesystem
 
-		else:
+			else:
-			raise UnknownFilesystemFormat(f"Fileformat '{filesystem}' is not yet implemented.")
+				raise UnknownFilesystemFormat(f"Fileformat '{filesystem}' is not yet implemented.")
+		except SysCallError as error:
+			log(f"Formatting ran in to an error: {error}", level=logging.WARNING, fg="orange")
+			if retry is True:
+				log(f"Retrying in {storage.get('DISK_TIMEOUTS', 1)} seconds.", level=logging.WARNING, fg="orange")
+				time.sleep(storage.get('DISK_TIMEOUTS', 1))
+				
+				return self.format(filesystem, path, log_formatting, options, retry=False)
 
 		if get_filesystem_type(path) == 'crypto_LUKS' or get_filesystem_type(self.real_device) == 'crypto_LUKS':
 			self.encrypted = True

archinstall/lib/exceptions.py

@@ -48,4 +48,8 @@ class PackageError(BaseException):
 
 
 class TranslationError(BaseException):
+	pass
+
+
+class Deprecated(BaseException):
 	pass

archinstall/lib/general.py

@@ -363,7 +363,7 @@ class SysCommandWorker:
 			try:
 				try:
 					with open(f"{storage['LOG_PATH']}/cmd_history.txt", "a") as cmd_log:
-						cmd_log.write(f"{' '.join(self.cmd)}\n")
+						cmd_log.write(f"{self.cmd}\n")
 				except PermissionError:
 					pass
 

archinstall/lib/installer.py

@@ -20,7 +20,6 @@ from .storage import storage
 # from .user_interaction import *
 from .output import log
 from .profiles import Profile
-from .disk.btrfs import manage_btrfs_subvolumes
 from .disk.partition import get_mount_fs_type
 from .exceptions import DiskError, ServiceException, RequirementError, HardwareIncompatibilityError, SysCallError
 from .hsm import fido2_enroll
@@ -233,12 +232,17 @@ class Installer:
 
 	def mount_ordered_layout(self, layouts: Dict[str, Any]) -> None:
 		from .luks import luks2
+		from .disk.btrfs import setup_subvolumes, mount_subvolume
+
 		# set the partitions as a list not part of a tree (which we don't need anymore (i think)
 		list_part = []
 		list_luks_handles = []
 		for blockdevice in layouts:
 			list_part.extend(layouts[blockdevice]['partitions'])
 
+		# TODO: Implement a proper mount-queue system that does not depend on return values.
+		mount_queue = {}
+
 		# we manage the encrypted partititons
 		for partition in [entry for entry in list_part if entry.get('encrypted', False)]:
 			# open the luks device and all associate stuff
@@ -260,32 +264,61 @@ class Installer:
 				fido2_enroll(hsm_device_path, partition['device_instance'], password)
 
 		# we manage the btrfs partitions
-		for partition in [entry for entry in list_part if entry.get('btrfs', {}).get('subvolumes', {})]:
+		if any(btrfs_subvolumes := [entry for entry in list_part if entry.get('btrfs', {}).get('subvolumes', {})]):
-			if partition.get('filesystem',{}).get('mount_options',[]):
+			for partition in btrfs_subvolumes:
-				mount_options = ','.join(partition['filesystem']['mount_options'])
+				if mount_options := ','.join(partition.get('filesystem',{}).get('mount_options',[])):
-				self.mount(partition['device_instance'], "/", options=mount_options)
+					self.mount(partition['device_instance'], "/", options=mount_options)
-			else:
+				else:
-				self.mount(partition['device_instance'], "/")
+					self.mount(partition['device_instance'], "/")
-			try:
-				new_mountpoints = manage_btrfs_subvolumes(self,partition)
-			except Exception as e:
-				# every exception unmounts the physical volume. Otherwise we let the system in an unstable state
-				partition['device_instance'].unmount()
-				raise e
-			partition['device_instance'].unmount()
-			if new_mountpoints:
-				list_part.extend(new_mountpoints)
 
-		# we mount. We need to sort by mountpoint to get a good working order
+				setup_subvolumes(
-		for partition in sorted([entry for entry in list_part if entry.get('mountpoint',False)],key=lambda part: part['mountpoint']):
+					installation=self,
+					partition_dict=partition
+				)
+
+				partition['device_instance'].unmount()
+
+		# We then handle any special cases, such as btrfs
+		if any(btrfs_subvolumes := [entry for entry in list_part if entry.get('btrfs', {}).get('subvolumes', {})]):
+			for partition_information in btrfs_subvolumes:
+				for name, mountpoint in sorted(partition_information['btrfs']['subvolumes'].items(), key=lambda item: item[1]):
+					btrfs_subvolume_information = {}
+
+					match mountpoint:
+						case str(): # backwards-compatability
+							btrfs_subvolume_information['mountpoint'] = mountpoint
+							btrfs_subvolume_information['options'] = []
+						case dict():
+							btrfs_subvolume_information['mountpoint'] = mountpoint.get('mountpoint', None)
+							btrfs_subvolume_information['options'] = mountpoint.get('options', [])
+						case _:
+							continue
+
+					if mountpoint_parsed := btrfs_subvolume_information.get('mountpoint'):
+						# We cache the mount call for later
+						mount_queue[mountpoint_parsed] = lambda device=partition_information['device_instance'], \
+							name=name, \
+							subvolume_information=btrfs_subvolume_information: mount_subvolume(
+								installation=self,
+								device=device,
+								name=name,
+								subvolume_information=subvolume_information
+							)
+
+		# We mount ordinary partitions, and we sort them by the mountpoint
+		for partition in sorted([entry for entry in list_part if entry.get('mountpoint', False)], key=lambda part: part['mountpoint']):
 			mountpoint = partition['mountpoint']
 			log(f"Mounting {mountpoint} to {self.target}{mountpoint} using {partition['device_instance']}", level=logging.INFO)
 
 			if partition.get('filesystem',{}).get('mount_options',[]):
 				mount_options = ','.join(partition['filesystem']['mount_options'])
-				partition['device_instance'].mount(f"{self.target}{mountpoint}", options=mount_options)
+				mount_queue[mountpoint] = lambda target=f"{self.target}{mountpoint}", options=mount_options: partition['device_instance'].mount(target, options)
 			else:
-				partition['device_instance'].mount(f"{self.target}{mountpoint}")
+				mount_queue[mountpoint] = lambda target=f"{self.target}{mountpoint}": partition['device_instance'].mount(target)
+
+		# We mount everything by sorting on the mountpoint itself.
+		for mountpoint, frozen_func in sorted(mount_queue.items(), key=lambda item: item[0]):
+			frozen_func()
 
 			time.sleep(1)
 
@@ -979,10 +1012,14 @@ class Installer:
 				if plugin.on_add_bootloader(self):
 					return True
 
+		if type(self.target) == str:
+			self.target = pathlib.Path(self.target)
+
 		boot_partition = None
 		root_partition = None
 		for partition in self.partitions:
-			if partition.mountpoint == os.path.join(self.target, 'boot'):
+			print(partition, [partition.mountpoint], [self.target])
+			if partition.mountpoint == self.target / 'boot':
 				boot_partition = partition
 			elif partition.mountpoint == self.target:
 				root_partition = partition

archinstall/lib/luks.py

@@ -15,7 +15,10 @@ from .general import SysCommand, SysCommandWorker
 from .output import log
 from .exceptions import SysCallError, DiskError
 from .storage import storage
+from .disk.helpers import get_filesystem_type
 from .disk.mapperdev import MapperDev
+from .disk.btrfs import BTRFSPartition
+
 
 class luks2:
 	def __init__(self,
@@ -149,7 +152,6 @@ class luks2:
 		:param mountpoint: The name without absolute path, for instance "luksdev" will point to /dev/mapper/luksdev
 		:type mountpoint: str
 		"""
-		from .disk import get_filesystem_type
 
 		if '/' in mountpoint:
 			os.path.basename(mountpoint)  # TODO: Raise exception instead?
@@ -162,14 +164,22 @@ class luks2:
 		if os.path.islink(f'/dev/mapper/{mountpoint}'):
 			self.mapdev = f'/dev/mapper/{mountpoint}'
 
-			unlocked_partition = Partition(
+			if (filesystem_type := get_filesystem_type(pathlib.Path(self.mapdev))) == 'btrfs':
+				return BTRFSPartition(
+					self.mapdev,
+					block_device=MapperDev(mountpoint).partition.block_device,
+					encrypted=True,
+					filesystem=filesystem_type,
+					autodetect_filesystem=False
+				)
+
+			return Partition(
 				self.mapdev,
 				block_device=MapperDev(mountpoint).partition.block_device,
 				encrypted=True,
 				filesystem=get_filesystem_type(self.mapdev),
 				autodetect_filesystem=False
 			)
-			return unlocked_partition
 
 	def close(self, mountpoint :Optional[str] = None) -> bool:
 		if not mountpoint:

archinstall/lib/output.py

@@ -61,9 +61,11 @@ def stylize_output(text: str, *opts :str, **kwargs) -> str:
 		'magenta' : '5',
 		'cyan' : '6',
 		'white' : '7',
-		'orange' : '8;5;208',    # Extended 256-bit colors (not always supported)
+		'teal' : '8;5;109',      # Extended 256-bit colors (not always supported)
-		'darkorange' : '8;5;202',# https://www.lihaoyi.com/post/BuildyourownCommandLinewithANSIescapecodes.html#256-colors
+		'orange' : '8;5;208',    # https://www.lihaoyi.com/post/BuildyourownCommandLinewithANSIescapecodes.html#256-colors
+		'darkorange' : '8;5;202',
 		'gray' : '8;5;246',
+		'grey' : '8;5;246',
 		'darkgray' : '8;5;240',
 		'lightgray' : '8;5;256'
 	}

archinstall/lib/systemd.py

@@ -91,20 +91,25 @@ class Boot:
 			log(f"The error above occured in a temporary boot-up of the installation {self.instance}", level=logging.ERROR, fg="red")
 
 		shutdown = None
+		shutdown_exit_code = -1
 
 		try:
 			shutdown = SysCommand(f'systemd-run --machine={self.container_name} --pty shutdown now')
 		except SysCallError as error:
-			if error.exit_code == 256:
+			shutdown_exit_code = error.exit_code
-				pass
+			# if error.exit_code == 256:
+			# 	pass
 
 		while self.session.is_alive():
 			time.sleep(0.25)
 
-		if self.session.exit_code == 0 or (shutdown and shutdown.exit_code == 0):
+		if shutdown:
+			shutdown_exit_code = shutdown.exit_code
+
+		if self.session.exit_code == 0 or shutdown_exit_code == 0:
 			storage['active_boot'] = None
 		else:
-			raise SysCallError(f"Could not shut down temporary boot of {self.instance}: {shutdown}", exit_code=shutdown.exit_code)
+			raise SysCallError(f"Could not shut down temporary boot of {self.instance}: {self.session.exit_code}/{shutdown_exit_code}", exit_code=next(filter(bool, [self.session.exit_code, shutdown_exit_code])))
 
 	def __iter__(self) -> Iterator[str]:
 		if self.session: