RomanNum3ral
/
archinstall
mirror of https://github.com/archlinux/archinstall.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Btrfs II (#838) 

* Btrfs with encrypted partitions.
We have changed installer.mount_ordered_layout into a series of loops
* open the encrypted devices
* manage btrfs subvolumes
* mount whatever
* create kyefiles for encrypted volumes
We have simplified the btrfs subvolume manager

We merged the locale branch as it is needed here

* We allow only the creation of keyfiles if the partition does not contain the root mount point.
Also, adapt examples/only_hd to the new __init__.py
Also, assorted flake8 warnings

* Cleanup code

* Naming schema for encrypted volumes
revert global locale association (provisional)

* We introduce the option of defining mount options in the partition dictionary.
It has forced us to define two new entries in this dictionary:
* format_options (formerly options) for mkfs options and
* mount_options for mount -o ones.
The different meaning of compress between partition and subvolumes is treated

* Function lib/disk/btrfs.py mount_subvolume marked as deprecated
Code cleanup.

* format_options now filesystem.options

* format_options now filesystem.format_options
mount_options  nof filesystem.mount_options

* flake8 uncovered a slip in the code
This commit is contained in:
Werner Llácer 2022-01-07 11:29:30 +01:00 committed by GitHub
parent 08d7375e62
commit 2190321eb4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 140 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
archinstall/lib/disk/btrfs.py
View File

@ -11,7 +11,6 @@ from .helpers import get_mount_info
from ..exceptions import DiskError
from ..general import SysCommand
from ..output import log
from .partition import Partition
def mount_subvolume(installation :Installer, subvolume_location :Union[pathlib.Path, str], force=False) -> bool:
@ -21,8 +20,11 @@ def mount_subvolume(installation :Installer, subvolume_location :Union[pathlib.P
@installation: archinstall.Installer instance
@subvolume_location: a localized string or path inside the installation / or /boot for instance without specifying /mnt/boot
@force: overrides the check for weither or not the subvolume mountpoint is empty or not
"""
This function is DEPRECATED. you can get the same result creating a partition dict like any other partition, and using the standard mount procedure.
Only change partition['device_instance'].path with the apropriate bind name: real_partition_path[/subvolume_name]
"""
log("function btrfs.mount_subvolume DEPRECATED. See code for alternatives",fg="yellow",level=logging.WARNING)
installation_mountpoint = installation.target
if type(installation_mountpoint) == str:
installation_mountpoint = pathlib.Path(installation_mountpoint)
@ -80,27 +82,38 @@ def create_subvolume(installation :Installer, subvolume_location :Union[pathlib.
if (cmd := SysCommand(f"btrfs subvolume create {target}")).exit_code != 0:
raise DiskError(f"Could not create a subvolume at {target}: {cmd}")
def _has_option(option :str,options :list) -> bool:
""" auxiliary routine to check if an option is present in a list.
we check if the string appears in one of the options, 'cause it can appear in severl forms (option, option=val,...)
"""
if not options:
return False
for item in options:
if option in item:
return True
return False
def manage_btrfs_subvolumes(installation :Installer,
partition :Dict[str, str],
mountpoints :Dict[str, str],
subvolumes :Dict[str, str],
unlocked_device :Dict[str, str] = None
) -> None:
partition :Dict[str, str],) -> list:
from copy import deepcopy
""" we do the magic with subvolumes in a centralized place
parameters:
* the installation object
* the partition dictionary entry which represents the physical partition
* mountpoinst, the dictionary which contains all the partititon to be mounted
* subvolumes is the dictionary with the names of the subvolumes and its location
returns
* mountpoinst, the list which contains all the "new" partititon to be mounted
We expect the partition has been mounted as / , and it to be unmounted after the processing
Then we create all the subvolumes inside btrfs as demand
We clone then, both the partition dictionary and the object inside it and adapt it to the subvolume needs
Then we add it them to the mountpoints dictionary to be processed as "normal" partitions
Then we return a list of "new" partitions to be processed as "normal" partitions
# TODO For encrypted devices we need some special processing prior to it
"""
# We process each of the pairs <subvolume name: mount point | None | mount info dict>
# th mount info dict has an entry for the path of the mountpoint (named 'mountpoint') and 'options' which is a list
# of mount options (or similar used by brtfs)
mountpoints = []
subvolumes = partition['btrfs']['subvolumes']
for name, right_hand in subvolumes.items():
try:
# we normalize the subvolume name (getting rid of slash at the start if exists. In our implemenation has no semantic load - every subvolume is created from the top of the hierarchy- and simplifies its further use
@ -108,7 +121,7 @@ def manage_btrfs_subvolumes(installation :Installer,
name = name[1:]
# renormalize the right hand.
location = None
mount_options = []
subvol_options = []
# no contents, so it is not to be mounted
if not right_hand:
location = None
@ -118,38 +131,37 @@ def manage_btrfs_subvolumes(installation :Installer,
# a dict. two elements 'mountpoint' (obvious) and and a mount options list ¿?
elif isinstance(right_hand,dict):
location = right_hand.get('mountpoint',None)
mount_options = right_hand.get('options',[])
subvol_options = right_hand.get('options',[])
# we create the subvolume
create_subvolume(installation,name)
# Make the nodatacow processing now
# It will be the main cause of creation of subvolumes which are not to be mounted
# it is not an options which can be established by subvolume (but for whole file systems), and can be
# set up via a simple attribute change in a directory (if empty). And here the directories are brand new
if 'nodatacow' in mount_options:
if 'nodatacow' in subvol_options:
if (cmd := SysCommand(f"chattr +C {installation.target}/{name}")).exit_code != 0:
raise DiskError(f"Could not set nodatacow attribute at {installation.target}/{name}: {cmd}")
# entry is deleted so nodatacow doesn't propagate to the mount options
del mount_options[mount_options.index('nodatacow')]
del subvol_options[subvol_options.index('nodatacow')]
# Make the compress processing now
# it is not an options which can be established by subvolume (but for whole file systems), and can be
# set up via a simple attribute change in a directory (if empty). And here the directories are brand new
# in this way only zstd compression is activaded
# TODO WARNING it is not clear if it should be a standard feature, so it might need to be deactivated
if 'compress' in mount_options:
if (cmd := SysCommand(f"chattr +c {installation.target}/{name}")).exit_code != 0:
raise DiskError(f"Could not set compress attribute at {installation.target}/{name}: {cmd}")
# entry is deleted so nodatacow doesn't propagate to the mount options
del mount_options[mount_options.index('compress')]
if 'compress' in subvol_options:
if not _has_option('compress',partition.get('filesystem',{}).get('mount_options',[])):
if (cmd := SysCommand(f"chattr +c {installation.target}/{name}")).exit_code != 0:
raise DiskError(f"Could not set compress attribute at {installation.target}/{name}: {cmd}")
# entry is deleted so compress doesn't propagate to the mount options
del subvol_options[subvol_options.index('compress')]
# END compress processing.
# we do not mount if THE basic partition will be mounted or if we exclude explicitly this subvolume
if not partition['mountpoint'] and location is not None:
# we begin to create a fake partition entry. First we copy the original -the one that corresponds to
# the primary partition
fake_partition = partition.copy()
# the primary partition. We make a deepcopy to avoid altering the original content in any case
fake_partition = deepcopy(partition)
# we start to modify entries in the "fake partition" to match the needs of the subvolumes
#
# to avoid any chance of entering in a loop (not expected) we delete the list of subvolumes in the copy
# and reset the encryption parameters
del fake_partition['btrfs']
fake_partition['encrypted'] = False
fake_partition['generate-encryption-key-file'] = False
@ -157,22 +169,16 @@ def manage_btrfs_subvolumes(installation :Installer,
fake_partition['mountpoint'] = location
# we load the name in an attribute called subvolume, but i think it is not needed anymore, 'cause the mount logic uses a different path.
fake_partition['subvolume'] = name
# here we add the mount options
fake_partition['options'] = mount_options
# Here comes the most exotic part. The dictionary attribute 'device_instance' contains an instance of Partition. This instance will be queried along the mount process at the installer.
# We instanciate a new object with following attributes coming / adapted from the instance which was in the primary partition entry (the one we are coping - partition['device_instance']
# * path, which will be expanded with the subvolume name to use the bind mount syntax the system uses for naming mounted subvolumes
# * size. When the OS queries all the subvolumes share the same size as the full partititon
# * uuid. All the subvolumes on a partition share the same uuid
if not unlocked_device:
fake_partition['device_instance'] = Partition(f"{partition['device_instance'].path}[/{name}]",partition['device_instance'].size,partition['device_instance'].uuid)
# here we add the special mount options for the subvolume, if any.
# if the original partition['options'] is not a list might give trouble
if fake_partition.get('filesystem',{}).get('mount_options',[]):
fake_partition['filesystem']['mount_options'].extend(subvol_options)
else:
# for subvolumes IN an encrypted partition we make our device instance from unlocked device instead of the raw partition.
# This time we make a copy (we should to the same above TODO) and alter the path by hand
from copy import copy
# KIDS DONT'T DO THIS AT HOME
fake_partition['device_instance'] = copy(unlocked_device)
fake_partition['device_instance'].path = f"{unlocked_device.path}[/{name}]"
fake_partition['filesystem']['mount_options'] = subvol_options
# Here comes the most exotic part. The dictionary attribute 'device_instance' contains an instance of Partition. This instance will be queried along the mount process at the installer.
# As the rest will query there the path of the "partition" to be mounted, we feed it with the bind name needed to mount subvolumes
# As we made a deepcopy we have a fresh instance of this object we can manipulate problemless
fake_partition['device_instance'].path = f"{partition['device_instance'].path}[/{name}]"
# we reset this attribute, which holds where the partition is actually mounted. Remember, the physical partition is mounted at this moment and therefore has the value '/'.
# If i don't reset it, process will abort as "already mounted' .
# TODO It works for this purpose, but the fact that this bevahiour can happed, should make think twice
@ -180,9 +186,7 @@ def manage_btrfs_subvolumes(installation :Installer,
#
# Well, now that this "fake partition" is ready, we add it to the list of the ones which are to be mounted,
# as "normal" ones
mountpoints[fake_partition['mountpoint']] = fake_partition
mountpoints.append(fake_partition)
except Exception as e:
raise e
# if the physical partition has been selected to be mounted, we include it at the list. Remmeber, all the above treatement won't happen except the creation of the subvolume
if partition['mountpoint']:
mountpoints[partition['mountpoint']] = partition
return mountpoints

15
archinstall/lib/disk/filesystem.py
View File

@ -90,6 +90,8 @@ class Filesystem:
raise ValueError(f"{self}.load_layout() doesn't know how to continue without a new partition definition or a UUID ({partition.get('PARTUUID')}) on the device ({self.blockdevice.get_partition(uuid=partition_uuid)}).")
if partition.get('filesystem', {}).get('format', False):
# needed for backward compatibility with the introduction of the new "format_options"
format_options = partition.get('options',[]) + partition.get('filesystem',{}).get('format_options',[])
if partition.get('encrypted', False):
if not partition.get('!password'):
if not storage['arguments'].get('!encryption-password'):
@ -100,15 +102,12 @@ class Filesystem:
storage['arguments']['!encryption-password'] = get_password(f"Enter a encryption password for {partition['device_instance']}")
partition['!password'] = storage['arguments']['!encryption-password']
# to be able to generate an unique name in case the partition will not be mounted
if partition.get('mountpoint',None):
ppath = partition['mountpoint']
loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(partition['mountpoint']).name}loop"
else:
ppath = partition['device_instance'].path
loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(ppath).name}loop"
loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(partition['device_instance'].path).name}"
partition['device_instance'].encrypt(password=partition['!password'])
# Immediately unlock the encrypted device to format the inner volume
with luks2(partition['device_instance'], loopdev, partition['!password'], auto_unmount=True) as unlocked_device:
if not partition.get('format'):
@ -126,9 +125,9 @@ class Filesystem:
continue
break
unlocked_device.format(partition['filesystem']['format'], options=partition.get('options', []))
unlocked_device.format(partition['filesystem']['format'], options=format_options)
elif partition.get('format', False):
partition['device_instance'].format(partition['filesystem']['format'], options=partition.get('options', []))
partition['device_instance'].format(partition['filesystem']['format'], options=format_options)
if partition.get('boot', False):
log(f"Marking partition {partition['device_instance']} as bootable.")

4
archinstall/lib/general.py
View File

@ -485,10 +485,10 @@ def pid_exists(pid: int) -> bool:
def run_custom_user_commands(commands :List[str], installation :Installer) -> None:
for index, command in enumerate(commands):
log(f'Executing custom command "{command}" ...', level=logging.INFO)
with open(f"{installation.target}/var/tmp/user-command.{index}.sh", "w") as temp_script:
temp_script.write(command)
execution_output = SysCommand(f"arch-chroot {installation.target} bash /var/tmp/user-command.{index}.sh")
log(execution_output)

148
archinstall/lib/installer.py
View File

@ -180,80 +180,101 @@ class Installer:
return True
def _create_keyfile(self,luks_handle , partition :dict, password :str):
""" roiutine to create keyfiles, so it can be moved elsewere
"""
if partition.get('generate-encryption-key-file'):
if not (cryptkey_dir := pathlib.Path(f"{self.target}/etc/cryptsetup-keys.d")).exists():
cryptkey_dir.mkdir(parents=True)
# Once we store the key as ../xyzloop.key systemd-cryptsetup can automatically load this key
# if we name the device to "xyzloop".
if partition.get('mountpoint',None):
encryption_key_path = f"/etc/cryptsetup-keys.d/{pathlib.Path(partition['mountpoint']).name}loop.key"
else:
encryption_key_path = f"/etc/cryptsetup-keys.d/{pathlib.Path(partition['device_instance'].path).name}.key"
with open(f"{self.target}{encryption_key_path}", "w") as keyfile:
keyfile.write(generate_password(length=512))
os.chmod(f"{self.target}{encryption_key_path}", 0o400)
luks_handle.add_key(pathlib.Path(f"{self.target}{encryption_key_path}"), password=password)
luks_handle.crypttab(self, encryption_key_path, options=["luks", "key-slot=1"])
def _has_root(self, partition :dict) -> bool:
"""
Determine if an encrypted partition contains root in it
"""
if partition.get("mountpoint") is None:
if (sub_list := partition.get("btrfs",{}).get('subvolumes',{})):
for mountpoint in [sub_list[subvolume] if isinstance(sub_list[subvolume],str) else sub_list[subvolume].get("mountpoint") for subvolume in sub_list]:
if mountpoint == '/':
return True
return False
else:
return False
elif partition.get("mountpoint") == '/':
return True
else:
return False
def mount_ordered_layout(self, layouts: Dict[str, Any]) -> None:
from .luks import luks2
mountpoints = {}
# set the partitions as a list not part of a tree (which we don't need anymore (i think)
list_part = []
list_luks_handles = []
for blockdevice in layouts:
for partition in layouts[blockdevice]['partitions']:
if (subvolumes := partition.get('btrfs', {}).get('subvolumes', {})):
if partition.get('encrypted',False):
if partition.get('mountpoint',None):
ppath = partition['mountpoint']
else:
ppath = partition['device_instance'].path
loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(ppath).name}loop"
# Immediately unlock the encrypted device to format the inner volume
with luks2(partition['device_instance'], loopdev, partition['!password'], auto_unmount=False) as unlocked_device:
unlocked_device.mount(f"{self.target}/")
try:
manage_btrfs_subvolumes(self,partition,mountpoints,subvolumes,unlocked_device)
except Exception as e:
# every exception unmounts the physical volume. Otherwise we let the system in an unstable state
unlocked_device.unmount()
raise e
unlocked_device.unmount()
# TODO generate key
else:
self.mount(partition['device_instance'],"/")
try:
manage_btrfs_subvolumes(self,partition,mountpoints,subvolumes)
except Exception as e:
# every exception unmounts the physical volume. Otherwise we let the system in an unstable state
partition['device_instance'].unmount()
raise e
partition['device_instance'].unmount()
else:
mountpoints[partition['mountpoint']] = partition
for mountpoint in sorted([mnt_dest for mnt_dest in mountpoints.keys() if mnt_dest is not None]):
partition = mountpoints[mountpoint]
if partition.get('encrypted', False) and not partition.get('subvolume',None):
list_part.extend(layouts[blockdevice]['partitions'])
# we manage the encrypted partititons
for partition in [entry for entry in list_part if entry.get('encrypted',False)]:
# open the luks device and all associate stuff
if not (password := partition.get('!password', None)):
raise RequirementError(f"Missing partition {partition['device_instance'].path} encryption password in layout: {partition}")
# i change a bit the naming conventions for the loop device
loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(partition['mountpoint']).name}loop"
if not (password := partition.get('!password', None)):
raise RequirementError(f"Missing mountpoint {mountpoint} encryption password in layout: {partition}")
with (luks_handle := luks2(partition['device_instance'], loopdev, password, auto_unmount=False)) as unlocked_device:
if partition.get('generate-encryption-key-file'):
if not (cryptkey_dir := pathlib.Path(f"{self.target}/etc/cryptsetup-keys.d")).exists():
cryptkey_dir.mkdir(parents=True)
# Once we store the key as ../xyzloop.key systemd-cryptsetup can automatically load this key
# if we name the device to "xyzloop".
encryption_key_path = f"/etc/cryptsetup-keys.d/{pathlib.Path(partition['mountpoint']).name}loop.key"
with open(f"{self.target}{encryption_key_path}", "w") as keyfile:
keyfile.write(generate_password(length=512))
os.chmod(f"{self.target}{encryption_key_path}", 0o400)
luks_handle.add_key(pathlib.Path(f"{self.target}{encryption_key_path}"), password=password)
luks_handle.crypttab(self, encryption_key_path, options=["luks", "key-slot=1"])
log(f"Mounting {mountpoint} to {self.target}{mountpoint} using {unlocked_device}", level=logging.INFO)
unlocked_device.mount(f"{self.target}{mountpoint}")
else:
log(f"Mounting {mountpoint} to {self.target}{mountpoint} using {partition['device_instance']}", level=logging.INFO)
if partition.get('options',[]):
mount_options = ','.join(partition['options'])
partition['device_instance'].mount(f"{self.target}{mountpoint}",options=mount_options)
else:
partition['device_instance'].mount(f"{self.target}{mountpoint}")
loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(partition['device_instance'].path).name}"
# note that we DON'T auto_unmount (i.e. close the encrypted device so it can be used
with (luks_handle := luks2(partition['device_instance'], loopdev, password, auto_unmount=False)) as unlocked_device:
if partition.get('generate-encryption-key-file',False) and not self._has_root(partition):
list_luks_handles.append([luks_handle,partition,password])
# this way all the requesrs will be to the dm_crypt device and not to the physical partition
partition['device_instance'] = unlocked_device
# we manage the btrfs partitions
for partition in [entry for entry in list_part if entry.get('btrfs', {}).get('subvolumes', {})]:
self.mount(partition['device_instance'],"/")
try:
new_mountpoints = manage_btrfs_subvolumes(self,partition)
except Exception as e:
# every exception unmounts the physical volume. Otherwise we let the system in an unstable state
partition['device_instance'].unmount()
raise e
partition['device_instance'].unmount()
if new_mountpoints:
list_part.extend(new_mountpoints)
# we mount. We need to sort by mountpoint to get a good working order
for partition in sorted([entry for entry in list_part if entry.get('mountpoint',False)],key=lambda part: part['mountpoint']):
mountpoint = partition['mountpoint']
log(f"Mounting {mountpoint} to {self.target}{mountpoint} using {partition['device_instance']}", level=logging.INFO)
if partition.get('filesystem',{}).get('mount_options',[]):
mount_options = ','.join(partition['filesystem']['mount_options'])
partition['device_instance'].mount(f"{self.target}{mountpoint}",options=mount_options)
else:
partition['device_instance'].mount(f"{self.target}{mountpoint}")
time.sleep(1)
try:
get_mount_info(f"{self.target}{mountpoint}", traverse=False)
except DiskError:
raise DiskError(f"Target {self.target}{mountpoint} never got mounted properly (unable to get mount information using findmnt).")
# once everything is mounted, we generate the key files in the correct place
for handle in list_luks_handles:
ppath = handle[1]['device_instance'].path
log(f"creating key-file for {ppath}",level=logging.INFO)
self._create_keyfile(handle[0],handle[1],handle[2])
def mount(self, partition :Partition, mountpoint :str, create_mountpoint :bool = True) -> None:
if create_mountpoint and not os.path.isdir(f'{self.target}{mountpoint}'):
os.makedirs(f'{self.target}{mountpoint}')
@ -692,6 +713,7 @@ class Installer:
base_path,bind_path = split_bind_name(str(root_partition.path))
if bind_path is not None: # and root_fs_type == 'btrfs':
options_entry = f"rootflags=subvol={bind_path} " + options_entry
if real_device := self.detect_encryption(root_partition):
# TODO: We need to detect if the encrypted device is a whole disk encryption,
# or simply a partition encryption. Right now we assume it's a partition (and we always have)