Starting to rework the entire codebase to be context friendly. Annotations is next after one successful install.

.gitignore

@@ -1,2 +1,4 @@
 **/**__pycache__
 SAFETY_LOCK
+**/**old.*
+**/**.img

exceptions.py

@@ -0,0 +1,4 @@
+class RequirementError(BaseException):
+	pass
+class DiskError(BaseException):
+	pass

helpers/disk.py

@@ -0,0 +1,109 @@
+import glob, re, os, json
+from collections import OrderedDict
+from helpers.general import sys_command
+
+ROOT_DIR_PATTERN = re.compile('^.*?/devices')
+GPT = 0b00000001
+
+class BlockDevice():
+	def __init__(self, path, info):
+		self.path = path
+		self.info = info
+		if not 'backplane' in self.info:
+			self.info['backplane'] = self.find_backplane(self.info)
+
+	def find_backplane(self, info):
+		if not 'type' in info: raise DiskError(f'Could not locate backplane info for "{self.path}"')
+
+		if info['type'] == 'loop':
+			for drive in json.loads(b''.join(sys_command(f'losetup --json', hide_from_log=True)).decode('UTF_8'))['loopdevices']:
+				if not drive['name'] == self.path: continue
+
+				return drive['back-file']
+		elif info['type'] == 'disk':
+			return self.path
+		elif info['type'] == 'crypt':
+			if not 'pkname' in info: raise DiskError(f'A crypt device ({self.path}) without a parent kernel device name.')
+			return f"/dev/{info['pkname']}"
+
+	def __repr__(self, *args, **kwargs):
+		return f'BlockDevice(path={self.path})'
+
+	def __getitem__(self, key, *args, **kwargs):
+		if not key in self.info:
+			raise KeyError(f'{self} does not contain information: "{key}"')
+		return self.info[key]
+
+#	def __enter__(self, *args, **kwargs):
+#		return self
+#
+#	def __exit__(self, *args, **kwargs):
+#		print('Exit:', args, kwargs)
+#		b''.join(sys_command(f'sync', *args, **kwargs, hide_from_log=True))
+
+class Formatter():
+	def __init__(self, blockdevice, mode=GPT):
+		self.blockdevice = blockdevice
+		self.mode = mode
+
+	def __enter__(self, *args, **kwargs):
+		print(f'Formatting {self.blockdevice} as {self.mode}:', args, kwargs)
+		return self
+
+	def __exit__(self, *args, **kwargs):
+		print('Exit:', args, kwargs)
+		b''.join(sys_command(f'sync', *args, **kwargs, hide_from_log=True))
+
+	def format_disk(drive='drive', start='start', end='size', emulate=False, *positionals, **kwargs):
+		drive = args[drive]
+		start = args[start]
+		end = args[end]
+		if not drive:
+			raise ValueError('Need to supply a drive path, for instance: /dev/sdx')
+
+		if not SAFETY_LOCK:
+			# dd if=/dev/random of=args['drive'] bs=4096 status=progress
+			# https://github.com/dcantrell/pyparted	would be nice, but isn't officially in the repo's #SadPanda
+			#if sys_command(f'/usr/bin/parted -s {drive} mklabel gpt', emulate=emulate, *positionals, **kwargs).exit_code != 0:
+			#	return None
+			if sys_command(f'/usr/bin/parted -s {drive} mklabel gpt', emulate=emulate, *positionals, **kwargs).exit_code != 0:
+				return None
+			if sys_command(f'/usr/bin/parted -s {drive} mkpart primary FAT32 1MiB {start}', emulate=emulate, *positionals, **kwargs).exit_code != 0:
+				return None
+			if sys_command(f'/usr/bin/parted -s {drive} name 1 "EFI"', emulate=emulate, *positionals, **kwargs).exit_code != 0:
+				return None
+			if sys_command(f'/usr/bin/parted -s {drive} set 1 esp on', emulate=emulate, *positionals, **kwargs).exit_code != 0:
+				return None
+			if sys_command(f'/usr/bin/parted -s {drive} set 1 boot on', emulate=emulate, *positionals, **kwargs).exit_code != 0:
+				return None
+			if sys_command(f'/usr/bin/parted -s {drive} mkpart primary {start} {end}', emulate=emulate, *positionals, **kwargs).exit_code != 0:
+				return None
+
+
+def device_state(name, *args, **kwargs):
+	# Based out of: https://askubuntu.com/questions/528690/how-to-get-list-of-all-non-removable-disk-device-names-ssd-hdd-and-sata-ide-onl/528709#528709
+	if os.path.isfile('/sys/block/{}/device/block/{}/removable'.format(name, name)):
+		with open('/sys/block/{}/device/block/{}/removable'.format(name, name)) as f:
+			if f.read(1) == '1':
+				return
+
+	path = ROOT_DIR_PATTERN.sub('', os.readlink('/sys/block/{}'.format(name)))
+	hotplug_buses = ("usb", "ieee1394", "mmc", "pcmcia", "firewire")
+	for bus in hotplug_buses:
+		if os.path.exists('/sys/bus/{}'.format(bus)):
+			for device_bus in os.listdir('/sys/bus/{}/devices'.format(bus)):
+				device_link = ROOT_DIR_PATTERN.sub('', os.readlink('/sys/bus/{}/devices/{}'.format(bus, device_bus)))
+				if re.search(device_link, path):
+					return
+	return True
+
+# lsblk --json -l -n -o path
+def all_disks(*args, **kwargs):
+	if not 'partitions' in kwargs: kwargs['partitions'] = False
+	drives = OrderedDict()
+	#for drive in json.loads(sys_command(f'losetup --json', *args, **lkwargs, hide_from_log=True)).decode('UTF_8')['loopdevices']:
+	for drive in json.loads(b''.join(sys_command(f'lsblk --json -l -n -o path,size,type,mountpoint,label,pkname', *args, **kwargs, hide_from_log=True)).decode('UTF_8'))['blockdevices']:
+		if not kwargs['partitions'] and drive['type'] == 'part': continue
+
+		drives[drive['path']] = BlockDevice(drive['path'], drive)
+	return drives

helpers/general.py

@@ -0,0 +1,206 @@
+import os, json, hashlib, shlex
+import time, pty
+from subprocess import Popen, STDOUT, PIPE, check_output
+from select import epoll, EPOLLIN, EPOLLHUP
+
+def log(*args, **kwargs):
+	print(' '.join([str(x) for x in args]))
+
+def gen_uid(entropy_length=256):
+	return hashlib.sha512(os.urandom(entropy_length)).hexdigest()
+
+class sys_command():#Thread):
+	"""
+	Stolen from archinstall_gui
+	"""
+	def __init__(self, cmd, callback=None, start_callback=None, *args, **kwargs):
+		if not 'worker_id' in kwargs: kwargs['worker_id'] = gen_uid()
+		if not 'emulate' in kwargs: kwargs['emulate'] = False
+		#Thread.__init__(self)
+		if kwargs['emulate']:
+			log(f"Starting command '{cmd}' in emulation mode.")
+		self.cmd = shlex.split(cmd)
+		self.args = args
+		self.kwargs = kwargs
+		if not 'worker' in self.kwargs: self.kwargs['worker'] = None
+		self.callback = callback
+		self.pid = None
+		self.exit_code = None
+		self.started = time.time()
+		self.ended = None
+		self.worker_id = kwargs['worker_id']
+		self.trace_log = b''
+		self.status = 'starting'
+
+		user_catalogue = os.path.expanduser('~')
+		self.cwd = f"{user_catalogue}/archinstall/cache/workers/{kwargs['worker_id']}/"
+		self.exec_dir = f'{self.cwd}/{os.path.basename(self.cmd[0])}_workingdir'
+
+		if not self.cmd[0][0] == '/':
+			log('Worker command is not executed with absolute path, trying to find: {}'.format(self.cmd[0]), origin='spawn', level=5)
+			o = check_output(['/usr/bin/which', self.cmd[0]])
+			log('This is the binary {} for {}'.format(o.decode('UTF-8'), self.cmd[0]), origin='spawn', level=5)
+			self.cmd[0] = o.decode('UTF-8').strip()
+
+		if not os.path.isdir(self.exec_dir):
+			os.makedirs(self.exec_dir)
+
+		if self.kwargs['emulate']:
+			commandlog.append(cmd + ' # (emulated)')
+		elif 'hide_from_log' in self.kwargs and self.kwargs['hide_from_log']:
+			pass
+		else:
+			commandlog.append(cmd)
+		if start_callback: start_callback(self, *args, **kwargs)
+		#self.start()
+		self.run()
+
+	def __iter__(self, *args, **kwargs):
+		for line in self.trace_log.split(b'\n'):
+			yield line
+
+	def __repr__(self, *args, **kwargs):
+		return f"{self.cmd, self.trace_log}"
+
+	def decode(self, fmt='UTF-8'):
+		return self.trace_log.decode(fmt)
+
+	def dump(self):
+		return {
+			'status' : self.status,
+			'worker_id' : self.worker_id,
+			'worker_result' : self.trace_log.decode('UTF-8'),
+			'started' : self.started,
+			'ended' : self.ended,
+			'started_pprint' : '{}-{}-{} {}:{}:{}'.format(*time.localtime(self.started)),
+			'ended_pprint' : '{}-{}-{} {}:{}:{}'.format(*time.localtime(self.ended)) if self.ended else None,
+			'exit_code' : self.exit_code
+		}
+
+	def run(self):
+		#main = None
+		#for t in tenum():
+		#	if t.name == 'MainThread':
+		#		main = t
+		#		break
+
+		#if not main:
+		#	log('Main thread not existing')
+		#	return
+		
+		self.status = 'running'
+		old_dir = os.getcwd()
+		os.chdir(self.exec_dir)
+		self.pid, child_fd = pty.fork()
+		if not self.pid: # Child process
+			# Replace child process with our main process
+			if not self.kwargs['emulate']:
+				try:
+					os.execv(self.cmd[0], self.cmd)
+				except FileNotFoundError:
+					self.status = 'done'
+					log(f"{self.cmd[0]} does not exist.", origin='spawn', level=2)
+					self.exit_code = 1
+					return False
+
+		os.chdir(old_dir)
+
+		poller = epoll()
+		poller.register(child_fd, EPOLLIN | EPOLLHUP)
+
+		if 'events' in self.kwargs and 'debug' in self.kwargs:
+			log(f'[D] Using triggers for command: {self.cmd}')
+			log(json.dumps(self.kwargs['events']))
+
+		alive = True
+		last_trigger_pos = 0
+		while alive and not self.kwargs['emulate']:
+			for fileno, event in poller.poll(0.1):
+				try:
+					output = os.read(child_fd, 8192).strip()
+					self.trace_log += output
+				except OSError:
+					alive = False
+					break
+
+				if 'debug' in self.kwargs and self.kwargs['debug'] and len(output):
+					log(self.cmd[0], 'gave:', output.decode('UTF-8'))
+					log(self.cmd[0],'gave:', output.decode('UTF-8'), origin='spawn', level=4)
+
+				if 'on_output' in self.kwargs:
+					self.kwargs['on_output'](self.kwargs['worker'], output)
+
+				lower = output.lower()
+				broke = False
+				if 'events' in self.kwargs:
+					for trigger in list(self.kwargs['events']):
+						if type(trigger) != bytes:
+							original = trigger
+							trigger = bytes(original, 'UTF-8')
+							self.kwargs['events'][trigger] = self.kwargs['events'][original]
+							del(self.kwargs['events'][original])
+						if type(self.kwargs['events'][trigger]) != bytes:
+							self.kwargs['events'][trigger] = bytes(self.kwargs['events'][trigger], 'UTF-8')
+
+						if trigger.lower() in self.trace_log[last_trigger_pos:].lower():
+							trigger_pos = self.trace_log[last_trigger_pos:].lower().find(trigger.lower())
+
+							if 'debug' in self.kwargs and self.kwargs['debug']:
+								log(f"Writing to subprocess {self.cmd[0]}: {self.kwargs['events'][trigger].decode('UTF-8')}")
+								log(f"Writing to subprocess {self.cmd[0]}: {self.kwargs['events'][trigger].decode('UTF-8')}", origin='spawn', level=5)
+
+							last_trigger_pos = trigger_pos
+							os.write(child_fd, self.kwargs['events'][trigger])
+							del(self.kwargs['events'][trigger])
+							broke = True
+							break
+
+					if broke:
+						continue
+
+					## Adding a exit trigger:
+					if len(self.kwargs['events']) == 0:
+						if 'debug' in self.kwargs and self.kwargs['debug']:
+							log(f"Waiting for last command {self.cmd[0]} to finish.", origin='spawn', level=4)
+
+						if bytes(f']$'.lower(), 'UTF-8') in self.trace_log[0-len(f']$')-5:].lower():
+							if 'debug' in self.kwargs and self.kwargs['debug']:
+								log(f"{self.cmd[0]} has finished.", origin='spawn', level=4)
+							alive = False
+							break
+
+		self.status = 'done'
+
+		if 'debug' in self.kwargs and self.kwargs['debug']:
+			log(f"{self.cmd[0]} waiting for exit code.", origin='spawn', level=5)
+
+		if not self.kwargs['emulate']:
+			try:
+				self.exit_code = os.waitpid(self.pid, 0)[1]
+			except ChildProcessError:
+				try:
+					self.exit_code = os.waitpid(child_fd, 0)[1]
+				except ChildProcessError:
+					self.exit_code = 1
+		else:
+			self.exit_code = 0
+
+		if 'ignore_errors' in self.kwargs:
+			self.exit_code = 0
+
+		if self.exit_code != 0:
+			log(f"{self.cmd} did not exit gracefully, exit code {self.exit_code}.", origin='spawn', level=3)
+			log(self.trace_log.decode('UTF-8'), origin='spawn', level=3)
+		else:
+			log(f"{self.cmd[0]} exit nicely.", origin='spawn', level=5)
+
+		self.ended = time.time()
+		with open(f'{self.cwd}/trace.log', 'wb') as fh:
+			fh.write(self.trace_log)
+
+def prerequisit_check():
+	if not os.path.isdir('/sys/firmware/efi'):
+		raise RequirementError('Archinstall only supports machines in UEFI mode.')
+
+	return True
+

helpers/user_interaction.py

@@ -0,0 +1,17 @@
+from exceptions import *
+
+def select_disk(dict_o_disks):
+	drives = sorted(list(dict_o_disks.keys()))
+	if len(drives) > 1:
+		for index, drive in enumerate(drives):
+			print(f"{index}: {drive} ({dict_o_disks[drive]['size'], dict_o_disks[drive]['backplane'], dict_o_disks[drive]['label']})")
+		drive = input('Select one of the above disks (by number or full path): ')
+		if drive.isdigit():
+			drive = dict_o_disks[drives[int(drive)]]
+		elif drive in dict_o_disks:
+			drive = dict_o_disks[drive]
+		else:
+			raise DiskError(f'Selected drive does not exist: "{drive}"')
+		return drive
+
+	raise DiskError('select_disk() requires a non-empty dictionary of disks to select from.')

installer.py

@@ -0,0 +1,22 @@
+import archinstall, getpass
+
+selected_hdd = archinstall.select_disk(archinstall.all_disks())
+disk_password = getpass.getpass(prompt='Disk password (won\'t echo): ')
+
+with archinstall.Formatter(selected_hdd, archinstall.GPT) as formatter:
+	exit(1)
+	disk.encrypt('luks2', password=disk_password, key_size=512, hash_type='sha512', iter_time=10000, key_file='./pwfile')
+
+	root_partition = disk.partition['/']
+
+with archinstall.installer(root_partition, hostname='testmachine') as installation:
+	if installation.minimal_installation():
+		installation.add_bootloader()
+
+		installation.add_additional_packages(['nano', 'wget', 'git'])
+		installation.install_profile('desktop')
+
+		installation.user_create('anton', 'test')
+		installation.user_set_pw('root', 'toor')
+
+		installation.add_AUR_support()