Command locales (second batch) (#886)

* flexibilize the definition of execution locale for OS commands executed via the SysCommand* interface.
Defined a storage argument which holds the default
Added functions to
unset the program own locales
reset to the program default locales
set a specific locale
A decorator to execute functions in the host locale environment

* rename decorator local_environ to host_locale_environ
created a simmetric decorator c_locale_environ, to make a routine work with the C locale whatever is set

* Correct definition of btrfs standard layout

* Added error handling

* Fixed issue where archinstall.Boot() would raise an exception in vain

* Added debugging for SysCommandWorker()

* Added some debugging

* Tweaking debug a bit

* Tweaking debug

* Adding more debug

* Adding more debug

* Removed some debugging

* Adding more debug

* Adding more debug

* Adding more debug

* Adding more debug

* Adding more debug

* Adding more debug

* Adding more debug

* Adding more debug

* Adding more debug

* Adding more debug

* Adding more debug

* Removed soem debugging

* Removed soem debugging

* Testing a revert

* Adding back the reverted change, adding lofile

* Redirecting stdout to /dev/null for testing (to avoid interrupting the fork)

* Reverted debug changes

* Testing os.system()

Co-authored-by: Anton Hvornum <anton@hvornum.se>

archinstall/__init__.py

@@ -58,12 +58,12 @@ def define_arguments():
 					help="JSON disk layout file")
 	parser.add_argument("--silent", action="store_true",
 						help="WARNING: Disables all prompts for input and confirmation. If no configuration is provided, this is ignored")
-	parser.add_argument("--dry-run","--dry_run",action="store_true",
+	parser.add_argument("--dry-run", "--dry_run", action="store_true",
 						help="Generates a configuration file and then exits instead of performing an installation")
 	parser.add_argument("--script", default="guided", nargs="?", help="Script to run for installation", type=str)
-	parser.add_argument("--mount-point","--mount_point",nargs="?",type=str,help="Define an alternate mount point for installation")
+	parser.add_argument("--mount-point","--mount_point", nargs="?", type=str, help="Define an alternate mount point for installation")
-	parser.add_argument("--debug",action="store_true",help="Adds debug info into the log")
+	parser.add_argument("--debug", action="store_true", default=False, help="Adds debug info into the log")
-	parser.add_argument("--plugin",nargs="?",type=str)
+	parser.add_argument("--plugin", nargs="?", type=str)
 
 def parse_unspecified_argument_list(unknowns :list, multiple :bool = False, error :bool = False) -> dict:
 	"""We accept arguments not defined to the parser. (arguments "ad hoc").
@@ -170,7 +170,7 @@ def post_process_arguments(arguments):
 	if arguments.get('mount_point'):
 		storage['MOUNT_POINT'] = arguments['mount_point']
 
-	if arguments.get('debug',False):
+	if arguments.get('debug', False):
 		log(f"Warning: --debug mode will write certain credentials to {storage['LOG_PATH']}/{storage['LOG_FILE']}!", fg="red", level=logging.WARNING)
 
 	if arguments.get('plugin', None):

archinstall/lib/disk/helpers.py

@@ -140,11 +140,12 @@ def split_bind_name(path :Union[pathlib.Path, str]) -> list:
 
 def get_mount_info(path :Union[pathlib.Path, str], traverse :bool = False, return_real_path :bool = False) -> Dict[str, Any]:
 	device_path,bind_path = split_bind_name(path)
+	output = {}
+
 	for traversal in list(map(str, [str(device_path)] + list(pathlib.Path(str(device_path)).parents))):
 		try:
 			log(f"Getting mount information for device path {traversal}", level=logging.INFO)
-			output = SysCommand(f'/usr/bin/findmnt --json {traversal}').decode('UTF-8')
+			if (output := SysCommand(f'/usr/bin/findmnt --json {traversal}').decode('UTF-8')):
-			if output:
 				break
 		except SysCallError:
 			pass

archinstall/lib/general.py

@@ -203,7 +203,7 @@ class SysCommandWorker:
 		self.callbacks = callbacks
 		self.peak_output = peak_output
 		# define the standard locale for command outputs. For now the C ascii one. Can be overriden
-		self.environment_vars = {'LC_ALL':'C' , **environment_vars}
+		self.environment_vars = {**storage.get('CMD_LOCALE',{}),**environment_vars}
 		self.logfile = logfile
 		self.working_directory = working_directory
 
@@ -262,10 +262,10 @@ class SysCommandWorker:
 			sys.stdout.flush()
 
 		if len(args) >= 2 and args[1]:
-			log(args[1], level=logging.ERROR, fg='red')
+			log(args[1], level=logging.DEBUG, fg='red')
 
 		if self.exit_code != 0:
-			raise SysCallError(f"{self.cmd} exited with abnormal exit code: {self.exit_code}", self.exit_code)
+			raise SysCallError(f"{self.cmd} exited with abnormal exit code [{self.exit_code}]: {self._trace_log[:500]}", self.exit_code)
 
 	def is_alive(self) -> bool:
 		self.poll()
@@ -350,9 +350,11 @@ class SysCommandWorker:
 		#   and until os.close(), the traceback will get locked inside
 		#   stdout of the child_fd object. `os.read(self.child_fd, 8192)` is the
 		#   only way to get the traceback without loosing it.
+
 		self.pid, self.child_fd = pty.fork()
 		os.chdir(old_dir)
 
+		# https://stackoverflow.com/questions/4022600/python-pty-fork-how-does-it-work
 		if not self.pid:
 			try:
 				try:

archinstall/lib/installer.py

@@ -914,7 +914,7 @@ class Installer:
 			# Setting an empty keymap first, allows the subsequent call to set layout for both console and x11.
 			from .systemd import Boot
 			with Boot(self) as session:
-				session.SysCommand(["localectl", "set-keymap", '""'])
+				os.system('/usr/bin/systemd-run --machine=archinstall --pty localectl set-keymap ""')
 
 				if (output := session.SysCommand(["localectl", "set-keymap", language])).exit_code != 0:
 					raise ServiceException(f"Unable to set locale '{language}' for console: {output}")

archinstall/lib/locale_helpers.py

@@ -1,10 +1,10 @@
 import logging
-from typing import Iterator, List
+from typing import Iterator, List, Callable
 
 from .exceptions import ServiceException
 from .general import SysCommand
 from .output import log
-
+from .storage import storage
 
 def list_keyboard_languages() -> Iterator[str]:
 	for line in SysCommand("localectl --no-pager list-keymaps", environment_vars={'SYSTEMD_COLORS': '0'}):
@@ -28,6 +28,101 @@ def list_locales() -> List[str]:
 		locales.reverse()
 		return locales
 
+def get_locale_mode_text(mode):
+	if mode == 'LC_ALL':
+		mode_text = "general (LC_ALL)"
+	elif mode == "LC_CTYPE":
+		mode_text = "Character set"
+	elif mode == "LC_NUMERIC":
+		mode_text = "Numeric values"
+	elif mode == "LC_TIME":
+		mode_text = "Time Values"
+	elif mode == "LC_COLLATE":
+		mode_text = "sort order"
+	elif mode == "LC_MESSAGES":
+		mode_text = "text messages"
+	else:
+		mode_text = "Unassigned"
+	return mode_text
+
+def reset_cmd_locale():
+	""" sets the cmd_locale to its saved default """
+	storage['CMD_LOCALE'] = storage.get('CMD_LOCALE_DEFAULT',{})
+
+def unset_cmd_locale():
+	""" archinstall will use the execution environment default """
+	storage['CMD_LOCALE'] = {}
+
+def set_cmd_locale(general :str = None,
+				charset :str = 'C',
+				numbers :str = 'C',
+				time :str = 'C',
+				collate :str = 'C',
+				messages :str = 'C'):
+	"""
+	Set the cmd locale.
+	If the parameter general is specified, it takes precedence over the rest (might as well not exist)
+	The rest define some specific settings above the installed default language. If anyone of this parameters is none means the installation default
+	"""
+	installed_locales = list_installed_locales()
+	result = {}
+	if general:
+		if general in installed_locales:
+			storage['CMD_LOCALE'] = {'LC_ALL':general}
+		else:
+			log(f"{get_locale_mode_text('LC_ALL')} {general} is not installed. Defaulting to C",fg="yellow",level=logging.WARNING)
+		return
+
+	if numbers:
+		if numbers in installed_locales:
+			result["LC_NUMERIC"] = numbers
+		else:
+			log(f"{get_locale_mode_text('LC_NUMERIC')} {numbers} is not installed. Defaulting to installation language",fg="yellow",level=logging.WARNING)
+	if charset:
+		if charset in installed_locales:
+			result["LC_CTYPE"] = charset
+		else:
+			log(f"{get_locale_mode_text('LC_CTYPE')} {charset} is not installed. Defaulting to installation language",fg="yellow",level=logging.WARNING)
+	if time:
+		if time in installed_locales:
+			result["LC_TIME"] = time
+		else:
+			log(f"{get_locale_mode_text('LC_TIME')} {time} is not installed. Defaulting to installation language",fg="yellow",level=logging.WARNING)
+	if collate:
+		if collate in installed_locales:
+			result["LC_COLLATE"] = collate
+		else:
+			log(f"{get_locale_mode_text('LC_COLLATE')} {collate} is not installed. Defaulting to installation language",fg="yellow",level=logging.WARNING)
+	if messages:
+		if messages in installed_locales:
+			result["LC_MESSAGES"] = messages
+		else:
+			log(f"{get_locale_mode_text('LC_MESSAGES')} {messages} is not installed. Defaulting to installation language",fg="yellow",level=logging.WARNING)
+	storage['CMD_LOCALE'] = result
+
+def host_locale_environ(func :Callable):
+	""" decorator when we want a function executing in the host's locale environment """
+	def wrapper(*args, **kwargs):
+		unset_cmd_locale()
+		result = func(*args,**kwargs)
+		reset_cmd_locale()
+		return result
+	return wrapper
+
+def c_locale_environ(func :Callable):
+	""" decorator when we want a function executing in the C locale environment """
+	def wrapper(*args, **kwargs):
+		set_cmd_locale(general='C')
+		result = func(*args,**kwargs)
+		reset_cmd_locale()
+		return result
+	return wrapper
+
+def list_installed_locales() -> List[str]:
+	lista = []
+	for line in SysCommand('locale -a'):
+		lista.append(line.decode('UTF-8').strip())
+	return lista
 
 def list_x11_keyboard_languages() -> Iterator[str]:
 	for line in SysCommand("localectl --no-pager list-x11-keymap-layouts", environment_vars={'SYSTEMD_COLORS': '0'}):

archinstall/lib/storage.py

@@ -22,4 +22,6 @@ storage = {
 	'ENC_IDENTIFIER': 'ainst',
 	'DISK_TIMEOUTS' : 1, # seconds
 	'DISK_RETRY_ATTEMPTS' : 20, # RETRY_ATTEMPTS * DISK_TIMEOUTS is used in disk operations
+	'CMD_LOCALE':{'LC_ALL':'C'}, # default locale for execution commands. Can be overriden with set_cmd_locale()
+	'CMD_LOCALE_DEFAULT':{'LC_ALL':'C'}, # should be the same as the former. Not be used except in reset_cmd_locale()
 }

archinstall/lib/systemd.py

@@ -90,11 +90,18 @@ class Boot:
 			log(args[1], level=logging.ERROR, fg='red')
 			log(f"The error above occured in a temporary boot-up of the installation {self.instance}", level=logging.ERROR, fg="red")
 
-		shutdown = SysCommand(f'systemd-run --machine={self.container_name} --pty /bin/bash -c "shutdown now"')
+		shutdown = None
+
+		try:
+			shutdown = SysCommand(f'systemd-run --machine={self.container_name} --pty shutdown now')
+		except SysCallError as error:
+			if error.exit_code == 256:
+				pass
+
 		while self.session.is_alive():
 			time.sleep(0.25)
 
-		if shutdown.exit_code == 0:
+		if self.session.exit_code == 0 or (shutdown and shutdown.exit_code == 0):
 			storage['active_boot'] = None
 		else:
 			raise SysCallError(f"Could not shut down temporary boot of {self.instance}: {shutdown}", exit_code=shutdown.exit_code)

examples/guided.py

@@ -212,9 +212,11 @@ def perform_installation(mountpoint):
 		installation.log('Waiting for automatic mirror selection (reflector) to complete.', level=logging.INFO)
 		while archinstall.service_state('reflector') not in ('dead', 'failed'):
 			time.sleep(1)
+		
 		# Set mirrors used by pacstrap (outside of installation)
 		if archinstall.arguments.get('mirror-region', None):
 			archinstall.use_mirrors(archinstall.arguments['mirror-region'])  # Set the mirrors for the live medium
+		
 		if installation.minimal_installation():
 			installation.set_locale(archinstall.arguments['sys-language'], archinstall.arguments['sys-encoding'].upper())
 			installation.set_hostname(archinstall.arguments['hostname'])