Made sure remote sourcing works for --creds, --config and --disk-layout. (#1170)

* Made sure remote sourcing works for --creds, --config and --disk-layout.

* Spelling error when moving around source code.

archinstall/__init__.py

@@ -149,22 +149,13 @@ def get_arguments() -> Dict[str, Any]:
 	# preprocess the json files.
 	# TODO Expand the url access to the other JSON file arguments ?
 	if args.config is not None:
-		try:
+		if not json_stream_to_structure('--config', args.config, config):
-			# First, let's check if this is a URL scheme instead of a filename
+			exit(1)
-			parsed_url = urllib.parse.urlparse(args.config)
 
-			if not parsed_url.scheme:  # The Profile was not a direct match on a remote URL, it must be a local file.
+	if args.creds is not None:
-				if not json_stream_to_structure('--config',args.config,config):
+		if not json_stream_to_structure('--creds', args.creds, config):
-					exit(1)
+			exit(1)
-			else:  # Attempt to load the configuration from the URL.
+			
-				with urllib.request.urlopen(urllib.request.Request(args.config, headers={'User-Agent': 'ArchInstall'})) as response:
-					config.update(json.loads(response.read()))
-		except Exception as e:
-			raise ValueError(f"Could not load --config because: {e}")
-
-		if args.creds is not None:
-			if not json_stream_to_structure('--creds',args.creds,config):
-				exit(1)
 	# load the parameters. first the known, then the unknowns
 	config.update(vars(args))
 	config.update(parse_unspecified_argument_list(unknowns))

archinstall/lib/general.py

@@ -10,6 +10,9 @@ import string
 import sys
 import time
 import re
+import urllib.parse
+import urllib.request
+import pathlib
 from datetime import datetime, date
 from typing import Callable, Optional, Dict, Any, List, Union, Iterator, TYPE_CHECKING
 # https://stackoverflow.com/a/39757388/929999
@@ -532,32 +535,40 @@ def run_custom_user_commands(commands :List[str], installation :Installer) -> No
 		log(execution_output)
 		os.unlink(f"{installation.target}/var/tmp/user-command.{index}.sh")
 
-def json_stream_to_structure(id : str, stream :str, target :dict) -> bool :
+def json_stream_to_structure(configuration_identifier : str, stream :str, target :dict) -> bool :
-	""" Function to load a stream (file (as name) or valid JSON string into an existing dictionary
+	"""
+	Function to load a stream (file (as name) or valid JSON string into an existing dictionary
 	Returns true if it could be done
 	Return  false if operation could not be executed
-	+id is just a parameter to get meaningful, but not so long messages
+	+configuration_identifier is just a parameter to get meaningful, but not so long messages
 	"""
-	from pathlib import Path
+
-	if Path(stream).exists():
+	parsed_url = urllib.parse.urlparse(stream)
-		try:
+
-			with open(Path(stream)) as fh:
+	if parsed_url.scheme: # The stream is in fact a URL that should be grabed
-				target.update(json.load(fh))
+		with urllib.request.urlopen(urllib.request.Request(stream, headers={'User-Agent': 'ArchInstall'})) as response:
-		except Exception as e:
+			target.update(json.loads(response.read()))
-			log(f"{id} = {stream} does not contain a valid JSON format: {e}",level=logging.ERROR)
-			return False
 	else:
-		log(f"{id} = {stream} does not exists in the filesystem. Trying as JSON stream",level=logging.DEBUG)
+		if pathlib.Path(stream).exists():
-		# NOTE: failure of this check doesn't make stream 'real' invalid JSON, just it first level entry is not an object (i.e. dict), so it is not a format we handle.
-		if stream.strip().startswith('{') and stream.strip().endswith('}'):
 			try:
-				target.update(json.loads(stream))
+				with pathlib.Path(stream).open() as fh:
-			except Exception as e:
+					target.update(json.load(fh))
-				log(f" {id} Contains an invalid JSON format : {e}",level=logging.ERROR)
+			except Exception as error:
+				log(f"{configuration_identifier} = {stream} does not contain a valid JSON format: {error}", level=logging.ERROR, fg="red")
 				return False
 		else:
-			log(f" {id} is neither a file nor is a JSON string:",level=logging.ERROR)
+			# NOTE: This is a rudimentary check if what we're trying parse is a dict structure.
-			return False
+			# Which is the only structure we tolerate anyway.
+			if stream.strip().startswith('{') and stream.strip().endswith('}'):
+				try:
+					target.update(json.loads(stream))
+				except Exception as e:
+					log(f" {configuration_identifier} Contains an invalid JSON format : {e}",level=logging.ERROR, fg="red")
+					return False
+			else:
+				log(f" {configuration_identifier} is neither a file nor is a JSON string:",level=logging.ERROR, fg="red")
+				return False
+
 	return True
 
 def secret(x :str):