* New feature: deployment structure can now import/include other templates (Included templates will be installed first, since we'll assume those are baselines/requirements for slimmer deploy-structs)

* Deployment scripts strings now supports python string formatting, if parameter  or  is given as options to each command-line (TODO: Add as a global  parameter as well, and treat all strings as formatable)
* Added two templates: workstation + 00:11:22:33:44:55, a example of how to set up a workstation using a template and a custom addition. Also added a default template that is similar to the workstation but does everything in one template. They all use some awk magic to fix alt-tab in Awesome WM, it also creates a "desktop" environment and installs some useful tools and fixes the start menu in Awesome WM.
* Reordering mirrors is now a configurable option (Default turned off, to better suit offline environments)
* Disk/root password now configurable as either a fixed string, or ask for via STDIN (Syntax for now is: )
* deployment targets now a variable (todo: add a parameter/argument)
* SSL support (ignores certificate validation for now, I know this is bad, but until a parameter etc is in place, this has to be a nessecary evil for now)
* Cleaned up a lot of custom variables and baked them into  so it's easier to format strings based off information gathered along the way (such as partition names).
* Error handling: Mostly for missing dependencies and situations, such as not booted in UEFI environment
This commit is contained in:
Lord Anton Hvornum 2018-05-24 20:09:51 +02:00
parent 3b07cf27e8
commit 252af79e3d
5 changed files with 226 additions and 55 deletions

View File

@ -1,7 +1,7 @@
#!/usr/bin/python3
import traceback
import psutil, os, re, struct, sys, json
import urllib.request, urllib.parse
import urllib.request, urllib.parse, ssl
from glob import glob
#from select import epoll, EPOLLIN, EPOLLHUP
from socket import socket, inet_ntoa, AF_INET, AF_INET6, AF_PACKET
@ -9,9 +9,13 @@ from collections import OrderedDict as oDict
from subprocess import Popen, STDOUT, PIPE
from time import sleep
## FIXME: dependency checks (fdisk, lsblk etc)
rootdir_pattern = re.compile('^.*?/devices')
harddrives = oDict()
deploy_target = 'https://raw.githubusercontent.com/Torxed/archinstall/net-deploy/deployments'
args = {}
positionals = []
for arg in sys.argv[1:]:
@ -105,10 +109,11 @@ def grab_partitions(dev):
parts = oDict()
o = run('lsblk -o name -J -b {dev}'.format(dev=dev))
r = json.loads(o)
for part in r['blockdevices'][0]['children']:
parts[part['name'][len(drive_name):]] = {
# TODO: Grab partition info and store here?
}
if len(r['blockdevices']) and 'children' in r['blockdevices'][0]:
for part in r['blockdevices'][0]['children']:
parts[part['name'][len(drive_name):]] = {
# TODO: Grab partition info and store here?
}
return parts
@ -134,22 +139,64 @@ def multisplit(s, splitters):
def grab_url_data(path):
safe_path = path[:path.find(':')+1]+''.join([item if item in ('/', '?', '=', '&') else urllib.parse.quote(item) for item in multisplit(path[path.find(':')+1:], ('/', '?', '=', '&'))])
response = urllib.request.urlopen(safe_path)
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode=ssl.CERT_NONE
response = urllib.request.urlopen(safe_path, context=ssl_context)
return response.read()
def get_instructions(target):
instructions = {}
try:
instructions = grab_url_data('{}/{}.json'.format(deploy_target, target))
except urllib.error.HTTPError:
print('[N] No instructions found called: {}'.format(target))
return instructions
print('[N] Found net-deploy instructions called: {}'.format(target))
try:
instructions = json.loads(instructions.decode('UTF-8'), object_pairs_hook=oDict)
except:
print('[E] JSON instructions failed to load for {}'.format(target))
traceback.print_exc()
sleep(5)
return instructions
def merge_dicts(d1, d2, before=True, overwrite=False):
""" Merges d2 into d1 """
if before:
d1, d2 = d2.copy(), d1.copy()
overwrite = True
for key, val in d2.items():
if key in d1:
if type(d1[key]) in [dict, oDict] and type(d2[key]) in [dict, oDict]:
d1[key] = merge_dicts(d1[key] if not before else d2[key], d2[key] if not before else d1[key], before=before, overwrite=overwrite)
elif overwrite:
d1[key] = val
else:
d1[key] = val
return d1
if __name__ == '__main__':
update_git() # Breaks and restarts the script if an update was found.
update_drive_list()
if not os.path.isdir('/sys/firmware/efi'):
print('[E] This script only supports UEFI-booted machines.')
exit(1)
## Setup some defaults (in case no command-line parameters or netdeploy-params were given)
if not 'drive' in args: args['drive'] = list(harddrives.keys())[0] # First drive found
if not 'size' in args: args['size'] = '100%'
if not 'start' in args: args['start'] = '513MiB'
if not 'pwfile' in args: args['pwfile'] = '/tmp/diskpw'
if not 'hostname' in args: args['hostname'] = 'Arcinstall'
if not 'country' in args: args['country'] = 'SE' #all
if not 'packages' in args: args['packages'] = ''
if not 'country' in args: args['country'] = 'SE' # 'all' if we don't want country specific mirrors.
if not 'packages' in args: args['packages'] = '' # extra packages other than default
if not 'post' in args: args['post'] = 'reboot'
if not 'password' in args: args['password'] = '0000'
if not 'password' in args: args['password'] = '0000' # Default disk passord, can be <STDIN> or a fixed string
## == If we got networking,
# Try fetching instructions for this box and execute them.
@ -160,28 +207,27 @@ if __name__ == '__main__':
print('[N] No network interfaces - No net deploy.')
else:
for mac in locmac:
try:
instructions = grab_url_data('https://raw.githubusercontent.com/Torxed/archinstall/net-deploy/deployments/{}.json'.format(mac))
except urllib.error.HTTPError:
print('[N] No instructions for this box on this mac: {}'.format(mac))
continue
#print('Decoding:', instructions)
try:
instructions = json.loads(instructions.decode('UTF-8'), object_pairs_hook=oDict)
except:
print('[E] JSON instructions failed to load for {}'.format(mac))
traceback.print_exc()
instructions = {}
sleep(5)
continue
instructions = get_instructions(mac)
if 'args' in instructions:
## == Recursively fetch instructions if "include" is found under {args: ...}
while 'include' in instructions['args']:
includes = instructions['args']['include']
print('[!] Importing net-deploy target: {}'.format(includes))
del(instructions['args']['include'])
if type(includes) in (dict, list):
for include in includes:
instructions = merge_dicts(instructions, get_instructions(include), before=True)
else:
instructions = merge_dicts(instructions, get_instructions(includes), before=True)
## Update arguments if we found any
for key, val in instructions['args'].items():
args[key] = val
else:
print('[N] No gateway - No net deploy')
if args['password'] == '<STDIN>': args['password'] = input('Enter a disk (and root) password: ')
print(args)
if not os.path.isfile(args['pwfile']):
@ -207,32 +253,47 @@ if __name__ == '__main__':
o = run('parted -s {drive} set 1 boot on'.format(**args))
o = run('parted -s {drive} mkpart primary {start} {size}'.format(**args))
first, second = grab_partitions(args['drive']).keys()
o = run('mkfs.vfat -F32 {drive}{part1}'.format(**args, part1=first))
args['paritions'] = grab_partitions(args['drive'])
if len(args['paritions']) <= 0:
print('[E] No paritions were created on {drive}'.format(**args), o)
exit(1)
for index, part_name in enumerate(args['paritions']):
args['partition_{}'.format(index+1)] = part_name
o = run('mkfs.vfat -F32 {drive}{partition_1}'.format(**args))
if (b'mkfs.fat' not in o and b'mkfs.vfat' not in o) or b'command not found' in o:
print('[E] Could not setup {drive}{partition_1}'.format(**args), o)
exit(1)
# "--cipher sha512" breaks the shit.
# TODO: --use-random instead of --use-urandom
print('[N] Adding encryption to {drive}{part2}.'.format(**args, part2=second))
o = run('cryptsetup -q -v --type luks2 --pbkdf argon2i --hash sha512 --key-size 512 --iter-time 10000 --key-file {pwfile} --use-urandom luksFormat {drive}{part2}'.format(**args, part2=second))
print('[N] Adding encryption to {drive}{partition_2}.'.format(**args))
o = run('cryptsetup -q -v --type luks2 --pbkdf argon2i --hash sha512 --key-size 512 --iter-time 10000 --key-file {pwfile} --use-urandom luksFormat {drive}{partition_2}'.format(**args))
if not o.decode('UTF-8').strip() == 'Command successful.':
print('[E] Failed to setup disk encryption.')
print('[E] Failed to setup disk encryption.', o)
exit(1)
o = run('cryptsetup open {drive}{part2} luksdev --key-file {pwfile} --type luks2'.format(**args, part2=second))
o = run('cryptsetup open {drive}{partition_2} luksdev --key-file {pwfile} --type luks2'.format(**args))
o = run('file /dev/mapper/luksdev') # /dev/dm-0
if b'cannot open' in o:
print('[E] Could not mount encrypted device.')
print('[E] Could not mount encrypted device.', o)
exit(1)
print('[N] Creating btrfs filesystem inside {drive}{partition_2}'.format(**args))
o = run('mkfs.btrfs /dev/mapper/luksdev')
if not b'UUID' in o:
print('[E] Could not setup btrfs filesystem.', o)
exit(1)
o = run('mount /dev/mapper/luksdev /mnt')
print('[N] Reordering mirrors.')
os.makedirs('/mnt/boot')
o = run('mount {drive}{part1} /mnt/boot'.format(**args, part1=first))
o = run("wget 'https://www.archlinux.org/mirrorlist/?country={country}&protocol=https&ip_version=4&ip_version=6&use_mirror_status=on' -O /root/mirrorlist".format(**args))
o = run("sed -i 's/#Server/Server/' /root/mirrorlist")
o = run('rankmirrors -n 6 /root/mirrorlist > /etc/pacman.d/mirrorlist')
o = run('mount {drive}{partition_1} /mnt/boot'.format(**args))
print('[N] Reordering mirrors.')
if 'mirrors' in args and args['mirrors'] and get_default_gateway_linux():
o = run("wget 'https://www.archlinux.org/mirrorlist/?country={country}&protocol=https&ip_version=4&ip_version=6&use_mirror_status=on' -O /root/mirrorlist".format(**args))
o = run("sed -i 's/#Server/Server/' /root/mirrorlist")
o = run('rankmirrors -n 6 /root/mirrorlist > /etc/pacman.d/mirrorlist')
pre_conf = {}
if 'pre' in instructions:
@ -247,13 +308,21 @@ if __name__ == '__main__':
for title in pre_conf:
print('[N] Network prerequisit step: {}'.format(title))
for command in pre_conf[title]:
opts = pre_conf[title][command] if type(pre_conf[title][command]) in (dict, oDict) else {}
raw_command = command
opts = pre_conf[title][raw_command] if type(pre_conf[title][raw_command]) in (dict, oDict) else {}
if len(opts):
print('[-] Options: {}'.format(opts))
if 'pass-args' in opts or 'format' in opts:
command = command.format(**args)
if 'pass-args' in opts:
del(opts['pass-args'])
elif 'format' in opts:
del(opts['format'])
else:
print('[-] Options: {}'.format(opts))
#print('[N] Command: {} ({})'.format(command, opts))
#print('[N] Command: {} ({})'.format(raw_command, opts))
o = run('{c}'.format(c=command), opts)
if type(conf[title][command]) == bytes and len(conf[title][command]) and not conf[title][command] in o:
if type(conf[title][raw_command]) == bytes and len(conf[title][raw_command]) and not conf[title][raw_command] in o:
print('[W] Prerequisit step failed: {}'.format(o.decode('UTF-8')))
#print(o)
@ -261,6 +330,10 @@ if __name__ == '__main__':
o = run('pacman -Syy')
o = run('pacstrap /mnt base base-devel btrfs-progs efibootmgr nano wpa_supplicant dialog {packages}'.format(**args))
if not os.path.isdir('/mnt/etc'):
print('[E] Failed to strap in packages', o)
exit(1)
o = run('genfstab -pU /mnt >> /mnt/etc/fstab')
with open('/mnt/etc/fstab', 'a') as fstab:
fstab.write('\ntmpfs /tmp tmpfs defaults,noatime,mode=1777 0 0\n') # Redundant \n at the start? who knoes?
@ -299,8 +372,8 @@ if __name__ == '__main__':
## For some reason, blkid and /dev/disk/by-uuid are not getting along well.
## And blkid is wrong in terms of LUKS.
#UUID = run('blkid -s PARTUUID -o value {drive}{part2}'.format(**args, part2=second)).decode('UTF-8').strip()
UUID = run("ls -l /dev/disk/by-uuid/ | grep {basename}{part2} | awk '{awk}'".format(basename=os.path.basename(args['drive']), part2=second, awk='{print $9}')).decode('UTF-8').strip()
#UUID = run('blkid -s PARTUUID -o value {drive}{partition_2}'.format(**args)).decode('UTF-8').strip()
UUID = run("ls -l /dev/disk/by-uuid/ | grep {basename}{partition_2} | awk '{{print $9}}'".format(basename=os.path.basename(args['drive']), **args)).decode('UTF-8').strip()
with open('/mnt/boot/loader/entries/arch.conf', 'w') as entry:
entry.write('title Arch Linux\n')
entry.write('linux /vmlinuz-linux\n')
@ -316,13 +389,16 @@ if __name__ == '__main__':
for title in conf:
print('[N] Network Deploy: {}'.format(title))
for command in conf[title]:
raw_command = command
opts = conf[title][command] if type(conf[title][command]) in (dict, oDict) else {}
if len(opts):
print('[-] Options: {}'.format(opts))
if 'pass-args' in opts and opts['pass-args']:
command = command.format(**args)
#print('[N] Command: {} ({})'.format(command, opts))
o = run('arch-chroot /mnt {c}'.format(c=command), opts)
if type(conf[title][command]) == bytes and len(conf[title][command]) and not conf[title][command] in o:
if type(conf[title][raw_command]) == bytes and len(conf[title][raw_command]) and not conf[title][raw_command] in o:
print('[W] Post install command failed: {}'.format(o.decode('UTF-8')))
#print(o)

View File

@ -0,0 +1,20 @@
{
"args" : {
"include" : "workstation",
"user" : "anton",
"password" : "1111",
"drive" : "/dev/sdb",
"post" : "don't reboot"
},
"post" : {
"Setup a basic virtual environment": {
"mkdir -p /home/{user}/virts" : {"pass-args" : true},
"qemu-img create -f qcow2 /home/{user}/virts/test_deploy.qcow2 4G" : {"pass-args" : true},
"chown -R {user}.{user} /home/{user}/virts" : {"pass-args" : true}
},
"Setup user" : {
"useradd -m -G wheel -s /bin/bash anton" : null,
"sh -c \"echo {user}:{password} | chpasswd\"" : {"pass-args" : true}
}
}
}

37
deployments/default.json Normal file
View File

@ -0,0 +1,37 @@
{
"args" : {
"password" : "<STDIN>",
"user" : "anton",
"_webbrowser" : "chromium",
"_window_manager" : "awesome",
"_keyboard_layout" : "sv-latin1"
},
"post" : {
"Install workstation packages": {
"pacman -Syy --noconfirm {_webbrowser} {_window_manager} openssh sshfs git dhclient ttf-freefont xorg-server xorg-xrandr xorg-xinit xterm nano wget pulseaudio pulseaudio-alsa pavucontrol smbclient cifs-utils xscreensaver" : null
},
"Enable autostarts": {
"systemctl enable dhcpcd" : null
},
"Setup desktop environment" : {
"sed -i 's/^twm &/#&/' /etc/X11/xinit/xinitrc" : null,
"sed -i 's/^xclock/#&/' /etc/X11/xinit/xinitrc" : null,
"sed -i 's/^xterm/#&/' /etc/X11/xinit/xinitrc" : null,
"sed -i 's/^exec xterm/#&/' /etc/X11/xinit/xinitrc" : null,
"sh -c \"echo 'setxkbmap se' >> /etc/X11/xinit/xinitrc\"" : null,
"sh -c \"echo 'xscreensaver -no-splash &' >> /etc/X11/xinit/xinitrc\"" : null,
"sh -c \"echo 'exec {_window_manager}' >> /etc/X11/xinit/xinitrc\"" : {"pass-args" : true},
"sh -c \"echo 'KEYMAP={_keyboard_layout}\nFONT=lat9w-16' >> /etc/vconsole.conf\"" : {"pass-args" : true},
"sh -c \"sed -i 's/{ \\\"open terminal\\\", terminal/{ \\\"Chromium\\\", \\\"chromium\\\" },\n &1/' /etc/xdg/awesome/rc.lua\"" : null,
"sh -c \"sed -i 's/{ \\\"open terminal\\\", terminal/{ \\\"File handler\\\", \\\"nemo\\\" },\n &1/' /etc/xdg/awesome/rc.lua\"" : null,
"sh -c \"sed -i 's/^globalkeys = gears.table.join(/&\n awful.key({ modkey, }, \\\"l\\\", function() awful.spawn(\\\"xscreensaver-command -lock &\\\") end),\n/' /etc/xdg/awesome/rc.lua\"" : null,
"sh -c \"awk -i inplace -v RS='' '{gsub(/awful.key\\({ modkey,.*?}, \\\"Tab\\\",.*?\\\"client\\\"}\\),/, \\\"awful.key({ modkey, }, \\\"Tab\\\",\\\n function ()\\\n awful.client.focus.byidx(-1)\\\n if client.focus then\\\n client.focus:raise()\\\n end\\\n end),\\\n awful.key({ modkey, \\\"Shift\\\" }, \\\"Tab\\\",\\\n function ()\\\n awful.client.focus.byidx(1)\\\n if client.focus then\\\n client.focus.raise()\\\n end\\\n end),\\\"); print}' /etc/xdg/awesome/rc.lua\"" : null,
"gsettings set org.nemo.desktop show-desktop-icons false" : null,
"xdg-mime default nemo.desktop inode/directory application/x-gnome-saved-search" : null
},
"Setup users" : {
"useradd -m -G wheel -s /bin/bash anton" : null,
"sh -c \"echo {user}:{password} | chpasswd\"" : {"pass-args" : true}
}
}
}

View File

@ -0,0 +1,35 @@
{
"args" : {
"password" : "<STDIN>",
"_mediaplayer" : "lollypop gstreamer gst-plugins-good gnome-keyring",
"_webbrowser" : "chromium",
"_window_manager" : "awesome",
"_keyboard_layout" : "sv-latin1",
"_virtulization" : "qemu ovmf",
"post" : "don't reboot"
},
"post" : {
"Install workstation packages": {
"pacman -Syy --noconfirm openssh sshfs git {_webbrowser} {_mediaplayer} {_window_manager} {_virtulization} dhclient ttf-freefont xorg-server xorg-xrandr xorg-xinit xterm nano wget pulseaudio pulseaudio-alsa pavucontrol smbclient cifs-utils xscreensaver" : {"pass-args" : true}
},
"Setup virtulization" : {
"sh -c \"Description=\\\"Bridge for virtual machines\\\"\nInterface=br0\nConnection=bridge\nBindsToInterfaces=(eno1)\nIP=no\nExecUpPost=\\\"ip link set dev br0 address $(cat /sys/class/net/eno1/address); IP=dhcp; ip_set\\\"\nExecDownPre=\\\"IP=dhcp\\\"\n\n## Ignore (R)STP and immediately activate the bridge\nSkipForwardingDelay=yes\"" : null
},
"Setup desktop environment" : {
"sed -i 's/^twm &/#&/' /etc/X11/xinit/xinitrc" : null,
"sed -i 's/^xclock/#&/' /etc/X11/xinit/xinitrc" : null,
"sed -i 's/^xterm/#&/' /etc/X11/xinit/xinitrc" : null,
"sed -i 's/^exec xterm/#&/' /etc/X11/xinit/xinitrc" : null,
"sh -c \"echo 'setxkbmap se' >> /etc/X11/xinit/xinitrc\"" : null,
"sh -c \"echo 'xscreensaver -no-splash &' >> /etc/X11/xinit/xinitrc\"" : null,
"sh -c \"echo 'exec {_window_manager}' >> /etc/X11/xinit/xinitrc\"" : {"pass-args" : true},
"sh -c \"echo 'KEYMAP={_keyboard_layout}\nFONT=lat9w-16' >> /etc/vconsole.conf\"" : {"pass-args" : true},
"sh -c \"sed -i 's/{ \\\"open terminal\\\", terminal/{ \\\"Chromium\\\", \\\"chromium\\\" },\n &1/' /etc/xdg/awesome/rc.lua\"" : null,
"sh -c \"sed -i 's/{ \\\"open terminal\\\", terminal/{ \\\"File handler\\\", \\\"nemo\\\" },\n &1/' /etc/xdg/awesome/rc.lua\"" : null,
"sh -c \"sed -i 's/^globalkeys = gears.table.join(/&\n awful.key({ modkey, }, \\\"l\\\", function() awful.spawn(\\\"xscreensaver-command -lock &\\\") end),\n/' /etc/xdg/awesome/rc.lua\"" : null,
"sh -c \"awk -i inplace -v RS='' '{gsub(/awful.key\\({ modkey,.*?}, \\\"Tab\\\",.*?\\\"client\\\"}\\),/, \\\"awful.key({ modkey, }, \\\"Tab\\\",\n function ()\n awful.client.focus.byidx(-1)\n if client.focus then\n client.focus:raise()\n end\n end),\n awful.key({ modkey, \\\"Shift\\\" }, \\\"Tab\\\",\n function ()\n awful.client.focus.byidx(1)\n if client.focus then\n client.focus.raise()\n end\n end),\\\"); print}' /etc/xdg/awesome/rc.lua\"" : null,
"gsettings set org.nemo.desktop show-desktop-icons false" : null,
"xdg-mime default nemo.desktop inode/directory application/x-gnome-saved-search" : null
}
}
}

View File

@ -1,17 +1,24 @@
#!/bin/bash
# offline_mirror_path - is used to temporarily store build AUR packages
# in order to "host" them to the build process.
# The path will be on the build machine, not inside the build itself.
work_dir=$1
arch=$2
offline_mirror_path="/tmp/aur_offline"
# A func to download, build ...
# A func to download, build and host AUR packages to the ISO build process
build_aur () {
old_dir=`pwd`
package=$1
# Prep with a build-user:
# Prep with a build-user (removed at the end):
# TODO: Check if already exists, if so, randomize name/don't remove at the end.
# TODO: Don't give permission to wheel, give it only to this user (easy, but needs debugging first)
useradd -m -G wheel builder
sed -i 's/# %wheel ALL=(ALL) NO/%wheel ALL=(ALL) NO/' /etc/sudoers
# Extract the AUR package.
cd /tmp
rm -rf ${package} ${package}.tar.gz
wget "https://aur.archlinux.org/cgit/aur.git/snapshot/${package}.tar.gz"
@ -24,18 +31,12 @@ build_aur () {
su - builder -c "(cd ${build_dir}; makepkg -s --noconfirm)" >/dev/null 2>&1
echo " => Adding ${package} to local AUR mirror"
echo " => Adding ${package} to local AUR hosting directory ${offline_mirror_path}"
mkdir -p ${offline_mirror_path}
sh -c "cp *.xz ${offline_mirror_path}/"
sh -c "repo-add ${offline_mirror_path}/aur_offline.db.tar.gz ${offline_mirror_path}/*.xz"
if [[ -z $(cat ${old_dir}/packages.both | grep ${package}) ]]; then
# TODO: save a copy of ${old_dir}/packages.both ONCE, if it doesn't excist already.
# This in order to revert our AUR changes which will affect a re-build.
echo " => Adding ${package} to packages.both (from AUR)"
echo "${package}" >> ${old_dir}/packages.both
fi
## Long term storage inside the ISO? (if we want to install to disk, we need to pass this along)
## Long term storage inside the ISO? (if we want to install from CD to disk or host it to others)
# sh -c "mv *.xz ${old_dir}/$2/$1.pkg.tar.xz"
cd ${old_dir}
@ -48,8 +49,10 @@ build_aur () {
echo "Starting to sync upstream changes to offline mirror."
rm -rf /tmp/sync /tmp/local
echo " => Building AUR (Adding packages to packages.both as we go along)"
build_aur "lighttpd2-git"
echo " => Building AUR packages (found in packages.aur)"
for package in $(cat ${work_dir}/packages.aur); do
build_aur package
done
if [[ -z $(cat ${work_dir}/pacman.conf | grep '\[aur_offline\]') ]]; then
echo " => Adding offline mirror to the chroot environment"