From 2063a65ffe616ad157f70ee6a076cd3d9453266d Mon Sep 17 00:00:00 2001 From: Torxed Date: Mon, 13 May 2024 08:26:18 +0200 Subject: [PATCH] Adding base functions for end to end testing using qemu tooling --- tests/qemu/machines.py | 51 ++++++++ tests/qemu/run_test.py | 259 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 310 insertions(+) create mode 100644 tests/qemu/machines.py create mode 100644 tests/qemu/run_test.py diff --git a/tests/qemu/machines.py b/tests/qemu/machines.py new file mode 100644 index 00000000..add91c08 --- /dev/null +++ b/tests/qemu/machines.py @@ -0,0 +1,51 @@ +import pathlib + +parameters = { + "pci_emulation" : [ + '/usr/bin/qemu-system-x86_64', + '-name', '"archinstall-test"', + '-display', 'none', + '-monitor', 'none', + #'-serial', 'none', + '-nographic', + '-usb', '-device', 'usb-host,vendorid=0x1050,productid=0x0407', + '-pidfile', f'{pathlib.Path(__file__).parent}/archinstall-test.pid', + '-cpu', 'host,topoext,kvm=off,hv_relaxed,hv_spinlocks=0x1fff,hv_vapic,hv_time', + '-enable-kvm', + '-object', 'rng-random,filename=/dev/urandom,id=rng0', + '-device', 'virtio-rng-pci,rng=rng0', + '-usbdevice', 'mouse', + '-global', 'driver=cfi.pflash01,property=secure,value=on', + '-machine', 'type=q35,accel=kvm,kernel_irqchip=on', + '-smbios', '"type=0,vendor=American Megatrends Inc.,version=P4.60,date=08/03/2021,release=08.03.2021"', + '-smbios', '"type=1,manufacturer=Inet_AB,product=To Be Filled By O.E.M.,version=To Be Filled By O.E.M.,serial=245797,uuid=4154a2b8-7b7f-0000-0000-000000000000,sku=To Be Filled By O.E.M.,family=To Be Filled By O.E.M."', + '-smbios', '"type=2,manufacturer=ASRock,product=X570 Taichi,version=,serial=M80-D00000012340asset=,location="', + '-smbios', '"type=3,manufacturer=To Be Filled By O.E.M.,version=To Be Filled By O.E.M.,serial=214345,asset=To Be Filled By O.E.M.,sku=To Be Filled By O.E.M."', + '-smbios', '"type=4,sock_pfx=AM4,manufacturer=Advanced Micro Devices,, Inc.,version=AMD Ryzen 9 5900X 12-Core Processor,serial=Unknown,asset=Unknown,part=Unknown"', + '-smbios', '"type=17,loc_pfx=DIMM 1,bank=P0 CHANNEL A,manufacturer=Unknown,serial=00000000,asset=Not Specified,part=CMK64GX4M2E3200C16,speed=2133"', + '-smbios', '"type=17,loc_pfx=DIMM 1,bank=P0 CHANNEL B,manufacturer=Unknown,serial=00000000,asset=Not Specified,part=CMK64GX4M2E3200C16,speed=2133"', + '-m', '4096', + '-smp', '4,sockets=1,dies=1,cores=4,threads=1', + '-device', 'intel-iommu,device-iotlb=on,caching-mode=on', + '-device', 'pcie-root-port,port=0xe,chassis=8,id=pci.8,bus=pcie.0,multifunction=on,addr=0x6', + '-device', 'virtio-keyboard-pci,id=input1,bus=pci.8,addr=0x0', + '-device', 'pcie-root-port,port=0xf,chassis=9,id=pci.9,bus=pcie.0,addr=0x6.0x1', + '-device', 'pcie-root-port,port=0x11,chassis=13,id=pci.13,bus=pcie.0,addr=0x6.0x3', + '-drive', 'if=pflash,format=raw,readonly=on,file=/usr/share/ovmf/x64/OVMF_CODE.secboot.fd', + '-drive', 'if=pflash,format=raw,file=/home/anton/archtest.fd', + #'-tpmdev', 'passthrough,id=tpm0,path=/dev/tpm0,cancel-path=/tmp/foo-cancel2', + #'-device', 'tpm-tis,tpmdev=tpm0', + '-object', 'iothread,id=iothread1', + '-device', 'virtio-scsi-pci,bus=pcie.0,id=scsi2,addr=0x8', + '-device', 'virtio-scsi-pci,iothread=iothread1,id=scsi0,num_queues=8,bus=pci.13,addr=0x0', + '-device', '"scsi-hd,serial=S5GBAD12345ABCE,drive=libvirt-1-format,bus=scsi2.0,id=scsi0-0-0-0,channel=0,scsi-id=0,lun=0,device_id=drive-scsi0-0-0-0,bootindex=2,write-cache=on"', + '-blockdev', '\'{"driver":"file","filename":"/home/anton/archtest.img","aio":"threads","node-name":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"}\'', + '-blockdev', '\'{"node-name":"libvirt-1-format","read-only":false,"discard":"unmap","cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":"libvirt-1-storage","backing":null}\'', + '-device', 'pcie-root-port,multifunction=on,bus=pcie.0,id=port9-0,addr=0x9,chassis=0', + '-device', 'virtio-net-pci,mac=FE:00:00:00:00:01,id=network0,netdev=network0.0,status=on,bus=port9-0', + '-netdev', 'tap,ifname=tap0,id=network0.0,script=/home/anton/Archtest_up.sh,downscript=no', + '-audiodev', 'pipewire,id=win11', + '-device', 'ich9-intel-hda,id=sound0,bus=pcie.0,addr=0x1b', + '-device', 'hda-micro,audiodev=win11', + ] +} \ No newline at end of file diff --git a/tests/qemu/run_test.py b/tests/qemu/run_test.py new file mode 100644 index 00000000..43051240 --- /dev/null +++ b/tests/qemu/run_test.py @@ -0,0 +1,259 @@ +import time +import asyncio +import threading +import pathlib +import socket +import select +import os +import json +import logging +import sys +from subprocess import Popen, PIPE, STDOUT +from qemu.qmp import QMPClient, Message +from machines import parameters + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +handler = logging.StreamHandler(sys.stdout) +handler.setLevel(logging.DEBUG) +logger.addHandler(handler) + +class QMPClientMonitor: + def __init__(self, name: str, qmp_socket): + self.qmp = QMPClient(name) + self.qmp.logger = logger + self.qmp_socket = qmp_socket + self.loop = None + + async def watch_events(self): + try: + async for event in self.qmp.events: + print(f"Event: {event['event']}") + except asyncio.CancelledError: + return + + async def run(self): + self.loop = asyncio.get_event_loop() + await self.qmp.connect(self.qmp_socket) + + asyncio.create_task(self.watch_events()) + + await self.qmp.runstate_changed() + try: + await self.qmp.disconnect() + except: + pass + + +class SerialMonitor(threading.Thread): + def __init__(self, profile, QMP, serial_socket_path): + self.profile = profile + self.QMP = QMP + self.serial_socket_path = serial_socket_path + + threading.Thread.__init__(self) + self.start() + + async def edit_boot(self): + logger.info("Sending 'e'") + + # https://github.com/coreos/qemu/blob/master/qmp-commands.hx + await self.QMP.qmp.execute_msg( + self.QMP.qmp.make_execute_msg( + 'send-key', + arguments={ + 'keys': [ + { "type": "qcode", "data": "e" } + ] + } + ) + ) + + logger.info("Sending 'end'") + asyncio.sleep(1) + ret = await self.QMP.qmp.execute_msg( + self.QMP.qmp.make_execute_msg( + 'send-key', + arguments={ + 'keys': [ + { "type": "qcode", "data": "end" } + ] + } + ) + ) + asyncio.sleep(1) + + keys = [] + # https://gist.github.com/mvidner/8939289 + keys.append({ "type": "qcode", "data": "spc" }) + for character in list('console=tty0 console=ttyS0,115200'): + if character.isupper(): + keys.append({ "type": "qcode", "data": 'caps_lock' }) + keys.append({ "type": "qcode", "data": character.lower().replace('=', 'equal').replace(',', 'comma').replace(' ', 'spc') }) + if character.isupper(): + keys.append({ "type": "qcode", "data": 'caps_lock' }) + # keys.append({ "type": "qcode", "data": "kp_enter" }) + + ret = await self.QMP.qmp.execute_msg( + self.QMP.qmp.make_execute_msg( + 'send-key', + arguments={ + 'keys': keys + } + ) + ) + + await asyncio.sleep(1) + ret = await self.QMP.qmp.execute_msg( + self.QMP.qmp.make_execute_msg( + 'send-key', + arguments={ + 'keys': [ + { "type": "qcode", "data": "kp_enter" } + ] + } + ) + ) + + async def login_root(self): + keys = [] + for character in list('root'): + if character.isupper(): + keys.append({ "type": "qcode", "data": 'caps_lock' }) + keys.append({ "type": "qcode", "data": character.lower().replace('=', 'equal').replace(',', 'comma').replace(' ', 'spc') }) + if character.isupper(): + keys.append({ "type": "qcode", "data": 'caps_lock' }) + + ret = await self.QMP.qmp.execute_msg( + self.QMP.qmp.make_execute_msg( + 'send-key', + arguments={ + 'keys': keys + } + ) + ) + + await asyncio.sleep(1) + ret = await self.QMP.qmp.execute_msg( + self.QMP.qmp.make_execute_msg( + 'send-key', + arguments={ + 'keys': [ + { "type": "qcode", "data": "kp_enter" } + ] + } + ) + ) + + await asyncio.sleep(1) + ret = await self.QMP.qmp.execute_msg( + self.QMP.qmp.make_execute_msg( + 'send-key', + arguments={ + 'keys': [ + { "type": "qcode", "data": "kp_enter" } + ] + } + ) + ) + + def run(self): + client_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client_socket.connect(str(self.serial_socket_path)) + + alive = True + with open('serial.log', 'wb') as fh: + while alive: + r, w, x = select.select([client_socket], [], [], 0.2) + for fd in r: + if (output := client_socket.recv(8192)): + if b'Boot in' in output: + logger.info("Found booting") + asyncio.run_coroutine_threadsafe(self.edit_boot(), loop=self.QMP.loop) + elif b'archiso login:' in output: + logger.info("Found login prompt") + asyncio.run_coroutine_threadsafe(self.login_root(), loop=self.QMP.loop) + + fh.write(output) + fh.flush() + + else: + client_socket.close() + alive = False + break + time.sleep(0.025) + +class QemuSession(threading.Thread): + def __init__(self, cmd, qmp_socket, serial_socket): + self.cmd = cmd + self.qmp_socket = qmp_socket + self.serial_socket = serial_socket + + threading.Thread.__init__(self) + self.start() + + def run(self): + #print(self.cmd) + self.handle = Popen( + ' '.join(self.cmd), + stdout=PIPE, + stderr=STDOUT, + stdin=PIPE, + shell=True, + cwd=str(pathlib.Path(__file__).parent), + pass_fds=[self.qmp_socket.fileno(), self.serial_socket.fileno()] + ) + + while self.handle.poll() is None: + r, w, x = select.select([self.handle.stdout.fileno(), self.handle.stdout.fileno()], [], [], 0.2) + for fd in r: + if fd == self.handle.stdout.fileno(): + if (output := self.handle.stdout.read()): + print(output) + #elif fd == self.handle.stderr.fileno(): + # if (output := self.handle.stderr.read()): + # print(output) + # No exit signal yet + time.sleep(0.25) + + r, w, x = select.select([self.handle.stdout.fileno(), self.handle.stdout.fileno()], [], [], 0.2) + for fd in r: + if fd == self.handle.stdout.fileno(): + if (output := self.handle.stdout.read()): + print(output) + #elif fd == self.handle.stderr.fileno(): + # if (output := self.handle.stderr.read()): + # print(output) + + self.handle.stdin.close() + self.handle.stdout.close() + # self.handle.stderr.close() + logger.warning("Qemu closed..") + +for profile in parameters: + qmp_socket_path = pathlib.Path(__file__).parent / "qmp.socket" + serial_socket_path = pathlib.Path(__file__).parent / "serial.socket" + + qmp_socket_path.unlink(missing_ok=True) + serial_socket_path.unlink(missing_ok=True) + + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as qmp_socket: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as serial_socket: + qmp_socket.bind(str(qmp_socket_path)) + serial_socket.bind(str(serial_socket_path)) + qmp_socket.listen(2) + serial_socket.listen(2) + + args = parameters[profile] + [ + '-chardev', f'socket,id=qmp1,fd={qmp_socket.fileno()},server=on,wait=off', + '-chardev', f'socket,id=serial1,fd={serial_socket.fileno()},server=on,wait=on', + '-mon', f'chardev=qmp1,mode=control,pretty=off', + '-serial', f"chardev:serial1", + '-drive', f'file=/home/anton/Downloads/archlinux-2024.05.11-x86_64.iso,media=cdrom,cache=none,id=cdrom0,index=0' + ] + + session = QemuSession(args, qmp_socket, serial_socket) + monitor = QMPClientMonitor(profile, str(qmp_socket_path)) + serial = SerialMonitor(profile, monitor, serial_socket_path) + + asyncio.run(monitor.run()) \ No newline at end of file