From 8ab2590e5b106d907fc4b98085179f84dd88b9eb Mon Sep 17 00:00:00 2001 From: Liam <33645555+lj3954@users.noreply.github.com> Date: Sun, 24 Dec 2023 01:01:53 -0600 Subject: [PATCH] inbuilt macOS downloading * Fix quickget show-iso-url and test-iso-url creating unnecessary directory * Beautify output, add show-iso-url and test-iso-url for Windows (fully) and macOS (sorta) * (NON-FUNCTIONAL) macrecovery shell script. * Semi-functional (although incomplete) macrecovery shell script Rough draft. To be completed, cleaned up and simplified (very much so) hoping to merge into quickemu & replace the python macrecovery dependency. * macrecovery shell script now successfully downloads the image. TODO: Verification * Merged macrecovery functions into quickget. Chunkcheck (C) to replace macrecovery's image verification Chunkcheck written by MCJack123: https://gist.github.com/MCJack123/943eaca762730ca4b7ae460b731b68e7 * Replace C chunkcheck binary with the Python equivalent. Re-add python to dependencies. * force macOS guests to usually boot with core counts which are powers of 2; fix #865 * Add support for macOS Sonoma * Fix issue where script would be unable to find chunkcheck if installed system-wide * Update README verbiage * Add headers to web_get function; macOS can now be downloaded via aria2; clean up code & output * Add support for macOS Sonoma * Fix use of wrong operator (>) which touches a file * Small correction to README * macOS switched from wget to default downloader (aria2/wget) * Replace wget with cURL for downloading macOS chunklist file * Fix variable naming in generate_id function --- README.md | 10 +- chunkcheck | 64 +++++++ macrecovery | 518 ---------------------------------------------------- quickemu | 11 +- quickget | 126 +++++++++---- 5 files changed, 174 insertions(+), 555 deletions(-) create mode 100755 chunkcheck delete mode 100755 macrecovery diff --git a/README.md b/README.md index 7c5bd7b..bd999b8 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ QEMU](https://img.youtube.com/vi/AOTYWEgw0hI/0.jpg)](https://www.youtube.com/wat - [LSB](https://wiki.linuxfoundation.org/lsb/start) - [procps](https://gitlab.com/procps-ng/procps) - [python3](https://www.python.org/) -- [macrecovery](https://github.com/acidanthera/OpenCorePkg/tree/master/Utilities/macrecovery) +- [chunkcheck](https://gist.github.com/MCJack123/943eaca762730ca4b7ae460b731b68e7) - [mkisofs](http://cdrtools.sourceforge.net/private/cdrecord.html) - [usbutils](https://github.com/gregkh/usbutils) - [util-linux](https://github.com/karelzak/util-linux) @@ -373,8 +373,8 @@ quickget macos catalina quickemu --vm macos-catalina.conf ``` -macOS `high-sierra`, `mojave`, `catalina`, `big-sur`, `monterey` and -`ventura` are supported. +macOS `high-sierra`, `mojave`, `catalina`, `big-sur`, `monterey`, `ventura` and +`sonoma` are supported. - Use cursor keys and enter key to select the **macOS Base System** - From **macOS Utilities** @@ -384,7 +384,7 @@ macOS `high-sierra`, `mojave`, `catalina`, `big-sur`, `monterey` and click **Erase**. - Enter a `Name:` for the disk - If you are installing macOS Mojave or later (Catalina, Big - Sur, Monterey and Ventura), choose any of the APFS options + Sur, Monterey, Ventura and Sonoma), choose any of the APFS options as the filesystem. MacOS Extended may not work. - Click **Erase**. - Click **Done**. @@ -462,6 +462,7 @@ There are some considerations when running macOS via Quickemu. - Big Sur - Monterey - Ventura + - Sonoma (Not recommended) - `quickemu` will automatically download the required [OpenCore](https://github.com/acidanthera/OpenCorePkg) bootloader and OVMF firmware from [OSX-KVM](https://github.com/kholia/OSX-KVM). @@ -910,6 +911,7 @@ Useful reference that assisted the development of Quickemu. - - - + - - - - diff --git a/chunkcheck b/chunkcheck new file mode 100755 index 0000000..af8c9ab --- /dev/null +++ b/chunkcheck @@ -0,0 +1,64 @@ +from pathlib import Path +import struct +import hashlib +import argparse +v1_prod_pubkey = 0xC3E748CAD9CD384329E10E25A91E43E1A762FF529ADE578C935BDDF9B13F2179D4855E6FC89E9E29CA12517D17DFA1EDCE0BEBF0EA7B461FFE61D94E2BDF72C196F89ACD3536B644064014DAE25A15DB6BB0852ECBD120916318D1CCDEA3C84C92ED743FC176D0BACA920D3FCF3158AFF731F88CE0623182A8ED67E650515F75745909F07D415F55FC15A35654D118C55A462D37A3ACDA08612F3F3F6571761EFCCBCC299AEE99B3A4FD6212CCFFF5EF37A2C334E871191F7E1C31960E010A54E86FA3F62E6D6905E1CD57732410A3EB0C6B4DEFDABE9F59BF1618758C751CD56CEF851D1C0EAA1C558E37AC108DA9089863D20E2E7E4BF475EC66FE6B3EFDCF +# v2_prod_pubkey = 0xCB45C5E53217D4499FB80B2D96AA4F964EB551F1DA4EBFA4F5E23F87BFE82FC113590E536757F329D6EAD1F267771EE342F5A5E61514DD3D3383187E663929D577D94648F262EBA1157E152DB5273D10AE3A6A058CB9CD64D01267DAC82ED3B7BC1631D078C911414129CDAAA0FFB0A8E2A7ADD6F32FB09A7E98D259BFF6ED10808D1BDA58CAF7355DFF1A085A18B11657D2617447BF657140D599364E5AC8E626276AC03BC2417831D9E61B25154AFE9F2D8271E9CE22D2783803083A5A7A575774688721097DC5E4B32D118CF6317A7083BA15BA608430A8C8C6B7DA2D932D81F571603A9363AC0197AB670242D9C9180D97A10900F11FE3D9246CF14F0883 +# v2_dev_pubkey = 0xB372CEC9E05E71FB3FAA08C34E3256FB312EA821638A243EF8A5DEA46FCDA33F00F88FC2933FB276D37B914F89BAD5B5D75771E342265B771995AE8F43B4DFF3F21A877FE777A8B419587C8718D36204FA1922A575AD5207D5D6B8C10F84DDCA661B731E7E7601D64D4A894F487FE1AA1DDC2A1697A3553B1DD85D5750DF2AA9D988E83C4C70BBBE4747219F9B92B199FECB16091896EBB441606DEC20F446249D5568BB51FC87BA7F85E6295FBE811B0A314408CD31921C360608A0FF7F87BD733560FE1C96E472834CAB6BE016C35727754273125089BE043FD3B26F0B2DE141E05990CE922F1702DA0A2F4E9F8760D0FA712DDB9928E0CDAC14501ED5E2C3 + +ChunkListHeader = struct.Struct('<4sIBBBxQQQ') +assert ChunkListHeader.size == 0x24 + +Chunk = struct.Struct(' 0 + assert chunk_offset == 0x24 + assert signature_offset == chunk_offset + Chunk.size * chunk_count + for i in range(chunk_count): + data = f.read(Chunk.size) + hash_ctx.update(data) + chunk_size, chunk_sha256 = Chunk.unpack(data) + yield chunk_size, chunk_sha256 + digest = hash_ctx.digest() + if signature_method == 1: + data = f.read(256) + assert len(data) == 256 + signature = int.from_bytes(data, 'little') + plaintext = 0x1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff003031300d0609608648016503040201050004200000000000000000000000000000000000000000000000000000000000000000 | int.from_bytes(digest, 'big') + assert pow(signature, 0x10001, v1_prod_pubkey) == plaintext + elif signature_method == 2: + data = f.read(32) + assert data == digest + else: + raise NotImplementedError + assert f.read(1) == b'' + +def check_chunklist(path, chunklist_path): + with open(path, 'rb') as f: + for chunk_size, chunk_sha256 in parse_chunklist(chunklist_path): + chunk = f.read(chunk_size) + assert len(chunk) == chunk_size + assert hashlib.sha256(chunk).digest() == chunk_sha256 + assert f.read(1) == b'' + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('vmdir', type=Path) + args = parser.parse_args() + vmdir = args.vmdir + check_chunklist(vmdir / 'RecoveryImage.dmg', vmdir / 'RecoveryImage.chunklist') + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/macrecovery b/macrecovery deleted file mode 100755 index e6799b2..0000000 --- a/macrecovery +++ /dev/null @@ -1,518 +0,0 @@ -#!/usr/bin/env python3 - -""" -Gather recovery information for Macs. - -Copyright (c) 2019, vit9696 -""" - -from __future__ import print_function - -import argparse -import binascii -import datetime -import hashlib -import json -import linecache -import os -import random -import struct -import sys -import textwrap -import time - -try: - from urllib.request import Request,HTTPError,urlopen - from urllib.parse import urlencode,urlparse -except ImportError: - from urllib2 import Request,HTTPError,urlopen - from urllib import urlencode - from urlparse import urlparse - -SELF_DIR = os.path.dirname(os.path.realpath(__file__)) - -RECENT_MAC = 'Mac-7BA5B2D9E42DDD94' -MLB_ZERO = '00000000000000000' -MLB_VALID = 'C02749200YGJ803AX' -MLB_PRODUCT = '00000000000J80300' - -TYPE_SID = 16 -TYPE_K = 64 -TYPE_FG = 64 - -INFO_PRODUCT = 'AP' -INFO_IMAGE_LINK = 'AU' -INFO_IMAGE_HASH = 'AH' -INFO_IMAGE_SESS = 'AT' -INFO_SIGN_LINK = 'CU' -INFO_SIGN_HASH = 'CH' -INFO_SIGN_SESS = 'CT' -INFO_REQURED = [ INFO_PRODUCT, INFO_IMAGE_LINK, INFO_IMAGE_HASH, INFO_IMAGE_SESS, - INFO_SIGN_LINK, INFO_SIGN_HASH, INFO_SIGN_SESS ] - -def run_query(url, headers, post=None, raw=False): - if post is not None: - data = '\n'.join([entry + '=' + post[entry] for entry in post]) - if sys.version_info[0] >= 3: - data = data.encode('utf-8') - else: - data = None - - req = Request(url=url, headers=headers, data=data) - try: - response = urlopen(req) - if raw: return response - return dict(response.info()), response.read() - except HTTPError as e: - print('ERROR: "{}" when connecting to {}'.format(e, url)) - sys.exit(1) - -def generate_id(type, id=None): - valid_chars = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'] - if id is None: - return ''.join(random.choice(valid_chars) for i in range(type)) - return id - -def product_mlb(mlb): - return '00000000000' + mlb[11] + mlb[12] + mlb[13] + mlb[14] + '00' - -def mlb_from_eeee(eeee): - if len(eeee) != 4: - print('ERROR: Invalid EEEE code length!') - sys.exit(1) - - return '00000000000' + eeee + '00' - -def int_from_unsigned_bytes(bytes, byteorder): - if byteorder == 'little': bytes = bytes[::-1] - encoded = binascii.hexlify(bytes) - return int(encoded, 16) - -# zhangyoufu https://gist.github.com/MCJack123/943eaca762730ca4b7ae460b731b68e7#gistcomment-3061078 2021-10-08 -Apple_EFI_ROM_public_key_1 = 0xC3E748CAD9CD384329E10E25A91E43E1A762FF529ADE578C935BDDF9B13F2179D4855E6FC89E9E29CA12517D17DFA1EDCE0BEBF0EA7B461FFE61D94E2BDF72C196F89ACD3536B644064014DAE25A15DB6BB0852ECBD120916318D1CCDEA3C84C92ED743FC176D0BACA920D3FCF3158AFF731F88CE0623182A8ED67E650515F75745909F07D415F55FC15A35654D118C55A462D37A3ACDA08612F3F3F6571761EFCCBCC299AEE99B3A4FD6212CCFFF5EF37A2C334E871191F7E1C31960E010A54E86FA3F62E6D6905E1CD57732410A3EB0C6B4DEFDABE9F59BF1618758C751CD56CEF851D1C0EAA1C558E37AC108DA9089863D20E2E7E4BF475EC66FE6B3EFDCF - -ChunkListHeader = struct.Struct('<4sIBBBxQQQ') -assert ChunkListHeader.size == 0x24 - -Chunk = struct.Struct(' 0 - assert chunk_offset == 0x24 - assert signature_offset == chunk_offset + Chunk.size * chunk_count - for i in range(chunk_count): - data = f.read(Chunk.size) - hash_ctx.update(data) - chunk_size, chunk_sha256 = Chunk.unpack(data) - yield chunk_size, chunk_sha256 - digest = hash_ctx.digest() - if signature_method == 1: - data = f.read(256) - assert len(data) == 256 - signature = int_from_unsigned_bytes(data, 'little') - plaintext = 0x1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff003031300d0609608648016503040201050004200000000000000000000000000000000000000000000000000000000000000000 | int_from_unsigned_bytes(digest, 'big') - assert pow(signature, 0x10001, Apple_EFI_ROM_public_key_1) == plaintext - elif signature_method == 2: - data = f.read(32) - assert data == digest - raise RuntimeError('Chunklist missing digital signature') - else: - raise NotImplementedError - assert f.read(1) == b'' - -def get_session(args): - headers = { - 'Host' : 'osrecovery.apple.com', - 'Connection': 'close', - 'User-Agent': 'InternetRecovery/1.0', - } - - headers, output = run_query('http://osrecovery.apple.com/', headers) - - if args.verbose: - print('Session headers:') - for header in headers: - print('{}: {}'.format(header, headers[header])) - - for header in headers: - if header.lower() == 'set-cookie': - cookies = headers[header].split('; ') - for cookie in cookies: - if cookie.startswith('session='): - return cookie - - raise RuntimeError('No session in headers ' + str(headers)) - -def get_image_info(session, bid, mlb=MLB_ZERO, diag = False, os_type = 'default', cid=None): - headers = { - 'Host' : 'osrecovery.apple.com', - 'Connection' : 'close', - 'User-Agent' : 'InternetRecovery/1.0', - 'Cookie' : session, - 'Content-Type': 'text/plain', - } - - post = { - 'cid': generate_id(TYPE_SID, cid), - 'sn' : mlb, - 'bid': bid, - 'k' : generate_id(TYPE_K), - 'fg' : generate_id(TYPE_FG) - } - - if diag: - url = 'http://osrecovery.apple.com/InstallationPayload/Diagnostics' - else: - url = 'http://osrecovery.apple.com/InstallationPayload/RecoveryImage' - post['os'] = os_type - - headers, output = run_query(url, headers, post) - - if sys.version_info[0] >= 3: - output = output.decode('utf-8') - - info = {} - for line in output.split('\n'): - try: - key, value = line.split(': ') - info[key] = value - except: - continue - - for k in INFO_REQURED: - if k not in info: - raise RuntimeError('Missing key ' + k) - - return info - -def save_image(url, sess, filename='', dir=''): - purl = urlparse(url) - headers = { - 'Host' : purl.hostname, - 'Connection': 'close', - 'User-Agent': 'InternetRecovery/1.0', - 'Cookie' : '='.join(['AssetToken', sess]) - } - - if filename == '': - filename = os.path.basename(purl.path) - if filename.find('/') >= 0 or filename == '': - raise RuntimeError('Invalid save path ' + filename) - - print('Saving ' + url + ' to ' + filename + '...') - - with open (os.path.join(dir, filename), 'wb') as fh: - response = run_query(url, headers, raw=True) - size = 0 - while True: - chunk = response.read(2**20) - if not chunk: - break - fh.write(chunk) - size += len(chunk) - print('\r{} MBs downloaded...'.format(size / (2**20)), end='') - sys.stdout.flush() - print('\rDownload complete!') - - return os.path.join(dir, os.path.basename(filename)) - -def verify_image(dmgpath, cnkpath): - print('Verifying image with chunklist...') - - with open (dmgpath, 'rb') as dmgf: - cnkcount = 0 - for cnksize, cnkhash in verify_chunklist(cnkpath): - cnkcount += 1 - print('\rChunk {} ({} bytes)'.format(cnkcount, cnksize), end='') - sys.stdout.flush() - cnk = dmgf.read(cnksize) - if len(cnk) != cnksize: - raise RuntimeError('Invalid chunk {} size: expected {}, read {}'.format(cnkcount, cnksize, len(cnk))) - if hashlib.sha256(cnk).digest() != cnkhash: - raise RuntimeError('Invalid chunk {}: hash mismatch'.format(cnkcount)) - if dmgf.read(1) != b'': - raise RuntimeError('Invalid image: larger than chunklist') - print('\rImage verification complete!') - -def action_download(args): - """ - Reference information for queries: - - Recovery latest: - cid=3076CE439155BA14 - sn=... - bid=Mac-E43C1C25D4880AD6 - k=4BE523BB136EB12B1758C70DB43BDD485EBCB6A457854245F9E9FF0587FB790C - os=latest - fg=B2E6AA07DB9088BE5BDB38DB2EA824FDDFB6C3AC5272203B32D89F9D8E3528DC - - Recovery default: - cid=4A35CB95FF396EE7 - sn=... - bid=Mac-E43C1C25D4880AD6 - k=0A385E6FFC3DDD990A8A1F4EC8B98C92CA5E19C9FF1DD26508C54936D8523121 - os=default - fg=B2E6AA07DB9088BE5BDB38DB2EA824FDDFB6C3AC5272203B32D89F9D8E3528DC - - Diagnostics: - cid=050C59B51497CEC8 - sn=... - bid=Mac-E43C1C25D4880AD6 - k=37D42A8282FE04A12A7D946304F403E56A2155B9622B385F3EB959A2FBAB8C93 - fg=B2E6AA07DB9088BE5BDB38DB2EA824FDDFB6C3AC5272203B32D89F9D8E3528DC - """ - - session = get_session(args) - info = get_image_info(session, bid=args.board_id, mlb=args.mlb, - diag=args.diagnostics, os_type=args.os_type) - if args.verbose: - print(info) - print('Downloading ' + info[INFO_PRODUCT] + '...') - dmgname = '' if args.basename == '' else args.basename + '.dmg' - dmgpath = save_image(info[INFO_IMAGE_LINK], info[INFO_IMAGE_SESS], dmgname, args.outdir) - cnkname = '' if args.basename == '' else args.basename + '.chunklist' - cnkpath = save_image(info[INFO_SIGN_LINK], info[INFO_SIGN_SESS], cnkname, args.outdir) - try: - verify_image(dmgpath, cnkpath) - return 0 - except Exception as err: - if isinstance(err, AssertionError) and str(err)=='': - try: - tb = sys.exc_info()[2] - while tb.tb_next: - tb = tb.tb_next - err = linecache.getline(tb.tb_frame.f_code.co_filename, tb.tb_lineno, tb.tb_frame.f_globals).strip() - except: - err = "Invalid chunklist" - print('\rImage verification failed. ({})'.format(err)) - return 1 - -def action_selfcheck(args): - """ - Sanity check server logic for recovery: - - if not valid(bid): - return error() - ppp = get_ppp(sn) - if not valid(ppp): - return latest_recovery(bid = bid) # Returns newest for bid. - if valid(sn): - if os == 'default': - return default_recovery(sn = sn, ppp = ppp) # Returns oldest for sn. - else: - return latest_recovery(sn = sn, ppp = ppp) # Returns newest for sn. - return default_recovery(ppp = ppp) # Returns oldest. - """ - - session = get_session(args) - valid_default = get_image_info(session, bid=RECENT_MAC, mlb=MLB_VALID, - diag=False, os_type='default') - valid_latest = get_image_info(session, bid=RECENT_MAC, mlb=MLB_VALID, - diag=False, os_type='latest') - product_default = get_image_info(session, bid=RECENT_MAC, mlb=MLB_PRODUCT, - diag=False, os_type='default') - product_latest = get_image_info(session, bid=RECENT_MAC, mlb=MLB_PRODUCT, - diag=False, os_type='latest') - generic_default = get_image_info(session, bid=RECENT_MAC, mlb=MLB_ZERO, - diag=False, os_type='default') - generic_latest = get_image_info(session, bid=RECENT_MAC, mlb=MLB_ZERO, - diag=False, os_type='latest') - - if args.verbose: - print(valid_default) - print(valid_latest) - print(product_default) - print(product_latest) - print(generic_default) - print(generic_latest) - - if valid_default[INFO_PRODUCT] == valid_latest[INFO_PRODUCT]: - # Valid MLB must give different default and latest if this is not a too new product. - print('ERROR: Cannot determine any previous product, got {}'.format(valid_default[INFO_PRODUCT])) - return 1 - - if product_default[INFO_PRODUCT] != product_latest[INFO_PRODUCT]: - # Product-only MLB must give the same value for default and latest. - print('ERROR: Latest and default do not match for product MLB, got {} and {}'.format( - product_default[INFO_PRODUCT], product_latest[INFO_PRODUCT])) - return 1 - - if generic_default[INFO_PRODUCT] != generic_latest[INFO_PRODUCT]: - # Zero MLB always give the same value for default and latest. - print('ERROR: Generic MLB gives different product, got {} and {}'.format( - generic_default[INFO_PRODUCT], generic_latest[INFO_PRODUCT])) - return 1 - - if valid_latest[INFO_PRODUCT] != generic_latest[INFO_PRODUCT]: - # Valid MLB must always equal generic MLB. - print('ERROR: Cannot determine unified latest product, got {} and {}'.format( - valid_latest[INFO_PRODUCT], generic_latest[INFO_PRODUCT])) - return 1 - - if product_default[INFO_PRODUCT] != valid_default[INFO_PRODUCT]: - # Product-only MLB can give the same value with valid default MLB. - # This is not an error for all models, but for our chosen code it is. - print('ERROR: Valid and product MLB give mismatch, got {} and {}'.format( - product_default[INFO_PRODUCT], valid_default[INFO_PRODUCT])) - return 1 - - print('SUCCESS: Found no discrepancies with MLB validation algorithm!') - return 0 - -def action_verify(args): - """ - Try to verify MLB serial number. - """ - session = get_session(args) - generic_latest = get_image_info(session, bid=RECENT_MAC, mlb=MLB_ZERO, - diag=False, os_type='latest') - uvalid_default = get_image_info(session, bid=args.board_id, mlb=args.mlb, - diag=False, os_type='default') - uvalid_latest = get_image_info(session, bid=args.board_id, mlb=args.mlb, - diag=False, os_type='latest') - uproduct_default = get_image_info(session, bid=args.board_id, mlb=product_mlb(args.mlb), - diag=False, os_type='default') - - if args.verbose: - print(generic_latest) - print(uvalid_default) - print(uvalid_latest) - print(uproduct_default) - - # Verify our MLB number. - if uvalid_default[INFO_PRODUCT] != uvalid_latest[INFO_PRODUCT]: - if uvalid_latest[INFO_PRODUCT] == generic_latest[INFO_PRODUCT]: - print('SUCCESS: {} MLB looks valid and supported!'.format(args.mlb)) - else: - print('SUCCESS: {} MLB looks valid, but probably unsupported!'.format(args.mlb)) - return 0 - - print('UNKNOWN: Run selfcheck, check your board-id, or try again later!') - - # Here we have matching default and latest products. This can only be true for very - # new models. These models get either latest or special builds. - if uvalid_default[INFO_PRODUCT] == generic_latest[INFO_PRODUCT]: - print('UNKNOWN: {} MLB can be valid if very new!'.format(args.mlb)) - return 0 - if uproduct_default[INFO_PRODUCT] != uvalid_default[INFO_PRODUCT]: - print('UNKNOWN: {} MLB looks invalid, other models use product {} instead of {}!'.format( - args.mlb, uproduct_default[INFO_PRODUCT], uvalid_default[INFO_PRODUCT])) - return 0 - print('UNKNOWN: {} MLB can be valid if very new and using special builds!'.format(args.mlb)) - return 0 - -def action_guess(args): - """ - Attempt to guess which model does this MLB belong. - """ - - mlb = args.mlb - anon = mlb.startswith('000') - - with open(args.board_db, 'r') as fh: - db = json.load(fh) - - supported = {} - - session = get_session(args) - - generic_latest = get_image_info(session, bid=RECENT_MAC, mlb=MLB_ZERO, - diag=False, os_type='latest') - - for model in db: - try: - if anon: - # For anonymous lookup check when given model does not match latest. - model_latest = get_image_info(session, bid=model, mlb=MLB_ZERO, - diag=False, os_type='latest') - - if model_latest[INFO_PRODUCT] != generic_latest[INFO_PRODUCT]: - if db[model] == 'current': - print('WARN: Skipped {} due to using latest product {} instead of {}'.format( - model, model_latest[INFO_PRODUCT], generic_latest[INFO_PRODUCT])) - continue - - user_default = get_image_info(session, bid=model, mlb=mlb, - diag=False, os_type='default') - - if user_default[INFO_PRODUCT] != generic_latest[INFO_PRODUCT]: - supported[model] = [db[model], user_default[INFO_PRODUCT], generic_latest[INFO_PRODUCT]] - else: - # For normal lookup check when given model has mismatching normal and latest. - user_latest = get_image_info(session, bid=model, mlb=mlb, - diag=False, os_type='latest') - - user_default = get_image_info(session, bid=model, mlb=mlb, - diag=False, os_type='default') - - if user_latest[INFO_PRODUCT] != user_default[INFO_PRODUCT]: - supported[model] = [db[model], user_default[INFO_PRODUCT], user_latest[INFO_PRODUCT]] - - except Exception as e: - print('WARN: Failed to check {}, exception: {}'.format(model, str(e))) - - if len(supported) > 0: - print('SUCCESS: MLB {} looks supported for:'.format(mlb)) - for model in supported: - print('- {}, up to {}, default: {}, latest: {}'.format(model, supported[model][0], - supported[model][1], supported[model][2])) - return 0 - - print('UNKNOWN: Failed to determine supported models for MLB {}!'.format(mlb)) - -def main(): - parser = argparse.ArgumentParser(description='Gather recovery information for Macs') - parser.add_argument('action', choices = ['download', 'selfcheck', 'verify', 'guess'], - help='Action to perform: "download" - performs recovery downloading,' - ' "selfcheck" checks whether MLB serial validation is possible, "verify" performs' - ' MLB serial verification, "guess" tries to find suitable mac model for MLB.') - parser.add_argument('-o', '--outdir', type=str, default=os.getcwd(), - help='customise output directory for downloading, defaults to current directory') - parser.add_argument('-n', '--basename', type=str, default='', - help='customise base name for downloading, defaults to remote name') - parser.add_argument('-b', '--board-id', type=str, default=RECENT_MAC, - help='use specified board identifier for downloading, defaults to ' + RECENT_MAC) - parser.add_argument('-m', '--mlb', type=str, default=MLB_ZERO, - help='use specified logic board serial for downloading, defaults to ' + MLB_ZERO) - parser.add_argument('-e', '--code', type=str, default='', - help='generate product logic board serial with specified product EEEE code') - parser.add_argument('-os', '--os-type', type=str, default='default', choices = ['default', 'latest'], - help='use specified os type, defaults to default ' + MLB_ZERO) - parser.add_argument('-diag', '--diagnostics', action='store_true', help='download diagnostics image') - parser.add_argument('-v', '--verbose', action='store_true', help='print debug information') - parser.add_argument('-db', '--board-db', type=str, default=os.path.join(SELF_DIR, 'boards.json'), - help='use custom board list for checking, defaults to boards.json') - - args = parser.parse_args() - - if args.code != '': - args.mlb = mlb_from_eeee(args.code) - - if len(args.mlb) != 17: - print('ERROR: Cannot use MLBs in non 17 character format!') - sys.exit(1) - - if args.action == 'download': - return action_download(args) - elif args.action == 'selfcheck': - return action_selfcheck(args) - elif args.action == 'verify': - return action_verify(args) - elif args.action == 'guess': - return action_guess(args) - else: - assert(False) - -if __name__ == '__main__': - sys.exit(main()) diff --git a/quickemu b/quickemu index c00388c..3842da4 100755 --- a/quickemu +++ b/quickemu @@ -288,6 +288,13 @@ function vm_boot() { GUEST_CPU_CORES="${cpu_cores}" fi + if [ "${guest_os}" == "macos" ] && [ "${GUEST_CPU_CORES}" -gt 10 ] || [ "${GUEST_CPU_CORES}" -eq 6 ] || [ "${GUEST_CPU_CORES}" -eq 7 ]; then + # macOS guests cannot boot with most core counts not powers of 2. This will fix the issue by rounding the core count down to a power of 2. Uses wc and factor from coreutils. + factorCPUCores=$(factor "${GUEST_CPU_CORES}") + GUEST_CPU_CORES=$(( 2 ** $(echo "${factorCPUCores#*:}" | grep -o '[0-9]' | wc -l) )) + fi + + # Account for Hyperthreading/SMT. if [ -e /sys/devices/system/cpu/smt/control ] && [ "${GUEST_CPU_CORES}" -ge 2 ]; then HOST_CPU_SMT=$(cat /sys/devices/system/cpu/smt/control) @@ -520,7 +527,7 @@ function vm_boot() { # A CPU with SSE4.1 support is required for >= macOS Sierra # A CPU with AVX2 support is required for >= macOS Ventura case ${macos_release} in - ventura) + ventura|sonoma) if check_cpu_flag sse4_1 && check_cpu_flag avx2; then CPU="-cpu Haswell,kvm=on,vendor=GenuineIntel,+sse3,+sse4.2,+aes,+xsave,+avx,+xsaveopt,+xsavec,+xgetbv1,+avx2,+bmi2,+smep,+bmi1,+fma,+movbe,+invtsc,+avx2" else @@ -557,7 +564,7 @@ function vm_boot() { NET_DEVICE="vmxnet3" USB_HOST_PASSTHROUGH_CONTROLLER="usb-ehci" ;; - big-sur|monterey|ventura) + big-sur|monterey|ventura|sonoma) BALLOON="-device virtio-balloon" MAC_DISK_DEV="virtio-blk-pci" NET_DEVICE="virtio-net" diff --git a/quickget b/quickget index f09168b..63fda38 100755 --- a/quickget +++ b/quickget @@ -178,9 +178,7 @@ function list_csv() { SVG="https://quickemu-project.github.io/quickemu-icons/svg/${FUNC}/${FUNC}-quickemu-white-pinkbg.svg" for RELEASE in $("releases_${FUNC}" | sed -Ee 's/eol-\S+//g' ); do # hide eol releases - if [ "${OS}" == "macos" ]; then - DOWNLOADER="macrecovery" - elif [[ "${OS}" == *"ubuntu"* ]] && [ "${RELEASE}" == "devel" ] && [ ${HAS_ZSYNC} -eq 1 ]; then + if [[ "${OS}" == *"ubuntu"* ]] && [ "${RELEASE}" == "devel" ] && [ ${HAS_ZSYNC} -eq 1 ]; then DOWNLOADER="zsync" else DOWNLOADER="${DL}" @@ -646,7 +644,7 @@ function editions_manjaro(){ } function releases_macos() { - echo high-sierra mojave catalina big-sur monterey ventura + echo high-sierra mojave catalina big-sur monterey ventura sonoma } function releases_manjaro() { @@ -930,6 +928,15 @@ function web_get() { else FILE="${URL##*/}" fi + + while (( "$#" )); do + if [[ $1 == --header ]]; then + HEADERS+=("$1" "$2") + shift 2 + else + shift + fi + done # Test mode for ISO if [ "${show_iso_url}" == 'on' ]; then @@ -945,14 +952,18 @@ function web_get() { exit 1 fi + if [[ ${OS} != windows && ${OS} != macos ]]; then + echo Downloading $(pretty_name "${OS}") ${RELEASE} ${EDITION:+ $EDITION} from ${URL} + fi + if command -v aria2c &>/dev/null; then - if ! aria2c --stderr -x16 --continue=true --summary-interval=0 --download-result=hide --console-log-level=error "${URL}" --dir "${DIR}" -o "${FILE}"; then + if ! aria2c --stderr -x16 --continue=true --summary-interval=0 --download-result=hide --console-log-level=error "${URL}" --dir "${DIR}" -o "${FILE}" "${HEADERS[@]}"; then echo #Necessary as aria2c in suppressed mode does not have new lines echo "ERROR! Failed to download ${URL} with aria2c. Try running 'quickget' again." exit 1 fi echo #Necessary as aria2c in suppressed mode does not have new lines - elif ! wget --quiet --continue --tries=3 --read-timeout=10 --show-progress --progress=bar:force:noscroll "${URL}" -O "${DIR}/${FILE}"; then + elif ! wget --quiet --continue --tries=3 --read-timeout=10 --show-progress --progress=bar:force:noscroll "${URL}" -O "${DIR}/${FILE}" "${HEADERS[@]}"; then echo "ERROR! Failed to download ${URL} with wget. Try running 'quickget' again." exit 1 fi @@ -983,6 +994,7 @@ function zsync_get() { exit 1 fi + echo -e Downloading $(pretty_name "${OS}") ${RELEASE} ${EDITION+ ${EDITION}} from ${URL}'\n' # Only force http for zsync - not earlier because we might fall through here if ! zsync "${URL/https/http}.zsync" -i "${DIR}/${OUT}" -o "${DIR}/${OUT}" 2>/dev/null; then echo "ERROR! Failed to download ${URL/https/http}.zsync" @@ -1657,10 +1669,20 @@ function get_lmde() { echo "${URL}/${ISO} ${HASH}" } +function generate_id() { + local macRecoveryID="" + local TYPE="${1}" + local valid_chars=("0" "1" "2" "3" "4" "5" "6" "7" "8" "9" "A" "B" "C" "D" "E" "F") + for ((i=0; i<$TYPE; i++)); do + macRecoveryID+="${valid_chars[$((RANDOM % 16))]}" + done + echo "${macRecoveryID}" +} + function get_macos() { local BOARD_ID="" local CWD="" - local MACRECOVERY="" + local CHUNKCHECK="" local MLB="00000000000000000" local OS_TYPE="default" @@ -1697,43 +1719,74 @@ function get_macos() { BOARD_ID="Mac-E43C1C25D4880AD6";; ventura) #13 BOARD_ID="Mac-BE088AF8C5EB4FA2";; + sonoma) + BOARD_ID="Mac-53FDB3D8DB8CA971";; *) echo "ERROR! Unknown release: ${RELEASE}" releases_macos exit 1;; esac - # Use a bundled macrecovery if possible CWD="$(dirname "${0}")" - if [ -x "${CWD}/macrecovery" ]; then - MACRECOVERY="${CWD}/macrecovery" - elif [ -x /usr/bin/macrecovery ]; then - MACRECOVERY="/usr/bin/macrecovery" + if [ -x "${CWD}/chunkcheck" ]; then + CHUNKCHECK="${CWD}/chunkcheck" + elif [ -x /usr/bin/chunkcheck ]; then + CHUNKCHECK="/usr/bin/chunkcheck" else - web_get "https://raw.githubusercontent.com/wimpysworld/quickemu/master/macrecovery" "${HOME}/.quickemu" - MACRECOVERY="python3 ${HOME}/.quickemu/macrecovery" + web_get "https://raw.githubusercontent.com/wimpysworld/quickemu/master/chunkcheck" "${HOME}/.quickemu" + CHUNKCHECK="${HOME}/.quickemu/chunkcheck" fi - if [ -z "${MACRECOVERY}" ]; then - echo "ERROR! Can not find a usable macrecovery." - exit 1 + if [ -z "${CHUNKCHECK}" ]; then + read -p "ERROR! Can not find chunkcheck. Will not be able to verify image. Proceed anyways?" skipVerification + if [ "${skipVerification,,}" != "y" ] && [ "${skipVerification,,}" != "yes" ]; then + exit 1 + fi + echo 'Skipping verification' && skipVerification=true fi - # Get firmware - web_get "https://github.com/kholia/OSX-KVM/raw/master/OpenCore/OpenCore.qcow2" "${VM_PATH}" - web_get "https://github.com/kholia/OSX-KVM/raw/master/OVMF_CODE.fd" "${VM_PATH}" + OpenCore_qcow2="https://github.com/kholia/OSX-KVM/raw/master/OpenCore/OpenCore.qcow2" + OVMF_CODE="https://github.com/kholia/OSX-KVM/raw/master/OVMF_CODE.fd" + OVMF_VARS="https://github.com/kholia/OSX-KVM/raw/master/OVMF_VARS-1920x1080.fd" + + local appleSession=$(curl -v -H "Host: osrecovery.apple.com" -H "Connection: close" -A "InternetRecovery/1.0" http://osrecovery.apple.com/ 2>&1 | tr ';' '\n' | awk -F'session=|;' '{print $2}' | grep 1) + local info=$(curl -s -X POST -H "Host: osrecovery.apple.com" -H "Connection: close" -A "InternetRecovery/1.0" -b "session=\"${appleSession}\"" -H "Content-Type: text/plain"\ + -d $'cid='$(generate_id 16)$'\nsn='${MLB}$'\nbid='${BOARD_ID}$'\nk='$(generate_id 64)$'\nfg='$(generate_id 64)$'\nos='${OS_TYPE} \ + http://osrecovery.apple.com/InstallationPayload/RecoveryImage | tr ' ' '\n') + local downloadLink=$(echo "$info" | grep 'oscdn' | grep 'dmg') + local downloadSession=$(echo "$info" | grep 'expires' | grep 'dmg') + local chunkListLink=$(echo "$info" | grep 'oscdn' | grep 'chunklist') + local chunkListSession=$(echo "$info" | grep 'expires' | grep 'chunklist') + + if [ "${show_iso_url}" == 'on' ]; then + echo -e "Recovery URL (inaccessible through normal browser):\n${downloadLink}\nChunklist (used for verifying the Recovery Image):\n${chunkListLink}\nFirmware URLs:\n${OpenCore_qcow2}\n${OVMF_CODE}\n${OVMF_VARS}" + exit 0 + elif [ "${test_iso_url}" == 'on' ]; then + wget --spider --header "Host: oscdn.apple.com" --header "Connection: close" --header "User-Agent: InternetRecovery/1.0" --header "Cookie: AssetToken=${downloadSession}" "${downloadLink}" + wget --spider --header "Host: oscdn.apple.com" --header "Connection: close" --header "User-Agent: InternetRecovery/1.0" --header "Cookie: AssetToken=${chunkListSession}" "${chunkListLink}" + exit 0 + fi + + echo Downloading macOS firmware + web_get "${OpenCore_qcow2}" "${VM_PATH}" + web_get "${OVMF_CODE}" "${VM_PATH}" if [ ! -e "${VM_PATH}/OVMF_VARS-1920x1080.fd" ]; then - web_get "https://github.com/kholia/OSX-KVM/raw/master/OVMF_VARS-1920x1080.fd" "${VM_PATH}" + web_get "${OVMF_VARS}" "${VM_PATH}" fi if [ ! -e "${VM_PATH}/RecoveryImage.chunklist" ]; then - echo "Downloading ${RELEASE}..." - ${MACRECOVERY} \ - --board-id "${BOARD_ID}" \ - --mlb "${MLB}" \ - --os-type "${OS_TYPE}" \ - --basename RecoveryImage \ - --outdir "${VM_PATH}" \ - download + echo "Downloading macOS ${RELEASE} from ${downloadLink}" + web_get "${downloadLink}" "${VM_PATH}" RecoveryImage.dmg --header "Host: oscdn.apple.com" --header "Connection: close" --header "User-Agent: InternetRecovery/1.0" --header "Cookie: AssetToken=${downloadSession}" + curl --progress-bar "${chunkListLink}" -o "${VM_PATH}/RecoveryImage.chunklist" --header "Host: oscdn.apple.com" --header "Connection: close" --header "User-Agent: InternetRecovery/1.0" --header "Cookie: AssetToken=${chunkListSession}" + fi + + if [ $skipVerification != true ]; then + if ! python3 "${CHUNKCHECK}" "${VM_PATH}" 2> /dev/null; then + echo Verification failed. + exit 1 + fi + echo Verified macOS ${RELEASE} image using chunklist. + else + echo Skipping verification of image. fi if [ -e "${VM_PATH}/RecoveryImage.dmg" ] && [ ! -e "${VM_PATH}/RecoveryImage.img" ]; then @@ -2636,6 +2689,10 @@ function download_windows() { fi if echo "$iso_download_link_html" | grep -q "We are unable to complete your request at this time."; then + if [ "${show_iso_url}" == 'on' ] || [ "${test_iso_url}" == 'on' ]; then + echo " - Failed to get URL: Microsoft blocked the automated download request based on your IP address." + exit 1 + fi echo " - Microsoft blocked the automated download request based on your IP address." failed=1 fi @@ -2659,7 +2716,15 @@ function download_windows() { return 1 fi - echo " - Got latest ISO download link (valid for 24 hours): $iso_download_link" + if [ "${show_iso_url}" == 'on' ]; then + echo -e " Windows ${RELEASE} Download (valid for 24 hours):\n${iso_download_link}" + exit 0 + elif [ "${test_iso_url}" == 'on' ]; then + wget --spider "${iso_download_link}" + exit 0 + fi + + echo Downloading Windows ${RELEASE} from "$iso_download_link" # Download ISO FILE_NAME="$(echo "$iso_download_link" | cut -d'?' -f1 | cut -d'/' -f5)" @@ -2667,7 +2732,6 @@ function download_windows() { } function get_windows() { - echo "Downloading Windows ${RELEASE}..." download_windows "${RELEASE}" echo "Downloading VirtIO drivers..."