From b2ccf6c0d2313d794468f7ae04beaddf5d930eb8 Mon Sep 17 00:00:00 2001 From: rdb Date: Mon, 19 Dec 2016 23:35:05 +0100 Subject: [PATCH] Add ability to produce .whl file Based on original version by pennomi Closes: #83 --- makepanda/makepanda.py | 21 +- makepanda/makewheel.py | 558 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 577 insertions(+), 2 deletions(-) create mode 100644 makepanda/makewheel.py diff --git a/makepanda/makepanda.py b/makepanda/makepanda.py index 956e8db52c..f2746881bd 100755 --- a/makepanda/makepanda.py +++ b/makepanda/makepanda.py @@ -39,6 +39,7 @@ import sys COMPILER=0 INSTALLER=0 +WHEEL=0 GENMAN=0 COMPRESSOR="zlib" THREADCOUNT=0 @@ -124,6 +125,7 @@ def usage(problem): print(" --verbose (print out more information)") print(" --runtime (build a runtime build instead of an SDK build)") print(" --installer (build an installer)") + print(" --wheel (build a pip-installable .whl)") print(" --optimize X (optimization level can be 1,2,3,4)") print(" --version X (set the panda version number)") print(" --lzma (use lzma compression when building Windows installer)") @@ -159,13 +161,13 @@ def usage(problem): os._exit(1) def parseopts(args): - global INSTALLER,RTDIST,RUNTIME,GENMAN,DISTRIBUTOR,VERSION + global INSTALLER,WHEEL,RTDIST,RUNTIME,GENMAN,DISTRIBUTOR,VERSION global COMPRESSOR,THREADCOUNT,OSXTARGET,OSX_ARCHS,HOST_URL global DEBVERSION,RPMRELEASE,GIT_COMMIT,P3DSUFFIX,RTDIST_VERSION global STRDXSDKVERSION, WINDOWS_SDK, MSVC_VERSION, BOOUSEINTELCOMPILER longopts = [ "help","distributor=","verbose","runtime","osxtarget=", - "optimize=","everything","nothing","installer","rtdist","nocolor", + "optimize=","everything","nothing","installer","wheel","rtdist","nocolor", "version=","lzma","no-python","threads=","outputdir=","override=", "static","host=","debversion=","rpmrelease=","p3dsuffix=","rtdist-version=", "directx-sdk=", "windows-sdk=", "msvc-version=", "clean", "use-icl", @@ -188,6 +190,7 @@ def parseopts(args): if (option=="--help"): raise Exception elif (option=="--optimize"): optimize=value elif (option=="--installer"): INSTALLER=1 + elif (option=="--wheel"): WHEEL=1 elif (option=="--verbose"): SetVerbose(True) elif (option=="--distributor"): DISTRIBUTOR=value elif (option=="--rtdist"): RTDIST=1 @@ -416,9 +419,18 @@ if (RUNTIME): if (INSTALLER and RTDIST): exit("Cannot build an installer for the rtdist build!") +if (WHEEL and RUNTIME): + exit("Cannot build a wheel for the runtime build!") + +if (WHEEL and RTDIST): + exit("Cannot build a wheel for the rtdist build!") + if (INSTALLER) and (PkgSkip("PYTHON")) and (not RUNTIME) and GetTarget() == 'windows': exit("Cannot build installer on Windows without python") +if WHEEL and PkgSkip("PYTHON"): + exit("Cannot build wheel without Python") + if (RTDIST) and (PkgSkip("WX") and PkgSkip("FLTK")): exit("Cannot build rtdist without wx or fltk") @@ -7239,6 +7251,11 @@ try: MakeInstallerFreeBSD() else: exit("Do not know how to make an installer for this platform") + + if WHEEL: + ProgressOutput(100.0, "Building wheel") + from makewheel import makewheel + makewheel(VERSION, GetOutputDir()) finally: SaveDependencyCache() diff --git a/makepanda/makewheel.py b/makepanda/makewheel.py new file mode 100644 index 0000000000..81caf24701 --- /dev/null +++ b/makepanda/makewheel.py @@ -0,0 +1,558 @@ +""" +Generates a wheel (.whl) file from the output of makepanda. + +Since the wheel requires special linking, this will only work if compiled with +the `--wheel` parameter. +""" +from __future__ import print_function, unicode_literals +from distutils.util import get_platform as get_dist +import json + +import sys +import os +from os.path import join +import shutil +import zipfile +import hashlib +import tempfile +import subprocess +from sysconfig import get_config_var +from optparse import OptionParser +from makepandacore import ColorText, LocateBinary, ParsePandaVersion, GetExtensionSuffix, SetVerbose, GetVerbose +from base64 import urlsafe_b64encode + + +def get_platform(): + p = get_dist().replace('-', '_').replace('.', '_') + #if "linux" in p: + # print(ColorText("red", "WARNING:") + + # " Linux-specific wheel files are not supported." + # " We will generate this wheel as a generic package instead.") + # return "any" + return p + + +def get_abi_tag(): + if sys.version_info >= (3, 0): + soabi = get_config_var('SOABI') + if soabi and soabi.startswith('cpython-'): + return 'cp' + soabi.split('-')[1] + elif soabi: + return soabi.replace('.', '_').replace('-', '_') + + soabi = 'cp%d%d' % (sys.version_info[:2]) + + debug_flag = get_config_var('Py_DEBUG') + if (debug_flag is None and hasattr(sys, 'gettotalrefcount')) or debug_flag: + soabi += 'd' + + malloc_flag = get_config_var('WITH_PYMALLOC') + if malloc_flag is None or malloc_flag: + soabi += 'm' + + if sys.version_info < (3, 3): + usize = get_config_var('Py_UNICODE_SIZE') + if (usize is None and sys.maxunicode == 0x10ffff) or usize == 4: + soabi += 'u' + + return soabi + + +def is_exe_file(path): + return os.path.isfile(path) and path.lower().endswith('.exe') + + +def is_elf_file(path): + base = os.path.basename(path) + return os.path.isfile(path) and '.' not in base and \ + open(path, 'rb').read(4) == b'\x7FELF' + + +def is_mach_o_file(path): + base = os.path.basename(path) + return os.path.isfile(path) and '.' not in base and \ + open(path, 'rb').read(4) == b'\xCA\xFE\xBA\xBE' + + +if sys.platform in ('win32', 'cygwin'): + is_executable = is_exe_file +elif sys.platform == 'darwin': + is_executable = is_mach_o_file +else: + is_executable = is_elf_file + + +# Other global parameters +PY_VERSION = "cp{}{}".format(sys.version_info.major, sys.version_info.minor) +ABI_TAG = get_abi_tag() +PLATFORM_TAG = get_platform() +EXCLUDE_EXT = [".pyc", ".pyo", ".N", ".prebuilt", ".xcf", ".plist", ".vcproj", ".sln"] + +# Plug-ins to install. +PLUGIN_LIBS = ["pandagl", "pandagles", "pandagles2", "p3ptloader", "p3assimp", "p3ffmpeg", "p3openal_audio", "p3fmod_audio"] + +WHEEL_DATA = """Wheel-Version: 1.0 +Generator: makepanda +Root-Is-Purelib: false +Tag: {}-{}-{} +""" + +METADATA = { + "license": "BSD", + "name": "Panda3D", + "metadata_version": "2.0", + "generator": "makepanda", + "summary": "Panda3D is a game engine, a framework for 3D rendering and " + "game development for Python and C++ programs.", + "extensions": { + "python.details": { + "project_urls": { + "Home": "https://www.panda3d.org/" + }, + "document_names": { + "license": "LICENSE.txt" + }, + "contacts": [ + { + "role": "author", + "email": "etc-panda3d@lists.andrew.cmu.edu", + "name": "Panda3D Team" + } + ] + } + }, + "classifiers": [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: End Users/Desktop", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: C++", + "Programming Language :: Python", + "Topic :: Games/Entertainment", + "Topic :: Multimedia", + "Topic :: Multimedia :: Graphics", + "Topic :: Multimedia :: Graphics :: 3D Rendering" + ] +} + +PANDA3D_TOOLS_INIT = """import os, sys +import panda3d + +if sys.platform in ('win32', 'cygwin'): + path_var = 'PATH' +elif sys.platform == 'darwin': + path_var = 'DYLD_LIBRARY_PATH' +else: + path_var = 'LD_LIBRARY_PATH' + +dir = os.path.dirname(panda3d.__file__) +del panda3d +if not os.environ.get(path_var): + os.environ[path_var] = dir +else: + os.environ[path_var] = dir + os.pathsep + os.environ[path_var] + +del os, sys, path_var, dir + + +def _exec_tool(tool): + import os, sys + from subprocess import Popen + tools_dir = os.path.dirname(__file__) + handle = Popen(sys.argv, executable=os.path.join(tools_dir, tool)) + try: + try: + return handle.wait() + except KeyboardInterrupt: + # Give the program a chance to handle the signal gracefully. + return handle.wait() + except: + handle.kill() + handle.wait() + raise + +# Register all the executables in this directory as global functions. +{0} +""" + + +def parse_dependencies_windows(data): + """ Parses the given output from dumpbin /dependents to determine the list + of dll's this executable file depends on. """ + + lines = data.splitlines() + li = 0 + while li < len(lines): + line = lines[li] + li += 1 + if line.find(' has the following dependencies') != -1: + break + + if li < len(lines): + line = lines[li] + if line.strip() == '': + # Skip a blank line. + li += 1 + + # Now we're finding filenames, until the next blank line. + filenames = [] + while li < len(lines): + line = lines[li] + li += 1 + line = line.strip() + if line == '': + # We're done. + return filenames + filenames.append(line) + + # At least we got some data. + return filenames + + +def parse_dependencies_unix(data): + """ Parses the given output from otool -XL or ldd to determine the list of + libraries this executable file depends on. """ + + lines = data.splitlines() + filenames = [] + for l in lines: + l = l.strip() + if l != "statically linked": + filenames.append(l.split(' ', 1)[0]) + return filenames + + +def scan_dependencies(pathname): + """ Checks the named file for DLL dependencies, and adds any appropriate + dependencies found into pluginDependencies and dependentFiles. """ + + if sys.platform == "darwin": + command = ['otool', '-XL', pathname] + elif sys.platform in ("win32", "cygwin"): + command = ['dumpbin', '/dependents', pathname] + else: + command = ['ldd', pathname] + + output = subprocess.check_output(command, universal_newlines=True) + filenames = None + + if sys.platform in ("win32", "cygwin"): + filenames = parse_dependencies_windows(output) + else: + filenames = parse_dependencies_unix(output) + + if filenames is None: + sys.exit("Unable to determine dependencies from %s" % (pathname)) + + return filenames + + +class WheelFile(object): + def __init__(self, name, version): + self.name = name + self.version = version + + wheel_name = "{}-{}-{}-{}-{}.whl".format( + name, version, PY_VERSION, ABI_TAG, PLATFORM_TAG) + + print("Writing %s" % (wheel_name)) + self.zip_file = zipfile.ZipFile(wheel_name, 'w', zipfile.ZIP_DEFLATED) + self.records = [] + + # Used to locate dependency libraries. + self.lib_path = [] + self.dep_paths = {} + + def consider_add_dependency(self, target_path, dep, search_path=None): + """Considers adding a dependency library. + Returns the target_path if it was added, which may be different from + target_path if it was already added earlier, or None if it wasn't.""" + + if dep in self.dep_paths: + # Already considered this. + return self.dep_paths[dep] + + self.dep_paths[dep] = None + + if dep.lower().startswith("python"): + # Don't include the Python library. + return + + source_path = None + + if search_path is None: + search_path = self.lib_path + + for lib_dir in search_path: + # Ignore static stuff. + path = os.path.join(lib_dir, dep) + if os.path.isfile(path): + source_path = os.path.normpath(path) + break + + if not source_path: + # Couldn't find library in the panda3d lib dir. + #print("Ignoring %s" % (dep)) + return + + self.dep_paths[dep] = target_path + self.write_file(target_path, source_path) + return target_path + + def write_file(self, target_path, source_path): + """Adds the given file to the .whl file.""" + + # If this is a .so file, we should set the rpath appropriately. + temp = None + ext = os.path.splitext(source_path)[1] + if ext in ('.so', '.dylib') or '.so.' in os.path.basename(source_path) or \ + (not ext and is_executable(source_path)): + # Scan and add Unix dependencies. + deps = scan_dependencies(source_path) + for dep in deps: + # Only include dependencies with relative path. Otherwise we + # end up overwriting system files like /lib/ld-linux.so.2! + # Yes, it happened to me. + if '/' not in dep: + target_dep = os.path.dirname(target_path) + '/' + dep + self.consider_add_dependency(target_dep, dep) + + suffix = '' + if '.so' in os.path.basename(source_path): + suffix = '.so' + elif ext == '.dylib': + suffix = '.dylib' + + temp = tempfile.NamedTemporaryFile(suffix=suffix, prefix='whl', delete=False) + temp.write(open(source_path, 'rb').read()) + os.fchmod(temp.fileno(), os.fstat(temp.fileno()).st_mode | 0o111) + temp.close() + + # Fix things like @loader_path/../lib references + if sys.platform == "darwin": + loader_path = [os.path.dirname(source_path)] + for dep in deps: + if '@loader_path' not in dep: + continue + + dep_path = dep.replace('@loader_path', '.') + target_dep = os.path.dirname(target_path) + '/' + os.path.basename(dep) + target_dep = self.consider_add_dependency(target_dep, dep_path, loader_path) + if not target_dep: + # It won't be included, so no use adjusting the path. + continue + + new_dep = os.path.join('@loader_path', os.path.relpath(target_dep, os.path.dirname(target_path))) + subprocess.call(["install_name_tool", "-change", dep, new_dep, temp.name]) + else: + subprocess.call(["strip", "-s", temp.name]) + subprocess.call(["patchelf", "--set-rpath", "$ORIGIN", temp.name]) + + source_path = temp.name + + ext = ext.lower() + if ext in ('.dll', '.pyd', '.exe'): + # Scan and add Win32 dependencies. + for dep in scan_dependencies(source_path): + target_dep = os.path.dirname(target_path) + '/' + dep + self.consider_add_dependency(target_dep, dep) + + # Calculate the SHA-256 hash and size. + sha = hashlib.sha256() + fp = open(source_path, 'rb') + size = 0 + data = fp.read(1024 * 1024) + while data: + size += len(data) + sha.update(data) + data = fp.read(1024 * 1024) + fp.close() + + # Save it in PEP-0376 format for writing out later. + digest = str(urlsafe_b64encode(sha.digest())) + digest = digest.rstrip('=') + self.records.append("{},sha256={},{}\n".format(target_path, digest, size)) + + if GetVerbose(): + print("Adding %s from %s" % (target_path, source_path)) + self.zip_file.write(source_path, target_path) + + #if temp: + # os.unlink(temp.name) + + def write_file_data(self, target_path, source_data): + """Adds the given file from a string.""" + + sha = hashlib.sha256() + sha.update(source_data.encode()) + digest = str(urlsafe_b64encode(sha.digest())) + digest = digest.rstrip('=') + self.records.append("{},sha256={},{}\n".format(target_path, digest, len(source_data))) + + if GetVerbose(): + print("Adding %s from data" % target_path) + self.zip_file.writestr(target_path, source_data) + + def write_directory(self, target_dir, source_dir): + """Adds the given directory recursively to the .whl file.""" + + for root, dirs, files in os.walk(source_dir): + for file in files: + if os.path.splitext(file)[1] in EXCLUDE_EXT: + continue + + source_path = os.path.join(root, file) + target_path = os.path.join(target_dir, os.path.relpath(source_path, source_dir)) + target_path = target_path.replace('\\', '/') + self.write_file(target_path, source_path) + + def close(self): + # Write the RECORD file. + record_file = "{}-{}.dist-info/RECORD".format(self.name, self.version) + self.records.append(record_file + ",,\n") + + self.zip_file.writestr(record_file, "".join(self.records)) + self.zip_file.close() + + +def makewheel(version, output_dir): + if sys.platform not in ("win32", "darwin") and not sys.platform.startswith("cygwin"): + if not LocateBinary("patchelf"): + raise Exception("patchelf is required when building a Linux wheel.") + + # Global filepaths + panda3d_dir = join(output_dir, "panda3d") + pandac_dir = join(output_dir, "pandac") + direct_dir = join(output_dir, "direct") + models_dir = join(output_dir, "models") + etc_dir = join(output_dir, "etc") + bin_dir = join(output_dir, "bin") + if sys.platform == "win32": + libs_dir = join(output_dir, "bin") + else: + libs_dir = join(output_dir, "lib") + license_src = "LICENSE" + readme_src = "README.md" + + # Update relevant METADATA entries + METADATA['version'] = version + version_classifiers = [ + "Programming Language :: Python :: {}".format(*sys.version_info), + "Programming Language :: Python :: {}.{}".format(*sys.version_info), + ] + METADATA['classifiers'].extend(version_classifiers) + + # Build out the metadata + details = METADATA["extensions"]["python.details"] + homepage = details["project_urls"]["Home"] + author = details["contacts"][0]["name"] + email = details["contacts"][0]["email"] + metadata = ''.join([ + "Metadata-Version: {metadata_version}\n" \ + "Name: {name}\n" \ + "Version: {version}\n" \ + "Summary: {summary}\n" \ + "License: {license}\n".format(**METADATA), + "Home-page: {}\n".format(homepage), + "Author: {}\n".format(author), + "Author-email: {}\n".format(email), + "Platform: {}\n".format(PLATFORM_TAG), + ] + ["Classifier: {}\n".format(c) for c in METADATA['classifiers']]) + + # Zip it up and name it the right thing + whl = WheelFile('panda3d', version) + whl.lib_path = [libs_dir] + + # Add the trees with Python modules. + whl.write_directory('direct', direct_dir) + + # Write the panda3d tree. We use a custom empty __init__ since the + # default one adds the bin directory to the PATH, which we don't have. + whl.write_file_data('panda3d/__init__.py', '') + + ext_suffix = GetExtensionSuffix() + + for file in os.listdir(panda3d_dir): + if file == '__init__.py': + pass + elif file.endswith(ext_suffix) or file.endswith('.py'): + source_path = os.path.join(panda3d_dir, file) + + if file.endswith('.pyd') and PLATFORM_TAG.startswith('cygwin'): + # Rename it to .dll for cygwin Python to be able to load it. + target_path = 'panda3d/' + os.path.splitext(file)[0] + '.dll' + else: + target_path = 'panda3d/' + file + whl.write_file(target_path, source_path) + + # Add plug-ins. + for lib in PLUGIN_LIBS: + plugin_name = 'lib' + lib + if sys.platform in ('win32', 'cygwin'): + plugin_name += '.dll' + elif sys.platform == 'darwin': + plugin_name += '.dylib' + else: + plugin_name += '.so' + plugin_path = os.path.join(libs_dir, plugin_name) + if os.path.isfile(plugin_path): + whl.write_file('panda3d/' + plugin_name, plugin_path) + + # Add the pandac tree for backward compatibility. + for file in os.listdir(pandac_dir): + if file.endswith('.py'): + whl.write_file('pandac/' + file, os.path.join(pandac_dir, file)) + + # Add a panda3d-tools directory containing the executables. + entry_points = '[console_scripts]\n' + tools_init = '' + for file in os.listdir(bin_dir): + source_path = os.path.join(bin_dir, file) + + if is_executable(source_path): + # Put the .exe files inside the panda3d-tools directory. + whl.write_file('panda3d_tools/' + file, source_path) + + # Tell pip to create a wrapper script. + basename = os.path.splitext(file)[0] + funcname = basename.replace('-', '_') + entry_points += '{0} = panda3d_tools:{1}\n'.format(basename, funcname) + tools_init += '{0} = lambda: _exec_tool({1!r})\n'.format(funcname, file) + + whl.write_file_data('panda3d_tools/__init__.py', PANDA3D_TOOLS_INIT.format(tools_init)) + + # Add the .data directory, containing additional files. + data_dir = 'panda3d-{}.data'.format(version) + #whl.write_directory(data_dir + '/data/etc', etc_dir) + #whl.write_directory(data_dir + '/data/models', models_dir) + + # Actually, let's not. That seems to install the files to the strangest + # places in the user's filesystem. Let's instead put them in panda3d. + whl.write_directory('panda3d/etc', etc_dir) + whl.write_directory('panda3d/models', models_dir) + + # Add the dist-info directory last. + info_dir = 'panda3d-{}.dist-info'.format(version) + whl.write_file_data(info_dir + '/entry_points.txt', entry_points) + whl.write_file_data(info_dir + '/metadata.json', json.dumps(METADATA, indent=4, separators=(',', ': '))) + whl.write_file_data(info_dir + '/METADATA', metadata) + whl.write_file_data(info_dir + '/WHEEL', WHEEL_DATA.format(PY_VERSION, ABI_TAG, PLATFORM_TAG)) + whl.write_file(info_dir + '/LICENSE.txt', license_src) + whl.write_file(info_dir + '/README.md', readme_src) + whl.write_file_data(info_dir + '/top_level.txt', 'direct\npanda3d\npandac\npanda3d_tools\n') + + whl.close() + + +if __name__ == "__main__": + version = ParsePandaVersion("dtool/PandaVersion.pp") + + parser = OptionParser() + parser.add_option('', '--version', dest = 'version', help = 'Panda3D version number (default: %s)' % (version), default = version) + parser.add_option('', '--outputdir', dest = 'outputdir', help = 'Makepanda\'s output directory (default: built)', default = 'built') + parser.add_option('', '--verbose', dest = 'verbose', help = 'Enable verbose output', action = 'store_true', default = False) + (options, args) = parser.parse_args() + + SetVerbose(options.verbose) + makewheel(options.version, options.outputdir)