Allow running test suite with emscripten via node.js

.github/workflows/ci.yml

@@ -480,22 +480,59 @@ jobs:
 
   emscripten:
     if: "!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[ci skip]')"
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v4
 
     - name: Install dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install build-essential ninja-build bison flex
+        sudo apt-get install build-essential ninja-build bison flex nodejs python3-pip
 
     - name: Setup emsdk
       uses: mymindstorm/setup-emsdk@v14
       with:
-        version: 3.1.65
+        version: 3.1.70
         actions-cache-folder: 'emsdk-cache'
 
+    - name: Restore Python build cache
+      id: cache-emscripten-python-restore
+      uses: actions/cache/restore@v4
+      with:
+        path: ~/python
+        key: cache-emscripten-python-3.12.7
+
+    - name: Build Python 3.12
+      if: steps.cache-emscripten-python-restore.outputs.cache-hit != 'true'
+      run: |
+        wget https://www.python.org/ftp/python/3.12.7/Python-3.12.7.tar.xz
+        tar -xJf Python-3.12.7.tar.xz
+        (cd Python-3.12.7 && EM_CONFIG=$EMSDK/.emscripten python3 Tools/wasm/wasm_build.py emscripten-browser)
+        (cd Python-3.12.7/builddir/emscripten-browser && make install DESTDIR=~/python)
+        cp Python-3.12.7/builddir/emscripten-browser/Modules/_hacl/libHacl_Hash_SHA2.a ~/python/usr/local/lib
+        cp Python-3.12.7/builddir/emscripten-browser/Modules/_decimal/libmpdec/libmpdec.a ~/python/usr/local/lib
+        cp Python-3.12.7/builddir/emscripten-browser/Modules/expat/libexpat.a ~/python/usr/local/lib
+        rm -rf Python-3.12.7
+
+    - name: Save Python build cache
+      id: cache-emscripten-python-save
+      if: steps.cache-emscripten-python-restore.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v4
+      with:
+        path: ~/python
+        key: ${{ steps.cache-emscripten-python-restore.outputs.cache-primary-key }}
+
     - name: Build for Emscripten
       shell: bash
       run: |
-        python3 makepanda/makepanda.py --git-commit=${{github.sha}} --target emscripten --outputdir=built --everything --no-python --verbose --threads=4
+        python3 makepanda/makepanda.py --git-commit=${{github.sha}} --target emscripten --python-incdir=~/python/usr/local/include --python-libdir=~/python/usr/local/lib --static --everything --no-pandatool --no-deploytools --verbose --threads=4
+
+    - name: Install test dependencies
+      shell: bash
+      run: |
+        python3 -m pip install -t ~/python/usr/local/lib/python3.12/site-packages -r requirements-test.txt
+
+    - name: Test in node.js
+      shell: bash
+      run: |
+        PYTHONHOME=~/python/usr/local PYTHONPATH=built node built/bin/run_tests.js tests

makepanda/installpanda.py

@@ -29,6 +29,9 @@ APP_INFO = (
   ("pstats", "Panda3D Profiling Tool", ("pstats",), False),
 )
 
+EXCLUDE_BINARIES = ["deploy-stub", "deploy-stubw", "run_tests"]
+
+
 def WriteApplicationsFile(fname, appinfo, mimeinfo, bindir):
     fhandle = open(fname, "w")
     for app, desc, exts, multiple in appinfo:
@@ -265,7 +268,7 @@ def InstallPanda(destdir="", prefix="/usr", outputdir="built", libdir=GetLibDir(
             oscmd(f"cp -R -P {outputdir}/lib/{base} {dest_libdir}/panda3d/{base}")
 
     for base in os.listdir(outputdir + "/bin"):
-        if not base.startswith("deploy-stub"):
+        if base not in EXCLUDE_BINARIES:
             oscmd(f"cp -R -P {outputdir}/bin/{base} {dest_prefix}/bin/{base}")
 
     DeleteVCS(dest_prefix + "/share/panda3d")

makepanda/makepackage.py

@@ -125,6 +125,8 @@ MACOS_SCRIPT_POSTFIX = """\
 fi
 """
 
+EXCLUDE_BINARIES = ["deploy-stub", "deploy-stubw", "run_tests"]
+
 
 def MakeInstallerNSIS(version, file, title, installdir, compressor="lzma", **kwargs):
     outputdir = GetOutputDir()
@@ -404,7 +406,7 @@ def MakeInstallerLinux(version, debversion=None, rpmversion=None, rpmrelease=1,
 
         # Add the binaries in /usr/bin explicitly to the spec file
         for base in os.listdir(outputdir + "/bin"):
-            if not base.startswith("deploy-stub"):
+            if base not in EXCLUDE_BINARIES:
                 txt += "/usr/bin/%s\n" % (base)
 
         # Write out the spec file.
@@ -470,7 +472,7 @@ def MakeInstallerOSX(version, python_versions=[], installdir=None, **kwargs):
     oscmd("install -m 0644 doc/man/*.1 dstroot/tools/usr/local/share/man/man1/")
 
     for base in os.listdir(outputdir + "/bin"):
-        if not base.startswith("deploy-stub"):
+        if base not in EXCLUDE_BINARIES:
             binname = ("dstroot/tools/%s/bin/" % installdir) + base
             # OSX needs the -R argument to copy symbolic links correctly, it doesn't have -d. How weird.
             oscmd("cp -R " + outputdir + "/bin/" + base + " " + binname)

makepanda/makepanda.py

@@ -5944,6 +5944,43 @@ if PkgSkip("PYTHON") == 0:
         PyTargetAdd('libdeploy-stubw.dll', input='libp3android.dll')
         PyTargetAdd('libdeploy-stubw.dll', opts=['DEPLOYSTUB', 'ANDROID'])
 
+#
+# Build the test runner for static builds
+#
+if GetLinkAllStatic():
+    if GetTarget() == 'emscripten':
+        LinkFlag('RUN_TESTS_FLAGS', '-s NODERAWFS')
+        LinkFlag('RUN_TESTS_FLAGS', '-s FORCE_FILESYSTEM -lnodefs.js')
+        LinkFlag('RUN_TESTS_FLAGS', '-s ASSERTIONS=2')
+        LinkFlag('RUN_TESTS_FLAGS', '-s ALLOW_MEMORY_GROWTH')
+        LinkFlag('RUN_TESTS_FLAGS', '-s INITIAL_HEAP=585302016')
+        LinkFlag('RUN_TESTS_FLAGS', '-s STACK_SIZE=1048576')
+        LinkFlag('RUN_TESTS_FLAGS', '--minify 0')
+
+    if not PkgSkip('DIRECT'):
+        DefSymbol('RUN_TESTS_FLAGS', 'HAVE_DIRECT')
+    if not PkgSkip('PANDAPHYSICS'):
+        DefSymbol('RUN_TESTS_FLAGS', 'HAVE_PHYSICS')
+    if not PkgSkip('EGG'):
+        DefSymbol('RUN_TESTS_FLAGS', 'HAVE_EGG')
+    if not PkgSkip('BULLET'):
+        DefSymbol('RUN_TESTS_FLAGS', 'HAVE_BULLET')
+
+    OPTS=['DIR:tests', 'PYTHON', 'RUN_TESTS_FLAGS']
+    PyTargetAdd('run_tests-main.obj', opts=OPTS, input='main.c')
+    PyTargetAdd('run_tests.exe', input='run_tests-main.obj')
+    PyTargetAdd('run_tests.exe', input='core.pyd')
+    if not PkgSkip('DIRECT'):
+        PyTargetAdd('run_tests.exe', input='direct.pyd')
+    if not PkgSkip('PANDAPHYSICS'):
+        PyTargetAdd('run_tests.exe', input='physics.pyd')
+    if not PkgSkip('EGG'):
+        PyTargetAdd('run_tests.exe', input='egg.pyd')
+    if not PkgSkip('BULLET'):
+        PyTargetAdd('run_tests.exe', input='bullet.pyd')
+    PyTargetAdd('run_tests.exe', input=COMMON_PANDA_LIBS)
+    PyTargetAdd('run_tests.exe', opts=['PYTHON', 'BULLET', 'RUN_TESTS_FLAGS'])
+
 #
 # Generate the models directory and samples directory
 #
@@ -6105,8 +6142,16 @@ finally:
 
 # Run the test suite.
 if RUNTESTS:
-    cmdstr = BracketNameWithQuotes(SDK["PYTHONEXEC"].replace('\\', '/'))
-    cmdstr += " -B -m pytest tests"
+    if GetLinkAllStatic():
+        runner = FindLocation("run_tests.exe", [])
+        if runner.endswith(".js"):
+            cmdstr = "node " + BracketNameWithQuotes(runner)
+        else:
+            cmdstr = BracketNameWithQuotes(runner)
+    else:
+        cmdstr = BracketNameWithQuotes(SDK["PYTHONEXEC"].replace('\\', '/'))
+        cmdstr += " -B -m pytest"
+    cmdstr += " tests"
     if GetVerbose():
         cmdstr += " --verbose"
     oscmd(cmdstr)

makepanda/makepandacore.py

@@ -3078,6 +3078,9 @@ def SetupBuildEnvironment(compiler):
 
     # If we're cross-compiling, no point in putting our output dirs on the path.
     if CrossCompiling():
+        if GetTarget() == 'emscripten' and not PkgSkip("PYTHON"):
+            AddToPathEnv("PYTHONPATH", GetOutputDir())
+            os.environ["PYTHONHOME"] = os.path.dirname(os.path.dirname(SDK["PYTHON"]))
         return
 
     # Add our output directories to the environment.

makepanda/makewheel.py

@@ -104,6 +104,7 @@ EXCLUDE_BINARIES = [
     'interrogate_module',
     'test_interrogate',
     'parse_file',
+    'run_tests',
 ]
 
 WHEEL_DATA = """Wheel-Version: 1.0

tests/gui/test_DirectGui.py

@@ -8,7 +8,7 @@ from direct.gui.DirectGui import DirectButton, DirectDialog, DirectEntry, Direct
 def test_DirectGui(base):
     # EXAMPLE CODE
     # Load a model
-    smiley = base.loader.loadModel('models/misc/smiley')
+    smiley = base.loader.loadModel('models/misc/smiley.egg')
 
     # Here we specify the button's command
     def dummyCmd(index):

tests/main.c

@@ -0,0 +1,164 @@
+/**
+ * PANDA 3D SOFTWARE
+ * Copyright (c) Carnegie Mellon University.  All rights reserved.
+ *
+ * All use of this software is subject to the terms of the revised BSD
+ * license.  You should have received a copy of this license along
+ * with this source code in a file named "LICENSE."
+ *
+ * @file main.c
+ * @author rdb
+ * @date 2024-11-03
+ */
+
+/**
+ * This script embeds the Python interpreter and runs the unit testing suite.
+ * It is designed for use with a statically built Panda3D, where the Panda3D
+ * modules are linked directly into the interpreter.
+ */
+
+#include <Python.h>
+
+#ifdef __EMSCRIPTEN__
+#include <emscripten/emscripten.h>
+#include <emscripten/em_asm.h>
+#endif
+
+#include "pandabase.h"
+
+#ifdef LINK_ALL_STATIC
+extern PyObject *PyInit_core();
+
+#ifdef HAVE_DIRECT
+extern PyObject *PyInit_direct();
+#endif
+
+#ifdef HAVE_PHYSICS
+extern PyObject *PyInit_physics();
+#endif
+
+#ifdef HAVE_EGG
+extern PyObject *PyInit_egg();
+extern EXPCL_PANDAEGG void init_libpandaegg();
+#endif
+
+#ifdef HAVE_BULLET
+extern PyObject *PyInit_bullet();
+#endif
+
+extern EXPCL_PANDA_PNMIMAGETYPES void init_libpnmimagetypes();
+#endif  // LINK_ALL_STATIC
+
+
+int main(int argc, char **argv) {
+  PyStatus status;
+  PyConfig config;
+  PyConfig_InitPythonConfig(&config);
+
+#ifdef __EMSCRIPTEN__
+  // getenv does not work with emscripten, instead read the PYTHONPATH and
+  // PYTHONHOME from the process.env variable if we're running in node.js.
+  char path[4096], home[4096];
+  path[0] = 0;
+  home[0] = 0;
+
+  EM_ASM({
+    if (process && process.env) {
+      var path = process.env.PYTHONPATH;
+      var home = process.env.PYTHONHOME;
+      if (path) {
+        if (process.platform === 'win32') {
+          path = path.replace(/;/g, ':');
+        }
+        stringToUTF8(path, $0, $1);
+      }
+      if (home) {
+        stringToUTF8(home, $2, $3);
+      }
+    }
+  }, path, sizeof(path), home, sizeof(home));
+
+  if (path[0] != 0) {
+    status = PyConfig_SetBytesString(&config, &config.pythonpath_env, path);
+    if (PyStatus_Exception(status)) {
+      goto exception;
+    }
+  }
+  if (home[0] != 0) {
+    status = PyConfig_SetBytesString(&config, &config.home, home);
+    if (PyStatus_Exception(status)) {
+      goto exception;
+    }
+  }
+#endif
+
+  PyConfig_SetBytesString(&config, &config.run_module, "pytest");
+  config.parse_argv = 0;
+
+  status = PyConfig_SetBytesArgv(&config, argc, argv);
+  if (PyStatus_Exception(status)) {
+    goto exception;
+  }
+
+  status = Py_InitializeFromConfig(&config);
+  if (PyStatus_Exception(status)) {
+    goto exception;
+  }
+  PyConfig_Clear(&config);
+
+#ifdef LINK_ALL_STATIC
+#ifdef HAVE_EGG
+  init_libpandaegg();
+#endif
+
+  init_libpnmimagetypes();
+
+  {
+    PyObject *panda3d_module = PyImport_ImportModule("panda3d");
+    PyObject *panda3d_dict = PyModule_GetDict(panda3d_module);
+    PyObject *sys_modules = PySys_GetObject("modules");
+
+    PyObject *core_module = PyInit_core();
+    PyDict_SetItemString(panda3d_dict, "core", core_module);
+    PyDict_SetItemString(sys_modules, "panda3d.core", core_module);
+
+#ifdef HAVE_DIRECT
+    PyObject *direct_module = PyInit_direct();
+    PyDict_SetItemString(panda3d_dict, "direct", direct_module);
+    PyDict_SetItemString(sys_modules, "panda3d.direct", direct_module);
+#endif
+
+#ifdef HAVE_PHYSICS
+    PyObject *physics_module = PyInit_physics();
+    PyDict_SetItemString(panda3d_dict, "physics", physics_module);
+    PyDict_SetItemString(sys_modules, "panda3d.physics", physics_module);
+#endif
+
+#ifdef HAVE_EGG
+    PyObject *egg_module = PyInit_egg();
+    PyDict_SetItemString(panda3d_dict, "egg", egg_module);
+    PyDict_SetItemString(sys_modules, "panda3d.egg", egg_module);
+#endif
+
+#ifdef HAVE_BULLET
+    PyObject *bullet_module = PyInit_bullet();
+    PyDict_SetItemString(panda3d_dict, "bullet", bullet_module);
+    PyDict_SetItemString(sys_modules, "panda3d.bullet", bullet_module);
+#endif
+  }
+#endif  // LINK_ALL_STATIC
+
+#ifdef __EMSCRIPTEN__
+  // Default fd capturing doesn't work on emscripten
+  PyRun_SimpleString("import sys; sys.argv.insert(1, '--capture=sys')");
+#endif
+
+  return Py_RunMain();
+
+exception:
+  PyConfig_Clear(&config);
+  if (PyStatus_IsExit(status)) {
+    return status.exitcode;
+  }
+  Py_ExitStatusException(status);
+}

tests/pgraph/test_nodepath.py

@@ -106,7 +106,7 @@ def test_nodepath_transform_composition():
     leg1 = node2.get_transform().compose(node3.get_transform())
     leg2 = node1.get_transform().compose(node3.get_transform())
     relative_transform = leg1.get_inverse().compose(leg2)
-    assert np1.get_transform(np2) == relative_transform
+    assert np1.get_transform(np2).compare_to(relative_transform, True) == 0
 
 
 def test_nodepath_comparison():

tests/physics/test_fall.py

@@ -23,7 +23,7 @@ class FallTest(NodePath):
         #self.setPos(avatarNodePath, Vec3(0))
         #self.setHpr(avatarNodePath, Vec3(0))
 
-        avatarNodePath = base.loader.loadModel("models/misc/smiley")
+        avatarNodePath = base.loader.loadModel("models/misc/smiley.egg")
         assert not avatarNodePath.isEmpty()
 
         # camLL = base.render.find("**/camLL")

tests/physics/test_rotation.py

@@ -24,7 +24,7 @@ class RotationTest(NodePath):
         #self.setPos(avatarNodePath, Vec3(0))
         #self.setHpr(avatarNodePath, Vec3(0))
 
-        avatarNodePath = base.loader.loadModel("models/misc/smiley")
+        avatarNodePath = base.loader.loadModel("models/misc/smiley.egg")
         assert not avatarNodePath.isEmpty()
 
         # camLL = base.render.find("**/camLL")

tests/stdpy/test_threading.py

@@ -9,6 +9,7 @@ def test_threading_error():
         threading.stack_size()
 
 
+@pytest.mark.skipif(sys.platform == "emscripten", reason="No threading")
 def test_threading():
     from collections import deque
 

tests/stdpy/test_threading2.py

@@ -1,8 +1,11 @@
+import sys
 from collections import deque
 from panda3d import core
 from direct.stdpy import threading2
+import pytest
 
 
+@pytest.mark.skipif(sys.platform == "emscripten", reason="No threading")
 def test_threading2():
     class BoundedQueue(threading2._Verbose):
 

tests/test_imports.py

@@ -18,7 +18,7 @@ def test_imports_panda3d():
         if mod.startswith('panda3d.'):
             importlib.import_module(mod)
 
-    if panda3d.__spec__.origin != 'frozen':
+    if hasattr(panda3d, '__file__') and panda3d.__spec__.origin != 'frozen':
         dir = os.path.dirname(panda3d.__file__)
 
         # Iterate over the things in the panda3d package that look like modules.
@@ -41,6 +41,9 @@ def test_imports_panda3d():
 
 def test_imports_panda3d_net():
     from panda3d import core
+    if not hasattr(core, 'ConnectionWriter'):
+        pytest.skip("Build without HAVE_NET")
+
     from panda3d import net
     assert core.ConnectionWriter == net.ConnectionWriter
     assert core.ConnectionWriter.__module__ == 'panda3d.net'

tests/test_tools.py

@@ -5,6 +5,9 @@ import os
 import tempfile
 import panda3d
 
+if sys.platform == "emscripten":
+    pytest.skip(allow_module_level=True)
+
 try:
     panda3d_tools = pytest.importorskip("panda3d_tools")
 except: