From 3eab54374cab573f30983a404b71ff323411d47c Mon Sep 17 00:00:00 2001 From: doria <93405247+dr-frmr@users.noreply.github.com> Date: Fri, 6 Feb 2026 21:10:34 -0500 Subject: [PATCH] chore: parallelize tests for speed (#374) * chore: parallelize tests for speed * fix: truncate all tables after each test --------- Co-authored-by: Vineeth Voruganti <13438633+VVoruganti@users.noreply.github.com> --- pyproject.toml | 3 ++- tests/conftest.py | 63 +++++++++++++++++++++++++++++++++++------------ uv.lock | 24 ++++++++++++++++++ 3 files changed, 73 insertions(+), 17 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 660fc11f..2bba0d65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ dev = [ "fakeredis>=2.32.0", "scipy>=1.15.3", "boto3>=1.42.5", + "pytest-xdist>=3.8.0", ] [tool.uv.workspace] @@ -90,7 +91,7 @@ extend-immutable-calls = ["fastapi.Depends"] [tool.pytest.ini_options] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "session" -addopts = "--strict-markers --cov=src/ --cov=sdks/python/src/honcho --cov-report=term-missing --ignore=tests/alembic" +addopts = "--strict-markers -n auto --ignore=tests/alembic" testpaths = ["tests"] pythonpath = ["src"] diff --git a/tests/conftest.py b/tests/conftest.py index 38f9f216..46cb67ca 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -66,8 +66,14 @@ DB_URI = ( or "postgresql+psycopg://postgres:postgres@localhost:5432/postgres" ) CONNECTION_URI = make_url(DB_URI) -TEST_DB_URL = CONNECTION_URI.set(database="test_db") -DEFAULT_DB_URL = str(CONNECTION_URI.set(database="postgres")) + + +def _get_test_db_url(worker_id: str) -> URL: + """Get a worker-specific test database URL for pytest-xdist parallelism.""" + + db_name = "test_db" if worker_id == "master" else f"test_db_{worker_id}" + return CONNECTION_URI.set(database=db_name) + # Test API authorization - no longer needed as module-level constants # We'll use settings.AUTH directly where needed @@ -127,10 +133,29 @@ async def setup_test_database(db_url: URL): return engine +async def _truncate_all_tables(engine: AsyncEngine) -> None: + """Remove all data from every mapped table while resetting identities.""" + + table_names: list[str] = [] + for table in Base.metadata.sorted_tables: + if table.schema: + table_names.append(f'"{table.schema}"."{table.name}"') + else: + table_names.append(f'"{table.name}"') + + if not table_names: + return + + joined_names = ", ".join(table_names) + async with engine.begin() as conn: + await conn.execute(text(f"TRUNCATE {joined_names} RESTART IDENTITY CASCADE")) + + @pytest_asyncio.fixture(scope="session") -async def db_engine(): - create_test_database(TEST_DB_URL) - engine = await setup_test_database(TEST_DB_URL) +async def db_engine(worker_id: str): + test_db_url = _get_test_db_url(worker_id) + create_test_database(test_db_url) + engine = await setup_test_database(test_db_url) # Force the schema to 'public' for tests # Save the original schema to restore later @@ -147,25 +172,31 @@ async def db_engine(): # Then create all tables with current models await conn.run_sync(Base.metadata.create_all) - yield engine + try: + yield engine + finally: + await engine.dispose() - await engine.dispose() + # Restore original schema + Base.metadata.schema = original_schema + for table in Base.metadata.tables.values(): + table.schema = original_schema - # Restore original schema - Base.metadata.schema = original_schema - for table in Base.metadata.tables.values(): - table.schema = original_schema - - drop_database(TEST_DB_URL) + drop_database(test_db_url) @pytest_asyncio.fixture(scope="function") async def db_session(db_engine: AsyncEngine): """Create a database session for the scope of a single test function""" Session = async_sessionmaker(bind=db_engine, expire_on_commit=False) - async with Session() as session: - yield session - await session.rollback() + try: + async with Session() as session: + try: + yield session + finally: + await session.rollback() + finally: + await _truncate_all_tables(db_engine) @pytest_asyncio.fixture(scope="session") diff --git a/uv.lock b/uv.lock index 634827d0..8817b2e7 100644 --- a/uv.lock +++ b/uv.lock @@ -681,6 +681,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "fakeredis" version = "2.32.0" @@ -1075,6 +1084,7 @@ dev = [ { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, + { name = "pytest-xdist" }, { name = "ruff" }, { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "scipy", version = "1.16.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, @@ -1129,6 +1139,7 @@ dev = [ { name = "pytest", specifier = ">=8.2.2" }, { name = "pytest-asyncio", specifier = ">=0.23.7" }, { name = "pytest-cov", specifier = ">=6.2.1" }, + { name = "pytest-xdist", specifier = ">=3.8.0" }, { name = "ruff", specifier = ">=0.11.2" }, { name = "scipy", specifier = ">=1.15.3" }, { name = "sqlalchemy-utils", specifier = ">=0.41.2" }, @@ -2812,6 +2823,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/16/4ea354101abb1287856baa4af2732be351c7bee728065aed451b678153fd/pytest_cov-6.2.1-py3-none-any.whl", hash = "sha256:f5bc4c23f42f1cdd23c70b1dab1bbaef4fc505ba950d53e0081d0730dd7e86d5", size = 24644, upload-time = "2025-06-12T10:47:45.932Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0"