chore: parallelize tests for speed (#374)
* chore: parallelize tests for speed * fix: truncate all tables after each test --------- Co-authored-by: Vineeth Voruganti <13438633+VVoruganti@users.noreply.github.com>
This commit is contained in:
doria
GitHub
parent
e93e57175e
commit
3eab54374c
|
|
@ -56,6 +56,7 @@ dev = [
|
|||
"fakeredis>=2.32.0",
|
||||
"scipy>=1.15.3",
|
||||
"boto3>=1.42.5",
|
||||
"pytest-xdist>=3.8.0",
|
||||
]
|
||||
|
||||
[tool.uv.workspace]
|
||||
|
|
@ -90,7 +91,7 @@ extend-immutable-calls = ["fastapi.Depends"]
|
|||
[tool.pytest.ini_options]
|
||||
asyncio_mode = "auto"
|
||||
asyncio_default_fixture_loop_scope = "session"
|
||||
addopts = "--strict-markers --cov=src/ --cov=sdks/python/src/honcho --cov-report=term-missing --ignore=tests/alembic"
|
||||
addopts = "--strict-markers -n auto --ignore=tests/alembic"
|
||||
testpaths = ["tests"]
|
||||
pythonpath = ["src"]
|
||||
|
||||
|
|
|
|||
|
|
@ -66,8 +66,14 @@ DB_URI = (
|
|||
or "postgresql+psycopg://postgres:postgres@localhost:5432/postgres"
|
||||
)
|
||||
CONNECTION_URI = make_url(DB_URI)
|
||||
TEST_DB_URL = CONNECTION_URI.set(database="test_db")
|
||||
DEFAULT_DB_URL = str(CONNECTION_URI.set(database="postgres"))
|
||||
|
||||
|
||||
def _get_test_db_url(worker_id: str) -> URL:
|
||||
"""Get a worker-specific test database URL for pytest-xdist parallelism."""
|
||||
|
||||
db_name = "test_db" if worker_id == "master" else f"test_db_{worker_id}"
|
||||
return CONNECTION_URI.set(database=db_name)
|
||||
|
||||
|
||||
# Test API authorization - no longer needed as module-level constants
|
||||
# We'll use settings.AUTH directly where needed
|
||||
|
|
@ -127,10 +133,29 @@ async def setup_test_database(db_url: URL):
|
|||
return engine
|
||||
|
||||
|
||||
async def _truncate_all_tables(engine: AsyncEngine) -> None:
|
||||
"""Remove all data from every mapped table while resetting identities."""
|
||||
|
||||
table_names: list[str] = []
|
||||
for table in Base.metadata.sorted_tables:
|
||||
if table.schema:
|
||||
table_names.append(f'"{table.schema}"."{table.name}"')
|
||||
else:
|
||||
table_names.append(f'"{table.name}"')
|
||||
|
||||
if not table_names:
|
||||
return
|
||||
|
||||
joined_names = ", ".join(table_names)
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(text(f"TRUNCATE {joined_names} RESTART IDENTITY CASCADE"))
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope="session")
|
||||
async def db_engine():
|
||||
create_test_database(TEST_DB_URL)
|
||||
engine = await setup_test_database(TEST_DB_URL)
|
||||
async def db_engine(worker_id: str):
|
||||
test_db_url = _get_test_db_url(worker_id)
|
||||
create_test_database(test_db_url)
|
||||
engine = await setup_test_database(test_db_url)
|
||||
|
||||
# Force the schema to 'public' for tests
|
||||
# Save the original schema to restore later
|
||||
|
|
@ -147,25 +172,31 @@ async def db_engine():
|
|||
# Then create all tables with current models
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
yield engine
|
||||
try:
|
||||
yield engine
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
await engine.dispose()
|
||||
# Restore original schema
|
||||
Base.metadata.schema = original_schema
|
||||
for table in Base.metadata.tables.values():
|
||||
table.schema = original_schema
|
||||
|
||||
# Restore original schema
|
||||
Base.metadata.schema = original_schema
|
||||
for table in Base.metadata.tables.values():
|
||||
table.schema = original_schema
|
||||
|
||||
drop_database(TEST_DB_URL)
|
||||
drop_database(test_db_url)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope="function")
|
||||
async def db_session(db_engine: AsyncEngine):
|
||||
"""Create a database session for the scope of a single test function"""
|
||||
Session = async_sessionmaker(bind=db_engine, expire_on_commit=False)
|
||||
async with Session() as session:
|
||||
yield session
|
||||
await session.rollback()
|
||||
try:
|
||||
async with Session() as session:
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
await session.rollback()
|
||||
finally:
|
||||
await _truncate_all_tables(db_engine)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope="session")
|
||||
|
|
|
|||
24
uv.lock
24
uv.lock
|
|
@ -681,6 +681,15 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "execnet"
|
||||
version = "2.1.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fakeredis"
|
||||
version = "2.32.0"
|
||||
|
|
@ -1075,6 +1084,7 @@ dev = [
|
|||
{ name = "pytest" },
|
||||
{ name = "pytest-asyncio" },
|
||||
{ name = "pytest-cov" },
|
||||
{ name = "pytest-xdist" },
|
||||
{ name = "ruff" },
|
||||
{ name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
|
||||
{ name = "scipy", version = "1.16.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
|
||||
|
|
@ -1129,6 +1139,7 @@ dev = [
|
|||
{ name = "pytest", specifier = ">=8.2.2" },
|
||||
{ name = "pytest-asyncio", specifier = ">=0.23.7" },
|
||||
{ name = "pytest-cov", specifier = ">=6.2.1" },
|
||||
{ name = "pytest-xdist", specifier = ">=3.8.0" },
|
||||
{ name = "ruff", specifier = ">=0.11.2" },
|
||||
{ name = "scipy", specifier = ">=1.15.3" },
|
||||
{ name = "sqlalchemy-utils", specifier = ">=0.41.2" },
|
||||
|
|
@ -2812,6 +2823,19 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/bc/16/4ea354101abb1287856baa4af2732be351c7bee728065aed451b678153fd/pytest_cov-6.2.1-py3-none-any.whl", hash = "sha256:f5bc4c23f42f1cdd23c70b1dab1bbaef4fc505ba950d53e0081d0730dd7e86d5", size = 24644, upload-time = "2025-06-12T10:47:45.932Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pytest-xdist"
|
||||
version = "3.8.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "execnet" },
|
||||
{ name = "pytest" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "python-dateutil"
|
||||
version = "2.9.0.post0"
|
||||
|
|
|
|||
Loading…
Reference in New Issue