diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 36d9139cc..6035365c3 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -11,124 +11,7 @@ concurrency: cancel-in-progress: true jobs: - # ── Delta pre-check ──────────────────────────────────────────────────────── - # Runs only the tests affected by the diff. The full matrix below is gated - # behind this job – if Delta fails, the expensive tox runners never start. - delta: - name: Delta – affected tests only - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Set up Python 3.13 - uses: actions/setup-python@v6 - with: - python-version: "3.13" - - - name: Install system libraries - run: | - sudo apt-get update - sudo apt-get install -y libxml2-dev libxslt-dev - - - name: Install Scrapy and runtime dependencies - run: | - pip install -U pip - pip install -e . - - - name: Install test dependencies - run: | - pip install \ - "pytest>=8.4.1" \ - "pytest-cov>=7.0.0" \ - pytest-xdist \ - "pytest-twisted>=1.14.3" \ - attrs \ - "coverage>=7.10.6" \ - httpx \ - "pexpect>=4.8.0" \ - pygments \ - "pyftpdlib>=2.0.1" \ - "sybil>=1.3.0" \ - testfixtures - - - name: Install optional/extra test dependencies - run: | - pip install \ - Pillow \ - "Twisted[http2]" \ - boto3 \ - "brotli>=1.2.0" \ - google-cloud-storage \ - "httpx[http2,socks]" \ - ipython \ - robotexclusionrulesparser \ - uvloop \ - zstandard \ - pytest-asyncio - - - name: Install Delta - run: pip install deltatest-cli - - # Write config.toml so Delta connects to the remote mapping DB. - # IMPORTANT: Delta reads from ~/.delta/config.toml (Path.home()/'.delta'), - # NOT from the repo-local .delta/ directory. Writing to the wrong path - # causes Delta to fall back to local-init mode (4131 unmapped tests). - - name: Configure Delta Cloud - env: - DELTA_API_KEY: ${{ secrets.DELTA_API_KEY }} - DELTA_REPO_ID: ${{ secrets.DELTA_REPO_ID }} - run: | - mkdir -p ~/.delta - { - echo '[cloud]' - echo "api_key = \"${DELTA_API_KEY}\"" - echo 'api_url = "https://api.deltatest.dev"' - echo "repo_id = \"${DELTA_REPO_ID}\"" - echo 'branch = "master"' - echo 'test_dir = "tests"' - } > ~/.delta/config.toml - # Verify structure (values masked) - sed 's/= ".*"/= "***"/' ~/.delta/config.toml - - - name: Run Delta (affected tests only) - run: | - BASE=${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || 'master' }} - echo "Diffing against: origin/${BASE}" - delta run \ - --base-branch "origin/${BASE}" \ - -- \ - --cov-config=pyproject.toml \ - --cov=scrapy \ - --cov-report= \ - --cov-report=term-missing \ - --cov-report=xml \ - --junitxml=delta.junit.xml \ - -o junit_family=legacy \ - --durations=10 \ - scrapy tests \ - --doctest-modules - - - name: Push mapping to Delta Cloud - if: always() - run: delta push - - - name: Upload coverage report - if: always() - uses: codecov/codecov-action@v5 - - - name: Upload test results - if: always() - uses: codecov/codecov-action@v5 - with: - report_type: test_results - files: delta.junit.xml - - # ── Full tox matrix ──────────────────────────────────────────────────────── - # Only starts after Delta passes. tests: - needs: delta runs-on: ubuntu-latest env: PYTEST_ADDOPTS: -n auto @@ -217,6 +100,21 @@ jobs: sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev + # Write ~/.delta/config.toml so tox-spawned delta processes connect to + # the remote mapping DB (Delta reads from Path.home()/'.delta/config.toml'). + - name: Configure Delta Cloud + run: | + mkdir -p ~/.delta + { + echo '[cloud]' + echo "api_key = \"${DELTA_API_KEY}\"" + echo 'api_url = "https://api.deltatest.dev"' + echo "repo_id = \"${DELTA_REPO_ID}\"" + echo 'branch = "master"' + echo 'test_dir = "tests"' + } > ~/.delta/config.toml + sed 's/= ".*"/= "***"/' ~/.delta/config.toml + - name: Run tests env: ${{ matrix.env }} run: | diff --git a/tox.ini b/tox.ini index 23d053304..a7bdab16e 100644 --- a/tox.ini +++ b/tox.ini @@ -151,10 +151,19 @@ deps = w3lib==1.17.0 zope.interface==5.1.0 {[test-requirements]deps} + deltatest-cli +passenv = + PYTEST_ADDOPTS + S3_TEST_FILE_URI + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN + DELTA_* + GITHUB_* setenv = _SCRAPY_MIN=true commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=min.junit.xml -o junit_family=legacy --durations=10 scrapy tests} + delta run -- {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=min.junit.xml -o junit_family=legacy --durations=10 scrapy tests} [testenv:min] basepython = {[min]basepython} @@ -240,7 +249,7 @@ setenv = basepython = pypy3 commands = ; not enabling coverage as it significantly increases the run time - pytest {posargs:--durations=10 scrapy tests} + delta run -- {posargs:--durations=10 scrapy tests} [testenv:pypy3-extra-deps] basepython = pypy3 @@ -271,9 +280,14 @@ deps = # rely on that encoding. w3lib==2.1.0 zope.interface==5.1.0 + deltatest-cli +passenv = + PYTEST_ADDOPTS + DELTA_* + GITHUB_* commands = ; disabling coverage - pytest {posargs:--durations=10 scrapy tests} + delta run -- {posargs:--durations=10 scrapy tests} setenv = {[min]setenv} @@ -308,7 +322,7 @@ deps = {[testenv]deps} botocore>=1.13.45 commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_botocore + delta run -- {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_botocore [testenv:min-botocore] basepython = {[min]basepython} @@ -320,7 +334,7 @@ deps = setenv = {[min]setenv} commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore + delta run -- {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore # Run proxy tests that use mitmproxy in a separate env to avoid installing @@ -334,4 +348,4 @@ deps = mitmproxy; implementation_name != "pypy" httpx[http2,socks] commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=mitmproxy.junit.xml -o junit_family=legacy} -m requires_mitmproxy + delta run -- {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=mitmproxy.junit.xml -o junit_family=legacy} -m requires_mitmproxy