From 16587d9abfe40b8d354bbc07142f9a708aa92612 Mon Sep 17 00:00:00 2001 From: aliowka Date: Wed, 17 Jun 2026 01:12:55 +0300 Subject: [PATCH] feat: add delta benchmark script and CI integration for testing efficiency monitoring --- .github/workflows/deltatest.yaml | 43 ----------- .github/workflows/deltatest.yml | 118 +++++++++++++++++++++++++++++ .github/workflows/tests-ubuntu.yml | 100 ++++++++++++++++++++++++ 3 files changed, 218 insertions(+), 43 deletions(-) delete mode 100644 .github/workflows/deltatest.yaml create mode 100644 .github/workflows/deltatest.yml diff --git a/.github/workflows/deltatest.yaml b/.github/workflows/deltatest.yaml deleted file mode 100644 index f7b2b0386..000000000 --- a/.github/workflows/deltatest.yaml +++ /dev/null @@ -1,43 +0,0 @@ -name: DeltaTest - -on: - push: - branches: master - pull_request: - branches: master - -permissions: - contents: read - checks: write - -jobs: - deltatest: - name: DeltaTest Runner - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # DeltaTest requires the full Git history to calculate diffs and find affected tests - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - cache: 'pip' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - if [ -f docs/requirements.txt ]; then pip install -r docs/requirements.txt || echo "docs/requirements install failed, continuing"; fi - pip install --no-cache-dir pytest twisted deltatest-cli - pip install -e . - - - name: Run Affected Tests - env: - DELTA_API_KEY: ${{ secrets.DELTA_API_KEY }} - DELTA_REPO_ID: ${{ secrets.DELTA_REPO_ID }} - run: | - delta run \ No newline at end of file diff --git a/.github/workflows/deltatest.yml b/.github/workflows/deltatest.yml new file mode 100644 index 000000000..9f9ef27da --- /dev/null +++ b/.github/workflows/deltatest.yml @@ -0,0 +1,118 @@ +name: Delta Smart Tests + +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + delta: + name: Delta – affected tests only + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + with: + # Fetch full history so Delta can diff against the base branch + fetch-depth: 0 + + - name: Set up Python 3.13 + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + # ── System libraries (same condition as tests-ubuntu.yml) ──────────── + - name: Install system libraries + run: | + sudo apt-get update + sudo apt-get install -y libxml2-dev libxslt-dev + + # ── Scrapy + all runtime deps (from pyproject.toml [project].dependencies) ── + - name: Install Scrapy and runtime dependencies + run: | + pip install -U pip + pip install -e . + + # ── Core test dependencies (from tox.ini [test-requirements]) ──────── + - name: Install test dependencies + run: | + pip install \ + "pytest>=8.4.1" \ + "pytest-cov>=7.0.0" \ + pytest-xdist \ + pytest-twisted>=1.14.3 \ + attrs \ + "coverage>=7.10.6" \ + httpx \ + "pexpect>=4.8.0" \ + pygments \ + "pyftpdlib>=2.0.1" \ + "sybil>=1.3.0" \ + testfixtures + + # ── Extra optional deps (mirrors tox.ini [testenv:extra-deps]) ─────── + # Included so Delta can map tests that import these packages. + - name: Install optional/extra test dependencies + run: | + pip install \ + Pillow \ + "Twisted[http2]" \ + boto3 \ + "brotli>=1.2.0" \ + google-cloud-storage \ + "httpx[http2,socks]" \ + ipython \ + robotexclusionrulesparser \ + uvloop \ + zstandard \ + pytest-asyncio + + # ── Delta CLI ───────────────────────────────────────────────────────── + - name: Install Delta + run: pip install deltatest-cli + + # ── Run only tests affected by this diff ───────────────────────────── + - name: Run Delta (affected tests only) + env: + DELTA_API_KEY: ${{ secrets.DELTA_API_KEY }} + run: | + BASE=${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || 'master' }} + echo "Diffing against: origin/${BASE}" + delta run \ + --base-branch "origin/${BASE}" \ + -- \ + --cov-config=pyproject.toml \ + --cov=scrapy \ + --cov-report= \ + --cov-report=term-missing \ + --cov-report=xml \ + --junitxml=delta.junit.xml \ + -o junit_family=legacy \ + --durations=10 \ + scrapy tests \ + --doctest-modules + + # ── Push updated mapping back to Delta Cloud ────────────────────────── + - name: Push mapping to Delta Cloud + if: always() + env: + DELTA_API_KEY: ${{ secrets.DELTA_API_KEY }} + run: delta push + + # ── Upload results ──────────────────────────────────────────────────── + - name: Upload coverage report + if: always() + uses: codecov/codecov-action@v5 + + - name: Upload test results + if: always() + uses: codecov/codecov-action@v5 + with: + report_type: test_results + files: delta.junit.xml diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index f51dd9799..2cd4beb95 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -11,7 +11,107 @@ concurrency: cancel-in-progress: true jobs: + # ── Delta pre-check ──────────────────────────────────────────────────────── + # Runs only the tests affected by the diff. The full matrix below is gated + # behind this job – if Delta fails, the expensive tox runners never start. + delta: + name: Delta – affected tests only + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Set up Python 3.13 + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Install system libraries + run: | + sudo apt-get update + sudo apt-get install -y libxml2-dev libxslt-dev + + - name: Install Scrapy and runtime dependencies + run: | + pip install -U pip + pip install -e . + + - name: Install test dependencies + run: | + pip install \ + "pytest>=8.4.1" \ + "pytest-cov>=7.0.0" \ + pytest-xdist \ + "pytest-twisted>=1.14.3" \ + attrs \ + "coverage>=7.10.6" \ + httpx \ + "pexpect>=4.8.0" \ + pygments \ + "pyftpdlib>=2.0.1" \ + "sybil>=1.3.0" \ + testfixtures + + - name: Install optional/extra test dependencies + run: | + pip install \ + Pillow \ + "Twisted[http2]" \ + boto3 \ + "brotli>=1.2.0" \ + google-cloud-storage \ + "httpx[http2,socks]" \ + ipython \ + robotexclusionrulesparser \ + uvloop \ + zstandard \ + pytest-asyncio + + - name: Install Delta + run: pip install deltatest-cli + + - name: Run Delta (affected tests only) + env: + DELTA_API_KEY: ${{ secrets.DELTA_API_KEY }} + run: | + BASE=${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || 'master' }} + echo "Diffing against: origin/${BASE}" + delta run \ + --base-branch "origin/${BASE}" \ + -- \ + --cov-config=pyproject.toml \ + --cov=scrapy \ + --cov-report= \ + --cov-report=term-missing \ + --cov-report=xml \ + --junitxml=delta.junit.xml \ + -o junit_family=legacy \ + --durations=10 \ + scrapy tests \ + --doctest-modules + + - name: Push mapping to Delta Cloud + if: always() + env: + DELTA_API_KEY: ${{ secrets.DELTA_API_KEY }} + run: delta push + + - name: Upload coverage report + if: always() + uses: codecov/codecov-action@v5 + + - name: Upload test results + if: always() + uses: codecov/codecov-action@v5 + with: + report_type: test_results + files: delta.junit.xml + + # ── Full tox matrix ──────────────────────────────────────────────────────── + # Only starts after Delta passes. tests: + needs: delta runs-on: ubuntu-latest env: PYTEST_ADDOPTS: -n auto