mirror of https://github.com/scrapy/scrapy.git
231 lines
6.6 KiB
YAML
231 lines
6.6 KiB
YAML
name: Ubuntu
|
||
on:
|
||
push:
|
||
branches:
|
||
- master
|
||
- '[0-9]+.[0-9]+'
|
||
pull_request:
|
||
|
||
concurrency:
|
||
group: ${{github.workflow}}-${{ github.ref }}
|
||
cancel-in-progress: true
|
||
|
||
jobs:
|
||
# ── Delta pre-check ────────────────────────────────────────────────────────
|
||
# Runs only the tests affected by the diff. The full matrix below is gated
|
||
# behind this job – if Delta fails, the expensive tox runners never start.
|
||
delta:
|
||
name: Delta – affected tests only
|
||
runs-on: ubuntu-latest
|
||
steps:
|
||
- uses: actions/checkout@v6
|
||
with:
|
||
fetch-depth: 0
|
||
|
||
- name: Set up Python 3.13
|
||
uses: actions/setup-python@v6
|
||
with:
|
||
python-version: "3.13"
|
||
|
||
- name: Install system libraries
|
||
run: |
|
||
sudo apt-get update
|
||
sudo apt-get install -y libxml2-dev libxslt-dev
|
||
|
||
- name: Install Scrapy and runtime dependencies
|
||
run: |
|
||
pip install -U pip
|
||
pip install -e .
|
||
|
||
- name: Install test dependencies
|
||
run: |
|
||
pip install \
|
||
"pytest>=8.4.1" \
|
||
"pytest-cov>=7.0.0" \
|
||
pytest-xdist \
|
||
"pytest-twisted>=1.14.3" \
|
||
attrs \
|
||
"coverage>=7.10.6" \
|
||
httpx \
|
||
"pexpect>=4.8.0" \
|
||
pygments \
|
||
"pyftpdlib>=2.0.1" \
|
||
"sybil>=1.3.0" \
|
||
testfixtures
|
||
|
||
- name: Install optional/extra test dependencies
|
||
run: |
|
||
pip install \
|
||
Pillow \
|
||
"Twisted[http2]" \
|
||
boto3 \
|
||
"brotli>=1.2.0" \
|
||
google-cloud-storage \
|
||
"httpx[http2,socks]" \
|
||
ipython \
|
||
robotexclusionrulesparser \
|
||
uvloop \
|
||
zstandard \
|
||
pytest-asyncio
|
||
|
||
- name: Install Delta
|
||
run: pip install deltatest-cli
|
||
|
||
# Write config.toml so Delta connects to the remote mapping DB.
|
||
# IMPORTANT: Delta reads from ~/.delta/config.toml (Path.home()/'.delta'),
|
||
# NOT from the repo-local .delta/ directory. Writing to the wrong path
|
||
# causes Delta to fall back to local-init mode (4131 unmapped tests).
|
||
- name: Configure Delta Cloud
|
||
env:
|
||
DELTA_API_KEY: ${{ secrets.DELTA_API_KEY }}
|
||
DELTA_REPO_ID: ${{ secrets.DELTA_REPO_ID }}
|
||
run: |
|
||
mkdir -p ~/.delta
|
||
{
|
||
echo '[cloud]'
|
||
echo "api_key = \"${DELTA_API_KEY}\""
|
||
echo 'api_url = "https://api.deltatest.dev"'
|
||
echo "repo_id = \"${DELTA_REPO_ID}\""
|
||
echo 'branch = "master"'
|
||
echo 'test_dir = "tests"'
|
||
} > ~/.delta/config.toml
|
||
# Verify structure (values masked)
|
||
sed 's/= ".*"/= "***"/' ~/.delta/config.toml
|
||
|
||
- name: Run Delta (affected tests only)
|
||
run: |
|
||
BASE=${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || 'master' }}
|
||
echo "Diffing against: origin/${BASE}"
|
||
delta run \
|
||
--base-branch "origin/${BASE}" \
|
||
-- \
|
||
--cov-config=pyproject.toml \
|
||
--cov=scrapy \
|
||
--cov-report= \
|
||
--cov-report=term-missing \
|
||
--cov-report=xml \
|
||
--junitxml=delta.junit.xml \
|
||
-o junit_family=legacy \
|
||
--durations=10 \
|
||
scrapy tests \
|
||
--doctest-modules
|
||
|
||
- name: Push mapping to Delta Cloud
|
||
if: always()
|
||
run: delta push
|
||
|
||
- name: Upload coverage report
|
||
if: always()
|
||
uses: codecov/codecov-action@v5
|
||
|
||
- name: Upload test results
|
||
if: always()
|
||
uses: codecov/codecov-action@v5
|
||
with:
|
||
report_type: test_results
|
||
files: delta.junit.xml
|
||
|
||
# ── Full tox matrix ────────────────────────────────────────────────────────
|
||
# Only starts after Delta passes.
|
||
tests:
|
||
needs: delta
|
||
runs-on: ubuntu-latest
|
||
env:
|
||
PYTEST_ADDOPTS: -n auto
|
||
strategy:
|
||
fail-fast: false
|
||
matrix:
|
||
include:
|
||
- python-version: "3.10"
|
||
env:
|
||
TOXENV: py
|
||
- python-version: "3.11"
|
||
env:
|
||
TOXENV: py
|
||
- python-version: "3.12"
|
||
env:
|
||
TOXENV: py
|
||
- python-version: "3.13"
|
||
env:
|
||
TOXENV: py
|
||
- python-version: "3.14"
|
||
env:
|
||
TOXENV: py
|
||
- python-version: "3.14"
|
||
env:
|
||
TOXENV: default-reactor
|
||
- python-version: "3.14"
|
||
env:
|
||
TOXENV: no-reactor
|
||
# pinned due to https://github.com/pypy/pypy/issues/5388
|
||
- python-version: pypy3.11-7.3.20
|
||
env:
|
||
TOXENV: pypy3
|
||
|
||
# min deps
|
||
- python-version: "3.10.19"
|
||
env:
|
||
TOXENV: min
|
||
- python-version: "3.10.19"
|
||
env:
|
||
TOXENV: min-default-reactor
|
||
- python-version: "3.10.19"
|
||
env:
|
||
TOXENV: min-no-reactor
|
||
# pinned due to https://github.com/pypy/pypy/issues/5388
|
||
- python-version: pypy3.11-7.3.20
|
||
env:
|
||
TOXENV: min-pypy3
|
||
- python-version: "3.10.19"
|
||
env:
|
||
TOXENV: min-extra-deps
|
||
- python-version: "3.10.19"
|
||
env:
|
||
TOXENV: min-botocore
|
||
|
||
- python-version: "3.14"
|
||
env:
|
||
TOXENV: extra-deps
|
||
- python-version: "3.14"
|
||
env:
|
||
TOXENV: no-reactor-extra-deps
|
||
# pinned due to https://github.com/pypy/pypy/issues/5388
|
||
- python-version: pypy3.11-7.3.20
|
||
env:
|
||
TOXENV: pypy3-extra-deps
|
||
- python-version: "3.14"
|
||
env:
|
||
TOXENV: botocore
|
||
- python-version: "3.14"
|
||
env:
|
||
TOXENV: mitmproxy
|
||
|
||
steps:
|
||
- uses: actions/checkout@v6
|
||
|
||
- name: Set up Python ${{ matrix.python-version }}
|
||
uses: actions/setup-python@v6
|
||
with:
|
||
python-version: ${{ matrix.python-version }}
|
||
|
||
- name: Install system libraries
|
||
if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'min')
|
||
run: |
|
||
sudo apt-get update
|
||
sudo apt-get install libxml2-dev libxslt-dev
|
||
|
||
- name: Run tests
|
||
env: ${{ matrix.env }}
|
||
run: |
|
||
pip install -U tox
|
||
tox
|
||
|
||
- name: Upload coverage report
|
||
uses: codecov/codecov-action@v5
|
||
|
||
- name: Upload test results
|
||
if: ${{ !cancelled() }}
|
||
uses: codecov/codecov-action@v5
|
||
with:
|
||
report_type: test_results
|