RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into nameless-spiders

This commit is contained in:
Adrian Chaves 2026-07-31 20:17:30 +02:00
parent 44d8d56f7c 24de06bcd3
commit 93882d9823
12 changed files with 182 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.github/dependabot.yml vendored Normal file
View File

@ -0,0 +1,12 @@
version: 2
updates:
- package-ecosystem: github-actions
directory: "/"
schedule:
interval: monthly
groups:
github-actions:
patterns:
- "*"
cooldown:
default-days: 7

6
.github/workflows/auto-close-llm-pr.yml vendored
View File

@ -1,5 +1,7 @@
name: Auto-close LLM PRs
on:
# The workflow only reads the pull request body through the API, it never
# checks out or runs pull request code, so pull_request_target is safe here.
on: # zizmor: ignore[dangerous-triggers]
pull_request_target:
types: [opened]
permissions:
@ -11,7 +13,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check PR body and close if LLM-written
uses: actions/github-script@v6
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |

32
.github/workflows/checks.yml vendored
View File

@ -1,4 +1,8 @@
name: Checks
permissions:
contents: read
on:
push:
branches:
@ -13,6 +17,10 @@ concurrency:
jobs:
checks:
runs-on: ubuntu-latest
env:
# Make uv use the interpreter that actions/setup-python installed instead
# of downloading one of its own.
UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
@ -38,21 +46,31 @@ jobs:
TOXENV: twinecheck
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
- name: Set up uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
cache-dependency-glob: |
docs/requirements.txt
pyproject.toml
tox.ini
- name: Run check
env: ${{ matrix.env }}
run: |
pip install -U tox
tox
run: uvx --with tox-uv tox
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: pre-commit/action@v3.0.1
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1

24
.github/workflows/codspeed.yml vendored
View File

@ -22,24 +22,38 @@ permissions: {}
jobs:
benchmark:
runs-on: ubuntu-latest
env:
# Make uv use the interpreter that actions/setup-python installed
# instead of downloading one of its own.
UV_PYTHON_PREFERENCE: only-system
permissions:
contents: read
id-token: write # OIDC authentication with CodSpeed
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python 3.14
uses: actions/setup-python@v6
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: '3.14'
- name: Set up uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
cache-dependency-glob: |
pyproject.toml
tox.ini
- name: Install dependencies
# tox must stay on PATH for the CodSpeed action to invoke it.
run: |
pip install --upgrade pip
pip install --upgrade tox
uv tool install --with tox-uv tox
tox -n -e benchmark
- name: Run benchmarks
uses: CodSpeedHQ/action@v4
uses: CodSpeedHQ/action@f22792bfac16f3e14eb9fbea76f4a48e9cc22b93 # v4.19.1
with:
mode: simulation
run: tox -e benchmark

35
.github/workflows/publish.yml vendored
View File

@ -1,4 +1,8 @@
name: Publish
permissions:
contents: read
on:
push:
tags:
@ -9,8 +13,28 @@ concurrency:
cancel-in-progress: true
jobs:
build:
name: Build distribution
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.14"
- run: |
python -m pip install --upgrade build
python -m build
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: python-package-distributions
path: dist/
publish:
name: Upload release to PyPI
needs:
- build
runs-on: ubuntu-latest
environment:
name: pypi
@ -18,12 +42,9 @@ jobs:
permissions:
id-token: write
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
python-version: "3.14"
- run: |
python -m pip install --upgrade build
python -m build
name: python-package-distributions
path: dist/
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2

35
.github/workflows/tests-macos.yml vendored
View File

@ -1,4 +1,8 @@
name: macOS
permissions:
contents: read
on:
push:
branches:
@ -15,6 +19,9 @@ jobs:
runs-on: macos-latest
env:
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
# Make uv use the interpreter that actions/setup-python installed instead
# of downloading one of its own.
UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
@ -31,25 +38,39 @@ jobs:
TOXENV: no-reactor
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
- name: Set up uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
cache-dependency-glob: |
pyproject.toml
tox.ini
- name: Install mitmproxy
env:
# mitmproxy needs a newer Python than the oldest matrix entries, so let
# uv download one where no system interpreter is new enough.
UV_PYTHON_PREFERENCE: system
run: uv tool install mitmproxy
- name: Run tests
env: ${{ matrix.env }}
run: |
pip install -U tox
tox
run: uvx --with tox-uv tox
- name: Upload coverage report
if: ${{ matrix.coverage }}
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
- name: Upload test results
if: ${{ !cancelled() }}
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
report_type: test_results

36
.github/workflows/tests-ubuntu.yml vendored
View File

@ -1,4 +1,8 @@
name: Ubuntu
permissions:
contents: read
on:
push:
branches:
@ -15,6 +19,9 @@ jobs:
runs-on: ubuntu-latest
env:
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
# Make uv use the interpreter that actions/setup-python installed instead
# of downloading one of its own.
UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
@ -92,10 +99,12 @@ jobs:
coverage: true
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
@ -105,21 +114,32 @@ jobs:
sudo apt-get update
sudo apt-get install libxml2-dev libxslt-dev
- name: Set up uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
cache-dependency-glob: |
pyproject.toml
tox.ini
- name: Install mitmproxy
run: pipx install mitmproxy
env:
# mitmproxy needs a newer Python than the oldest matrix entries, so let
# uv download one where no system interpreter is new enough.
UV_PYTHON_PREFERENCE: system
# mitmproxy has no PyPy wheels, so run it on CPython regardless of the
# interpreter under test.
run: uv tool install --python cpython mitmproxy
- name: Run tests
env: ${{ matrix.env }}
run: |
pip install -U tox
tox
run: uvx --with tox-uv tox
- name: Upload coverage report
if: ${{ matrix.coverage }}
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
- name: Upload test results
if: ${{ !cancelled() }}
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
report_type: test_results

35
.github/workflows/tests-windows.yml vendored
View File

@ -1,4 +1,8 @@
name: Windows
permissions:
contents: read
on:
push:
branches:
@ -15,6 +19,9 @@ jobs:
runs-on: windows-latest
env:
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
# Make uv use the interpreter that actions/setup-python installed instead
# of downloading one of its own.
UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
@ -55,25 +62,39 @@ jobs:
TOXENV: extra-deps
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
- name: Set up uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
cache-dependency-glob: |
pyproject.toml
tox.ini
- name: Install mitmproxy
env:
# mitmproxy needs a newer Python than the oldest matrix entries, so let
# uv download one where no system interpreter is new enough.
UV_PYTHON_PREFERENCE: system
run: uv tool install mitmproxy
- name: Run tests
env: ${{ matrix.env }}
run: |
pip install -U tox
tox
run: uvx --with tox-uv tox
- name: Upload coverage report
if: ${{ matrix.coverage }}
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
- name: Upload test results
if: ${{ !cancelled() }}
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
report_type: test_results

7
.pre-commit-config.yaml
View File

@ -27,6 +27,11 @@ repos:
hooks:
- id: sphinx-lint
- repo: https://github.com/scrapy/sphinx-scrapy
rev: 0.8.9
rev: 0.8.10
hooks:
- id: sphinx-scrapy
- repo: https://github.com/zizmorcore/zizmor-pre-commit
rev: v1.28.0
hooks:
- id: zizmor
args: [--no-progress, --fix]

2
docs/requirements.in
View File

@ -5,4 +5,4 @@ sphinx
sphinx-notfound-page
sphinx-rtd-theme
sphinx-rtd-dark-mode
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.9
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10

2
docs/requirements.txt
View File

@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0
# via
# -r docs/requirements.in
# sphinx-rtd-dark-mode
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@912ed0507405e16ac60a47dd08195a1cd0ced984
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@fe176adc1a8577601bc3fa39b590ebed71a7e9b8
# via -r docs/requirements.in
sphinx-sitemap==2.9.0
# via sphinx-scrapy

3
tox.ini
View File

@ -5,7 +5,8 @@
[tox]
requires =
sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.9
sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10
tox-uv
envlist =
pre-commit
pylint