diff --git a/.bandit.yml b/.bandit.yml deleted file mode 100644 index 2aae8a0aa..000000000 --- a/.bandit.yml +++ /dev/null @@ -1,21 +0,0 @@ -skips: -- B101 -- B113 # https://github.com/PyCQA/bandit/issues/1010 -- B105 -- B301 -- B303 -- B306 -- B307 -- B311 -- B320 -- B321 -- B324 -- B402 # https://github.com/scrapy/scrapy/issues/4180 -- B403 -- B404 -- B406 -- B410 -- B503 -- B603 -- B605 -exclude_dirs: ['tests'] diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index f76bf783d..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[bumpversion] -current_version = 2.11.0 -commit = True -tag = True -tag_name = {new_version} - -[bumpversion:file:scrapy/VERSION] diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index bbe5cebc3..000000000 --- a/.coveragerc +++ /dev/null @@ -1,7 +0,0 @@ -[run] -branch = true -; https://github.com/pytest-dev/pytest-cov/issues/369#issuecomment-1053702088 -disable_warnings=include-ignored -include = scrapy/* -omit = - tests/* diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 544d72956..000000000 --- a/.flake8 +++ /dev/null @@ -1,22 +0,0 @@ -[flake8] - -max-line-length = 119 -ignore = W503, E203 - -exclude = - docs/conf.py - -per-file-ignores = -# Exclude files that are meant to provide top-level imports -# E402: Module level import not at top of file -# F401: Module imported but unused - scrapy/__init__.py:E402 - scrapy/core/downloader/handlers/http.py:F401 - scrapy/http/__init__.py:F401 - scrapy/linkextractors/__init__.py:E402,F401 - scrapy/selector/__init__.py:F401 - scrapy/spiders/__init__.py:E402,F401 - - # Issues pending a review: - scrapy/utils/url.py:F403,F405 - tests/test_loader.py:E741 diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index dbcebfa0a..1f062eef2 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,7 +1,7 @@ # .git-blame-ignore-revs # adding black formatter to all the code e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d -# re applying black to the code with default line length +# reapplying black to the code with default line length 303f0a70fcf8067adf0a909c2096a5009162383a -# reaplying black again and removing line length on pre-commit black config -c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962 \ No newline at end of file +# reapplying black again and removing line length on pre-commit black config +c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..98a74f8ce --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,31 @@ + diff --git a/.github/workflows/auto-close-llm-pr.yml b/.github/workflows/auto-close-llm-pr.yml new file mode 100644 index 000000000..160b39488 --- /dev/null +++ b/.github/workflows/auto-close-llm-pr.yml @@ -0,0 +1,48 @@ +name: Auto-close LLM PRs +on: + pull_request_target: + types: [opened] +permissions: + contents: read + pull-requests: write +jobs: + close-llm-pr: + name: Close PR if marked as LLM-written + runs-on: ubuntu-latest + steps: + - name: Check PR body and close if LLM-written + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const marker = "This PR was written entirely using an LLM"; + const { owner, repo } = context.repo; + const prNumber = context.payload.pull_request && context.payload.pull_request.number; + if (!prNumber) { + console.log('No pull request number found in context; exiting.'); + return; + } + const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: prNumber }); + const body = pr.body || ""; + if (body.includes(marker)) { + if (pr.state === 'closed') { + console.log(`PR #${prNumber} already closed.`); + return; + } + await github.rest.issues.addLabels({ + owner, + repo, + issue_number: prNumber, + labels: ['spam'] + }); + await github.rest.issues.createComment({ + owner, + repo, + issue_number: prNumber, + body: "Closing this PR because it contains the disclosure: \"This PR was written entirely using an LLM\"." + }); + await github.rest.pulls.update({ owner, repo, pull_number: prNumber, state: 'closed' }); + console.log(`Closed PR #${prNumber} because marker was found.`); + } else { + console.log(`Marker not found in PR #${prNumber}; nothing to do.`); + } diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d6fc0f6c5..49ea3277a 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,5 +1,10 @@ name: Checks -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} @@ -12,24 +17,31 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.12" + - python-version: "3.14" env: TOXENV: pylint - - python-version: 3.8 + - python-version: "3.10" env: TOXENV: typing - - python-version: "3.11" # Keep in sync with .readthedocs.yml + - python-version: "3.10" + env: + TOXENV: typing-tests + # Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version. + - python-version: "3.14" env: TOXENV: docs - - python-version: "3.12" + - python-version: "3.13" + env: + TOXENV: docs-tests + - python-version: "3.14" env: TOXENV: twinecheck steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} @@ -42,5 +54,5 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: pre-commit/action@v3.0.0 + - uses: actions/checkout@v6 + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index affaa32a5..7779bbb6b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,16 +10,20 @@ concurrency: jobs: publish: + name: Upload release to PyPI runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/Scrapy + permissions: + id-token: write steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 with: - python-version: 3.12 - - run: | - pip install --upgrade build twine + python-version: "3.14" + - run: | + python -m pip install --upgrade build python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.6.4 - with: - password: ${{ secrets.PYPI_TOKEN }} + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 252176464..0409b3ef2 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,5 +1,10 @@ name: macOS -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} @@ -7,24 +12,39 @@ concurrency: jobs: tests: - runs-on: macos-11 + runs-on: macos-latest + env: + PYTEST_ADDOPTS: -n auto strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + env: + - TOXENV: py + include: + - python-version: '3.14' + env: + TOXENV: no-reactor steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Run tests + env: ${{ matrix.env }} run: | pip install -U tox - tox -e py + tox - name: Upload coverage report - run: bash <(curl -s https://codecov.io/bash) + uses: codecov/codecov-action@v5 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/codecov-action@v5 + with: + report_type: test_results diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index f50a4d104..524c79cdb 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,5 +1,10 @@ name: Ubuntu -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} @@ -8,13 +13,12 @@ concurrency: jobs: tests: runs-on: ubuntu-latest + env: + PYTEST_ADDOPTS: -n auto strategy: fail-fast: false matrix: include: - - python-version: 3.9 - env: - TOXENV: py - python-version: "3.10" env: TOXENV: py @@ -24,45 +28,66 @@ jobs: - python-version: "3.12" env: TOXENV: py - - python-version: "3.12" + - python-version: "3.13" env: - TOXENV: asyncio - - python-version: pypy3.9 + TOXENV: py + - python-version: "3.14" env: - TOXENV: pypy3 - - python-version: pypy3.10 + TOXENV: py + - python-version: "3.14" + env: + TOXENV: default-reactor + - python-version: "3.14" + env: + TOXENV: no-reactor + # pinned due to https://github.com/pypy/pypy/issues/5388 + - python-version: pypy3.11-7.3.20 env: TOXENV: pypy3 # pinned deps - - python-version: 3.8.17 + - python-version: "3.10.19" env: TOXENV: pinned - - python-version: 3.8.17 + - python-version: "3.10.19" env: - TOXENV: asyncio-pinned - - python-version: pypy3.8 + TOXENV: default-reactor-pinned + - python-version: "3.10.19" + env: + TOXENV: no-reactor-pinned + # pinned due to https://github.com/pypy/pypy/issues/5388 + - python-version: pypy3.11-7.3.20 env: TOXENV: pypy3-pinned - - python-version: 3.8.17 + - python-version: "3.10.19" env: TOXENV: extra-deps-pinned - - python-version: 3.8.17 + - python-version: "3.10.19" env: TOXENV: botocore-pinned - - python-version: "3.12" + - python-version: "3.14" env: TOXENV: extra-deps - - python-version: "3.12" + - python-version: "3.14" + env: + TOXENV: no-reactor-extra-deps + # pinned due to https://github.com/pypy/pypy/issues/5388 + - python-version: pypy3.11-7.3.20 + env: + TOXENV: pypy3-extra-deps + - python-version: "3.14" env: TOXENV: botocore + - python-version: "3.14" + env: + TOXENV: mitmproxy steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} @@ -79,4 +104,10 @@ jobs: tox - name: Upload coverage report - run: bash <(curl -s https://codecov.io/bash) + uses: codecov/codecov-action@v5 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/codecov-action@v5 + with: + report_type: test_results diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 757d62285..840c7f68e 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,5 +1,10 @@ name: Windows -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} @@ -8,16 +13,12 @@ concurrency: jobs: tests: runs-on: windows-latest + env: + PYTEST_ADDOPTS: -n auto strategy: fail-fast: false matrix: include: - - python-version: 3.8 - env: - TOXENV: windows-pinned - - python-version: 3.9 - env: - TOXENV: py - python-version: "3.10" env: TOXENV: py @@ -27,15 +28,36 @@ jobs: - python-version: "3.12" env: TOXENV: py - - python-version: "3.12" + - python-version: "3.13" env: - TOXENV: asyncio + TOXENV: py + - python-version: "3.14" + env: + TOXENV: py + - python-version: "3.14" + env: + TOXENV: default-reactor + - python-version: "3.14" + env: + TOXENV: no-reactor + + # pinned deps + - python-version: "3.10.11" + env: + TOXENV: pinned + - python-version: "3.10.11" + env: + TOXENV: extra-deps-pinned + + - python-version: "3.14" + env: + TOXENV: extra-deps steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} @@ -44,3 +66,12 @@ jobs: run: | pip install -U tox tox + + - name: Upload coverage report + uses: codecov/codecov-action@v5 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/codecov-action@v5 + with: + report_type: test_results diff --git a/.gitignore b/.gitignore index 6c5c50e08..5e52ecf1e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,18 +3,21 @@ *.pyc _trial_temp* dropin.cache -docs/build +docs/_build *egg-info -.tox -venv -build -dist -.idea +.tox/ +venv/ +.venv/ +build/ +dist/ +.idea/ +.vscode/ htmlcov/ -.coverage .pytest_cache/ +.coverage .coverage.* coverage.* +*.junit.xml test-output.* .cache/ .mypy_cache/ @@ -25,4 +28,4 @@ test-output.* Thumbs.db # OSX miscellaneous -.DS_Store \ No newline at end of file +.DS_Store diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index f238bf7ea..000000000 --- a/.isort.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[settings] -profile = black diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0cff5cc73..6b9ef3c04 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,24 +1,32 @@ +exclude: | + (?x)( + ^docs/_static| + ^docs/_tests| + ^tests/sample_data + ) repos: -- repo: https://github.com/PyCQA/bandit - rev: 1.7.5 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.2 hooks: - - id: bandit - args: [-r, -c, .bandit.yml] -- repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 - hooks: - - id: flake8 -- repo: https://github.com/psf/black.git - rev: 23.9.1 - hooks: - - id: black -- repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort + - id: ruff-check + args: [ --fix ] + - id: ruff-format - repo: https://github.com/adamchainz/blacken-docs - rev: 1.16.0 + rev: 1.20.0 hooks: - id: blacken-docs additional_dependencies: - - black==23.9.1 + - black==25.9.0 +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v1.0.2 + hooks: + - id: sphinx-lint +- repo: https://github.com/scrapy/sphinx-scrapy + rev: 0.8.6 + hooks: + - id: sphinx-scrapy diff --git a/.readthedocs.yml b/.readthedocs.yml index e71d34f3a..a2773dcf2 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,17 +1,10 @@ version: 2 -formats: all -sphinx: - configuration: docs/conf.py - fail_on_warning: true - build: - os: ubuntu-20.04 + os: ubuntu-24.04 tools: - # For available versions, see: - # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.11" # Keep in sync with .github/workflows/checks.yml - -python: - install: - - requirements: docs/requirements.txt - - path: . + python: "3.14" + commands: + - pip install tox + - tox -e docs + - mkdir -p $READTHEDOCS_OUTPUT/html + - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/ diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..24a426d36 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,6 @@ +cff-version: 1.2.0 +message: If you use Scrapy in published research, please cite it as below. +title: Scrapy +authors: + - name: Scrapy contributors +url: https://scrapy.org diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 4920dc0c3..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,27 +0,0 @@ -include README.rst -include AUTHORS -include INSTALL -include LICENSE -include MANIFEST.in -include NEWS - -include scrapy/VERSION -include scrapy/mime.types -include scrapy/py.typed - -include codecov.yml -include conftest.py -include pytest.ini -include requirements-*.txt -include tox.ini - -recursive-include scrapy/templates * -recursive-include scrapy license.txt -recursive-include docs * -prune docs/build - -recursive-include extras * -recursive-include bin * -recursive-include tests * - -global-exclude __pycache__ *.py[cod] diff --git a/README.rst b/README.rst index 14adff648..6235cb20c 100644 --- a/README.rst +++ b/README.rst @@ -1,114 +1,62 @@ -.. image:: https://scrapy.org/img/scrapylogo.png - :target: https://scrapy.org/ - -====== -Scrapy -====== +|logo| -.. image:: https://img.shields.io/pypi/v/Scrapy.svg - :target: https://pypi.python.org/pypi/Scrapy +.. |logo| image:: https://raw.githubusercontent.com/scrapy/scrapy/master/docs/_static/logo.svg + :target: https://scrapy.org + :alt: Scrapy + :width: 480px + +|version| |python_version| |ubuntu| |macos| |windows| |coverage| |conda| |deepwiki| + +.. |version| image:: https://img.shields.io/pypi/v/Scrapy.svg + :target: https://pypi.org/pypi/Scrapy :alt: PyPI Version -.. image:: https://img.shields.io/pypi/pyversions/Scrapy.svg - :target: https://pypi.python.org/pypi/Scrapy +.. |python_version| image:: https://img.shields.io/pypi/pyversions/Scrapy.svg + :target: https://pypi.org/pypi/Scrapy :alt: Supported Python Versions -.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg +.. |ubuntu| image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu :alt: Ubuntu -.. .. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg - .. :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS - .. :alt: macOS +.. |macos| image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS + :alt: macOS - -.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg +.. |windows| image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows :alt: Windows -.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg - :target: https://pypi.python.org/pypi/Scrapy - :alt: Wheel Status - -.. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg +.. |coverage| image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg :target: https://codecov.io/github/scrapy/scrapy?branch=master :alt: Coverage report -.. image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg +.. |conda| image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg :target: https://anaconda.org/conda-forge/scrapy :alt: Conda Version +.. |deepwiki| image:: https://deepwiki.com/badge.svg + :target: https://deepwiki.com/scrapy/scrapy + :alt: Ask DeepWiki -Overview -======== - -Scrapy is a BSD-licensed fast high-level web crawling and web scraping framework, used to -crawl websites and extract structured data from their pages. It can be used for -a wide range of purposes, from data mining to monitoring and automated testing. - -Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other -contributors`_. +Scrapy_ is a web scraping framework to extract structured data from websites. +It is cross-platform, and requires Python 3.10+. It is maintained by Zyte_ +(formerly Scrapinghub) and `many other contributors`_. .. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors +.. _Scrapy: https://scrapy.org/ .. _Zyte: https://www.zyte.com/ -Check the Scrapy homepage at https://scrapy.org for more information, -including a list of features. - - -Requirements -============ - -* Python 3.8+ -* Works on Linux, Windows, macOS, BSD - -Install -======= - -The quick way: +Install with: .. code:: bash pip install scrapy -See the install section in the documentation at -https://docs.scrapy.org/en/latest/intro/install.html for more details. +And follow the documentation_ to learn how to use it. -Documentation -============= +.. _documentation: https://docs.scrapy.org/en/latest/ -Documentation is available online at https://docs.scrapy.org/ and in the ``docs`` -directory. +If you wish to contribute, see Contributing_. -Releases -======== - -You can check https://docs.scrapy.org/en/latest/news.html for the release notes. - -Community (blog, twitter, mail list, IRC) -========================================= - -See https://scrapy.org/community/ for details. - -Contributing -============ - -See https://docs.scrapy.org/en/master/contributing.html for details. - -Code of Conduct ---------------- - -Please note that this project is released with a Contributor `Code of Conduct `_. - -By participating in this project you agree to abide by its terms. -Please report unacceptable behavior to opensource@zyte.com. - -Companies using Scrapy -====================== - -See https://scrapy.org/companies/ for a list. - -Commercial Support -================== - -See https://scrapy.org/support/ for details. \ No newline at end of file +.. _Contributing: https://docs.scrapy.org/en/master/contributing.html diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..49cafdf9e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,12 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +| ------- | ------------------ | +| 2.16.x | :white_check_mark: | +| < 2.16.x | :x: | + +## Reporting a Vulnerability + +Please report the vulnerability using https://github.com/scrapy/scrapy/security/advisories/new. diff --git a/artwork/README.rst b/artwork/README.rst deleted file mode 100644 index c1880ef6c..000000000 --- a/artwork/README.rst +++ /dev/null @@ -1,20 +0,0 @@ -============== -Scrapy artwork -============== - -This folder contains the Scrapy artwork resources such as logos and fonts. - -scrapy-logo.jpg ---------------- - -The main Scrapy logo, in JPEG format. - -qlassik.zip ------------ - -The font used for the Scrapy logo. Homepage: https://www.dafont.com/qlassik.font - -scrapy-blog.logo.xcf --------------------- - -The logo used in the Scrapy blog, in Gimp format. diff --git a/artwork/qlassik.zip b/artwork/qlassik.zip deleted file mode 100644 index 2885c06ef..000000000 Binary files a/artwork/qlassik.zip and /dev/null differ diff --git a/artwork/scrapy-blog-logo.xcf b/artwork/scrapy-blog-logo.xcf deleted file mode 100644 index 320102604..000000000 Binary files a/artwork/scrapy-blog-logo.xcf and /dev/null differ diff --git a/artwork/scrapy-logo.jpg b/artwork/scrapy-logo.jpg deleted file mode 100644 index 4315ef8e1..000000000 Binary files a/artwork/scrapy-logo.jpg and /dev/null differ diff --git a/conftest.py b/conftest.py index 2bfa46f5a..1674086ec 100644 --- a/conftest.py +++ b/conftest.py @@ -1,14 +1,20 @@ -import platform -import sys +from __future__ import annotations + +import importlib from pathlib import Path +from typing import TYPE_CHECKING import pytest -from twisted import version as twisted_version -from twisted.python.versions import Version from twisted.web.http import H2_ENABLED -from scrapy.utils.reactor import install_reactor +from scrapy.utils.reactor import set_asyncio_event_loop_policy +from scrapy.utils.reactorless import install_reactor_import_hook from tests.keys import generate_keys +from tests.mockserver.http import MockServer +from tests.mockserver.mitm_proxy import MitmProxy + +if TYPE_CHECKING: + from collections.abc import Generator def _py_files(folder): @@ -16,19 +22,21 @@ def _py_files(folder): collect_ignore = [ - # not a test, but looks like a test - "scrapy/utils/testsite.py", - "tests/ftpserver.py", - "tests/mockserver.py", - "tests/pipelines.py", - "tests/spiders.py", + # may need extra deps + "docs/_ext", + # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess + *_py_files("tests/AsyncCrawlerProcess"), + # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess + *_py_files("tests/AsyncCrawlerRunner"), # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess *_py_files("tests/CrawlerProcess"), # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess *_py_files("tests/CrawlerRunner"), ] -with Path("tests/ignores.txt").open(encoding="utf-8") as reader: +base_dir = Path(__file__).parent +ignore_file_path = base_dir / "tests" / "ignores.txt" +with ignore_file_path.open(encoding="utf-8") as reader: for line in reader: file_path = line.strip() if file_path and file_path[0] != "#": @@ -42,62 +50,113 @@ if not H2_ENABLED: ) ) - -@pytest.fixture() -def chdir(tmpdir): - """Change to pytest-provided temporary directory""" - tmpdir.chdir() +try: + import httpx # noqa: F401 +except ImportError: + collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py") -def pytest_addoption(parser): +def pytest_addoption(parser, pluginmanager): + if pluginmanager.hasplugin("twisted"): + return + # add the full choice set so that pytest doesn't complain about invalid choices in some cases parser.addoption( "--reactor", - default="default", - choices=["default", "asyncio"], + default="none", + choices=["asyncio", "default", "none"], ) -@pytest.fixture(scope="class") -def reactor_pytest(request): - if not request.cls: - # doctests - return - request.cls.reactor_pytest = request.config.getoption("--reactor") - return request.cls.reactor_pytest +@pytest.fixture(scope="session") +def mockserver() -> Generator[MockServer]: + with MockServer() as mockserver: + yield mockserver -@pytest.fixture(autouse=True) -def only_asyncio(request, reactor_pytest): - if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio": - pytest.skip("This test is only run with --reactor=asyncio") +@pytest.fixture # function scope because it modifies os.environ +def mitm_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]: + proxy = MitmProxy() + url = proxy.start() + monkeypatch.setenv("http_proxy", url) + monkeypatch.setenv("https_proxy", url) + + try: + yield proxy + finally: + proxy.stop() -@pytest.fixture(autouse=True) -def only_not_asyncio(request, reactor_pytest): - if ( - request.node.get_closest_marker("only_not_asyncio") - and reactor_pytest == "asyncio" - ): - pytest.skip("This test is only run without --reactor=asyncio") +@pytest.fixture # function scope because it modifies os.environ +def mitm_proxy_server_https(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]: + proxy = MitmProxy() + url = proxy.start().replace("http://", "https://") + monkeypatch.setenv("http_proxy", url) + monkeypatch.setenv("https_proxy", url) + + try: + yield proxy + finally: + proxy.stop() -@pytest.fixture(autouse=True) -def requires_uvloop(request): - if not request.node.get_closest_marker("requires_uvloop"): - return - if sys.implementation.name == "pypy": - pytest.skip("uvloop does not support pypy properly") - if platform.system() == "Windows": - pytest.skip("uvloop does not support Windows") - if twisted_version == Version("twisted", 21, 2, 0): - pytest.skip("https://twistedmatrix.com/trac/ticket/10106") - if sys.version_info >= (3, 12): - pytest.skip("uvloop doesn't support Python 3.12 yet") +@pytest.fixture # function scope because it modifies os.environ +def socks5_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]: + proxy = MitmProxy(mode="socks5") + url = proxy.start() + monkeypatch.setenv("http_proxy", url) + monkeypatch.setenv("https_proxy", url) + + try: + yield proxy + finally: + proxy.stop() + + +@pytest.fixture(scope="session") +def reactor_pytest(request) -> str: + return request.config.getoption("--reactor") def pytest_configure(config): if config.getoption("--reactor") == "asyncio": - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + # Needed on Windows to switch from proactor to selector for Twisted reactor compatibility. + # If we decide to run tests with both, we will need to add a new option and check it here. + set_asyncio_event_loop_policy() + elif config.getoption("--reactor") == "none": + install_reactor_import_hook() + + +def pytest_runtest_setup(item): + # Skip tests based on reactor markers + reactor = item.config.getoption("--reactor") + + if item.get_closest_marker("requires_reactor") and reactor == "none": + pytest.skip('This test is only run when the --reactor value is not "none"') + + if item.get_closest_marker("only_asyncio") and reactor not in {"asyncio", "none"}: + pytest.skip( + 'This test is only run when the --reactor value is "asyncio" (default) or "none"' + ) + + if item.get_closest_marker("only_not_asyncio") and reactor in {"asyncio", "none"}: + pytest.skip( + 'This test is only run when the --reactor value is not "asyncio" (default) or "none"' + ) + + # Skip tests requiring optional dependencies + optional_deps = [ + "uvloop", + "botocore", + "boto3", + "mitmproxy", + ] + + for module in optional_deps: + if item.get_closest_marker(f"requires_{module}"): + try: + importlib.import_module(module) + except ImportError: + pytest.skip(f"{module} is not installed") # Generate localhost certificate files, needed by some tests diff --git a/docs/Makefile b/docs/Makefile index 48401bac8..ed8809902 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,96 +1,20 @@ -# -# Makefile for Scrapy documentation [based on Python documentation Makefile] -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Minimal makefile for Sphinx documentation # -# You can set these variables from the command line. -PYTHON = python -SPHINXOPTS = -PAPER = -SOURCES = -SHELL = /usr/bin/env bash - -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \ - -D latex_elements.papersize=$(PAPER) \ - $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES) - -.PHONY: help update build html htmlhelp clean +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = build +# Put it first so that "make" without argument is like "make help". help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " text to make plain text files" - @echo " changes to make an overview over all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " watch build HTML docs, open in browser and watch for changes" + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -build-dirs: - mkdir -p build/$(BUILDER) build/doctrees +.PHONY: help Makefile -build: build-dirs - sphinx-build $(ALLSPHINXOPTS) - @echo - -build-ignore-errors: build-dirs - -sphinx-build $(ALLSPHINXOPTS) - @echo - - -html: BUILDER = html -html: build - @echo "Build finished. The HTML pages are in build/html." - -htmlhelp: BUILDER = htmlhelp -htmlhelp: build - @echo "Build finished; now you can run HTML Help Workshop with the" \ - "build/htmlhelp/pydoc.hhp project file." - -latex: BUILDER = latex -latex: build - @echo "Build finished; the LaTeX files are in build/latex." - @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ - "run these through (pdf)latex." - -text: BUILDER = text -text: build - @echo "Build finished; the text files are in build/text." - -changes: BUILDER = changes -changes: build - @echo "The overview file is in build/changes." - -linkcheck: BUILDER = linkcheck -linkcheck: build - @echo "Link check complete; look for any errors in the above output " \ - "or in build/$(BUILDER)/output.txt" - -linkfix: BUILDER = linkcheck -linkfix: build-ignore-errors - $(PYTHON) utils/linkfix.py - @echo "Fixing redirecting links in docs has finished; check all " \ - "replacements before committing them" - -doctest: BUILDER = doctest -doctest: build - @echo "Testing of doctests in the sources finished, look at the " \ - "results in build/doctest/output.txt" - -pydoc-topics: BUILDER = pydoc-topics -pydoc-topics: build - @echo "Building finished; now copy build/pydoc-topics/pydoc_topics.py " \ - "into the Lib/ directory" - -coverage: BUILDER = coverage -coverage: build - -htmlview: html - $(PYTHON) -c "import webbrowser; from pathlib import Path; \ - webbrowser.open(Path('build/html/index.html').resolve().as_uri())" - -clean: - -rm -rf build/* - -watch: htmlview - watchmedo shell-command -p '*.rst' -c 'make html' -R -D +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/README.rst b/docs/README.rst index 36dd5aea4..4d2236b53 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -65,4 +65,4 @@ To compile the documentation to HTML run the following command:: tox -e docs -Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir. +Documentation will be generated inside the ``docs/_build/all`` dir. diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index c23a89089..edb91bfb9 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,62 +1,67 @@ +# pylint: disable=import-error +from collections.abc import Sequence from operator import itemgetter +from typing import Any, TypedDict from docutils import nodes +from docutils.nodes import Element, General, Node, document from docutils.parsers.rst import Directive -from docutils.parsers.rst.roles import set_classes +from sphinx.application import Sphinx from sphinx.util.nodes import make_refnode -class settingslist_node(nodes.General, nodes.Element): +class SettingData(TypedDict): + docname: str + setting_name: str + refid: str + + +class SettingslistNode(General, Element): pass class SettingsListDirective(Directive): - def run(self): - return [settingslist_node("")] + def run(self) -> Sequence[Node]: + return [SettingslistNode()] -def is_setting_index(node): - if node.tagname == "index" and node["entries"]: +def is_setting_index(node: Node) -> bool: + if node.tagname == "index" and node["entries"]: # type: ignore[index,attr-defined] # index entries for setting directives look like: # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')] - entry_type, info, refid = node["entries"][0][:3] + entry_type, info, _ = node["entries"][0][:3] # type: ignore[index] return entry_type == "pair" and info.endswith("; setting") return False -def get_setting_target(node): - # target nodes are placed next to the node in the doc tree - return node.parent[node.parent.index(node) + 1] - - -def get_setting_name_and_refid(node): +def get_setting_name_and_refid(node: Node) -> tuple[str, str]: """Extract setting name from directive index node""" - entry_type, info, refid = node["entries"][0][:3] + _, info, refid = node["entries"][0][:3] # type: ignore[index] return info.replace("; setting", ""), refid -def collect_scrapy_settings_refs(app, doctree): +def collect_scrapy_settings_refs(app: Sphinx, doctree: document) -> None: env = app.builder.env if not hasattr(env, "scrapy_all_settings"): - env.scrapy_all_settings = [] - - for node in doctree.traverse(is_setting_index): - targetnode = get_setting_target(node) - assert isinstance(targetnode, nodes.target), "Next node is not a target" + emptyList: list[SettingData] = [] + env.scrapy_all_settings = emptyList # type: ignore[attr-defined] + for node in doctree.findall(is_setting_index): setting_name, refid = get_setting_name_and_refid(node) - env.scrapy_all_settings.append( - { - "docname": env.docname, - "setting_name": setting_name, - "refid": refid, - } + env.scrapy_all_settings.append( # type: ignore[attr-defined] + SettingData( + docname=env.docname, + setting_name=setting_name, + refid=refid, + ) ) -def make_setting_element(setting_data, app, fromdocname): +def make_setting_element( + setting_data: SettingData, app: Sphinx, fromdocname: str +) -> Any: refnode = make_refnode( app.builder, fromdocname, @@ -72,77 +77,106 @@ def make_setting_element(setting_data, app, fromdocname): return item -def replace_settingslist_nodes(app, doctree, fromdocname): +def make_setting_markdown_item( + setting_data: SettingData, app: Sphinx, fromdocname: str +) -> str: + uri = app.builder.get_relative_uri(fromdocname, setting_data["docname"]) + if uri.startswith("#"): + target = f"#{setting_data['refid']}" + else: + target = f"{uri}#{setting_data['refid']}" + return f"* [{setting_data['setting_name']}]({target})" + + +def _iter_sorted_settings(env: Any, fromdocname: str) -> list[SettingData]: + return [ + d + for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) # type: ignore[attr-defined] + if fromdocname != d["docname"] + ] + + +def replace_settingslist_nodes( + app: Sphinx, doctree: document, fromdocname: str +) -> None: env = app.builder.env - for node in doctree.traverse(settingslist_node): + for node in doctree.findall(SettingslistNode): settings_list = nodes.bullet_list() settings_list.extend( [ make_setting_element(d, app, fromdocname) - for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) - if fromdocname != d["docname"] + for d in _iter_sorted_settings(env, fromdocname) ] ) node.replace_self(settings_list) -def setup(app): - app.add_crossref_type( - directivename="setting", - rolename="setting", - indextemplate="pair: %s; setting", - ) - app.add_crossref_type( - directivename="signal", - rolename="signal", - indextemplate="pair: %s; signal", - ) - app.add_crossref_type( - directivename="command", - rolename="command", - indextemplate="pair: %s; command", - ) - app.add_crossref_type( - directivename="reqmeta", - rolename="reqmeta", - indextemplate="pair: %s; reqmeta", - ) +def visit_settingslist_node_markdown(translator: Any, _node: Node) -> None: + builder = translator.builder + env = builder.env + fromdocname = getattr(builder, "current_doc_name", env.docname) + lines = [ + make_setting_markdown_item(setting_data, builder.app, fromdocname) + for setting_data in _iter_sorted_settings(env, fromdocname) + ] + if lines: + translator.add("\n".join(lines), prefix_eol=2, suffix_eol=2) + raise nodes.SkipNode + + +def depart_settingslist_node_markdown(_translator: Any, _node: Node) -> None: + return None + + +def source_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "https://github.com/scrapy/scrapy/blob/master/" + text + node = nodes.reference(rawtext, text, refuri=ref, **options) + return [node], [] + + +def issue_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "https://github.com/scrapy/scrapy/issues/" + text + node = nodes.reference(rawtext, "issue " + text, refuri=ref) + return [node], [] + + +def commit_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "https://github.com/scrapy/scrapy/commit/" + text + node = nodes.reference(rawtext, "commit " + text, refuri=ref) + return [node], [] + + +def rev_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "http://hg.scrapy.org/scrapy/changeset/" + text + node = nodes.reference(rawtext, "r" + text, refuri=ref) + return [node], [] + + +def setup(app: Sphinx) -> dict[str, Any]: app.add_role("source", source_role) app.add_role("commit", commit_role) app.add_role("issue", issue_role) app.add_role("rev", rev_role) - app.add_node(settingslist_node) + app.add_node( + SettingslistNode, + markdown=(visit_settingslist_node_markdown, depart_settingslist_node_markdown), + singlemarkdown=( + visit_settingslist_node_markdown, + depart_settingslist_node_markdown, + ), + ) app.add_directive("settingslist", SettingsListDirective) app.connect("doctree-read", collect_scrapy_settings_refs) app.connect("doctree-resolved", replace_settingslist_nodes) - - -def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "https://github.com/scrapy/scrapy/blob/master/" + text - set_classes(options) - node = nodes.reference(rawtext, text, refuri=ref, **options) - return [node], [] - - -def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "https://github.com/scrapy/scrapy/issues/" + text - set_classes(options) - node = nodes.reference(rawtext, "issue " + text, refuri=ref, **options) - return [node], [] - - -def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "https://github.com/scrapy/scrapy/commit/" + text - set_classes(options) - node = nodes.reference(rawtext, "commit " + text, refuri=ref, **options) - return [node], [] - - -def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "http://hg.scrapy.org/scrapy/changeset/" + text - set_classes(options) - node = nodes.reference(rawtext, "r" + text, refuri=ref, **options) - return [node], [] + return {"parallel_read_safe": True} diff --git a/docs/_ext/scrapyfixautodoc.py b/docs/_ext/scrapyfixautodoc.py new file mode 100644 index 000000000..e342e92cf --- /dev/null +++ b/docs/_ext/scrapyfixautodoc.py @@ -0,0 +1,21 @@ +""" +Must be included after 'sphinx.ext.autodoc'. Fixes unwanted 'alias of' behavior. +https://github.com/sphinx-doc/sphinx/issues/4422 +""" + +from typing import Any + +# pylint: disable=import-error +from sphinx.application import Sphinx + + +def maybe_skip_member(app: Sphinx, what, name: str, obj, skip: bool, options) -> bool: + if not skip: + # autodoc was generating the text "alias of" for the following members + return name in {"default_item_class", "default_selector_class"} + return skip + + +def setup(app: Sphinx) -> dict[str, Any]: + app.connect("autodoc-skip-member", maybe_skip_member) + return {"parallel_read_safe": True} diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 64f16939c..1c2859deb 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -7,4 +7,50 @@ } .rst-content dl p + ol, .rst-content dl p + ul { margin-top: -6px; /* Compensates margin-top: 12px of p */ -} \ No newline at end of file +} + +/*override some styles in +sphinx-rtd-dark-mode/static/dark_mode_css/general.css*/ +.theme-switcher { + right: 0.4em !important; + top: 0.6em !important; + -webkit-box-shadow: 0px 3px 14px 4px rgba(0, 0, 0, 0.30) !important; + box-shadow: 0px 3px 14px 4px rgba(0, 0, 0, 0.30) !important; + height: 2em !important; + width: 2em !important; +} + +/*place the toggle button for dark mode +at the bottom right corner on small screens*/ +@media (max-width: 768px) { + .theme-switcher { + right: 0.4em !important; + bottom: 2.6em !important; + top: auto !important; + } +} + +/*persist blue color at the top left used in +default rtd theme*/ +html[data-theme="dark"] .wy-side-nav-search, +html[data-theme="dark"] .wy-nav-top { + background-color: #1d577d !important; +} + +/*all the styles below used to present +API objects nicely in dark mode*/ +html[data-theme="dark"] .sig.sig-object { + border-left-color: #3e4446 !important; + background-color: #202325 !important +} + +html[data-theme="dark"] .sig-name, +html[data-theme="dark"] .sig-prename, +html[data-theme="dark"] .property, +html[data-theme="dark"] .sig-param, +html[data-theme="dark"] .sig-paren, +html[data-theme="dark"] .sig-return-icon, +html[data-theme="dark"] .sig-return-typehint, +html[data-theme="dark"] .optional { + color: #e8e6e3 !important +} diff --git a/docs/_static/logo.svg b/docs/_static/logo.svg new file mode 100644 index 000000000..04b2d18a7 --- /dev/null +++ b/docs/_static/logo.svg @@ -0,0 +1 @@ + diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 000000000..29394799b --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,23 @@ +{% extends "!layout.html" %} + +{# Overridden to include a link to scrapy.org, not just to the docs root #} +{%- block sidebartitle %} + +{# the logo helper function was removed in Sphinx 6 and deprecated since Sphinx 4 #} +{# the master_doc variable was renamed to root_doc in Sphinx 4 (master_doc still exists in later Sphinx versions) #} +{%- set _logo_url = logo_url|default(pathto('_static/' + (logo or ""), 1)) %} +{%- set _root_doc = root_doc|default(master_doc) %} +scrapy.org / docs + +{%- if READTHEDOCS or DEBUG %} + {%- if theme_version_selector or theme_language_selector %} +
+
+
+
+ {%- endif %} +{%- endif %} + +{%- include "searchbox.html" %} + +{%- endblock %} diff --git a/docs/_tests/quotes.html b/docs/_tests/quotes.html index f4002ecd1..d1cfd9020 100644 --- a/docs/_tests/quotes.html +++ b/docs/_tests/quotes.html @@ -16,13 +16,13 @@

- + Login - +

- +
@@ -34,16 +34,16 @@
Tags: - - + + change - + deep-thoughts - + thinking - + world - +
@@ -54,12 +54,12 @@
Tags: - - + + abilities - + choices - +
@@ -70,18 +70,18 @@
Tags: - - + + inspirational - + life - + live - + miracle - + miracles - +
@@ -92,16 +92,16 @@
Tags: - - + + aliteracy - + books - + classic - + humor - +
@@ -112,12 +112,12 @@
Tags: - - + + be-yourself - + inspirational - +
@@ -128,14 +128,14 @@
Tags: - - + + adulthood - + success - + value - +
@@ -146,12 +146,12 @@
Tags: - - + + life - + love - +
@@ -162,16 +162,16 @@
Tags: - - + + edison - + failure - + inspirational - + paraphrased - +
@@ -182,10 +182,10 @@
Tags: - - + + misattributed-eleanor-roosevelt - +
@@ -196,73 +196,73 @@
Tags: - - + + humor - + obvious - + simile - +
- +

Top Ten tags

- + love - + inspirational - + life - + humor - + books - + reading - + friendship - + friends - + truth - + simile - - + +
diff --git a/docs/_tests/quotes1.html b/docs/_tests/quotes1.html index f4002ecd1..d1cfd9020 100644 --- a/docs/_tests/quotes1.html +++ b/docs/_tests/quotes1.html @@ -16,13 +16,13 @@

- + Login - +

- +
@@ -34,16 +34,16 @@
Tags: - - + + change - + deep-thoughts - + thinking - + world - +
@@ -54,12 +54,12 @@
Tags: - - + + abilities - + choices - +
@@ -70,18 +70,18 @@
Tags: - - + + inspirational - + life - + live - + miracle - + miracles - +
@@ -92,16 +92,16 @@
Tags: - - + + aliteracy - + books - + classic - + humor - +
@@ -112,12 +112,12 @@
Tags: - - + + be-yourself - + inspirational - +
@@ -128,14 +128,14 @@
Tags: - - + + adulthood - + success - + value - +
@@ -146,12 +146,12 @@
Tags: - - + + life - + love - +
@@ -162,16 +162,16 @@
Tags: - - + + edison - + failure - + inspirational - + paraphrased - +
@@ -182,10 +182,10 @@
Tags: - - + + misattributed-eleanor-roosevelt - +
@@ -196,73 +196,73 @@
Tags: - - + + humor - + obvious - + simile - +
- +

Top Ten tags

- + love - + inspirational - + life - + humor - + books - + reading - + friendship - + friends - + truth - + simile - - + +
diff --git a/docs/conf.py b/docs/conf.py index 9ca0f817a..99d5df7da 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,16 +1,11 @@ -# Scrapy documentation build configuration file, created by -# sphinx-quickstart on Mon Nov 24 12:02:52 2008. +# Configuration file for the Sphinx documentation builder. # -# This file is execfile()d with the current directory set to its containing dir. -# -# The contents of this file are pickled, so don't put values in the namespace -# that aren't pickleable (module imports are okay, they're removed automatically). -# -# All configuration values have a default; values that are commented out -# serve to show the default. +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html +import os import sys -from datetime import datetime +from collections.abc import Sequence from pathlib import Path # If your extensions are in another directory, add it here. If the directory @@ -19,36 +14,28 @@ sys.path.append(str(Path(__file__).parent / "_ext")) sys.path.insert(0, str(Path(__file__).parent.parent)) -# General configuration -# --------------------- +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "Scrapy" +project_copyright = "Scrapy developers" +author = "Scrapy developers" + + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ - "hoverxref.extension", "notfound.extension", "scrapydocs", - "sphinx.ext.autodoc", + "sphinx_scrapy", + "scrapyfixautodoc", # Must be after "sphinx.ext.autodoc" "sphinx.ext.coverage", - "sphinx.ext.intersphinx", - "sphinx.ext.viewcode", + "sphinx_rtd_dark_mode", ] -# Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8' - -# The master toctree document. -master_doc = "index" - -# General information about the project. -project = "Scrapy" -copyright = f"2008–{datetime.now().year}, Scrapy developers" +exclude_patterns = ["build", "Thumbs.db", ".DS_Store"] # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -64,138 +51,34 @@ except ImportError: version = "" release = "" -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -language = "en" - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of documents that shouldn't be included in the build. -# unused_docs = [] - -exclude_patterns = ["build"] - -# List of directories, relative to source directory, that shouldn't be searched -# for source files. -exclude_trees = [".build"] - -# The reST default role (used for this markup: `text`) to use for all documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# List of Sphinx warnings that will not be raised suppress_warnings = ["epub.unknown_project_files"] -# Options for HTML output -# ----------------------- +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. html_theme = "sphinx_rtd_theme" - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -# Add path to the RTD explicitly to robustify builds (otherwise might -# fail in a clean Debian build env) -import sphinx_rtd_theme - -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - -# The style sheet to use for HTML and HTML Help pages. A file of that name -# must exist either in Sphinx' static/ path, or in one of the custom paths -# given in html_static_path. -# html_style = 'scrapydoc.css' - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -# html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. html_last_updated_fmt = "%b %d, %Y" -# Custom sidebar templates, maps document names to template names. -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_use_modindex = True - -# If false, no index is generated. -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, the reST sources are included in the HTML build as _sources/. -html_copy_source = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = '' - -# Output file base name for HTML help builder. -htmlhelp_basename = "Scrapydoc" - html_css_files = [ "custom.css", ] +html_context = { + "display_github": True, + "github_user": "scrapy", + "github_repo": "scrapy", + "github_version": "master", + "conf_py_path": "/docs/", +} -# Options for LaTeX output -# ------------------------ +# Set canonical URL from the Read the Docs Domain +html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "") -# The paper size ('letter' or 'a4'). -# latex_paper_size = 'letter' - -# The font size ('10pt', '11pt' or '12pt'). -# latex_font_size = '10pt' +# -- Options for LaTeX output ------------------------------------------------ +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-latex-output # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). @@ -203,38 +86,22 @@ latex_documents = [ ("index", "Scrapy.tex", "Scrapy Documentation", "Scrapy developers", "manual"), ] -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# latex_logo = None -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False +# -- Options for the linkcheck builder --------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-the-linkcheck-builder -# Additional stuff for the LaTeX preamble. -# latex_preamble = '' - -# Documents to append as an appendix to all manuals. -# latex_appendices = [] - -# If false, no module index is generated. -# latex_use_modindex = True - - -# Options for the linkcheck builder -# --------------------------------- - -# A list of regular expressions that match URIs that should not be checked when -# doing a linkcheck build. linkcheck_ignore = [ - "http://localhost:\d+", + r"http://localhost:\d+", "http://hg.scrapy.org", - "http://directory.google.com/", + r"https://github.com/scrapy/scrapy/commit/\w+", + r"https://github.com/scrapy/scrapy/issues/\d+", ] +linkcheck_anchors_ignore_for_url = ["https://github.com/pyca/cryptography/issues/2692"] + +# -- Options for the Coverage extension -------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/coverage.html#configuration -# Options for the Coverage extension -# ---------------------------------- coverage_ignore_pyobjects = [ # Contract’s add_pre_hook and add_post_hook are not documented because # they should be transparent to contract developers, for whom pre_hook and @@ -254,6 +121,10 @@ coverage_ignore_pyobjects = [ # Base classes of downloader middlewares are implementation details that # are not meant for users. r"^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware", + # The interface methods of duplicate request filtering classes are already + # covered in the interface documentation part of the DUPEFILTER_CLASS + # setting documentation. + r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_crawler|request_seen|open|close|log)$", # Private exception used by the command-line interface implementation. r"^scrapy\.exceptions\.UsageError", # Methods of BaseItemExporter subclasses are only documented in @@ -271,51 +142,28 @@ coverage_ignore_pyobjects = [ ] -# Options for the InterSphinx extension -# ------------------------------------- +# -- Options for the InterSphinx extension ----------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration -intersphinx_mapping = { - "attrs": ("https://www.attrs.org/en/stable/", None), - "coverage": ("https://coverage.readthedocs.io/en/latest", None), - "cryptography": ("https://cryptography.io/en/latest/", None), - "cssselect": ("https://cssselect.readthedocs.io/en/latest", None), - "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None), - "pytest": ("https://docs.pytest.org/en/latest", None), - "python": ("https://docs.python.org/3", None), - "sphinx": ("https://www.sphinx-doc.org/en/master", None), - "tox": ("https://tox.wiki/en/latest/", None), - "twisted": ("https://docs.twisted.org/en/stable/", None), - "twistedapi": ("https://docs.twisted.org/en/stable/api/", None), - "w3lib": ("https://w3lib.readthedocs.io/en/latest", None), -} -intersphinx_disabled_reftypes = [] +intersphinx_disabled_reftypes: Sequence[str] = [] +# sphinx-scrapy --------------------------------------------------------------- -# Options for sphinx-hoverxref options -# ------------------------------------ +scrapy_intersphinx_enable = [ + "attrs", + "coverage", + "cryptography", + "cssselect", + "form2request", + "itemloaders", + "parsel", + "pytest", + "sphinx", + "tox", + "twisted", + "twistedapi", + "w3lib", +] -hoverxref_auto_ref = True -hoverxref_role_types = { - "class": "tooltip", - "command": "tooltip", - "confval": "tooltip", - "hoverxref": "tooltip", - "mod": "tooltip", - "ref": "tooltip", - "reqmeta": "tooltip", - "setting": "tooltip", - "signal": "tooltip", -} -hoverxref_roles = ["command", "reqmeta", "setting", "signal"] - - -def setup(app): - app.connect("autodoc-skip-member", maybe_skip_member) - - -def maybe_skip_member(app, what, name, obj, skip, options): - if not skip: - # autodocs was generating a text "alias of" for the following members - # https://github.com/sphinx-doc/sphinx/issues/4422 - return name in {"default_item_class", "default_selector_class"} - return skip +# -- Other options ------------------------------------------------------------ +default_dark_mode = False diff --git a/docs/contributing.rst b/docs/contributing.rst index d728338da..c868a0ac4 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -6,8 +6,13 @@ Contributing to Scrapy .. important:: - Double check that you are reading the most recent version of this document at - https://docs.scrapy.org/en/master/contributing.html + Double check that you are reading the most recent version of this document + at https://docs.scrapy.org/en/master/contributing.html + + By participating in this project you agree to abide by the terms of our + `Code of Conduct + `_. Please + report unacceptable behavior to opensource@zyte.com. There are many ways to contribute to Scrapy. Here are some of them: @@ -74,18 +79,81 @@ guidelines when you're going to report a new bug. .. _Minimal, Complete, and Verifiable example: https://stackoverflow.com/help/mcve +.. _find-work: + +Finding work +============ + +If you have decided to make a contribution to Scrapy, but you do not know what +to contribute, you have a few options to find pending work: + +- Check out the `contribution GitHub page`_, which lists open issues tagged + as **good first issue**. + + .. _contribution GitHub page: https://github.com/scrapy/scrapy/contribute + + There are also `help wanted issues`_ but mind that some may require + familiarity with the Scrapy code base. You can also target any other issue + provided it is not tagged as **discuss**. + +- If you enjoy writing documentation, there are `documentation issues`_ as + well, but mind that some may require familiarity with the Scrapy code base + as well. + + .. _documentation issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3Adocs+ + +- If you enjoy :ref:`writing automated tests `, you can work on + increasing our `test coverage`_. + +- If you enjoy code cleanup, we welcome fixes for issues detected by our + static analysis tools. See ``pyproject.toml`` for silenced issues that may + need addressing. + + Mind that some issues we do not aim to address at all, and usually include + a comment on them explaining the reason; not to confuse with comments that + state what the issue is about, for non-descriptive issue codes. + +If you have found an issue, make sure you read the entire issue thread before +you ask questions. That includes related issues and pull requests that show up +in the issue thread when the issue is mentioned elsewhere. + +We do not assign issues, and you do not need to announce that you are going to +start working on an issue either. If you want to work on an issue, just go +ahead and :ref:`write a patch for it `. + +Do not discard an issue simply because there is an open pull request for it. +Check if open pull requests are active first. And even if some are active, if +you think you can build a better implementation, feel free to create a pull +request with your approach. + +If you decide to work on something without an open issue, please: + +- Do not create an issue to work on code coverage or code cleanup, create a + pull request directly. + +- Do not create both an issue and a pull request right away. Either open an + issue first to get feedback on whether or not the issue is worth + addressing, and create a pull request later only if the feedback from the + team is positive, or create only a pull request, if you think a discussion + will be easier over your code. + +- Do not add docstrings for the sake of adding docstrings, or only to address + silenced Ruff issues. We expect docstrings to exist only when they add + something significant to readers, such as explaining something that is not + easier to understand from reading the corresponding code, summarizing a + long, hard-to-read implementation, providing context about calling code, or + indicating purposely uncaught exceptions from called code. + +- Do not add tests that use as much mocking as possible just to touch a given + line of code and hence improve line coverage. While we do aim to maximize + test coverage, tests should be written for real scenarios, with minimum + mocking. We usually prefer end-to-end tests. + .. _writing-patches: Writing patches =============== -Scrapy has a list of `good first issues`_ and `help wanted issues`_ that you -can work on. These issues are a great way to get started with contributing to -Scrapy. If you're new to the codebase, you may want to focus on documentation -or testing-related issues, as they are always useful and can help you get -more familiar with the project. You can also check Scrapy's `test coverage`_ -to see which areas may benefit from more tests. - The better a patch is written, the higher the chances that it'll get accepted and the sooner it will be merged. Well-written patches should: @@ -131,6 +199,14 @@ Remember to explain what was fixed or the new functionality (what it is, why it's needed, etc). The more info you include, the easier will be for core developers to understand and accept your patch. +If your pull request aims to resolve an open issue, `link it accordingly +`__, +e.g.: + +.. code-block:: none + + Resolves #123 + You can also discuss the new functionality (or bug fix) before creating the patch, but it's always good to have a patch ready to illustrate your arguments and show that you have put some additional thought into the subject. A good @@ -154,7 +230,7 @@ by running ``git fetch upstream pull/$PR_NUMBER/head:$BRANCH_NAME_TO_CREATE`` (replace 'upstream' with a remote name for scrapy repository, ``$PR_NUMBER`` with an ID of the pull request, and ``$BRANCH_NAME_TO_CREATE`` with a name of the branch you want to create locally). -See also: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally. +See also: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally. When writing GitHub pull requests, try to keep titles short but descriptive. E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests" @@ -175,15 +251,15 @@ Coding style Please follow these coding conventions when writing code for inclusion in Scrapy: -* We use `black `_ for code formatting. +* We use `Ruff `_ for code formatting. There is a hook in the pre-commit config that will automatically format your code before every commit. You can also - run black manually with ``tox -e pre-commit``. + run Ruff manually with ``tox -e pre-commit``. * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. - See https://help.github.com/en/github/using-git/setting-your-username-in-git for - setup instructions. + See https://docs.github.com/en/get-started/git-basics/setting-your-username-in-git + for setup instructions. .. _scrapy-pre-commit: @@ -242,6 +318,7 @@ Documentation about deprecated features must be removed as those features are deprecated, so that new readers do not run into it. New deprecations and deprecation removals are documented in the :ref:`release notes `. +.. _write-tests: Tests ===== @@ -313,13 +390,11 @@ And their unit-tests are in:: .. _issue tracker: https://github.com/scrapy/scrapy/issues .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users -.. _Scrapy subreddit: https://reddit.com/r/scrapy -.. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS +.. _Scrapy subreddit: https://www.reddit.com/r/scrapy/ .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues -.. _PEP 257: https://www.python.org/dev/peps/pep-0257/ -.. _pull request: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request +.. _PEP 257: https://peps.python.org/pep-0257/ +.. _pull request: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist -.. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22 .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 .. _test coverage: https://app.codecov.io/gh/scrapy/scrapy diff --git a/docs/faq.rst b/docs/faq.rst index 2113b0964..df90122f5 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -23,7 +23,7 @@ comparing `jinja2`_ to `Django`_. .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/ .. _lxml: https://lxml.de/ -.. _jinja2: https://palletsprojects.com/p/jinja/ +.. _jinja2: https://palletsprojects.com/projects/jinja/ .. _Django: https://www.djangoproject.com/ Can I use Scrapy with BeautifulSoup? @@ -82,10 +82,18 @@ to steal from us! Does Scrapy work with HTTP proxies? ----------------------------------- -Yes. Support for HTTP proxies is provided (since Scrapy 0.8) through the HTTP -Proxy downloader middleware. See +Yes. Support for HTTP proxies is provided through the HTTP Proxy downloader +middleware. See :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`. +Does Scrapy work with SOCKS proxies? +------------------------------------ + +Yes, when using +:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. See +:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and the +handler documentation. + How can I scrape an item with attributes in different pages? ------------------------------------------------------------ @@ -96,30 +104,13 @@ How can I simulate a user login in my spider? See :ref:`topics-request-response-ref-request-userlogin`. + .. _faq-bfo-dfo: Does Scrapy crawl in breadth-first or depth-first order? -------------------------------------------------------- -By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which -basically means that it crawls in `DFO order`_. This order is more convenient -in most cases. - -If you do want to crawl in true `BFO order`_, you can do it by -setting the following settings: - -.. code-block:: python - - DEPTH_PRIORITY = 1 - SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue" - SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" - -While pending requests are below the configured values of -:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or -:setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent -concurrently. As a result, the first few requests of a crawl rarely follow the -desired order. Lowering those settings to ``1`` enforces the desired order, but -it significantly slows down the crawl as a whole. +:ref:`DFO by default, but other orders are possible `. My Scrapy crawler has memory leaks. What can I do? @@ -138,39 +129,36 @@ See previous question. How can I prevent memory errors due to many allowed domains? ------------------------------------------------------------ -If you have a spider with a long list of -:attr:`~scrapy.Spider.allowed_domains` (e.g. 50,000+), consider -replacing the default -:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware -with a :ref:`custom spider middleware ` that requires -less memory. For example: +If you have a spider with a long list of :attr:`~scrapy.Spider.allowed_domains` +(e.g. 50,000+), consider replacing the default +:class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` downloader +middleware with a :ref:`custom downloader middleware +` that requires less memory. For example: - If your domain names are similar enough, use your own regular expression - instead joining the strings in - :attr:`~scrapy.Spider.allowed_domains` into a complex regular - expression. + instead joining the strings in :attr:`~scrapy.Spider.allowed_domains` into + a complex regular expression. -- If you can `meet the installation requirements`_, use pyre2_ instead of +- If you can meet the installation requirements, use pyre2_ instead of Python’s re_ to compile your URL-filtering regular expression. See :issue:`1908`. -See also other suggestions at `StackOverflow`_. +See also `other suggestions at StackOverflow +`__. .. note:: Remember to disable - :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable - your custom implementation: + :class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` when you + enable your custom implementation: .. code-block:: python - SPIDER_MIDDLEWARES = { - "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None, - "myproject.middlewares.CustomOffsiteMiddleware": 500, + DOWNLOADER_MIDDLEWARES = { + "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": None, + "myproject.middlewares.CustomOffsiteMiddleware": 50, } -.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation .. _pyre2: https://github.com/andreasvc/pyre2 -.. _re: https://docs.python.org/library/re.html -.. _StackOverflow: https://stackoverflow.com/q/36440681/939364 +.. _re: https://docs.python.org/3/library/re.html Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- @@ -206,12 +194,10 @@ I get "Filtered offsite request" messages. How can I fix them? Those messages (logged with ``DEBUG`` level) don't necessarily mean there is a problem, so you may not need to fix them. -Those messages are thrown by the Offsite Spider Middleware, which is a spider -middleware (enabled by default) whose purpose is to filter out requests to -domains outside the ones covered by the spider. - -For more info see: -:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware`. +Those messages are thrown by +:class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`, which is a +downloader middleware (enabled by default) whose purpose is to filter out +requests to domains outside the ones covered by the spider. What is the recommended way to deploy a Scrapy crawler in production? --------------------------------------------------------------------- @@ -273,7 +259,7 @@ To dump into a CSV file:: scrapy crawl myspider -O items.csv -To dump into a XML file:: +To dump into an XML file:: scrapy crawl myspider -O items.xml @@ -286,7 +272,7 @@ The ``__VIEWSTATE`` parameter is used in sites built with ASP.NET/VB.NET. For more info on how it works see `this page`_. Also, here's an `example spider`_ which scrapes one of these sites. -.. _this page: https://metacpan.org/pod/release/ECARROLL/HTML-TreeBuilderX-ASP_NET-0.09/lib/HTML/TreeBuilderX/ASP_NET.pm +.. _this page: https://metacpan.org/release/ECARROLL/HTML-TreeBuilderX-ASP_NET-0.09/view/lib/HTML/TreeBuilderX/ASP_NET.pm .. _example spider: https://github.com/AmbientLighter/rpn-fas/blob/master/fas/spiders/rnp.py What's the best way to parse big XML/CSV data feeds? @@ -297,9 +283,13 @@ build the DOM of the entire feed in memory, and this can be quite slow and consume a lot of memory. In order to avoid parsing all the entire feed at once in memory, you can use -the functions ``xmliter`` and ``csviter`` from ``scrapy.utils.iterators`` -module. In fact, this is what the feed spiders (see :ref:`topics-spiders`) use -under the cover. +the :func:`~scrapy.utils.iterators.xmliter_lxml` and +:func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what +:class:`~scrapy.spiders.XMLFeedSpider` uses. + +.. autofunction:: scrapy.utils.iterators.xmliter_lxml + +.. autofunction:: scrapy.utils.iterators.csviter Does Scrapy manage cookies automatically? ----------------------------------------- @@ -362,21 +352,26 @@ method for this purpose. For example: from copy import deepcopy - from itemadapter import is_item, ItemAdapter + from itemadapter import ItemAdapter + from scrapy import Request class MultiplyItemsMiddleware: - def process_spider_output(self, response, result, spider): - for item in result: - if is_item(item): - adapter = ItemAdapter(item) - for _ in range(adapter["multiply_by"]): - yield deepcopy(item) + def process_spider_output(self, response, result): + for item_or_request in result: + if isinstance(item_or_request, Request): + continue + adapter = ItemAdapter(item) + for _ in range(adapter["multiply_by"]): + yield deepcopy(item) Does Scrapy support IPv6 addresses? ----------------------------------- -Yes, by setting :setting:`DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``. +Yes, but when using +:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` or +:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` you need to +set :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``. Note that by doing so, you lose the ability to set a specific timeout for DNS requests (the value of the :setting:`DNS_TIMEOUT` setting is ignored). @@ -411,14 +406,14 @@ How can I make a blank request? ------------------------------- .. code-block:: python - + from scrapy import Request blank_request = Request("data:,") -In this case, the URL is set to a data URI scheme. Data URLs allow you to include data -in-line in web pages as if they were external resources. The "data:" scheme with an empty +In this case, the URL is set to a data URI scheme. Data URLs allow you to include data +inline within web pages, similar to external resources. The "data:" scheme with an empty content (",") essentially creates a request to a data URL without any specific content. @@ -432,9 +427,5 @@ See :issue:`2680`. .. _has been reported: https://github.com/scrapy/scrapy/issues/2905 -.. _Python standard library modules: https://docs.python.org/py-modindex.html +.. _Python standard library modules: https://docs.python.org/3/py-modindex.html .. _Python package: https://pypi.org/ -.. _user agents: https://en.wikipedia.org/wiki/User_agent -.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) -.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search -.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search diff --git a/docs/index.rst b/docs/index.rst index 8798aebd1..a46a2ad9f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -33,7 +33,7 @@ Having trouble? We'd like to help! .. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy .. _issue tracker: https://github.com/scrapy/scrapy/issues -.. _Scrapy Discord: https://discord.gg/mv3yErfpvq +.. _Scrapy Discord: https://discord.com/invite/mv3yErfpvq First steps @@ -128,18 +128,14 @@ Built-in services topics/logging topics/stats - topics/email topics/telnetconsole :doc:`topics/logging` - Learn how to use Python's builtin logging on Scrapy. + Learn how to use Python's built-in logging on Scrapy. :doc:`topics/stats` Collect statistics about your scraping crawler. -:doc:`topics/email` - Send email notifications when certain events occur. - :doc:`topics/telnetconsole` Inspect a running crawler using a built-in Python console. @@ -229,6 +225,7 @@ Extending Scrapy topics/signals topics/scheduler topics/exporters + topics/download-handlers topics/components topics/api @@ -257,6 +254,9 @@ Extending Scrapy :doc:`topics/exporters` Quickly export your scraped items to a file (XML, CSV, etc). +:doc:`topics/download-handlers` + Customize how requests are downloaded or add support for new URL schemes. + :doc:`topics/components` Learn the common API and some good practices when building custom Scrapy components. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index c90c1d2bf..8cef04ff1 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,7 +9,7 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.8+, either the CPython implementation (default) or +Scrapy requires Python 3.10+, either the CPython implementation (default) or the PyPy implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: @@ -37,7 +37,7 @@ Note that sometimes this may require solving compilation issues for some Scrapy dependencies depending on your operating system, so be sure to check the :ref:`intro-install-platform-notes`. -For more detailed and platform specifics instructions, as well as +For more detailed and platform-specific instructions, as well as troubleshooting information, read on. @@ -101,7 +101,7 @@ Windows ------- Though it's possible to install Scrapy on Windows using pip, we recommend you -to install `Anaconda`_ or `Miniconda`_ and use the package from the +install `Anaconda`_ or `Miniconda`_ and use the package from the `conda-forge`_ channel, which will avoid most installation issues. Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: @@ -111,7 +111,7 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: To install Scrapy on Windows using ``pip``: .. warning:: - This installation method requires “Microsoft Visual C++†for installing some + This installation method requires “Microsoft Visual C++†for installing some Scrapy dependencies, which demands significantly more disk space than Anaconda. #. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer. @@ -123,7 +123,7 @@ To install Scrapy on Windows using ``pip``: #. Check the installation details and make sure following packages are selected as optional components: * **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) ) - + * **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0)) #. Install the Visual Studio Build Tools. @@ -141,7 +141,7 @@ But it should support older versions of Ubuntu too, like Ubuntu 14.04, albeit with potential issues with TLS connections. **Don't** use the ``python-scrapy`` package provided by Ubuntu, they are -typically too old and slow to catch up with latest Scrapy. +typically too old and slow to catch up with the latest Scrapy release. To install Scrapy on Ubuntu (or Ubuntu-based) systems, you need to install @@ -170,7 +170,7 @@ macOS Building Scrapy's dependencies requires the presence of a C compiler and development headers. On macOS this is typically provided by Apple’s Xcode -development tools. To install the Xcode command line tools open a terminal +development tools. To install the Xcode command-line tools, open a terminal window and run:: xcode-select --install @@ -200,11 +200,6 @@ solutions: brew install python - * Latest versions of python have ``pip`` bundled with them so you won't need - to install it separately. If this is not the case, upgrade python:: - - brew update; brew upgrade python - * *(Optional)* :ref:`Install Scrapy inside a Python virtual environment `. @@ -268,18 +263,16 @@ reinstall Twisted with the :code:`tls` extra option:: For details, see `Issue #2473 `_. .. _Python: https://www.python.org/ -.. _pip: https://pip.pypa.io/en/latest/installing/ .. _lxml: https://lxml.de/index.html .. _parsel: https://pypi.org/project/parsel/ .. _w3lib: https://pypi.org/project/w3lib/ -.. _twisted: https://twistedmatrix.com/trac/ +.. _twisted: https://twisted.org/ .. _cryptography: https://cryptography.io/en/latest/ .. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/ -.. _setuptools: https://pypi.python.org/pypi/setuptools +.. _setuptools: https://pypi.org/pypi/setuptools .. _homebrew: https://brew.sh/ .. _zsh: https://www.zsh.org/ -.. _Anaconda: https://docs.anaconda.com/anaconda/ +.. _Anaconda: https://www.anaconda.com/docs/main .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html -.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio .. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/ .. _conda-forge: https://conda-forge.org/ diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 542760b4f..ee91ce7ca 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -44,13 +44,13 @@ https://quotes.toscrape.com, following the pagination: if next_page is not None: yield response.follow(next_page, self.parse) -Put this in a text file, name it to something like ``quotes_spider.py`` +Put this in a text file, name it something like ``quotes_spider.py`` and run the spider using the :command:`runspider` command:: scrapy runspider quotes_spider.py -o quotes.jsonl When this finishes you will have in the ``quotes.jsonl`` file a list of the -quotes in JSON Lines format, containing text and author, looking like this:: +quotes in JSON Lines format, containing the text and author, which will look like this:: {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"} {"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"} @@ -65,27 +65,27 @@ When you ran the command ``scrapy runspider quotes_spider.py``, Scrapy looked fo Spider definition inside it and ran it through its crawler engine. The crawl started by making requests to the URLs defined in the ``start_urls`` -attribute (in this case, only the URL for quotes in *humor* category) +attribute (in this case, only the URL for quotes in the *humor* category) and called the default callback method ``parse``, passing the response object as an argument. In the ``parse`` callback, we loop through the quote elements using a CSS Selector, yield a Python dict with the extracted quote text and author, look for a link to the next page and schedule another request using the same ``parse`` method as callback. -Here you notice one of the main advantages about Scrapy: requests are +Here you will notice one of the main advantages of Scrapy: requests are :ref:`scheduled and processed asynchronously `. This means that Scrapy doesn't need to wait for a request to be finished and processed, it can send another request or do other things in the meantime. This -also means that other requests can keep going even if some request fails or an +also means that other requests can keep going even if a request fails or an error happens while handling it. While this enables you to do very fast crawls (sending multiple concurrent requests at the same time, in a fault-tolerant way) Scrapy also gives you control over the politeness of the crawl through :ref:`a few settings `. You can do things like setting a download delay between -each request, limiting amount of concurrent requests per domain or per IP, and +each request, limiting the amount of concurrent requests per domain or per IP, and even :ref:`using an auto-throttling extension ` that tries -to figure out these automatically. +to figure these settings out automatically. .. note:: @@ -106,10 +106,10 @@ scraping easy and efficient, such as: * Built-in support for :ref:`selecting and extracting ` data from HTML/XML sources using extended CSS selectors and XPath expressions, - with helper methods to extract using regular expressions. + with helper methods for extraction using regular expressions. * An :ref:`interactive shell console ` (IPython aware) for trying - out the CSS and XPath expressions to scrape data, very useful when writing or + out the CSS and XPath expressions to scrape data, which is very useful when writing or debugging your spiders. * Built-in support for :ref:`generating feed exports ` in @@ -124,7 +124,7 @@ scraping easy and efficient, such as: well-defined API (middlewares, :ref:`extensions `, and :ref:`pipelines `). -* Wide range of built-in extensions and middlewares for handling: +* A wide range of built-in extensions and middlewares for handling: - cookies and session handling - HTTP features like compression, authentication, caching @@ -150,8 +150,8 @@ The next steps for you are to :ref:`install Scrapy `, a full-blown Scrapy project and `join the community`_. Thanks for your interest! -.. _join the community: https://scrapy.org/community/ +.. _join the community: https://www.scrapy.org/community .. _web scraping: https://en.wikipedia.org/wiki/Web_scraping -.. _Amazon Associates Web Services: https://affiliate-program.amazon.com/gp/advertising/api/detail/main.html +.. _Amazon Associates Web Services: https://affiliate-program.amazon.com/welcome/ecs .. _Amazon S3: https://aws.amazon.com/s3/ .. _Sitemaps: https://www.sitemaps.org/index.html diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 8ea98f29b..c4e04364b 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -18,11 +18,11 @@ This tutorial will walk you through these tasks: 4. Changing spider to recursively follow links 5. Using spider arguments -Scrapy is written in Python_. If you're new to the language you might want to -start by getting an idea of what the language is like, to get the most out of -Scrapy. +Scrapy is written in Python_. The more you learn about Python, the more you +can get out of Scrapy. -If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource. +If you're already familiar with other languages and want to learn Python quickly, the +`Python Tutorial`_ is a good resource. If you're new to programming and want to start with Python, the following books may be useful to you: @@ -76,10 +76,9 @@ This will create a ``tutorial`` directory with the following contents:: Our first Spider ================ -Spiders are classes that you define and that Scrapy uses to scrape information -from a website (or a group of websites). They must subclass -:class:`~scrapy.Spider` and define the initial requests to make, -optionally how to follow links in the pages, and how to parse the downloaded +Spiders are classes that you define and that Scrapy uses to scrape information from a website +(or a group of websites). They must subclass :class:`~scrapy.Spider` and define the initial +requests to be made, and optionally, how to follow links in pages and parse the downloaded page content to extract data. This is the code for our first Spider. Save it in a file named @@ -95,7 +94,7 @@ This is the code for our first Spider. Save it in a file named class QuotesSpider(scrapy.Spider): name = "quotes" - def start_requests(self): + async def start(self): urls = [ "https://quotes.toscrape.com/page/1/", "https://quotes.toscrape.com/page/2/", @@ -117,10 +116,10 @@ and defines some attributes and methods: unique within a project, that is, you can't set the same name for different Spiders. -* :meth:`~scrapy.Spider.start_requests`: must return an iterable of - Requests (you can return a list of requests or write a generator function) - which the Spider will begin to crawl from. Subsequent requests will be - generated successively from these initial requests. +* :meth:`~scrapy.Spider.start`: must be an asynchronous generator that + yields requests (and, optionally, items) for the spider to start crawling. + Subsequent requests will be generated successively from these initial + requests. * :meth:`~scrapy.Spider.parse`: a method that will be called to handle the response downloaded for each of the requests made. The response parameter @@ -138,7 +137,7 @@ To put our spider to work, go to the project's top level directory and run:: scrapy crawl quotes -This command runs the spider with name ``quotes`` that we've just added, that +This command runs the spider named ``quotes`` that we've just added, that will send some requests for the ``quotes.toscrape.com`` domain. You will get an output similar to this:: @@ -165,21 +164,22 @@ for the respective URLs, as our ``parse`` method instructs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Scrapy schedules the :class:`scrapy.Request ` objects -returned by the ``start_requests`` method of the Spider. Upon receiving a -response for each one, it instantiates :class:`~scrapy.http.Response` objects -and calls the callback method associated with the request (in this case, the -``parse`` method) passing the response as argument. +Scrapy sends the first :class:`scrapy.Request ` objects yielded +by the :meth:`~scrapy.Spider.start` spider method. Upon receiving a +response for each one, Scrapy calls the callback method associated with the +request (in this case, the ``parse`` method) with a +:class:`~scrapy.http.Response` object. -A shortcut to the start_requests method ---------------------------------------- -Instead of implementing a :meth:`~scrapy.Spider.start_requests` method -that generates :class:`scrapy.Request ` objects from URLs, -you can just define a :attr:`~scrapy.Spider.start_urls` class attribute -with a list of URLs. This list will then be used by the default implementation -of :meth:`~scrapy.Spider.start_requests` to create the initial requests -for your spider. +A shortcut to the ``start`` method +---------------------------------- + +Instead of implementing a :meth:`~scrapy.Spider.start` method that yields +:class:`~scrapy.Request` objects from URLs, you can define a +:attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This +list will then be used by the default implementation of +:meth:`~scrapy.Spider.start` to create the initial requests for your +spider. .. code-block:: python @@ -217,8 +217,8 @@ using the :ref:`Scrapy shell `. Run:: .. note:: - Remember to always enclose urls in quotes when running Scrapy shell from - command-line, otherwise urls containing arguments (i.e. ``&`` character) + Remember to always enclose URLs in quotes when running Scrapy shell from the + command line, otherwise URLs containing arguments (i.e. ``&`` character) will not work. On Windows, use double quotes instead:: @@ -257,7 +257,7 @@ object: The result of running ``response.css('title')`` is a list-like object called :class:`~scrapy.selector.SelectorList`, which represents a list of :class:`~scrapy.Selector` objects that wrap around XML/HTML elements -and allow you to run further queries to fine-grain the selection or extract the +and allow you to run further queries to refine the selection or extract the data. To extract the text from the title above, you can do: @@ -293,7 +293,7 @@ As an alternative, you could've written: >>> response.css("title::text")[0].get() 'Quotes to Scrape' -Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will +Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will raise an :exc:`IndexError` exception if there are no results: .. code-block:: pycon @@ -303,8 +303,8 @@ raise an :exc:`IndexError` exception if there are no results: ... IndexError: list index out of range -You might want to use ``.get()`` directly on the -:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` +You might want to use ``.get()`` directly on the +:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` if there are no results: .. code-block:: pycon @@ -354,12 +354,12 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions: XPath expressions are very powerful, and are the foundation of Scrapy Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You -can see that if you read closely the text representation of the selector -objects in the shell. +can see that if you read the text representation of the selector +objects in the shell closely. While perhaps not as popular as CSS selectors, XPath expressions offer more power because besides navigating the structure, it can also look at the -content. Using XPath, you're able to select things like: *select the link +content. Using XPath, you're able to select things like: *the link that contains the text "Next Page"*. This makes XPath very fitting to the task of scraping, and we encourage you to learn XPath even if you already know how to construct CSS selectors, it will make scraping much easier. @@ -370,7 +370,7 @@ recommend `this tutorial to learn XPath through examples `_, and `this tutorial to learn "how to think in XPath" `_. -.. _XPath: https://www.w3.org/TR/xpath/all/ +.. _XPath: https://www.w3.org/TR/xpath-10/ .. _CSS: https://www.w3.org/TR/selectors Extracting quotes and authors @@ -422,7 +422,7 @@ variable, so that we can run our CSS selectors directly on a particular quote: >>> quote = response.css("div.quote")[0] -Now, let's extract ``text``, ``author`` and the ``tags`` from that quote +Now, let's extract the ``text``, ``author`` and ``tags`` from that quote using the ``quote`` object we just created: .. code-block:: pycon @@ -448,7 +448,7 @@ to get all of them: from sys import version_info Having figured out how to extract each bit, we can now iterate over all the -quotes elements and put them together into a Python dictionary: +quote elements and put them together into a Python dictionary: .. code-block:: pycon @@ -465,8 +465,8 @@ quotes elements and put them together into a Python dictionary: Extracting data in our spider ----------------------------- -Let's get back to our spider. Until now, it doesn't extract any data in -particular, just saves the whole HTML page to a local file. Let's integrate the +Let's get back to our spider. Until now, it hasn't extracted any data in +particular, just saving the whole HTML page to a local file. Let's integrate the extraction logic above into our spider. A Scrapy spider typically generates many dictionaries containing the data @@ -529,8 +529,8 @@ using a different serialization format, such as `JSON Lines`_:: scrapy crawl quotes -o quotes.jsonl -The `JSON Lines`_ format is useful because it's stream-like, you can easily -append new records to it. It doesn't have the same problem of JSON when you run +The `JSON Lines`_ format is useful because it's stream-like, so you can easily +append new records to it. It doesn't have the same problem as JSON when you run twice. Also, as each record is a separate line, you can process big files without having to fit everything in memory, there are tools like `JQ`_ to help do that at the command-line. @@ -542,7 +542,7 @@ for Item Pipelines has been set up for you when the project is created, in ``tutorial/pipelines.py``. Though you don't need to implement any item pipelines if you just want to store the scraped items. -.. _JSON Lines: http://jsonlines.org +.. _JSON Lines: https://jsonlines.org .. _JQ: https://stedolan.github.io/jq @@ -555,7 +555,7 @@ from https://quotes.toscrape.com, you want quotes from all the pages in the webs Now that you know how to extract data from pages, let's see how to follow links from them. -First thing is to extract the link to the page we want to follow. Examining +The first thing to do is extract the link to the page we want to follow. Examining our page, we can see there is a link to the next page with the following markup: @@ -589,7 +589,7 @@ There is also an ``attrib`` property available >>> response.css("li.next a").attrib["href"] '/page/2/' -Let's see now our spider modified to recursively follow the link to the next +Now let's see our spider, modified to recursively follow the link to the next page, extracting data from it: .. code-block:: python @@ -756,8 +756,8 @@ Another interesting thing this spider demonstrates is that, even if there are many quotes from the same author, we don't need to worry about visiting the same author page multiple times. By default, Scrapy filters out duplicated requests to URLs already visited, avoiding the problem of hitting servers too -much because of a programming mistake. This can be configured by the setting -:setting:`DUPEFILTER_CLASS`. +much because of a programming mistake. This can be configured in the +:setting:`DUPEFILTER_CLASS` setting. Hopefully by now you have a good understanding of how to use the mechanism of following links and callbacks with Scrapy. @@ -795,7 +795,7 @@ with a specific tag, building the URL based on the argument: class QuotesSpider(scrapy.Spider): name = "quotes" - def start_requests(self): + async def start(self): url = "https://quotes.toscrape.com/" tag = getattr(self, "tag", None) if tag is not None: @@ -824,12 +824,12 @@ Next steps ========== This tutorial covered only the basics of Scrapy, but there's a lot of other -features not mentioned here. Check the :ref:`topics-whatelse` section in +features not mentioned here. Check the :ref:`topics-whatelse` section in the :ref:`intro-overview` chapter for a quick overview of the most important ones. You can continue from the section :ref:`section-basics` to know more about the command-line tool, spiders, selectors and other things the tutorial hasn't covered like -modeling the scraped data. If you prefer to play with an example project, check +modeling the scraped data. If you'd prefer to play with an example project, check the :ref:`intro-examples` section. .. _JSON: https://en.wikipedia.org/wiki/JSON diff --git a/docs/news.rst b/docs/news.rst index d90e32560..b8b976df9 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,2936 @@ Release notes ============= +.. _release-2.16.0: + +Scrapy 2.16.0 (2026-05-19) +-------------------------- + +Highlights: + +- Official support for Python 3.14 + +- Support for Twisted 26.4.0+ + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Increased the minimum versions of the following dependencies: + + - service_identity_: 18.1.0 → 23.1.0 + + (:issue:`7347`) + +- Added support for Twisted 26.4.0+. + (:issue:`7347`, :issue:`7505`, :issue:`7520`) + +- Added support for Python 3.14. + (:issue:`6604`, :issue:`7460`) + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The following classes and functions, intended for internal use by + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + and :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`, have + been made private: + + - ``scrapy.core.downloader.handlers.http11.ScrapyAgent`` + + - ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent`` + + - ``scrapy.core.downloader.handlers.http11.TunnelingAgent`` + + - ``scrapy.core.downloader.handlers.http11.TunnelingTCP4ClientEndpoint`` + + - ``scrapy.core.downloader.handlers.http11.tunnel_request_data()`` + + - ``scrapy.core.downloader.handlers.http2.ScrapyH2Agent`` + + (:issue:`7496`, :issue:`7510`) + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.FormRequest`` is deprecated. You can use the :doc:`form2request + ` library instead, see :ref:`form`. + (:issue:`6438`) + +- ``scrapy.utils.python.MutableChain`` is deprecated. + (:issue:`7504`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- The ``start_requests()`` method of :class:`~scrapy.Spider`, deprecated in + 2.13.0, is removed and no longer called. Use :meth:`~scrapy.Spider.start` + instead, or both to maintain support for lower Scrapy versions. + (:issue:`7490`) + +- Support for ``process_start_requests()`` methods of :ref:`spider middlewares + `, deprecated in 2.13.0, is removed. Use + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead, + or both to maintain support for lower Scrapy versions. + (:issue:`7490`) + +- Support for synchronous ``process_spider_output()`` methods of spider + middlewares, deprecated in Scrapy 2.13.0, is removed. You should upgrade + the affected middlewares to have asynchronous ``process_spider_output()`` + methods. + (:issue:`7504`) + +- The ``spider`` arguments of the following methods of + :class:`~scrapy.core.scraper.Scraper`, deprecated in Scrapy 2.13.0, are + removed: + + - ``close_spider()`` + + - ``enqueue_scrape()`` + + - ``handle_spider_error()`` + + - ``handle_spider_output()`` + + (:issue:`7487`) + +- HTTP/1.0 support code, deprecated in Scrapy 2.13.0, is removed. This + includes: + + - ``scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler`` + + - The ``scrapy.core.downloader.webclient`` module. + + - The ``DOWNLOADER_HTTPCLIENTFACTORY`` setting. + + (:issue:`7486`) + +- The following functions, deprecated in Scrapy 2.13.0, are removed, you + should import them from :mod:`w3lib.url` directly instead: + + - ``scrapy.utils.url.add_or_replace_parameter()`` + + - ``scrapy.utils.url.add_or_replace_parameters()`` + + - ``scrapy.utils.url.any_to_uri()`` + + - ``scrapy.utils.url.canonicalize_url()`` + + - ``scrapy.utils.url.file_uri_to_path()`` + + - ``scrapy.utils.url.is_url()`` + + - ``scrapy.utils.url.parse_data_uri()`` + + - ``scrapy.utils.url.parse_url()`` + + - ``scrapy.utils.url.path_to_file_uri()`` + + - ``scrapy.utils.url.safe_download_url()`` + + - ``scrapy.utils.url.safe_url_string()`` + + - ``scrapy.utils.url.url_query_cleaner()`` + + - ``scrapy.utils.url.url_query_parameter()`` + + (:issue:`7487`) + +- The following test-related code, deprecated in Scrapy 2.13.0, is removed: + + - the ``scrapy.utils.testproc`` module + + - the ``scrapy.utils.testsite`` module + + - ``scrapy.utils.test.assert_gcs_environ()`` + + - ``scrapy.utils.test.get_ftp_content_and_delete()`` + + - ``scrapy.utils.test.get_gcs_content_and_delete()`` + + - ``scrapy.utils.test.mock_google_cloud_storage()`` + + - ``scrapy.utils.test.skip_if_no_boto()`` + + - ``scrapy.utils.test.TestSpider`` + + (:issue:`7487`) + +- ``scrapy.utils.versions.scrapy_components_versions()``, deprecated in + Scrapy 2.13.0, is removed, you can use + :func:`scrapy.utils.versions.get_versions` instead. + (:issue:`7487`) + +- ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` and + ``scrapy.utils.url.escape_ajax()``, deprecated in Scrapy 2.13.0, are + removed. + (:issue:`7487`) + +- The ``__init__()`` method of priority queue classes (see + :setting:`SCHEDULER_PRIORITY_QUEUE`) now needs to support a keyword-only + ``start_queue_cls`` parameter, not supporting it was deprecated in Scrapy + 2.13.0. + (:issue:`7487`) + +- ``scrapy.spiders.init.InitSpider``, deprecated in Scrapy 2.13.0, is + removed. + (:issue:`7487`) + +New features +~~~~~~~~~~~~ + +- New features and improvements for + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`: + + - Support for proxies. + + - Support for the :reqmeta:`download_latency` meta key. + + - Support for :attr:`Response.certificate + `. + + - Default headers set by the ``httpx`` library are no longer added to + requests. + + (:issue:`7441`, :issue:`7524`) + +- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now + skips HTTPS proxy certificate verification when the + :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting is set to ``False``. + (:issue:`7496`) + +Improvements +~~~~~~~~~~~~ + +- :func:`time.monotonic` is used instead of :func:`time.time` to calculate + elapsed time in various places. + (:issue:`7377`) + +- Improved extraction of the file extension from the URL in + :class:`~scrapy.pipelines.files.FilesPipeline`. + (:issue:`4225`, :issue:`7414`) + +- Other code refactoring and improvements. + (:issue:`7401`) + +Bug fixes +~~~~~~~~~ + +- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now + raises an exception when a request has an ``https://`` destination and an + ``https://`` proxy, which is not supported by this handler. Previously it + tried to connect to the proxy via HTTP in this case. + (:issue:`7496`) + +- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` now + raises an exception for requests with ``http://`` URLs instead of trying to + connect, which is not supported by this handler. + (:issue:`7496`) + +- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` no longer + adds the ``:status`` pseudo-header to :attr:`Response.headers + `. + (:issue:`7441`) + +- Fixed :func:`scrapy.utils.response.open_in_browser` removing the ```` + tag when adding the ```` tag. + (:issue:`7459`) + +Documentation +~~~~~~~~~~~~~ + +- Documented that + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + doesn't support HTTPS proxies for HTTPS destinations and that + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` doesn't + support proxies at all. + (:issue:`7496`) + +- Added an example of using + :class:`logging.handlers.TimedRotatingFileHandler` to rotate Scrapy logs. + (:issue:`3628`, :issue:`7501`) + +- Added a ``CITATION.cff`` file. + (:issue:`7502`, :issue:`7519`) + +- Mentioned ``DOWNLOADER_CLIENT_TLS_METHOD`` in :ref:`bans`. + (:issue:`5232`, :issue:`7518`) + +- Other documentation improvements and fixes. + (:issue:`7417`, + :issue:`7463`, + :issue:`7472`, + :issue:`7480`, + :issue:`7489`, + :issue:`7503`, + :issue:`7507`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added tests that connect to https://books.toscrape.com/ to test the + behavior with a real website. These tests are marked with the + ``requires_internet`` pytest mark and can be skipped with e.g. + ``-m 'not requires_internet'`` if you cannot or don't want to run them. + (:issue:`7520`) + +- Type hints improvements and fixes. + (:issue:`7492`, :issue:`7532`) + +- CI and test improvements and fixes. + (:issue:`7441`, :issue:`7466`, :issue:`7491`, :issue:`7496`) + +.. _release-2.15.2: + +Scrapy 2.15.2 (2026-04-28) +-------------------------- + +Bug fixes +~~~~~~~~~ + +- Fixed links in https://docs.scrapy.org/llms.txt (:issue:`7467`) + +.. _release-2.15.1: + +Scrapy 2.15.1 (2026-04-23) +-------------------------- + +Bug fixes +~~~~~~~~~ + +- Sharing of the SSL context between multiple connections, introduced in + Scrapy 2.15.0, is reverted as it caused problems and wasn't actually + needed. + (:issue:`7445`, :issue:`7450`) + +- Fixed :meth:`scrapy.settings.BaseSettings.getwithbase` failing on keys with + dots that aren't import names. It now works the way it worked before Scrapy + 2.15.0, without trying to match class objects and import path. A separate + method, + :func:`~scrapy.settings.BaseSettings.get_component_priority_dict_with_base`, + was added that does that, and it is now used for :ref:`component priority + dictionaries `. + (:issue:`7426`, :issue:`7449`) + +- Documentation rendering improvements. + (:issue:`7452`, :issue:`7454`) + +.. _release-2.15.0: + +Scrapy 2.15.0 (2026-04-09) +-------------------------- + +Highlights: + +- Experimental support for running without a Twisted reactor + +- Experimental ``httpx``-based download handler + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The built-in HTTP :ref:`download handlers ` now + raise Scrapy-specific exceptions instead of implementation-specific ones, + see :ref:`download-handlers-exceptions`. This can affect user code that + handles downloader exceptions, such as ``process_exception()`` methods of + custom :ref:`downloader middlewares `. + (:issue:`7208`) + +- In order to fix a long-standing bug with handling of asynchronous storages, + the following changes were made to media pipeline classes, which can impact + some of the user code that subclasses them or calls their methods directly: + + - overrides of :meth:`scrapy.pipelines.media.MediaPipeline.media_downloaded` + and :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` can now + return coroutines + + - :meth:`~scrapy.pipelines.files.FilesPipeline.media_downloaded`, + :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` and + :meth:`~scrapy.pipelines.images.ImagesPipeline.image_downloaded` now + return coroutines + + (:issue:`2183`, :issue:`6369`, :issue:`7182`) + +- ``Request`` and ``Response`` objects: ``__slots__`` and setter changes: + + - :class:`scrapy.http.Request` and :class:`scrapy.http.Response` now + define ``__slots__``. Assigning arbitrary attributes to instances (for + example, ``response.foo = 1``) will raise ``AttributeError``. Store + per-request/response data in the request/response ``meta`` mapping + instead of attaching new attributes to the objects. + + - If you maintain custom ``Request`` or ``Response`` subclasses that + relied on dynamic instance attributes, either add ``'__dict__'`` to + your subclass ``__slots__`` to allow dynamic attributes, or migrate + per-instance state to ``meta`` or explicit documented attributes. + + - The setters for ``headers``, ``flags`` and ``cookies`` no longer coerce + falsy values into ``None``. For example, ``request.headers = {}`` now + stores an empty :class:`scrapy.http.headers.Headers` instance (not + ``None``), and ``request.flags = []`` remains an empty list instead of + being set to ``None``. Update code that relied on ``is None`` checks or + the previous coercion behaviour. + + (:issue:`7036`, :issue:`7367`, :issue:`7374`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- The context factory class set as the value of the + ``DOWNLOADER_CLIENTCONTEXTFACTORY`` setting is now required to support the + ``method`` argument of ``__init__()``, recommended since Scrapy 1.2.0. + (:issue:`7353`) + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.mail.MailSender`` is deprecated. Please use :mod:`smtplib`, + :mod:`twisted.mail.smtp` or other 3rd party email libraries. + (:issue:`7249`, :issue:`7263`) + +- The ``scrapy.extensions.statsmailer.StatsMailer`` extension is deprecated. + You can instead implement your own notifications by handling the + :signal:`spider_closed` signal. + (:issue:`7249`, :issue:`7263`) + +- The ``MEMUSAGE_NOTIFY_MAIL`` setting is deprecated. You can instead + implement your own notifications by handling the + :signal:`memusage_warning_reached` and :signal:`spider_closed` signals. + (:issue:`7249`, :issue:`7263`) + +- The ``DNS_RESOLVER`` setting was renamed to :setting:`TWISTED_DNS_RESOLVER` + and the old name is deprecated. + (:issue:`7350`, :issue:`7361`) + +- The ``DOWNLOADER_CLIENTCONTEXTFACTORY`` setting is deprecated. If you were + using it to switch to + ``scrapy.core.downloader.contextfactory.BrowserLikeContextFactory``, please + use the new :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting instead. If you + cannot use the default context factory for some other reason, please + subclass the :ref:`download handler ` instead. + (:issue:`7352`, :issue:`7379`) + +- ``scrapy.core.downloader.contextfactory.BrowserLikeContextFactory`` is + deprecated. You can set the new :setting:`DOWNLOAD_VERIFY_CERTIFICATES` + setting to ``True`` instead. + (:issue:`7379`) + +- The following implementation details of the context factory handling code + are deprecated: + + - ``scrapy.core.downloader.contextfactory.AcceptableProtocolsContextFactory`` + + - ``scrapy.core.downloader.contextfactory.load_context_factory_from_settings()`` + + - ``scrapy.core.downloader.contextfactory.ScrapyClientContextFactory`` + + - ``scrapy.core.downloader.tls.ScrapyClientTLSOptions`` + + (:issue:`7353`, :issue:`7391`) + +- Passing :class:`str` instead of :class:`bytes` to + :class:`scrapy.utils.sitemap.Sitemap` and + :func:`scrapy.utils.sitemap.sitemap_urls_from_robots` is deprecated. + (:issue:`7007`) + +- ``scrapy.utils.misc.walk_modules()`` is deprecated. You can use + :func:`scrapy.utils.misc.walk_modules_iter` instead. + (:issue:`7388`) + +- ``scrapy.shell.Shell.inthread`` is deprecated. You can use + :attr:`scrapy.shell.Shell.fetch_available` instead to check if + :func:`~scrapy.shell.Shell.fetch` can be used. + (:issue:`7395`) + +- ``scrapy.commands.ScrapyCommand.set_crawler()`` is deprecated. + (:issue:`7276`) + +New features +~~~~~~~~~~~~ + +- Added an *experimental* mode for running Scrapy without installing a + Twisted reactor: set :setting:`TWISTED_REACTOR_ENABLED` to ``False`` to + enable it. This mode has limitations, refer to :ref:`its documentation + ` for details. As long as it's experimental, its + behavior and related features and APIs may change in future Scrapy releases + in a breaking way. + (:issue:`6219`, + :issue:`7185`, + :issue:`7186`, + :issue:`7187`, + :issue:`7188`, + :issue:`7190`, + :issue:`7197`, + :issue:`7199`, + :issue:`7209`, + :issue:`7228`, + :issue:`7355`, + :issue:`7366`, + :issue:`7385`, + :issue:`7395`) + +- Added the :func:`scrapy.utils.reactorless.is_reactorless` function that + checks if there is a running asyncio event loop but no Twisted reactor. + (:issue:`7185`, :issue:`7199`) + +- Changed :func:`scrapy.utils.asyncio.is_asyncio_available` to return + ``True`` if there is a running asyncio loop, even if no Twisted reactor is + installed. + (:issue:`7185`, :issue:`7199`) + +- Added an *experimental* download handler that uses the httpx_ library and + doesn't require a Twisted reactor: + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. As + long as it's experimental, its behavior may change in future Scrapy + releases in a breaking way. + (:issue:`6805`, :issue:`7239`, :issue:`7368`, :issue:`7384`) + + .. _httpx: https://www.python-httpx.org/ + +- Added the :setting:`DOWNLOAD_BIND_ADDRESS` setting as a global counterpart + to the per-request :reqmeta:`bindaddress` meta key. + (:issue:`7266`, :issue:`7283`) + +- Added the :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting that can be set + to ``True`` to make Scrapy abort HTTPS requests when the server certificate + is invalid or doesn't match the domain. + (:issue:`7379`) + +- The built-in HTTP :ref:`download handlers ` now + raise Scrapy-specific exceptions instead of implementation-specific ones, + to allow unified handling of similar problems caused by different + implementations. The default value of the :setting:`RETRY_EXCEPTIONS` + setting was updated replacing Twisted-specific exceptions with these new + ones. The exceptions: + + - :exc:`~scrapy.exceptions.CannotResolveHostError` + + - :exc:`~scrapy.exceptions.DownloadCancelledError` + + - :exc:`~scrapy.exceptions.DownloadConnectionRefusedError` + + - :exc:`~scrapy.exceptions.DownloadFailedError` + + - :exc:`~scrapy.exceptions.DownloadTimeoutError` + + - :exc:`~scrapy.exceptions.ResponseDataLossError` + + - :exc:`~scrapy.exceptions.UnsupportedURLSchemeError` + + (:issue:`7208`) + +- Added the :signal:`memusage_warning_reached` signal emitted by the + :class:`~scrapy.extensions.memusage.MemoryUsage` extension when the memory + usage reaches :setting:`MEMUSAGE_WARNING_MB`. + (:issue:`7249`, :issue:`7263`) + +- Added + :meth:`Headers.to_tuple_list() ` + that returns headers as a list of ``(key, value)`` tuples. + (:issue:`7239`) + +- :class:`~scrapy.core.downloader.handlers.s3.S3DownloadHandler` now uses the + download handler configured for the ``"https"`` scheme to make requests + instead of always using + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`. + (:issue:`7369`, :issue:`7370`) + +- Added :func:`scrapy.utils.misc.walk_modules_iter` as a replacement for + ``scrapy.utils.misc.walk_modules()`` that returns an iterable instead of a + list. + (:issue:`7388`) + +Improvements +~~~~~~~~~~~~ + +- :func:`asyncio.to_thread` is now used instead of + :func:`twisted.internet.threads.deferToThread` in the built-in feed + storages, media pipeline storages and the + :func:`scrapy.utils.decorators.inthread` decorator when available. + (:issue:`7183`, :issue:`7184`, :issue:`7349`) + +- Improved memory footprint of :class:`~scrapy.Request` and + :class:`~scrapy.http.Response` objects by adding ``__slots__`` and omitting + empty lists and dicts in some internal attributes. + (:issue:`7036`, :issue:`7367`, :issue:`7374`) + +- :class:`~scrapy.core.downloader.contextfactory._ScrapyClientContextFactory` + no longer mutates the SSL context, to avoid the behavior that was + deprecated in pyOpenSSL 25.1.0. + (:issue:`6859`, :issue:`7353`) + +- Improved memory usage of :class:`~scrapy.spiders.sitemap.SitemapSpider` and + :class:`scrapy.utils.sitemap.Sitemap`. + (:issue:`3529`, :issue:`7007`) + +- Improved the scheduling behavior of + :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue` when crawling + multiple domains. + (:issue:`7293`, :issue:`7351`) + +- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` and + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` now handle + TLS verbose logging (see :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING`) + directly instead of relying on + :class:`~scrapy.core.downloader.contextfactory._ScrapyClientContextFactory`. + (:issue:`7387`) + +- The server certificate verification code now correctly handles certificates + with IP addresses in ``subjectAltName``. + (:issue:`7353`) + +- Improved reliability of :func:`scrapy.utils.trackref.get_oldest`. + (:issue:`1758`, :issue:`7375`) + +- Other code refactoring and improvements. + (:issue:`7210`, :issue:`7238`, :issue:`7376`, :issue:`7386`, :issue:`7395`, + :issue:`7405`, :issue:`7410`) + +Bug fixes +~~~~~~~~~ + +- :ref:`Media pipelines ` should now wait for uploads + to asynchronous storages (e.g. + :class:`~scrapy.pipelines.files.S3FilesStore`) to complete. + (:issue:`2183`, :issue:`6369`, :issue:`7182`) + +- Fixed merging ``*_BASE`` settings (e.g. merging + :setting:`DOWNLOADER_MIDDLEWARES` with + :setting:`DOWNLOADER_MIDDLEWARES_BASE`) when a component is referred to by + a class object in one setting and by a string import path in the other one. + (:issue:`6912`, :issue:`6993`) + +- ``scrapy runspider`` and ``scrapy crawl`` now set the exit code to 1 if an + exception happened early (this was broken since Scrapy 2.13.0). + (:issue:`6820`, :issue:`7255`) + +- Fixed repeated warnings about data loss (see + :setting:`DOWNLOAD_FAIL_ON_DATALOSS`) not being suppressed in + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`. + (:issue:`7222`) + +- Improved FTP connection management in + :class:`scrapy.pipelines.files.FTPFilesStore`. + (:issue:`7256`) + +- Fixed the ``spider`` variable in the :ref:`shell `, which + wasn't available since Scrapy 2.13.0. + (:issue:`7395`) + +Documentation +~~~~~~~~~~~~~ + +- The ``llms.txt`` and ``llms-full.txt`` files and Markdown versions of pages + are now generated when the HTML documentation is built. + (:issue:`7380`) + +- Added a "Copy as Markdown" button to the HTML documentation. + (:issue:`7380`) + +- Added :ref:`docs for using Pydantic models as items `. + (:issue:`6955`, :issue:`6966`) + +- Documented :ref:`job directory contents `. + (:issue:`4842`, :issue:`5260`) + +- Improved docs for :attr:`~scrapy.Request.dont_filter`. + (:issue:`6398`, :issue:`7245`) + +- Clarified that settings related to :setting:`TWISTED_DNS_RESOLVER` are only + taken into account if the selected resolver supports them. + (:issue:`7385`) + +- Other documentation improvements and fixes. + (:issue:`7248`, :issue:`7274`, :issue:`7406`, :issue:`7408`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added the ``no-reactor`` test environment that doesn't install a Twisted + reactor and uses ``pytest-asyncio`` instead of ``pytest-twisted`` to run + asynchronous test functions. + (:issue:`6952`, :issue:`7189`, :issue:`7233`, :issue:`7234`, :issue:`7254`, + :issue:`7259`) + +- Fixed running tests with ``pytest-xdist``. + (:issue:`7216`, :issue:`7257`) + +- Type hints improvements and fixes. + (:issue:`7300`, :issue:`7331`) + +- CI and test improvements and fixes. + (:issue:`7060`, + :issue:`7223`, + :issue:`7232`, + :issue:`7241`, + :issue:`7250`, + :issue:`7256`, + :issue:`7276`, + :issue:`7277`, + :issue:`7279`, + :issue:`7329`, + :issue:`7363`, + :issue:`7381`, + :issue:`7402`) + +.. _release-2.14.2: + +Scrapy 2.14.2 (2026-03-12) +-------------------------- + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- Values from the ``Referrer-Policy`` header of HTTP responses are no longer + executed as Python callables. See the `cwxj-rr6w-m6w7`_ security advisory + for details. + + .. _cwxj-rr6w-m6w7: https://github.com/scrapy/scrapy/security/advisories/GHSA-cwxj-rr6w-m6w7 + +- In line with the `standard + `__, 301 redirects of + ``POST`` requests are converted into ``GET`` requests. + + Converting to a ``GET`` request implies not only a method change, but also + omitting the body and ``Content-*`` headers in the redirect request. On + cross-origin redirects (for example, cross-domain redirects), this is + effectively a security bug fix for scenarios where the body contains + secrets. + +Deprecations +~~~~~~~~~~~~ + +- Passing a response URL string as the first positional argument to + :meth:`scrapy.spidermiddlewares.referer.RefererMiddleware.policy` is + deprecated. Pass a :class:`~scrapy.http.Response` instead. + + The parameter has also been renamed to ``response`` to reflect this change. + The old parameter name (``resp_or_url``) is deprecated. + +New features +~~~~~~~~~~~~ + +- Added a new setting, :setting:`REFERER_POLICIES`, to allow customizing + supported referrer policies. + +Bug fixes +~~~~~~~~~ + +- Made additional redirect scenarios convert to ``GET`` in line with the + `standard `__: + + - Only ``POST`` 302 redirects are converted into ``GET`` requests; other + methods are preserved. + + - ``HEAD`` 303 redirects are not converted into ``GET`` requests. + + - ``GET`` 303 redirects do not have their body or standard ``Content-*`` + headers removed. + +- Redirects where the original request body is dropped now also have their + ``Content-Encoding``, ``Content-Language`` and ``Content-Location`` headers + removed, in addition to the ``Content-Type`` and ``Content-Length`` headers + that were already being removed. + +- Redirects now preserve the source URL fragment if the redirect URL does not + include one. This is useful when using browser-based download handlers, + such as `scrapy-playwright`_ or `scrapy-zyte-api`_, while letting Scrapy + handle redirects. + + .. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright + .. _scrapy-zyte-api: https://scrapy-zyte-api.readthedocs.io/en/latest/ + +- The ``Referer`` header is now removed on redirect if + :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` is disabled. + +- The handling of the ``Referer`` header on redirects now takes into account + the ``Referer-Policy`` header of the response that triggers the redirect. + +.. _release-2.14.1: + +Scrapy 2.14.1 (2026-01-12) +-------------------------- + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.utils.defer.maybeDeferred_coro()`` is deprecated. (:issue:`7212`) + +Bug fixes +~~~~~~~~~ + +- Fixed custom stats collectors that require a ``spider`` argument in their + ``open_spider()`` and ``close_spider()`` methods not receiving the + argument when called by the engine. + + Note, however, that the ``spider`` argument is now deprecated and will stop + being passed in a future version of Scrapy. + + (:issue:`7213`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Replaced deprecated ``codecov/test-results-action@v1`` GitHub Action with + ``codecov/codecov-action@v5``. + (:issue:`7180`, :issue:`7215`) + +.. _release-2.14.0: + +Scrapy 2.14.0 (2026-01-05) +-------------------------- + +Highlights: + +- More coroutine-based replacements for Deferred-based APIs + +- The default priority queue is now ``DownloaderAwarePriorityQueue`` + +- Dropped support for Python 3.9 and PyPy 3.10 + +- Improved and documented the API for custom download handlers + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for Python 3.9. + (:issue:`7121`) + +- Dropped support for PyPy 3.10. + (:issue:`7050`) + +- Increased the minimum versions of the following dependencies: + + - lxml_: 4.6.0 → 4.6.4 + + - Pillow_ (optional dependency): 8.0.0 → 8.3.2 + + - botocore_ (optional dependency): 1.4.87 → 1.13.45 + +- Restored support for ``brotlicffi`` dropped in Scrapy 2.13.4. Its minimum + supported version is now ``1.2.0.0``. + (:issue:`7160`) + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- If you set the :setting:`TWISTED_REACTOR` setting to a :ref:`non-asyncio + value ` at the :ref:`spider level `, you + may now need to set the :setting:`FORCE_CRAWLER_PROCESS` setting to + ``True`` when running Scrapy via :ref:`its command-line tool + ` to avoid a reactor mismatch exception. + (:issue:`6845`) + +- The ``log_count/*`` stats no longer count some of the early messages that + they counted before. While the earliest log messages, emitted before the + counter is initialized, were never counted, the counter initialization now + happens later than in previous Scrapy versions. You may need to adjust + expected values if you retrieve and compare values of these stats in your + code. + (:issue:`7046`) + +- The classes listed below are now :term:`abstract base classes `. They cannot be instantiated directly and their subclasses + need to override the abstract methods listed below to be able to be + instantiated. If you previously instantiated these classes directly, you + will now need to subclass them and provide trivial (e.g. empty) + implementations for the abstract methods. + + - :class:`scrapy.commands.ScrapyCommand` + + - :meth:`~scrapy.commands.ScrapyCommand.run` + + - :meth:`~scrapy.commands.ScrapyCommand.short_desc` + + - :class:`scrapy.exporters.BaseItemExporter` + + - :meth:`~scrapy.exporters.BaseItemExporter.export_item` + + - :class:`scrapy.extensions.feedexport.BlockingFeedStorage` + + - :meth:`~scrapy.extensions.feedexport.BlockingFeedStorage._store_in_thread` + + - :class:`scrapy.middleware.MiddlewareManager` + + - :meth:`~scrapy.middleware.MiddlewareManager._get_mwlist_from_settings` + + - :class:`scrapy.spidermiddlewares.referer.ReferrerPolicy` + + - :meth:`~scrapy.spidermiddlewares.referer.ReferrerPolicy.referrer` + + (:issue:`6930`) + +- Scrapy no longer passes a ``spider`` argument to any methods of the + :setting:`stats collector `. It wasn't passed in many of the + calls even in older Scrapy versions, so we don't expect existing custom + stats collector implementations to require a ``spider`` argument. If your + implementation needs a :class:`~scrapy.Spider` instance, you can get it + from the :class:`~scrapy.crawler.Crawler` instance passed to the + constructor. + (:issue:`7011`) + +- :class:`scrapy.middleware.MiddlewareManager` no longer includes code for + handling ``open_spider()`` and ``close_spider()`` component methods. As + this code was only used for pipelines it was moved into + :class:`scrapy.pipelines.ItemPipelineManager`. This change should only + affect custom subclasses of :class:`~scrapy.middleware.MiddlewareManager`. + The following code was moved: + + - ``scrapy.middleware.MiddlewareManager.open_spider()`` + + - ``scrapy.middleware.MiddlewareManager.close_spider()`` + + - Code in ``scrapy.middleware.MiddlewareManager._add_middleware()`` that + processes ``open_spider()`` and ``close_spider()`` component methods. + + (:issue:`7006`) + +- :meth:`scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware.process_request` + now returns a coroutine, previously it returned a + :class:`~twisted.internet.defer.Deferred` object or ``None``. The + ``robot_parser()`` method was also changed to return a coroutine. This + change only impacts code that subclasses + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` or + calls its methods directly. + (:issue:`6802`) + +- The built-in :ref:`download handlers ` have been + refactored, changing the signatures of their methods. This change should + only affect user code that subclasses any of these handlers or calls their + methods directly. + (:issue:`6778`, :issue:`7164`) + +- :meth:`scrapy.pipelines.media.MediaPipeline.process_item` now returns a + coroutine, previously it returned a + :class:`~twisted.internet.defer.Deferred` object. This + change only impacts code that calls this method directly. + (:issue:`7177`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- The ``from_settings()`` method of the following components, deprecated in + Scrapy 2.12.0, is removed. You should use ``from_crawler()`` instead. + + - :class:`scrapy.dupefilters.RFPDupeFilter` + - :class:`scrapy.mail.MailSender` + - :class:`scrapy.middleware.MiddlewareManager` + - :class:`scrapy.core.downloader.contextfactory.ScrapyClientContextFactory` + - :class:`scrapy.pipelines.files.FilesPipeline` + - :class:`scrapy.pipelines.images.ImagesPipeline` + + (:issue:`7126`) + +- Scrapy no longer calls ``from_settings()`` methods of 3rd-party + :ref:`components `, deprecated in Scrapy 2.12.0. You + should define a ``from_crawler()`` method instead. + (:issue:`7126`) + +- The initialization flow of :class:`scrapy.pipelines.media.MediaPipeline` + and its subclasses was simplified, it now mandates ``from_crawler()`` + methods and ``crawler`` arguments of ``__init__()`` methods. Not using + these was deprecated in Scrapy 2.12.0. + (:issue:`7126`) + +- The ``REQUEST_FINGERPRINTER_IMPLEMENTATION`` setting, deprecated in Scrapy + 2.12.0, is removed. + (:issue:`7126`) + +- The ``scrapy.utils.misc.create_instance()`` function, deprecated in Scrapy + 2.12.0, is removed. Use :func:`scrapy.utils.misc.build_from_crawler` + instead. + (:issue:`7126`) + +- The ``scrapy.core.downloader.Downloader._get_slot_key()`` function, + deprecated in Scrapy 2.12.0, is removed. Use + :meth:`scrapy.core.downloader.Downloader.get_slot_key` instead. + (:issue:`7126`) + +- The ``scrapy.twisted_version`` attribute, deprecated in Scrapy 2.12.0, is + removed. You should instead use the :attr:`twisted.version` attribute + directly. + (:issue:`7126`) + +- The following utility functions, deprecated in Scrapy 2.12.0, are removed: + + - ``scrapy.utils.defer.process_chain_both()`` + - ``scrapy.utils.python.equal_attributes()`` + - ``scrapy.utils.python.flatten()`` + - ``scrapy.utils.python.iflatten()`` + - ``scrapy.utils.request.request_authenticate()`` + - ``scrapy.utils.test.assert_samelines()`` + + (:issue:`7126`) + +- ``scrapy.utils.serialize.ScrapyJSONDecoder``, deprecated in Scrapy 2.12.0, + is removed. + (:issue:`7126`) + +- The ``scrapy.extensions.feedexport.build_storage()`` function, deprecated + in Scrapy 2.12.0, is removed, you can instead call the builder callable + directly. + (:issue:`7126`) + +- ``scrapy.spidermiddlewares.offsite.OffsiteMiddleware``, deprecated in + Scrapy 2.11.2, is removed. + :class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` should be + used instead. + (:issue:`6926`) + +Deprecations +~~~~~~~~~~~~ + +- The following methods that return a + :class:`~twisted.internet.defer.Deferred` are deprecated in favor of their + coroutine-based replacements: + + - :class:`scrapy.core.downloader.handlers.DownloadHandlers` + + - ``download_request()`` (use + :meth:`~scrapy.core.downloader.handlers.DownloadHandlers.download_request_async`) + + - :class:`scrapy.core.downloader.middleware.DownloaderMiddlewareManager` + + - ``download()`` (use + :meth:`~scrapy.core.downloader.middleware.DownloaderMiddlewareManager.download_async`) + + - :class:`scrapy.core.engine.ExecutionEngine` + + - ``start()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.start_async`) + + - ``stop()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.stop_async`) + + - ``close()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.close_async`) + + - ``open_spider()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.open_spider_async`) + + - ``close_spider()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.close_spider_async`) + + - ``download()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.download_async`) + + - :class:`scrapy.core.scraper.Scraper` + + - ``open_spider()`` (use + :meth:`~scrapy.core.scraper.Scraper.open_spider_async`) + + - ``call_spider()`` (use + :meth:`~scrapy.core.scraper.Scraper.call_spider_async`) + + - ``close_spider()`` (use + :meth:`~scrapy.core.scraper.Scraper.close_spider_async`) + + - ``handle_spider_output()`` (use + :meth:`~scrapy.core.scraper.Scraper.handle_spider_output_async`) + + - ``start_itemproc()`` (use + :meth:`~scrapy.core.scraper.Scraper.start_itemproc_async`) + + - :class:`scrapy.core.spidermw.SpiderMiddlewareManager` + + - ``scrape_response()`` (use + :meth:`~scrapy.core.spidermw.SpiderMiddlewareManager.scrape_response_async`) + + - :class:`scrapy.crawler.Crawler` + + - ``stop()`` (use :meth:`~scrapy.crawler.Crawler.stop_async`) + + - :class:`scrapy.pipelines.ItemPipelineManager` + + - ``process_item()`` (use + :meth:`~scrapy.pipelines.ItemPipelineManager.process_item_async`) + + - ``open_spider()`` (use + :meth:`~scrapy.pipelines.ItemPipelineManager.open_spider_async`) + + - ``close_spider()`` (use + :meth:`~scrapy.pipelines.ItemPipelineManager.close_spider_async`) + + - :class:`scrapy.signalmanager.SignalManager` + + - ``send_catch_log_deferred()`` (use + :meth:`~scrapy.signalmanager.SignalManager.send_catch_log_async`) + + - ``scrapy.utils.signal.send_catch_log_deferred()`` (use + :func:`scrapy.utils.signal.send_catch_log_async`) + + (:issue:`6791`, :issue:`6842`, :issue:`6979`, :issue:`6997`, :issue:`6999`, + :issue:`7005`, :issue:`7043`, :issue:`7069`, :issue:`7161`, :issue:`7164`) + +- The following spider attributes are deprecated in favor of settings: + + - ``download_maxsize`` (use :setting:`DOWNLOAD_MAXSIZE`) + + - ``download_timeout`` (use :setting:`DOWNLOAD_TIMEOUT`) + + - ``download_warnsize`` (use :setting:`DOWNLOAD_WARNSIZE`) + + - ``max_concurrent_requests`` (use :setting:`CONCURRENT_REQUESTS`) + + - ``user_agent`` (use :setting:`USER_AGENT`) + + (:issue:`6988`, :issue:`6994`, :issue:`7038`, :issue:`7039`, :issue:`7117`, + :issue:`7176`) + +- Returning a :class:`~twisted.internet.defer.Deferred` from the following + user-defined functions is deprecated in favor of defining them as coroutine + functions: + + - spider callbacks and errbacks (which was never officially supported and + may work incorrectly) + + - the ``process_request()``, ``process_response()`` and + ``process_exception()`` methods of custom downloader middlewares + + - the ``process_item()``, ``open_spider()`` and ``close_spider()`` methods + of custom pipelines + + - signal handlers + + - the ``download_request()`` and ``close()`` methods of custom download + handlers + + (:issue:`6718`, :issue:`6778`, :issue:`7069`, :issue:`7147`, :issue:`7148`, + :issue:`7149`, :issue:`7150`, :issue:`7151`, :issue:`7161`, :issue:`7164`, + :issue:`7179`) + +- Passing a ``spider`` argument to the following methods is deprecated: + + - :meth:`scrapy.core.spidermw.SpiderMiddlewareManager.process_start` + + - :meth:`scrapy.core.downloader.Downloader.fetch` + + - :meth:`scrapy.core.downloader.Downloader._get_slot` + + - :meth:`scrapy.core.downloader.handlers.DownloadHandlers.download_request` + + - all public methods of :class:`scrapy.statscollectors.StatsCollector` + + - :meth:`scrapy.spidermiddlewares.base.BaseSpiderMiddleware.process_spider_output` + + - :meth:`scrapy.spidermiddlewares.base.BaseSpiderMiddleware.process_spider_output_async` + + - all ``process_*()`` methods of built-in downloader middlewares + + - all ``process_*()`` methods of built-in spider middlewares + + - :meth:`scrapy.pipelines.media.MediaPipeline.open_spider` + + - :meth:`scrapy.pipelines.media.MediaPipeline.process_item` + + (:issue:`6750`, :issue:`6927`, :issue:`6984`, :issue:`7006`, :issue:`7011`, + :issue:`7033`, :issue:`7037`, :issue:`7045`, :issue:`7178`) + +- Instantiating subclasses of :class:`scrapy.middleware.MiddlewareManager` + without a :class:`~scrapy.crawler.Crawler` instance is deprecated. + (:issue:`6984`) + +- For the following user-defined functions and methods requiring a ``spider`` + argument is deprecated, if you need a :class:`~scrapy.Spider` instance + inside them you should get it from the :class:`~scrapy.crawler.Crawler` + instance (you may need to refactor your code to save that instance in e.g. + the ``from_crawler()`` method): + + - the ``process_request()``, ``process_response()`` and + ``process_exception()`` methods of custom downloader middlewares + + - the ``process_spider_input()``, ``process_spider_output()``, + ``process_spider_output_async()`` and ``process_spider_exception()`` + methods of custom spider middlewares + + - the ``process_item()`` method of custom pipelines + + - the ``fetch()`` method of a custom :setting:`DOWNLOADER` + + (:issue:`6927`, :issue:`6984`, :issue:`7006`, :issue:`7037`) + +- The following things in custom download handlers are deprecated: + + - not having a ``lazy`` attribute (you should define it as ``True`` if you + want to keep the current behavior) + + - returning a :class:`~twisted.internet.defer.Deferred` from the + ``download_request()`` method (you should refactor it to return a + coroutine; you also need to remove the ``spider`` argument when doing + this) + + - not having a ``close()`` method, having a synchronous one or one that + returns a :class:`~twisted.internet.defer.Deferred` (you should refactor + it to return a coroutine or add an empty one if you don't have it) + + (:issue:`6778`, :issue:`7164`) + +- Custom implementations of :setting:`ITEM_PROCESSOR` should now define + ``process_item_async()``, ``open_spider_async()`` and + ``close_spider_async()`` methods instead of, or in addition to, + ``process_item()``, ``open_spider()`` and ``close_spider()``. + (:issue:`7005`, :issue:`7043`) + +- The ``CONCURRENT_REQUESTS_PER_IP`` setting is deprecated, use + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` instead. + (:issue:`6917`, :issue:`6921`) + +- The ``scrapy.core.downloader.handlers.http`` module is deprecated. You + should import + :class:`scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + directly instead of importing the + ``scrapy.core.downloader.handlers.http.HTTPDownloadHandler`` alias. + (:issue:`7079`) + +- The ``scrapy.utils.decorators.defers()`` decorator is deprecated, you can + use :func:`twisted.internet.defer.maybeDeferred` directly or reimplement + this decorator in your code. + (:issue:`7164`) + +- ``scrapy.spiders.CrawlSpider._parse_response()`` is deprecated, use + :meth:`scrapy.spiders.CrawlSpider.parse_with_rules` instead. + (:issue:`4463`, :issue:`6804`) + +- The functions that add a delay to a Deferred are deprecated, their + underlying Twisted functions can be used instead, either directly if a + delay isn't needed, or with some explicit way to add a delay if it's + needed: + + - ``scrapy.utils.defer.mustbe_deferred()`` (you can use + :func:`twisted.internet.defer.maybeDeferred`) + + - ``scrapy.utils.defer.defer_succeed()`` (you can use + :func:`twisted.internet.defer.succeed`) + + - ``scrapy.utils.defer.defer_fail()`` (you can use + :func:`twisted.internet.defer.fail`) + + - ``scrapy.utils.defer.defer_result()`` (you can use + :func:`twisted.internet.defer.succeed` and + :func:`twisted.internet.defer.fail`) + + (:issue:`6937`) + +New features +~~~~~~~~~~~~ + +- Added :class:`scrapy.crawler.AsyncCrawlerProcess` and + :class:`scrapy.crawler.AsyncCrawlerRunner` as counterparts to + :class:`~scrapy.crawler.CrawlerProcess` and + :class:`~scrapy.crawler.CrawlerRunner` that offer coroutine-based APIs. + (:issue:`6789`, :issue:`6790`, :issue:`6796`, :issue:`6817`, :issue:`6845`, + :issue:`7034`) + +- Added coroutine counterparts to some of the Deferred-based APIs: + + - :class:`scrapy.core.downloader.handlers.DownloadHandlers` + + - :meth:`~scrapy.core.downloader.handlers.DownloadHandlers.download_request_async` + (to ``download_request()``) + + - :class:`scrapy.core.downloader.middleware.DownloaderMiddlewareManager` + + - :meth:`~scrapy.core.downloader.middleware.DownloaderMiddlewareManager.download_async` + (to ``download()``) + + - :class:`scrapy.core.engine.ExecutionEngine` + + - :meth:`~scrapy.core.engine.ExecutionEngine.start_async` (to + ``start()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.stop_async` (to + ``stop()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.close_async` (to + ``close()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.open_spider_async` (to + ``open_spider()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.close_spider_async` (to + ``close_spider()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.download_async` (to + ``download()``) + + - :class:`scrapy.core.scraper.Scraper` + + - :meth:`~scrapy.core.scraper.Scraper.open_spider_async` (to + ``open_spider()``) + + - :meth:`~scrapy.core.scraper.Scraper.close_spider_async` (to + ``close_spider()``) + + - :meth:`~scrapy.core.scraper.Scraper.start_itemproc_async` (to + ``start_itemproc()``) + + - :class:`scrapy.crawler.Crawler` + + - :meth:`~scrapy.crawler.Crawler.crawl_async` (to ``crawl()``) + + - :meth:`~scrapy.crawler.Crawler.stop_async` (to ``stop()``) + + - :class:`scrapy.pipelines.ItemPipelineManager` + + - :meth:`~scrapy.pipelines.ItemPipelineManager.process_item_async` (to + ``process_item()``) + + - :meth:`~scrapy.pipelines.ItemPipelineManager.open_spider_async` (to + ``open_spider()``) + + - :meth:`~scrapy.pipelines.ItemPipelineManager.close_spider_async` (to + ``close_spider()``) + + - :class:`scrapy.signalmanager.SignalManager` + + - :meth:`~scrapy.signalmanager.SignalManager.send_catch_log_async` (to + ``send_catch_log_deferred()``) + + (:issue:`6781`, :issue:`6791`, :issue:`6792`, :issue:`6795`, :issue:`6801`, + :issue:`6817`, :issue:`6842`, :issue:`6997`, :issue:`7005`, :issue:`7043`, + :issue:`7069`,:issue:`7164`, :issue:`7202`) + +- The default value of the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is now + ``'scrapy.pqueues.DownloaderAwarePriorityQueue'``. + (:issue:`6924`, :issue:`6940`) + +- Added :class:`scrapy.extensions.logcount.LogCount`, an enabled-by-default + extension that is responsible for the ``log_count/*`` stats. Previously, + this code was in :class:`scrapy.crawler.Crawler` and couldn't be disabled. + (:issue:`7046`) + +- Added :meth:`scrapy.spiders.CrawlSpider.parse_with_rules` as a public + replacement for ``_parse_response()``. + (:issue:`4463`, :issue:`6804`) + +- Added :func:`scrapy.utils.asyncio.is_asyncio_available` as an alternative + to :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` with a + future-proof name and semantics. + (:issue:`6827`) + +- The API for :ref:`download handlers `, previously + undocumented, has been modernized and documented. An optional base class, + :class:`scrapy.core.downloader.handlers.base.BaseDownloadHandler`, has been + added to simplify writing custom download handlers that conform to the + current API. + (:issue:`4944`, :issue:`6778`, :issue:`7164`) + +- Added :func:`scrapy.utils.defer.ensure_awaitable`, which can be helpful to + call user-defined functions that can return coroutines, Deferreds or + values directly. + (:issue:`7005`) + +- The ``requests.seen`` file, written by + :class:`~scrapy.dupefilters.RFPDupeFilter` when :ref:`job persistence + ` is enabled, now uses line buffering to reduce data loss in + spider crashes. + (:issue:`6019`, :issue:`7094`) + +- Images downloaded by :class:`~scrapy.pipelines.images.ImagesPipeline` are + now automatically transposed based on EXIF data. + (:issue:`6525`, :issue:`6975`) + +Improvements +~~~~~~~~~~~~ + +- Refactored internal functions to use coroutines instead of Deferreds. + (:issue:`6795`, :issue:`6852`, :issue:`6855`, :issue:`6858`, :issue:`7159`) + +- Commands that don't need a :class:`~scrapy.crawler.CrawlerProcess` instance + no longer create it. + (:issue:`6824`) + +- Improved :command:`shell` help formatting when using IPython 9+. + (:issue:`6915`, :issue:`6980`) + +Bug fixes +~~~~~~~~~ + +- Setting :setting:`FILES_STORE` or :setting:`IMAGES_STORE` to ``None`` now + correctly disables the respective pipeline. + (:issue:`6964`, :issue:`6969`) + +- :class:`~scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware` now + uses the URL set in the ```` tag as the base URL when redirecting to + a relative URL. + (:issue:`7042`, :issue:`7047`) + +- Passing ``None`` as a value of the :reqmeta:`download_slot` request meta + key is now handled in the same way as not setting this meta key at all. + (:issue:`7172`) + +- Fixed parsing of the first line of ``robots.txt`` files that have a BOM. + (:issue:`6195`, :issue:`7095`) + +Documentation +~~~~~~~~~~~~~ + +- Added :ref:`documentation ` about download + handlers, their API and built-in handlers. + (:issue:`4944`, :issue:`7164`) + +- Added a section about the `scrapy-spider-metadata`_ library to the + :ref:`spider argument docs `. + (:issue:`6676`, :issue:`6957`, :issue:`7116`) + + .. _scrapy-spider-metadata: https://scrapy-spider-metadata.readthedocs.io/en/latest/ + +- Improved :ref:`the docs ` about coroutine-based + and Deferred-based APIs. + (:issue:`6800`, :issue:`7146`) + +- Other documentation improvements and fixes. + (:issue:`7058`, :issue:`7076`, :issue:`7109`, :issue:`7195`, :issue:`7198`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Switched from ``twisted.trial`` to ``pytest-twisted`` and replaced + remaining ``unittest`` and ``twisted.trial`` features with ``pytest`` ones. + (:issue:`6658`, :issue:`6873`, :issue:`6884`, :issue:`6938`) + +- Enabled fancy ``pytest`` asserts. + (:issue:`6888`) + +- Added `Sphinx Lint`_ to the ``pre-commit`` configuration. + (:issue:`6920`) + + .. _Sphinx Lint: https://github.com/sphinx-contrib/sphinx-lint + +- CI and test improvements and fixes. + (:issue:`6649`, + :issue:`6769`, + :issue:`6821`, + :issue:`6835`, + :issue:`6836`, + :issue:`6846`, + :issue:`6883`, + :issue:`6885`, + :issue:`6889`, + :issue:`6905`, + :issue:`6928`, + :issue:`6933`, + :issue:`6941`, + :issue:`6942`, + :issue:`6945`, + :issue:`6947`, + :issue:`6960`, + :issue:`6968`, + :issue:`6972`, + :issue:`6974`, + :issue:`6996`, + :issue:`7003`, + :issue:`7012`, + :issue:`7013`, + :issue:`7050`, + :issue:`7059`, + :issue:`7070`, + :issue:`7073`, + :issue:`7118`, + :issue:`7127`, + :issue:`7141`, + :issue:`7143`, + :issue:`7145`, + :issue:`7173`) + +- Code cleanups. + (:issue:`6803`, + :issue:`6838`, + :issue:`6849`, + :issue:`6875`, + :issue:`6876`, + :issue:`6892`, + :issue:`6930`, + :issue:`6949`, + :issue:`6970`, + :issue:`6977`, + :issue:`6986`, + :issue:`7008`, + :issue:`7177`) + +.. _release-2.13.4: + +Scrapy 2.13.4 (2025-11-17) +-------------------------- + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- Improved protection against decompression bombs in + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + for responses compressed using the ``br`` and ``deflate`` methods: if a + single compressed chunk would be larger than the response size limit (see + :setting:`DOWNLOAD_MAXSIZE`) when decompressed, decompression is no longer + carried out. This is especially important for the ``br`` (Brotli) method + that can provide a very high compression ratio. Please, see the + `CVE-2025-6176`_ and `GHSA-2qfp-q593-8484`_ security advisories for more + information. + (:issue:`7134`) + + .. _CVE-2025-6176: https://nvd.nist.gov/vuln/detail/CVE-2025-6176 + .. _GHSA-2qfp-q593-8484: https://github.com/advisories/GHSA-2qfp-q593-8484 + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- The minimum supported version of the optional ``brotli`` package is now + ``1.2.0``. + (:issue:`7134`) + +- The ``brotlicffi`` and ``brotlipy`` packages can no longer be used to + decompress Brotli-compressed responses. Please install the ``brotli`` + package instead. + (:issue:`7134`) + +Other changes +~~~~~~~~~~~~~ + +- Restricted the maximum supported Twisted version to ``25.5.0``, as Scrapy + currently uses some private APIs changed in later Twisted versions. + (:issue:`7142`) + +- Stopped setting the ``COVERAGE_CORE`` environment variable in tests, it + didn't have an effect but caused the ``coverage`` module to produce a + warning or an error. + (:issue:`7137`) + +- Removed the documentation build dependency on the deprecated + ``sphinx-hoverxref`` module. + (:issue:`6786`, :issue:`6922`) + +.. _release-2.13.3: + +Scrapy 2.13.3 (2025-07-02) +-------------------------- + +- Changed the values for :setting:`DOWNLOAD_DELAY` (from ``0`` to ``1``) and + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (from ``8`` to ``1``) in the + default project template. + (:issue:`6597`, :issue:`6918`, :issue:`6923`) + +- Improved :class:`scrapy.core.engine.ExecutionEngine` logic related to + initialization and exception handling, fixing several cases where the + spider would crash, hang or log an unhandled exception. + (:issue:`6783`, :issue:`6784`, :issue:`6900`, :issue:`6908`, :issue:`6910`, + :issue:`6911`) + +- Fixed a Windows issue with :ref:`feed exports ` using + :class:`scrapy.extensions.feedexport.FileFeedStorage` that caused the file + to be created on the wrong drive. + (:issue:`6894`, :issue:`6897`) + +- Allowed running tests with Twisted 25.5.0+ again. Pytest 8.4.1+ is now + required for running tests in non-pinned envs as support for the new + Twisted version was added in that version. + (:issue:`6893`) + +- Fixed running tests with lxml 6.0.0+. + (:issue:`6919`) + +- Added a deprecation notice for + ``scrapy.spidermiddlewares.offsite.OffsiteMiddleware`` to :ref:`the Scrapy + 2.11.2 release notes `. + (:issue:`6926`) + +- Updated :ref:`contribution docs ` to refer to ruff_ + instead of black_. + (:issue:`6903`) + +- Added ``.venv/`` and ``.vscode/`` to ``.gitignore``. + (:issue:`6901`, :issue:`6907`) + + +.. _release-2.13.2: + +Scrapy 2.13.2 (2025-06-09) +-------------------------- + +- Fixed a bug introduced in Scrapy 2.13.0 that caused results of request + errbacks to be ignored when the errback was called because of a downloader + error. + (:issue:`6861`, :issue:`6863`) + +- Added a note about the behavior change of + :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to its docs and + to the "Backward-incompatible changes" section of :ref:`the Scrapy 2.13.0 + release notes `. + (:issue:`6866`) + +- Improved the message in the exception raised by + :func:`scrapy.utils.test.get_reactor_settings` when there is no reactor + installed. + (:issue:`6866`) + +- Updated the :class:`scrapy.crawler.CrawlerRunner` examples in + :ref:`topics-practices` to install the reactor explicitly, to fix + reactor-related errors with Scrapy 2.13.0 and later. + (:issue:`6865`) + +- Fixed ``scrapy fetch`` not working with scrapy-poet_. + (:issue:`6872`) + +- Fixed an exception produced by :class:`scrapy.core.engine.ExecutionEngine` + when it's closed before being fully initialized. + (:issue:`6857`, :issue:`6867`) + +- Improved the README, updated the Scrapy logo in it. + (:issue:`6831`, :issue:`6833`, :issue:`6839`) + +- Restricted the Twisted version used in tests to below 25.5.0, as some tests + fail with 25.5.0. + (:issue:`6878`, :issue:`6882`) + +- Updated type hints for Twisted 25.5.0 changes. + (:issue:`6882`) + +- Removed the old artwork. + (:issue:`6874`) + + +.. _release-2.13.1: + +Scrapy 2.13.1 (2025-05-28) +-------------------------- + +- Give callback requests precedence over start requests when priority values + are the same. + + This makes changes from 2.13.0 to start request handling more intuitive and + backward compatible. For scenarios where all requests have the same + priorities, in 2.13.0 all start requests were sent before the first + callback request. In 2.13.1, same as in 2.12 and lower, start requests are + only sent when there are not enough pending callback requests to reach + concurrency limits. + + (:issue:`6828`) + +- Added a deepwiki_ badge to the README. (:issue:`6793`) + + .. _deepwiki: https://deepwiki.com/scrapy/scrapy + +- Fixed a typo in the code example of :ref:`start-requests-lazy`. + (:issue:`6812`, :issue:`6815`) + +- Fixed a typo in the :ref:`coroutine-support` section of the documentation. + (:issue:`6822`) + +- Made this page more prominently listed in PyPI project links. + (:issue:`6826`) + + +.. _release-2.13.0: + +Scrapy 2.13.0 (2025-05-08) +-------------------------- + +Highlights: + +- The asyncio reactor is now enabled by default + +- Replaced ``start_requests()`` (sync) with :meth:`~scrapy.Spider.start` + (async) and changed how it is iterated + +- Added the :reqmeta:`allow_offsite` request meta key + +- Spider middlewares that don't support asynchronous spider output are + deprecated + +- Added a base class for :ref:`universal spider middlewares + ` + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for PyPy 3.9. + (:issue:`6613`) + +- Added support for PyPy 3.11. + (:issue:`6697`) + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The default value of the :setting:`TWISTED_REACTOR` setting was changed + from ``None`` to + ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``. This value + was used in newly generated projects since Scrapy 2.7.0 but now existing + projects that don't explicitly set this setting will also use the asyncio + reactor. You can :ref:`change this setting in your project + ` to use a different reactor. + (:issue:`6659`, :issue:`6713`) + +- The iteration of start requests and items no longer stops once there are + requests in the scheduler, and instead runs continuously until all start + requests have been scheduled. + + To reproduce the previous behavior, see :ref:`start-requests-lazy`. + (:issue:`6729`) + +- An unhandled exception from the + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.open_spider` method of a + :ref:`spider middleware ` no longer stops the + crawl. + (:issue:`6729`) + +- In ``scrapy.core.engine.ExecutionEngine``: + + - The second parameter of ``open_spider()``, ``start_requests``, has been + removed. The start requests are determined by the ``spider`` parameter + instead (see :meth:`~scrapy.Spider.start`). + + - The ``slot`` attribute has been renamed to ``_slot`` and should not be + used. + + (:issue:`6729`) + +- In ``scrapy.core.engine``, the ``Slot`` class has been renamed to ``_Slot`` + and should not be used. + (:issue:`6729`) + +- The ``slot`` :ref:`telnet variable ` has been removed. + (:issue:`6729`) + +- In ``scrapy.core.spidermw.SpiderMiddlewareManager``, + ``process_start_requests()`` has been replaced by ``process_start()``. + (:issue:`6729`) + +- The now-deprecated ``start_requests()`` method, when it returns an iterable + instead of being defined as a generator, is now executed *after* the + :ref:`scheduler ` instance has been created. + (:issue:`6729`) + +- When using :setting:`JOBDIR`, :ref:`start requests ` are + now serialized into their own, ``s``-suffixed priority folders. You can set + :setting:`SCHEDULER_START_DISK_QUEUE` to ``None`` or ``""`` to change that, + but the side effects may be undesirable. See + :setting:`SCHEDULER_START_DISK_QUEUE` for details. + (:issue:`6729`) + +- The URL length limit, set by the :setting:`URLLENGTH_LIMIT` setting, is now + also enforced for start requests. + (:issue:`6777`) + +- Calling :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` without + an installed reactor now raises an exception instead of installing a + reactor. This shouldn't affect normal Scrapy use cases, but it may affect + 3rd-party test suites that use Scrapy internals such as + :class:`~scrapy.crawler.Crawler` and don't install a reactor explicitly. If + you are affected by this change, you most likely need to install the + reactor before running Scrapy code that expects it to be installed. + (:issue:`6732`, :issue:`6735`) + +- The ``from_settings()`` method of + :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`, + deprecated in Scrapy 2.12.0, is removed earlier than the usual deprecation + period (this was needed because after the introduction of the + :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware` base class and + switching built-in spider middlewares to it those middlewares need the + :class:`~scrapy.crawler.Crawler` instance at run time). Please use + ``from_crawler()`` instead. + (:issue:`6693`) + +- ``scrapy.utils.url.escape_ajax()`` is no longer called when a + :class:`~scrapy.Request` instance is created. It was only useful for + websites supporting the ``_escaped_fragment_`` feature which most modern + websites don't support. If you still need this you can modify the URLs + before passing them to :class:`~scrapy.Request`. + (:issue:`6523`, :issue:`6651`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed old deprecated name aliases for some signals: + + - ``stats_spider_opened`` (use ``spider_opened`` instead) + + - ``stats_spider_closing`` and ``stats_spider_closed`` (use + ``spider_closed`` instead) + + - ``item_passed`` (use ``item_scraped`` instead) + + - ``request_received`` (use ``request_scheduled`` instead) + + (:issue:`6654`, :issue:`6655`) + +Deprecations +~~~~~~~~~~~~ + +- The ``start_requests()`` method of :class:`~scrapy.Spider` is deprecated, + use :meth:`~scrapy.Spider.start` instead, or both to maintain support for + lower Scrapy versions. + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- The ``process_start_requests()`` method of :ref:`spider middlewares + ` is deprecated, use + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead, + or both to maintain support for lower Scrapy versions. + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- The ``__init__`` method of priority queue classes (see + :setting:`SCHEDULER_PRIORITY_QUEUE`) should now support a keyword-only + ``start_queue_cls`` parameter. + (:issue:`6752`) + +- Spider middlewares that don't support asynchronous spider output are + deprecated. The async iterable downgrading feature, needed for using such + middlewares with asynchronous callbacks and with other spider middlewares + that produce asynchronous iterables, is also deprecated. Please update all + such middlewares to support asynchronous spider output. (:issue:`6664`) + +- Functions that were imported from :mod:`w3lib.url` and re-exported in + :mod:`scrapy.utils.url` are now deprecated, you should import them from + :mod:`w3lib.url` directly. They are: + + - ``scrapy.utils.url.add_or_replace_parameter()`` + + - ``scrapy.utils.url.add_or_replace_parameters()`` + + - ``scrapy.utils.url.any_to_uri()`` + + - ``scrapy.utils.url.canonicalize_url()`` + + - ``scrapy.utils.url.file_uri_to_path()`` + + - ``scrapy.utils.url.is_url()`` + + - ``scrapy.utils.url.parse_data_uri()`` + + - ``scrapy.utils.url.parse_url()`` + + - ``scrapy.utils.url.path_to_file_uri()`` + + - ``scrapy.utils.url.safe_download_url()`` + + - ``scrapy.utils.url.safe_url_string()`` + + - ``scrapy.utils.url.url_query_cleaner()`` + + - ``scrapy.utils.url.url_query_parameter()`` + + (:issue:`4577`, :issue:`6583`, :issue:`6586`) + +- HTTP/1.0 support code is deprecated. It was disabled by default and + couldn't be used together with HTTP/1.1. If you still need it, you should + write your own download handler or copy the code from Scrapy. The + deprecations include: + + - ``scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler`` + + - ``scrapy.core.downloader.webclient.ScrapyHTTPClientFactory`` + + - ``scrapy.core.downloader.webclient.ScrapyHTTPPageGetter`` + + - Overriding + ``scrapy.core.downloader.contextfactory.ScrapyClientContextFactory.getContext()`` + + (:issue:`6634`) + +- The following modules and functions used only in tests are deprecated: + + - the ``scrapy.utils.testproc`` module + + - the ``scrapy.utils.testsite`` module + + - ``scrapy.utils.test.assert_gcs_environ()`` + + - ``scrapy.utils.test.get_ftp_content_and_delete()`` + + - ``scrapy.utils.test.get_gcs_content_and_delete()`` + + - ``scrapy.utils.test.mock_google_cloud_storage()`` + + - ``scrapy.utils.test.skip_if_no_boto()`` + + If you need to use them in your tests or code, you can copy the code from Scrapy. + (:issue:`6696`) + +- ``scrapy.utils.test.TestSpider`` is deprecated. If you need an empty spider + class you can use :class:`scrapy.utils.spider.DefaultSpider` or create your + own subclass of :class:`scrapy.Spider`. + (:issue:`6678`) + +- ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` is + deprecated. It was disabled by default and isn't useful for most of the + existing websites. + (:issue:`6523`, :issue:`6651`, :issue:`6656`) + +- ``scrapy.utils.url.escape_ajax()`` is deprecated. + (:issue:`6523`, :issue:`6651`) + +- ``scrapy.spiders.init.InitSpider`` is deprecated. If you find it useful, + you can copy its code from Scrapy. + (:issue:`6708`, :issue:`6714`) + +- ``scrapy.utils.versions.scrapy_components_versions()`` is deprecated, use + :func:`scrapy.utils.versions.get_versions` instead. + (:issue:`6582`) + +- ``BaseDupeFilter.log()`` is deprecated. It does nothing and shouldn't be + called. + (:issue:`4151`) + +- Passing the ``spider`` argument to the following methods of + :class:`~scrapy.core.scraper.Scraper` is deprecated: + + - ``close_spider()`` + + - ``enqueue_scrape()`` + + - ``handle_spider_error()`` + + - ``handle_spider_output()`` + + (:issue:`6764`) + +New features +~~~~~~~~~~~~ + +- You can now yield the start requests and items of a spider from the + :meth:`~scrapy.Spider.start` spider method and from the + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` spider + middleware method, both :term:`asynchronous generators `. + + This makes it possible to use asynchronous code to generate those start + requests and items, e.g. reading them from a queue service or database + using an asynchronous client, without workarounds. + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- Start requests are now :ref:`scheduled ` as soon as + possible. + + As a result, their :attr:`~scrapy.Request.priority` is now taken into + account as soon as :setting:`CONCURRENT_REQUESTS` is reached. + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- :class:`Crawler.signals ` has a new + :meth:`~scrapy.signalmanager.SignalManager.wait_for` method. + (:issue:`6729`) + +- Added a new :signal:`scheduler_empty` signal. + (:issue:`6729`) + +- Added new settings: :setting:`SCHEDULER_START_DISK_QUEUE` and + :setting:`SCHEDULER_START_MEMORY_QUEUE`. + (:issue:`6729`) + +- Added :class:`~scrapy.spidermiddlewares.start.StartSpiderMiddleware`, which + sets :reqmeta:`is_start_request` to ``True`` on :ref:`start requests + `. + (:issue:`6729`) + +- Exposed a new method of :class:`Crawler.engine + `: + :meth:`~scrapy.core.engine.ExecutionEngine.needs_backout`. + (:issue:`6729`) + +- Added the :reqmeta:`allow_offsite` request meta key that can be used + instead of the more general :attr:`~scrapy.Request.dont_filter` request + attribute to skip processing of the request by + :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` (but not + by other code that checks :attr:`~scrapy.Request.dont_filter`). + (:issue:`3690`, :issue:`6151`, :issue:`6366`) + +- Added an optional base class for spider middlewares, + :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware`, which can be + helpful for writing :ref:`universal spider middlewares + ` without boilerplate and code duplication. + The built-in spider middlewares now inherit from this class. + (:issue:`6693`, :issue:`6777`) + +- :ref:`Scrapy add-ons ` can now define a class method called + ``update_pre_crawler_settings()`` to update :ref:`pre-crawler settings + `. + (:issue:`6544`, :issue:`6568`) + +- Added :ref:`helpers ` for modifying :ref:`component + priority dictionary ` settings. + (:issue:`6614`) + +- Responses that use an unknown/unsupported encoding now produce a warning. + If Scrapy knows that installing an additional package (such as brotli_) + will allow decoding the response, that will be mentioned in the warning. + (:issue:`4697`, :issue:`6618`) + +- Added the ``spider_exceptions/count`` stat which tracks the total count of + exceptions (tracked also by per-type ``spider_exceptions/*`` stats). + (:issue:`6739`, :issue:`6740`) + +- Added the :setting:`DEFAULT_DROPITEM_LOG_LEVEL` setting and the + :attr:`scrapy.exceptions.DropItem.log_level` attribute that allow + customizing the log level of the message that is logged when an item is + dropped. + (:issue:`6603`, :issue:`6608`) + +- Added support for the ``-b, --cookie`` curl argument to + :meth:`scrapy.Request.from_curl`. + (:issue:`6684`) + +- Added the :setting:`LOG_VERSIONS` setting that allows customizing the + list of software whose versions are logged when the spider starts. + (:issue:`6582`) + +- Added the :setting:`WARN_ON_GENERATOR_RETURN_VALUE` setting that allows + disabling run time analysis of callback code used to warn about incorrect + ``return`` statements in generator-based callbacks. You may need to disable + this setting if this analysis breaks on your callback code. + (:issue:`6731`, :issue:`6738`) + +Improvements +~~~~~~~~~~~~ + +- Removed or postponed some calls of :func:`itemadapter.is_item` to increase + performance. + (:issue:`6719`) + +- Improved the error message when running a ``scrapy`` command that requires + a project (such as ``scrapy crawl``) outside of a project directory. + (:issue:`2349`, :issue:`3426`) + +- Added an empty :setting:`ADDONS` setting to the ``settings.py`` template + for new projects. + (:issue:`6587`) + +Bug fixes +~~~~~~~~~ + +- Yielding an item from :meth:`Spider.start ` or from + :meth:`SpiderMiddleware.process_start + ` no longer delays + the next iteration of starting requests and items by up to 5 seconds. + (:issue:`6729`) + +- Fixed calculation of ``items_per_minute`` and ``responses_per_minute`` + stats. + (:issue:`6599`) + +- Fixed an error initializing + :class:`scrapy.extensions.feedexport.GCSFeedStorage`. + (:issue:`6617`, :issue:`6628`) + +- Fixed an error running ``scrapy bench``. + (:issue:`6632`, :issue:`6633`) + +- Fixed duplicated log messages about the reactor and the event loop. + (:issue:`6636`, :issue:`6657`) + +- Fixed resolving type annotations of ``SitemapSpider._parse_sitemap()`` at + run time, required by tools such as scrapy-poet_. + (:issue:`6665`, :issue:`6671`) + + .. _scrapy-poet: https://github.com/scrapinghub/scrapy-poet + +- Calling :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` without + an installed reactor now raises an exception instead of installing a + reactor. + (:issue:`6732`, :issue:`6735`) + +- Restored support for the ``x-gzip`` content encoding. + (:issue:`6618`) + +Documentation +~~~~~~~~~~~~~ + +- Documented the setting values set in the default project template. + (:issue:`6762`, :issue:`6775`) + +- Improved the docs about asynchronous iterable support in spider + middlewares. (:issue:`6688`) + +- Improved the :ref:`docs ` about using + :class:`~twisted.internet.defer.Deferred`-based APIs in coroutine-based + code and included a list of such APIs. + (:issue:`6677`, :issue:`6734`, :issue:`6776`) + +- Improved the :ref:`contribution docs `. + (:issue:`6561`, :issue:`6575`) + +- Removed the ``Splash`` recommendation from the :ref:`headless browser + ` suggestion. We no longer recommend using + ``Splash`` and recommend using other headless browser solutions instead. + (:issue:`6642`, :issue:`6701`) + +- Added the dark mode to the HTML documentation. + (:issue:`6653`) + +- Other documentation improvements and fixes. + (:issue:`4151`, + :issue:`6526`, + :issue:`6620`, + :issue:`6621`, + :issue:`6622`, + :issue:`6623`, + :issue:`6624`, + :issue:`6721`, + :issue:`6723`, + :issue:`6780`) + +Packaging +~~~~~~~~~ + +- Switched from ``setup.py`` to ``pyproject.toml``. + (:issue:`6514`, :issue:`6547`) + +- Switched the build backend from setuptools_ to hatchling_. + (:issue:`6771`) + + .. _hatchling: https://pypi.org/project/hatchling/ + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Replaced most linters with ruff_. + (:issue:`6565`, + :issue:`6576`, + :issue:`6577`, + :issue:`6581`, + :issue:`6584`, + :issue:`6595`, + :issue:`6601`, + :issue:`6631`) + + .. _ruff: https://docs.astral.sh/ruff/ + +- Improved accuracy and performance of collecting test coverage. + (:issue:`6255`, :issue:`6610`) + +- Fixed an error that prevented running tests from directories other than the + top level source directory. + (:issue:`6567`) + +- Reduced the amount of ``mockserver`` calls in tests to improve the overall + test run time. + (:issue:`6637`, :issue:`6648`) + +- Fixed tests that were running the same test code more than once. + (:issue:`6646`, :issue:`6647`, :issue:`6650`) + +- Refactored tests to use more ``pytest`` features instead of ``unittest`` + ones where possible. + (:issue:`6678`, + :issue:`6680`, + :issue:`6695`, + :issue:`6699`, + :issue:`6700`, + :issue:`6702`, + :issue:`6709`, + :issue:`6710`, + :issue:`6711`, + :issue:`6712`, + :issue:`6725`) + +- Type hints improvements and fixes. + (:issue:`6578`, + :issue:`6579`, + :issue:`6593`, + :issue:`6605`, + :issue:`6694`) + +- CI and test improvements and fixes. + (:issue:`5360`, + :issue:`6271`, + :issue:`6547`, + :issue:`6560`, + :issue:`6602`, + :issue:`6607`, + :issue:`6609`, + :issue:`6613`, + :issue:`6619`, + :issue:`6626`, + :issue:`6679`, + :issue:`6703`, + :issue:`6704`, + :issue:`6716`, + :issue:`6720`, + :issue:`6722`, + :issue:`6724`, + :issue:`6741`, + :issue:`6743`, + :issue:`6766`, + :issue:`6770`, + :issue:`6772`, + :issue:`6773`) + +- Code cleanups. + (:issue:`6600`, + :issue:`6606`, + :issue:`6635`, + :issue:`6764`) + + +.. _release-2.12.0: + +Scrapy 2.12.0 (2024-11-18) +-------------------------- + +Highlights: + +- Dropped support for Python 3.8, added support for Python 3.13 + +- ``scrapy.Spider.start_requests()`` can now yield items + +- Added :class:`~scrapy.http.JsonResponse` + +- Added :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM` + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for Python 3.8. + (:issue:`6466`, :issue:`6472`) + +- Added support for Python 3.13. + (:issue:`6166`) + +- Minimum versions increased for these dependencies: + + - Twisted_: 18.9.0 → 21.7.0 + + - cryptography_: 36.0.0 → 37.0.0 + + - pyOpenSSL_: 21.0.0 → 22.0.0 + + - lxml_: 4.4.1 → 4.6.0 + +- Removed ``setuptools`` from the dependency list. + (:issue:`6487`) + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- User-defined cookies for HTTPS requests will have the ``secure`` flag set + to ``True`` unless it's set to ``False`` explicitly. This is important when + these cookies are reused in HTTP requests, e.g. after a redirect to an HTTP + URL. + (:issue:`6357`) + +- The Reppy-based ``robots.txt`` parser, + ``scrapy.robotstxt.ReppyRobotParser``, was removed, as it doesn't support + Python 3.9+. + (:issue:`5230`, :issue:`6099`, :issue:`6499`) + +- The initialization API of :class:`scrapy.pipelines.media.MediaPipeline` and + its subclasses was improved and it's possible that some previously working + usage scenarios will no longer work. It can only affect you if you define + custom subclasses of ``MediaPipeline`` or create instances of these + pipelines via ``from_settings()`` or ``__init__()`` calls instead of + ``from_crawler()`` calls. + + Previously, ``MediaPipeline.from_crawler()`` called the ``from_settings()`` + method if it existed or the ``__init__()`` method otherwise, and then did + some additional initialization using the ``crawler`` instance. If the + ``from_settings()`` method existed (like in ``FilesPipeline``) it called + ``__init__()`` to create the instance. It wasn't possible to override + ``from_crawler()`` without calling ``MediaPipeline.from_crawler()`` from it + which, in turn, couldn't be called in some cases (including subclasses of + ``FilesPipeline``). + + Now, in line with the general usage of ``from_crawler()`` and + ``from_settings()`` and the deprecation of the latter the recommended + initialization order is the following one: + + - All ``__init__()`` methods should take a ``crawler`` argument. If they + also take a ``settings`` argument they should ignore it, using + ``crawler.settings`` instead. When they call ``__init__()`` of the base + class they should pass the ``crawler`` argument to it too. + - A ``from_settings()`` method shouldn't be defined. Class-specific + initialization code should go into either an overridden ``from_crawler()`` + method or into ``__init__()``. + - It's now possible to override ``from_crawler()`` and it's not necessary + to call ``MediaPipeline.from_crawler()`` in it if other recommendations + were followed. + - If pipeline instances were created with ``from_settings()`` or + ``__init__()`` calls (which wasn't supported even before, as it missed + important initialization code), they should now be created with + ``from_crawler()`` calls. + + (:issue:`6540`) + +- The ``response_body`` argument of :meth:`ImagesPipeline.convert_image + ` is now + positional-only, as it was changed from optional to required. + (:issue:`6500`) + +- The ``convert`` argument of :func:`scrapy.utils.conf.build_component_list` + is now positional-only, as the preceding argument (``custom``) was removed. + (:issue:`6500`) + +- The ``overwrite_output`` argument of + :func:`scrapy.utils.conf.feed_process_params_from_cli` is now + positional-only, as the preceding argument (``output_format``) was removed. + (:issue:`6500`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed the ``scrapy.utils.request.request_fingerprint()`` function, + deprecated in Scrapy 2.7.0. + (:issue:`6212`, :issue:`6213`) + +- Removed support for value ``"2.6"`` of setting + ``REQUEST_FINGERPRINTER_IMPLEMENTATION``, deprecated in Scrapy 2.7.0. + (:issue:`6212`, :issue:`6213`) + +- :class:`~scrapy.dupefilters.RFPDupeFilter` subclasses now require + supporting the ``fingerprinter`` parameter in their ``__init__`` method, + introduced in Scrapy 2.7.0. + (:issue:`6102`, :issue:`6113`) + +- Removed the ``scrapy.downloadermiddlewares.decompression`` module, + deprecated in Scrapy 2.7.0. + (:issue:`6100`, :issue:`6113`) + +- Removed the ``scrapy.utils.response.response_httprepr()`` function, + deprecated in Scrapy 2.6.0. + (:issue:`6111`, :issue:`6116`) + +- Spiders with spider-level HTTP authentication, i.e. with the ``http_user`` + or ``http_pass`` attributes, must now define ``http_auth_domain`` as well, + which was introduced in Scrapy 2.5.1. + (:issue:`6103`, :issue:`6113`) + +- :ref:`Media pipelines ` methods ``file_path()``, + ``file_downloaded()``, ``get_images()``, ``image_downloaded()``, + ``media_downloaded()``, ``media_to_download()``, and ``thumb_path()`` must + now support an ``item`` parameter, added in Scrapy 2.4.0. + (:issue:`6107`, :issue:`6113`) + +- The ``__init__()`` and ``from_crawler()`` methods of :ref:`feed storage + backend classes ` must now support the keyword-only + ``feed_options`` parameter, introduced in Scrapy 2.4.0. + (:issue:`6105`, :issue:`6113`) + +- Removed the ``scrapy.loader.common`` and ``scrapy.loader.processors`` + modules, deprecated in Scrapy 2.3.0. + (:issue:`6106`, :issue:`6113`) + +- Removed the ``scrapy.utils.misc.extract_regex()`` function, deprecated in + Scrapy 2.3.0. + (:issue:`6106`, :issue:`6113`) + +- Removed the ``scrapy.http.JSONRequest`` class, replaced with + ``JsonRequest`` in Scrapy 1.8.0. + (:issue:`6110`, :issue:`6113`) + +- ``scrapy.utils.log.logformatter_adapter`` no longer supports missing + ``args``, ``level``, or ``msg`` parameters, and no longer supports a + ``format`` parameter, all scenarios that were deprecated in Scrapy 1.0.0. + (:issue:`6109`, :issue:`6116`) + +- A custom class assigned to the :setting:`SPIDER_LOADER_CLASS` setting that + does not implement the :class:`~scrapy.interfaces.ISpiderLoader` interface + will now raise a :exc:`zope.interface.verify.DoesNotImplement` exception at + run time. Non-compliant classes have been triggering a deprecation warning + since Scrapy 1.0.0. + (:issue:`6101`, :issue:`6113`) + +- Removed the ``--output-format``/``-t`` command line option, deprecated in + Scrapy 2.1.0. ``-O :`` should be used instead. + (:issue:`6500`) + +- Running :meth:`~scrapy.crawler.Crawler.crawl` more than once on the same + :class:`~scrapy.crawler.Crawler` instance, deprecated in Scrapy 2.11.0, now + raises an exception. + (:issue:`6500`) + +- Subclassing + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + without support for the ``crawler`` argument in ``__init__()`` and without + a custom ``from_crawler()`` method, deprecated in Scrapy 2.5.0, is no + longer allowed. + (:issue:`6500`) + +- Removed the ``EXCEPTIONS_TO_RETRY`` attribute of + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware`, deprecated in + Scrapy 2.10.0. + (:issue:`6500`) + +- Removed support for :ref:`S3 feed exports ` without + the boto3_ package installed, deprecated in Scrapy 2.10.0. + (:issue:`6500`) + +- Removed the ``scrapy.extensions.feedexport._FeedSlot`` class, deprecated in + Scrapy 2.10.0. + (:issue:`6500`) + +- Removed the ``scrapy.pipelines.images.NoimagesDrop`` exception, deprecated + in Scrapy 2.8.0. + (:issue:`6500`) + +- The ``response_body`` argument of :meth:`ImagesPipeline.convert_image + ` is now required, + not passing it was deprecated in Scrapy 2.8.0. + (:issue:`6500`) + +- Removed the ``custom`` argument of + :func:`scrapy.utils.conf.build_component_list`, deprecated in Scrapy + 2.10.0. + (:issue:`6500`) + +- Removed the ``scrapy.utils.reactor.get_asyncio_event_loop_policy()`` + function, deprecated in Scrapy 2.9.0. Use :func:`asyncio.get_event_loop` + and related standard library functions instead. + (:issue:`6500`) + +Deprecations +~~~~~~~~~~~~ + +- The ``from_settings()`` methods of the :ref:`Scrapy components + ` that have them are now deprecated. ``from_crawler()`` + should now be used instead. Affected components: + + - :class:`scrapy.dupefilters.RFPDupeFilter` + - :class:`scrapy.mail.MailSender` + - :class:`scrapy.middleware.MiddlewareManager` + - :class:`scrapy.core.downloader.contextfactory.ScrapyClientContextFactory` + - :class:`scrapy.pipelines.files.FilesPipeline` + - :class:`scrapy.pipelines.images.ImagesPipeline` + - :class:`scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` + + (:issue:`6540`) + +- It's now deprecated to have a ``from_settings()`` method but no + ``from_crawler()`` method in 3rd-party :ref:`Scrapy components + `. You can define a simple ``from_crawler()`` method + that calls ``cls.from_settings(crawler.settings)`` to fix this if you don't + want to refactor the code. Note that if you have a ``from_crawler()`` + method Scrapy will not call the ``from_settings()`` method so the latter + can be removed. + (:issue:`6540`) + +- The initialization API of :class:`scrapy.pipelines.media.MediaPipeline` and + its subclasses was improved and some old usage scenarios are now deprecated + (see also the "Backward-incompatible changes" section). Specifically: + + - It's deprecated to define an ``__init__()`` method that doesn't take a + ``crawler`` argument. + - It's deprecated to call an ``__init__()`` method without passing a + ``crawler`` argument. If it's passed, it's also deprecated to pass a + ``settings`` argument, which will be ignored anyway. + - Calling ``from_settings()`` is deprecated, use ``from_crawler()`` + instead. + - Overriding ``from_settings()`` is deprecated, override ``from_crawler()`` + instead. + + (:issue:`6540`) + +- The ``REQUEST_FINGERPRINTER_IMPLEMENTATION`` setting is now deprecated. + (:issue:`6212`, :issue:`6213`) + +- The ``scrapy.utils.misc.create_instance()`` function is now deprecated, use + :func:`scrapy.utils.misc.build_from_crawler` instead. + (:issue:`5523`, :issue:`5884`, :issue:`6162`, :issue:`6169`, :issue:`6540`) + +- ``scrapy.core.downloader.Downloader._get_slot_key()`` is deprecated, use + :meth:`scrapy.core.downloader.Downloader.get_slot_key` instead. + (:issue:`6340`, :issue:`6352`) + +- ``scrapy.utils.defer.process_chain_both()`` is now deprecated. + (:issue:`6397`) + +- ``scrapy.twisted_version`` is now deprecated, you should instead use + :attr:`twisted.version` directly (but note that it's an + ``incremental.Version`` object, not a tuple). + (:issue:`6509`, :issue:`6512`) + +- ``scrapy.utils.python.flatten()`` and ``scrapy.utils.python.iflatten()`` + are now deprecated. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.utils.python.equal_attributes()`` is now deprecated. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.utils.request.request_authenticate()`` is now deprecated, you + should instead just set the ``Authorization`` header directly. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.utils.serialize.ScrapyJSONDecoder`` is now deprecated, it didn't + contain any code since Scrapy 1.0.0. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.utils.test.assert_samelines()`` is now deprecated. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.extensions.feedexport.build_storage()`` is now deprecated. You can + instead call the builder callable directly. + (:issue:`6540`) + +New features +~~~~~~~~~~~~ + +- ``scrapy.Spider.start_requests()`` can now yield items. + (:issue:`5289`, :issue:`6417`) + + .. note:: Some spider middlewares may need to be updated for Scrapy 2.12 + support before you can use them in combination with the ability to + yield items from ``start_requests()``. + +- Added a new :class:`~scrapy.http.Response` subclass, + :class:`~scrapy.http.JsonResponse`, for responses with a `JSON MIME type + `_. + (:issue:`6069`, :issue:`6171`, :issue:`6174`) + +- The :class:`~scrapy.extensions.logstats.LogStats` extension now adds + ``items_per_minute`` and ``responses_per_minute`` to the :ref:`stats + ` when the spider closes. + (:issue:`4110`, :issue:`4111`) + +- Added :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM` which allows closing the + spider if no items were scraped in a set amount of time. + (:issue:`6434`) + +- User-defined cookies can now include the ``secure`` field. + (:issue:`6357`) + +- Added component getters to :class:`~scrapy.crawler.Crawler`: + :meth:`~scrapy.crawler.Crawler.get_addon`, + :meth:`~scrapy.crawler.Crawler.get_downloader_middleware`, + :meth:`~scrapy.crawler.Crawler.get_extension`, + :meth:`~scrapy.crawler.Crawler.get_item_pipeline`, + :meth:`~scrapy.crawler.Crawler.get_spider_middleware`. + (:issue:`6181`) + +- Slot delay updates by the :ref:`AutoThrottle extension + ` based on response latencies can now be disabled for + specific requests via the :reqmeta:`autothrottle_dont_adjust_delay` meta + key. + (:issue:`6246`, :issue:`6527`) + +- If :setting:`SPIDER_LOADER_WARN_ONLY` is set to ``True``, + :class:`~scrapy.spiderloader.SpiderLoader` does not raise + :exc:`SyntaxError` but emits a warning instead. + (:issue:`6483`, :issue:`6484`) + +- Added support for multiple-compressed responses (ones with several + encodings in the ``Content-Encoding`` header). + (:issue:`5143`, :issue:`5964`, :issue:`6063`) + +- Added support for multiple standard values in :setting:`REFERRER_POLICY`. + (:issue:`6381`) + +- Added support for brotlicffi_ (previously named brotlipy_). brotli_ is + still recommended but only brotlicffi_ works on PyPy. + (:issue:`6263`, :issue:`6269`) + + .. _brotlicffi: https://github.com/python-hyper/brotlicffi + +- Added :class:`~scrapy.contracts.default.MetadataContract` that sets the + request meta. + (:issue:`6468`, :issue:`6469`) + +Improvements +~~~~~~~~~~~~ + +- Extended the list of file extensions that + :class:`LinkExtractor ` + ignores by default. + (:issue:`6074`, :issue:`6125`) + +- :func:`scrapy.utils.httpobj.urlparse_cached` is now used in more places + instead of :func:`urllib.parse.urlparse`. + (:issue:`6228`, :issue:`6229`) + +Bug fixes +~~~~~~~~~ + +- :class:`~scrapy.pipelines.media.MediaPipeline` is now an abstract class and + its methods that were expected to be overridden in subclasses are now + abstract methods. + (:issue:`6365`, :issue:`6368`) + +- Fixed handling of invalid ``@``-prefixed lines in contract extraction. + (:issue:`6383`, :issue:`6388`) + +- Importing ``scrapy.extensions.telnet`` no longer installs the default + reactor. + (:issue:`6432`) + +- Reduced log verbosity for dropped requests that was increased in 2.11.2. + (:issue:`6433`, :issue:`6475`) + +Documentation +~~~~~~~~~~~~~ + +- Added ``SECURITY.md`` that documents the security policy. + (:issue:`5364`, :issue:`6051`) + +- Example code for :ref:`running Scrapy from a script ` no + longer imports ``twisted.internet.reactor`` at the top level, which caused + problems with non-default reactors when this code was used unmodified. + (:issue:`6361`, :issue:`6374`) + +- Documented the :class:`~scrapy.extensions.spiderstate.SpiderState` + extension. + (:issue:`6278`, :issue:`6522`) + +- Other documentation improvements and fixes. + (:issue:`5920`, + :issue:`6094`, + :issue:`6177`, + :issue:`6200`, + :issue:`6207`, + :issue:`6216`, + :issue:`6223`, + :issue:`6317`, + :issue:`6328`, + :issue:`6389`, + :issue:`6394`, + :issue:`6402`, + :issue:`6411`, + :issue:`6427`, + :issue:`6429`, + :issue:`6440`, + :issue:`6448`, + :issue:`6449`, + :issue:`6462`, + :issue:`6497`, + :issue:`6506`, + :issue:`6507`, + :issue:`6524`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added ``py.typed``, in line with `PEP 561 + `_. + (:issue:`6058`, :issue:`6059`) + +- Fully covered the code with type hints (except for the most complicated + parts, mostly related to ``twisted.web.http`` and other Twisted parts + without type hints). + (:issue:`5989`, + :issue:`6097`, + :issue:`6127`, + :issue:`6129`, + :issue:`6130`, + :issue:`6133`, + :issue:`6143`, + :issue:`6191`, + :issue:`6268`, + :issue:`6274`, + :issue:`6275`, + :issue:`6276`, + :issue:`6279`, + :issue:`6325`, + :issue:`6326`, + :issue:`6333`, + :issue:`6335`, + :issue:`6336`, + :issue:`6337`, + :issue:`6341`, + :issue:`6353`, + :issue:`6356`, + :issue:`6370`, + :issue:`6371`, + :issue:`6384`, + :issue:`6385`, + :issue:`6387`, + :issue:`6391`, + :issue:`6395`, + :issue:`6414`, + :issue:`6422`, + :issue:`6460`, + :issue:`6466`, + :issue:`6472`, + :issue:`6494`, + :issue:`6498`, + :issue:`6516`) + +- Improved Bandit_ checks. + (:issue:`6260`, :issue:`6264`, :issue:`6265`) + +- Added pyupgrade_ to the ``pre-commit`` configuration. + (:issue:`6392`) + + .. _pyupgrade: https://github.com/asottile/pyupgrade + +- Added ``flake8-bugbear``, ``flake8-comprehensions``, ``flake8-debugger``, + ``flake8-docstrings``, ``flake8-string-format`` and + ``flake8-type-checking`` to the ``pre-commit`` configuration. + (:issue:`6406`, :issue:`6413`) + +- CI and test improvements and fixes. + (:issue:`5285`, + :issue:`5454`, + :issue:`5997`, + :issue:`6078`, + :issue:`6084`, + :issue:`6087`, + :issue:`6132`, + :issue:`6153`, + :issue:`6154`, + :issue:`6201`, + :issue:`6231`, + :issue:`6232`, + :issue:`6235`, + :issue:`6236`, + :issue:`6242`, + :issue:`6245`, + :issue:`6253`, + :issue:`6258`, + :issue:`6259`, + :issue:`6270`, + :issue:`6272`, + :issue:`6286`, + :issue:`6290`, + :issue:`6296` + :issue:`6367`, + :issue:`6372`, + :issue:`6403`, + :issue:`6416`, + :issue:`6435`, + :issue:`6489`, + :issue:`6501`, + :issue:`6504`, + :issue:`6511`, + :issue:`6543`, + :issue:`6545`) + +- Code cleanups. + (:issue:`6196`, + :issue:`6197`, + :issue:`6198`, + :issue:`6199`, + :issue:`6254`, + :issue:`6257`, + :issue:`6285`, + :issue:`6305`, + :issue:`6343`, + :issue:`6349`, + :issue:`6386`, + :issue:`6415`, + :issue:`6463`, + :issue:`6470`, + :issue:`6499`, + :issue:`6505`, + :issue:`6510`, + :issue:`6531`, + :issue:`6542`) + +Other +~~~~~ + +- Issue tracker improvements. (:issue:`6066`) + + +.. _release-2.11.2: + +Scrapy 2.11.2 (2024-05-14) +-------------------------- + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- Redirects to non-HTTP protocols are no longer followed. Please, see the + `23j4-mw76-5v7h security advisory`_ for more information. (:issue:`457`) + + .. _23j4-mw76-5v7h security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-23j4-mw76-5v7h + +- The ``Authorization`` header is now dropped on redirects to a different + scheme (``http://`` or ``https://``) or port, even if the domain is the + same. Please, see the `4qqq-9vqf-3h3f security advisory`_ for more + information. + + .. _4qqq-9vqf-3h3f security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-4qqq-9vqf-3h3f + +- When using system proxy settings that are different for ``http://`` and + ``https://``, redirects to a different URL scheme will now also trigger the + corresponding change in proxy settings for the redirected request. Please, + see the `jm3v-qxmh-hxwv security advisory`_ for more information. + (:issue:`767`) + + .. _jm3v-qxmh-hxwv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-jm3v-qxmh-hxwv + +- :attr:`Spider.allowed_domains ` is now + enforced for all requests, and not only requests from spider callbacks. + (:issue:`1042`, :issue:`2241`, :issue:`6358`) + +- :func:`~scrapy.utils.iterators.xmliter_lxml` no longer resolves XML + entities. (:issue:`6265`) + +- defusedxml_ is now used to make + :class:`scrapy.http.request.rpc.XmlRpcRequest` more secure. + (:issue:`6250`, :issue:`6251`) + + .. _defusedxml: https://github.com/tiran/defusedxml + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.spidermiddlewares.offsite.OffsiteMiddleware`` (a spider + middleware) is now deprecated and not enabled by default. The new + downloader middleware with the same functionality, + :class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`, is enabled + instead. + (:issue:`2241`, :issue:`6358`) + + +Bug fixes +~~~~~~~~~ + +- Restored support for brotlipy_, which had been dropped in Scrapy 2.11.1 in + favor of brotli_. (:issue:`6261`) + + .. note:: brotlipy is deprecated, both in Scrapy and upstream. Use brotli + instead if you can. + +- Make :setting:`METAREFRESH_IGNORE_TAGS` ``["noscript"]`` by default. This + prevents + :class:`~scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware` from + following redirects that would not be followed by web browsers with + JavaScript enabled. (:issue:`6342`, :issue:`6347`) + +- During :ref:`feed export `, do not close the + underlying file from :ref:`built-in post-processing plugins + `. + (:issue:`5932`, :issue:`6178`, :issue:`6239`) + +- :class:`LinkExtractor ` + now properly applies the ``unique`` and ``canonicalize`` parameters. + (:issue:`3273`, :issue:`6221`) + +- Do not initialize the scheduler disk queue if :setting:`JOBDIR` is an empty + string. (:issue:`6121`, :issue:`6124`) + +- Fix :attr:`Spider.logger ` not logging custom extra + information. (:issue:`6323`, :issue:`6324`) + +- ``robots.txt`` files with a non-UTF-8 encoding no longer prevent parsing + the UTF-8-compatible (e.g. ASCII) parts of the document. + (:issue:`6292`, :issue:`6298`) + +- :meth:`scrapy.http.cookies.WrappedRequest.get_header` no longer raises an + exception if ``default`` is ``None``. + (:issue:`6308`, :issue:`6310`) + +- :class:`~scrapy.Selector` now uses + :func:`scrapy.utils.response.get_base_url` to determine the base URL of a + given :class:`~scrapy.http.Response`. (:issue:`6265`) + +- The :meth:`media_to_download` method of :ref:`media pipelines + ` now logs exceptions before stripping them. + (:issue:`5067`, :issue:`5068`) + +- When passing a callback to the :command:`parse` command, build the callback + callable with the right signature. + (:issue:`6182`) + +Documentation +~~~~~~~~~~~~~ + +- Add a FAQ entry about :ref:`creating blank requests `. + (:issue:`6203`, :issue:`6208`) + +- Document that :attr:`scrapy.Selector.type` can be ``"json"``. + (:issue:`6328`, :issue:`6334`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Make builds reproducible. (:issue:`5019`, :issue:`6322`) + +- Packaging and test fixes. + (:issue:`6286`, :issue:`6290`, :issue:`6312`, :issue:`6316`, :issue:`6344`) + + +.. _release-2.11.1: + +Scrapy 2.11.1 (2024-02-14) +-------------------------- + +Highlights: + +- Security bug fixes. + +- Support for Twisted >= 23.8.0. + +- Documentation improvements. + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- Addressed `ReDoS vulnerabilities`_: + + - ``scrapy.utils.iterators.xmliter`` is now deprecated in favor of + :func:`~scrapy.utils.iterators.xmliter_lxml`, which + :class:`~scrapy.spiders.XMLFeedSpider` now uses. + + To minimize the impact of this change on existing code, + :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating + the node namespace with a prefix in the node name, and big files with + highly nested trees when using libxml2 2.7+. + + - Fixed regular expressions in the implementation of the + :func:`~scrapy.utils.response.open_in_browser` function. + + Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. + + .. _ReDoS vulnerabilities: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS + .. _cc65-xxvf-f7r9 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9 + +- :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` now also apply + to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security + advisory`_ for more information. + + .. _7j7m-v7m3-jqm7 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-7j7m-v7m3-jqm7 + +- Also in relation with the `7j7m-v7m3-jqm7 security advisory`_, the + deprecated ``scrapy.downloadermiddlewares.decompression`` module has been + removed. + +- The ``Authorization`` header is now dropped on redirects to a different + domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more + information. + + .. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- The Twisted dependency is no longer restricted to < 23.8.0. (:issue:`6024`, + :issue:`6064`, :issue:`6142`) + +Bug fixes +~~~~~~~~~ + +- The OS signal handling code was refactored to no longer use private Twisted + functions. (:issue:`6024`, :issue:`6064`, :issue:`6112`) + +Documentation +~~~~~~~~~~~~~ + +- Improved documentation for :class:`~scrapy.crawler.Crawler` initialization + changes made in the 2.11.0 release. (:issue:`6057`, :issue:`6147`) + +- Extended documentation for :attr:`.Request.meta`. + (:issue:`5565`) + +- Fixed the :reqmeta:`dont_merge_cookies` documentation. (:issue:`5936`, + :issue:`6077`) + +- Added a link to Zyte's export guides to the :ref:`feed exports + ` documentation. (:issue:`6183`) + +- Added a missing note about backward-incompatible changes in + :class:`~scrapy.exporters.PythonItemExporter` to the 2.11.0 release notes. + (:issue:`6060`, :issue:`6081`) + +- Added a missing note about removing the deprecated + ``scrapy.utils.boto.is_botocore()`` function to the 2.8.0 release notes. + (:issue:`6056`, :issue:`6061`) + +- Other documentation improvements. (:issue:`6128`, :issue:`6144`, + :issue:`6163`, :issue:`6190`, :issue:`6192`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added Python 3.12 to the CI configuration, re-enabled tests that were + disabled when the pre-release support was added. (:issue:`5985`, + :issue:`6083`, :issue:`6098`) + +- Fixed a test issue on PyPy 7.3.14. (:issue:`6204`, :issue:`6205`) + + .. _release-2.11.0: Scrapy 2.11.0 (2023-09-18) @@ -34,7 +2964,7 @@ Backward-incompatible changes in :meth:`scrapy.Spider.from_crawler`. If you want to access the final setting values and the initialized :class:`~scrapy.crawler.Crawler` attributes in the spider code as early as possible you can do this in - :meth:`~scrapy.Spider.start_requests` or in a handler of the + ``scrapy.Spider.start_requests()`` or in a handler of the :signal:`engine_started` signal. (:issue:`6038`) - The :meth:`TextResponse.json ` method now @@ -173,10 +3103,10 @@ Modified requirements Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` - instead of ``False``. In earlier Scrapy versions empty files were created - even when this setting was ``False`` (which was a bug that is now fixed), - so the new default should keep the old behavior. (:issue:`872`, +- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` + instead of ``False``. In earlier Scrapy versions empty files were created + even when this setting was ``False`` (which was a bug that is now fixed), + so the new default should keep the old behavior. (:issue:`872`, :issue:`5847`) Deprecation removals @@ -334,7 +3264,7 @@ New features :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) -- Added :meth:`TextResponse.jmespath`, a shortcut for JMESPath selectors +- Added :meth:`.TextResponse.jmespath`, a shortcut for JMESPath selectors available since parsel_ 1.8.1. (:issue:`5894`, :issue:`5915`) - Added :signal:`feed_slot_closed` and :signal:`feed_exporter_closed` @@ -389,7 +3319,7 @@ Bug fixes (:issue:`5914`, :issue:`5917`) - Fixed an error breaking user handling of send failures in - :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`) + :meth:`scrapy.mail.MailSender.send`. (:issue:`1611`, :issue:`5880`) Documentation ~~~~~~~~~~~~~ @@ -514,7 +3444,7 @@ New features avoid confusion. (:issue:`5717`, :issue:`5722`, :issue:`5727`) -- The ``callback`` parameter of :class:`~scrapy.http.Request` can now be set +- The ``callback`` parameter of :class:`~scrapy.Request` can now be set to :func:`scrapy.http.request.NO_CALLBACK`, to distinguish it from ``None``, as the latter indicates that the default spider callback (:meth:`~scrapy.Spider.parse`) is to be used. @@ -808,7 +3738,7 @@ Bug fixes that does not match the asyncio event loop actually installed (:issue:`5529`). -- Fixed :meth:`Headers.getlist ` +- Fixed :meth:`Headers.getlist() ` returning only the last header (:issue:`5515`, :issue:`5526`). - Fixed :class:`LinkExtractor @@ -840,7 +3770,7 @@ Documentation (:issue:`3582`, :issue:`5432`). .. _Common Crawl: https://commoncrawl.org/ - .. _Google cache: http://www.googleguide.com/cached_pages.html + .. _Google cache: https://www.googleguide.com/cached_pages.html - The new :ref:`topics-components` topic covers enforcing requirements on Scrapy components, like :ref:`downloader middlewares @@ -1011,17 +3941,17 @@ Highlights: Security bug fixes ~~~~~~~~~~~~~~~~~~ -- When a :class:`~scrapy.http.Request` object with cookies defined gets a - redirect response causing a new :class:`~scrapy.http.Request` object to be +- When a :class:`~scrapy.Request` object with cookies defined gets a + redirect response causing a new :class:`~scrapy.Request` object to be scheduled, the cookies defined in the original - :class:`~scrapy.http.Request` object are no longer copied into the new - :class:`~scrapy.http.Request` object. + :class:`~scrapy.Request` object are no longer copied into the new + :class:`~scrapy.Request` object. If you manually set the ``Cookie`` header on a - :class:`~scrapy.http.Request` object and the domain name of the redirect + :class:`~scrapy.Request` object and the domain name of the redirect URL is not an exact match for the domain of the URL of the original - :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped - from the new :class:`~scrapy.http.Request` object. + :class:`~scrapy.Request` object, your ``Cookie`` header is now dropped + from the new :class:`~scrapy.Request` object. The old behavior could be exploited by an attacker to gain access to your cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more @@ -1034,10 +3964,10 @@ Security bug fixes ``example.com`` and any subdomain) by defining the shared domain suffix (e.g. ``example.com``) as the cookie domain when defining your cookies. See the documentation of the - :class:`~scrapy.http.Request` class for more information. + :class:`~scrapy.Request` class for more information. - When the domain of a cookie, either received in the ``Set-Cookie`` header - of a response or defined in a :class:`~scrapy.http.Request` object, is set + of a response or defined in a :class:`~scrapy.Request` object, is set to a `public suffix `_, the cookie is now ignored unless the cookie domain is the same as the request domain. @@ -1053,7 +3983,7 @@ Modified requirements ~~~~~~~~~~~~~~~~~~~~~ - The h2_ dependency is now optional, only needed to - :ref:`enable HTTP/2 support `. (:issue:`5113`) + :ref:`enable HTTP/2 support `. (:issue:`5113`) .. _h2: https://pypi.org/project/h2/ @@ -1088,7 +4018,7 @@ Backward-incompatible changes meet expectations, :exc:`TypeError` is now raised at startup time. Before, other exceptions would be raised at run time. (:issue:`3559`) -- The ``_encoding`` field of serialized :class:`~scrapy.http.Request` objects +- The ``_encoding`` field of serialized :class:`~scrapy.Request` objects is now named ``encoding``, in line with all other fields (:issue:`5130`) @@ -1118,7 +4048,7 @@ Deprecations - :mod:`scrapy.utils.reqser` is deprecated. (:issue:`5130`) - Instead of :func:`~scrapy.utils.reqser.request_to_dict`, use the new - :meth:`Request.to_dict ` method. + :meth:`.Request.to_dict` method. - Instead of :func:`~scrapy.utils.reqser.request_from_dict`, use the new :func:`scrapy.utils.request.request_from_dict` function. @@ -1197,7 +4127,7 @@ New features (:setting:`AWS_SESSION_TOKEN`) and endpoint customization (:setting:`AWS_ENDPOINT_URL`). (:issue:`4998`, :issue:`5210`) - .. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys + .. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html - New :setting:`LOG_FILE_APPEND` setting to allow truncating the log file. (:issue:`5279`) @@ -1223,9 +4153,9 @@ New features using ``queuelib`` 1.6.1 or later), the ``peek`` method raises :exc:`NotImplementedError`. -- :class:`~scrapy.http.Request` and :class:`~scrapy.http.Response` now have +- :class:`~scrapy.Request` and :class:`~scrapy.http.Response` now have an ``attributes`` attribute that makes subclassing easier. For - :class:`~scrapy.http.Request`, it also allows subclasses to work with + :class:`~scrapy.Request`, it also allows subclasses to work with :func:`scrapy.utils.request.request_from_dict`. (:issue:`1877`, :issue:`5130`, :issue:`5218`) @@ -1301,7 +4231,7 @@ Documentation - Provided better context and instructions to disable the :setting:`URLLENGTH_LIMIT` setting. (:issue:`5135`, :issue:`5250`) -- Documented that :ref:`reppy-parser` does not support Python 3.9+. +- Documented that Reppy parser does not support Python 3.9+. (:issue:`5226`, :issue:`5231`) - Documented :ref:`the scheduler component `. @@ -1343,7 +4273,7 @@ Documentation - ``quotes.toscrape.com`` references now use HTTPS instead of HTTP. (:issue:`5395`, :issue:`5396`) -- Added a link to `our Discord server `_ +- Added a link to `our Discord server `_ to :ref:`getting-help`. (:issue:`5421`, :issue:`5422`) - The pronunciation of the project name is now :ref:`officially @@ -1425,8 +4355,6 @@ Scrapy 2.5.1 (2021-10-05) need to upgrade scrapy-splash to a greater version for it to continue to work. -.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash - .. _release-2.5.0: @@ -1437,7 +4365,7 @@ Highlights: - Official Python 3.9 support -- Experimental :ref:`HTTP/2 support ` +- Experimental :ref:`HTTP/2 support ` - New :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function to retry requests from spider callbacks @@ -1468,7 +4396,7 @@ Deprecations New features ~~~~~~~~~~~~ -- Experimental :ref:`HTTP/2 support ` through a new download handler +- Experimental :ref:`HTTP/2 support ` through a new download handler that can be assigned to the ``https`` protocol in the :setting:`DOWNLOAD_HANDLERS` setting. (:issue:`1854`, :issue:`4769`, :issue:`5058`, :issue:`5059`, :issue:`5066`) @@ -1534,7 +4462,7 @@ Bug fixes with lower indentation than the following code. (:issue:`4477`, :issue:`4935`) -- The `Content-Length `_ +- The `Content-Length `_ header is no longer omitted from responses when using the default, HTTP/1.1 download handler (see :setting:`DOWNLOAD_HANDLERS`). (:issue:`5009`, :issue:`5034`, :issue:`5045`, :issue:`5057`, :issue:`5062`) @@ -1691,14 +4619,13 @@ Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` once again - discards cookies defined in :attr:`Request.headers - `. + discards cookies defined in :attr:`.Request.headers`. We decided to revert this bug fix, introduced in Scrapy 2.2.0, because it was reported that the current implementation could break existing code. If you need to set cookies for a request, use the :class:`Request.cookies - ` parameter. + ` parameter. A future version of Scrapy will include a new, better implementation of the reverted bug fix. @@ -1819,16 +4746,16 @@ New features :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response` or :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception` - with a custom :class:`~scrapy.http.Request` object assigned to + with a custom :class:`~scrapy.Request` object assigned to :class:`response.request `: - The response is handled by the callback of that custom - :class:`~scrapy.http.Request` object, instead of being handled by the - callback of the original :class:`~scrapy.http.Request` object + :class:`~scrapy.Request` object, instead of being handled by the + callback of the original :class:`~scrapy.Request` object - - That custom :class:`~scrapy.http.Request` object is now sent as the + - That custom :class:`~scrapy.Request` object is now sent as the ``request`` argument to the :signal:`response_received` signal, instead - of the original :class:`~scrapy.http.Request` object + of the original :class:`~scrapy.Request` object (:issue:`4529`, :issue:`4632`) @@ -1999,7 +4926,7 @@ New features * The :command:`parse` command now allows specifying an output file (:issue:`4317`, :issue:`4377`) -* :meth:`Request.from_curl ` and +* :meth:`.Request.from_curl` and :func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support ``--data-raw`` (:issue:`4612`) @@ -2015,7 +4942,7 @@ Bug fixes :ref:`dataclass items ` and :ref:`attr.s items ` (:issue:`4667`, :issue:`4668`) -* :meth:`Request.from_curl ` and +* :meth:`.Request.from_curl` and :func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request method to ``POST`` when a request body is specified and no request method is specified (:issue:`4612`) @@ -2034,7 +4961,7 @@ Documentation * Simplified the code example in :ref:`topics-loaders-dataclass` (:issue:`4652`) -.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT +.. _OpenSSL cipher list format: https://docs.openssl.org/master/man1/openssl-ciphers/#cipher-list-format Quality assurance @@ -2100,8 +5027,7 @@ Backward-incompatible changes Deprecations ~~~~~~~~~~~~ -* :meth:`TextResponse.body_as_unicode - ` is now deprecated, use +* ``TextResponse.body_as_unicode()`` is now deprecated, use :attr:`TextResponse.text ` instead (:issue:`4546`, :issue:`4555`, :issue:`4579`) @@ -2140,9 +5066,8 @@ New features * :ref:`Link extractors ` are now serializable, as long as you do not use :ref:`lambdas ` for parameters; for - example, you can now pass link extractors in :attr:`Request.cb_kwargs - ` or - :attr:`Request.meta ` when :ref:`persisting + example, you can now pass link extractors in :attr:`.Request.cb_kwargs` + or :attr:`.Request.meta` when :ref:`persisting scheduled requests ` (:issue:`4554`) * Upgraded the :ref:`pickle protocol ` that Scrapy uses @@ -2161,11 +5086,11 @@ Bug fixes * :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer discards cookies defined in :attr:`Request.headers - ` (:issue:`1992`, :issue:`2400`) + ` (:issue:`1992`, :issue:`2400`) * :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer re-encodes cookies defined as :class:`bytes` in the ``cookies`` parameter - of the ``__init__`` method of :class:`~scrapy.http.Request` + of the ``__init__`` method of :class:`~scrapy.Request` (:issue:`2400`, :issue:`3575`) * When :setting:`FEEDS` defines multiple URIs, :setting:`FEED_STORE_EMPTY` is @@ -2174,7 +5099,7 @@ Bug fixes * :class:`~scrapy.spiders.Spider` callbacks defined using :doc:`coroutine syntax ` no longer need to return an iterable, and may - instead return a :class:`~scrapy.http.Request` object, an + instead return a :class:`~scrapy.Request` object, an :ref:`item `, or ``None`` (:issue:`4609`) * The :command:`startproject` command now ensures that the generated project @@ -2215,8 +5140,8 @@ Documentation :issue:`4587`) * The display-on-hover behavior of internal documentation references now also - covers links to :ref:`commands `, :attr:`Request.meta - ` keys, :ref:`settings ` and + covers links to :ref:`commands `, :attr:`.Request.meta` + keys, :ref:`settings ` and :ref:`signals ` (:issue:`4495`, :issue:`4563`) * It is again possible to download the documentation for offline reading @@ -2261,7 +5186,7 @@ Quality assurance * Added a `Pylint `_ job to Travis CI (:issue:`3727`) -* Added a `Mypy `_ job to Travis CI (:issue:`4637`) +* Added a `Mypy `_ job to Travis CI (:issue:`4637`) * Made use of set literals in tests (:issue:`4573`) @@ -2445,7 +5370,7 @@ Highlights: * :ref:`FTP support ` for media pipelines * New :attr:`Response.certificate ` attribute -* IPv6 support through :setting:`DNS_RESOLVER` +* IPv6 support through ``DNS_RESOLVER`` Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2501,7 +5426,7 @@ Deprecation removals ~~~~~~~~~~~~~~~~~~~~ * The :ref:`Scrapy shell ` no longer provides a `sel` proxy - object, use :meth:`response.selector ` + object, use :meth:`response.selector ` instead (:issue:`4347`) * LevelDB support has been removed (:issue:`4112`) @@ -2553,7 +5478,7 @@ New features :class:`twisted.internet.ssl.Certificate` object for HTTPS responses (:issue:`2726`, :issue:`4054`) -* A new :setting:`DNS_RESOLVER` setting allows enabling IPv6 support +* A new ``DNS_RESOLVER`` setting allows enabling IPv6 support (:issue:`1031`, :issue:`4227`) * A new :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` setting allows configuring @@ -2571,10 +5496,10 @@ New features * The new :attr:`Response.cb_kwargs ` attribute serves as a shortcut for :attr:`Response.request.cb_kwargs - ` (:issue:`4331`) + ` (:issue:`4331`) * :meth:`Response.follow ` now supports a - ``flags`` parameter, for consistency with :class:`~scrapy.http.Request` + ``flags`` parameter, for consistency with :class:`~scrapy.Request` (:issue:`4277`, :issue:`4279`) * :ref:`Item loader processors ` can now be @@ -2583,7 +5508,7 @@ New features * :class:`~scrapy.spiders.Rule` now accepts an ``errback`` parameter (:issue:`4000`) -* :class:`~scrapy.http.Request` no longer requires a ``callback`` parameter +* :class:`~scrapy.Request` no longer requires a ``callback`` parameter when an ``errback`` parameter is specified (:issue:`3586`, :issue:`4008`) * :class:`~scrapy.logformatter.LogFormatter` now supports some additional @@ -2613,7 +5538,7 @@ New features * :class:`~scrapy.spiders.Spider` objects now raise an :exc:`AttributeError` exception if they do not have a :class:`~scrapy.spiders.Spider.start_urls` - attribute nor reimplement :class:`~scrapy.spiders.Spider.start_requests`, + attribute nor reimplement ``scrapy.spiders.Spider.start_requests()``, but have a ``start_url`` attribute (:issue:`4133`, :issue:`4170`) * :class:`~scrapy.exporters.BaseItemExporter` subclasses may now use @@ -2633,7 +5558,7 @@ New features components already supported (:issue:`4126`) * :class:`scrapy.utils.python.MutableChain.__iter__` now returns ``self``, - `allowing it to be used as a sequence `_ + allowing it to be used as a sequence. (:issue:`4153`) @@ -2655,7 +5580,7 @@ Bug fixes * Redirects to URLs starting with 3 slashes (``///``) are now supported (:issue:`4032`, :issue:`4042`) -* :class:`~scrapy.http.Request` no longer accepts strings as ``url`` simply +* :class:`~scrapy.Request` no longer accepts strings as ``url`` simply because they have a colon (:issue:`2552`, :issue:`4094`) * The correct encoding is now used for attach names in @@ -2701,7 +5626,7 @@ Documentation using :class:`~scrapy.crawler.CrawlerProcess` (:issue:`2149`, :issue:`2352`, :issue:`3146`, :issue:`3960`) -* Clarified the requirements for :class:`~scrapy.http.Request` objects +* Clarified the requirements for :class:`~scrapy.Request` objects :ref:`when using persistence ` (:issue:`4124`, :issue:`4139`) @@ -2768,7 +5693,7 @@ Quality assurance * Cleaned up code (:issue:`3937`, :issue:`4208`, :issue:`4209`, :issue:`4210`, :issue:`4212`, :issue:`4369`, :issue:`4376`, :issue:`4378`) -.. _Bandit: https://bandit.readthedocs.io/ +.. _Bandit: https://bandit.readthedocs.io/en/latest/ .. _Flake8: https://flake8.pycqa.org/en/latest/ @@ -2877,6 +5802,38 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.4: + +Scrapy 1.8.4 (2024-02-14) +------------------------- + +**Security bug fixes:** + +- Due to its `ReDoS vulnerabilities`_, ``scrapy.utils.iterators.xmliter`` is + now deprecated in favor of :func:`~scrapy.utils.iterators.xmliter_lxml`, + which :class:`~scrapy.spiders.XMLFeedSpider` now uses. + + To minimize the impact of this change on existing code, + :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating + the node namespace as a prefix in the node name, and big files with highly + nested trees when using libxml2 2.7+. + + Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. + +- :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` now also apply + to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security + advisory`_ for more information. + +- Also in relation with the `7j7m-v7m3-jqm7 security advisory`_, use of the + ``scrapy.downloadermiddlewares.decompression`` module is discouraged and + will trigger a warning. + +- The ``Authorization`` header is now dropped on redirects to a different + domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more + information. + + .. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv + .. _release-1.8.3: @@ -2938,17 +5895,17 @@ Scrapy 1.8.2 (2022-03-01) **Security bug fixes:** -- When a :class:`~scrapy.http.Request` object with cookies defined gets a - redirect response causing a new :class:`~scrapy.http.Request` object to be +- When a :class:`~scrapy.Request` object with cookies defined gets a + redirect response causing a new :class:`~scrapy.Request` object to be scheduled, the cookies defined in the original - :class:`~scrapy.http.Request` object are no longer copied into the new - :class:`~scrapy.http.Request` object. + :class:`~scrapy.Request` object are no longer copied into the new + :class:`~scrapy.Request` object. If you manually set the ``Cookie`` header on a - :class:`~scrapy.http.Request` object and the domain name of the redirect + :class:`~scrapy.Request` object and the domain name of the redirect URL is not an exact match for the domain of the URL of the original - :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped - from the new :class:`~scrapy.http.Request` object. + :class:`~scrapy.Request` object, your ``Cookie`` header is now dropped + from the new :class:`~scrapy.Request` object. The old behavior could be exploited by an attacker to gain access to your cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more @@ -2961,10 +5918,10 @@ Scrapy 1.8.2 (2022-03-01) ``example.com`` and any subdomain) by defining the shared domain suffix (e.g. ``example.com``) as the cookie domain when defining your cookies. See the documentation of the - :class:`~scrapy.http.Request` class for more information. + :class:`~scrapy.Request` class for more information. - When the domain of a cookie, either received in the ``Set-Cookie`` header - of a response or defined in a :class:`~scrapy.http.Request` object, is set + of a response or defined in a :class:`~scrapy.Request` object, is set to a `public suffix `_, the cookie is now ignored unless the cookie domain is the same as the request domain. @@ -3022,7 +5979,7 @@ Highlights: * Dropped Python 3.4 support and updated minimum requirements; made Python 3.8 support official -* New :meth:`Request.from_curl ` class method +* New :meth:`.Request.from_curl` class method * New :setting:`ROBOTSTXT_PARSER` and :setting:`ROBOTSTXT_USER_AGENT` settings * New :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` and :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` settings @@ -3030,6 +5987,8 @@ Highlights: Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. skip: start + * Python 3.4 is no longer supported, and some of the minimum requirements of Scrapy have also changed: @@ -3049,7 +6008,7 @@ Backward-incompatible changes consistency with similar classes (:issue:`3929`, :issue:`3982`) * If you are using a custom context factory - (:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`), its ``__init__`` method must + (``DOWNLOADER_CLIENTCONTEXTFACTORY``), its ``__init__`` method must accept two new parameters: ``tls_verbose_logging`` and ``tls_ciphers`` (:issue:`2111`, :issue:`3392`, :issue:`3442`, :issue:`3450`) @@ -3070,20 +6029,22 @@ Backward-incompatible changes (:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`, :issue:`3998`, :issue:`4036`) +.. skip: end + See also :ref:`1.8-deprecation-removals` below. New features ~~~~~~~~~~~~ -* A new :meth:`Request.from_curl ` class +* A new :meth:`Request.from_curl ` class method allows :ref:`creating a request from a cURL command ` (:issue:`2985`, :issue:`3862`) * A new :setting:`ROBOTSTXT_PARSER` setting allows choosing which robots.txt_ parser to use. It includes built-in support for :ref:`RobotFileParser `, - :ref:`Protego ` (default), :ref:`Reppy `, and + :ref:`Protego ` (default), Reppy, and :ref:`Robotexclusionrulesparser `, and allows you to :ref:`implement support for additional parsers ` (:issue:`754`, :issue:`2669`, @@ -3105,9 +6066,8 @@ New features ``True`` to enable debug-level messages about TLS connection parameters after establishing HTTPS connections (:issue:`2111`, :issue:`3450`) -* Callbacks that receive keyword arguments - (see :attr:`Request.cb_kwargs `) can now be - tested using the new :class:`@cb_kwargs +* Callbacks that receive keyword arguments (see :attr:`.Request.cb_kwargs`) + can now be tested using the new :class:`@cb_kwargs ` :ref:`spider contract ` (:issue:`3985`, :issue:`3988`) @@ -3296,7 +6256,7 @@ Backward-incompatible changes * Non-default values for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting may stop working. Scheduler priority queue classes now need to handle - :class:`~scrapy.http.Request` objects instead of arbitrary Python data + :class:`~scrapy.Request` objects instead of arbitrary Python data structures. * An additional ``crawler`` parameter has been added to the ``__init__`` @@ -3318,7 +6278,7 @@ New features scheduling improvement on crawls targeting multiple web domains, at the cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`) -* A new :attr:`Request.cb_kwargs ` attribute +* A new :attr:`.Request.cb_kwargs` attribute provides a cleaner way to pass keyword arguments to callback methods (:issue:`1138`, :issue:`3563`) @@ -3399,7 +6359,7 @@ Bug fixes * Requests with private callbacks are now correctly unserialized from disk (:issue:`3790`) -* :meth:`FormRequest.from_response() ` +* :meth:`.FormRequest.from_response` now handles invalid methods like major web browsers (:issue:`3777`, :issue:`3794`) @@ -3479,13 +6439,13 @@ The following deprecated APIs have been removed (:issue:`3578`): * From both ``scrapy.selector`` and ``scrapy.selector.lxmlsel``: - * ``HtmlXPathSelector`` (use :class:`~scrapy.selector.Selector`) + * ``HtmlXPathSelector`` (use :class:`~scrapy.Selector`) - * ``XmlXPathSelector`` (use :class:`~scrapy.selector.Selector`) + * ``XmlXPathSelector`` (use :class:`~scrapy.Selector`) - * ``XPathSelector`` (use :class:`~scrapy.selector.Selector`) + * ``XPathSelector`` (use :class:`~scrapy.Selector`) - * ``XPathSelectorList`` (use :class:`~scrapy.selector.Selector`) + * ``XPathSelectorList`` (use :class:`~scrapy.Selector`) * From ``scrapy.selector.csstranslator``: @@ -3495,7 +6455,7 @@ The following deprecated APIs have been removed (:issue:`3578`): * ``ScrapyXPathExpr`` (use parsel.csstranslator.XPathExpr_) -* From :class:`~scrapy.selector.Selector`: +* From :class:`~scrapy.Selector`: * ``_root`` (both the ``__init__`` method argument and the object property, use ``root``) @@ -3909,9 +6869,9 @@ Docs - Added missing bullet point for the ``AUTOTHROTTLE_TARGET_CONCURRENCY`` setting. (:issue:`2756`) - Update Contributing docs, document new support channels - (:issue:`2762`, issue:`3038`) + (:issue:`2762`, :issue:`3038`) - Include references to Scrapy subreddit in the docs -- Fix broken links; use https:// for external links +- Fix broken links; use ``https://`` for external links (:issue:`2978`, :issue:`2982`, :issue:`2958`) - Document CloseSpider extension better (:issue:`2759`) - Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example @@ -4025,7 +6985,7 @@ New Features (:issue:`2535`) - New :ref:`response.follow ` shortcut for creating requests (:issue:`1940`) -- Added ``flags`` argument and attribute to :class:`Request ` +- Added ``flags`` argument and attribute to :class:`~scrapy.Request` objects (:issue:`2047`) - Support Anonymous FTP (:issue:`2342`) - Added ``retry/count``, ``retry/max_reached`` and ``retry/reason_count/`` @@ -4067,7 +7027,7 @@ Bug fixes - LinkExtractor now strips leading and trailing whitespaces from attributes (:issue:`2547`, fixes :issue:`1614`) - Properly handle whitespaces in action attribute in - :class:`~scrapy.http.FormRequest` (:issue:`2548`) + :class:`~scrapy.FormRequest` (:issue:`2548`) - Buffer CONNECT response bytes from proxy until all HTTP headers are received (:issue:`2495`, fixes :issue:`2491`) - FTP downloader now works on Python 3, provided you use Twisted>=17.1 @@ -4109,8 +7069,7 @@ Documentation ~~~~~~~~~~~~~ - Binary mode is required for exporters (:issue:`2564`, fixes :issue:`2553`) -- Mention issue with :meth:`FormRequest.from_response - ` due to bug in lxml (:issue:`2572`) +- Mention issue with :meth:`.FormRequest.from_response` due to bug in lxml (:issue:`2572`) - Use single quotes uniformly in templates (:issue:`2596`) - Document :reqmeta:`ftp_user` and :reqmeta:`ftp_password` meta keys (:issue:`2587`) - Removed section on deprecated ``contrib/`` (:issue:`2636`) @@ -4505,14 +7464,14 @@ This 1.1 release brings a lot of interesting features and bug fixes: selectors engine without needing to upgrade Scrapy. - HTTPS downloader now does TLS protocol negotiation by default, instead of forcing TLS 1.0. You can also set the SSL/TLS method - using the new :setting:`DOWNLOADER_CLIENT_TLS_METHOD`. + using the new ``DOWNLOADER_CLIENT_TLS_METHOD`` setting. - These bug fixes may require your attention: - Don't retry bad requests (HTTP 400) by default (:issue:`1289`). If you need the old behavior, add ``400`` to :setting:`RETRY_HTTP_CODES`. - Fix shell files argument handling (:issue:`1710`, :issue:`1550`). - If you try ``scrapy shell index.html`` it will try to load the URL http://index.html, + If you try ``scrapy shell index.html`` it will try to load the URL ``http://index.html``, use ``scrapy shell ./index.html`` to load a local file. - Robots.txt compliance is now enabled by default for newly-created projects (:issue:`1724`). Scrapy will also wait for robots.txt to be downloaded @@ -4540,8 +7499,7 @@ Keep reading for more details on other improvements and bug fixes. Beta Python 3 Support ~~~~~~~~~~~~~~~~~~~~~ -We have been `hard at work to make Scrapy run on Python 3 -`_. As a result, now +We have been hard at work to make Scrapy run on Python 3. As a result, now you can run spiders on Python 3.3, 3.4 and 3.5 (Twisted >= 15.5 required). Some features are still missing (and some may never be ported). @@ -4609,7 +7567,7 @@ Additional New Features and Enhancements - Other refactoring, optimizations and cleanup (:issue:`1476`, :issue:`1481`, :issue:`1477`, :issue:`1315`, :issue:`1290`, :issue:`1750`, :issue:`1881`). -.. _`Code of Conduct`: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md +.. _Code of Conduct: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md Deprecations and Removals @@ -4649,7 +7607,7 @@ Bugfixes - Support empty password for http_proxy config (:issue:`1274`). - Interpret ``application/x-json`` as ``TextResponse`` (:issue:`1333`). - Support link rel attribute with multiple values (:issue:`1201`). -- Fixed ``scrapy.http.FormRequest.from_response`` when there is a ```` +- Fixed ``scrapy.FormRequest.from_response`` when there is a ```` tag (:issue:`1564`). - Fixed :setting:`TEMPLATES_DIR` handling (:issue:`1575`). - Various ``FormRequest`` fixes (:issue:`1595`, :issue:`1596`, :issue:`1597`). @@ -5188,7 +8146,7 @@ Scrapy 0.24.5 (2015-02-25) Scrapy 0.24.4 (2014-08-09) -------------------------- -- pem file is used by mockserver and required by scrapy bench (:commit:`5eddc68`) +- pem file is used by mockserver and required by scrapy bench (:commit:`5eddc68b63`) - scrapy bench needs scrapy.tests* (:commit:`d6cb999`) Scrapy 0.24.3 (2014-08-09) @@ -5371,7 +8329,7 @@ Enhancements - Make ``RFPDupeFilter`` class easily subclassable (:issue:`533`) - Improve test coverage and forthcoming Python 3 support (:issue:`525`) - Promote startup info on settings and middleware to INFO level (:issue:`520`) -- Support partials in ``get_func_args`` util (:issue:`506`, issue:`504`) +- Support partials in ``get_func_args`` util (:issue:`506`, :issue:`504`) - Allow running individual tests via tox (:issue:`503`) - Update extensions ignored by link extractors (:issue:`498`) - Add middleware methods to get files/images/thumbs paths (:issue:`490`) @@ -5504,7 +8462,7 @@ Scrapy 0.18.4 (released 2013-10-10) - IPython refuses to update the namespace. fix #396 (:commit:`3d32c4f`) - Fix AlreadyCalledError replacing a request in shell command. closes #407 (:commit:`b1d8919`) -- Fix start_requests laziness and early hangs (:commit:`89faf52`) +- Fix ``start_requests()`` laziness and early hangs (:commit:`89faf52`) Scrapy 0.18.3 (released 2013-10-03) ----------------------------------- @@ -5576,7 +8534,7 @@ Scrapy 0.18.0 (released 2013-08-09) - Moved persistent (on disk) queues to a separate project (queuelib_) which Scrapy now depends on - Add Scrapy commands using external libraries (:issue:`260`) - Added ``--pdb`` option to ``scrapy`` command line tool -- Added :meth:`XPathSelector.remove_namespaces ` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`. +- Added :meth:`XPathSelector.remove_namespaces ` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`. - Several improvements to spider contracts - New default middleware named MetaRefreshMiddleware that handles meta-refresh html tag redirections, - MetaRefreshMiddleware and RedirectMiddleware have different priorities to address #62 @@ -5697,7 +8655,7 @@ Scrapy changes: - added options ``-o`` and ``-t`` to the :command:`runspider` command - documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED` - major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals. -- added :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests` method to spider middlewares +- added a ``process_start_requests()`` method to spider middlewares - dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info. - dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info. - documented :ref:`topics-api` @@ -5709,7 +8667,7 @@ Scrapy changes: - nested items now fully supported in JSON and JSONLines exporters - added :reqmeta:`cookiejar` Request meta key to support multiple cookie sessions per spider - decoupled encoding detection code to `w3lib.encoding`_, and ported Scrapy code to use that module -- dropped support for Python 2.5. See https://blog.scrapinghub.com/2012/02/27/scrapy-0-15-dropping-support-for-python-2-5/ +- dropped support for Python 2.5. See https://www.zyte.com/blog/scrapy-0-15-dropping-support-for-python-2-5/ - dropped support for Twisted 2.5 - added :setting:`REFERER_ENABLED` setting, to control referer middleware - changed default user agent to: ``Scrapy/VERSION (+http://scrapy.org)`` @@ -5760,7 +8718,7 @@ Scrapy 0.14.2 - fixed bug in MemoryUsage extension: get_engine_status() takes exactly 1 argument (0 given) (:commit:`11133e9`) - fixed struct.error on http compression middleware. closes #87 (:commit:`1423140`) - ajax crawling wasn't expanding for unicode urls (:commit:`0de3fb4`) -- Catch start_requests iterator errors. refs #83 (:commit:`454a21d`) +- Catch ``start_requests()`` iterator errors. refs #83 (:commit:`454a21d`) - Speed-up libxml2 XPathSelector (:commit:`2fbd662`) - updated versioning doc according to recent changes (:commit:`0a070f5`) - scrapyd: fixed documentation link (:commit:`2b4e4c3`) @@ -5787,7 +8745,7 @@ Scrapy 0.14 New features and settings ~~~~~~~~~~~~~~~~~~~~~~~~~ -- Support for `AJAX crawlable urls`_ +- Support for AJAX crawlable urls - New persistent scheduler that stores requests on disk, allowing to suspend and resume crawls (:rev:`2737`) - added ``-o`` option to ``scrapy crawl``, a shortcut for dumping scraped items into a file (or standard output using ``-``) - Added support for passing custom settings to Scrapyd ``schedule.json`` api (:rev:`2779`, :rev:`2783`) @@ -5939,7 +8897,7 @@ API changes - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler`` - ``scrapy.core.manager.scrapymanager`` singleton moved to ``scrapy.project.crawler`` - Moved module: ``scrapy.contrib.spidermanager`` to ``scrapy.spidermanager`` -- Spider Manager singleton moved from ``scrapy.spider.spiders`` to the ``spiders` attribute of ``scrapy.project.crawler`` singleton. +- Spider Manager singleton moved from ``scrapy.spider.spiders`` to the ``spiders`` attribute of ``scrapy.project.crawler`` singleton. - moved Stats Collector classes: (#204) - ``scrapy.stats.collector.StatsCollector`` to ``scrapy.statscol.StatsCollector`` - ``scrapy.stats.collector.SimpledbStatsCollector`` to ``scrapy.contrib.statscol.SimpledbStatsCollector`` @@ -6058,11 +9016,10 @@ Scrapy 0.7 First release of Scrapy. -.. _AJAX crawlable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1 .. _boto3: https://github.com/boto/boto3 .. _botocore: https://github.com/boto/botocore .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding -.. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/ +.. _ClientForm: https://pypi.org/project/ClientForm/ .. _Creating a pull request: https://help.github.com/en/articles/creating-a-pull-request .. _cryptography: https://cryptography.io/en/latest/ .. _docstrings: https://docs.python.org/3/glossary.html#term-docstring @@ -6074,8 +9031,8 @@ First release of Scrapy. .. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator .. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator .. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr -.. _PEP 257: https://www.python.org/dev/peps/pep-0257/ -.. _Pillow: https://python-pillow.org/ +.. _PEP 257: https://peps.python.org/pep-0257/ +.. _Pillow: https://github.com/python-pillow/Pillow .. _pyOpenSSL: https://www.pyopenssl.org/en/stable/ .. _queuelib: https://github.com/scrapy/queuelib .. _registered with IANA: https://www.iana.org/assignments/media-types/media-types.xhtml @@ -6086,7 +9043,7 @@ First release of Scrapy. .. _service_identity: https://service-identity.readthedocs.io/en/stable/ .. _six: https://six.readthedocs.io/ .. _tox: https://pypi.org/project/tox/ -.. _Twisted: https://twistedmatrix.com/trac/ +.. _Twisted: https://twisted.org/ .. _w3lib: https://github.com/scrapy/w3lib .. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py .. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 diff --git a/docs/requirements.in b/docs/requirements.in new file mode 100644 index 000000000..140791641 --- /dev/null +++ b/docs/requirements.in @@ -0,0 +1,8 @@ +h2 +pydantic +scrapy-spider-metadata +sphinx +sphinx-notfound-page +sphinx-rtd-theme +sphinx-rtd-dark-mode +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.6 diff --git a/docs/requirements.txt b/docs/requirements.txt index 5f683d34c..9c93dacd0 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,197 @@ -sphinx==6.2.1 -sphinx-hoverxref==1.3.0 -sphinx-notfound-page==1.0.0 -sphinx-rtd-theme==2.0.0 +# This file was autogenerated by uv via the following command: +# uv pip compile -p 3.13 docs/requirements.in -o docs/requirements.txt +alabaster==1.0.0 + # via sphinx +annotated-types==0.7.0 + # via pydantic +attrs==26.1.0 + # via + # service-identity + # twisted +automat==25.4.16 + # via twisted +babel==2.18.0 + # via sphinx +certifi==2026.2.25 + # via requests +cffi==2.0.0 + # via cryptography +charset-normalizer==3.4.6 + # via requests +constantly==23.10.4 + # via twisted +cryptography==46.0.6 + # via + # pyopenssl + # scrapy + # service-identity +cssselect==1.4.0 + # via + # parsel + # scrapy +defusedxml==0.7.1 + # via scrapy +docutils==0.22.4 + # via + # sphinx + # sphinx-markdown-builder + # sphinx-rtd-theme +filelock==3.25.2 + # via tldextract +h2==4.3.0 + # via -r docs/requirements.in +hpack==4.1.0 + # via h2 +hyperframe==6.1.0 + # via h2 +hyperlink==21.0.0 + # via twisted +idna==3.11 + # via + # hyperlink + # requests + # tldextract +imagesize==2.0.0 + # via sphinx +incremental==24.11.0 + # via twisted +itemadapter==0.13.1 + # via + # itemloaders + # scrapy +itemloaders==1.4.0 + # via scrapy +jinja2==3.1.6 + # via sphinx +jmespath==1.1.0 + # via + # itemloaders + # parsel +lxml==6.0.2 + # via + # parsel + # scrapy +markupsafe==3.0.3 + # via jinja2 +packaging==26.0 + # via + # incremental + # parsel + # scrapy + # scrapy-spider-metadata + # sphinx + # sphinx-scrapy +parsel==1.11.0 + # via + # itemloaders + # scrapy +protego==0.6.0 + # via scrapy +pyasn1==0.6.3 + # via + # pyasn1-modules + # service-identity +pyasn1-modules==0.4.2 + # via service-identity +pycparser==3.0 + # via cffi +pydantic==2.12.5 + # via + # -r docs/requirements.in + # scrapy-spider-metadata +pydantic-core==2.41.5 + # via pydantic +pydispatcher==2.0.7 + # via scrapy +pygments==2.19.2 + # via sphinx +pyopenssl==26.0.0 + # via scrapy +queuelib==1.9.0 + # via scrapy +requests==2.33.0 + # via + # requests-file + # sphinx + # tldextract +requests-file==3.0.1 + # via tldextract +roman-numerals==4.1.0 + # via sphinx +scrapy==2.14.2 + # via scrapy-spider-metadata +scrapy-spider-metadata==0.2.0 + # via -r docs/requirements.in +service-identity==24.2.0 + # via scrapy +snowballstemmer==3.0.1 + # via sphinx +sphinx==9.1.0 + # via + # -r docs/requirements.in + # sphinx-copybutton + # sphinx-last-updated-by-git + # sphinx-llms-txt + # sphinx-markdown-builder + # sphinx-notfound-page + # sphinx-rtd-theme + # sphinx-scrapy + # sphinxcontrib-jquery +sphinx-copybutton==0.5.2 + # via sphinx-scrapy +sphinx-last-updated-by-git==0.3.8 + # via sphinx-sitemap +sphinx-llms-txt @ git+https://github.com/zytedata/sphinx-llms-txt.git@5e8866cb0cc249aa2017ad9050b3b83a7ca16f69 + # via sphinx-scrapy +sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builder.git@cfe4c0bfd7b4542f7e6b65a58cdf9ec765829940 + # via sphinx-scrapy +sphinx-notfound-page==1.1.0 + # via -r docs/requirements.in +sphinx-rtd-dark-mode==1.3.0 + # via -r docs/requirements.in +sphinx-rtd-theme==3.1.0 + # via + # -r docs/requirements.in + # sphinx-rtd-dark-mode +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@b1d55db4d16a5425fc68576d63519bbfe26dd9c0 + # via -r docs/requirements.in +sphinx-sitemap==2.9.0 + # via sphinx-scrapy +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jquery==4.1 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +tabulate==0.10.0 + # via sphinx-markdown-builder +tldextract==5.3.1 + # via scrapy +twisted==25.5.0 + # via scrapy +typing-extensions==4.15.0 + # via + # pydantic + # pydantic-core + # twisted + # typing-inspection +typing-inspection==0.4.2 + # via pydantic +urllib3==2.6.3 + # via requests +w3lib==2.4.1 + # via + # parsel + # scrapy +zope-interface==8.2 + # via + # scrapy + # twisted diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 1bf2172bd..01e4bcac1 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -32,7 +32,8 @@ This is an example where two add-ons are enabled in a project's Writing your own add-ons ======================== -Add-ons are Python classes that include the following method: +Add-ons are :ref:`components ` that include one or both of +the following methods: .. method:: update_settings(settings) @@ -45,19 +46,14 @@ Add-ons are Python classes that include the following method: :param settings: The settings object storing Scrapy/component configuration :type settings: :class:`~scrapy.settings.Settings` -They can also have the following method: +.. classmethod:: update_pre_crawler_settings(cls, settings) -.. classmethod:: from_crawler(cls, crawler) - :noindex: + Use this class method instead of the :meth:`update_settings` method to + update :ref:`pre-crawler settings ` whose value is + used before the :class:`~scrapy.crawler.Crawler` object is created. - If present, this class method is called to create an add-on instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the add-on. The crawler object provides access to all Scrapy core - components like settings and signals; it is a way for the add-on to access - them and hook its functionality into Scrapy. - - :param crawler: The crawler that uses this add-on - :type crawler: :class:`~scrapy.crawler.Crawler` + :param settings: The settings object storing Scrapy/component configuration + :type settings: :class:`~scrapy.settings.BaseSettings` The settings set by the add-on should use the ``addon`` priority (see :ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):: @@ -67,15 +63,11 @@ The settings set by the add-on should use the ``addon`` priority (see settings.set("DNSCACHE_ENABLED", True, "addon") This allows users to override these settings in the project or spider -configuration. This is not possible with settings that are mutable objects, -such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases -you can provide an add-on-specific setting that governs whether the add-on will -modify :setting:`ITEM_PIPELINES`:: +configuration. - class MyAddon: - def update_settings(self, settings): - if settings.getbool("MYADDON_ENABLE_PIPELINE"): - settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 +When editing the value of a setting instead of overriding it entirely, it is +usually best to leave its priority unchanged. For example, when editing a +:ref:`component priority dictionary `. If the ``update_settings`` method raises :exc:`scrapy.exceptions.NotConfigured`, the add-on will be skipped. This makes @@ -96,7 +88,7 @@ recommend that such custom components should be written in the following way: 1. The custom component (e.g. ``MyDownloadHandler``) shouldn't inherit from the default Scrapy one (e.g. - ``scrapy.core.downloader.handlers.http.HTTPDownloadHandler``), but instead + ``scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler``), but instead be able to load the class of the fallback component from a special setting (e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use it. @@ -106,9 +98,9 @@ recommend that such custom components should be written in the following way: (``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default setting to the component provided by the add-on (e.g. ``MyDownloadHandler``). If the fallback setting is already set by the user, - they shouldn't change it. + it should not be changed. 3. This way, if there are several add-ons that want to modify the same setting, - all of them will fallback to the component from the previous one and then to + all of them will fall back to the component from the previous one and then to the Scrapy default. The order of that depends on the priority order in the ``ADDONS`` setting. @@ -118,12 +110,30 @@ Add-on examples Set some basic configuration: +.. skip: next .. code-block:: python + from myproject.pipelines import MyPipeline + + class MyAddon: def update_settings(self, settings): - settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 settings.set("DNSCACHE_ENABLED", True, "addon") + settings.remove_from_list("METAREFRESH_IGNORE_TAGS", "noscript") + settings.setdefault_in_component_priority_dict( + "ITEM_PIPELINES", MyPipeline, 200 + ) + +.. _priority-dict-helpers: + +.. tip:: When editing a :ref:`component priority dictionary + ` setting, like :setting:`ITEM_PIPELINES`, + consider using setting methods like + :meth:`~scrapy.settings.BaseSettings.replace_in_component_priority_dict`, + :meth:`~scrapy.settings.BaseSettings.set_in_component_priority_dict` + and + :meth:`~scrapy.settings.BaseSettings.setdefault_in_component_priority_dict` + to avoid mistakes. Check dependencies: @@ -150,14 +160,13 @@ Access the crawler instance: def from_crawler(cls, crawler): return cls(crawler) - def update_settings(self, settings): - ... + def update_settings(self, settings): ... Use a fallback component: .. code-block:: python - from scrapy.core.downloader.handlers.http import HTTPDownloadHandler + from scrapy.utils.misc import build_from_crawler, load_object FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" @@ -166,20 +175,19 @@ Use a fallback component: class MyHandler: lazy = False - def __init__(self, settings, crawler): - dhcls = load_object(settings.get(FALLBACK_SETTING)) - self._fallback_handler = create_instance( - dhcls, - settings=None, - crawler=crawler, - ) + def __init__(self, crawler): + dhcls = load_object(crawler.settings.get(FALLBACK_SETTING)) + self._fallback_handler = build_from_crawler(dhcls, crawler) - def download_request(self, request, spider): + async def download_request(self, request): if request.meta.get("my_params"): # handle the request ... else: - return self._fallback_handler.download_request(request, spider) + return await self._fallback_handler.download_request(request) + + async def close(self): + pass class MyAddon: diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 175c877de..19082d9d7 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -12,10 +12,11 @@ extensions and middlewares. Crawler API =========== -The main entry point to Scrapy API is the :class:`~scrapy.crawler.Crawler` -object, passed to extensions through the ``from_crawler`` class method. This -object provides access to all Scrapy core components, and it's the only way for -extensions to access them and hook their functionality into Scrapy. +The main entry point to the Scrapy API is the :class:`~scrapy.crawler.Crawler` +object, which :ref:`components ` can :ref:`get for +initialization `. It provides access to all Scrapy core +components, and it is the only way for components to access them and hook their +functionality into Scrapy. .. module:: scrapy.crawler :synopsis: The Scrapy crawler @@ -26,7 +27,9 @@ contains a dictionary of all available extensions and their order similar to how you :ref:`configure the downloader middlewares `. -.. class:: Crawler(spidercls, settings) +.. autoclass:: Crawler + :members: get_addon, get_downloader_middleware, get_extension, + get_item_pipeline, get_spider_middleware The Crawler object must be instantiated with a :class:`scrapy.Spider` subclass and a @@ -86,7 +89,7 @@ how you :ref:`configure the downloader middlewares The execution engine, which coordinates the core crawling logic between the scheduler, downloader and spiders. - Some extension may want to access the Scrapy engine, to inspect or + Some extension may want to access the Scrapy engine, to inspect or modify the downloader and scheduler behaviour, although this is an advanced use and this API is not yet stable. @@ -96,19 +99,25 @@ how you :ref:`configure the downloader middlewares provided while constructing the crawler, and it is created after the arguments given in the :meth:`crawl` method. - .. method:: crawl(*args, **kwargs) + .. automethod:: crawl_async - Starts the crawler by instantiating its spider class with the given - ``args`` and ``kwargs`` arguments, while setting the execution engine in - motion. Should be called only once. + .. automethod:: crawl - Returns a deferred that is fired when the crawl is finished. + .. automethod:: stop_async .. automethod:: stop +.. autoclass:: AsyncCrawlerRunner + :members: + .. autoclass:: CrawlerRunner :members: +.. autoclass:: AsyncCrawlerProcess + :show-inheritance: + :members: + :inherited-members: + .. autoclass:: CrawlerProcess :show-inheritance: :members: @@ -204,6 +213,8 @@ SpiderLoader API :param request: queried request :type request: :class:`~scrapy.Request` instance +.. autoclass:: DummySpiderLoader + .. _topics-api-signals: Signals API @@ -269,11 +280,17 @@ class (which they all inherit from). The following methods are not part of the stats collection api but instead used when implementing custom stats collectors: - .. method:: open_spider(spider) + .. method:: open_spider() - Open the given spider for stats collection. + Open the spider for stats collection. - .. method:: close_spider(spider) + .. method:: close_spider() - Close the given spider. After this is called, no more specific stats + Close the spider. After this is called, no more specific stats can be accessed or collected. + +Engine API +========== + +.. autoclass:: scrapy.core.engine.ExecutionEngine() + :members: needs_backout diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 0c3a7ed88..c60c43f3c 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -63,7 +63,7 @@ this: :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`). 8. The :ref:`Engine ` sends processed items to - :ref:`Item Pipelines `, then send processed Requests to + :ref:`Item Pipelines `, then sends processed Requests to the :ref:`Scheduler ` and asks for possible next Requests to crawl. @@ -87,8 +87,8 @@ of the system, and triggering events when certain actions occur. See the Scheduler --------- -The :ref:`scheduler ` receives requests from the engine and -enqueues them for feeding them later (also to the engine) when the engine +The :ref:`scheduler ` receives requests from the engine and +enqueues them for feeding them later (also to the engine) when the engine requests them. .. _component-downloader: @@ -150,7 +150,7 @@ requests). Use a Spider middleware if you need to * post-process output of spider callbacks - change/add/remove requests or items; -* post-process start_requests; +* post-process start requests or items; * handle spider exceptions; * call errback instead of callback for some of the requests based on response content. @@ -168,9 +168,7 @@ For more information about asynchronous programming and Twisted see these links: * :doc:`twisted:core/howto/defer-intro` -* `Twisted - hello, asynchronous programming`_ * `Twisted Introduction - Krondo`_ -.. _Twisted: https://twistedmatrix.com/trac/ -.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/ -.. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/ +.. _Twisted: https://twisted.org/ +.. _Twisted Introduction - Krondo: https://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/ diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 07baea071..8efa2559e 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -4,8 +4,6 @@ asyncio ======= -.. versionadded:: 2.0 - Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the asyncio reactor `, you may use :mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine `. @@ -16,15 +14,20 @@ asyncio reactor `, you may use :mod:`asyncio` and Installing the asyncio reactor ============================== -To enable :mod:`asyncio` support, set the :setting:`TWISTED_REACTOR` setting to -``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``. +To enable :mod:`asyncio` support, your :setting:`TWISTED_REACTOR` setting needs +to be set to ``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, +which is the default value. -If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to +If you are using :class:`~scrapy.crawler.AsyncCrawlerRunner` or +:class:`~scrapy.crawler.CrawlerRunner`, you also need to install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` reactor manually. You can do that using -:func:`~scrapy.utils.reactor.install_reactor`:: +:func:`~scrapy.utils.reactor.install_reactor`: - install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') +.. skip: next +.. code-block:: python + + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") .. _asyncio-preinstalled-reactor: @@ -44,6 +47,7 @@ You can usually fix the issue by moving those offending module-level Twisted imports to the method or function definitions where they are used. For example, if you have something like: +.. skip: next .. code-block:: python from twisted.internet import reactor @@ -68,24 +72,33 @@ those imports happen. .. _asyncio-await-dfd: -Awaiting on Deferreds -===================== +Integrating Deferred code and asyncio code +========================================== -When the asyncio reactor isn't installed, you can await on Deferreds in the -coroutines directly. When it is installed, this is not possible anymore, due to -specifics of the Scrapy coroutine integration (the coroutines are wrapped into -:class:`asyncio.Future` objects, not into -:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into -Futures. Scrapy provides two helpers for this: +Coroutine functions can await on Deferreds by wrapping them into +:class:`asyncio.Future` objects. Scrapy provides two helpers for this: .. autofunction:: scrapy.utils.defer.deferred_to_future .. autofunction:: scrapy.utils.defer.maybe_deferred_to_future + +.. tip:: If you don't need to support reactors other than the default + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, you + can use :func:`~scrapy.utils.defer.deferred_to_future`, otherwise you + should use :func:`~scrapy.utils.defer.maybe_deferred_to_future`. + .. tip:: If you need to use these functions in code that aims to be compatible with lower versions of Scrapy that do not provide these functions, down to Scrapy 2.0 (earlier versions do not support :mod:`asyncio`), you can copy the implementation of these functions into your own code. +Coroutines and futures can be wrapped into Deferreds (for example, when a +Scrapy API requires passing a Deferred to it) using the following helpers: + +.. autofunction:: scrapy.utils.defer.deferred_from_coro +.. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f +.. autofunction:: scrapy.utils.defer.ensure_awaitable + .. _enforce-asyncio-requirement: @@ -93,25 +106,195 @@ Enforcing asyncio as a requirement ================================== If you are writing a :ref:`component ` that requires asyncio -to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to +to work, use :func:`scrapy.utils.asyncio.is_asyncio_available` to :ref:`enforce it as a requirement `. For example: .. code-block:: python - from scrapy.utils.reactor import is_asyncio_reactor_installed + from scrapy.utils.asyncio import is_asyncio_available class MyComponent: def __init__(self): - if not is_asyncio_reactor_installed(): + if not is_asyncio_available(): raise ValueError( - f"{MyComponent.__qualname__} requires the asyncio Twisted " - f"reactor. Make sure you have it configured in the " + f"{MyComponent.__qualname__} requires the asyncio support. " + f"Make sure you have configured the asyncio reactor in the " f"TWISTED_REACTOR setting. See the asyncio documentation " f"of Scrapy for more information." ) +.. autofunction:: scrapy.utils.asyncio.is_asyncio_available +.. autofunction:: scrapy.utils.reactor.is_asyncio_reactor_installed + + +.. _asyncio-without-reactor: + +Using Scrapy without a Twisted reactor +====================================== + +.. versionadded:: 2.15.0 + +.. warning:: + This is currently experimental and may not be suitable for production use. + +It's possible to use Scrapy without installing a Twisted reactor at all, by +setting the :setting:`TWISTED_REACTOR_ENABLED` setting to ``False``. In this +mode Scrapy will use the asyncio event loop directly, and most of the Scrapy +functionality will work in the same way. + +Doing this provides several benefits in certain use cases: + +* A Twisted reactor, once stopped, cannot be started again. This prevents, for + example, using several instances of + :class:`~scrapy.crawler.AsyncCrawlerProcess` in the same process when they + use a reactor, but with ``TWISTED_REACTOR_ENABLED=False`` it becomes + possible. +* There may be limitations imposed by + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` and related + Twisted code, such as the requirement of using + :class:`~asyncio.SelectorEventLoop` on Windows (see :ref:`asyncio-windows`), + that do not apply if the reactor is not used. +* :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` manages the + underlying event loop, and while :class:`~scrapy.crawler.AsyncCrawlerRunner` + can use a pre-existing reactor which, in turn, can use a pre-existing event + loop, it's easier to use :class:`~scrapy.crawler.AsyncCrawlerRunner` with a + pre-existing loop directly. +* Omitting the reactor machinery may improve performance and reliability. + +Limitations +----------- + +As some Scrapy features and components require a reactor, they don't work and +are disabled without it. Replacements that don't require a reactor may be added +in future Scrapy versions. The following features are not available: + +* The default HTTP(S) download handler, + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` (this + is likely the biggest difference; Scrapy provides an HTTP(S) download handler + that doesn't require a reactor and will be used instead of it: + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`) +* :class:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler` +* :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` +* :ref:`topics-telnetconsole` +* :class:`~scrapy.crawler.CrawlerRunner` and + :class:`~scrapy.crawler.CrawlerProcess` + (:class:`~scrapy.crawler.AsyncCrawlerProcess` and + :class:`~scrapy.crawler.AsyncCrawlerRunner` are available) +* Twisted-specific DNS resolvers (the :setting:`DNS_RESOLVER` setting) +* User and 3rd-party code that requires a reactor (see :ref:`below + ` for examples) + +Note that importing Twisted modules and, among other things, creating and using +:class:`~twisted.internet.defer.Deferred` objects doesn't require a reactor, so +code that uses :class:`~twisted.internet.defer.Deferred`, +:class:`~twisted.python.failure.Failure` and some other Twisted APIs will not +necessarily stop working. + +Other differences +----------------- + +When :setting:`TWISTED_REACTOR_ENABLED` is set to ``False``, Scrapy will change +the defaults of some other settings: + +* :setting:`TELNETCONSOLE_ENABLED` is set to ``False``. +* The ``"http"`` and ``"https"`` keys in :setting:`DOWNLOAD_HANDLERS_BASE` are + set to ``"scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler"``. +* The ``"ftp"`` key in :setting:`DOWNLOAD_HANDLERS_BASE` is set to ``None``. + +Thus, :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` is +used by default for making HTTP(S) requests. Please refer to its documentation +for its differences and limitations compared to +:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`. + +Additionally, :class:`~scrapy.crawler.AsyncCrawlerProcess` will install a +:term:`meta path finder` that prevents :mod:`twisted.internet.reactor` from +being imported. + +.. _asyncio-without-reactor-migrate: + +Adding support to existing code +------------------------------- + +Code that doesn't directly use Twisted APIs or APIs that depend on Twisted ones +doesn't need special support for running without a reactor. + +Here are some examples of APIs and patterns that need a replacement: + +* Using :meth:`reactor.callLater() + ` for sleeping or delayed calls. + You can use :meth:`asyncio.loop.call_later` instead. +* Using :func:`twisted.internet.threads.deferToThread`, + :meth:`reactor.callFromThread() + ` and related APIs to + execute code in other threads. You can use :func:`asyncio.to_thread`, + :meth:`asyncio.loop.call_soon_threadsafe` and related APIs instead. +* Using :class:`twisted.internet.task.LoopingCall` for scheduling repeated + tasks. As there is no direct replacement in the standard library, you may + need to write your own one using :func:`asyncio.sleep` in a task. +* Using Twisted network client and server APIs (:meth:`reactor.connectTCP() + `, + :meth:`reactor.listenTCP() + `, + :mod:`twisted.web.client`, :mod:`twisted.mail.smtp` etc.). You can use other + built-in or 3rd-party libraries for this. +* Using :class:`~scrapy.crawler.CrawlerProcess` or + :class:`~scrapy.crawler.CrawlerRunner`. You should use + :class:`~scrapy.crawler.AsyncCrawlerProcess` or + :class:`~scrapy.crawler.AsyncCrawlerRunner` respectively instead. +* Checking whether ``asyncio`` support is available with + :func:`scrapy.utils.reactor.is_asyncio_reactor_installed`. You should use + :func:`scrapy.utils.asyncio.is_asyncio_available` instead. + +Scrapy provides unified helpers for some of these examples: + +.. autofunction:: scrapy.utils.asyncio.call_later +.. autofunction:: scrapy.utils.asyncio.create_looping_call +.. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall +.. autofunction:: scrapy.utils.asyncio.run_in_thread + +If your code needs to know whether the reactor is available, you can either +check for the value of the :setting:`TWISTED_REACTOR_ENABLED` setting (you need +access to the :class:`~scrapy.crawler.Crawler` instance to do this) or use the +following function: + +.. autofunction:: scrapy.utils.reactorless.is_reactorless + +In general, code that doesn't use the reactor (directly or indirectly) can be +used unmodified both with the asyncio reactor and without a reactor. This +includes code that converts Deferreds to futures and vice versa as described in +:ref:`asyncio-await-dfd`. + +Troubleshooting +--------------- + +**ImportError: Import of twisted.internet.reactor is forbidden when running +without a Twisted reactor [...]:** Scrapy is configured to run without a +reactor, but some code imported :mod:`twisted.internet.reactor`, most likely +because that code needs a reactor to be used. You need to stop using this code +or set :setting:`TWISTED_REACTOR_ENABLED` back to ``True``. It's also possible +that the reactor isn't really needed but was installed due to the problem +described in :ref:`asyncio-preinstalled-reactor`, in which case it should be +enough to fix the problematic imports. + +**RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is +installed:** Scrapy is configured to run without a reactor, but a reactor is +already installed before the Scrapy code is executed. If you are trying to set +:setting:`TWISTED_REACTOR_ENABLED` via :ref:`per-spider settings +`, it's currently unsupported. + +**RuntimeError: We expected a Twisted reactor to be installed but it isn't:** +Scrapy is configured to run with a reactor and not to install one, but a +reactor wasn't installed before the Scrapy code is executed. If you are trying +to set :setting:`TWISTED_REACTOR_ENABLED` via :ref:`per-spider settings +`, it's currently unsupported. + +**RuntimeError: doesn't support TWISTED_REACTOR_ENABLED=False:** The +listed class cannot be used with :setting:`TWISTED_REACTOR_ENABLED` set to +``False``. There may be a replacement in the :ref:`documentation above +` or the documentation of the affected class. + .. _asyncio-windows: @@ -133,6 +316,9 @@ automatically when you change the :setting:`TWISTED_REACTOR` setting or call them together with Scrapy on Windows (but you should be able to use them on WSL or native Linux). +.. note:: This problem doesn't apply when not using the reactor, see + :ref:`asyncio-without-reactor`. + .. _playwright: https://github.com/microsoft/playwright-python @@ -144,3 +330,18 @@ Using custom asyncio loops You can also use custom asyncio event loops with the asyncio reactor. Set the :setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to use it instead of the default asyncio event loop. + + +.. _disable-asyncio: + +Switching to a non-asyncio reactor +================================== + +If for some reason your code doesn't work with the asyncio reactor, you can use +a different reactor by setting the :setting:`TWISTED_REACTOR` setting to its +import path (e.g. ``'twisted.internet.epollreactor.EPollReactor'``) or to +``None``, which will use the default reactor for your platform. If you are +using :class:`~scrapy.crawler.AsyncCrawlerRunner` or +:class:`~scrapy.crawler.AsyncCrawlerProcess` you also need to switch to their +Deferred-based counterparts: :class:`~scrapy.crawler.CrawlerRunner` or +:class:`~scrapy.crawler.CrawlerProcess` respectively. diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 8e6aae65c..d0321c906 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -21,9 +21,14 @@ Design goals How it works ============ -AutoThrottle extension adjusts download delays dynamically to make spider send -:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent requests on average -to each remote website. +Scrapy allows defining the concurrency and delay of different download slots, +e.g. through the :setting:`DOWNLOAD_SLOTS` setting. By default requests are +assigned to slots based on their URL domain, although it is possible to +customize the download slot of any request. + +The AutoThrottle extension adjusts the delay of each download slot dynamically, +to make your spider send :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent +requests on average to each remote website. It uses download latency to compute the delays. The main idea is the following: if a server needs ``latency`` seconds to respond, a client @@ -32,8 +37,7 @@ processed in parallel. Instead of adjusting the delays one can just set a small fixed download delay and impose hard limits on concurrency using -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or -:setting:`CONCURRENT_REQUESTS_PER_IP` options. It will provide a similar +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. It will provide a similar effect, but there are some important differences: * because the download delay is small there will be occasional bursts @@ -66,7 +70,6 @@ AutoThrottle algorithm adjusts download delays based on the following rules: .. note:: The AutoThrottle extension honours the standard Scrapy settings for concurrency and delay. This means that it will respect :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and - :setting:`CONCURRENT_REQUESTS_PER_IP` options and never set a download delay lower than :setting:`DOWNLOAD_DELAY`. .. _download-latency: @@ -80,6 +83,33 @@ callback, for example, and unable to attend downloads. However, these latencies should still give a reasonable estimate of how busy Scrapy (and ultimately, the server) is, and this extension builds on that premise. +.. reqmeta:: autothrottle_dont_adjust_delay + +Prevent specific requests from triggering slot delay adjustments +================================================================ + +AutoThrottle adjusts the delay of download slots based on the latencies of +responses that belong to that download slot. The only exceptions are non-200 +responses, which are only taken into account to increase that delay, but +ignored if they would decrease that delay. + +You can also set the ``autothrottle_dont_adjust_delay`` request metadata key to +``True`` in any request to prevent its response latency from impacting the +delay of its download slot: + +.. code-block:: python + + from scrapy import Request + + Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True}) + +Note, however, that AutoThrottle still determines the starting delay of every +download slot by setting the ``download_delay`` attribute on the running +spider. If you want AutoThrottle not to impact a download slot at all, in +addition to setting this meta key in all requests that use that download slot, +you might want to set a custom value for the ``delay`` attribute of that +download slot, e.g. using :setting:`DOWNLOAD_SLOTS`. + Settings ======== @@ -91,7 +121,6 @@ The settings used to control the AutoThrottle extension are: * :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` * :setting:`AUTOTHROTTLE_DEBUG` * :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` -* :setting:`CONCURRENT_REQUESTS_PER_IP` * :setting:`DOWNLOAD_DELAY` For more information see :ref:`autothrottle-algorithm`. @@ -131,7 +160,7 @@ AUTOTHROTTLE_TARGET_CONCURRENCY Default: ``1.0`` Average number of requests Scrapy should be sending in parallel to remote -websites. +websites. It must be higher than ``0.0``. By default, AutoThrottle adjusts the delay to send a single concurrent request to each of the remote websites. Set this option to @@ -139,12 +168,10 @@ a higher value (e.g. ``2.0``) to increase the throughput and the load on remote servers. A lower ``AUTOTHROTTLE_TARGET_CONCURRENCY`` value (e.g. ``0.5``) makes the crawler more conservative and polite. -Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` -and :setting:`CONCURRENT_REQUESTS_PER_IP` options are still respected +Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is still respected when AutoThrottle extension is enabled. This means that if ``AUTOTHROTTLE_TARGET_CONCURRENCY`` is set to a value higher than -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or -:setting:`CONCURRENT_REQUESTS_PER_IP`, the crawler won't reach this number +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, the crawler won't reach this number of concurrent requests. At every given time point Scrapy can be sending more or less concurrent diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 0643df6a6..e8ddec00c 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -24,7 +24,8 @@ You should see an output like this:: 'scrapy.extensions.telnet.TelnetConsole', 'scrapy.extensions.corestats.CoreStats'] 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled downloader middlewares: - ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', + ['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware', + 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', @@ -37,7 +38,6 @@ You should see an output like this:: 'scrapy.downloadermiddlewares.stats.DownloaderStats'] 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled spider middlewares: ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', - 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', 'scrapy.spidermiddlewares.referer.RefererMiddleware', 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', 'scrapy.spidermiddlewares.depth.DepthMiddleware'] @@ -83,4 +83,4 @@ and how well it's written. Use scrapy-bench_ for more complex benchmarking. -.. _scrapy-bench: https://github.com/scrapy/scrapy-bench \ No newline at end of file +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 8be89feb2..cace1f883 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -41,19 +41,6 @@ efficient broad crawl. .. _broad-crawls-scheduler-priority-queue: -Use the right :setting:`SCHEDULER_PRIORITY_QUEUE` -================================================= - -Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. -It works best during single-domain crawl. It does not work well with crawling -many different domains in parallel - -To apply the recommended priority queue use: - -.. code-block:: python - - SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.DownloaderAwarePriorityQueue" - .. _broad-crawls-concurrency: Increase concurrency @@ -61,12 +48,7 @@ Increase concurrency Concurrency is the number of requests that are processed in parallel. There is a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that -can be set either per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) or per -IP (:setting:`CONCURRENT_REQUESTS_PER_IP`). - -.. note:: The scheduler priority queue :ref:`recommended for broad crawls - ` does not support - :setting:`CONCURRENT_REQUESTS_PER_IP`. +can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`). The default global concurrency limit in Scrapy is not suitable for crawling many different domains in parallel, so you will want to increase it. How much @@ -116,7 +98,7 @@ Reduce log level When doing broad crawls you are often only interested in the crawl rates you get and any errors found. These stats are reported by Scrapy when using the ``INFO`` log level. In order to save CPU (and log storage requirements) you -should not use ``DEBUG`` log level when preforming large broad crawls in +should not use ``DEBUG`` log level when performing large broad crawls in production. Using ``DEBUG`` level when developing your (broad) crawler may be fine though. @@ -143,7 +125,7 @@ To disable cookies use: Disable retries =============== -Retrying failed HTTP requests can slow down the crawls substantially, specially +Retrying failed HTTP requests can slow down the crawls substantially, especially when sites causes are very slow (or fail) to respond, thus causing a timeout error which gets retried many times, unnecessarily, preventing crawler capacity to be reused for other domains. @@ -182,32 +164,6 @@ To disable redirects use: REDIRECT_ENABLED = False -Enable crawling of "Ajax Crawlable Pages" -========================================= - -Some pages (up to 1%, based on empirical data from year 2013) declare -themselves as `ajax crawlable`_. This means they provide plain HTML -version of content that is usually available only via AJAX. -Pages can indicate it in two ways: - -1) by using ``#!`` in URL - this is the default way; -2) by using a special meta tag - this way is used on - "main", "index" website pages. - -Scrapy handles (1) automatically; to handle (2) enable -:ref:`AjaxCrawlMiddleware `: - -.. code-block:: python - - AJAXCRAWL_ENABLED = True - -When doing broad crawls it's common to crawl a lot of "index" web pages; -AjaxCrawlMiddleware helps to crawl them correctly. -It is turned OFF by default because it has some performance overhead, -and enabling it for focused crawls doesn't make much sense. - -.. _ajax crawlable: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started - .. _broad-crawls-bfo: Crawl in BFO order diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 1d37895c2..8d1351eb9 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -6,7 +6,7 @@ Command line tool ================= -Scrapy is controlled through the ``scrapy`` command-line tool, to be referred +Scrapy is controlled through the ``scrapy`` command-line tool, to be referred to here as the "Scrapy tool" to differentiate it from the sub-commands, which we just call "commands" or "Scrapy commands". @@ -163,8 +163,8 @@ information on which commands must be run from inside projects, and which not. Also keep in mind that some commands may have slightly different behaviours when running them from inside projects. For example, the fetch command will use -spider-overridden behaviours (such as the ``user_agent`` attribute to override -the user-agent) if the url being fetched is associated with some specific +spider-overridden behaviours (such as the ``custom_settings`` attribute to +override settings) if the url being fetched is associated with some specific spider. This is intentional, as the ``fetch`` command is meant to be used to check how spiders are downloading pages. @@ -185,8 +185,8 @@ And you can see all available commands with:: There are two kinds of commands, those that only work from inside a Scrapy project (Project-specific commands) and those that also work without an active -Scrapy project (Global commands), though they may behave slightly different -when running from inside a project (as they would use the project overridden +Scrapy project (Global commands), though they may behave slightly differently +when run from inside a project (as they would use the project overridden settings). Global commands: @@ -233,10 +233,7 @@ genspider * Syntax: ``scrapy genspider [-t template] `` * Requires project: *no* -.. versionadded:: 2.6.0 - The ability to pass a URL instead of a domain. - -Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. +Creates a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. Usage example:: @@ -253,7 +250,7 @@ Usage example:: $ scrapy genspider -t crawl scrapyorg scrapy.org Created spider 'scrapyorg' using template 'crawl' -This is just a convenience shortcut command for creating spiders based on +This is just a convenient shortcut command for creating spiders based on pre-defined templates, but certainly not the only way to create spiders. You can just create the spider source code files yourself, instead of using this command. @@ -274,11 +271,9 @@ Supported options: * ``-a NAME=VALUE``: set a spider argument (may be repeated) -* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout), to define format set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``) +* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout). To define the output format, set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``) -* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file, to define format set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``) - -* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O`` +* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file. To define the output format, set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``) Usage examples:: @@ -291,9 +286,6 @@ Usage examples:: $ scrapy crawl -O myfile:json myspider [ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ] - $ scrapy crawl -o myfile -t csv myspider - [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ] - .. command:: check check @@ -353,7 +345,7 @@ edit Edit the given spider using the editor defined in the ``EDITOR`` environment variable or (if unset) the :setting:`EDITOR` setting. -This command is provided only as a convenience shortcut for the most common +This command is provided only as a convenient shortcut for the most common case, the developer is of course free to choose any tool or IDE to write and debug spiders. @@ -372,7 +364,7 @@ fetch Downloads the given URL using the Scrapy downloader and writes the contents to standard output. -The interesting thing about this command is that it fetches the page how the +The interesting thing about this command is that it fetches the page the way the spider would download it. For example, if the spider has a ``USER_AGENT`` attribute which overrides the User Agent, it will use that one. @@ -514,8 +506,6 @@ Supported options: * ``--output`` or ``-o``: dump scraped items to a file - .. versionadded:: 2.3 - .. skip: start Usage example:: @@ -592,6 +582,44 @@ bench Run a quick benchmark test. :ref:`benchmarking`. +.. _topics-commands-crawlerprocess: + +Commands that run a crawl +========================= + +Many commands need to run a crawl of some kind, running either a user-provided +spider or a special internal one: + +* :command:`bench` +* :command:`check` +* :command:`crawl` +* :command:`fetch` +* :command:`parse` +* :command:`runspider` +* :command:`shell` +* :command:`view` + +They use an internal instance of :class:`scrapy.crawler.AsyncCrawlerProcess` or +:class:`scrapy.crawler.CrawlerProcess` for this. In most cases this detail +shouldn't matter to the user running the command, but when the user :ref:`needs +a non-default Twisted reactor `, it may be important. + +Scrapy decides which of these two classes to use based on the value of the +:setting:`TWISTED_REACTOR` setting. If the setting value is the default one +(``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``), +:class:`~scrapy.crawler.AsyncCrawlerProcess` will be used, otherwise +:class:`~scrapy.crawler.CrawlerProcess` will be used. The :ref:`spider settings +` are not taken into account when doing this, as they are +loaded after this decision is made. This may cause an error if the +project-level setting is set to :ref:`the asyncio reactor ` +(:ref:`explicitly ` or :ref:`by using the Scrapy default +`) and :ref:`the setting of the spider being run +` is set to :ref:`a different one `, because +:class:`~scrapy.crawler.AsyncCrawlerProcess` only supports the asyncio reactor. +In this case you should set the :setting:`FORCE_CRAWLER_PROCESS` setting to +``True`` (at the project level or via the command line) so that Scrapy uses +:class:`~scrapy.crawler.CrawlerProcess` which supports all reactors. + Custom project commands ======================= diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 478dd9647..c0df86922 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -4,17 +4,17 @@ Components ========== -A Scrapy component is any class whose objects are created using -:func:`scrapy.utils.misc.create_instance`. +A Scrapy component is any class whose objects are built using +:func:`~scrapy.utils.misc.build_from_crawler`. That includes the classes that you may assign to the following settings: -- :setting:`DNS_RESOLVER` +- :setting:`ADDONS` + +- :setting:`TWISTED_DNS_RESOLVER` - :setting:`DOWNLOAD_HANDLERS` -- :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY` - - :setting:`DOWNLOADER_MIDDLEWARES` - :setting:`DUPEFILTER_CLASS` @@ -35,16 +35,90 @@ That includes the classes that you may assign to the following settings: - :setting:`SCHEDULER_PRIORITY_QUEUE` +- :setting:`SCHEDULER_START_DISK_QUEUE` + +- :setting:`SCHEDULER_START_MEMORY_QUEUE` + - :setting:`SPIDER_MIDDLEWARES` Third-party Scrapy components may also let you define additional Scrapy components, usually configurable through :ref:`settings `, to modify their behavior. +.. _from-crawler: + +Initializing from the crawler +============================= + +Any Scrapy component may optionally define the following class method: + +.. classmethod:: from_crawler(cls, crawler: scrapy.crawler.Crawler, *args, **kwargs) + + Return an instance of the component based on *crawler*. + + *args* and *kwargs* are component-specific arguments that some components + receive. However, most components do not get any arguments, and instead + :ref:`use settings `. + + If a component class defines this method, this class method is called to + create any instance of the component. + + The *crawler* object provides access to all Scrapy core components like + :ref:`settings ` and :ref:`signals `, + allowing the component to access them and hook its functionality into + Scrapy. + +.. _component-settings: + +Settings +======== + +Components can be configured through :ref:`settings `. + +Components can read any setting from the +:attr:`~scrapy.crawler.Crawler.settings` attribute of the +:class:`~scrapy.crawler.Crawler` object they can :ref:`get for initialization +`. That includes both built-in and custom settings. + +For example: + +.. code-block:: python + + class MyExtension: + @classmethod + def from_crawler(cls, crawler): + settings = crawler.settings + return cls(settings.getbool("LOG_ENABLED")) + + def __init__(self, log_is_enabled=False): + if log_is_enabled: + print("log is enabled!") + +Components do not need to declare their custom settings programmatically. +However, they should document them, so that users know they exist and how to +use them. + +It is a good practice to prefix custom settings with the name of the component, +to avoid collisions with custom settings of other existing (or future) +components. For example, an extension called ``WarcCaching`` could prefix its +custom settings with ``WARC_CACHING_``. + +Another good practice, mainly for components meant for :ref:`component priority +dictionaries `, is to provide a boolean setting +called ``_ENABLED`` (e.g. ``WARC_CACHING_ENABLED``) to allow toggling +that component on and off without changing the component priority dictionary +setting. You can usually check the value of such a setting during +initialization, and if ``False``, raise +:exc:`~scrapy.exceptions.NotConfigured`. + +When choosing a name for a custom setting, it is also a good idea to have a +look at the names of :ref:`built-in settings `, to try to +maintain consistency with them. + .. _enforce-component-requirements: -Enforcing component requirements -================================ +Enforcing requirements +====================== Sometimes, your components may only be intended to work under certain conditions. For example, they may require a minimum version of Scrapy to work as @@ -58,8 +132,8 @@ In the case of :ref:`downloader middlewares `, :ref:`extensions `, :ref:`item pipelines `, and :ref:`spider middlewares `, you should raise -:exc:`scrapy.exceptions.NotConfigured`, passing a description of the issue as a -parameter to the exception so that it is printed in the logs, for the user to +:exc:`~scrapy.exceptions.NotConfigured`, passing a description of the issue as +a parameter to the exception so that it is printed in the logs, for the user to see. For other components, feel free to raise whatever other exception feels right to you; for example, :exc:`RuntimeError` would make sense for a Scrapy version mismatch, while :exc:`ValueError` may be better if the issue is the @@ -84,3 +158,15 @@ If your requirement is a minimum Scrapy version, you may use f"method of spider middlewares as an asynchronous " f"generator." ) + +API reference +============= + +The following function can be used to create an instance of a component class: + +.. autofunction:: scrapy.utils.misc.build_from_crawler + +The following function can also be useful when implementing a component, to +report the import path of the component class, e.g. when reporting problems: + +.. autofunction:: scrapy.utils.python.global_object_name diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 2d61026e9..61aef4bbb 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -20,13 +20,13 @@ following example: This function parses a sample response. Some contracts are mingled with this docstring. - @url http://www.amazon.com/s?field-keywords=selfish+gene + @url http://www.example.com/s?field-keywords=selfish+gene @returns items 1 16 @returns requests 0 0 @scrapes Title Author Year Price """ -This callback is tested using three built-in contracts: +You can use the following contracts: .. module:: scrapy.contracts.default @@ -46,6 +46,14 @@ This callback is tested using three built-in contracts: @cb_kwargs {"arg1": "value1", "arg2": "value2", ...} +.. class:: MetadataContract + + This contract (``@meta``) sets the :attr:`meta ` + attribute for the sample request. It must be a valid JSON dictionary. + :: + + @meta {"arg1": "value1", "arg2": "value2", ...} + .. class:: ReturnsContract This contract (``@returns``) sets lower and upper bounds for the items and diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index a65bab3ca..ba68f0dbc 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -4,10 +4,9 @@ Coroutines ========== -.. versionadded:: 2.0 +Scrapy :ref:`supports ` the :ref:`coroutine syntax ` +(i.e. ``async def``). -Scrapy has :ref:`partial support ` for the -:ref:`coroutine syntax `. .. _coroutine-support: @@ -17,15 +16,13 @@ Supported callables The following callables may be defined as coroutines using ``async def``, and hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): +- The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be + defined as an :term:`asynchronous generator`. + + .. versionadded:: 2.13 + - :class:`~scrapy.Request` callbacks. - If you are using any custom or third-party :ref:`spider middleware - `, see :ref:`sync-async-spider-middleware`. - - .. versionchanged:: 2.7 - Output of async callbacks is now processed asynchronously instead of - collecting all of it first. - - The :meth:`process_item` method of :ref:`item pipelines `. @@ -37,19 +34,102 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): methods of :ref:`downloader middlewares `. -- :ref:`Signal handlers that support deferreds `. - - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` - method of :ref:`spider middlewares `. + method of :ref:`spider middlewares `, which + *must* be defined as an :term:`asynchronous generator` except in + :ref:`universal spider middlewares `. - It must be defined as an :term:`asynchronous generator`. The input - ``result`` parameter is an :term:`asynchronous iterable`. +- The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method + of :ref:`spider middlewares `, which *must* be + defined as an :term:`asynchronous generator`. - See also :ref:`sync-async-spider-middleware` and - :ref:`universal-spider-middleware`. + .. versionadded:: 2.13 + +- :ref:`Signal handlers that support deferreds `. + +- Methods of :ref:`download handlers `. + + .. versionadded:: 2.14 + + +.. _coroutine-deferred-apis: + +Using Deferred-based APIs +========================= + +In addition to native coroutine APIs Scrapy has some APIs that return a +:class:`~twisted.internet.defer.Deferred` object or take a user-supplied +function that returns a :class:`~twisted.internet.defer.Deferred` object. These +APIs are also asynchronous but don't yet support native ``async def`` syntax. +In the future we plan to add support for the ``async def`` syntax to these APIs +or replace them with other APIs where changing the existing ones isn't +possible. + +These APIs have a coroutine-based implementation and a Deferred-based one: + +- :class:`scrapy.crawler.Crawler`: + + - :meth:`~scrapy.crawler.Crawler.crawl_async` (coroutine-based) and + :meth:`~scrapy.crawler.Crawler.crawl` (Deferred-based): the former + may be inconvenient to use in Deferred-based code so both are available, + this may change in a future Scrapy version. + +- :class:`scrapy.crawler.AsyncCrawlerRunner` and its subclass + :class:`scrapy.crawler.AsyncCrawlerProcess` (coroutine-based) and + :class:`scrapy.crawler.CrawlerRunner` and its subclass + :class:`scrapy.crawler.CrawlerProcess` (Deferred-based): the former + doesn't support non-default reactors and so the latter should be used + with those. + +The following user-supplied methods can return +:class:`~twisted.internet.defer.Deferred` objects (the methods that can also +return coroutines are listed in :ref:`coroutine-support`): + +- Custom downloader implementations (see :setting:`DOWNLOADER`): + + - ``fetch()`` + +- Custom scheduler implementations (see :setting:`SCHEDULER`): + + - :meth:`~scrapy.core.scheduler.BaseScheduler.open` + + - :meth:`~scrapy.core.scheduler.BaseScheduler.close` + +- Custom dupefilters (see :setting:`DUPEFILTER_CLASS`): + + - ``open()`` + + - ``close()`` + +- Custom feed storages (see :setting:`FEED_STORAGES`): + + - ``store()`` + +- Subclasses of :class:`scrapy.pipelines.media.MediaPipeline`: + + - ``media_to_download()`` + + - ``item_completed()`` + +- Custom storages used by subclasses of + :class:`scrapy.pipelines.files.FilesPipeline`: + + - ``persist_file()`` + + - ``stat_file()`` + +In most cases you can use these APIs in code that otherwise uses coroutines, by +wrapping a :class:`~twisted.internet.defer.Deferred` object into a +:class:`~asyncio.Future` object or vice versa. See :ref:`asyncio-await-dfd` for +more information about this. + +For example: a custom scheduler needs to define an ``open()`` method that can +return a :class:`~twisted.internet.defer.Deferred` object. You can write a +method that works with Deferreds and returns one directly, or you can write a +coroutine and convert it into a function that returns a Deferred with +:func:`~scrapy.utils.defer.deferred_f_from_coro_f`. - .. versionadded:: 2.7 General usage ============= @@ -71,7 +151,7 @@ shorter and cleaner: adapter["field"] = data return item - def process_item(self, item, spider): + def process_item(self, item): adapter = ItemAdapter(item) dfd = db.get_some_data(adapter["id"]) dfd.addCallback(self._update_item, item) @@ -85,7 +165,7 @@ becomes: class DbPipeline: - async def process_item(self, item, spider): + async def process_item(self, item): adapter = ItemAdapter(item) adapter["field"] = await db.get_some_data(adapter["id"]) return item @@ -123,8 +203,9 @@ This means you can use many useful Python libraries providing such code: Common use cases for asynchronous code include: -* requesting data from websites, databases and other services (in callbacks, - pipelines and middlewares); +* requesting data from websites, databases and other services (in + :meth:`~scrapy.spiders.Spider.start`, callbacks, pipelines and + middlewares); * storing data in databases (in pipelines and middlewares); * delaying the spider initialization until some external event (in the :signal:`spider_opened` handler); @@ -145,7 +226,6 @@ within a spider callback: .. code-block:: python from scrapy import Spider, Request - from scrapy.utils.defer import maybe_deferred_to_future class SingleRequestSpider(Spider): @@ -154,8 +234,9 @@ within a spider callback: async def parse(self, response, **kwargs): additional_request = Request("https://example.org/price") - deferred = self.crawler.engine.download(additional_request) - additional_response = await maybe_deferred_to_future(deferred) + additional_response = await self.crawler.engine.download_async( + additional_request + ) yield { "h1": response.css("h1").get(), "price": additional_response.css("#price").get(), @@ -165,9 +246,9 @@ You can also send multiple requests in parallel: .. code-block:: python + import asyncio + from scrapy import Spider, Request - from scrapy.utils.defer import maybe_deferred_to_future - from twisted.internet.defer import DeferredList class MultipleRequestsSpider(Spider): @@ -179,108 +260,13 @@ You can also send multiple requests in parallel: Request("https://example.com/price"), Request("https://example.com/color"), ] - deferreds = [] + tasks = [] for r in additional_requests: - deferred = self.crawler.engine.download(r) - deferreds.append(deferred) - responses = await maybe_deferred_to_future(DeferredList(deferreds)) + task = self.crawler.engine.download_async(r) + tasks.append(task) + responses = await asyncio.gather(*tasks) yield { "h1": response.css("h1::text").get(), "price": responses[0][1].css(".price::text").get(), "price2": responses[1][1].css(".color::text").get(), } - - -.. _sync-async-spider-middleware: - -Mixing synchronous and asynchronous spider middlewares -====================================================== - -.. versionadded:: 2.7 - -The output of a :class:`~scrapy.Request` callback is passed as the ``result`` -parameter to the -:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method -of the first :ref:`spider middleware ` from the -:ref:`list of active spider middlewares `. -Then the output of that ``process_spider_output`` method is passed to the -``process_spider_output`` method of the next spider middleware, and so on for -every active spider middleware. - -Scrapy supports mixing :ref:`coroutine methods ` and synchronous methods -in this chain of calls. - -However, if any of the ``process_spider_output`` methods is defined as a -synchronous method, and the previous ``Request`` callback or -``process_spider_output`` method is a coroutine, there are some drawbacks to -the asynchronous-to-synchronous conversion that Scrapy does so that the -synchronous ``process_spider_output`` method gets a synchronous iterable as its -``result`` parameter: - -- The whole output of the previous ``Request`` callback or - ``process_spider_output`` method is awaited at this point. - -- If an exception raises while awaiting the output of the previous - ``Request`` callback or ``process_spider_output`` method, none of that - output will be processed. - - This contrasts with the regular behavior, where all items yielded before - an exception raises are processed. - -Asynchronous-to-synchronous conversions are supported for backward -compatibility, but they are deprecated and will stop working in a future -version of Scrapy. - -To avoid asynchronous-to-synchronous conversions, when defining ``Request`` -callbacks as coroutine methods or when using spider middlewares whose -``process_spider_output`` method is an :term:`asynchronous generator`, all -active spider middlewares must either have their ``process_spider_output`` -method defined as an asynchronous generator or :ref:`define a -process_spider_output_async method `. - -.. note:: When using third-party spider middlewares that only define a - synchronous ``process_spider_output`` method, consider - :ref:`making them universal ` through - :ref:`subclassing `. - - -.. _universal-spider-middleware: - -Universal spider middlewares -============================ - -.. versionadded:: 2.7 - -To allow writing a spider middleware that supports asynchronous execution of -its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding -:ref:`asynchronous-to-synchronous conversions `) -while maintaining support for older Scrapy versions, you may define -``process_spider_output`` as a synchronous method and define an -:term:`asynchronous generator` version of that method with an alternative name: -``process_spider_output_async``. - -For example: - -.. code-block:: python - - class UniversalSpiderMiddleware: - def process_spider_output(self, response, result, spider): - for r in result: - # ... do something with r - yield r - - async def process_spider_output_async(self, response, result, spider): - async for r in result: - # ... do something with r - yield r - -.. note:: This is an interim measure to allow, for a time, to write code that - works in Scrapy 2.7 and later without requiring - asynchronous-to-synchronous conversions, and works in earlier Scrapy - versions as well. - - In some future version of Scrapy, however, this feature will be - deprecated and, eventually, in a later version of Scrapy, this - feature will be removed, and all spider middlewares will be expected - to define their ``process_spider_output`` method as an asynchronous - generator. diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 49c5b0410..988e37bbd 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -125,25 +125,15 @@ Fortunately, the :command:`shell` is your bread and butter in this case (see See also: :ref:`topics-shell-inspect-response`. + Open in browser =============== Sometimes you just want to see how a certain response looks in a browser, you -can use the ``open_in_browser`` function for that. Here is an example of how -you would use it: +can use the :func:`~scrapy.utils.response.open_in_browser` function for that: -.. code-block:: python +.. autofunction:: scrapy.utils.response.open_in_browser - from scrapy.utils.response import open_in_browser - - - def parse_details(self, response): - if "item name" not in response.body: - open_in_browser(response) - -``open_in_browser`` will open a browser with the response received by Scrapy at -that point, adjusting the `base tag`_ so that images and styles are displayed -properly. Logging ======= @@ -163,8 +153,6 @@ available in all future runs should they be necessary again: For more information, check the :ref:`topics-logging` section. -.. _base tag: https://www.w3schools.com/tags/tag_base.asp - .. _debug-vscode: Visual Studio Code diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index 961d6dc01..f3515b4be 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -54,6 +54,6 @@ just like ``scrapyd-deploy``. .. _scrapyd-client: https://github.com/scrapy/scrapyd-client .. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html .. _shub: https://shub.readthedocs.io/en/latest/ -.. _Zyte: https://zyte.com/ +.. _Zyte: https://www.zyte.com/ .. _Zyte Scrapy Cloud: https://www.zyte.com/scrapy-cloud/ .. _Zyte Scrapy Cloud documentation: https://docs.zyte.com/scrapy-cloud.html diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index a15ee1059..05dffcdda 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -278,9 +278,9 @@ into our ``url``. In more complex websites, it could be difficult to easily reproduce the requests, as we could need to add ``headers`` or ``cookies`` to make it work. -In those cases you can export the requests in `cURL `_ +In those cases you can export the requests in `cURL `_ format, by right-clicking on each of them in the network tool and using the -:meth:`~scrapy.Request.from_curl()` method to generate an equivalent +:meth:`~scrapy.Request.from_curl` method to generate an equivalent request: .. code-block:: python @@ -317,4 +317,3 @@ to identifying the correct request and replicating it in your spider. .. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll .. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10 .. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions - diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst new file mode 100644 index 000000000..888bfaf08 --- /dev/null +++ b/docs/topics/download-handlers.rst @@ -0,0 +1,389 @@ +.. _topics-download-handlers: + +================= +Download handlers +================= + +Download handlers are Scrapy :ref:`components ` used to +download :ref:`requests ` and produce responses from +them. + +Using download handlers +======================= + +The :setting:`DOWNLOAD_HANDLERS_BASE` and :setting:`DOWNLOAD_HANDLERS` settings +tell Scrapy which handler is responsible for a given URL scheme. Their values +are merged into a mapping from scheme names to handler classes. When Scrapy +initializes it creates instances of all configured download handlers (except +for :ref:`lazy ones `) and stores them in a similar +mapping. When Scrapy needs to download a request it extracts the scheme from +its URL, finds the handler for this scheme, passes the request to it and gets a +response from it. If there is no handler for the scheme, the request is not +downloaded and a :exc:`~scrapy.exceptions.NotSupported` exception is raised. + +The :setting:`DOWNLOAD_HANDLERS_BASE` setting contains the default mapping of +handlers. You can use the :setting:`DOWNLOAD_HANDLERS` setting to add handlers +for additional schemes and to replace or disable default ones: + +.. code-block:: python + + DOWNLOAD_HANDLERS = { + # disable support for ftp:// requests + "ftp": None, + # replace the default one for http:// + "http": "my.download_handlers.HttpHandler", + # http:// and https:// are different schemes, + # even though they may use the same handler + "https": "my.download_handlers.HttpHandler", + # support for any custom scheme can be added + "sftp": "my.download_handlers.SftpHandler", + } + +Replacing HTTP(S) download handlers +----------------------------------- + +While Scrapy provides a default handler for ``http`` and ``https`` schemes, +users may want to use a different handler, provided by Scrapy or by some +3rd-party package. There are several considerations to keep in mind related to +this. + +First of all, as ``http`` and ``https`` are separate schemes, they need +separate entries in the :setting:`DOWNLOAD_HANDLERS` setting, even though it's +likely that the same handler class will be used for both schemes. + +Additionally, some of the Scrapy settings, like :setting:`DOWNLOAD_MAXSIZE`, +are honored by the default HTTP(S) handler but not necessarily by alternative +ones. The same may apply to other Scrapy features, e.g. the +:signal:`bytes_received` and :signal:`headers_received` signals. + +.. _lazy-download-handlers: + +Lazy instantiation of download handlers +--------------------------------------- + +A download handler can be marked as "lazy" by setting its ``lazy`` class +attribute to ``True``. Such handlers are only instantiated when they need to +download their first request. This may be useful when the instantiation is slow +or requires dependencies that are not always available, and the handler is not +needed on every spider run. For example, :class:`the built-in S3 handler +<.S3DownloadHandler>` is lazy. + +Writing your own download handler +================================= + +A download handler is a :ref:`component ` that defines +the following API: + +.. class:: SampleDownloadHandler + + .. attribute:: lazy + :type: bool + + If ``False``, the handler will be instantiated when Scrapy is + initialized. + + If ``True``, the handler will only be instantiated when the first + request handled by it needs to be downloaded. + + .. method:: download_request(request: Request) -> Response: + :async: + + Download the given request and return a response. + + .. method:: close() -> None + :async: + + Clean up any resources used by the handler. + +An optional base class for custom handlers is provided: + +.. autoclass:: scrapy.core.downloader.handlers.base.BaseDownloadHandler + :members: + :undoc-members: + :member-order: bysource + +.. _download-handlers-exceptions: + +Exceptions raised by download handlers +====================================== + +.. versionadded:: 2.15.0 + +The built-in download handlers raise Scrapy-specific exceptions instead of +implementation-specific ones, so that code that handles these exceptions can be +written in a generic way. We recommend custom download handlers to also use +these exceptions. + +.. autoexception:: scrapy.exceptions.CannotResolveHostError + +.. autoexception:: scrapy.exceptions.DownloadCancelledError + +.. autoexception:: scrapy.exceptions.DownloadConnectionRefusedError + +.. autoexception:: scrapy.exceptions.DownloadFailedError + +.. autoexception:: scrapy.exceptions.DownloadTimeoutError + +.. autoexception:: scrapy.exceptions.ResponseDataLossError + +.. autoexception:: scrapy.exceptions.UnsupportedURLSchemeError + +.. _download-handlers-ref: + +Built-in HTTP download handlers reference +========================================= + +Scrapy ships several handlers for HTTP and HTTPS requests. While all of them +support basic features, they may differ in support of specific Scrapy features +and settings and HTTP protocol features. See the documentation of specific +handlers and specific settings for more information. Additionally, as the +underlying HTTP client implementations differ between handlers, the behavior of +specific websites may be different when doing the same Scrapy requests but +using different handlers. + +Here is a comparison of some features of the built-in HTTP handlers, see the +individual handler docs for more differences: + +================== ================= ===================== ==================== +Feature H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler +================== ================= ===================== ==================== +Requires asyncio No No Yes +Requires a reactor Yes Yes No +HTTP/1.1 No Yes Yes +HTTP/2 Yes No Yes +TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl`` +HTTP proxies No Yes Yes +SOCKS proxies No No Yes +================== ================= ===================== ==================== + +You can find additional HTTP download handlers in the +scrapy-download-handlers-incubator_ package. This package is made by the Scrapy +developers and contains experimental handlers that may be included in some +later Scrapy version but can already be used. Please refer to the documentation +of this package for more information. + +.. _scrapy-download-handlers-incubator: https://github.com/scrapy-plugins/scrapy-download-handlers-incubator + +.. _twisted-http2-handler: + +H2DownloadHandler +----------------- + +.. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler + +| Supported scheme: ``https``. +| :ref:`Lazy `: yes. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. + +This handler supports ``https://host/path`` URLs and uses the HTTP/2 protocol +for them. + +It's implemented using :mod:`twisted.web.client` and the ``h2`` library. + +For this handler to work you need to install the ``Twisted[http2]`` extra +dependency. + +If you want to use this handler you need to replace the default one for the +``https`` scheme: + +.. code-block:: python + + DOWNLOAD_HANDLERS = { + "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler", + } + +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. warning:: + + This handler is experimental, and not yet recommended for production + environments. Future Scrapy versions may introduce related changes without + a deprecation period or warning. + +=========================== ================================================ +HTTP proxies No (not implemented) +SOCKS proxies No (not supported by the library) +HTTP/2 Yes +``response.certificate`` :class:`twisted.internet.ssl.Certificate` object +Per-request ``bindaddress`` Yes +TLS implementation ``pyOpenSSL``/``cryptography`` +=========================== ================================================ + +Other limitations: + +- No support for HTTP/1.1. + +- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER` + to ``scrapy.resolver.CachingHostnameResolver``. + +- No support for the :signal:`bytes_received` and :signal:`headers_received` + signals. + +Known limitations of the HTTP/2 support: + +- No support for HTTP/2 Cleartext (h2c), since no major browser supports + HTTP/2 unencrypted (refer `http2 faq`_). + +- No setting to specify a maximum `frame size`_ larger than the default + value, 16384. Connections to servers that send a larger frame will fail. + +- No support for `server pushes`_, which are ignored. + +.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2 +.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption +.. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2 + +HTTP11DownloadHandler +--------------------- + +.. autoclass:: scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler + +| Supported schemes: ``http``, ``https``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. + +This handler supports ``http://host/path`` and ``https://host/path`` URLs and +uses the HTTP/1.1 protocol for them. + +It's implemented using :mod:`twisted.web.client`. + +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + +=========================== ================================================ +HTTP proxies Yes +SOCKS proxies No (not supported by the library) +HTTP/2 No (implemented as a separate handler) +``response.certificate`` :class:`twisted.internet.ssl.Certificate` object +Per-request ``bindaddress`` Yes +TLS implementation ``pyOpenSSL``/``cryptography`` +=========================== ================================================ + +Other limitations: + +- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER` + to ``scrapy.resolver.CachingHostnameResolver``. + +- HTTPS proxies to HTTPS destinations are not supported. + +HttpxDownloadHandler +-------------------- + +.. versionadded:: 2.15.0 + +.. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler + +| Supported schemes: ``http``, ``https``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: yes. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports ``http://host/path`` and ``https://host/path`` URLs and +uses the HTTP/1.1 or HTTP/2 protocol for them. + +It's implemented using the ``httpx`` library and needs it to be installed. + +If you want to use this handler you need to replace the default ones for the +``http`` and ``https`` schemes: + +.. code-block:: python + + DOWNLOAD_HANDLERS = { + "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + } + +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. warning:: + + This handler is experimental, and not yet recommended for production + environments. Future Scrapy versions may introduce related changes without + a deprecation period or warning or even remove it altogether. + +=========================== ======================================= +HTTP proxies Yes +SOCKS proxies Yes (SOCKS5; requires ``httpx[socks]``) +HTTP/2 Yes (requires ``httpx[http2]``) +``response.certificate`` DER bytes +Per-request ``bindaddress`` No (not supported by the library) +TLS implementation Standard library ``ssl`` +=========================== ======================================= + +Other limitations: + +- The handler creates a separate connection pool for each proxy URL (due to + limitations of ``httpx``) which may lead to higher resource usage when + using proxy rotation. + +.. setting:: HTTPX_HTTP2_ENABLED + +HTTPX_HTTP2_ENABLED +^^^^^^^^^^^^^^^^^^^ + +Default: ``False`` + +Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra +needs to be installed if you want to enable this setting. + +.. versionadded:: VERSION + +Built-in non-HTTP download handlers reference +============================================= + +DataURIDownloadHandler +---------------------- + +.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler + +| Supported scheme: ``data``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports RFC 2397 ``data:content/type;base64,`` data URIs. + +FileDownloadHandler +------------------- + +.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler + +| Supported scheme: ``file``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports ``file:///path`` local file URIs. It doesn't +support remote files. + +FTPDownloadHandler +------------------ + +.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler + +| Supported scheme: ``ftp``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. + +This handler supports ``ftp://host/path`` FTP URIs. + +It's implemented using :mod:`twisted.protocols.ftp`. + +S3DownloadHandler +----------------- + +.. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler + +| Supported scheme: ``s3``. +| :ref:`Lazy `: yes. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports ``s3://bucket/path`` S3 URIs. + +It's implemented using the ``botocore`` library and needs it to be installed. diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1abbc4968..8cb29deff 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -61,26 +61,23 @@ particular setting. See each middleware documentation for more info. Writing your own downloader middleware ====================================== -Each downloader middleware is a Python class that defines one or more of the -methods defined below. - -The main entry point is the ``from_crawler`` class method, which receives a -:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` -object gives you access, for example, to the :ref:`settings `. +Each downloader middleware is a :ref:`component ` that +defines one or more of these methods: .. module:: scrapy.downloadermiddlewares .. class:: DownloaderMiddleware - .. note:: Any of the downloader middleware methods may also return a deferred. + .. note:: Any of the downloader middleware methods may be defined as a + coroutine function (``async def``). - .. method:: process_request(request, spider) + .. method:: process_request(request) This method is called for each request that goes through the download middleware. :meth:`process_request` should either: return ``None``, return a - :class:`~scrapy.Response` object, return a :class:`~scrapy.http.Request` + :class:`~scrapy.http.Response` object, return a :class:`~scrapy.Request` object, or raise :exc:`~scrapy.exceptions.IgnoreRequest`. If it returns ``None``, Scrapy will continue processing this request, executing all @@ -106,10 +103,7 @@ object gives you access, for example, to the :ref:`settings `. :param request: the request being processed :type request: :class:`~scrapy.Request` object - :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.Spider` object - - .. method:: process_response(request, response, spider) + .. method:: process_response(request, response) :meth:`process_response` should either: return a :class:`~scrapy.http.Response` object, return a :class:`~scrapy.Request` object or @@ -133,14 +127,12 @@ object gives you access, for example, to the :ref:`settings `. :param response: the response being processed :type response: :class:`~scrapy.http.Response` object - :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.Spider` object + .. method:: process_exception(request, exception) - .. method:: process_exception(request, exception, spider) - - Scrapy calls :meth:`process_exception` when a download handler - or a :meth:`process_request` (from a downloader middleware) raises an - exception (including an :exc:`~scrapy.exceptions.IgnoreRequest` exception) + Scrapy calls :meth:`process_exception` when a :ref:`download handler + ` or a :meth:`process_request` (from a + downloader middleware) raises an exception (including an + :exc:`~scrapy.exceptions.IgnoreRequest` exception). :meth:`process_exception` should return: either ``None``, a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.Request` object. @@ -164,20 +156,6 @@ object gives you access, for example, to the :ref:`settings `. :param exception: the raised exception :type exception: an ``Exception`` object - :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.Spider` object - - .. method:: from_crawler(cls, crawler) - - If present, this classmethod is called to create a middleware instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the middleware. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for middleware to - access them and hook its functionality into Scrapy. - - :param crawler: crawler that uses this middleware - :type crawler: :class:`~scrapy.crawler.Crawler` object - .. _topics-downloader-middleware-ref: Built-in downloader middleware reference @@ -313,13 +291,12 @@ DownloadTimeoutMiddleware .. class:: DownloadTimeoutMiddleware This middleware sets the download timeout for requests specified in the - :setting:`DOWNLOAD_TIMEOUT` setting or :attr:`download_timeout` - spider attribute. + :setting:`DOWNLOAD_TIMEOUT` setting. .. note:: - You can also set download timeout per-request using - :reqmeta:`download_timeout` Request.meta key; this is supported + You can also set download timeout per-request using the + :reqmeta:`download_timeout` :attr:`.Request.meta` key; this is supported even when DownloadTimeoutMiddleware is disabled. HttpAuthMiddleware @@ -749,7 +726,7 @@ HttpProxyMiddleware .. class:: HttpProxyMiddleware This middleware sets the HTTP proxy to use for requests, by setting the - ``proxy`` meta value for :class:`~scrapy.Request` objects. + :reqmeta:`proxy` meta value for :class:`~scrapy.Request` objects. Like the Python standard library module :mod:`urllib.request`, it obeys the following environment variables: @@ -758,11 +735,96 @@ HttpProxyMiddleware * ``https_proxy`` * ``no_proxy`` - You can also set the meta key ``proxy`` per-request, to a value like + You can also set the meta key :reqmeta:`proxy` per-request, to a value like ``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``. Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` environment variables, and it will also ignore ``no_proxy`` environment variable. +.. note:: + + Handling of this meta key needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. It's currently unsupported by + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`. + +.. note:: + + Usually a proxy URL uses the ``http://`` scheme. More rarely, it uses the + ``https://`` one. While both kinds of proxy URLs can be used with both HTTP + and HTTPS destination URLs, the specifics of the network exchange are + different for all 4 cases and it's possible that HTTPS proxies are fully or + partially unsupported by a given download handler. Currently, + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + supports HTTPS proxies only for HTTP destinations. + +.. note:: + + If the download handler supports it, you can use a SOCKS proxy URL (e.g. + ``socks5://username:password@some_proxy_server:port``). + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` + supports SOCKS proxies while other built-in handlers don't. + +HttpProxyMiddleware settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. setting:: HTTPPROXY_ENABLED +.. setting:: HTTPPROXY_AUTH_ENCODING + +HTTPPROXY_ENABLED +^^^^^^^^^^^^^^^^^ + +Default: ``True`` + +Whether or not to enable the :class:`HttpProxyMiddleware`. + +HTTPPROXY_AUTH_ENCODING +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``"latin-1"`` + +The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. + +OffsiteMiddleware +----------------- + +.. module:: scrapy.downloadermiddlewares.offsite + :synopsis: Offsite Middleware + +.. class:: OffsiteMiddleware + + .. versionadded:: 2.11.2 + + Filters out Requests for URLs outside the domains covered by the spider. + + This middleware filters out every request whose host names aren't in the + spider's :attr:`~scrapy.Spider.allowed_domains` attribute. + All subdomains of any domain in the list are also allowed. + E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` + but not ``www2.example.com`` nor ``example.com``. + + When your spider returns a request for a domain not belonging to those + covered by the spider, this middleware will log a debug message similar to + this one:: + + DEBUG: Filtered offsite request to 'offsite.example': + + To avoid filling the log with too much noise, it will only print one of + these messages for each new domain filtered. So, for example, if another + request for ``offsite.example`` is filtered, no log message will be + printed. But if a request for ``other.example`` is filtered, a message + will be printed (but only for the first request filtered). + + If the spider doesn't define an + :attr:`~scrapy.Spider.allowed_domains` attribute, or the + attribute is empty, the offsite middleware will allow all requests. + + .. reqmeta:: allow_offsite + + If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to + ``True`` or :attr:`Request.meta` has ``allow_offsite`` set to ``True``, then + the OffsiteMiddleware will allow the request even if its domain is not listed + in allowed domains. + RedirectMiddleware ------------------ @@ -838,7 +900,7 @@ REDIRECT_MAX_TIMES Default: ``20`` The maximum number of redirections that will be followed for a single request. -After this maximum, the request's response is returned as is. +If maximum redirections are exceeded, the request is aborted and ignored. MetaRefreshMiddleware --------------------- @@ -880,9 +942,9 @@ Default: ``[]`` Meta tags within these tags are ignored. -.. versionchanged:: 2.0 +.. versionchanged:: 2.11.2 The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from - ``['script', 'noscript']`` to ``[]``. + ``[]`` to ``["noscript"]``. .. setting:: METAREFRESH_MAXDELAY @@ -975,15 +1037,14 @@ RETRY_EXCEPTIONS Default:: [ - 'twisted.internet.defer.TimeoutError', - 'twisted.internet.error.TimeoutError', - 'twisted.internet.error.DNSLookupError', - 'twisted.internet.error.ConnectionRefusedError', + 'scrapy.exceptions.CannotResolveHostError', + 'scrapy.exceptions.DownloadConnectionRefusedError', + 'scrapy.exceptions.DownloadFailedError', + 'scrapy.exceptions.DownloadTimeoutError', + 'scrapy.exceptions.ResponseDataLossError', 'twisted.internet.error.ConnectionDone', 'twisted.internet.error.ConnectError', 'twisted.internet.error.ConnectionLost', - 'twisted.internet.error.TCPTimedOutError', - 'twisted.web.client.ResponseFailed', IOError, 'scrapy.core.downloader.handlers.http11.TunnelError', ] @@ -998,6 +1059,21 @@ has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught exception propagation, see :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`. +.. setting:: RETRY_GIVE_UP_LOG_LEVEL + +RETRY_GIVE_UP_LOG_LEVEL +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``"ERROR"`` + +:ref:`Logging level ` used for the message logged when a request +exceeds its retries. + +Can be a level name (e.g. ``"WARNING"``) or a number (e.g. ``logging.WARNING`` +or ``30``). + +See also: :reqmeta:`give_up_log_level`, :func:`get_retry_request`. + .. setting:: RETRY_PRIORITY_ADJUST RETRY_PRIORITY_ADJUST @@ -1040,7 +1116,6 @@ RobotsTxtMiddleware * :ref:`Protego ` (default) * :ref:`RobotFileParser ` * :ref:`Robotexclusionrulesparser ` - * :ref:`Reppy ` (deprecated) You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` setting. Or you can also :ref:`implement support for a new parser `. @@ -1060,7 +1135,7 @@ Parsers vary in several aspects: * Support for wildcard matching -* Usage of `length based rule `_: +* Usage of `length based rule `_: in particular for ``Allow`` and ``Disallow`` directives, where the most specific rule based on the length of the path trumps the less specific (shorter) rule @@ -1078,7 +1153,7 @@ Based on `Protego `_: * implemented in Python * is compliant with `Google's Robots.txt Specification - `_ + `_ * supports wildcard matching @@ -1098,9 +1173,9 @@ Based on :class:`~urllib.robotparser.RobotFileParser`: * is compliant with `Martijn Koster's 1996 draft specification `_ -* lacks support for wildcard matching +* lacks support for wildcard matching (before Python 3.14.5) -* doesn't use the length based rule +* doesn't use the length based rule (before Python 3.14.5) It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0. @@ -1108,43 +1183,12 @@ In order to use this parser, set: * :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser`` -.. _reppy-parser: - -Reppy parser -~~~~~~~~~~~~ - -Based on `Reppy `_: - -* is a Python wrapper around `Robots Exclusion Protocol Parser for C++ - `_ - -* is compliant with `Martijn Koster's 1996 draft specification - `_ - -* supports wildcard matching - -* uses the length based rule - -Native implementation, provides better speed than Protego. - -In order to use this parser: - -* Install `Reppy `_ by running ``pip install reppy`` - - .. warning:: `Upstream issue #122 - `_ prevents reppy usage in Python 3.9+. - Because of this the Reppy parser is deprecated. - -* Set :setting:`ROBOTSTXT_PARSER` setting to - ``scrapy.robotstxt.ReppyRobotParser`` - - .. _rerp-parser: Robotexclusionrulesparser ~~~~~~~~~~~~~~~~~~~~~~~~~ -Based on `Robotexclusionrulesparser `_: +Based on `Robotexclusionrulesparser `_: * implemented in Python @@ -1157,7 +1201,7 @@ Based on `Robotexclusionrulesparser `_: In order to use this parser: -* Install `Robotexclusionrulesparser `_ by running +* Install ``Robotexclusionrulesparser`` by running ``pip install robotexclusionrulesparser`` * Set :setting:`ROBOTSTXT_PARSER` setting to @@ -1202,64 +1246,8 @@ UserAgentMiddleware .. class:: UserAgentMiddleware - Middleware that allows spiders to override the default user agent. - - In order for a spider to override the default user agent, its ``user_agent`` - attribute must be set. - -.. _ajaxcrawl-middleware: - -AjaxCrawlMiddleware -------------------- - -.. module:: scrapy.downloadermiddlewares.ajaxcrawl - -.. class:: AjaxCrawlMiddleware - - Middleware that finds 'AJAX crawlable' page variants based - on meta-fragment html tag. See - https://developers.google.com/search/docs/ajax-crawling/docs/getting-started - for more info. - - .. note:: - - Scrapy finds 'AJAX crawlable' pages for URLs like - ``'http://example.com/!#foo=bar'`` even without this middleware. - AjaxCrawlMiddleware is necessary when URL doesn't contain ``'!#'``. - This is often a case for 'index' or 'main' website pages. - -AjaxCrawlMiddleware Settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. setting:: AJAXCRAWL_ENABLED - -AJAXCRAWL_ENABLED -^^^^^^^^^^^^^^^^^ - -Default: ``False`` - -Whether the AjaxCrawlMiddleware will be enabled. You may want to -enable it for :ref:`broad crawls `. - -HttpProxyMiddleware settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. setting:: HTTPPROXY_ENABLED -.. setting:: HTTPPROXY_AUTH_ENCODING - -HTTPPROXY_ENABLED -^^^^^^^^^^^^^^^^^ - -Default: ``True`` - -Whether or not to enable the :class:`HttpProxyMiddleware`. - -HTTPPROXY_AUTH_ENCODING -^^^^^^^^^^^^^^^^^^^^^^^ - -Default: ``"latin-1"`` - -The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. + Middleware that sets the ``User-Agent`` header. + The header value is taken from the :setting:`USER_AGENT` setting. .. _DBM: https://en.wikipedia.org/wiki/Dbm diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index a0f4b4411..5a399e094 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -14,7 +14,7 @@ from it. If you fail to do that, and you can nonetheless access the desired data through the :ref:`DOM ` from your web browser, see -:ref:`topics-javascript-rendering`. +:ref:`topics-headless-browsing`. .. _topics-finding-data-source: @@ -83,11 +83,10 @@ request with Scrapy. It might be enough to yield a :class:`~scrapy.Request` with the same HTTP method and URL. However, you may also need to reproduce the body, headers and -form parameters (see :class:`~scrapy.FormRequest`) of that request. +form parameters (see :ref:`form`) of that request. -As all major browsers allow to export the requests in `cURL -`_ format, Scrapy incorporates the method -:meth:`~scrapy.Request.from_curl()` to generate an equivalent +As all major browsers allow to export the requests in curl_ format, Scrapy +incorporates the method :meth:`~scrapy.Request.from_curl` to generate an equivalent :class:`~scrapy.Request` from a cURL command. To get more information visit :ref:`request from curl ` inside the network tool section. @@ -98,7 +97,7 @@ it `. You can reproduce any request with Scrapy. However, some times reproducing all necessary requests may not seem efficient in developer time. If that is your case, and crawling speed is not a major concern for you, you can alternatively -consider :ref:`JavaScript pre-rendering `. +consider :ref:`using a headless browser `. If you get the expected response `sometimes`, but not always, the issue is probably not your request, but the target server. The target server might be @@ -112,18 +111,20 @@ you may use `curl2scrapy `_. Handling different response formats =================================== +.. skip: start + Once you have a response with the desired data, how you extract the desired data from it depends on the type of response: -- If the response is HTML or XML, use :ref:`selectors +- If the response is HTML, XML or JSON, use :ref:`selectors ` as usual. -- If the response is JSON, use :func:`json.loads` to load the desired data from - :attr:`response.text `: +- If the response is JSON, use :func:`response.json() + ` to load the desired data: .. code-block:: python - data = json.loads(response.text) + data = response.json() If the desired data is inside HTML or XML code embedded within JSON data, you can load that HTML or XML code into a @@ -145,7 +146,7 @@ data from it depends on the type of response: - If the response is an image or another format based on images (e.g. PDF), read the response as bytes from - :attr:`response.body ` and use an OCR + :attr:`response.body ` and use an OCR solution to extract the desired data as text. For example, you can use pytesseract_. To read a table from a PDF, @@ -158,11 +159,15 @@ data from it depends on the type of response: Otherwise, you might need to convert the SVG code into a raster image, and :ref:`handle that raster image `. +.. skip: end + .. _topics-parsing-javascript: Parsing JavaScript code ======================= +.. skip: start + If the desired data is hardcoded in JavaScript, you first need to get the JavaScript code: @@ -221,9 +226,11 @@ data from it: >>> selector.css('var[name="data"]').get() 'value' -.. _topics-javascript-rendering: +.. skip: end -Pre-rendering JavaScript +.. _topics-headless-browsing: + +Using a headless browser ======================== On webpages that fetch data from additional requests, reproducing those @@ -233,35 +240,17 @@ network transfer. However, sometimes it can be really hard to reproduce certain requests. Or you may need something that no request can give you, such as a screenshot of a -webpage as seen in a web browser. +webpage as seen in a web browser. In this case using a `headless browser`_ will +help. -In these cases use the Splash_ JavaScript-rendering service, along with -`scrapy-splash`_ for seamless integration. - -Splash returns as HTML the :ref:`DOM ` of a webpage, so that -you can parse it with :ref:`selectors `. It provides great -flexibility through configuration_ or scripting_. - -If you need something beyond what Splash offers, such as interacting with the -DOM on-the-fly from Python code instead of using a previously-written script, -or handling multiple web browser windows, you might need to -:ref:`use a headless browser ` instead. - -.. _configuration: https://splash.readthedocs.io/en/stable/api.html -.. _scripting: https://splash.readthedocs.io/en/stable/scripting-tutorial.html - -.. _topics-headless-browsing: - -Using a headless browser -======================== - -A `headless browser`_ is a special web browser that provides an API for +A headless browser is a special web browser that provides an API for automation. By installing the :ref:`asyncio reactor `, it is possible to integrate ``asyncio``-based libraries which handle headless browsers. One such library is `playwright-python`_ (an official Python port of `playwright`_). The following is a simple snippet to illustrate its usage within a Scrapy spider: +.. skip: next .. code-block:: python import scrapy @@ -285,20 +274,15 @@ However, using `playwright-python`_ directly as in the above example circumvents most of the Scrapy components (middlewares, dupefilter, etc). We recommend using `scrapy-playwright`_ for a better integration. -.. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29 .. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets -.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript -.. _Splash: https://github.com/scrapinghub/splash .. _chompjs: https://github.com/Nykakin/chompjs -.. _curl: https://curl.haxx.se/ +.. _curl: https://curl.se/ .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser .. _js2xml: https://github.com/scrapinghub/js2xml .. _playwright-python: https://github.com/microsoft/playwright-python .. _playwright: https://github.com/microsoft/playwright -.. _pyppeteer: https://pyppeteer.github.io/pyppeteer/ .. _pytesseract: https://github.com/madmaze/pytesseract .. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright -.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash .. _tabula-py: https://github.com/chezou/tabula-py .. _wget: https://www.gnu.org/software/wget/ .. _wgrep: https://github.com/stav/wgrep diff --git a/docs/topics/email.rst b/docs/topics/email.rst deleted file mode 100644 index d6a7ad354..000000000 --- a/docs/topics/email.rst +++ /dev/null @@ -1,193 +0,0 @@ -.. _topics-email: - -============== -Sending e-mail -============== - -.. module:: scrapy.mail - :synopsis: Email sending facility - -Although Python makes sending e-mails relatively easy via the :mod:`smtplib` -library, Scrapy provides its own facility for sending e-mails which is very -easy to use and it's implemented using :doc:`Twisted non-blocking IO -`, to avoid interfering with the non-blocking -IO of the crawler. It also provides a simple API for sending attachments and -it's very easy to configure, with a few :ref:`settings -`. - -Quick example -============= - -There are two ways to instantiate the mail sender. You can instantiate it using -the standard ``__init__`` method: - -.. code-block:: python - - from scrapy.mail import MailSender - - mailer = MailSender() - -Or you can instantiate it passing a Scrapy settings object, which will respect -the :ref:`settings `: - -.. skip: start -.. code-block:: python - - mailer = MailSender.from_settings(settings) - -And here is how to use it to send an e-mail (without attachments): - -.. code-block:: python - - mailer.send( - to=["someone@example.com"], - subject="Some subject", - body="Some body", - cc=["another@example.com"], - ) -.. skip: end - -MailSender class reference -========================== - -MailSender is the preferred class to use for sending emails from Scrapy, as it -uses :doc:`Twisted non-blocking IO `, like the -rest of the framework. - -.. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None) - - :param smtphost: the SMTP host to use for sending the emails. If omitted, the - :setting:`MAIL_HOST` setting will be used. - :type smtphost: str - - :param mailfrom: the address used to send emails (in the ``From:`` header). - If omitted, the :setting:`MAIL_FROM` setting will be used. - :type mailfrom: str - - :param smtpuser: the SMTP user. If omitted, the :setting:`MAIL_USER` - setting will be used. If not given, no SMTP authentication will be - performed. - :type smtphost: str or bytes - - :param smtppass: the SMTP pass for authentication. - :type smtppass: str or bytes - - :param smtpport: the SMTP port to connect to - :type smtpport: int - - :param smtptls: enforce using SMTP STARTTLS - :type smtptls: bool - - :param smtpssl: enforce using a secure SSL connection - :type smtpssl: bool - - .. classmethod:: from_settings(settings) - - Instantiate using a Scrapy settings object, which will respect - :ref:`these Scrapy settings `. - - :param settings: the e-mail recipients - :type settings: :class:`scrapy.settings.Settings` object - - .. method:: send(to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None) - - Send email to the given recipients. - - :param to: the e-mail recipients as a string or as a list of strings - :type to: str or list - - :param subject: the subject of the e-mail - :type subject: str - - :param cc: the e-mails to CC as a string or as a list of strings - :type cc: str or list - - :param body: the e-mail body - :type body: str - - :param attachs: an iterable of tuples ``(attach_name, mimetype, - file_object)`` where ``attach_name`` is a string with the name that will - appear on the e-mail's attachment, ``mimetype`` is the mimetype of the - attachment and ``file_object`` is a readable file object with the - contents of the attachment - :type attachs: collections.abc.Iterable - - :param mimetype: the MIME type of the e-mail - :type mimetype: str - - :param charset: the character encoding to use for the e-mail contents - :type charset: str - - -.. _topics-email-settings: - -Mail settings -============= - -These settings define the default ``__init__`` method values of the :class:`MailSender` -class, and can be used to configure e-mail notifications in your project without -writing any code (for those extensions and code that uses :class:`MailSender`). - -.. setting:: MAIL_FROM - -MAIL_FROM ---------- - -Default: ``'scrapy@localhost'`` - -Sender email to use (``From:`` header) for sending emails. - -.. setting:: MAIL_HOST - -MAIL_HOST ---------- - -Default: ``'localhost'`` - -SMTP host to use for sending emails. - -.. setting:: MAIL_PORT - -MAIL_PORT ---------- - -Default: ``25`` - -SMTP port to use for sending emails. - -.. setting:: MAIL_USER - -MAIL_USER ---------- - -Default: ``None`` - -User to use for SMTP authentication. If disabled no SMTP authentication will be -performed. - -.. setting:: MAIL_PASS - -MAIL_PASS ---------- - -Default: ``None`` - -Password to use for SMTP authentication, along with :setting:`MAIL_USER`. - -.. setting:: MAIL_TLS - -MAIL_TLS --------- - -Default: ``False`` - -Enforce using STARTTLS. STARTTLS is a way to take an existing insecure connection, and upgrade it to a secure connection using SSL/TLS. - -.. setting:: MAIL_SSL - -MAIL_SSL --------- - -Default: ``False`` - -Enforce connecting using an SSL encrypted connection diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index ea64edbe6..cc690d609 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -83,8 +83,6 @@ This exception is raised to indicate an unsupported feature. StopDownload ------------- -.. versionadded:: 2.2 - .. exception:: StopDownload(fail=True) Raised from a :class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received` @@ -105,7 +103,7 @@ response: In both cases, the response could have its body truncated: the body contains all bytes received up until the exception is raised, including the bytes received in the signal handler that raises the exception. Also, the response -object is marked with ``"download_stopped"`` in its :attr:`Response.flags` +object is marked with ``"download_stopped"`` in its :attr:`~scrapy.http.Response.flags` attribute. .. note:: ``fail`` is a keyword-only parameter, i.e. raising diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 7a85c099b..c4cd05683 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -67,7 +67,7 @@ value of one of their fields: self.year_to_exporter[year] = (exporter, xml_file) return self.year_to_exporter[year][0] - def process_item(self, item, spider): + def process_item(self, item): exporter = self._exporter_for_item(item) exporter.export_item(item) return item @@ -93,33 +93,34 @@ described next. 1. Declaring a serializer in the field -------------------------------------- -If you use :class:`~scrapy.Item` you can declare a serializer in the -:ref:`field metadata `. The serializer must be -a callable which receives a value and returns its serialized form. +Every :ref:`item type ` except :class:`dict` lets you declare a +serializer in the :ref:`field metadata `. The serializer +must be a callable which receives a value and returns its serialized form. Example: .. code-block:: python - import scrapy + from dataclasses import dataclass, field def serialize_price(value): return f"$ {str(value)}" - class Product(scrapy.Item): - name = scrapy.Field() - price = scrapy.Field(serializer=serialize_price) + @dataclass + class Product: + name: str + price: float = field(metadata={"serializer": serialize_price}) 2. Overriding the serialize_field() method ------------------------------------------ -You can also override the :meth:`~BaseItemExporter.serialize_field()` method to +You can also override the :meth:`~BaseItemExporter.serialize_field` method to customize how your field value will be exported. -Make sure you call the base class :meth:`~BaseItemExporter.serialize_field()` method +Make sure you call the base class :meth:`~BaseItemExporter.serialize_field` method after your custom code. Example: @@ -163,9 +164,6 @@ BaseItemExporter populate their respective instance attributes: :attr:`fields_to_export`, :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`. - .. versionadded:: 2.0 - The *dont_fail* parameter. - .. method:: export_item(item) Exports the given item. This method must be implemented in subclasses. @@ -224,7 +222,7 @@ BaseItemExporter .. [1] Not all exporters respect the specified field order. .. [2] When using :ref:`item objects ` that do not expose all their possible fields, exporters that do not support exporting - a different subset of fields per item will only export the fields + a different subset of fields per item will only export the fields found in the first item exported. .. attribute:: export_empty_fields diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index f7b2f3799..735e46c29 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -4,34 +4,21 @@ Extensions ========== -The extensions framework provides a mechanism for inserting your own -custom functionality into Scrapy. +Extensions are :ref:`components ` that allow inserting your +own custom functionality into Scrapy. -Extensions are just regular classes. +Unlike other components, extensions do not have a specific role in Scrapy. They +are “wildcard†components that can be used for anything that does not fit the +role of any other type of component. -Extension settings -================== +Loading and activating extensions +================================= -Extensions use the :ref:`Scrapy settings ` to manage their -settings, just like any other Scrapy code. +Extensions are loaded at startup by creating a single instance of the extension +class per spider being run. -It is customary for extensions to prefix their settings with their own name, to -avoid collision with existing (and future) extensions. For example, a -hypothetical extension to handle `Google Sitemaps`_ would use settings like -``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on. - -.. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps - -Loading & activating extensions -=============================== - -Extensions are loaded and activated at startup by instantiating a single -instance of the extension class per spider being run. All the extension -initialization code must be performed in the class ``__init__`` method. - -To make an extension available, add it to the :setting:`EXTENSIONS` setting in -your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented -by a string: the full Python path to the extension's class name. For example: +To enable an extension, add it to the :setting:`EXTENSIONS` setting. For +example: .. code-block:: python @@ -40,55 +27,24 @@ by a string: the full Python path to the extension's class name. For example: "scrapy.extensions.telnet.TelnetConsole": 500, } - -As you can see, the :setting:`EXTENSIONS` setting is a dict where the keys are -the extension paths, and their values are the orders, which define the -extension *loading* order. The :setting:`EXTENSIONS` setting is merged with the -:setting:`EXTENSIONS_BASE` setting defined in Scrapy (and not meant to be -overridden) and then sorted by order to get the final sorted list of enabled -extensions. +:setting:`EXTENSIONS` is merged with :setting:`EXTENSIONS_BASE` (not meant to +be overridden), and the priorities in the resulting value determine the +*loading* order. As extensions typically do not depend on each other, their loading order is irrelevant in most cases. This is why the :setting:`EXTENSIONS_BASE` setting -defines all extensions with the same order (``0``). However, this feature can -be exploited if you need to add an extension which depends on other extensions -already loaded. - -Available, enabled and disabled extensions -========================================== - -Not all available extensions will be enabled. Some of them usually depend on a -particular setting. For example, the HTTP Cache extension is available by default -but disabled unless the :setting:`HTTPCACHE_ENABLED` setting is set. - -Disabling an extension -====================== - -In order to disable an extension that comes enabled by default (i.e. those -included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to -``None``. For example: - -.. code-block:: python - - EXTENSIONS = { - "scrapy.extensions.corestats.CoreStats": None, - } +defines all extensions with the same order (``0``). However, you may need to +carefully use priorities if you add an extension that depends on other +extensions being already loaded. Writing your own extension ========================== -Each extension is a Python class. The main entry point for a Scrapy extension -(this also includes middlewares and pipelines) is the ``from_crawler`` -class method which receives a ``Crawler`` instance. Through the Crawler object -you can access settings, signals, stats, and also control the crawling behaviour. +Each extension is a :ref:`component `. Typically, extensions connect to :ref:`signals ` and perform tasks triggered by them. -Finally, if the ``from_crawler`` method raises the -:exc:`~scrapy.exceptions.NotConfigured` exception, the extension will be -disabled. Otherwise, the extension will be enabled. - Sample extension ---------------- @@ -180,8 +136,29 @@ Core Stats extension Enable the collection of core statistics, provided the stats collection is enabled (see :ref:`topics-stats`). +The following stats are collected: + +* ``start_time``: start date/time of the crawl (:class:`~datetime.datetime`). +* ``finish_time``: end date/time of the crawl (:class:`~datetime.datetime`). +* ``elapsed_time_seconds``: total crawl duration in seconds (:class:`float`). +* ``finish_reason``: the closing reason string (e.g. ``"finished"``, + ``"closespider_timeout"``). +* ``item_scraped_count``: total number of items that passed all pipelines. +* ``item_dropped_count``: total number of items dropped by a pipeline. +* ``item_dropped_reasons_count/``: per-exception drop count + (e.g. ``item_dropped_reasons_count/DropItem``). +* ``response_received_count``: total number of HTTP responses received. + .. _topics-extensions-ref-telnetconsole: +Log Count extension +~~~~~~~~~~~~~~~~~~~ + +.. module:: scrapy.extensions.logcount + :synopsis: Basic stats logging + +.. autoclass:: LogCount + Telnet console extension ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -211,20 +188,16 @@ Memory usage extension Monitors the memory used by the Scrapy process that runs the spider and: -1. sends a notification e-mail when it exceeds a certain value -2. closes the spider when it exceeds a certain value - -The notification e-mails can be triggered when a certain warning value is -reached (:setting:`MEMUSAGE_WARNING_MB`) and when the maximum value is reached -(:setting:`MEMUSAGE_LIMIT_MB`) which will also cause the spider to be closed -and the Scrapy process to be terminated. +1. sends a :signal:`memusage_warning_reached` signal when it exceeds + :setting:`MEMUSAGE_WARNING_MB` +2. closes the spider with the `"memusage_exceeded"` reason when it exceeds + :setting:`MEMUSAGE_LIMIT_MB` This extension is enabled by the :setting:`MEMUSAGE_ENABLED` setting and can be configured with the following settings: * :setting:`MEMUSAGE_LIMIT_MB` * :setting:`MEMUSAGE_WARNING_MB` -* :setting:`MEMUSAGE_NOTIFY_MAIL` * :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS` Memory debugger extension @@ -243,6 +216,32 @@ An extension for debugging memory usage. It collects information about: To enable this extension, turn on the :setting:`MEMDEBUG_ENABLED` setting. The info will be stored in the stats. +.. _topics-extensions-ref-spiderstate: + +Spider state extension +~~~~~~~~~~~~~~~~~~~~~~ + +.. module:: scrapy.extensions.spiderstate + :synopsis: Spider state extension + +.. class:: SpiderState + +Manages spider state data by loading it before a crawl and saving it after. + +Give a value to the :setting:`JOBDIR` setting to enable this extension. +When enabled, this extension manages the :attr:`~scrapy.Spider.state` +attribute of your :class:`~scrapy.Spider` instance: + +- When your spider closes (:signal:`spider_closed`), the contents of its + :attr:`~scrapy.Spider.state` attribute are serialized into a file named + ``spider.state`` in the :setting:`JOBDIR` folder. +- When your spider opens (:signal:`spider_opened`), if a previously-generated + ``spider.state`` file exists in the :setting:`JOBDIR` folder, it is loaded + into the :attr:`~scrapy.Spider.state` attribute. + + +For an example, see :ref:`topics-keeping-persistent-state-between-batches`. + Close spider extension ~~~~~~~~~~~~~~~~~~~~~~ @@ -261,12 +260,13 @@ settings: * :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM` * :setting:`CLOSESPIDER_ITEMCOUNT` * :setting:`CLOSESPIDER_PAGECOUNT` +* :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM` * :setting:`CLOSESPIDER_ERRORCOUNT` .. note:: - When a certain closing condition is met, requests which are - currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS` + When a certain closing condition is met, requests which are + currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS` requests) are still processed. .. setting:: CLOSESPIDER_TIMEOUT @@ -274,12 +274,11 @@ settings: CLOSESPIDER_TIMEOUT """"""""""""""""""" -Default: ``0`` +Default: ``0.0`` -An integer which specifies a number of seconds. If the spider remains open for -more than that number of second, it will be automatically closed with the -reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by -timeout. +If the spider remains open for more than this number of seconds, it will be +automatically closed with the reason ``closespider_timeout``. If zero (or non +set), spiders won't be closed by timeout. .. setting:: CLOSESPIDER_TIMEOUT_NO_ITEM @@ -317,6 +316,19 @@ crawls more than that, the spider will be closed with the reason ``closespider_pagecount``. If zero (or non set), spiders won't be closed by number of crawled responses. +.. setting:: CLOSESPIDER_PAGECOUNT_NO_ITEM + +CLOSESPIDER_PAGECOUNT_NO_ITEM +""""""""""""""""""""""""""""" + +Default: ``0`` + +An integer which specifies the maximum number of consecutive responses to crawl +without items scraped. If the spider crawls more consecutive responses than that +and no items are scraped in the meantime, the spider will be closed with the +reason ``closespider_pagecount_no_item``. If zero (or not set), spiders won't be +closed by number of crawled responses with no items. + .. setting:: CLOSESPIDER_ERRORCOUNT CLOSESPIDER_ERRORCOUNT @@ -329,24 +341,6 @@ closing the spider. If the spider generates more than that number of errors, it will be closed with the reason ``closespider_errorcount``. If zero (or non set), spiders won't be closed by number of errors. -StatsMailer extension -~~~~~~~~~~~~~~~~~~~~~ - -.. module:: scrapy.extensions.statsmailer - :synopsis: StatsMailer extension - -.. class:: StatsMailer - -This simple extension can be used to send a notification e-mail every time a -domain has finished scraping, including the Scrapy stats collected. The email -will be sent to all recipients specified in the :setting:`STATSMAILER_RCPTS` -setting. - -Emails can be sent using the :class:`~scrapy.mail.MailSender` class. To see a -full list of parameters, including examples on how to instantiate -:class:`~scrapy.mail.MailSender` and use mail settings, see -:ref:`topics-email`. - .. module:: scrapy.extensions.debug :synopsis: Extensions for debugging Scrapy @@ -507,8 +501,4 @@ Invokes a :doc:`Python debugger ` inside a running Scrapy process w signal is received. After the debugger is exited, the Scrapy process continues running normally. -For more info see `Debugging in Python`_. - This extension only works on POSIX-compliant platforms (i.e. not Windows). - -.. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/ diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index f64bbac06..6ce5255b9 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -180,7 +180,7 @@ FTP supports two different connection modes: `active or passive mode by default. To use the active connection mode instead, set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. -The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. .. caution:: The value ``True`` in ``overwrite`` will cause you to lose the @@ -213,7 +213,7 @@ passed through the following settings: - :setting:`AWS_SECRET_ACCESS_KEY` - :setting:`AWS_SESSION_TOKEN` (only needed for `temporary security credentials`_) -.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys +.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html You can also define a custom ACL, custom endpoint, and region name for exported feeds using these settings: @@ -222,7 +222,7 @@ feeds using these settings: - :setting:`AWS_ENDPOINT_URL` - :setting:`AWS_REGION_NAME` -The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. .. caution:: The value ``True`` in ``overwrite`` will cause you to lose the @@ -236,8 +236,6 @@ This storage backend uses :ref:`delayed file delivery `. Google Cloud Storage (GCS) -------------------------- -.. versionadded:: 2.3 - The feeds are stored on `Google Cloud Storage`_. - URI scheme: ``gs`` @@ -248,14 +246,14 @@ The feeds are stored on `Google Cloud Storage`_. - Required external libraries: `google-cloud-storage`_. -For more information about authentication, please refer to `Google Cloud documentation `_. +For more information about authentication, please refer to `Google Cloud documentation `_. You can set a *Project ID* and *Access Control List (ACL)* through the following settings: - :setting:`FEED_STORAGE_GCS_ACL` - :setting:`GCS_PROJECT_ID` -The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. .. caution:: The value ``True`` in ``overwrite`` will cause you to lose the @@ -263,7 +261,7 @@ storage backend is: ``True``. This storage backend uses :ref:`delayed file delivery `. -.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python +.. _google-cloud-storage: https://docs.cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python .. _topics-feed-storage-stdout: @@ -303,8 +301,6 @@ feed URI, allowing item delivery to start way before the end of the crawl. Item filtering ============== -.. versionadded:: 2.6.0 - You can filter items that you want to allow for a particular feed by using the ``item_classes`` option in :ref:`feeds options `. Only items of the specified types will be added to the feed. @@ -344,8 +340,6 @@ ItemFilter Post-Processing =============== -.. versionadded:: 2.6.0 - Scrapy provides an option to activate plugins to post-process feeds before they are exported to feed storages. In addition to using :ref:`builtin plugins `, you can create your own :ref:`plugins `. @@ -390,7 +384,13 @@ Each plugin is a class that must implement the following methods: .. method:: close(self) - Close the target file object. + Clean up the plugin. + + For example, you might want to close a file wrapper that you might have + used to compress data written into the file received in the ``__init__`` + method. + + .. warning:: Do not close the file from the ``__init__`` method. To pass a parameter to your plugin, use :ref:`feed options `. You can then access those parameters from the ``__init__`` method of your plugin. @@ -419,8 +419,6 @@ These are the settings used for configuring the feed exports: FEEDS ----- -.. versionadded:: 2.1 - Default: ``{}`` A dictionary in which every key is a feed URI (or a :class:`pathlib.Path` @@ -473,8 +471,6 @@ as a fallback value if that key is not provided for a specific feed definition: - ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. - .. versionadded:: 2.3.0 - - ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`. - ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`. @@ -483,20 +479,14 @@ as a fallback value if that key is not provided for a specific feed definition: If undefined or empty, all items are exported. - .. versionadded:: 2.6.0 - - ``item_filter``: a :ref:`filter class ` to filter items to export. :class:`~scrapy.extensions.feedexport.ItemFilter` is used be default. - .. versionadded:: 2.6.0 - - ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. - ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class `. - .. versionadded:: 2.4.0 - - ``overwrite``: whether to overwrite the file if it already exists (``True``) or append to its content (``False``). @@ -510,15 +500,12 @@ as a fallback value if that key is not provided for a specific feed definition: .. note:: Some FTP servers may not support appending to files (the ``APPE`` FTP command). - - :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported - `_) + - :ref:`topics-feed-storage-s3`: ``True`` (appending is not supported) - :ref:`topics-feed-storage-gcs`: ``True`` (appending is not supported) - :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported) - .. versionadded:: 2.4.0 - - ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`. - ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`. @@ -527,25 +514,23 @@ as a fallback value if that key is not provided for a specific feed definition: The plugins will be used in the order of the list passed. - .. versionadded:: 2.6.0 - .. setting:: FEED_EXPORT_ENCODING FEED_EXPORT_ENCODING -------------------- -Default: ``None`` +Default: ``"utf-8"`` (:ref:`fallback `: ``None``) The encoding to be used for the feed. -If unset or set to ``None`` (default) it uses UTF-8 for everything except JSON output, -which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons. +If set to ``None``, it uses UTF-8 for everything except JSON output, which uses +safe numeric encoding (``\uXXXX`` sequences) for historic reasons. -Use ``utf-8`` if you want UTF-8 for JSON too. +Use ``"utf-8"`` if you want UTF-8 for JSON too. .. versionchanged:: 2.8 The :command:`startproject` command now sets this setting to - ``utf-8`` in the generated ``settings.py`` file. + ``"utf-8"`` in the generated ``settings.py`` file. .. setting:: FEED_EXPORT_FIELDS @@ -582,8 +567,8 @@ FEED_STORE_EMPTY Default: ``True`` Whether to export empty feeds (i.e. feeds with no items). -If ``False``, and there are no items to export, no new files are created and -existing files are not modified, even if the :ref:`overwrite feed option +If ``False``, and there are no items to export, no new files are created and +existing files are not modified, even if the :ref:`overwrite feed option ` is enabled. .. setting:: FEED_STORAGES @@ -695,8 +680,6 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter FEED_EXPORT_BATCH_ITEM_COUNT ---------------------------- -.. versionadded:: 2.3.0 - Default: ``0`` If assigned an integer number higher than ``0``, Scrapy generates multiple output files @@ -766,15 +749,11 @@ The function signature should be as follows: If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id`` is always ``1``. - .. versionadded:: 2.3.0 - - ``batch_time``: UTC date and time, in ISO format with ``:`` replaced with ``-``. See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. - .. versionadded:: 2.3.0 - - ``time``: ``batch_time``, with microseconds set to ``0``. :type params: dict @@ -810,5 +789,5 @@ source spider in the feed URI: .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier .. _Amazon S3: https://aws.amazon.com/s3/ .. _boto3: https://github.com/boto/boto3 -.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl .. _Google Cloud Storage: https://cloud.google.com/storage/ diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index a5f6e07b8..9a53f88fb 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -23,53 +23,35 @@ Typical uses of item pipelines are: Writing your own item pipeline ============================== -Each item pipeline component is a Python class that must implement the following method: +Each item pipeline is a :ref:`component ` that must +implement the following method: -.. method:: process_item(self, item, spider) +.. method:: process_item(self, item) This method is called for every item pipeline component. `item` is an :ref:`item object `, see :ref:`supporting-item-types`. - :meth:`process_item` must either: return an :ref:`item object `, - return a :class:`~twisted.internet.defer.Deferred` or raise a - :exc:`~scrapy.exceptions.DropItem` exception. + :meth:`process_item` must either return an :ref:`item object ` + or raise a :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer processed by further pipeline components. :param item: the scraped item :type item: :ref:`item object ` - :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.Spider` object - Additionally, they may also implement the following methods: -.. method:: open_spider(self, spider) +.. method:: open_spider(self) This method is called when the spider is opened. - :param spider: the spider which was opened - :type spider: :class:`~scrapy.Spider` object - -.. method:: close_spider(self, spider) +.. method:: close_spider(self) This method is called when the spider is closed. - :param spider: the spider which was closed - :type spider: :class:`~scrapy.Spider` object - -.. classmethod:: from_crawler(cls, crawler) - - If present, this class method is called to create a pipeline instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the pipeline. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for pipeline to - access them and hook its functionality into Scrapy. - - :param crawler: crawler that uses this pipeline - :type crawler: :class:`~scrapy.crawler.Crawler` object +Any of these methods may be defined as a coroutine function (``async def``). Item pipeline example @@ -92,14 +74,14 @@ contain a price: class PricePipeline: vat_factor = 1.15 - def process_item(self, item, spider): + def process_item(self, item): adapter = ItemAdapter(item) if adapter.get("price"): if adapter.get("price_excludes_vat"): adapter["price"] = adapter["price"] * self.vat_factor return item else: - raise DropItem(f"Missing price in {item}") + raise DropItem("Missing price") Write items to a JSON lines file @@ -117,13 +99,13 @@ format: class JsonWriterPipeline: - def open_spider(self, spider): + def open_spider(self): self.file = open("items.jsonl", "w") - def close_spider(self, spider): + def close_spider(self): self.file.close() - def process_item(self, item, spider): + def process_item(self, item): line = json.dumps(ItemAdapter(item).asdict()) + "\n" self.file.write(line) return item @@ -139,8 +121,8 @@ In this example we'll write items to MongoDB_ using pymongo_. MongoDB address and database name are specified in Scrapy settings; MongoDB collection is named after item class. -The main point of this example is to show how to use :meth:`from_crawler` -method and how to clean up the resources properly. +The main point of this example is to show how to :ref:`get the crawler +` and how to clean up the resources properly. .. skip: next .. code-block:: python @@ -163,19 +145,19 @@ method and how to clean up the resources properly. mongo_db=crawler.settings.get("MONGO_DATABASE", "items"), ) - def open_spider(self, spider): + def open_spider(self): self.client = pymongo.MongoClient(self.mongo_uri) self.db = self.client[self.mongo_db] - def close_spider(self, spider): + def close_spider(self): self.client.close() - def process_item(self, item, spider): + def process_item(self, item): self.db[self.collection_name].insert_one(ItemAdapter(item).asdict()) return item .. _MongoDB: https://www.mongodb.com/ -.. _pymongo: https://api.mongodb.com/python/current/ +.. _pymongo: https://pymongo.readthedocs.io/en/stable/ .. _ScreenshotPipeline: @@ -200,7 +182,6 @@ item. import scrapy from itemadapter import ItemAdapter from scrapy.http.request import NO_CALLBACK - from scrapy.utils.defer import maybe_deferred_to_future class ScreenshotPipeline: @@ -209,14 +190,19 @@ item. SPLASH_URL = "http://localhost:8050/render.png?url={}" - async def process_item(self, item, spider): + def __init__(self, crawler): + self.crawler = crawler + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + async def process_item(self, item): adapter = ItemAdapter(item) encoded_item_url = quote(adapter["url"]) screenshot_url = self.SPLASH_URL.format(encoded_item_url) request = scrapy.Request(screenshot_url, callback=NO_CALLBACK) - response = await maybe_deferred_to_future( - spider.crawler.engine.download(request) - ) + response = await self.crawler.engine.download_async(request) if response.status != 200: # Error happened, return item. @@ -251,10 +237,10 @@ returns multiples items with the same id: def __init__(self): self.ids_seen = set() - def process_item(self, item, spider): + def process_item(self, item): adapter = ItemAdapter(item) if adapter["id"] in self.ids_seen: - raise DropItem(f"Duplicate item found: {item!r}") + raise DropItem(f"Item ID already seen: {adapter['id']}") else: self.ids_seen.add(adapter["id"]) return item diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 3c38ac2dc..7a1c1f907 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -42,39 +42,27 @@ Item objects :class:`Item` provides a :class:`dict`-like API plus additional features that make it the most feature-complete item type: -.. class:: scrapy.item.Item([arg]) -.. class:: scrapy.Item([arg]) +.. autoclass:: scrapy.Item + :members: copy, deepcopy, fields + :undoc-members: - :class:`Item` objects replicate the standard :class:`dict` API, including - its ``__init__`` method. +:class:`Item` objects replicate the standard :class:`dict` API, including +its ``__init__`` method. - :class:`Item` allows defining field names, so that: +:class:`Item` allows the defining of field names, so that: - - :class:`KeyError` is raised when using undefined field names (i.e. - prevents typos going unnoticed) +- :class:`KeyError` is raised when using undefined field names (i.e. + prevents typos going unnoticed) - - :ref:`Item exporters ` can export all fields by - default even if the first scraped object does not have values for all - of them +- :ref:`Item exporters ` can export all fields by + default even if the first scraped object does not have values for all + of them - :class:`Item` also allows defining field metadata, which can be used to - :ref:`customize serialization `. +:class:`Item` also allows the defining of field metadata, which can be used to +:ref:`customize serialization `. - :mod:`trackref` tracks :class:`Item` objects to help find memory leaks - (see :ref:`topics-leaks-trackrefs`). - - :class:`Item` objects also provide the following additional API members: - - .. automethod:: copy - - .. automethod:: deepcopy - - .. attribute:: fields - - A dictionary containing *all declared fields* for this Item, not only - those populated. The keys are the field names and the values are the - :class:`Field` objects used in the :ref:`Item declaration - `. +:mod:`trackref` tracks :class:`Item` objects to help find memory leaks +(see :ref:`topics-leaks-trackrefs`). Example: @@ -92,13 +80,11 @@ Example: Dataclass objects ----------------- -.. versionadded:: 2.2 - -:func:`~dataclasses.dataclass` allows defining item classes with field names, +:func:`~dataclasses.dataclass` allows the defining of item classes with field names, so that :ref:`item exporters ` can export all fields by default even if the first scraped object does not have values for all of them. -Additionally, ``dataclass`` items also allow to: +Additionally, ``dataclass`` items also allow you to: * define the type and default value of each defined field. @@ -124,9 +110,7 @@ Example: attr.s objects -------------- -.. versionadded:: 2.2 - -:func:`attr.s` allows defining item classes with field names, +:func:`attr.s` allows the defining of item classes with field names, so that :ref:`item exporters ` can export all fields by default even if the first scraped object does not have values for all of them. @@ -152,6 +136,45 @@ Example: another_field = attr.ib() +.. _pydantic-items: + +Pydantic models +--------------- + +`Pydantic `_ models allow the defining of item +classes with field names, so that :ref:`item exporters ` can +export all fields by default even if the first scraped object does not have +values for all of them. + +Additionally, ``pydantic`` items also allow you to: + +* define the type and default value of each defined field with run-time type + validation. + +* define custom field metadata through `pydantic.Field + `_, which can be used to + :ref:`customize serialization `. + +* benefit from automatic data validation and conversion based on type + annotations. + +In order to use this type, the `pydantic package `_ +needs to be installed. + +Example: + +.. code-block:: python + + from pydantic import BaseModel, Field + + + class CustomItem(BaseModel): + one_field: str = Field(default="", description="First field") + another_field: int = Field(default=0, description="Second field") + +.. note:: Unlike other item types, Pydantic models enforce field types at + run time and will raise validation errors for invalid data types. + Working with Item objects ========================= @@ -205,10 +228,9 @@ documentation to see which metadata keys are used by each component. It's important to note that the :class:`Field` objects used to declare the item do not stay assigned as class attributes. Instead, they can be accessed through -the :attr:`Item.fields` attribute. +the :attr:`~scrapy.Item.fields` attribute. -.. class:: scrapy.item.Field([arg]) -.. class:: scrapy.Field([arg]) +.. autoclass:: scrapy.Field The :class:`Field` class is just an alias to the built-in :class:`dict` class and doesn't provide any extra functionality or attributes. In other words, @@ -221,12 +243,14 @@ the :attr:`Item.fields` attribute. `attr.ib`_ for additional information. .. _dataclasses.field: https://docs.python.org/3/library/dataclasses.html#dataclasses.field - .. _attr.ib: https://www.attrs.org/en/stable/api.html#attr.ib + .. _attr.ib: https://www.attrs.org/en/stable/api-attr.html#attr.ib Working with Item objects ------------------------- +.. skip: start + Here are some examples of common tasks performed with items, using the ``Product`` item :ref:`declared above `. You will notice the API is very similar to the :class:`dict` API. @@ -388,6 +412,8 @@ appending more values, or changing existing values, like this: That adds (or replaces) the ``serializer`` metadata key for the ``name`` field, keeping all the previously existing metadata values. +.. skip: end + .. _supporting-item-types: @@ -397,14 +423,8 @@ Supporting All Item Types In code that receives an item, such as methods of :ref:`item pipelines ` or :ref:`spider middlewares `, it is a good practice to use the -:class:`~itemadapter.ItemAdapter` class and the -:func:`~itemadapter.is_item` function to write code that works for -any :ref:`supported item type `: - -.. autoclass:: itemadapter.ItemAdapter - -.. autofunction:: itemadapter.is_item - +:class:`~itemadapter.ItemAdapter` class to write code that works for any +supported item type. Other classes related to items ============================== diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index c7fc1ea48..769925dd5 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -17,15 +17,25 @@ facilities: * an extension that keeps some spider state (key/value pairs) persistent between batches +.. _job-dir: + Job directory ============= -To enable persistence support you just need to define a *job directory* through -the ``JOBDIR`` setting. This directory will be for storing all required data to -keep the state of a single job (i.e. a spider run). It's important to note that -this directory must not be shared by different spiders, or even different -jobs/runs of the same spider, as it's meant to be used for storing the state of -a *single* job. +To enable persistence support, define a *job directory* through the +:setting:`JOBDIR` setting. + +The job directory will store all required data to keep the state of a *single* +job (i.e. a spider run), so that if stopped cleanly, it can be resumed later. + +.. warning:: This directory must *not* be shared by different spiders, or even + different jobs of the same spider. + +.. warning:: Treat the job directory with the same security care as your + Scrapy project source code. Do not point ``JOBDIR`` to a path that + untrusted parties can write to. + +See also :ref:`job-dir-contents`. How to use it ============= @@ -46,9 +56,9 @@ Keeping persistent state between batches Sometimes you'll want to keep some persistent spider state between pause/resume batches. You can use the ``spider.state`` attribute for that, which should be a -dict. There's a built-in extension that takes care of serializing, storing and -loading that attribute from the job directory, when the spider starts and -stops. +dict. There's :ref:`a built-in extension ` +that takes care of serializing, storing and loading that attribute from the job +directory, when the spider starts and stops. Here's an example of a callback that uses the spider state (other spider code is omitted for brevity): @@ -65,6 +75,14 @@ Persistence gotchas There are a few things to keep in mind if you want to be able to use the Scrapy persistence support: +Pause limitations +----------------- + +Job pausing and resuming is only supported when the spider is paused by +stopping it cleanly. Forced, sudden or otherwise unclean shutdown can lead to +data corruption in the job directory, which may prevent the spider from +resuming correctly. + Cookies expiration ------------------ @@ -72,7 +90,6 @@ Cookies may expire. So, if you don't resume your spider quickly the requests scheduled may no longer work. This won't be an issue if your spider doesn't rely on cookies. - .. _request-serialization: Request serialization @@ -86,3 +103,52 @@ running :class:`~scrapy.Spider` class. If you wish to log the requests that couldn't be serialized, you can set the :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. It is ``False`` by default. + +.. _job-dir-contents: + +Job directory contents +====================== + +The contents of a job directory depend on the components used during the job. +Components known to write in the job directory include the :ref:`scheduler +` and the :class:`~scrapy.extensions.spiderstate.SpiderState` +extension. See the reference documentation of the corresponding components for +details. + +For example, with default settings, the job directory may look like this: + +.. code-block:: none + + ├── requests.queue + | ├── active.json + | └── {hostname}-{hash} + | └── {priority}{s?} + | ├── q{00000} + | └── info.json + ├── requests.seen + └── spider.state + +Where: + +- :class:`~scrapy.core.scheduler.Scheduler` creates the ``requests.queue/`` + directory and the ``active.json`` file, the latter containing the state + data returned by :meth:`DownloaderAwarePriorityQueue.close() + ` the last time the job + was paused. + +- :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue` creates the + ``{hostname}-{hash}`` directories. + +- :class:`~scrapy.pqueues.ScrapyPriorityQueue` creates the ``{priority}{s?}`` + directories. + +- :class:`scrapy.squeues.PickleLifoDiskQueue`, a subclass of + :class:`queuelib.LifoDiskQueue` that uses :mod:`pickle` to serialize + :class:`dict` representations of :class:`scrapy.Request` objects, creates + the ``info.json`` and ``q{00000}`` files. + +- :class:`~scrapy.dupefilters.RFPDupeFilter` creates the ``requests.seen`` + file. + +- :class:`~scrapy.extensions.spiderstate.SpiderState` creates the + ``spider.state`` file. diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index cd8914644..0913a3310 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -60,25 +60,29 @@ in control. Debugging memory leaks with ``trackref`` ======================================== +.. skip: start + :mod:`trackref` is a module provided by Scrapy to debug the most common cases of memory leaks. It basically tracks the references to all live Request, Response, Item, Spider and Selector objects. You can enter the telnet console and inspect how many objects (of the classes mentioned above) are currently alive using the ``prefs()`` function which is an -alias to the :func:`~scrapy.utils.trackref.print_live_refs` function:: +alias to the :func:`~scrapy.utils.trackref.print_live_refs` function: + +.. code-block:: bash telnet localhost 6023 - .. code-block:: pycon +.. code-block:: pycon - >>> prefs() - Live References + >>> prefs() + Live References - ExampleSpider 1 oldest: 15s ago - HtmlResponse 10 oldest: 1s ago - Selector 2 oldest: 0s ago - FormRequest 878 oldest: 7s ago + ExampleSpider 1 oldest: 15s ago + HtmlResponse 10 oldest: 1s ago + Selector 2 oldest: 0s ago + Request 878 oldest: 7s ago As you can see, that report also shows the "age" of the oldest object in each class. If you're running multiple spiders per process chances are you can @@ -160,7 +164,7 @@ Too many spiders? ----------------- If your project has too many spiders executed in parallel, -the output of :func:`prefs()` can be difficult to read. +the output of :func:`prefs` can be difficult to read. For this reason, that function has a ``ignore`` argument which can be used to ignore a particular class (and all its subclasses). For example, this won't show any live references to spiders: @@ -203,6 +207,8 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module. ``None`` if none is found. Use :func:`print_live_refs` first to get a list of all tracked live objects per class name. +.. skip: end + .. _topics-leaks-muppy: Debugging memory leaks with muppy @@ -226,6 +232,7 @@ If you use ``pip``, you can install muppy with the following command:: Here's an example to view all Python objects available in the heap using muppy: +.. skip: start .. code-block:: pycon >>> from pympler import muppy @@ -253,6 +260,8 @@ the heap using muppy: `, using the :ref:`Product item ` declared in the :ref:`Items chapter `: +.. skip: next .. code-block:: python from scrapy.loader import ItemLoader @@ -101,14 +102,13 @@ One approach to overcome this is to define items using the .. code-block:: python from dataclasses import dataclass, field - from typing import Optional @dataclass class InventoryItem: - name: Optional[str] = field(default=None) - price: Optional[float] = field(default=None) - stock: Optional[int] = field(default=None) + name: str | None = field(default=None) + price: float | None = field(default=None) + stock: int | None = field(default=None) .. _topics-loaders-processors: @@ -130,6 +130,7 @@ assigned to the item. Let's see an example to illustrate how the input and output processors are called for a particular field (the same applies for any other field): +.. skip: next .. code-block:: python l = ItemLoader(Product(), some_selector) @@ -172,9 +173,6 @@ with the data to be parsed, and return a parsed value. So you can use any function as input or output processor. The only requirement is that they must accept one (and only one) positional argument, which will be an iterable. -.. versionchanged:: 2.0 - Processors no longer need to be methods. - .. note:: Both input and output processors must receive an iterable as their first argument. The output of those functions can be anything. The result of input processors will be appended to an internal list (in the Loader) @@ -229,7 +227,8 @@ metadata. Here is an example: .. code-block:: python - import scrapy + from dataclasses import dataclass, field + from itemloaders.processors import Join, MapCompose, TakeFirst from w3lib.html import remove_tags @@ -239,17 +238,25 @@ metadata. Here is an example: return value - class Product(scrapy.Item): - name = scrapy.Field( - input_processor=MapCompose(remove_tags), - output_processor=Join(), + @dataclass + class Product: + name: str | None = field( + default=None, + metadata={ + "input_processor": MapCompose(remove_tags), + "output_processor": Join(), + }, ) - price = scrapy.Field( - input_processor=MapCompose(remove_tags, filter_price), - output_processor=TakeFirst(), + price: str | None = field( + default=None, + metadata={ + "input_processor": MapCompose(remove_tags, filter_price), + "output_processor": TakeFirst(), + }, ) +.. skip: start .. code-block:: pycon >>> from scrapy.loader import ItemLoader @@ -259,6 +266,8 @@ metadata. Here is an example: >>> il.load_item() {'name': 'Welcome to my website', 'price': '1000'} +.. skip: end + The precedence order, for both input and output processors, is as follows: 1. Item Loader field-specific attributes: ``field_in`` and ``field_out`` (most @@ -294,6 +303,8 @@ the Item Loader that it's able to receive an Item Loader context, so the Item Loader passes the currently active context when calling it, and the processor function (``parse_length`` in this case) can thus use them. +.. skip: start + There are several ways to modify Item Loader context values: 1. By modifying the currently active Item Loader context @@ -320,6 +331,8 @@ There are several ways to modify Item Loader context values: class ProductLoader(ItemLoader): length_out = MapCompose(parse_length, unit="cm") +.. skip: end + ItemLoader objects ================== @@ -350,6 +363,7 @@ that you wish to extract. Example: +.. skip: next .. code-block:: python loader = ItemLoader(item=Item()) @@ -364,6 +378,7 @@ the footer selector. Example: +.. skip: next .. code-block:: python loader = ItemLoader(item=Item()) @@ -401,6 +416,7 @@ those dashes in the final product names. Here's how you can remove those dashes by reusing and extending the default Product Item Loader (``ProductLoader``): +.. skip: next .. code-block:: python from itemloaders.processors import MapCompose @@ -418,6 +434,7 @@ Another case where extending Item Loaders can be very helpful is when you have multiple source formats, for example XML and HTML. In the XML version you may want to remove ``CDATA`` occurrences. Here's an example of how to do it: +.. skip: next .. code-block:: python from itemloaders.processors import MapCompose @@ -442,4 +459,3 @@ organization of your Loaders collection - that's up to you and your project's needs. .. _itemloaders: https://itemloaders.readthedocs.io/en/latest/ -.. _processors: https://itemloaders.readthedocs.io/en/latest/built-in-processors.html diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index fe1c4d162..bbb5d0458 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -194,6 +194,48 @@ If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy component that prints the log. It is unset by default, hence logs contain the Scrapy component responsible for that log output. +Rotating log files +------------------ + +Scrapy's :setting:`LOG_FILE` setting writes logs to a single file. It does not +rotate log files automatically, but you can use Python's standard +:mod:`logging.handlers` module when running Scrapy from a script. + +For example, to rotate the log file every day: + +.. skip: next + +.. code-block:: python + + import logging + from logging.handlers import TimedRotatingFileHandler + + from scrapy.crawler import CrawlerProcess + from scrapy.utils.project import get_project_settings + + from myproject.spiders.myspider import MySpider + + settings = get_project_settings() + process = CrawlerProcess(settings, install_root_handler=False) + + handler = TimedRotatingFileHandler( + "scrapy.log", + when="midnight", + backupCount=7, + encoding=settings.get("LOG_ENCODING"), + ) + handler.setFormatter( + logging.Formatter(settings.get("LOG_FORMAT"), settings.get("LOG_DATEFORMAT")) + ) + + root_logger = logging.getLogger() + root_logger.setLevel(settings.get("LOG_LEVEL")) + root_logger.addHandler(handler) + + process.crawl(MySpider) + process.start() + + Command-line options -------------------- @@ -266,9 +308,9 @@ e.g. in the spider's ``__init__`` method: If you run this spider again then INFO messages from ``scrapy.spidermiddlewares.httperror`` logger will be gone. -You can also filter log records by :class:`~logging.LogRecord` data. For +You can also filter log records by :class:`~logging.LogRecord` data. For example, you can filter log records by message content using a substring or -a regular expression. Create a :class:`logging.Filter` subclass +a regular expression. Create a :class:`logging.Filter` subclass and equip it with a regular expression pattern to filter out unwanted messages: @@ -284,8 +326,8 @@ filter out unwanted messages: if match: return False -A project-level filter may be attached to the root -handler created by Scrapy, this is a wieldy way to +A project-level filter may be attached to the root +handler created by Scrapy, this is a wieldy way to filter all loggers in different parts of the project (middlewares, spider, etc.): @@ -301,7 +343,7 @@ filter all loggers in different parts of the project for handler in logging.root.handlers: handler.addFilter(ContentFilter()) -Alternatively, you may choose a specific logger +Alternatively, you may choose a specific logger and hide it without affecting other loggers: .. code-block:: python diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index da0587aa4..037fe87fa 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you can configure some extra functions like generating thumbnails and filtering the images based on their size. -The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for +The Images Pipeline requires Pillow_ 8.3.2 or greater. It is used for thumbnailing and normalizing images to JPEG/RGB format. .. _Pillow: https://github.com/python-pillow/Pillow @@ -212,8 +212,6 @@ Where: FTP server storage ------------------ -.. versionadded:: 2.0 - :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can point to an FTP server. Scrapy will automatically upload the files to the server. @@ -238,7 +236,7 @@ Amazon S3 storage .. setting:: FILES_STORE_S3_ACL .. setting:: IMAGES_STORE_S3_ACL -If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and +If botocore_ >= 1.13.45 is installed, :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will automatically upload the files to the bucket. @@ -261,7 +259,7 @@ policy: For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide. You can also use other S3-like storages. Storages like self-hosted `Minio`_ or -`s3.scality`_. All you need to do is set endpoint option in you Scrapy +`Zenko CloudServer`_. All you need to do is set endpoint option in you Scrapy settings: .. code-block:: python @@ -276,9 +274,9 @@ For self-hosting you also might feel the need not to use SSL and not to verify S AWS_VERIFY = False # or True (None by default) .. _botocore: https://github.com/boto/botocore -.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl .. _Minio: https://github.com/minio/minio -.. _s3.scality: https://s3.scality.com/ +.. _Zenko CloudServer: https://www.zenko.io/cloudserver/ .. _media-pipeline-gcs: @@ -292,7 +290,7 @@ Google Cloud Storage :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ ) -.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python +.. _google-cloud-storage: https://docs.cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings: @@ -303,7 +301,7 @@ For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_I For information about authentication, see this `documentation`_. -.. _documentation: https://cloud.google.com/docs/authentication/production +.. _documentation: https://docs.cloud.google.com/docs/authentication You can modify the Access Control List (ACL) policy used for the stored files, which is defined by the :setting:`FILES_STORE_GCS_ACL` and @@ -318,7 +316,7 @@ policy: For more information, see `Predefined ACLs`_ in the Google Cloud Platform Developer Guide. -.. _Predefined ACLs: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl +.. _Predefined ACLs: https://docs.cloud.google.com/storage/docs/access-control/lists#predefined-acl Usage example ============= @@ -339,17 +337,18 @@ respectively), the pipeline will put the results under the respective field When using :ref:`item types ` for which fields are defined beforehand, you must define both the URLs field and the results field. For example, when using the images pipeline, items must define both the ``image_urls`` and the -``images`` field. For instance, using the :class:`~scrapy.Item` class: +``images`` field. For instance, using a dataclass: .. code-block:: python - import scrapy + from dataclasses import dataclass, field - class MyItem(scrapy.Item): + @dataclass + class MyItem: # ... other item fields ... - image_urls = scrapy.Field() - images = scrapy.Field() + image_urls: list[str] = field(default_factory=list) + images: list[dict] = field(default_factory=list) If you want to use another field name for the URLs key or for the results key, it is also possible to override it. @@ -414,7 +413,7 @@ class name. E.g. given pipeline class called MyPipeline you can set setting key: and pipeline class MyPipeline will have expiration time set to 180. -The last modified time from the file is used to determine the age of the file in days, +The last modified time from the file is used to determine the age of the file in days, which is then compared to the set expiration time to determine if the file is expired. .. _topics-images-thumbnails: @@ -519,7 +518,7 @@ See here the methods that you can override in your custom Files Pipeline: In addition to ``response``, this method receives the original :class:`request `, - :class:`info ` and + :class:`info ` and :class:`item ` You can override this method to customize the download path of each file. @@ -532,24 +531,21 @@ See here the methods that you can override in your custom Files Pipeline: .. code-block:: python from pathlib import PurePosixPath - from urllib.parse import urlparse + from scrapy.utils.httpobj import urlparse_cached from scrapy.pipelines.files import FilesPipeline class MyFilesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None, *, item=None): - return "files/" + PurePosixPath(urlparse(request.url).path).name + return "files/" + PurePosixPath(urlparse_cached(request).path).name - Similarly, you can use the ``item`` to determine the file path based on some item + Similarly, you can use the ``item`` to determine the file path based on some item property. - + By default the :meth:`file_path` method returns ``full/.``. - .. versionadded:: 2.4 - The *item* parameter. - .. method:: FilesPipeline.get_media_requests(item, info) As seen on the workflow, the pipeline will get the URLs of the images to @@ -590,8 +586,6 @@ See here the methods that you can override in your custom Files Pipeline: * ``status`` - the file status indication. - .. versionadded:: 2.2 - It can be one of the following: * ``downloaded`` - file was downloaded. @@ -677,7 +671,7 @@ See here the methods that you can override in your custom Images Pipeline: In addition to ``response``, this method receives the original :class:`request `, - :class:`info ` and + :class:`info ` and :class:`item ` You can override this method to customize the download path of each file. @@ -690,24 +684,21 @@ See here the methods that you can override in your custom Images Pipeline: .. code-block:: python from pathlib import PurePosixPath - from urllib.parse import urlparse + from scrapy.utils.httpobj import urlparse_cached from scrapy.pipelines.images import ImagesPipeline class MyImagesPipeline(ImagesPipeline): def file_path(self, request, response=None, info=None, *, item=None): - return "files/" + PurePosixPath(urlparse(request.url).path).name + return "files/" + PurePosixPath(urlparse_cached(request).path).name - Similarly, you can use the ``item`` to determine the file path based on some item + Similarly, you can use the ``item`` to determine the file path based on some item property. - + By default the :meth:`file_path` method returns ``full/.``. - .. versionadded:: 2.4 - The *item* parameter. - .. method:: ImagesPipeline.thumb_path(self, request, thumb_id, response=None, info=None, *, item=None) This method is called for every item of :setting:`IMAGES_THUMBS` per downloaded item. It returns the diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index b1b8c9e9c..11c2656da 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -21,16 +21,21 @@ Remember that Scrapy is built on top of the Twisted asynchronous networking library, so you need to run it inside the Twisted reactor. The first utility you can use to run your spiders is -:class:`scrapy.crawler.CrawlerProcess`. This class will start a Twisted reactor -for you, configuring the logging and setting shutdown handlers. This class is -the one used by all Scrapy commands. +:class:`scrapy.crawler.AsyncCrawlerProcess` or +:class:`scrapy.crawler.CrawlerProcess`. These classes will start a Twisted +reactor for you, configuring the logging and setting shutdown handlers. These +classes are the ones used by all Scrapy commands. They have similar +functionality, differing in their asynchronous API style: +:class:`~scrapy.crawler.AsyncCrawlerProcess` returns coroutines from its +asynchronous methods while :class:`~scrapy.crawler.CrawlerProcess` returns +:class:`~twisted.internet.defer.Deferred` objects. Here's an example showing how to run a single spider with it. .. code-block:: python import scrapy - from scrapy.crawler import CrawlerProcess + from scrapy.crawler import AsyncCrawlerProcess class MySpider(scrapy.Spider): @@ -38,7 +43,7 @@ Here's an example showing how to run a single spider with it. ... - process = CrawlerProcess( + process = AsyncCrawlerProcess( settings={ "FEEDS": { "items.json": {"format": "json"}, @@ -49,53 +54,182 @@ Here's an example showing how to run a single spider with it. process.crawl(MySpider) process.start() # the script will block here until the crawling is finished -Define settings within dictionary in CrawlerProcess. Make sure to check :class:`~scrapy.crawler.CrawlerProcess` +You can define :ref:`settings ` within the dictionary passed +to :class:`~scrapy.crawler.AsyncCrawlerProcess`. Make sure to check the +:class:`~scrapy.crawler.AsyncCrawlerProcess` documentation to get acquainted with its usage details. If you are inside a Scrapy project there are some additional helpers you can use to import those components within the project. You can automatically import -your spiders passing their name to :class:`~scrapy.crawler.CrawlerProcess`, and -use ``get_project_settings`` to get a :class:`~scrapy.settings.Settings` -instance with your project settings. +your spiders passing their name to +:class:`~scrapy.crawler.AsyncCrawlerProcess`, and use +:func:`scrapy.utils.project.get_project_settings` to get a +:class:`~scrapy.settings.Settings` instance with your project settings. What follows is a working example of how to do that, using the `testspiders`_ project as example. .. code-block:: python - from scrapy.crawler import CrawlerProcess + from scrapy.crawler import AsyncCrawlerProcess from scrapy.utils.project import get_project_settings - process = CrawlerProcess(get_project_settings()) + process = AsyncCrawlerProcess(get_project_settings()) # 'followall' is the name of one of the spiders of the project. process.crawl("followall", domain="scrapy.org") process.start() # the script will block here until the crawling is finished There's another Scrapy utility that provides more control over the crawling -process: :class:`scrapy.crawler.CrawlerRunner`. This class is a thin wrapper -that encapsulates some simple helpers to run multiple crawlers, but it won't -start or interfere with existing reactors in any way. +process: :class:`scrapy.crawler.AsyncCrawlerRunner` or +:class:`scrapy.crawler.CrawlerRunner`. These classes are thin wrappers +that encapsulate some simple helpers to run multiple crawlers, but they won't +start or interfere with existing reactors in any way. Just like +:class:`scrapy.crawler.AsyncCrawlerProcess` and +:class:`scrapy.crawler.CrawlerProcess` they differ in their asynchronous API +style. -Using this class the reactor should be explicitly run after scheduling your -spiders. It's recommended you use :class:`~scrapy.crawler.CrawlerRunner` -instead of :class:`~scrapy.crawler.CrawlerProcess` if your application is -already using Twisted and you want to run Scrapy in the same reactor. +When using these classes the reactor should be explicitly run after scheduling +your spiders. It's recommended that you use +:class:`~scrapy.crawler.AsyncCrawlerRunner` or +:class:`~scrapy.crawler.CrawlerRunner` instead of +:class:`~scrapy.crawler.AsyncCrawlerProcess` or +:class:`~scrapy.crawler.CrawlerProcess` if your application is already using +Twisted and you want to run Scrapy in the same reactor. -Note that you will also have to shutdown the Twisted reactor yourself after the -spider is finished. This can be achieved by adding callbacks to the deferred -returned by the :meth:`CrawlerRunner.crawl -` method. +If you want to stop the reactor or run any other code right after the spider +finishes you can do that after the task returned from +:meth:`AsyncCrawlerRunner.crawl() ` +completes (or the Deferred returned from :meth:`CrawlerRunner.crawl() +` fires). In the simplest case you can also +use :func:`twisted.internet.task.react` to start and stop the reactor, though +it may be easier to just use :class:`~scrapy.crawler.AsyncCrawlerProcess` or +:class:`~scrapy.crawler.CrawlerProcess` instead. -Here's an example of its usage, along with a callback to manually stop the -reactor after ``MySpider`` has finished running. +Here's an example of using :class:`~scrapy.crawler.AsyncCrawlerRunner` together +with simple reactor management code: + +.. code-block:: python + + import scrapy + from scrapy.crawler import AsyncCrawlerRunner + from scrapy.utils.defer import deferred_f_from_coro_f + from scrapy.utils.log import configure_logging + from scrapy.utils.reactor import install_reactor + from twisted.internet.task import react + + + class MySpider(scrapy.Spider): + # Your spider definition + ... + + + async def crawl(_): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = AsyncCrawlerRunner() + await runner.crawl(MySpider) # completes when the spider finishes + + + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + react(deferred_f_from_coro_f(crawl)) + +Same example but using :class:`~scrapy.crawler.CrawlerRunner` and a +different reactor (:class:`~scrapy.crawler.AsyncCrawlerRunner` only works +with :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`): .. code-block:: python - from twisted.internet import reactor import scrapy from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging + from scrapy.utils.reactor import install_reactor + from twisted.internet.task import react + + + class MySpider(scrapy.Spider): + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.epollreactor.EPollReactor", + } + # Your spider definition + ... + + + def crawl(_): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = CrawlerRunner() + d = runner.crawl(MySpider) + return d # this Deferred fires when the spider finishes + + + install_reactor("twisted.internet.epollreactor.EPollReactor") + react(crawl) + +.. seealso:: :doc:`twisted:core/howto/reactor-basics` + +And here are examples of using these classes with +:setting:`TWISTED_REACTOR_ENABLED` set to ``False``. + +Simple usage of :class:`~scrapy.crawler.AsyncCrawlerProcess`: + +.. code-block:: python + + import scrapy + from scrapy.crawler import AsyncCrawlerProcess + + + class MySpider(scrapy.Spider): + # Your spider definition + ... + + + process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR_ENABLED": False, + } + ) + + process.crawl(MySpider) + process.start() # the script will block here until the crawling is finished + +With ``TWISTED_REACTOR_ENABLED=False`` you can use several instances of +:class:`~scrapy.crawler.AsyncCrawlerProcess` in the same process: + +.. code-block:: python + + import scrapy + from scrapy.crawler import AsyncCrawlerProcess + + + class MySpider(scrapy.Spider): + # Your spider definition + ... + + + process1 = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR_ENABLED": False, + } + ) + process1.crawl(MySpider) + process1.start() + + process2 = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR_ENABLED": False, + } + ) + process2.crawl(MySpider) + process2.start() + +Using :func:`asyncio.run` with :class:`~scrapy.crawler.AsyncCrawlerRunner`: + +.. code-block:: python + + import asyncio + + import scrapy + from scrapy.crawler import AsyncCrawlerRunner + from scrapy.utils.log import configure_logging class MySpider(scrapy.Spider): @@ -103,14 +237,14 @@ reactor after ``MySpider`` has finished running. ... - configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) - runner = CrawlerRunner() + async def main(): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) + await runner.crawl(MySpider) # completes when the spider finishes - d = runner.crawl(MySpider) - d.addBoth(lambda _: reactor.stop()) - reactor.run() # the script will block here until the crawling is finished -.. seealso:: :doc:`twisted:core/howto/reactor-basics` + asyncio.run(main()) + .. _run-multiple-spiders: @@ -126,7 +260,7 @@ Here is an example that runs multiple spiders simultaneously: .. code-block:: python import scrapy - from scrapy.crawler import CrawlerProcess + from scrapy.crawler import AsyncCrawlerProcess from scrapy.utils.project import get_project_settings @@ -141,20 +275,21 @@ Here is an example that runs multiple spiders simultaneously: settings = get_project_settings() - process = CrawlerProcess(settings) + process = AsyncCrawlerProcess(settings) process.crawl(MySpider1) process.crawl(MySpider2) process.start() # the script will block here until all crawling jobs are finished -Same example using :class:`~scrapy.crawler.CrawlerRunner`: +Same example using :class:`~scrapy.crawler.AsyncCrawlerRunner`: .. code-block:: python import scrapy - from twisted.internet import reactor - from scrapy.crawler import CrawlerRunner + from scrapy.crawler import AsyncCrawlerRunner + from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.log import configure_logging - from scrapy.utils.project import get_project_settings + from scrapy.utils.reactor import install_reactor + from twisted.internet.task import react class MySpider1(scrapy.Spider): @@ -167,24 +302,29 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: ... - configure_logging() - settings = get_project_settings() - runner = CrawlerRunner(settings) - runner.crawl(MySpider1) - runner.crawl(MySpider2) - d = runner.join() - d.addBoth(lambda _: reactor.stop()) + async def crawl(_): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = AsyncCrawlerRunner() + runner.crawl(MySpider1) + runner.crawl(MySpider2) + await runner.join() # completes when both spiders finish - reactor.run() # the script will block here until all crawling jobs are finished -Same example but running the spiders sequentially by chaining the deferreds: + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + react(deferred_f_from_coro_f(crawl)) + + +Same example but running the spiders sequentially by awaiting until each one +finishes before starting the next one: .. code-block:: python - from twisted.internet import reactor, defer - from scrapy.crawler import CrawlerRunner + import scrapy + from scrapy.crawler import AsyncCrawlerRunner + from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.log import configure_logging - from scrapy.utils.project import get_project_settings + from scrapy.utils.reactor import install_reactor + from twisted.internet.task import react class MySpider1(scrapy.Spider): @@ -197,39 +337,20 @@ Same example but running the spiders sequentially by chaining the deferreds: ... - settings = get_project_settings() - configure_logging(settings) - runner = CrawlerRunner(settings) + async def crawl(_): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = AsyncCrawlerRunner() + await runner.crawl(MySpider1) + await runner.crawl(MySpider2) - @defer.inlineCallbacks - def crawl(): - yield runner.crawl(MySpider1) - yield runner.crawl(MySpider2) - reactor.stop() + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + react(deferred_f_from_coro_f(crawl)) - - crawl() - reactor.run() # the script will block here until the last crawl call is finished - -Different spiders can set different values for the same setting, but when they -run in the same process it may be impossible, by design or because of some -limitations, to use these different values. What happens in practice is -different for different settings: - -* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value - (:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the - default one) cannot be read from the per-spider settings. These are applied - when the :class:`~scrapy.crawler.CrawlerRunner` or - :class:`~scrapy.crawler.CrawlerProcess` object is created. -* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first - available value is used, and if a spider requests a different reactor an - exception will be raised. These are applied when the reactor is installed. -* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the - ones used by the resolver (:setting:`DNSCACHE_ENABLED`, - :setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy) - the first available value is used. These are applied when the reactor is - started. +.. note:: When running multiple spiders in the same process, :ref:`reactor + settings ` should not have a different value per spider. + Also, :ref:`pre-crawler settings ` cannot be defined + per spider. .. seealso:: :ref:`run-from-script`. @@ -240,7 +361,7 @@ different for different settings: Distributed crawls ================== -Scrapy doesn't provide any built-in facility for running crawls in a distribute +Scrapy doesn't provide any built-in facility for running crawls in a distributed (multi-server) manner. However, there are some ways to distribute crawls, which vary depending on how you plan to distribute them. @@ -248,10 +369,10 @@ If you have many spiders, the obvious way to distribute the load is to setup many Scrapyd instances and distribute spider runs among those. If you instead want to run a single (big) spider through many machines, what -you usually do is partition the urls to crawl and send them to each separate +you usually do is partition the URLs to crawl and send them to each separate spider. Here is a concrete example: -First, you prepare the list of urls to crawl and put them into separate +First, you prepare the list of URLs to crawl and put them into separate files/urls:: http://somedomain.com/urls-to-crawl/spider1/part1.list @@ -278,7 +399,7 @@ consider contacting `commercial support`_ if in doubt. Here are some tips to keep in mind when dealing with these kinds of sites: -* rotate your user agent from a pool of well-known ones from browsers (google +* rotate your user agent from a pool of well-known ones from browsers (Google around to get a list of them) * disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use cookies to spot bot behaviour @@ -288,14 +409,19 @@ Here are some tips to keep in mind when dealing with these kinds of sites: * use a pool of rotating IPs. For example, the free `Tor project`_ or paid services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a super proxy that you can attach your own proxies to. +* for HTTPS websites, if blocking appears related to TLS behavior, consider + adjusting the :setting:`DOWNLOAD_TLS_MIN_VERSION` and + :setting:`DOWNLOAD_TLS_MAX_VERSION` settings, since some websites may respond + differently depending on the TLS method used by the client. * use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy - plugin `__ + plugin `__ and additional + features, like `AI web scraping `__ If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. .. _Tor project: https://www.torproject.org/ -.. _commercial support: https://scrapy.org/support/ +.. _commercial support: https://www.scrapy.org/companies .. _ProxyMesh: https://proxymesh.com/ .. _Common Crawl: https://commoncrawl.org/ .. _testspiders: https://github.com/scrapinghub/testspiders diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 4440783cf..4968256bc 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -7,15 +7,15 @@ Requests and Responses .. module:: scrapy.http :synopsis: Request and Response classes -Scrapy uses :class:`Request` and :class:`Response` objects for crawling web +Scrapy uses :class:`~scrapy.Request` and :class:`Response` objects for crawling web sites. -Typically, :class:`Request` objects are generated in the spiders and pass +Typically, :class:`~scrapy.Request` objects are generated in the spiders and pass across the system until they reach the Downloader, which executes the request and returns a :class:`Response` object which travels back to the spider that issued the request. -Both :class:`Request` and :class:`Response` classes have subclasses which add +Both :class:`~scrapy.Request` and :class:`Response` classes have subclasses which add functionality not required in the base classes. These are described below in :ref:`topics-request-response-ref-request-subclasses` and :ref:`topics-request-response-ref-response-subclasses`. @@ -24,35 +24,20 @@ below in :ref:`topics-request-response-ref-request-subclasses` and Request objects =============== -.. autoclass:: Request +.. autoclass:: scrapy.Request :param url: the URL of this request If the URL is invalid, a :exc:`ValueError` exception is raised. :type url: str - :param callback: the function that will be called with the response of this - request (once it's downloaded) as its first parameter. - - In addition to a function, the following values are supported: - - - ``None`` (default), which indicates that the spider's - :meth:`~scrapy.Spider.parse` method must be used. - - - :func:`~scrapy.http.request.NO_CALLBACK` - - For more information, see - :ref:`topics-request-response-ref-request-callback-arguments`. - - .. note:: If exceptions are raised during processing, ``errback`` is - called instead. - - :type callback: collections.abc.Callable + :param callback: sets :attr:`callback`, defaults to ``None``. + :type callback: Callable[Concatenate[Response, ...], Any] | None :param method: the HTTP method of this request. Defaults to ``'GET'``. :type method: str - :param meta: the initial values for the :attr:`Request.meta` attribute. If + :param meta: the initial values for the :attr:`.Request.meta` attribute. If given, the dict passed in this parameter will be shallow copied. :type meta: dict @@ -67,10 +52,10 @@ Request objects (for single valued headers) or lists (for multi-valued headers). If ``None`` is passed as value, the HTTP header will not be sent at all. - .. caution:: Cookies set via the ``Cookie`` header are not considered by the - :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known - current limitation that is being worked on. + .. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + ``cookies`` argument. This is a known current limitation that is being + worked on. :type headers: dict @@ -94,13 +79,14 @@ Request objects .. code-block:: python request_with_cookies = Request( - url="http://www.example.com", + url="https://www.example.com", cookies=[ { "name": "currency", "value": "USD", "domain": "example.com", "path": "/currency", + "secure": True, }, ], ) @@ -123,43 +109,27 @@ Request objects .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`scrapy.Request.cookies ` parameter. This is a known current limitation that is being worked on. - .. versionadded:: 2.6.0 - Cookie values that are :class:`bool`, :class:`float` or :class:`int` - are casted to :class:`str`. - :type cookies: dict or list :param encoding: the encoding of this request (defaults to ``'utf-8'``). This encoding will be used to percent-encode the URL and to convert the body to bytes (if given as a string). + + To disable URL percent-encoding for a request, use the + :reqmeta:`verbatim_url` request meta key. :type encoding: str - :param priority: the priority of this request (defaults to ``0``). - The priority is used by the scheduler to define the order used to process - requests. Requests with a higher priority value will execute earlier. - Negative values are allowed in order to indicate relatively low-priority. + :param priority: sets :attr:`priority`, defaults to ``0``. :type priority: int - :param dont_filter: indicates that this request should not be filtered by - the scheduler. This is used when you want to perform an identical - request multiple times, to ignore the duplicates filter. Use it with - care, or you will get into crawling loops. Default to ``False``. + :param dont_filter: sets :attr:`dont_filter`, defaults to ``False``. :type dont_filter: bool - :param errback: a function that will be called if any exception was - raised while processing the request. This includes pages that failed - with 404 HTTP errors and such. It receives a - :exc:`~twisted.python.failure.Failure` as first parameter. - For more information, - see :ref:`topics-request-response-ref-errbacks` below. - - .. versionchanged:: 2.0 - The *callback* parameter is no longer required when the *errback* - parameter is specified. - :type errback: collections.abc.Callable + :param errback: sets :attr:`errback`, defaults to ``None``. + :type errback: Callable[[Failure], Any] | None :param flags: Flags sent to the request, can be used for logging or similar purposes. :type flags: list @@ -169,9 +139,13 @@ Request objects .. attribute:: Request.url - A string containing the URL of this request. Keep in mind that this - attribute contains the escaped URL, so it can differ from the URL passed in - the ``__init__`` method. + A string containing the URL of this request. + + Keep in mind that this attribute contains the escaped URL, so it can + differ from the URL passed in the ``__init__()`` method. + + If :reqmeta:`verbatim_url` is set to ``True``, the URL is kept as + passed to ``__init__()``. This attribute is read-only. To change the URL of a Request use :meth:`replace`. @@ -183,7 +157,8 @@ Request objects .. attribute:: Request.headers - A dictionary-like object which contains the request headers. + A dictionary-like (:class:`scrapy.http.headers.Headers`) object which contains + the request headers. .. attribute:: Request.body @@ -192,6 +167,27 @@ Request objects This attribute is read-only. To change the body of a Request use :meth:`replace`. + .. autoattribute:: callback + + .. autoattribute:: errback + + .. autoattribute:: priority + + .. attribute:: Request.cb_kwargs + + A dictionary that contains arbitrary metadata for this request. Its contents + will be passed to the Request's callback as keyword arguments. It is empty + for new Requests, which means by default callbacks only get a + :class:`~scrapy.http.Response` object as argument. + + This dict is :doc:`shallow copied ` when the request is + cloned using the ``copy()`` or ``replace()`` methods, and can also be + accessed, in your spider, from the ``response.cb_kwargs`` attribute. + + In case of a failure to process the request, this dict can be accessed as + ``failure.request.cb_kwargs`` in the request's errback. For more information, + see :ref:`errback-cb_kwargs`. + .. attribute:: Request.meta :value: {} @@ -235,20 +231,7 @@ Request objects Also mind that the :meth:`copy` and :meth:`replace` request methods :doc:`shallow-copy ` request metadata. - .. attribute:: Request.cb_kwargs - - A dictionary that contains arbitrary metadata for this request. Its contents - will be passed to the Request's callback as keyword arguments. It is empty - for new Requests, which means by default callbacks only get a :class:`Response` - object as argument. - - This dict is :doc:`shallow copied ` when the request is - cloned using the ``copy()`` or ``replace()`` methods, and can also be - accessed, in your spider, from the ``response.cb_kwargs`` attribute. - - In case of a failure to process the request, this dict can be accessed as - ``failure.request.cb_kwargs`` in the request's errback. For more information, - see :ref:`errback-cb_kwargs`. + .. autoattribute:: dont_filter .. autoattribute:: Request.attributes @@ -261,7 +244,7 @@ Request objects Return a Request object with the same members, except for those members given new values by whichever keyword arguments are specified. The - :attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are shallow + :attr:`~scrapy.Request.cb_kwargs` and :attr:`~scrapy.Request.meta` attributes are shallow copied by default (unless new values are given as arguments). See also :ref:`topics-request-response-ref-request-callback-arguments`. @@ -270,6 +253,78 @@ Request objects .. automethod:: to_dict +.. _form: + +Creating requests that submit HTML forms +---------------------------------------- + +Use :doc:`form2request ` to build request data from an HTML +``
`` element and convert it to a :class:`~scrapy.Request`. + +Install it with pip: + +.. code-block:: bash + + pip install form2request + +Select the desired form with CSS or XPath, then build and convert request +data: + +.. code-block:: python + + from form2request import form2request + + + def parse(self, response): + form = response.css("form#search") + request_data = form2request(form, data={"q": "scrapy"}) + yield request_data.to_scrapy(callback=self.parse_results) + +Use ``data`` to override field values. To drop a field from the resulting +request, set its value to ``None``. + +By default, form2request simulates clicking the first submit button. To submit +without clicking any button, pass ``click=False``. To click a specific submit +button, pass its element: + +.. code-block:: python + + def parse(self, response): + form = response.css("form#checkout") + submit = form.css('button[name="pay"]') + request_data = form2request(form, click=submit) + +.. _topics-request-response-ref-request-userlogin: + +Using form2request to simulate a user login +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It is usual for web sites to provide pre-populated form fields through ```` elements, such as session related data or authentication +tokens (for login pages). Build the request from the form and only override the +credentials: + +.. code-block:: python + + import scrapy + from form2request import form2request + + + class LoginSpider(scrapy.Spider): + name = "example.com" + start_urls = ["http://www.example.com/users/login.php"] + + def parse(self, response): + form = response.css("form") + request_data = form2request( + form, + data={"username": "john", "password": "secret"}, + ) + yield request_data.to_scrapy(callback=self.after_login) + + def after_login(self, response): ... + + Other functions related to requests ----------------------------------- @@ -304,7 +359,7 @@ Example: In some cases you may be interested in passing arguments to those callback functions so you can receive the arguments later, in the second callback. The following example shows how to achieve this by using the -:attr:`Request.cb_kwargs` attribute: +:attr:`.Request.cb_kwargs` attribute: .. code-block:: python @@ -325,10 +380,10 @@ The following example shows how to achieve this by using the foo=foo, ) -.. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``. - Prior to that, using :attr:`Request.meta` was recommended for passing - information around callbacks. After ``1.7``, :attr:`Request.cb_kwargs` - became the preferred way for handling user information, leaving :attr:`Request.meta` +.. caution:: :attr:`.Request.cb_kwargs` was introduced in version ``1.7``. + Prior to that, using :attr:`.Request.meta` was recommended for passing + information around callbacks. After ``1.7``, :attr:`.Request.cb_kwargs` + became the preferred way for handling user information, leaving :attr:`.Request.meta` for communication with components like middlewares and extensions. .. _topics-request-response-ref-errbacks: @@ -364,7 +419,7 @@ errors if needed: "https://example.invalid/", # DNS error expected ] - def start_requests(self): + async def start(self): for u in self.start_urls: yield scrapy.Request( u, @@ -440,7 +495,7 @@ Request fingerprints There are some aspects of scraping, such as filtering out duplicate requests (see :setting:`DUPEFILTER_CLASS`) or caching responses (see :setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short, -unique identifier from a :class:`~scrapy.http.Request` object: a request +unique identifier from a :class:`~scrapy.Request` object: a request fingerprint. You often do not need to worry about request fingerprints, the default request @@ -460,8 +515,6 @@ To change how request fingerprints are built for your requests, use the REQUEST_FINGERPRINTER_CLASS ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: 2.7 - Default: :class:`scrapy.utils.request.RequestFingerprinter` A :ref:`request fingerprinter class ` or its @@ -474,42 +527,17 @@ import path. Writing your own request fingerprinter ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A request fingerprinter is a class that must implement the following method: +A request fingerprinter is a :ref:`component ` that must +implement the following method: .. currentmodule:: None -.. method:: fingerprint(self, request) +.. method:: fingerprint(self, request: scrapy.Request) Return a :class:`bytes` object that uniquely identifies *request*. See also :ref:`request-fingerprint-restrictions`. - :param request: request to fingerprint - :type request: scrapy.http.Request - -Additionally, it may also implement the following methods: - -.. classmethod:: from_crawler(cls, crawler) - :noindex: - - If present, this class method is called to create a request fingerprinter - instance from a :class:`~scrapy.crawler.Crawler` object. It must return a - new instance of the request fingerprinter. - - *crawler* provides access to all Scrapy core components like settings and - signals; it is a way for the request fingerprinter to access them and hook - its functionality into Scrapy. - - :param crawler: crawler that uses this request fingerprinter - :type crawler: :class:`~scrapy.crawler.Crawler` object - -.. classmethod:: from_settings(cls, settings) - - If present, and ``from_crawler`` is not defined, this class method is called - to create a request fingerprinter instance from a - :class:`~scrapy.settings.Settings` object. It must return a new instance of - the request fingerprinter. - .. currentmodule:: scrapy.http The :meth:`fingerprint` method of the default request fingerprinter, @@ -520,6 +548,11 @@ in your :meth:`fingerprint` method implementation: .. autofunction:: scrapy.utils.request.fingerprint +By default, request fingerprinting canonicalizes the request URL. If +:reqmeta:`verbatim_url` is set to ``True``, fingerprinting does not +canonicalize the URL, and the ``keep_fragments`` parameter is ignored (it is +effectively true). + For example, to take the value of a request header named ``X-ID`` into account: @@ -572,7 +605,7 @@ URL canonicalization or taking the request method or body into account: If you need to be able to override the request fingerprinting for arbitrary requests from your spider callbacks, you may implement a request fingerprinter -that reads fingerprints from :attr:`request.meta ` +that reads fingerprints from :attr:`request.meta ` when available, and then falls back to :func:`scrapy.utils.request.fingerprint`. For example: @@ -587,10 +620,8 @@ when available, and then falls back to return request.meta["fingerprint"] return fingerprint(request) -If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6 -without using the deprecated ``'2.6'`` value of the -:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following -request fingerprinter: +If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6, use +the following request fingerprinter: .. code-block:: python @@ -634,7 +665,7 @@ The following built-in Scrapy components have such restrictions: :setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`, the following directory structure is created: - - :attr:`Spider.name ` + - :attr:`.Spider.name` - first byte of a request fingerprint as hexadecimal @@ -662,11 +693,13 @@ The following built-in Scrapy components have such restrictions: Request.meta special keys ========================= -The :attr:`Request.meta` attribute can contain any arbitrary data, but there +The :attr:`.Request.meta` attribute can contain any arbitrary data, but there are some special keys recognized by Scrapy and its built-in extensions. Those are: +* :reqmeta:`allow_offsite` +* :reqmeta:`autothrottle_dont_adjust_delay` * :reqmeta:`bindaddress` * :reqmeta:`cookiejar` * :reqmeta:`dont_cache` @@ -677,23 +710,60 @@ Those are: * :reqmeta:`download_fail_on_dataloss` * :reqmeta:`download_latency` * :reqmeta:`download_maxsize` +* :reqmeta:`download_warnsize` * :reqmeta:`download_timeout` * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) * ``ftp_user`` (See :setting:`FTP_USER` for more info) +* :reqmeta:`give_up_log_level` * :reqmeta:`handle_httpstatus_all` * :reqmeta:`handle_httpstatus_list` +* :reqmeta:`is_start_request` * :reqmeta:`max_retry_times` * :reqmeta:`proxy` * :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` * :reqmeta:`referrer_policy` +* :reqmeta:`verbatim_url` .. reqmeta:: bindaddress bindaddress ----------- -The IP of the outgoing IP address to use for the performing the request. +The default local outgoing address for download-handler connections. + +This meta value can be either: + +- a host address as a string (e.g. ``"127.0.0.2"``), in which case the local + port is chosen automatically, or + +- a ``(host, port)`` tuple (e.g. ``("127.0.0.2", 50000)``) to bind to both a + specific local interface and a specific local port. + +For example: + +.. code-block:: python + + Request( + "https://example.org", + meta={"bindaddress": "127.0.0.2"}, + ) + +.. code-block:: python + + Request( + "https://example.org", + meta={"bindaddress": ("127.0.0.2", 50000)}, + ) + +If not set, built-in HTTP download handlers use the value of +:setting:`DOWNLOAD_BIND_ADDRESS` as the default bind address. +Set the :reqmeta:`bindaddress` request meta key to override it for a +specific request. + +This meta key is not supported by +:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`, but the +:setting:`DOWNLOAD_BIND_ADDRESS` is supported by it. .. reqmeta:: download_timeout @@ -721,15 +791,38 @@ download_fail_on_dataloss Whether or not to fail on broken responses. See: :setting:`DOWNLOAD_FAIL_ON_DATALOSS`. +.. reqmeta:: give_up_log_level + +give_up_log_level +----------------- + +:ref:`Logging level ` used for the message logged when a request +exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details. + .. reqmeta:: max_retry_times max_retry_times --------------- -The meta key is used set retry times per request. When initialized, the +The meta key is used set retry times per request. When set, the :reqmeta:`max_retry_times` meta key takes higher precedence over the :setting:`RETRY_TIMES` setting. +.. reqmeta:: verbatim_url + +verbatim_url +------------ + +Set this key to ``True`` to keep the request URL as passed to +:class:`~scrapy.Request`, without URL percent-encoding. + +When this key is enabled, :func:`~scrapy.utils.request.fingerprint` does not +canonicalize the request URL, so requests whose URLs differ only in +characters that would otherwise be canonicalized get different fingerprints. + +In this mode, the ``keep_fragments`` parameter is ignored, and it is +effectively true. + .. _topics-stop-response-download: @@ -784,177 +877,28 @@ call their callback instead, like in this example, pass ``fail=False`` to the Request subclasses ================== -Here is the list of built-in :class:`Request` subclasses. You can also subclass +Here is the list of built-in :class:`~scrapy.Request` subclasses. You can also subclass it to implement your own custom functionality. -FormRequest objects -------------------- - -The FormRequest class extends the base :class:`Request` with functionality for -dealing with HTML forms. It uses `lxml.html forms`_ to pre-populate form -fields with form data from :class:`Response` objects. - -.. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms - -.. class:: scrapy.http.request.form.FormRequest -.. class:: scrapy.http.FormRequest -.. class:: scrapy.FormRequest(url, [formdata, ...]) - - The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The - remaining arguments are the same as for the :class:`Request` class and are - not documented here. - - :param formdata: is a dictionary (or iterable of (key, value) tuples) - containing HTML Form data which will be url-encoded and assigned to the - body of the request. - :type formdata: dict or collections.abc.Iterable - - The :class:`FormRequest` objects support the following class method in - addition to the standard :class:`Request` methods: - - .. classmethod:: FormRequest.from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...]) - - Returns a new :class:`FormRequest` object with its form field values - pre-populated with those found in the HTML ```` element contained - in the given response. For an example see - :ref:`topics-request-response-ref-request-userlogin`. - - The policy is to automatically simulate a click, by default, on any form - control that looks clickable, like a ````. Even - though this is quite convenient, and often the desired behaviour, - sometimes it can cause problems which could be hard to debug. For - example, when working with forms that are filled and/or submitted using - javascript, the default :meth:`from_response` behaviour may not be the - most appropriate. To disable this behaviour you can set the - ``dont_click`` argument to ``True``. Also, if you want to change the - control clicked (instead of disabling it) you can also use the - ``clickdata`` argument. - - .. caution:: Using this method with select elements which have leading - or trailing whitespace in the option values will not work due to a - `bug in lxml`_, which should be fixed in lxml 3.8 and above. - - :param response: the response containing a HTML form which will be used - to pre-populate the form fields - :type response: :class:`Response` object - - :param formname: if given, the form with name attribute set to this value will be used. - :type formname: str - - :param formid: if given, the form with id attribute set to this value will be used. - :type formid: str - - :param formxpath: if given, the first form that matches the xpath will be used. - :type formxpath: str - - :param formcss: if given, the first form that matches the css selector will be used. - :type formcss: str - - :param formnumber: the number of form to use, when the response contains - multiple forms. The first one (and also the default) is ``0``. - :type formnumber: int - - :param formdata: fields to override in the form data. If a field was - already present in the response ```` element, its value is - overridden by the one passed in this parameter. If a value passed in - this parameter is ``None``, the field will not be included in the - request, even if it was present in the response ```` element. - :type formdata: dict - - :param clickdata: attributes to lookup the control clicked. If it's not - given, the form data will be submitted simulating a click on the - first clickable element. In addition to html attributes, the control - can be identified by its zero-based index relative to other - submittable inputs inside the form, via the ``nr`` attribute. - :type clickdata: dict - - :param dont_click: If True, the form data will be submitted without - clicking in any element. - :type dont_click: bool - - The other parameters of this class method are passed directly to the - :class:`FormRequest` ``__init__`` method. - -Request usage examples ----------------------- - -Using FormRequest to send data via HTTP POST -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you want to simulate a HTML Form POST in your spider and send a couple of -key-value fields, you can return a :class:`FormRequest` object (from your -spider) like this: - -.. skip: next -.. code-block:: python - - return [ - FormRequest( - url="http://www.example.com/post/action", - formdata={"name": "John Doe", "age": "27"}, - callback=self.after_post, - ) - ] - -.. _topics-request-response-ref-request-userlogin: - -Using FormRequest.from_response() to simulate a user login -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It is usual for web sites to provide pre-populated form fields through ```` elements, such as session related data or authentication -tokens (for login pages). When scraping, you'll want these fields to be -automatically pre-populated and only override a couple of them, such as the -user name and password. You can use the :meth:`FormRequest.from_response` -method for this job. Here's an example spider which uses it: - -.. code-block:: python - - import scrapy - - - def authentication_failed(response): - # TODO: Check the contents of the response and return True if it failed - # or False if it succeeded. - pass - - - class LoginSpider(scrapy.Spider): - name = "example.com" - start_urls = ["http://www.example.com/users/login.php"] - - def parse(self, response): - return scrapy.FormRequest.from_response( - response, - formdata={"username": "john", "password": "secret"}, - callback=self.after_login, - ) - - def after_login(self, response): - if authentication_failed(response): - self.logger.error("Login failed") - return - - # continue scraping with authenticated session... - JsonRequest ----------- -The JsonRequest class extends the base :class:`Request` class with functionality for +The JsonRequest class extends the base :class:`~scrapy.Request` class with functionality for dealing with JSON requests. .. class:: JsonRequest(url, [... data, dumps_kwargs]) - The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method. The - remaining arguments are the same as for the :class:`Request` class and are + The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__()`` method. The + remaining arguments are the same as for the :class:`~scrapy.Request` class and are not documented here. Using the :class:`JsonRequest` will set the ``Content-Type`` header to ``application/json`` and ``Accept`` header to ``application/json, text/javascript, */*; q=0.01`` :param data: is any JSON serializable object that needs to be JSON encoded and assigned to body. - if :attr:`Request.body` argument is provided this parameter will be ignored. - if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be + If the :attr:`~scrapy.Request.body` argument is provided this parameter will be ignored. + If the :attr:`~scrapy.Request.body` argument is not provided and the + ``data`` argument is provided the :attr:`~scrapy.Request.method` will be set to ``'POST'`` automatically. :type data: object @@ -1006,11 +950,11 @@ Response objects :type flags: list :param request: the initial value of the :attr:`Response.request` attribute. - This represents the :class:`Request` that generated this response. + This represents the :class:`~scrapy.Request` that generated this response. :type request: scrapy.Request :param certificate: an object representing the server's SSL certificate. - :type certificate: twisted.internet.ssl.Certificate + :type certificate: typing.Any :param ip_address: The IP address of the server from which the Response originated. :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` @@ -1019,15 +963,6 @@ Response objects For instance: "HTTP/1.0", "HTTP/1.1", "h2" :type protocol: :class:`str` - .. versionadded:: 2.0.0 - The ``certificate`` parameter. - - .. versionadded:: 2.1.0 - The ``ip_address`` parameter. - - .. versionadded:: 2.5.0 - The ``protocol`` parameter. - .. attribute:: Response.url A string containing the URL of the response. @@ -1042,11 +977,12 @@ Response objects .. attribute:: Response.headers - A dictionary-like object which contains the response headers. Values can - be accessed using :meth:`get` to return the first header value with the - specified name or :meth:`getlist` to return all header values with the - specified name. For example, this call will give you all cookies in the - headers:: + A dictionary-like (:class:`scrapy.http.headers.Headers`) object which contains + the response headers. Values can be accessed using + :meth:`~scrapy.http.headers.Headers.get` to return the first header value with + the specified name or :meth:`~scrapy.http.headers.Headers.getlist` to return + all header values with the specified name. For example, this call will give you + all cookies in the headers:: response.headers.getlist('Set-Cookie') @@ -1062,7 +998,7 @@ Response objects .. attribute:: Response.request - The :class:`Request` object that generated this response. This attribute is + The :class:`~scrapy.Request` object that generated this response. This attribute is assigned in the Scrapy engine, after the response and the request have passed through all :ref:`Downloader Middlewares `. In particular, this means that: @@ -1081,49 +1017,42 @@ Response objects .. attribute:: Response.meta - A shortcut to the :attr:`Request.meta` attribute of the + A shortcut to the :attr:`~scrapy.Request.meta` attribute of the :attr:`Response.request` object (i.e. ``self.request.meta``). Unlike the :attr:`Response.request` attribute, the :attr:`Response.meta` attribute is propagated along redirects and retries, so you will get - the original :attr:`Request.meta` sent from your spider. + the original :attr:`.Request.meta` sent from your spider. - .. seealso:: :attr:`Request.meta` attribute + .. seealso:: :attr:`.Request.meta` attribute .. attribute:: Response.cb_kwargs - .. versionadded:: 2.0 - - A shortcut to the :attr:`Request.cb_kwargs` attribute of the + A shortcut to the :attr:`~scrapy.Request.cb_kwargs` attribute of the :attr:`Response.request` object (i.e. ``self.request.cb_kwargs``). Unlike the :attr:`Response.request` attribute, the :attr:`Response.cb_kwargs` attribute is propagated along redirects and - retries, so you will get the original :attr:`Request.cb_kwargs` sent - from your spider. + retries, so you will get the original :attr:`.Request.cb_kwargs` sent from your spider. - .. seealso:: :attr:`Request.cb_kwargs` attribute + .. seealso:: :attr:`.Request.cb_kwargs` attribute .. attribute:: Response.flags A list that contains flags for this response. Flags are labels used for tagging Responses. For example: ``'cached'``, ``'redirected``', etc. And - they're shown on the string representation of the Response (`__str__` + they're shown on the string representation of the Response (``__str__()`` method) which is used by the engine for logging. .. attribute:: Response.certificate - .. versionadded:: 2.0.0 - - A :class:`twisted.internet.ssl.Certificate` object representing - the server's SSL certificate. + An object representing the server's SSL certificate. Its type and + contents depend on the download handler that produced the response. Only populated for ``https`` responses, ``None`` otherwise. .. attribute:: Response.ip_address - .. versionadded:: 2.1.0 - The IP address of the server from which the Response originated. This attribute is currently only populated by the HTTP 1.1 download @@ -1132,8 +1061,6 @@ Response objects .. attribute:: Response.protocol - .. versionadded:: 2.5.0 - The protocol that was used to download the response. For instance: "HTTP/1.0", "HTTP/1.1" @@ -1185,7 +1112,7 @@ TextResponse objects :class:`Response` class, which is meant to be used only for binary data, such as images, sounds or any media file. - :class:`TextResponse` objects support a new ``__init__`` method argument, in + :class:`TextResponse` objects support a new ``__init__()`` method argument, in addition to the base :class:`Response` objects. The remaining functionality is the same as for the :class:`Response` class and is not documented here. @@ -1223,7 +1150,7 @@ TextResponse objects A string with the encoding of this response. The encoding is resolved by trying the following mechanisms, in order: - 1. the encoding passed in the ``__init__`` method ``encoding`` argument + 1. the encoding passed in the ``__init__()`` method ``encoding`` argument 2. the encoding declared in the Content-Type HTTP header. If this encoding is not valid (i.e. unknown), it is ignored and the next @@ -1279,7 +1206,7 @@ TextResponse objects Constructs an absolute url by combining the Response's base url with a possible relative url. The base url shall be extracted from the - ```` tag, or just the Response's :attr:`url` if there is no such + ```` tag, or just :attr:`Response.url` if there is no such tag. @@ -1311,7 +1238,7 @@ JsonResponse objects .. class:: JsonResponse(url[, ...]) - The :class:`JsonResponse` class is a subclass of :class:`TextResponse` - that is used when the response has a `JSON MIME type - `_ in its `Content-Type` + The :class:`JsonResponse` class is a subclass of :class:`TextResponse` + that is used when the response has a `JSON MIME type + `_ in its `Content-Type` header. diff --git a/docs/topics/scheduler.rst b/docs/topics/scheduler.rst index 57c24b76a..b79d6de1f 100644 --- a/docs/topics/scheduler.rst +++ b/docs/topics/scheduler.rst @@ -26,9 +26,16 @@ Minimal scheduler interface :members: -Default Scrapy scheduler -======================== +Default scheduler +================= -.. autoclass:: Scheduler +.. autoclass:: Scheduler() :members: - :special-members: __len__ + :special-members: __init__, __len__ + + +Priority queues +=============== + +.. autoclass:: scrapy.pqueues.DownloaderAwarePriorityQueue +.. autoclass:: scrapy.pqueues.ScrapyPriorityQueue diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 4a64d530b..35ea2a200 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -308,6 +308,7 @@ Examples: * ``*::text`` selects all descendant text nodes of the current selector context: +..skip: next .. code-block:: pycon >>> response.css("#images *::text").getall() @@ -542,7 +543,7 @@ you may want to take a look first at this `XPath tutorial`_. .. note:: Some of the tips are based on `this post from Zyte's blog`_. -.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html +.. _XPath tutorial: http://www.zvon.org/comp/r/tut-XPath_1.html .. _this post from Zyte's blog: https://www.zyte.com/blog/xpath-tips-from-the-web-scraping-trenches/ @@ -559,7 +560,7 @@ For example, suppose you want to extract all ``

`` elements inside ``

`` elements. First, you would get all ``
`` elements: .. code-block:: pycon - + >>> divs = response.xpath("//div") At first, you may be tempted to use the following approach, which is wrong, as @@ -591,7 +592,7 @@ Another common case would be to extract all direct ``

`` children: For more details about relative XPaths see the `Location Paths`_ section in the XPath specification. -.. _Location Paths: https://www.w3.org/TR/xpath/all/#location-paths +.. _Location Paths: https://www.w3.org/TR/xpath-10/#location-paths When querying by class, consider using CSS ------------------------------------------ @@ -610,7 +611,7 @@ As it turns out, Scrapy selectors allow you to chain selectors, so most of the t you can just select by class using CSS and then switch to XPath when needed: .. code-block:: pycon - + >>> from scrapy import Selector >>> sel = Selector( ... text='

' @@ -727,7 +728,7 @@ But using the ``.`` to mean the node, works: >>> sel.xpath("//a[contains(., 'Next Page')]").getall() ['Click here to go to the Next Page'] -.. _`XPath string function`: https://www.w3.org/TR/xpath/all/#section-String-Functions +.. _XPath string function: https://www.w3.org/TR/xpath-10/#section-String-Functions .. _topics-selectors-xpath-variables: @@ -777,7 +778,7 @@ Removing namespaces When dealing with scraping projects, it is often quite convenient to get rid of namespaces altogether and just work with element names, to write more simple/convenient XPaths. You can use the -:meth:`Selector.remove_namespaces` method for that. +:meth:`.Selector.remove_namespaces` method for that. Let's show an example that illustrates this with the Python Insider blog atom feed. @@ -801,8 +802,8 @@ This is how the file starts:: ... You can see several namespace declarations including a default -"http://www.w3.org/2005/Atom" and another one using the "gd:" prefix for -"http://schemas.google.com/g/2005". +``"http://www.w3.org/2005/Atom"`` and another one using the ``gd:`` prefix for +``"http://schemas.google.com/g/2005"``. .. highlight:: python @@ -814,7 +815,7 @@ doesn't work (because the Atom XML namespace is obfuscating those nodes): >>> response.xpath("//link") [] -But once we call the :meth:`Selector.remove_namespaces` method, all +But once we call the :meth:`.Selector.remove_namespaces` method, all nodes can be accessed directly by their names: .. code-block:: pycon @@ -878,7 +879,7 @@ Example selecting links in list item with a "class" attribute ending with a digi >>> sel = Selector(text=doc, type="html") >>> sel.xpath("//li//@href").getall() ['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html'] - >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall() + >>> sel.xpath(r'//li[re:test(@class, "item-\d$")]//@href').getall() ['link1.html', 'link2.html', 'link4.html', 'link5.html'] .. warning:: C library ``libxslt`` doesn't natively support EXSLT regular @@ -982,9 +983,9 @@ Here we first iterate over ``itemscope`` elements, and for each one, we look for all ``itemprops`` elements and exclude those that are themselves inside another ``itemscope``. -.. _EXSLT: http://exslt.org/ -.. _regular expressions: http://exslt.org/regexp/index.html -.. _set manipulation: http://exslt.org/set/index.html +.. _EXSLT: https://exslt.github.io/ +.. _regular expressions: https://exslt.github.io/regexp/index.html +.. _set manipulation: https://exslt.github.io/set/index.html Other XPath extensions ---------------------- @@ -1032,10 +1033,8 @@ whereas the CSS lookup is translated into XPath and thus runs more efficiently, so performance-wise its uses are limited to situations that are not easily described with CSS selectors. -Parsel also simplifies adding your own XPath extensions. - -.. autofunction:: parsel.xpathfuncs.set_xpathfunc - +Parsel also simplifies adding your own XPath extensions with +:func:`~parsel.xpathfuncs.set_xpathfunc`. .. _topics-selectors-ref: @@ -1048,7 +1047,7 @@ Built-in Selectors reference Selector objects ---------------- -.. autoclass:: Selector +.. autoclass:: scrapy.Selector .. automethod:: xpath @@ -1062,6 +1061,12 @@ Selector objects For convenience, this method can be called as ``response.css()`` + .. automethod:: jmespath + + .. note:: + + For convenience, this method can be called as ``response.jmespath()`` + .. automethod:: get See also: :ref:`old-extraction-api` @@ -1094,6 +1099,8 @@ SelectorList objects .. automethod:: css + .. automethod:: jmespath + .. automethod:: getall See also: :ref:`old-extraction-api` @@ -1120,8 +1127,8 @@ Examples Selector examples on HTML response ---------------------------------- -Here are some :class:`Selector` examples to illustrate several concepts. -In all cases, we assume there is already a :class:`Selector` instantiated with +Here are some :class:`~scrapy.Selector` examples to illustrate several concepts. +In all cases, we assume there is already a :class:`~scrapy.Selector` instantiated with a :class:`~scrapy.http.HtmlResponse` object like this: .. code-block:: python @@ -1129,7 +1136,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this: sel = Selector(html_response) 1. Select all ``

`` elements from an HTML response body, returning a list of - :class:`Selector` objects (i.e. a :class:`SelectorList` object): + :class:`~scrapy.Selector` objects (i.e. a :class:`SelectorList` object): .. code-block:: python @@ -1159,7 +1166,7 @@ Selector examples on XML response .. skip: start -Here are some examples to illustrate concepts for :class:`Selector` objects +Here are some examples to illustrate concepts for :class:`~scrapy.Selector` objects instantiated with an :class:`~scrapy.http.XmlResponse` object: .. code-block:: python @@ -1167,7 +1174,7 @@ instantiated with an :class:`~scrapy.http.XmlResponse` object: sel = Selector(xml_response) 1. Select all ```` elements from an XML response body, returning a list - of :class:`Selector` objects (i.e. a :class:`SelectorList` object): + of :class:`~scrapy.Selector` objects (i.e. a :class:`SelectorList` object): .. code-block:: python @@ -1183,4 +1190,4 @@ instantiated with an :class:`~scrapy.http.XmlResponse` object: .. skip: end -.. _Google Base XML feed: https://support.google.com/merchants/answer/160589?hl=en&ref_topic=2473799 +.. _Google Base XML feed: https://support.google.com/merchants/answer/14987622 diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 06f8481ba..06de33e6f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -33,42 +33,48 @@ Python :ref:`import search path `. Populating the settings ======================= -Settings can be populated using different mechanisms, each of which having a -different precedence. Here is the list of them in decreasing order of -precedence: +Settings can be populated using different mechanisms, each of which has a +different precedence: - 1. Command line options (most precedence) - 2. Settings per-spider - 3. Project settings module - 4. Settings set by add-ons - 5. Default settings per-command - 6. Default global settings (less precedence) + 1. :ref:`Command-line settings ` (highest precedence) + 2. :ref:`Spider settings ` + 3. :ref:`Project settings ` + 4. :ref:`Add-on settings ` + 5. :ref:`Command-specific default settings ` + 6. :ref:`Global default settings ` (lowest precedence) -The population of these settings sources is taken care of internally, but a -manual handling is possible using API calls. See the -:ref:`topics-api-settings` topic for reference. +.. _cli-settings: -These mechanisms are described in more detail below. +1. Command-line settings +------------------------ -1. Command line options ------------------------ +Settings set in the command line have the highest precedence, overriding any +other settings. -Arguments provided by the command line are the ones that take most precedence, -overriding any other options. You can explicitly override one (or more) -settings using the ``-s`` (or ``--set``) command line option. +You can explicitly override one or more settings using the ``-s`` (or +``--set``) command-line option. .. highlight:: sh Example:: - scrapy crawl myspider -s LOG_FILE=scrapy.log + scrapy crawl myspider -s LOG_LEVEL=INFO -s LOG_FILE=scrapy.log -2. Settings per-spider ----------------------- +.. _spider-settings: -Spiders (See the :ref:`topics-spiders` chapter for reference) can define their -own settings that will take precedence and override the project ones. One way -to do so is by setting their :attr:`~scrapy.Spider.custom_settings` attribute: +2. Spider settings +------------------ + +:ref:`Spiders ` can define their own settings that will take +precedence and override the project ones. + +.. note:: :ref:`Pre-crawler settings ` cannot be defined + per spider, and :ref:`reactor settings ` should not have + a different value per spider when :ref:`running multiple spiders in the + same process `. + +One way to do so is by setting their :attr:`~scrapy.Spider.custom_settings` +attribute: .. code-block:: python @@ -83,7 +89,7 @@ to do so is by setting their :attr:`~scrapy.Spider.custom_settings` attribute: } It's often better to implement :meth:`~scrapy.Spider.update_settings` instead, -and settings set there should use the "spider" priority explicitly: +and settings set there should use the ``"spider"`` priority explicitly: .. code-block:: python @@ -121,33 +127,52 @@ arguments ` or other logic: ) return spider -3. Project settings module --------------------------- +.. _project-settings: -The project settings module is the standard configuration file for your Scrapy -project, it's where most of your custom settings will be populated. For a -standard Scrapy project, this means you'll be adding or changing the settings -in the ``settings.py`` file created for your project. +3. Project settings +------------------- -4. Settings set by add-ons --------------------------- +Scrapy projects include a settings module, usually a file called +``settings.py``, where you should populate most settings that apply to all your +spiders. + +.. seealso:: :ref:`topics-settings-module-envvar` + +.. _addon-settings: + +4. Add-on settings +------------------ :ref:`Add-ons ` can modify settings. They should do this with -this priority, though this is not enforced. +``"addon"`` priority where possible. -5. Default settings per-command -------------------------------- +.. _cmd-default-settings: -Each :doc:`Scrapy tool ` command can have its own default -settings, which override the global default settings. Those custom command -settings are specified in the ``default_settings`` attribute of the command -class. +5. Command-specific default settings +------------------------------------ + +Each :ref:`Scrapy command ` can have its own default settings, +which override the :ref:`global default settings `. + +Those command-specific default settings are specified in the +``default_settings`` attribute of each command class. + +.. _default-settings: 6. Default global settings -------------------------- -The global defaults are located in the ``scrapy.settings.default_settings`` -module and documented in the :ref:`topics-settings-ref` section. +The ``scrapy.settings.default_settings`` module defines global default values +for some :ref:`built-in settings `. + +.. note:: :command:`startproject` generates a ``settings.py`` file that sets + some settings to different values. + + The reference documentation of settings indicates the default value if one + exists. If :command:`startproject` sets a value, that value is documented + as default, and the value from ``scrapy.settings.default_settings`` is + documented as “fallbackâ€. + Compatibility with pickle ========================= @@ -157,8 +182,6 @@ Setting values must be :ref:`picklable `. Import paths and classes ======================== -.. versionadded:: 2.4.0 - When a setting references a callable object to be imported by Scrapy, such as a class or a function, there are two different ways you can specify that object: @@ -188,7 +211,7 @@ How to access settings .. highlight:: python -In a spider, the settings are available through ``self.settings``: +In a spider, settings are available through ``self.settings``: .. code-block:: python @@ -201,38 +224,142 @@ In a spider, the settings are available through ``self.settings``: .. note:: The ``settings`` attribute is set in the base Spider class after the spider - is initialized. If you want to use the settings before the initialization + is initialized. If you want to use settings before the initialization (e.g., in your spider's ``__init__()`` method), you'll need to override the :meth:`~scrapy.Spider.from_crawler` method. -Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings` -attribute of the Crawler that is passed to ``from_crawler`` method in -extensions, middlewares and item pipelines: +:ref:`Components ` can also :ref:`access settings +`. + +The ``settings`` object can be used like a :class:`dict` (e.g. +``settings["LOG_ENABLED"]``). However, to support non-string setting values, +which may be passed from the command line as strings, it is recommended to use +one of the methods provided by the :class:`~scrapy.settings.Settings` API. + + +.. _component-priority-dictionaries: + +Component priority dictionaries +=============================== + +A **component priority dictionary** is a :class:`dict` where keys are +:ref:`components ` and values are component priorities. For +example: + +.. skip: next +.. code-block:: python + + { + "path.to.ComponentA": None, + ComponentB: 100, + } + +A component can be specified either as a class object or through an import +path. + +.. warning:: Component priority dictionaries are regular :class:`dict` objects. + Be careful not to define the same component more than once, e.g. with + different import path strings or defining both an import path and a + :class:`type` object. + +A priority can be an :class:`int` or :data:`None`. + +A component with priority 1 goes *before* a component with priority 2. What +going before entails, however, depends on the corresponding setting. For +example, in the :setting:`DOWNLOADER_MIDDLEWARES` setting, components have +their +:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request` +method executed before that of later components, but have their +:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response` +method executed after that of later components. + +A component with priority :data:`None` is disabled. + +Some component priority dictionaries get merged with some built-in value. For +example, :setting:`DOWNLOADER_MIDDLEWARES` is merged with +:setting:`DOWNLOADER_MIDDLEWARES_BASE`. This is where :data:`None` comes in +handy, allowing you to disable a component from the base setting in the regular +setting: .. code-block:: python - class MyExtension: - def __init__(self, log_is_enabled=False): - if log_is_enabled: - print("log is enabled!") + DOWNLOADER_MIDDLEWARES = { + "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": None, + } - @classmethod - def from_crawler(cls, crawler): - settings = crawler.settings - return cls(settings.getbool("LOG_ENABLED")) -The settings object can be used like a dict (e.g., -``settings['LOG_ENABLED']``), but it's usually preferred to extract the setting -in the format you need it to avoid type errors, using one of the methods -provided by the :class:`~scrapy.settings.Settings` API. +Special settings +================ -Rationale for setting names -=========================== +The following settings work slightly differently than all other settings. -Setting names are usually prefixed with the component that they configure. For -example, proper setting names for a fictional robots.txt extension would be -``ROBOTSTXT_ENABLED``, ``ROBOTSTXT_OBEY``, ``ROBOTSTXT_CACHEDIR``, etc. +.. _pre-crawler-settings: +Pre-crawler settings +-------------------- + +**Pre-crawler settings** are settings used before the +:class:`~scrapy.crawler.Crawler` object is created. + +These settings cannot be :ref:`set from a spider `. + +These settings are: + +- :setting:`TWISTED_REACTOR_ENABLED` +- :setting:`SPIDER_LOADER_CLASS` and settings used by the corresponding + spider loader class, e.g. :setting:`SPIDER_MODULES` and + :setting:`SPIDER_LOADER_WARN_ONLY` for the default spider loader class. + +.. _reactor-settings: + +Reactor settings +---------------- + +**Reactor settings** are settings tied to the :doc:`Twisted reactor +`. + +These settings can be defined from a spider. However, because only 1 reactor +can be used per process, these settings cannot use a different value per spider +when :ref:`running multiple spiders in the same process +`. + +In general, if different spiders define different values, the first defined +value is used. However, if two spiders request a different reactor, an +exception is raised. + +These settings are: + +- :setting:`ASYNCIO_EVENT_LOOP` (not possible to set per-spider when using + :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below) + +- :setting:`TWISTED_DNS_RESOLVER` and settings used by the corresponding + component, e.g. :setting:`DNSCACHE_ENABLED`, :setting:`DNSCACHE_SIZE` + and :setting:`DNS_TIMEOUT` for the default one. + +- :setting:`REACTOR_THREADPOOL_MAXSIZE` + +- :setting:`TWISTED_REACTOR` (ignored when using + :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below) + +:setting:`ASYNCIO_EVENT_LOOP` and :setting:`TWISTED_REACTOR` are used upon +installing the reactor. The rest of the settings are applied when starting +the reactor. + +There is an additional restriction for :setting:`TWISTED_REACTOR` and +:setting:`ASYNCIO_EVENT_LOOP` when using +:class:`~scrapy.crawler.AsyncCrawlerProcess`: when this class is instantiated, +it installs :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, +ignoring the value of :setting:`TWISTED_REACTOR` and using the value of +:setting:`ASYNCIO_EVENT_LOOP` that was passed to +:meth:`AsyncCrawlerProcess.__init__() +`. If a different value for +:setting:`TWISTED_REACTOR` or :setting:`ASYNCIO_EVENT_LOOP` is provided later, +e.g. in :ref:`per-spider settings `, an exception will be +raised. + +All of these settings, except for :setting:`ASYNCIO_EVENT_LOOP`, are only used +when the Twisted reactor is used, i.e. when :setting:`TWISTED_REACTOR_ENABLED` +is ``True``. .. _topics-settings-ref: @@ -288,7 +415,7 @@ The AWS security token used by code that requires access to `Amazon Web services such as the :ref:`S3 feed storage backend `, when using `temporary security credentials`_. -.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys +.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html .. setting:: AWS_ENDPOINT_URL @@ -359,7 +486,7 @@ Note that the event loop class must inherit from :class:`asyncio.AbstractEventLo BOT_NAME -------- -Default: ``'scrapybot'`` +Default: ```` (:ref:`fallback `: ``'scrapybot'``) The name of the bot implemented by this Scrapy project (also known as the project name). This name will be used for the logging too. @@ -392,7 +519,7 @@ performed by the Scrapy downloader. CONCURRENT_REQUESTS_PER_DOMAIN ------------------------------ -Default: ``8`` +Default: ``1`` (:ref:`fallback `: ``8``) The maximum number of concurrent (i.e. simultaneous) requests that will be performed to any single domain. @@ -401,23 +528,38 @@ See also: :ref:`topics-autothrottle` and its :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` option. -.. setting:: CONCURRENT_REQUESTS_PER_IP +.. setting:: DEFAULT_DROPITEM_LOG_LEVEL -CONCURRENT_REQUESTS_PER_IP +DEFAULT_DROPITEM_LOG_LEVEL -------------------------- -Default: ``0`` +Default: ``"WARNING"`` -The maximum number of concurrent (i.e. simultaneous) requests that will be -performed to any single IP. If non-zero, the -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is ignored, and this one is -used instead. In other words, concurrency limits will be applied per IP, not -per domain. +Default :ref:`log level ` of messages about dropped items. -This setting also affects :setting:`DOWNLOAD_DELAY` and -:ref:`topics-autothrottle`: if :setting:`CONCURRENT_REQUESTS_PER_IP` -is non-zero, download delay is enforced per IP, not per domain. +When an item is dropped by raising :exc:`scrapy.exceptions.DropItem` from the +:func:`process_item` method of an :ref:`item pipeline `, +a message is logged, and by default its log level is the one configured in this +setting. +You may specify this log level as an integer (e.g. ``20``), as a log level +constant (e.g. ``logging.INFO``) or as a string with the name of a log level +constant (e.g. ``"INFO"``). + +When writing an item pipeline, you can force a different log level by setting +:attr:`scrapy.exceptions.DropItem.log_level` in your +:exc:`scrapy.exceptions.DropItem` exception. For example: + +.. code-block:: python + + from scrapy.exceptions import DropItem + + + class MyPipeline: + def process_item(self, item): + if not item.get("price"): + raise DropItem("Missing price data", log_level="INFO") + return item .. setting:: DEFAULT_ITEM_CLASS @@ -513,6 +655,13 @@ Default: ``True`` Whether to enable DNS in-memory cache. +.. note:: + This setting is only used by + :class:`~scrapy.resolver.CachingThreadedResolver` and + :class:`~scrapy.resolver.CachingHostnameResolver`. It has no effect when + :setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect + either when :setting:`DNS_RESOLVER` is set to a different resolver. + .. setting:: DNSCACHE_SIZE DNSCACHE_SIZE @@ -520,23 +669,25 @@ DNSCACHE_SIZE Default: ``10000`` -DNS in-memory cache size. +DNS in-memory cache size, see :setting:`DNSCACHE_ENABLED`. -.. setting:: DNS_RESOLVER +.. setting:: TWISTED_DNS_RESOLVER -DNS_RESOLVER ------------- - -.. versionadded:: 2.0 +TWISTED_DNS_RESOLVER +-------------------- Default: ``'scrapy.resolver.CachingThreadedResolver'`` -The class to be used to resolve DNS names. The default ``scrapy.resolver.CachingThreadedResolver`` -supports specifying a timeout for DNS requests via the :setting:`DNS_TIMEOUT` setting, -but works only with IPv4 addresses. Scrapy provides an alternative resolver, +The class to be used by Twisted to resolve DNS names. The default +``scrapy.resolver.CachingThreadedResolver`` supports specifying a timeout for +DNS requests via the :setting:`DNS_TIMEOUT` setting, but works only with IPv4 +addresses. Scrapy provides an alternative resolver, ``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not take the :setting:`DNS_TIMEOUT` setting into account. +.. note:: + This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. + .. setting:: DNS_TIMEOUT DNS_TIMEOUT @@ -546,6 +697,12 @@ Default: ``60`` Timeout for processing of DNS queries in seconds. Float is supported. +.. note:: + This setting is only used by + :class:`~scrapy.resolver.CachingThreadedResolver`. It has no effect when + :setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect + either when :setting:`DNS_RESOLVER` is set to a different resolver. + .. setting:: DOWNLOADER DOWNLOADER @@ -555,52 +712,6 @@ Default: ``'scrapy.core.downloader.Downloader'`` The downloader to use for crawling. -.. setting:: DOWNLOADER_HTTPCLIENTFACTORY - -DOWNLOADER_HTTPCLIENTFACTORY ----------------------------- - -Default: ``'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'`` - -Defines a Twisted ``protocol.ClientFactory`` class to use for HTTP/1.0 -connections (for ``HTTP10DownloadHandler``). - -.. note:: - - HTTP/1.0 is rarely used nowadays so you can safely ignore this setting, - unless you really want to use HTTP/1.0 and override - :setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly, - i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``. - -.. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY - -DOWNLOADER_CLIENTCONTEXTFACTORY -------------------------------- - -Default: ``'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'`` - -Represents the classpath to the ContextFactory to use. - -Here, "ContextFactory" is a Twisted term for SSL/TLS contexts, defining -the TLS/SSL protocol version to use, whether to do certificate verification, -or even enable client-side authentication (and various other things). - -.. note:: - - Scrapy default context factory **does NOT perform remote server - certificate verification**. This is usually fine for web scraping. - - If you do need remote server certificate verification enabled, - Scrapy also has another context factory class that you can set, - ``'scrapy.core.downloader.contextfactory.BrowserLikeContextFactory'``, - which uses the platform's certificates to validate remote endpoints. - -If you do use a custom ContextFactory, make sure its ``__init__`` method -accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping -:setting:`DOWNLOADER_CLIENT_TLS_METHOD`), a ``tls_verbose_logging`` -parameter (``bool``) and a ``tls_ciphers`` parameter (see -:setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`). - .. setting:: DOWNLOADER_CLIENT_TLS_CIPHERS DOWNLOADER_CLIENT_TLS_CIPHERS @@ -608,8 +719,8 @@ DOWNLOADER_CLIENT_TLS_CIPHERS Default: ``'DEFAULT'`` -Use this setting to customize the TLS/SSL ciphers used by the default -HTTP/1.1 downloader. +Use this setting to customize the TLS/SSL ciphers used by the HTTPS download +handler. The setting should contain a string in the `OpenSSL cipher list format`_, these ciphers will be used as client ciphers. Changing this setting may be @@ -617,28 +728,59 @@ necessary to access certain HTTPS websites: for example, you may need to use ``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a specific cipher that is not included in ``DEFAULT`` if a website requires it. -.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT +.. _OpenSSL cipher list format: https://docs.openssl.org/master/man1/openssl-ciphers/#cipher-list-format -.. setting:: DOWNLOADER_CLIENT_TLS_METHOD +.. note:: -DOWNLOADER_CLIENT_TLS_METHOD ----------------------------- + Handling of this setting needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. It's currently unsupported by + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. -Default: ``'TLS'`` +.. setting:: DOWNLOAD_TLS_MAX_VERSION -Use this setting to customize the TLS/SSL method used by the default -HTTP/1.1 downloader. +DOWNLOAD_TLS_MAX_VERSION +------------------------ -This setting must be one of these string values: +Default: ``None`` -- ``'TLS'``: maps to OpenSSL's ``TLS_method()`` (a.k.a ``SSLv23_method()``), - which allows protocol negotiation, starting from the highest supported - by the platform; **default, recommended** -- ``'TLSv1.0'``: this value forces HTTPS connections to use TLS version 1.0 ; - set this if you want the behavior of Scrapy<1.1 -- ``'TLSv1.1'``: forces TLS version 1.1 -- ``'TLSv1.2'``: forces TLS version 1.2 +Use this setting to change the maximum version of the TLS protocol allowed to +be used by Scrapy. +This setting must be either ``None``, in which case it doesn't affect the +version selection, or one of these string values: + +- ``'TLSv1.0'`` +- ``'TLSv1.1'`` +- ``'TLSv1.2'`` +- ``'TLSv1.3'`` + +The range of allowed TLS versions advertised by Scrapy when making TLS +connections will depend on the TLS implementation defaults and the values of +:setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION`. +It's possible to re-enable versions that are supported by the TLS +implementation but disabled by default by adjusting these settings, but it's +impossible to enable unsupported ones, such as any versions below 1.2 in many +modern environments. + +.. note:: + + Handling of this setting needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. Additionally, the set of supported TLS versions + depends on the TLS implementation being used by the handler. + +.. setting:: DOWNLOAD_TLS_MIN_VERSION + +DOWNLOAD_TLS_MIN_VERSION +------------------------ + +Default: ``None`` + +Use this setting to change the minimum version of the TLS protocol allowed to +be used by Scrapy. + +See :setting:`DOWNLOAD_TLS_MAX_VERSION` for the details and limitations. .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING @@ -649,17 +791,21 @@ Default: ``False`` Setting this to ``True`` will enable DEBUG level messages about TLS connection parameters after establishing HTTPS connections. The kind of information logged -depends on the versions of OpenSSL and pyOpenSSL. +depends on the implementation of the download handler and the versions of +the TLS-related libraries. -This setting is only used for the default -:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`. +.. note:: + + Handling of this setting needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. .. setting:: DOWNLOADER_MIDDLEWARES DOWNLOADER_MIDDLEWARES ---------------------- -Default:: ``{}`` +Default: ``{}`` A dict containing the downloader middlewares enabled in your project, and their orders. For more info see :ref:`topics-downloader-middleware-setting`. @@ -674,13 +820,13 @@ Default: .. code-block:: python { + "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50, "scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware": 100, "scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware": 300, "scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware": 350, "scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400, "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500, "scrapy.downloadermiddlewares.retry.RetryMiddleware": 550, - "scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware": 560, "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580, "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590, "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600, @@ -710,7 +856,7 @@ Whether to enable downloader stats collection. DOWNLOAD_DELAY -------------- -Default: ``0`` +Default: ``1`` (:ref:`fallback `: ``0``) Minimum seconds to wait between 2 consecutive requests to the same domain. @@ -725,9 +871,6 @@ every 10 seconds:: This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY` setting, which is enabled by default. -When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced -per IP address instead of per domain. - Note that :setting:`DOWNLOAD_DELAY` can lower the effective per-domain concurrency below :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. If the response time of a domain is lower than :setting:`DOWNLOAD_DELAY`, the effective @@ -747,6 +890,45 @@ It is also possible to change this setting per domain, although it requires non-trivial code. See the implementation of the :ref:`AutoThrottle ` extension for an example. +.. setting:: DOWNLOAD_BIND_ADDRESS + +DOWNLOAD_BIND_ADDRESS +--------------------- + +Default: ``None`` + +The default local outgoing address for download-handler connections. + +This setting can be either: + +- a host address as a string (e.g. ``"127.0.0.2"``), in which case the local + port is chosen automatically, or + +- a ``(host, port)`` tuple (e.g. ``("127.0.0.2", 50000)``) to bind to both a + specific local interface and a specific local port. + +For example: + +.. code-block:: python + + # Bind to this local address + DOWNLOAD_BIND_ADDRESS = "127.0.0.2" + +.. code-block:: python + + # Bind to this local address and local port + DOWNLOAD_BIND_ADDRESS = ("127.0.0.2", 5000) + +If set, built-in HTTP download handlers use this value by default. +Set the :reqmeta:`bindaddress` request meta key to override it for a specific +request. + +.. note:: + + Handling of this setting needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. Specifying the port is unsupported by + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. .. setting:: DOWNLOAD_HANDLERS @@ -755,7 +937,9 @@ DOWNLOAD_HANDLERS Default: ``{}`` -A dict containing the request downloader handlers enabled in your project. +A dict containing the :ref:`download handlers ` +enabled in your project. + See :setting:`DOWNLOAD_HANDLERS_BASE` for example format. .. setting:: DOWNLOAD_HANDLERS_BASE @@ -770,16 +954,30 @@ Default: { "data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler", "file": "scrapy.core.downloader.handlers.file.FileDownloadHandler", - "http": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", - "https": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", + "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", + "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "s3": "scrapy.core.downloader.handlers.s3.S3DownloadHandler", "ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler", } +(when :setting:`TWISTED_REACTOR_ENABLED` is ``True``) -A dict containing the request download handlers enabled by default in Scrapy. -You should never modify this setting in your project, modify -:setting:`DOWNLOAD_HANDLERS` instead. +.. code-block:: python + + { + "data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler", + "file": "scrapy.core.downloader.handlers.file.FileDownloadHandler", + "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + "s3": "scrapy.core.downloader.handlers.s3.S3DownloadHandler", + "ftp": None, + } + +(when :setting:`TWISTED_REACTOR_ENABLED` is ``False``) + +A dict containing the :ref:`download handlers ` +enabled by default in Scrapy. You should never modify this setting in your +project, modify :setting:`DOWNLOAD_HANDLERS` instead. You can disable any of these download handlers by assigning ``None`` to their URI scheme in :setting:`DOWNLOAD_HANDLERS`. E.g., to disable the built-in FTP @@ -791,51 +989,11 @@ handler (without replacement), place this in your ``settings.py``: "ftp": None, } -.. _http2: - -The default HTTPS handler uses HTTP/1.1. To use HTTP/2: - -#. Install ``Twisted[http2]>=17.9.0`` to install the packages required to - enable HTTP/2 support in Twisted. - -#. Update :setting:`DOWNLOAD_HANDLERS` as follows: - - .. code-block:: python - - DOWNLOAD_HANDLERS = { - "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler", - } - -.. warning:: - - HTTP/2 support in Scrapy is experimental, and not yet recommended for - production environments. Future Scrapy versions may introduce related - changes without a deprecation period or warning. - -.. note:: - - Known limitations of the current HTTP/2 implementation of Scrapy include: - - - No support for HTTP/2 Cleartext (h2c), since no major browser supports - HTTP/2 unencrypted (refer `http2 faq`_). - - - No setting to specify a maximum `frame size`_ larger than the default - value, 16384. Connections to servers that send a larger frame will - fail. - - - No support for `server pushes`_, which are ignored. - - - No support for the :signal:`bytes_received` and - :signal:`headers_received` signals. - -.. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2 -.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption -.. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2 .. setting:: DOWNLOAD_SLOTS DOWNLOAD_SLOTS ----------------- +-------------- Default: ``{}`` @@ -868,45 +1026,66 @@ The amount of time (in secs) that the downloader will wait before timing out. .. note:: - This timeout can be set per spider using :attr:`download_timeout` - spider attribute and per-request using :reqmeta:`download_timeout` - Request.meta key. + This timeout can be per-request using the :reqmeta:`download_timeout` + :attr:`.Request.meta` key. + +.. note:: + + Handling of this setting needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. .. setting:: DOWNLOAD_MAXSIZE +.. reqmeta:: download_maxsize DOWNLOAD_MAXSIZE ---------------- -Default: ``1073741824`` (1024MB) +Default: ``1073741824`` (1 GiB) -The maximum response size (in bytes) that downloader will download. +The maximum response body size (in bytes) allowed. Bigger responses are +aborted and ignored. -If you want to disable it set to 0. +This applies both before and after compression. If decompressing a response +body would exceed this limit, decompression is aborted and the response is +ignored. -.. reqmeta:: download_maxsize +Use ``0`` to disable this limit. .. note:: - This size can be set per spider using :attr:`download_maxsize` - spider attribute and per-request using :reqmeta:`download_maxsize` - Request.meta key. + This limit can be set per-request using the :reqmeta:`download_maxsize` + :attr:`.Request.meta` key. + +.. note:: + + Checking responses before decompressing them needs to be implemented inside + the :ref:`download handler `, so it's not + guaranteed to be supported by all 3rd-party handlers. .. setting:: DOWNLOAD_WARNSIZE +.. reqmeta:: download_warnsize DOWNLOAD_WARNSIZE ----------------- -Default: ``33554432`` (32MB) +Default: ``33554432`` (32 MiB) -The response size (in bytes) that downloader will start to warn. +If the size of a response exceeds this value, before or after compression, a +warning will be logged about it. -If you want to disable it set to 0. +Use ``0`` to disable this limit. .. note:: - This size can be set per spider using :attr:`download_warnsize` - spider attribute and per-request using :reqmeta:`download_warnsize` - Request.meta key. + This limit can be set per-request using the :reqmeta:`download_warnsize` + :attr:`.Request.meta` key. + +.. note:: + + Checking responses before decompressing them needs to be implemented inside + the :ref:`download handler `, so it's not + guaranteed to be supported by all 3rd-party handlers. .. setting:: DOWNLOAD_FAIL_ON_DATALOSS @@ -915,12 +1094,12 @@ DOWNLOAD_FAIL_ON_DATALOSS Default: ``True`` -Whether or not to fail on broken responses, that is, declared -``Content-Length`` does not match content sent by the server or chunked -response was not properly finish. If ``True``, these responses raise a -``ResponseFailed([_DataLoss])`` error. If ``False``, these responses -are passed through and the flag ``dataloss`` is added to the response, i.e.: -``'dataloss' in response.flags`` is ``True``. +Whether or not to fail on broken responses, that is, when the declared +``Content-Length`` does not match content sent by the server or a chunked +response was not properly finished. If ``True``, these responses raise a +:exc:`~scrapy.exceptions.ResponseDataLossError` exception. If ``False``, these +responses are passed through and the flag ``dataloss`` is added to the +response, i.e.: ``'dataloss' in response.flags`` is ``True``. Optionally, this can be set per-request basis by using the :reqmeta:`download_fail_on_dataloss` Request.meta key to ``False``. @@ -932,17 +1111,42 @@ Optionally, this can be set per-request basis by using the corruption. It is up to the user to decide if it makes sense to process broken responses considering they may contain partial or incomplete content. If :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``, - the ``ResponseFailed([_DataLoss])`` failure will be retried as usual. + the :exc:`~scrapy.exceptions.ResponseDataLossError` failure will be retried + as usual. + +.. note:: + + Handling of this setting needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. .. warning:: This setting is ignored by the :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` - download handler (see :setting:`DOWNLOAD_HANDLERS`). In case of a data loss + :ref:`download handler `. In case of a data loss error, the corresponding HTTP/2 connection may be corrupted, affecting other requests that use the same connection; hence, a ``ResponseFailed([InvalidBodyLengthError])`` failure is always raised for every request that was using that connection. +.. setting:: DOWNLOAD_VERIFY_CERTIFICATES + +DOWNLOAD_VERIFY_CERTIFICATES +---------------------------- + +Default: ``False`` + +Whether the HTTPS download handlers should verify the server TLS certificate +when making a request and abort the request if the verification fails. + +.. note:: + + Handling of this setting needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. The exact behavior of a handler (e.g. whether + certificate problems are logged when this setting is set to ``False``) + depends on its implementation. + .. setting:: DUPEFILTER_CLASS DUPEFILTER_CLASS @@ -952,15 +1156,79 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'`` The class used to detect and filter duplicate requests. -The default (``RFPDupeFilter``) filters based on the +The default, :class:`~scrapy.dupefilters.RFPDupeFilter`, filters based on the :setting:`REQUEST_FINGERPRINTER_CLASS` setting. -You can disable filtering of duplicate requests by setting -:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``. -Be very careful about this however, because you can get into crawling loops. -It's usually a better idea to set the ``dont_filter`` parameter to -``True`` on the specific :class:`~scrapy.Request` that should not be -filtered. +To change how duplicates are checked, you can point :setting:`DUPEFILTER_CLASS` +to a custom subclass of :class:`~scrapy.dupefilters.RFPDupeFilter` that +overrides its ``__init__`` method to use a :ref:`different request +fingerprinting class `. For example: + +.. code-block:: python + + from scrapy.dupefilters import RFPDupeFilter + from scrapy.utils.request import fingerprint + + + class CustomRequestFingerprinter: + def fingerprint(self, request): + return fingerprint(request, include_headers=["X-ID"]) + + + class CustomDupeFilter(RFPDupeFilter): + + def __init__(self, path=None, debug=False, *, fingerprinter=None): + super().__init__( + path=path, debug=debug, fingerprinter=CustomRequestFingerprinter() + ) + +To disable duplicate request filtering set :setting:`DUPEFILTER_CLASS` to +``'scrapy.dupefilters.BaseDupeFilter'``. Note that not filtering out duplicate +requests may cause crawling loops. It is usually better to set +the ``dont_filter`` parameter to ``True`` on the ``__init__`` method of a +specific :class:`~scrapy.Request` object that should not be filtered out. + +A class assigned to :setting:`DUPEFILTER_CLASS` must implement the following +interface:: + + class MyDupeFilter: + + @classmethod + def from_crawler(cls, crawler): + """Returns an instance of this duplicate request filtering class + based on the current Crawler instance.""" + return cls() + + def request_seen(self, request): + """Returns ``True`` if *request* is a duplicate of another request + seen in a previous call to :meth:`request_seen`, or ``False`` + otherwise.""" + return False + + def open(self): + """Called before the spider opens. It may return a deferred.""" + pass + + def close(self, reason): + """Called before the spider closes. It may return a deferred.""" + pass + + def log(self, request, spider): + """Logs that a request has been filtered out. + + It is called right after a call to :meth:`request_seen` that + returns ``True``. + + If :meth:`request_seen` always returns ``False``, such as in the + case of :class:`~scrapy.dupefilters.BaseDupeFilter`, this method + may be omitted. + """ + pass + +.. autoclass:: scrapy.dupefilters.BaseDupeFilter + +.. autoclass:: scrapy.dupefilters.RFPDupeFilter + .. setting:: DUPEFILTER_DEBUG @@ -988,9 +1256,10 @@ command will prefer it over the default setting. EXTENSIONS ---------- -Default:: ``{}`` +Default: ``{}`` -A dict containing the extensions enabled in your project, and their orders. +:ref:`Component priority dictionary ` of +enabled extensions. See :ref:`topics-extensions`. .. setting:: EXTENSIONS_BASE @@ -1035,7 +1304,27 @@ FEED_STORAGE_GCS_ACL -------------------- The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage `. -For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation `_. +For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation `_. + +.. setting:: FORCE_CRAWLER_PROCESS + +FORCE_CRAWLER_PROCESS +--------------------- + +Default: ``False`` + +If ``False``, :ref:`Scrapy commands that need a CrawlerProcess +` will decide between using +:class:`scrapy.crawler.AsyncCrawlerProcess` and +:class:`scrapy.crawler.CrawlerProcess` based on the value of the +:setting:`TWISTED_REACTOR` setting, but ignoring its value in :ref:`per-spider +settings `. + +If ``True``, these commands will always use +:class:`~scrapy.crawler.CrawlerProcess`. + +Set this to ``True`` if you want to set :setting:`TWISTED_REACTOR` to a +non-default value in :ref:`per-spider settings `. .. setting:: FTP_PASSIVE_MODE @@ -1046,6 +1335,12 @@ Default: ``True`` Whether or not to use passive mode when initiating FTP transfers. +.. note:: + + Handling of this setting needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. + .. reqmeta:: ftp_password .. setting:: FTP_PASSWORD @@ -1063,7 +1358,13 @@ in ``Request`` meta. some FTP servers explicitly ask for the user's e-mail address and will not allow login with the "guest" password. -.. _RFC 1635: https://tools.ietf.org/html/rfc1635 +.. _RFC 1635: https://datatracker.ietf.org/doc/html/rfc1635 + +.. note:: + + Handling of this setting needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. .. reqmeta:: ftp_user .. setting:: FTP_USER @@ -1076,6 +1377,12 @@ Default: ``"anonymous"`` The username to use for FTP connections when there is no ``"ftp_user"`` in ``Request`` meta. +.. note:: + + Handling of this setting needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. + .. setting:: GCS_PROJECT_ID GCS_PROJECT_ID @@ -1115,6 +1422,7 @@ Default: ``{}`` A dict containing the pipelines enabled by default in Scrapy. You should never modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead. + .. setting:: JOBDIR JOBDIR @@ -1125,6 +1433,7 @@ Default: ``None`` A string indicating the directory for storing the state of a crawl when :ref:`pausing and resuming crawls `. + .. setting:: LOG_ENABLED LOG_ENABLED @@ -1225,6 +1534,25 @@ Default: ``False`` If ``True``, the logs will just contain the root path. If it is set to ``False`` then it displays the component responsible for the log output +.. setting:: LOG_VERSIONS + +LOG_VERSIONS +------------ + +Default: ``["lxml", "libxml2", "cssselect", "parsel", "w3lib", "Twisted", "Python", "pyOpenSSL", "cryptography", "Platform"]`` + +Logs the installed versions of the specified items. + +An item can be any installed Python package. + +The following special items are also supported: + +- ``libxml2`` + +- ``Platform`` (:func:`platform.platform`) + +- ``Python`` + .. setting:: LOGSTATS_INTERVAL LOGSTATS_INTERVAL @@ -1266,13 +1594,12 @@ MEMUSAGE_ENABLED Default: ``True`` -Scope: ``scrapy.extensions.memusage`` +Scope: ``scrapy.extensions.memusage.MemoryUsage`` Whether to enable the memory usage extension. This extension keeps track of a peak memory used by the process (it writes it to stats). It can also optionally shutdown the Scrapy process when it exceeds a memory limit -(see :setting:`MEMUSAGE_LIMIT_MB`), and notify by email when that happened -(see :setting:`MEMUSAGE_NOTIFY_MAIL`). +(see :setting:`MEMUSAGE_LIMIT_MB`). See :ref:`topics-extensions-ref-memusage`. @@ -1283,10 +1610,11 @@ MEMUSAGE_LIMIT_MB Default: ``0`` -Scope: ``scrapy.extensions.memusage`` +Scope: ``scrapy.extensions.memusage.MemoryUsage`` The maximum amount of memory to allow (in megabytes) before shutting down -Scrapy (if MEMUSAGE_ENABLED is True). If zero, no check will be performed. +Scrapy (if :setting:`MEMUSAGE_ENABLED` is ``True``). If zero, no check will be +performed. See :ref:`topics-extensions-ref-memusage`. @@ -1297,7 +1625,7 @@ MEMUSAGE_CHECK_INTERVAL_SECONDS Default: ``60.0`` -Scope: ``scrapy.extensions.memusage`` +Scope: ``scrapy.extensions.memusage.MemoryUsage`` The :ref:`Memory usage extension ` checks the current memory usage, versus the limits set by @@ -1308,23 +1636,6 @@ This sets the length of these intervals, in seconds. See :ref:`topics-extensions-ref-memusage`. -.. setting:: MEMUSAGE_NOTIFY_MAIL - -MEMUSAGE_NOTIFY_MAIL --------------------- - -Default: ``False`` - -Scope: ``scrapy.extensions.memusage`` - -A list of emails to notify if the memory limit has been reached. - -Example:: - - MEMUSAGE_NOTIFY_MAIL = ['user@example.com'] - -See :ref:`topics-extensions-ref-memusage`. - .. setting:: MEMUSAGE_WARNING_MB MEMUSAGE_WARNING_MB @@ -1332,17 +1643,20 @@ MEMUSAGE_WARNING_MB Default: ``0`` -Scope: ``scrapy.extensions.memusage`` +Scope: ``scrapy.extensions.memusage.MemoryUsage`` -The maximum amount of memory to allow (in megabytes) before sending a warning -email notifying about it. If zero, no warning will be produced. +The maximum amount of memory to allow (in megabytes) before sending a +:signal:`memusage_warning_reached` signal (if :setting:`MEMUSAGE_ENABLED` is +``True``). If zero, no signal will be sent. + +See :ref:`topics-extensions-ref-memusage`. .. setting:: NEWSPIDER_MODULE NEWSPIDER_MODULE ---------------- -Default: ``''`` +Default: ``".spiders"`` (:ref:`fallback `: ``""``) Module where to create new spiders using the :command:`genspider` command. @@ -1401,9 +1715,7 @@ Adjust redirect request priority relative to original request: ROBOTSTXT_OBEY -------------- -Default: ``False`` - -Scope: ``scrapy.downloadermiddlewares.robotstxt`` +Default: ``True`` (:ref:`fallback `: ``False``) If enabled, Scrapy will respect robots.txt policies. For more information see :ref:`topics-dlmw-robots`. @@ -1441,10 +1753,10 @@ the user agent to use in the robots.txt file. SCHEDULER --------- -Default: ``'scrapy.core.scheduler.Scheduler'`` +Default: :class:`~scrapy.core.scheduler.Scheduler` -The scheduler class to be used for crawling. -See the :ref:`topics-scheduler` topic for details. +The scheduler class to be used for crawling. See :ref:`topics-scheduler` for +details. .. setting:: SCHEDULER_DEBUG @@ -1472,39 +1784,88 @@ SCHEDULER_DISK_QUEUE Default: ``'scrapy.squeues.PickleLifoDiskQueue'`` -Type of disk queue that will be used by scheduler. Other available types are -``scrapy.squeues.PickleFifoDiskQueue``, ``scrapy.squeues.MarshalFifoDiskQueue``, +Type of disk queue that will be used by the scheduler. Other available types +are ``scrapy.squeues.PickleFifoDiskQueue``, +``scrapy.squeues.MarshalFifoDiskQueue``, ``scrapy.squeues.MarshalLifoDiskQueue``. + .. setting:: SCHEDULER_MEMORY_QUEUE SCHEDULER_MEMORY_QUEUE ---------------------- + Default: ``'scrapy.squeues.LifoMemoryQueue'`` -Type of in-memory queue used by scheduler. Other available type is: +Type of in-memory queue used by the scheduler. Other available type is: ``scrapy.squeues.FifoMemoryQueue``. + .. setting:: SCHEDULER_PRIORITY_QUEUE SCHEDULER_PRIORITY_QUEUE ------------------------ -Default: ``'scrapy.pqueues.ScrapyPriorityQueue'`` -Type of priority queue used by the scheduler. Another available type is -``scrapy.pqueues.DownloaderAwarePriorityQueue``. -``scrapy.pqueues.DownloaderAwarePriorityQueue`` works better than -``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different -domains in parallel. But currently ``scrapy.pqueues.DownloaderAwarePriorityQueue`` -does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`. +Default: :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue` + +Type of priority queue used by the scheduler. + +Another available type is :class:`~scrapy.pqueues.ScrapyPriorityQueue`. + +:class:`~scrapy.pqueues.DownloaderAwarePriorityQueue` works better than +:class:`~scrapy.pqueues.ScrapyPriorityQueue` when you crawl many different +domains in parallel. + + +.. setting:: SCHEDULER_START_DISK_QUEUE + +SCHEDULER_START_DISK_QUEUE +-------------------------- + +Default: ``'scrapy.squeues.PickleFifoDiskQueue'`` + +Type of disk queue (see :setting:`JOBDIR`) that the :ref:`scheduler +` uses for :ref:`start requests `. + +For available choices, see :setting:`SCHEDULER_DISK_QUEUE`. + +.. queue-common-starts + +Use ``None`` or ``""`` to disable these separate queues entirely, and instead +have start requests share the same queues as other requests. + +.. note:: + + Disabling separate start request queues makes :ref:`start request order + ` unintuitive: start requests will be sent in order + only until :setting:`CONCURRENT_REQUESTS` is reached, then remaining start + requests will be sent in reverse order. + +.. queue-common-ends + + +.. setting:: SCHEDULER_START_MEMORY_QUEUE + +SCHEDULER_START_MEMORY_QUEUE +---------------------------- + +Default: ``'scrapy.squeues.FifoMemoryQueue'`` + +Type of in-memory queue that the :ref:`scheduler ` uses for +:ref:`start requests `. + +For available choices, see :setting:`SCHEDULER_MEMORY_QUEUE`. + +.. include:: settings.rst + :start-after: queue-common-starts + :end-before: queue-common-ends + .. setting:: SCRAPER_SLOT_MAX_ACTIVE_SIZE SCRAPER_SLOT_MAX_ACTIVE_SIZE ---------------------------- -.. versionadded:: 2.0 - Default: ``5_000_000`` Soft limit (in bytes) for response data being processed. @@ -1517,7 +1878,7 @@ Scrapy does not process new requests. SPIDER_CONTRACTS ---------------- -Default:: ``{}`` +Default: ``{}`` A dict containing the spider contracts enabled in your project, used for testing spiders. For more info see :ref:`topics-contracts`. @@ -1569,25 +1930,16 @@ SPIDER_LOADER_WARN_ONLY Default: ``False`` By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`, -it will fail loudly if there is any ``ImportError`` exception. +it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception. But you can choose to silence this exception and turn it into a simple warning by setting ``SPIDER_LOADER_WARN_ONLY = True``. -.. note:: - Some :ref:`scrapy commands ` run with this setting to ``True`` - already (i.e. they will only issue a warning and will not fail) - since they do not actually need to load spider classes to work: - :command:`scrapy runspider `, - :command:`scrapy settings `, - :command:`scrapy startproject `, - :command:`scrapy version `. - .. setting:: SPIDER_MIDDLEWARES SPIDER_MIDDLEWARES ------------------ -Default:: ``{}`` +Default: ``{}`` A dict containing the spider middlewares enabled in your project, and their orders. For more info see :ref:`topics-spider-middleware-setting`. @@ -1603,7 +1955,6 @@ Default: { "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50, - "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": 500, "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, @@ -1618,7 +1969,7 @@ the spider. For more info see :ref:`topics-spider-middleware-setting`. SPIDER_MODULES -------------- -Default: ``[]`` +Default: ``[".spiders"]`` (:ref:`fallback `: ``[]``) A list of modules where Scrapy will look for spiders. @@ -1650,22 +2001,12 @@ finishes. For more info see: :ref:`topics-stats`. -.. setting:: STATSMAILER_RCPTS - -STATSMAILER_RCPTS ------------------ - -Default: ``[]`` (empty list) - -Send Scrapy stats after spiders finish scraping. See -:class:`~scrapy.extensions.statsmailer.StatsMailer` for more info. - .. setting:: TELNETCONSOLE_ENABLED TELNETCONSOLE_ENABLED --------------------- -Default: ``True`` +Default: ``True`` (``False`` when :setting:`TWISTED_REACTOR_ENABLED` is ``False``) A boolean which specifies if the :ref:`telnet console ` will be enabled (provided its extension is also enabled). @@ -1684,22 +2025,51 @@ command. The project name must not conflict with the name of custom files or directories in the ``project`` subdirectory. +.. setting:: TWISTED_REACTOR_ENABLED + +TWISTED_REACTOR_ENABLED +----------------------- + +Default: ``True`` + +Whether to install and use the Twisted reactor. + +If this is set to ``True``, Scrapy will use the Twisted reactor and will +install one according to the :setting:`TWISTED_REACTOR` setting value when +appropriate (e.g. when running via :ref:`the command-line tool +`). This is the traditional mode of using Scrapy. + +If this is set to ``False``, Scrapy will use the asyncio event loop directly +and will not attempt to install or use a reactor. Features that require a +reactor won't be available, but Twisted APIs that don't require a reactor, +including :class:`~twisted.internet.defer.Deferred` and +:class:`~twisted.python.failure.Failure`, will still be available. On the other +hand, limitations related to Twisted reactors (such as not being able to start +a reactor in the same process where a reactor was previously started and +stopped) will not apply. This mode is currently experimental and may not be +suitable for production use. It may also not be supported by 3rd-party code. +See :ref:`asyncio-without-reactor` for more information about this mode. + +.. note:: This setting can't be set :ref:`per-spider `. + +.. versionadded:: 2.15.0 + .. setting:: TWISTED_REACTOR TWISTED_REACTOR --------------- -.. versionadded:: 2.0 - -Default: ``None`` +Default: ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"`` Import path of a given :mod:`~twisted.internet.reactor`. Scrapy will install this reactor if no other reactor is installed yet, such as when the ``scrapy`` CLI program is invoked or when using the +:class:`~scrapy.crawler.AsyncCrawlerProcess` class or the :class:`~scrapy.crawler.CrawlerProcess` class. -If you are using the :class:`~scrapy.crawler.CrawlerRunner` class, you also +If you are using the :class:`~scrapy.crawler.AsyncCrawlerRunner` class or the +:class:`~scrapy.crawler.CrawlerRunner` class, you also need to install the correct reactor manually. You can do that using :func:`~scrapy.utils.reactor.install_reactor`: @@ -1708,15 +2078,16 @@ need to install the correct reactor manually. You can do that using If a reactor is already installed, :func:`~scrapy.utils.reactor.install_reactor` has no effect. -:meth:`CrawlerRunner.__init__ ` raises -:exc:`Exception` if the installed reactor does not match the +:class:`~scrapy.crawler.AsyncCrawlerRunner` and other similar classes raise an +exception if the installed reactor does not match the :setting:`TWISTED_REACTOR` setting; therefore, having top-level :mod:`~twisted.internet.reactor` imports in project files and imported -third-party libraries will make Scrapy raise :exc:`Exception` when -it checks which reactor is installed. +third-party libraries will make Scrapy raise an exception when it checks which +reactor is installed. In order to use the reactor installed by Scrapy: +.. skip: next .. code-block:: python import scrapy @@ -1730,7 +2101,7 @@ In order to use the reactor installed by Scrapy: self.timeout = int(kwargs.pop("timeout", "60")) super(QuotesSpider, self).__init__(*args, **kwargs) - def start_requests(self): + async def start(self): reactor.callLater(self.timeout, self.stop) urls = ["https://quotes.toscrape.com/page/1"] @@ -1745,7 +2116,7 @@ In order to use the reactor installed by Scrapy: self.crawler.engine.close_spider(self, "timeout") -which raises :exc:`Exception`, becomes: +which raises an exception, becomes: .. code-block:: python @@ -1759,7 +2130,7 @@ which raises :exc:`Exception`, becomes: self.timeout = int(kwargs.pop("timeout", "60")) super(QuotesSpider, self).__init__(*args, **kwargs) - def start_requests(self): + async def start(self): from twisted.internet import reactor reactor.callLater(self.timeout, self.stop) @@ -1776,16 +2147,13 @@ which raises :exc:`Exception`, becomes: self.crawler.engine.close_spider(self, "timeout") -The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which -means that Scrapy will use the existing reactor if one is already installed, or -install the default reactor defined by Twisted for the current platform. This -is to maintain backward compatibility and avoid possible problems caused by -using a non-default reactor. +If this setting is set ``None``, Scrapy will use the existing reactor if one is +already installed, or install the default reactor defined by Twisted for the +current platform. -.. versionchanged:: 2.7 - The :command:`startproject` command now sets this setting to - ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated - ``settings.py`` file. +.. versionchanged:: 2.13 + The default value was changed from ``None`` to + ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``. For additional information, see :doc:`core/howto/choosing-reactor`. @@ -1811,7 +2179,7 @@ Use ``0`` to allow URLs of any length. The default value is copied from the `Microsoft Internet Explorer maximum URL length`_, even though this setting exists for different reasons. -.. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 +.. _Microsoft Internet Explorer maximum URL length: https://web.archive.org/web/20250206050143/https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 .. setting:: USER_AGENT @@ -1825,6 +2193,21 @@ also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and there is no overriding User-Agent header specified for the request. +.. setting:: WARN_ON_GENERATOR_RETURN_VALUE + +WARN_ON_GENERATOR_RETURN_VALUE +------------------------------ + +Default: ``True`` + +When enabled, Scrapy will warn if generator-based callback methods (like +``parse``) contain return statements with non-``None`` values. This helps detect +potential mistakes in spider development. + +Disable this setting to prevent syntax errors that may occur when dynamically +modifying generator function source code during runtime, skip AST parsing of +callback functions, or improve performance in auto-reloading development +environments. Settings documented elsewhere: ------------------------------ @@ -1835,6 +2218,4 @@ case to see how to enable and use them. .. settingslist:: .. _Amazon web services: https://aws.amazon.com/ -.. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search -.. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search .. _Google Cloud Storage: https://cloud.google.com/storage/ diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 4898843e4..5b826ed18 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -24,7 +24,7 @@ If you have `IPython`_ installed, the Scrapy shell will use it (instead of the standard Python console). The `IPython`_ console is much more powerful and provides smart auto-completion and colorized output, among other things. -We highly recommend you install `IPython`_, specially if you're working on +We highly recommend you install `IPython`_, especially if you're working on Unix systems (where `IPython`_ excels). See the `IPython installation guide`_ for more info. @@ -40,7 +40,7 @@ variable; or by defining it in your :ref:`scrapy.cfg `:: shell = bpython .. _IPython: https://ipython.org/ -.. _IPython installation guide: https://ipython.org/install.html +.. _IPython installation guide: https://ipython.org/install/ .. _bpython: https://bpython-interpreter.org/ Launch the shell @@ -111,7 +111,7 @@ Available Shortcuts Note, however, that this will create a temporary file in your computer, which won't be removed automatically. -.. _ tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/base +.. _ tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/base Available Scrapy objects ------------------------ @@ -142,8 +142,10 @@ Those objects are: Example of shell session ======================== +.. skip: start + Here's an example of a typical shell session where we start by scraping the -https://scrapy.org page, and then proceed to scrape the https://old.reddit.com/ +https://www.scrapy.org/ page, and then proceed to scrape the https://old.reddit.com/ page. Finally, we modify the (Reddit) request method to POST and re-fetch it getting an error. We end the session by typing Ctrl-D (in Unix systems) or Ctrl-Z in Windows. @@ -232,6 +234,8 @@ After that, we can start playing with the objects: 'X-Ua-Compatible': ['IE=edge'], 'X-Xss-Protection': ['1; mode=block']} +.. skip: end + .. _topics-shell-inspect-response: @@ -268,6 +272,8 @@ Here's an example of how you would call it from your spider: # Rest of parsing code. +.. skip: start + When you run the spider, you will get something similar to this:: 2014-01-23 17:48:31-0400 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) @@ -301,6 +307,8 @@ crawling:: 2014-01-23 17:50:03-0400 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) ... +.. skip: end + Note that you can't use the ``fetch`` shortcut here since the Scrapy engine is blocked by the shell. However, after you leave the shell, the spider will continue crawling where it stopped, as shown above. diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 9bfd1761c..d13733623 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -46,8 +46,8 @@ Here is a simple example showing how you can catch signals and perform some acti .. _signal-deferred: -Deferred signal handlers -======================== +Asynchronous signal handlers +============================ Some signals support returning :class:`~twisted.internet.defer.Deferred` or :term:`awaitable objects ` from their handlers, allowing @@ -57,9 +57,11 @@ operation to finish. Let's take an example using :ref:`coroutines `: +.. skip: next .. code-block:: python import scrapy + import treq class SignalSpider(scrapy.Spider): @@ -103,6 +105,7 @@ Built-in signals reference Here's the list of Scrapy built-in signals and their meaning. + Engine signals -------------- @@ -114,7 +117,7 @@ engine_started Sent when the Scrapy engine has started crawling. - This signal supports returning deferreds from its handlers. + This signal supports :ref:`asynchronous handlers `. .. note:: This signal may be fired *after* the :signal:`spider_opened` signal, depending on how the spider was started. So **don't** rely on this signal @@ -129,7 +132,23 @@ engine_stopped Sent when the Scrapy engine is stopped (for example, when a crawling process has finished). - This signal supports returning deferreds from its handlers. + This signal supports :ref:`asynchronous handlers `. + +scheduler_empty +~~~~~~~~~~~~~~~ + +.. signal:: scheduler_empty +.. function:: scheduler_empty() + + Sent whenever the engine asks for a pending request from the + :ref:`scheduler ` (i.e. calls its + :meth:`~scrapy.core.scheduler.BaseScheduler.next_request` method) and the + scheduler returns none. + + See :ref:`start-requests-lazy` for an example. + + This signal does not support :ref:`asynchronous handlers `. + Item signals ------------ @@ -151,7 +170,7 @@ item_scraped Sent when an item has been scraped, after it has passed all the :ref:`topics-item-pipeline` stages (without being dropped). - This signal supports returning deferreds from its handlers. + This signal supports :ref:`asynchronous handlers `. :param item: the scraped item :type item: :ref:`item object ` @@ -159,8 +178,9 @@ item_scraped :param spider: the spider which scraped the item :type spider: :class:`~scrapy.Spider` object - :param response: the response from where the item was scraped - :type response: :class:`~scrapy.http.Response` object + :param response: the response from where the item was scraped, or ``None`` + if it was yielded from :meth:`~scrapy.Spider.start`. + :type response: :class:`~scrapy.http.Response` | ``None`` item_dropped ~~~~~~~~~~~~ @@ -171,7 +191,7 @@ item_dropped Sent after an item has been dropped from the :ref:`topics-item-pipeline` when some stage raised a :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports returning deferreds from its handlers. + This signal supports :ref:`asynchronous handlers `. :param item: the item dropped from the :ref:`topics-item-pipeline` :type item: :ref:`item object ` @@ -179,8 +199,9 @@ item_dropped :param spider: the spider which scraped the item :type spider: :class:`~scrapy.Spider` object - :param response: the response from where the item was dropped - :type response: :class:`~scrapy.http.Response` object + :param response: the response from where the item was dropped, or ``None`` + if it was yielded from :meth:`~scrapy.Spider.start`. + :type response: :class:`~scrapy.http.Response` | ``None`` :param exception: the exception (which must be a :exc:`~scrapy.exceptions.DropItem` subclass) which caused the item @@ -196,13 +217,15 @@ item_error Sent when a :ref:`topics-item-pipeline` generates an error (i.e. raises an exception), except :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports returning deferreds from its handlers. + This signal supports :ref:`asynchronous handlers `. :param item: the item that caused the error in the :ref:`topics-item-pipeline` :type item: :ref:`item object ` - :param response: the response being processed when the exception was raised - :type response: :class:`~scrapy.http.Response` object + :param response: the response being processed when the exception was + raised, or ``None`` if it was yielded from + :meth:`~scrapy.Spider.start`. + :type response: :class:`~scrapy.http.Response` | ``None`` :param spider: the spider which raised the exception :type spider: :class:`~scrapy.Spider` object @@ -210,6 +233,7 @@ item_error :param failure: the exception raised :type failure: twisted.python.failure.Failure + Spider signals -------------- @@ -222,7 +246,7 @@ spider_closed Sent after a spider has been closed. This can be used to release per-spider resources reserved on :signal:`spider_opened`. - This signal supports returning deferreds from its handlers. + This signal supports :ref:`asynchronous handlers `. :param spider: the spider which has been closed :type spider: :class:`~scrapy.Spider` object @@ -246,7 +270,7 @@ spider_opened reserve per-spider resources, but can be used for any task that needs to be performed when a spider is opened. - This signal supports returning deferreds from its handlers. + This signal supports :ref:`asynchronous handlers `. :param spider: the spider which has been opened :type spider: :class:`~scrapy.Spider` object @@ -277,16 +301,16 @@ spider_idle accordingly (e.g. setting it to 'too_few_results' instead of 'finished'). - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers `. :param spider: the spider which has gone idle :type spider: :class:`~scrapy.Spider` object -.. note:: Scheduling some requests in your :signal:`spider_idle` handler does - **not** guarantee that it can prevent the spider from being closed, - although it sometimes can. That's because the spider may still remain idle - if all the scheduled requests are rejected by the scheduler (e.g. filtered - due to duplication). + .. note:: Scheduling some requests in your :signal:`spider_idle` handler does + **not** guarantee that it can prevent the spider from being closed, + although it sometimes can. That's because the spider may still remain idle + if all the scheduled requests are rejected by the scheduler (e.g. filtered + due to duplication). spider_error ~~~~~~~~~~~~ @@ -296,7 +320,7 @@ spider_error Sent when a spider callback generates an error (i.e. raises an exception). - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers `. :param failure: the exception raised :type failure: twisted.python.failure.Failure @@ -315,12 +339,11 @@ feed_slot_closed Sent when a :ref:`feed exports ` slot is closed. - This signal supports returning deferreds from its handlers. + This signal supports :ref:`asynchronous handlers `. :param slot: the slot closed :type slot: scrapy.extensions.feedexport.FeedSlot - feed_exporter_closed ~~~~~~~~~~~~~~~~~~~~ @@ -331,7 +354,19 @@ feed_exporter_closed during the handling of the :signal:`spider_closed` signal by the extension, after all feed exporting has been handled. - This signal supports returning deferreds from its handlers. + This signal supports :ref:`asynchronous handlers `. + +memusage_warning_reached +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. signal:: memusage_warning_reached + +.. function:: memusage_warning_reached() + + Sent by the :class:`~scrapy.extensions.memusage.MemoryUsage` extension when the + memory usage reaches the warning threshold (:setting:`MEMUSAGE_WARNING_MB`). + + This signal does not support :ref:`asynchronous handlers `. Request signals @@ -343,10 +378,17 @@ request_scheduled .. signal:: request_scheduled .. function:: request_scheduled(request, spider) - Sent when the engine schedules a :class:`~scrapy.Request`, to be - downloaded later. + Sent when the engine is asked to schedule a :class:`~scrapy.Request`, to be + downloaded later, before the request reaches the :ref:`scheduler + `. - This signal does not support returning deferreds from its handlers. + Raise :exc:`~scrapy.exceptions.IgnoreRequest` to drop a request before it + reaches the scheduler. + + This signal does not support :ref:`asynchronous handlers `. + + .. versionadded:: 2.11.2 + Allow dropping requests with :exc:`~scrapy.exceptions.IgnoreRequest`. :param request: the request that reached the scheduler :type request: :class:`~scrapy.Request` object @@ -363,7 +405,7 @@ request_dropped Sent when a :class:`~scrapy.Request`, scheduled by the engine to be downloaded later, is rejected by the scheduler. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers `. :param request: the request that reached the scheduler :type request: :class:`~scrapy.Request` object @@ -379,7 +421,7 @@ request_reached_downloader Sent when a :class:`~scrapy.Request` reached downloader. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers `. :param request: the request that reached downloader :type request: :class:`~scrapy.Request` object @@ -393,12 +435,10 @@ request_left_downloader .. signal:: request_left_downloader .. function:: request_left_downloader(request, spider) - .. versionadded:: 2.0 - Sent when a :class:`~scrapy.Request` leaves the downloader, even in case of failure. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers `. :param request: the request that reached the downloader :type request: :class:`~scrapy.Request` object @@ -409,8 +449,6 @@ request_left_downloader bytes_received ~~~~~~~~~~~~~~ -.. versionadded:: 2.2 - .. signal:: bytes_received .. function:: bytes_received(data, request, spider) @@ -425,7 +463,7 @@ bytes_received exception. Please refer to the :ref:`topics-stop-response-download` topic for additional information and examples. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers `. :param data: the data received by the download handler :type data: :class:`bytes` object @@ -439,8 +477,6 @@ bytes_received headers_received ~~~~~~~~~~~~~~~~ -.. versionadded:: 2.5 - .. signal:: headers_received .. function:: headers_received(headers, body_length, request, spider) @@ -452,7 +488,7 @@ headers_received exception. Please refer to the :ref:`topics-stop-response-download` topic for additional information and examples. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers `. :param headers: the headers received by the download handler :type headers: :class:`scrapy.http.headers.Headers` object @@ -466,6 +502,7 @@ headers_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.Spider` object + Response signals ---------------- @@ -478,7 +515,7 @@ response_received Sent when the engine receives a new :class:`~scrapy.http.Response` from the downloader. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers `. :param response: the response received :type response: :class:`~scrapy.http.Response` object @@ -500,9 +537,9 @@ response_downloaded .. signal:: response_downloaded .. function:: response_downloaded(response, request, spider) - Sent by the downloader right after a ``HTTPResponse`` is downloaded. + Sent by the downloader right after a :class:`~scrapy.http.Response` is downloaded. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers `. :param response: the response downloaded :type response: :class:`~scrapy.http.Response` object diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 3f16efea5..99bbdf292 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -51,8 +51,8 @@ value. For example, if you want to disable the off-site middleware: .. code-block:: python SPIDER_MIDDLEWARES = { - "myproject.middlewares.CustomSpiderMiddleware": 543, - "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None, + "scrapy.spidermiddlewares.referer.RefererMiddleware": None, + "myproject.middlewares.CustomRefererSpiderMiddleware": 700, } Finally, keep in mind that some middlewares may need to be enabled through a @@ -63,18 +63,38 @@ particular setting. See each middleware documentation for more info. Writing your own spider middleware ================================== -Each spider middleware is a Python class that defines one or more of the -methods defined below. - -The main entry point is the ``from_crawler`` class method, which receives a -:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` -object gives you access, for example, to the :ref:`settings `. +Each spider middleware is a :ref:`component ` that defines +one or more of these methods: .. module:: scrapy.spidermiddlewares .. class:: SpiderMiddleware - .. method:: process_spider_input(response, spider) + .. method:: process_start(start: AsyncIterator[Any], /) -> AsyncIterator[Any] + :async: + + Iterate over the output of :meth:`~scrapy.Spider.start` or that + of the :meth:`process_start` method of an earlier spider middleware, + overriding it. For example: + + .. code-block:: python + + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request + + You may yield the same type of objects as :meth:`~scrapy.Spider.start`. + + To write spider middlewares that work on Scrapy versions lower than + 2.13, define also a synchronous ``process_start_requests()`` method + that returns an iterable. For example: + + .. code-block:: python + + def process_start_requests(self, start, spider): + yield from start + + .. method:: process_spider_input(response) This method is called for each response that goes through the spider middleware and into the spider, for processing. @@ -96,51 +116,31 @@ object gives you access, for example, to the :ref:`settings `. :param response: the response being processed :type response: :class:`~scrapy.http.Response` object - :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.Spider` object + .. method:: process_spider_output(response, result) + :async: + This method is an :term:`asynchronous generator` called with the + results from the spider after the spider has processed the response. - .. method:: process_spider_output(response, result, spider) - - This method is called with the results returned from the Spider, after - it has processed the response. - - :meth:`process_spider_output` must return an iterable of - :class:`~scrapy.Request` objects and :ref:`item objects - `. - - .. versionchanged:: 2.7 - This method may be defined as an :term:`asynchronous generator`, in - which case ``result`` is an :term:`asynchronous iterable`. - - Consider defining this method as an :term:`asynchronous generator`, - which will be a requirement in a future version of Scrapy. However, if - you plan on sharing your spider middleware with other people, consider - either :ref:`enforcing Scrapy 2.7 ` - as a minimum requirement of your spider middleware, or :ref:`making - your spider middleware universal ` so that - it works with Scrapy versions earlier than Scrapy 2.7. + .. seealso:: :ref:`universal-spider-middleware`. :param response: the response which generated this output from the spider :type response: :class:`~scrapy.http.Response` object - :param result: the result returned by the spider - :type result: an iterable of :class:`~scrapy.Request` objects and - :ref:`item objects ` + :param result: the results from the spider + :type result: an :term:`asynchronous iterable` of + :class:`~scrapy.Request` objects and :ref:`item objects + ` - :param spider: the spider whose result is being processed - :type spider: :class:`~scrapy.Spider` object + .. method:: process_spider_output_async(response, result) + :async: - .. method:: process_spider_output_async(response, result, spider) + Alternative name for :meth:`process_spider_output` used when + implementing a :ref:`universal spider middleware + `. - .. versionadded:: 2.7 - - If defined, this method must be an :term:`asynchronous generator`, - which will be called instead of :meth:`process_spider_output` if - ``result`` is an :term:`asynchronous iterable`. - - .. method:: process_spider_exception(response, exception, spider) + .. method:: process_spider_exception(response, exception) This method is called when a spider or :meth:`process_spider_output` method (from a previous spider middleware) raises an exception. @@ -165,44 +165,46 @@ object gives you access, for example, to the :ref:`settings `. :param exception: the exception raised :type exception: :exc:`Exception` object - :param spider: the spider which raised the exception - :type spider: :class:`~scrapy.Spider` object - .. method:: process_start_requests(start_requests, spider) +.. _universal-spider-middleware: - This method is called with the start requests of the spider, and works - similarly to the :meth:`process_spider_output` method, except that it - doesn't have a response associated and must return only requests (not - items). +Universal spider middlewares +---------------------------- - It receives an iterable (in the ``start_requests`` parameter) and must - return another iterable of :class:`~scrapy.Request` objects. +In Scrapy 2.6.3 and lower, ``process_spider_output()`` must be a *synchronous* +generator. - .. note:: When implementing this method in your spider middleware, you - should always return an iterable (that follows the input one) and - not consume all ``start_requests`` iterator because it can be very - large (or even unbounded) and cause a memory overflow. The Scrapy - engine is designed to pull start requests while it has capacity to - process them, so the start requests iterator can be effectively - endless where there is some other condition for stopping the spider - (like a time limit or item/page count). +To support those versions and higher Scrapy versions in the same middleware, +rename your asynchronous :meth:`~SpiderMiddleware.process_spider_output` +method to :meth:`~SpiderMiddleware.process_spider_output_async`, and define a +synchronous ``process_spider_output()`` method to be used by 2.6.3 and lower +versions. - :param start_requests: the start requests - :type start_requests: an iterable of :class:`~scrapy.Request` +For example: - :param spider: the spider to whom the start requests belong - :type spider: :class:`~scrapy.Spider` object +.. code-block:: python - .. method:: from_crawler(cls, crawler) + class UniversalSpiderMiddleware: + async def process_spider_output_async(self, response, result): + async for r in result: + # ... do something with r + yield r - If present, this classmethod is called to create a middleware instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the middleware. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for middleware to - access them and hook its functionality into Scrapy. + def process_spider_output(self, response, result): + for r in result: + # ... do something with r + yield r - :param crawler: crawler that uses this middleware - :type crawler: :class:`~scrapy.crawler.Crawler` object +Base class for custom spider middlewares +---------------------------------------- + +Scrapy provides a base class for custom spider middlewares. It's not required +to use it but it can help with simplifying middleware implementations. + +.. module:: scrapy.spidermiddlewares.base + +.. autoclass:: BaseSpiderMiddleware + :members: .. _topics-spider-middleware-ref: @@ -313,42 +315,6 @@ Default: ``False`` Pass all responses, regardless of its status code. -OffsiteMiddleware ------------------ - -.. module:: scrapy.spidermiddlewares.offsite - :synopsis: Offsite Spider Middleware - -.. class:: OffsiteMiddleware - - Filters out Requests for URLs outside the domains covered by the spider. - - This middleware filters out every request whose host names aren't in the - spider's :attr:`~scrapy.Spider.allowed_domains` attribute. - All subdomains of any domain in the list are also allowed. - E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` - but not ``www2.example.com`` nor ``example.com``. - - When your spider returns a request for a domain not belonging to those - covered by the spider, this middleware will log a debug message similar to - this one:: - - DEBUG: Filtered offsite request to 'www.othersite.com': - - To avoid filling the log with too much noise, it will only print one of - these messages for each new domain filtered. So, for example, if another - request for ``www.othersite.com`` is filtered, no log message will be - printed. But if a request for ``someothersite.com`` is filtered, a message - will be printed (but only for the first request filtered). - - If the spider doesn't define an - :attr:`~scrapy.Spider.allowed_domains` attribute, or the - attribute is empty, the offsite middleware will allow all requests. - - If the request has the :attr:`~scrapy.Request.dont_filter` attribute - set, the offsite middleware will allow the request even if its domain is not - listed in allowed domains. - RefererMiddleware ----------------- @@ -392,9 +358,9 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` Acceptable values for REFERRER_POLICY ************************************* -- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` +- either a path to a :class:`scrapy.spidermiddlewares.referer.ReferrerPolicy` subclass — a custom policy or one of the built-in ones (see classes below), -- or one of the standard W3C-defined string values, +- or one or more comma-separated standard W3C-defined string values, - or the special ``"scrapy-default"``. ======================================= ======================================================================== @@ -411,6 +377,8 @@ String value Class name (as a string) `"unsafe-url"`_ :class:`scrapy.spidermiddlewares.referer.UnsafeUrlPolicy` ======================================= ======================================================================== +.. autoclass:: ReferrerPolicy + .. autoclass:: DefaultReferrerPolicy .. warning:: Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_, @@ -454,6 +422,33 @@ String value Class name (as a string) .. _"strict-origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin .. _"unsafe-url": https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url +.. setting:: REFERRER_POLICIES + +REFERRER_POLICIES +^^^^^^^^^^^^^^^^^ + +.. versionadded:: 2.14.2 + +Default: ``{}`` + +A dictionary mapping policy names to import paths of +:class:`scrapy.spidermiddlewares.referer.ReferrerPolicy` subclasses, or +``None`` to disable support for a given policy name. + +This allows overriding the policies triggered by the ``Referrer-Policy`` +response header. + +Use ``""`` to override the policy for responses with `no referrer policy +`__. + + +StartSpiderMiddleware +--------------------- + +.. module:: scrapy.spidermiddlewares.start + +.. autoclass:: StartSpiderMiddleware + UrlLengthMiddleware ------------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 30677fe74..bcef9d5f6 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -12,16 +12,16 @@ parsing pages for a particular site (or, in some cases, a group of sites). For spiders, the scraping cycle goes through something like this: -1. You start by generating the initial Requests to crawl the first URLs, and +1. You start by generating the initial requests to crawl the first URLs, and specify a callback function to be called with the response downloaded from those requests. - The first requests to perform are obtained by calling the - :meth:`~scrapy.Spider.start_requests` method which (by default) - generates :class:`~scrapy.Request` for the URLs specified in the - :attr:`~scrapy.Spider.start_urls` and the - :attr:`~scrapy.Spider.parse` method as callback function for the - Requests. + The first requests to perform are obtained by iterating the + :meth:`~scrapy.Spider.start` method, which by default yields a + :class:`~scrapy.Request` object for each URL in the + :attr:`~scrapy.Spider.start_urls` spider attribute, with the + :attr:`~scrapy.Spider.parse` method set as :attr:`~scrapy.Request.callback` + function to handle each :class:`~scrapy.http.Response`. 2. In the callback function, you parse the response (web page) and return :ref:`item objects `, @@ -48,14 +48,7 @@ scrapy.Spider ============= .. class:: scrapy.spiders.Spider -.. class:: scrapy.Spider() - - This is the simplest spider, and the one from which every other spider - must inherit (including spiders that come bundled with Scrapy, as well as spiders - that you write yourself). It doesn't provide any special functionality. It just - provides a default :meth:`start_requests` implementation which sends requests from - the :attr:`start_urls` spider attribute and calls the spider's method ``parse`` - for each of the resulting responses. +.. autoclass:: scrapy.Spider .. attribute:: name @@ -75,17 +68,13 @@ scrapy.Spider An optional list of strings containing domains that this spider is allowed to crawl. Requests for URLs not belonging to the domain names specified in this list (or their subdomains) won't be followed if - :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled. + :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` is + enabled. Let's say your target url is ``https://www.example.com/1.html``, then add ``'example.com'`` to the list. - .. attribute:: start_urls - - A list of URLs where the spider will begin to crawl from, when no - particular URLs are specified. So, the first pages downloaded will be those - listed here. The subsequent :class:`~scrapy.Request` will be generated successively from data - contained in the start URLs. + .. autoattribute:: start_urls .. attribute:: custom_settings @@ -148,7 +137,7 @@ scrapy.Spider The final settings and the initialized :class:`~scrapy.crawler.Crawler` attributes are available in the - :meth:`start_requests` method, handlers of the + :meth:`start` method, handlers of the :signal:`engine_started` signal and later. :param crawler: crawler to which the spider will be bound @@ -200,41 +189,7 @@ scrapy.Spider super().update_settings(settings) settings.setdefault("FEEDS", {}).update(cls.custom_feed) - .. method:: start_requests() - - This method must return an iterable with the first Requests to crawl for - this spider. It is called by Scrapy when the spider is opened for - scraping. Scrapy calls it only once, so it is safe to implement - :meth:`start_requests` as a generator. - - The default implementation generates ``Request(url, dont_filter=True)`` - for each url in :attr:`start_urls`. - - If you want to change the Requests used to start scraping a domain, this is - the method to override. For example, if you need to start by logging in using - a POST request, you could do: - - .. code-block:: python - - import scrapy - - - class MySpider(scrapy.Spider): - name = "myspider" - - def start_requests(self): - return [ - scrapy.FormRequest( - "http://www.example.com/login", - formdata={"user": "john", "pass": "secret"}, - callback=self.logged_in, - ) - ] - - def logged_in(self, response): - # here you would extract links to follow and return Requests for - # each of them, with another callback - pass + .. automethod:: start .. method:: parse(response) @@ -243,7 +198,7 @@ scrapy.Spider The ``parse`` method is in charge of processing the response and returning scraped data and/or more URLs to follow. Other Requests callbacks have - the same requirements as the :class:`Spider` class. + the same requirements as the :class:`~scrapy.Spider` class. This method, as well as any other Request callback, must return a :class:`~scrapy.Request` object, an :ref:`item object `, an @@ -306,8 +261,9 @@ Return multiple Requests and items from a single callback: for href in response.xpath("//a/@href").getall(): yield scrapy.Request(response.urljoin(href), self.parse) -Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; -to give data more structure you can use :class:`~scrapy.Item` objects: +Instead of :attr:`~.start_urls` you can use :meth:`~scrapy.Spider.start` +directly; to give data more structure you can use :class:`~scrapy.Item` +objects: .. skip: next .. code-block:: python @@ -320,7 +276,7 @@ to give data more structure you can use :class:`~scrapy.Item` objects: name = "example.com" allowed_domains = ["example.com"] - def start_requests(self): + async def start(self): yield scrapy.Request("http://www.example.com/1.html", self.parse) yield scrapy.Request("http://www.example.com/2.html", self.parse) yield scrapy.Request("http://www.example.com/3.html", self.parse) @@ -374,11 +330,11 @@ The above example can also be written as follows: class MySpider(scrapy.Spider): name = "myspider" - def start_requests(self): + async def start(self): yield scrapy.Request(f"http://www.example.com/categories/{self.category}") -If you are :ref:`running Scrapy from a script `, you can -specify spider arguments when calling +If you are :ref:`running Scrapy from a script `, you can +specify spider arguments when calling :class:`CrawlerProcess.crawl ` or :class:`CrawlerRunner.crawl `: @@ -399,15 +355,93 @@ Otherwise, you would cause iteration over a ``start_urls`` string resulting in each character being seen as a separate url. A valid use case is to set the http auth credentials -used by :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` -or the user agent -used by :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`:: +used by :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`:: - scrapy crawl myspider -a http_user=myuser -a http_pass=mypassword -a user_agent=mybot + scrapy crawl myspider -a http_user=myuser -a http_pass=mypassword Spider arguments can also be passed through the Scrapyd ``schedule.json`` API. See `Scrapyd documentation`_. +.. _spiderargs-scrapy-spider-metadata: + +scrapy-spider-metadata parameters +--------------------------------- + +Another alternative to pass spider arguments is the library `scrapy-spider-metadata`_. + +This allows for Scrapy spiders to define, validate, document and pre-process +their arguments as Pydantic models. + +The example shows how to define typed parameters where a string argument +is automatically converted to an integer: + +.. code-block:: python + + import scrapy + from pydantic import BaseModel + from scrapy_spider_metadata import Args + + + class MyParams(BaseModel): + pages: int + + + class BookSpider(Args[MyParams], scrapy.Spider): + name = "bookspider" + start_urls = ["http://books.toscrape.com/catalogue"] + + async def start(self): + for start_url in self.start_urls: + for index in range(1, self.args.pages + 1): + yield scrapy.Request(f"{start_url}/page-{index}.html") + + def parse(self, response): + book_links = response.css("article.product_pod h3 a::attr(href)").getall() + for book_link in book_links: + yield response.follow(book_link, self.parse_book) + + def parse_book(self, response): + yield { + "title": response.css("h1::text").get(), + "price": response.css("p.price_color::text").get(), + } + +This spider can be called from the command line:: + + scrapy crawl bookspider -a pages=2 + +.. _start-requests: + +Start requests +============== + +**Start requests** are :class:`~scrapy.Request` objects yielded from the +:meth:`~scrapy.Spider.start` method of a spider or from the +:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method of a +:ref:`spider middleware `. + +.. seealso:: :ref:`start-request-order` + +.. _start-requests-lazy: + +Delaying start request iteration +-------------------------------- + +You can override the :meth:`~scrapy.Spider.start` method as follows to pause +its iteration whenever there are scheduled requests: + +.. code-block:: python + + async def start(self): + async for item_or_request in super().start(): + if self.crawler.engine.needs_backout(): + await self.crawler.signals.wait_for(signals.scheduler_empty) + yield item_or_request + +This can help minimize the number of requests in the scheduler at any given +time, to minimize resource usage (memory or disk, depending on +:setting:`JOBDIR`). + .. _builtin-spiders: Generic Spiders @@ -423,13 +457,14 @@ with a ``TestItem`` declared in a ``myproject.items`` module: .. code-block:: python - import scrapy + from dataclasses import dataclass - class TestItem(scrapy.Item): - id = scrapy.Field() - name = scrapy.Field() - description = scrapy.Field() + @dataclass + class TestItem: + id: str | None = None + name: str | None = None + description: str | None = None .. currentmodule:: scrapy.spiders @@ -515,9 +550,6 @@ Crawling rules callbacks for new requests when writing :class:`CrawlSpider`-based spiders; unexpected behaviour can occur otherwise. - .. versionadded:: 2.0 - The *errback* parameter. - CrawlSpider example ~~~~~~~~~~~~~~~~~~~ @@ -525,7 +557,6 @@ Let's now take a look at an example CrawlSpider with rules: .. code-block:: python - import scrapy from scrapy.spiders import CrawlSpider, Rule from scrapy.linkextractors import LinkExtractor @@ -545,7 +576,7 @@ Let's now take a look at an example CrawlSpider with rules: def parse_item(self, response): self.logger.info("Hi, this is an item page! %s", response.url) - item = scrapy.Item() + item = {} item["id"] = response.xpath('//td[@id="item_id"]/text()').re(r"ID: (\d+)") item["name"] = response.xpath('//td[@id="item_name"]/text()').get() item["description"] = response.xpath( @@ -640,7 +671,7 @@ XMLFeedSpider This method is called for the nodes matching the provided tag name (``itertag``). Receives the response and an :class:`~scrapy.Selector` for each node. Overriding this - method is mandatory. Otherwise, you spider won't work. This method + method is mandatory. Otherwise, your spider won't work. This method must return an :ref:`item object `, a :class:`~scrapy.Request` object, or an iterable containing any of them. @@ -683,9 +714,9 @@ These spiders are pretty easy to use, let's have a look at one example: ) item = TestItem() - item["id"] = node.xpath("@id").get() - item["name"] = node.xpath("name").get() - item["description"] = node.xpath("description").get() + item.id = node.xpath("@id").get() + item.name = node.xpath("name").get() + item.description = node.xpath("description").get() return item Basically what we did up there was to create a spider that downloads a feed from @@ -747,9 +778,9 @@ Let's see an example similar to the previous one, but using a self.logger.info("Hi, this is a row!: %r", row) item = TestItem() - item["id"] = row["id"] - item["name"] = row["name"] - item["description"] = row["description"] + item.id = row["id"] + item.name = row["name"] + item.description = row["description"] return item @@ -938,10 +969,11 @@ Combine SitemapSpider with other sources of urls: other_urls = ["http://www.example.com/about"] - def start_requests(self): - requests = list(super(MySpider, self).start_requests()) - requests += [scrapy.Request(x, self.parse_other) for x in self.other_urls] - return requests + async def start(self): + async for item_or_request in super().start(): + yield item_or_request + for url in self.other_urls: + yield Request(url, self.parse_other) def parse_shop(self, response): pass # ... scrape shop here ... @@ -949,6 +981,7 @@ Combine SitemapSpider with other sources of urls: def parse_other(self, response): pass # ... scrape other here ... +.. _scrapy-spider-metadata: https://scrapy-spider-metadata.readthedocs.io/en/latest/params.html .. _Sitemaps: https://www.sitemaps.org/index.html .. _Sitemap index files: https://www.sitemaps.org/protocol.html#index .. _robots.txt: https://www.robotstxt.org/ diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index be8ecb7a5..ac714eacb 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -42,6 +42,8 @@ attribute. Here is an example of an extension that access stats: def from_crawler(cls, crawler): return cls(crawler.stats) +.. skip: start + Set stat value: .. code-block:: python @@ -80,13 +82,15 @@ Get all stats: >>> stats.get_stats() {'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} +.. skip: end + Available Stats Collectors ========================== Besides the basic :class:`StatsCollector` there are other Stats Collectors available in Scrapy which extend the basic Stats Collector. You can select which Stats Collector to use through the :setting:`STATS_CLASS` setting. The -default Stats Collector used is the :class:`MemoryStatsCollector`. +default Stats Collector used is the :class:`MemoryStatsCollector`. .. currentmodule:: scrapy.statscollectors @@ -117,4 +121,3 @@ DummyStatsCollector setting, to disable stats collect in order to improve performance. However, the performance penalty of stats collection is usually marginal compared to other Scrapy workload like parsing pages. - diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 832829b75..6d99c756e 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -26,6 +26,9 @@ disable it if you want. For more information about the extension itself see Please avoid using telnet console over insecure connections, or disable it completely using :setting:`TELNETCONSOLE_ENABLED` option. +.. note:: + This feature is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. + .. highlight:: none How to access the telnet console @@ -43,12 +46,12 @@ the console you need to type:: Password: >>> -By default Username is ``scrapy`` and Password is autogenerated. The -autogenerated Password can be seen on Scrapy logs like the example below:: +By default, the username is ``scrapy`` and the password is autogenerated. The +autogenerated password can be seen on Scrapy logs like the example below:: 2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326 -Default Username and Password can be overridden by the settings +The default username and password can be overridden by the settings :setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD`. .. warning:: @@ -59,6 +62,8 @@ Default Username and Password can be overridden by the settings You need the telnet program which comes installed by default in Windows, and most Linux distros. +.. _telnet-vars: + Available variables in the telnet console ========================================= @@ -77,8 +82,6 @@ convenience: +----------------+-------------------------------------------------------------------+ | ``spider`` | the active spider | +----------------+-------------------------------------------------------------------+ -| ``slot`` | the engine slot | -+----------------+-------------------------------------------------------------------+ | ``extensions`` | the Extension Manager (Crawler.extensions attribute) | +----------------+-------------------------------------------------------------------+ | ``stats`` | the Stats Collector (Crawler.stats attribute) | @@ -97,6 +100,8 @@ convenience: Telnet console usage examples ============================= +.. skip: start + Here are some example tasks you can do with the telnet console: View engine status @@ -114,10 +119,10 @@ using the telnet console:: engine.scraper.is_idle() : False engine.spider.name : followall engine.spider_is_idle() : False - engine.slot.closing : False - len(engine.slot.inprogress) : 16 - len(engine.slot.scheduler.dqs or []) : 0 - len(engine.slot.scheduler.mqs) : 92 + engine._slot.closing : False + len(engine._slot.inprogress) : 16 + len(engine._slot.scheduler.dqs or []) : 0 + len(engine._slot.scheduler.mqs) : 92 len(engine.scraper.slot.queue) : 0 len(engine.scraper.slot.active) : 0 engine.scraper.slot.active_size : 0 @@ -146,6 +151,8 @@ To stop:: >>> engine.stop() Connection closed by foreign host. +.. skip: end + Telnet Console signals ====================== @@ -172,8 +179,8 @@ TELNETCONSOLE_PORT Default: ``[6023, 6073]`` -The port range to use for the telnet console. If set to ``None`` or ``0``, a -dynamically assigned port is used. +The port range to use for the telnet console. If set to ``None``, a dynamically +assigned port is used. .. setting:: TELNETCONSOLE_HOST diff --git a/docs/versioning.rst b/docs/versioning.rst index 9d02757b0..0c6498244 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -23,7 +23,7 @@ Development releases do not follow 3-numbers version and are generally released as ``dev`` suffixed versions, e.g. ``1.3dev``. .. note:: - With Scrapy 0.* series, Scrapy used `odd-numbered versions for development releases`_. + With Scrapy 0.* series, Scrapy used odd-numbered versions for development releases. This is not the case anymore from Scrapy 1.0 onwards. Starting with Scrapy 1.0, all releases should be considered production-ready. @@ -63,7 +63,3 @@ feature. All deprecated features removed in a Scrapy release are explicitly mentioned in the :ref:`release notes `. - - -.. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases - diff --git a/extras/coverage-report.sh b/extras/coverage-report.sh index 842d0e46e..7eaa214cf 100755 --- a/extras/coverage-report.sh +++ b/extras/coverage-report.sh @@ -1,6 +1,6 @@ # Run tests, generate coverage report and open it on a browser # -# Requires: coverage 3.3 or above from https://pypi.python.org/pypi/coverage +# Requires: coverage 3.3 or above from https://pypi.org/pypi/coverage coverage run --branch $(which trial) --reporter=text tests coverage html -i diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py index 70c9003e5..569438574 100755 --- a/extras/qps-bench-server.py +++ b/extras/qps-bench-server.py @@ -2,7 +2,7 @@ from collections import deque from time import time -from twisted.internet import reactor +from twisted.internet import reactor # noqa: TID253 from twisted.web.resource import Resource from twisted.web.server import NOT_DONE_YET, Site @@ -18,7 +18,7 @@ class Root(Resource): self.tail.clear() self.start = self.lastmark = self.lasttime = time() - def getChild(self, request, name): + def getChild(self, path, request): return self def render(self, request): diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 119dfdabb..8e5001c1d 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -34,7 +34,7 @@ class QPSSpider(Spider): elif self.download_delay is not None: self.download_delay = float(self.download_delay) - def start_requests(self): + async def start(self): url = self.benchurl if self.latency is not None: url += f"?latency={self.latency}" diff --git a/extras/scrapy_zsh_completion b/extras/scrapy_zsh_completion index e2f2dc82b..82eb77cc0 100644 --- a/extras/scrapy_zsh_completion +++ b/extras/scrapy_zsh_completion @@ -41,7 +41,6 @@ _scrapy() { (runspider) local options=( {'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files' - {'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)' '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)' '1:spider file:_files -g \*.py' ) @@ -99,7 +98,6 @@ _scrapy() { (crawl) local options=( {'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files' - {'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)' '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)' '1:spider:_scrapy_spiders' ) diff --git a/pylintrc b/pylintrc deleted file mode 100644 index c8654b8d3..000000000 --- a/pylintrc +++ /dev/null @@ -1,99 +0,0 @@ -[MASTER] -persistent=no -jobs=1 # >1 hides results - -[MESSAGES CONTROL] -disable=abstract-method, - anomalous-backslash-in-string, - arguments-differ, - arguments-renamed, - attribute-defined-outside-init, - bad-classmethod-argument, - bad-mcs-classmethod-argument, - bare-except, - broad-except, - broad-exception-raised, - c-extension-no-member, - catching-non-exception, - cell-var-from-loop, - comparison-with-callable, - consider-using-dict-items, - consider-using-in, - consider-using-with, - cyclic-import, - dangerous-default-value, - disallowed-name, - duplicate-code, # https://github.com/PyCQA/pylint/issues/214 - eval-used, - expression-not-assigned, - fixme, - function-redefined, - global-statement, - implicit-str-concat, - import-error, - import-outside-toplevel, - import-self, - inconsistent-return-statements, - inherit-non-class, - invalid-name, - invalid-overridden-method, - isinstance-second-argument-not-valid-type, - keyword-arg-before-vararg, - line-too-long, - logging-format-interpolation, - logging-fstring-interpolation, - logging-not-lazy, - lost-exception, - method-hidden, - missing-docstring, - no-else-raise, - no-else-return, - no-member, - no-method-argument, - no-name-in-module, - no-self-argument, - no-value-for-parameter, - not-callable, - pointless-exception-statement, - pointless-statement, - pointless-string-statement, - protected-access, - raise-missing-from, - redefined-argument-from-local, - redefined-builtin, - redefined-outer-name, - reimported, - signature-differs, - super-init-not-called, - too-few-public-methods, - too-many-ancestors, - too-many-arguments, - too-many-branches, - too-many-format-args, - too-many-function-args, - too-many-instance-attributes, - too-many-lines, - too-many-locals, - too-many-public-methods, - too-many-return-statements, - unbalanced-tuple-unpacking, - undefined-variable, - undefined-loop-variable, - unexpected-special-method-signature, - unnecessary-comprehension, - unnecessary-dunder-call, - unnecessary-pass, - unreachable, - unsubscriptable-object, - unused-argument, - unused-import, - unused-private-member, - unused-variable, - unused-wildcard-import, - use-dict-literal, - used-before-assignment, - useless-object-inheritance, # Required for Python 2 support - useless-return, - useless-super-delegation, - wildcard-import, - wrong-import-position diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..50322b17a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,458 @@ +[build-system] +requires = ["hatchling>=1.27.0"] +build-backend = "hatchling.build" + +[project] +name = "Scrapy" +dynamic = ["version"] +description = "A high-level Web Crawling and Web Scraping framework" +dependencies = [ + "Twisted>=21.7.0", + "cryptography>=37.0.0", + "cssselect>=0.9.1", + "defusedxml>=0.7.1", + "itemadapter>=0.1.0", + "itemloaders>=1.0.1", + "lxml>=4.6.4", + "packaging", + "parsel>=1.5.0", + "protego>=0.1.15", + "pyOpenSSL>=22.0.0", + "queuelib>=1.4.2", + "service_identity>=23.1.0", + "tldextract", + "w3lib>=1.17.0", + "xtractmime>=0.2.0", + "zope.interface>=5.1.0", + # Platform-specific dependencies + 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"', + 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"', +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Framework :: Scrapy", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Software Development :: Libraries :: Application Frameworks", + "Topic :: Software Development :: Libraries :: Python Modules", +] +license = "BSD-3-Clause" +license-files = ["LICENSE", "AUTHORS"] +readme = "README.rst" +requires-python = ">=3.10" +authors = [{ name = "Scrapy developers", email = "pablo@pablohoffman.com" }] +maintainers = [{ name = "Pablo Hoffman", email = "pablo@pablohoffman.com" }] + +[project.urls] +Homepage = "https://scrapy.org/" +Documentation = "https://docs.scrapy.org/" +Source = "https://github.com/scrapy/scrapy" +Tracker = "https://github.com/scrapy/scrapy/issues" +"Release notes" = "https://docs.scrapy.org/en/latest/news.html" + +[project.scripts] +scrapy = "scrapy.cmdline:execute" + +[tool.hatch.build.targets.sdist] +include = [ + "/docs", + "/extras", + "/scrapy", + "/tests", + "/tests_typing", + "/CODE_OF_CONDUCT.md", + "/CONTRIBUTING.md", + "/INSTALL.md", + "/NEWS", + "/SECURITY.md", + "/codecov.yml", + "/conftest.py", + "/tox.ini", +] + +[tool.hatch.version] +path = "scrapy/VERSION" +pattern = "^(?P.+)$" + +[tool.mypy] +strict = true +extra_checks = false # weird addErrback() errors +untyped_calls_exclude = [ + "twisted", +] + +[[tool.mypy.overrides]] +module = "tests.*" +allow_untyped_defs = true +allow_incomplete_defs = true # 48 errors +check_untyped_defs = false + +# Interface classes are hard to support +[[tool.mypy.overrides]] +module = "twisted.internet.interfaces" +follow_imports = "skip" + +[[tool.mypy.overrides]] +module = "scrapy.interfaces" +ignore_errors = true + +[[tool.mypy.overrides]] +module = "twisted.internet.reactor" +follow_imports = "skip" + +# just for twisted.version +[[tool.mypy.overrides]] +module = "twisted" +implicit_reexport = true + +# TODO +[[tool.mypy.overrides]] +module = "scrapy.settings.default_settings" +ignore_errors = true + +# usually no type hints +[[tool.mypy.overrides]] +module = [ + "bpython", + "brotli", + "brotlicffi", + "google.*", + "pydispatch.*", + "pyftpdlib.*", + "pytest_twisted", + "robotexclusionrulesparser", + "testfixtures", + "zope.interface.*", +] +ignore_missing_imports = true + +[tool.bumpversion] +current_version = "2.16.0" +commit = true +tag = true +tag_name = "{new_version}" + +[[tool.bumpversion.files]] +filename = "docs/news.rst" +search = "\\(unreleased\\)$" +replace = "({now:%Y-%m-%d})" +regex = true + +[[tool.bumpversion.files]] +filename = "scrapy/VERSION" + +[[tool.bumpversion.files]] +filename = "SECURITY.md" +parse = """(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)""" +serialize = ["{major}.{minor}"] + +[tool.coverage.run] +# sysmon, default on 3.14, is too slow: https://github.com/coveragepy/coveragepy/issues/2172 +core = "ctrace" +branch = true +include = ["scrapy/*"] +omit = ["tests/*"] +disable_warnings = ["include-ignored"] +patch = [ + "subprocess", +] + +[tool.coverage.paths] +source = [ + "scrapy", + ".tox/**/site-packages/scrapy" +] + +[tool.coverage.report] +exclude_also = [ + "@(abc\\.)?abstractmethod", + '\A(?s:.*# pragma: no file cover.*)\Z', +] + +[tool.pylint.MASTER] +persistent = "no" +jobs = 1 # >1 hides results +extension-pkg-allow-list=[ + "lxml", +] +load-plugins = ["pylint_per_file_ignores"] + +[tool.pylint."MESSAGES CONTROL"] +enable = [ + "useless-suppression", +] +# Make INFO checks like useless-suppression also cause pylint to return a +# non-zero exit code. +fail-on = "I" +disable = [ + # Ones we want to ignore + "attribute-defined-outside-init", + "broad-exception-caught", + "consider-using-with", + "cyclic-import", + "disallowed-name", + "duplicate-code", # https://github.com/pylint-dev/pylint/issues/214 + "fixme", + "inherit-non-class", # false positives with create_deprecated_class() + "invalid-name", + "invalid-overridden-method", + "isinstance-second-argument-not-valid-type", # false positives with create_deprecated_class() + "line-too-long", + "logging-format-interpolation", + "logging-fstring-interpolation", + "logging-not-lazy", + "missing-docstring", + "no-member", + "no-value-for-parameter", # https://github.com/pylint-dev/pylint/issues/3268 + "not-callable", + "protected-access", + "redefined-outer-name", + "too-few-public-methods", + "too-many-ancestors", + "too-many-arguments", + "too-many-function-args", + "too-many-instance-attributes", + "too-many-lines", + "too-many-locals", + "too-many-positional-arguments", + "too-many-public-methods", + "too-many-return-statements", + "undefined-variable", + "unused-argument", + "unused-variable", + "useless-import-alias", # used as a hint to mypy + "useless-return", # https://github.com/pylint-dev/pylint/issues/6530 + "wrong-import-position", + + # Ones that are implemented in ruff and need to be disabled for some lines (listed here to avoid two disable comments) + "bare-except", + "eval-used", + "global-statement", + "import-outside-toplevel", + "import-self", + "inconsistent-return-statements", + "redefined-builtin", + "too-many-branches", + "unused-import", + + # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix") + "arguments-differ", + "keyword-arg-before-vararg", +] +# requires `pylint_per_file_ignores` plugin +per-file-ignores = [ + # Extended list of ones that we may want to address, only for tests + "./tests/*:abstract-method,arguments-renamed,dangerous-default-value,pointless-statement,raise-missing-from,unnecessary-dunder-call,used-before-assignment", +] + +[tool.pytest.ini_options] +addopts = [ + "--reactor=asyncio", +] +xfail_strict = true +python_files = ["test_*.py", "test_*/__init__.py"] +markers = [ + "only_asyncio: marks tests that require the asyncio loop to be used", + "only_not_asyncio: marks tests that require the asyncio loop to not be used", + "requires_reactor: marks tests that require a reactor", + "requires_uvloop: marks tests as only enabled when uvloop is known to be working", + "requires_botocore: marks tests that need botocore (but not boto3)", + "requires_boto3: marks tests that need botocore and boto3", + "requires_mitmproxy: marks tests that need mitmproxy", + "requires_internet: marks tests that need real Internet access", +] +filterwarnings = [ + "ignore::DeprecationWarning:twisted.web.static", + "ignore:scrapy.responsetypes is deprecated", + "ignore:scrapy.utils.response.get_base_url is deprecated", +] + +[tool.ruff.lint] +extend-select = [ + # flake8-builtins + "A", + # flake8-async + "ASYNC", + # flake8-bugbear + "B", + # flake8-comprehensions + "C4", + # flake8-commas + "COM", + # pydocstyle + "D", + # flake8-future-annotations + "FA", + # flynt + "FLY", + # refurb + "FURB", + # isort + "I", + # flake8-implicit-str-concat + "ISC", + # flake8-logging + "LOG", + # Perflint + "PERF", + # pygrep-hooks + "PGH", + # flake8-pie + "PIE", + # pylint + "PL", + # flake8-pytest-style + "PT", + # flake8-use-pathlib + "PTH", + # flake8-pyi + "PYI", + # flake8-quotes + "Q", + # flake8-return + "RET", + # flake8-raise + "RSE", + # Ruff-specific rules + "RUF", + # flake8-bandit + "S", + # flake8-simplify + "SIM", + # flake8-slots + "SLOT", + # flake8-debugger + "T10", + # flake8-type-checking + "TC", + # flake8-tidy-imports + "TID", + # pyupgrade + "UP", + # pycodestyle warnings + "W", + # flake8-2020 + "YTT", +] +ignore = [ + # Ones we want to ignore + + # Trailing comma missing + "COM812", + # Missing docstring in public module + "D100", + # Missing docstring in public class + "D101", + # Missing docstring in public method + "D102", + # Missing docstring in public function + "D103", + # Missing docstring in public package + "D104", + # Missing docstring in magic method + "D105", + # Missing docstring in public nested class + "D106", + # Missing docstring in __init__ + "D107", + # One-line docstring should fit on one line with quotes + "D200", + # No blank lines allowed after function docstring + "D202", + # 1 blank line required between summary line and description + "D205", + # Multi-line docstring closing quotes should be on a separate line + "D209", + # First line should end with a period + "D400", + # First line should be in imperative mood; try rephrasing + "D401", + # First line should not be the function's "signature" + "D402", + # First word of the first line should be properly capitalized + "D403", + # `try`-`except` within a loop incurs performance overhead + "PERF203", + # Too many return statements + "PLR0911", + # Too many arguments in function definition + "PLR0913", + # Magic value used in comparison + "PLR2004", + # String contains ambiguous {}. + "RUF001", + # Docstring contains ambiguous {}. + "RUF002", + # Comment contains ambiguous {}. + "RUF003", + # Use of `assert` detected; needed for mypy + "S101", + # FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180 + "S321", + # Use a context manager for opening files + "SIM115", + # Yoda condition detected + "SIM300", +] + +[tool.ruff.lint.flake8-tidy-imports] +banned-module-level-imports = [ + "twisted.internet.reactor", + # indirectly imports twisted.conch.insults.helper which imports twisted.internet.reactor + "twisted.conch.manhole", + # directly imports twisted.internet.reactor + "twisted.protocols.ftp", +] + +[tool.ruff.lint.isort] +split-on-trailing-comma = false + +[tool.ruff.lint.per-file-ignores] +# Circular import workarounds +"scrapy/linkextractors/__init__.py" = ["E402"] +"scrapy/spiders/__init__.py" = ["E402"] + +"tests/**" = [ + # Skip bandit and allow blocking file I/O in tests + "ASYNC240", + "S", + # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix") + # Assigning to `os.environ` doesn't clear the environment. + "B003", + # Do not use mutable data structures for argument defaults. + "B006", + # Found useless expression. + "B018", + # No explicit stacklevel argument found. + "B028", + # Within an `except` clause, raise exceptions with `raise ... from` + "B904", + # `for` loop variable overwritten by assignment target + "PLW2901", + # Mutable class attributes should be annotated with `typing.ClassVar` + "RUF012", + # Use capitalized environment variable + "SIM112", +] + +# Issues pending a review: +"docs/conf.py" = ["E402"] +"scrapy/utils/url.py" = ["F403", "F405"] +"tests/test_loader.py" = ["E741"] + +[tool.ruff.lint.pydocstyle] +convention = "pep257" + +[tool.sphinx-scrapy] +python-version = "3.14" # Keep in sync with .github/workflows/checks.yml. diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 579d377b7..000000000 --- a/pytest.ini +++ /dev/null @@ -1,29 +0,0 @@ -[pytest] -xfail_strict = true -usefixtures = chdir -python_files=test_*.py __init__.py -python_classes= -addopts = - --assert=plain - --ignore=docs/_ext - --ignore=docs/conf.py - --ignore=docs/news.rst - --ignore=docs/topics/dynamic-content.rst - --ignore=docs/topics/items.rst - --ignore=docs/topics/leaks.rst - --ignore=docs/topics/loaders.rst - --ignore=docs/topics/selectors.rst - --ignore=docs/topics/shell.rst - --ignore=docs/topics/stats.rst - --ignore=docs/topics/telnetconsole.rst - --ignore=docs/utils -markers = - only_asyncio: marks tests as only enabled when --reactor=asyncio is passed - only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed - requires_uvloop: marks tests as only enabled when uvloop is known to be working -filterwarnings = - ignore:Module scrapy.utils.reqser is deprecated - ignore:scrapy.downloadermiddlewares.decompression is deprecated - ignore:scrapy.responsetypes is deprecated - ignore:typing.re is deprecated - ignore:typing.io is deprecated diff --git a/scrapy/VERSION b/scrapy/VERSION index 46b81d815..752490696 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.11.0 +2.16.0 diff --git a/scrapy/__init__.py b/scrapy/__init__.py index cc0e539c4..c5b6483e3 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -6,8 +6,6 @@ import pkgutil import sys import warnings -from twisted import version as _txv - # Declare top-level shortcuts from scrapy.http import FormRequest, Request from scrapy.item import Field, Item @@ -15,28 +13,20 @@ from scrapy.selector import Selector from scrapy.spiders import Spider __all__ = [ + "Field", + "FormRequest", + "Item", + "Request", + "Selector", + "Spider", "__version__", "version_info", - "twisted_version", - "Spider", - "Request", - "FormRequest", - "Selector", - "Item", - "Field", ] # Scrapy and Twisted versions __version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip() version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split(".")) -twisted_version = (_txv.major, _txv.minor, _txv.micro) - - -# Check minimum required Python version -if sys.version_info < (3, 8): - print(f"Scrapy {__version__} requires Python 3.8+") - sys.exit(1) # Ignore noisy twisted deprecation warnings diff --git a/scrapy/addons.py b/scrapy/addons.py index 65d7a0310..2e12f8c8a 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,13 +1,16 @@ +from __future__ import annotations + import logging -from typing import TYPE_CHECKING, Any, List +from typing import TYPE_CHECKING, Any from scrapy.exceptions import NotConfigured -from scrapy.settings import Settings from scrapy.utils.conf import build_component_list from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings, Settings + logger = logging.getLogger(__name__) @@ -15,9 +18,9 @@ logger = logging.getLogger(__name__) class AddonManager: """This class facilitates loading and storing :ref:`topics-addons`.""" - def __init__(self, crawler: "Crawler") -> None: - self.crawler: "Crawler" = crawler - self.addons: List[Any] = [] + def __init__(self, crawler: Crawler) -> None: + self.crawler: Crawler = crawler + self.addons: list[Any] = [] def load_settings(self, settings: Settings) -> None: """Load add-ons and configurations from a settings object and apply them. @@ -33,7 +36,8 @@ class AddonManager: try: addoncls = load_object(clspath) addon = build_from_crawler(addoncls, self.crawler) - addon.update_settings(settings) + if hasattr(addon, "update_settings"): + addon.update_settings(settings) self.addons.append(addon) except NotConfigured as e: if e.args: @@ -49,3 +53,20 @@ class AddonManager: }, extra={"crawler": self.crawler}, ) + + @classmethod + def load_pre_crawler_settings(cls, settings: BaseSettings) -> None: + """Update early settings that do not require a crawler instance, such as SPIDER_MODULES. + + Similar to the load_settings method, this loads each add-on configured in the + ``ADDONS`` setting and calls their 'update_pre_crawler_settings' class method if present. + This method doesn't have access to the crawler instance or the addons list. + + :param settings: The :class:`~scrapy.settings.BaseSettings` object from \ + which to read the early add-on configuration + :type settings: :class:`~scrapy.settings.Settings` + """ + for clspath in build_component_list(settings["ADDONS"]): + addoncls = load_object(clspath) + if hasattr(addoncls, "update_pre_crawler_settings"): + addoncls.update_pre_crawler_settings(settings) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 6580ba9ce..6c306afdb 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,44 +1,58 @@ +from __future__ import annotations + import argparse import cProfile import inspect import os import sys from importlib.metadata import entry_points +from typing import TYPE_CHECKING, ParamSpec import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter -from scrapy.crawler import CrawlerProcess +from scrapy.crawler import AsyncCrawlerProcess, CrawlerProcess from scrapy.exceptions import UsageError -from scrapy.utils.misc import walk_modules +from scrapy.utils.misc import walk_modules_iter from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect +from scrapy.utils.reactor import _asyncio_reactor_path + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable + + from scrapy.settings import BaseSettings, Settings + +_P = ParamSpec("_P") class ScrapyArgumentParser(argparse.ArgumentParser): - def _parse_optional(self, arg_string): - # if starts with -: it means that is a parameter not a argument - if arg_string[:2] == "-:": + def _parse_optional( + self, arg_string: str + ) -> tuple[argparse.Action | None, str, str | None] | None: + # Support something like ‘-o -:json’, where ‘-:json’ is a value for + # ‘-o’, not another parameter. + if arg_string.startswith("-:"): return None return super()._parse_optional(arg_string) -def _iter_command_classes(module_name): +def _iter_command_classes(module_name: str) -> Iterable[type[ScrapyCommand]]: # TODO: add `name` attribute to commands and merge this function with # scrapy.utils.spider.iter_spider_classes - for module in walk_modules(module_name): + for module in walk_modules_iter(module_name): for obj in vars(module).values(): if ( inspect.isclass(obj) and issubclass(obj, ScrapyCommand) and obj.__module__ == module.__name__ - and obj not in (ScrapyCommand, BaseRunSpiderCommand) + and obj not in {ScrapyCommand, BaseRunSpiderCommand} ): yield obj -def _get_commands_from_module(module, inproject): - d = {} +def _get_commands_from_module(module: str, inproject: bool) -> dict[str, ScrapyCommand]: + d: dict[str, ScrapyCommand] = {} for cmd in _iter_command_classes(module): if inproject or not cmd.requires_project: cmdname = cmd.__module__.split(".")[-1] @@ -46,22 +60,22 @@ def _get_commands_from_module(module, inproject): return d -def _get_commands_from_entry_points(inproject, group="scrapy.commands"): - cmds = {} - if sys.version_info >= (3, 10): - eps = entry_points(group=group) - else: - eps = entry_points().get(group, ()) - for entry_point in eps: +def _get_commands_from_entry_points( + inproject: bool, group: str = "scrapy.commands" +) -> dict[str, ScrapyCommand]: + cmds: dict[str, ScrapyCommand] = {} + for entry_point in entry_points(group=group): obj = entry_point.load() if inspect.isclass(obj): cmds[entry_point.name] = obj() else: - raise Exception(f"Invalid entry point {entry_point.name}") + raise ValueError(f"Invalid entry point {entry_point.name}") return cmds -def _get_commands_dict(settings, inproject): +def _get_commands_dict( + settings: BaseSettings, inproject: bool +) -> dict[str, ScrapyCommand]: cmds = _get_commands_from_module("scrapy.commands", inproject) cmds.update(_get_commands_from_entry_points(inproject)) cmds_module = settings["COMMANDS_MODULE"] @@ -70,16 +84,20 @@ def _get_commands_dict(settings, inproject): return cmds -def _pop_command_name(argv): - i = 0 - for arg in argv[1:]: - if not arg.startswith("-"): - del argv[i] - return arg - i += 1 +def _get_project_only_cmds(settings: BaseSettings) -> set[str]: + return set(_get_commands_dict(settings, inproject=True)) - set( + _get_commands_dict(settings, inproject=False) + ) -def _print_header(settings, inproject): +def _pop_command_name(argv: list[str]) -> str | None: + for i in range(1, len(argv)): + if not argv[i].startswith("-"): + return argv.pop(i) + return None + + +def _print_header(settings: BaseSettings, inproject: bool) -> None: version = scrapy.__version__ if inproject: print(f"Scrapy {version} - active project: {settings['BOT_NAME']}\n") @@ -88,28 +106,56 @@ def _print_header(settings, inproject): print(f"Scrapy {version} - no active project\n") -def _print_commands(settings, inproject): +def _print_commands(settings: BaseSettings, inproject: bool) -> None: _print_header(settings, inproject) - print("Usage:") - print(" scrapy [options] [args]\n") - print("Available commands:") + print( + "Usage:\n", + " scrapy [options] [args]\n", + "Available commands:\n", + ) cmds = _get_commands_dict(settings, inproject) - for cmdname, cmdclass in sorted(cmds.items()): - print(f" {cmdname:<13} {cmdclass.short_desc()}") + print( + "\n".join( + f" {cmdname:<13} {cmdclass.short_desc()}" + for cmdname, cmdclass in sorted(cmds.items()) + ) + ) if not inproject: - print() - print(" [ more ] More commands available when run from project directory") - print() - print('Use "scrapy -h" to see more info about a command') + print( + "\n", + " [ more ] More commands available when run from project directory", + ) + print("\n", 'Use "scrapy -h" to see more info about a command') -def _print_unknown_command(settings, cmdname, inproject): +def _print_unknown_command_msg( + settings: BaseSettings, cmdname: str, inproject: bool +) -> None: + proj_only_cmds = _get_project_only_cmds(settings) + if cmdname in proj_only_cmds and not inproject: + cmd_list = ", ".join(sorted(proj_only_cmds)) + print( + f"The {cmdname} command is not available from this location.\n" + f"These commands are only available from within a project: {cmd_list}.\n" + ) + else: + print(f"Unknown command: {cmdname}\n") + + +def _print_unknown_command( + settings: BaseSettings, cmdname: str, inproject: bool +) -> None: _print_header(settings, inproject) - print(f"Unknown command: {cmdname}\n") + _print_unknown_command_msg(settings, cmdname, inproject) print('Use "scrapy" to see available commands') -def _run_print_help(parser, func, *a, **kw): +def _run_print_help( + parser: argparse.ArgumentParser, + func: Callable[_P, None], + *a: _P.args, + **kw: _P.kwargs, +) -> None: try: func(*a, **kw) except UsageError as e: @@ -120,7 +166,7 @@ def _run_print_help(parser, func, *a, **kw): sys.exit(2) -def execute(argv=None, settings=None): +def execute(argv: list[str] | None = None, settings: Settings | None = None) -> None: if argv is None: argv = sys.argv @@ -157,19 +203,28 @@ def execute(argv=None, settings=None): opts, args = parser.parse_known_args(args=argv[1:]) _run_print_help(parser, cmd.process_options, args, opts) - cmd.crawler_process = CrawlerProcess(settings) + if cmd.requires_crawler_process: + if ( + settings["TWISTED_REACTOR"] == _asyncio_reactor_path + and not settings.getbool("FORCE_CRAWLER_PROCESS") + ) or not settings.getbool("TWISTED_REACTOR_ENABLED"): + cmd.crawler_process = AsyncCrawlerProcess(settings) + else: + cmd.crawler_process = CrawlerProcess(settings) _run_print_help(parser, _run_command, cmd, args, opts) sys.exit(cmd.exitcode) -def _run_command(cmd, args, opts): +def _run_command(cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace) -> None: if opts.profile: _run_command_profiled(cmd, args, opts) else: cmd.run(args, opts) -def _run_command_profiled(cmd, args, opts): +def _run_command_profiled( + cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace +) -> None: if opts.profile: sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") loc = locals() diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 2aa569cdd..19b6f6681 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -1,65 +1,84 @@ """ Base class for Scrapy commands """ + +from __future__ import annotations + import argparse +import builtins import os +import warnings +from abc import ABC, abstractmethod from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, ClassVar from twisted.python import failure -from scrapy.crawler import CrawlerProcess -from scrapy.exceptions import UsageError +from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli +if TYPE_CHECKING: + from collections.abc import Iterable -class ScrapyCommand: - requires_project = False - crawler_process: Optional[CrawlerProcess] = None + from scrapy.crawler import Crawler, CrawlerProcessBase + from scrapy.settings import Settings + + +class ScrapyCommand(ABC): + requires_project: bool = False + requires_crawler_process: bool = True + crawler_process: CrawlerProcessBase | None = None # set in scrapy.cmdline # default settings to be used for this command instead of global defaults - default_settings: Dict[str, Any] = {} + default_settings: ClassVar[dict[str, Any]] = {} - exitcode = 0 + exitcode: int = 0 def __init__(self) -> None: - self.settings: Any = None # set in scrapy.cmdline + self.settings: Settings | None = None # set in scrapy.cmdline - def set_crawler(self, crawler): + def set_crawler(self, crawler: Crawler) -> None: # pragma: no cover + warnings.warn( + "ScrapyCommand.set_crawler() is deprecated", + ScrapyDeprecationWarning, + stacklevel=2, + ) if hasattr(self, "_crawler"): raise RuntimeError("crawler already set") - self._crawler = crawler + self._crawler: Crawler = crawler - def syntax(self): + def syntax(self) -> str: """ Command syntax (preferably one-line). Do not include command name. """ return "" - def short_desc(self): + @abstractmethod + def short_desc(self) -> str: """ A short description of the command """ return "" - def long_desc(self): + def long_desc(self) -> str: """A long description of the command. Return short description when not available. It cannot contain newlines since contents will be formatted by optparser which removes newlines and wraps text. """ return self.short_desc() - def help(self): + def help(self) -> str: """An extensive help for the command. It will be shown when using the "help" command. It can contain newlines since no post-formatting will be applied to its contents. """ return self.long_desc() - def add_options(self, parser): + def add_options(self, parser: argparse.ArgumentParser) -> None: """ Populate option parse with options available for this command """ + assert self.settings is not None group = parser.add_argument_group(title="Global Options") group.add_argument( "--logfile", metavar="FILE", help="log file. if omitted stderr will be used" @@ -91,11 +110,14 @@ class ScrapyCommand: ) group.add_argument("--pdb", action="store_true", help="enable pdb on failure") - def process_options(self, args, opts): + def process_options(self, args: list[str], opts: argparse.Namespace) -> None: + assert self.settings is not None try: self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline") except ValueError: - raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False) + raise UsageError( + "Invalid -s value, use -s NAME=VALUE", print_help=False + ) from None if opts.logfile: self.settings.set("LOG_ENABLED", True, priority="cmdline") @@ -116,7 +138,8 @@ class ScrapyCommand: if opts.pdb: failure.startDebugMode() - def run(self, args: List[str], opts: argparse.Namespace) -> None: + @abstractmethod + def run(self, args: list[str], opts: argparse.Namespace) -> None: """ Entry point for running commands """ @@ -128,8 +151,8 @@ class BaseRunSpiderCommand(ScrapyCommand): Common class used to share functionality between the crawl, parse and runspider commands """ - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-a", dest="spargs", @@ -154,25 +177,21 @@ class BaseRunSpiderCommand(ScrapyCommand): help="dump scraped items into FILE, overwriting any existing file," " to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)", ) - parser.add_argument( - "-t", - "--output-format", - metavar="FORMAT", - help="format to use for dumping items", - ) - def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) + def process_options(self, args: list[str], opts: argparse.Namespace) -> None: + super().process_options(args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: - raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) + raise UsageError( + "Invalid -a value, use -a NAME=VALUE", print_help=False + ) from None if opts.output or opts.overwrite_output: + assert self.settings is not None feeds = feed_process_params_from_cli( self.settings, opts.output, - opts.output_format, - opts.overwrite_output, + overwrite_output=opts.overwrite_output, ) self.settings.set("FEEDS", feeds, priority="cmdline") @@ -182,7 +201,13 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): Help Formatter for scrapy command line help messages. """ - def __init__(self, prog, indent_increment=2, max_help_position=24, width=None): + def __init__( + self, + prog: str, + indent_increment: int = 2, + max_help_position: int = 24, + width: int | None = None, + ): super().__init__( prog, indent_increment=indent_increment, @@ -190,11 +215,12 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): width=width, ) - def _join_parts(self, part_strings): - parts = self.format_part_strings(part_strings) + def _join_parts(self, part_strings: Iterable[str]) -> str: + # scrapy.commands.list shadows builtins.list + parts = self.format_part_strings(builtins.list(part_strings)) return super()._join_parts(parts) - def format_part_strings(self, part_strings): + def format_part_strings(self, part_strings: list[str]) -> list[str]: """ Underline and title case command line help message headers. """ @@ -203,7 +229,7 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): headings = [ i for i in range(len(part_strings)) if part_strings[i].endswith(":\n") ] - for index in headings[::-1]: + for index in reversed(headings): char = "-" if "Global Options" in part_strings[index] else "=" part_strings[index] = part_strings[index][:-2].title() underline = "".join(["\n", (char * len(part_strings[index])), "\n"]) diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index e1ccdc451..0a5b431a2 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,38 +1,49 @@ +from __future__ import annotations + import subprocess import sys import time +from typing import TYPE_CHECKING, Any, ClassVar from urllib.parse import urlencode import scrapy from scrapy.commands import ScrapyCommand +from scrapy.http import Response, TextResponse from scrapy.linkextractors import LinkExtractor +from scrapy.utils.test import get_testenv + +if TYPE_CHECKING: + import argparse + from collections.abc import AsyncIterator class Command(ScrapyCommand): - default_settings = { + default_settings: ClassVar[dict[str, Any]] = { "LOG_LEVEL": "INFO", "LOGSTATS_INTERVAL": 1, "CLOSESPIDER_TIMEOUT": 10, } - def short_desc(self): + def short_desc(self) -> str: return "Run quick benchmark test" - def run(self, args, opts): + def run(self, args: list[str], opts: argparse.Namespace) -> None: with _BenchServer(): + assert self.crawler_process self.crawler_process.crawl(_BenchSpider, total=100000) self.crawler_process.start() class _BenchServer: - def __enter__(self): - from scrapy.utils.test import get_testenv - + def __enter__(self) -> None: pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"] - self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE, env=get_testenv()) + self.proc = subprocess.Popen( # noqa: S603 + pargs, stdout=subprocess.PIPE, env=get_testenv() + ) + assert self.proc.stdout self.proc.stdout.readline() - def __exit__(self, exc_type, exc_value, traceback): + def __exit__(self, exc_type, exc_value, traceback) -> None: # type: ignore[no-untyped-def] self.proc.kill() self.proc.wait() time.sleep(0.2) @@ -47,11 +58,12 @@ class _BenchSpider(scrapy.Spider): baseurl = "http://localhost:8998" link_extractor = LinkExtractor() - def start_requests(self): + async def start(self) -> AsyncIterator[Any]: qargs = {"total": self.total, "show": self.show} url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}" - return [scrapy.Request(url, dont_filter=True)] + yield scrapy.Request(url, dont_filter=True) - def parse(self, response): + def parse(self, response: Response) -> Any: + assert isinstance(response, TextResponse) for link in self.link_extractor.extract_links(response): yield scrapy.Request(link.url, callback=self.parse) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index de54ca4d3..2113c19d2 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -1,8 +1,12 @@ +import argparse import time from collections import defaultdict +from collections.abc import AsyncIterator +from typing import Any, ClassVar from unittest import TextTestResult as _TextTestResult from unittest import TextTestRunner +from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.contracts import ContractsManager from scrapy.utils.conf import build_component_list @@ -10,7 +14,7 @@ from scrapy.utils.misc import load_object, set_environ class TextTestResult(_TextTestResult): - def printSummary(self, start, stop): + def printSummary(self, start: float, stop: float) -> None: write = self.stream.write writeln = self.stream.writeln @@ -40,16 +44,16 @@ class TextTestResult(_TextTestResult): class Command(ScrapyCommand): requires_project = True - default_settings = {"LOG_ENABLED": False} + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Check spider contracts" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-l", "--list", @@ -66,9 +70,12 @@ class Command(ScrapyCommand): help="print contract tests for all spiders", ) - def run(self, args, opts): + def run(self, args: list[str], opts: argparse.Namespace) -> None: # load contracts - contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS")) + assert self.settings is not None + contracts = build_component_list( + self.settings.get_component_priority_dict_with_base("SPIDER_CONTRACTS") + ) conman = ContractsManager(load_object(c) for c in contracts) runner = TextTestRunner(verbosity=2 if opts.verbose else 1) result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity) @@ -76,12 +83,17 @@ class Command(ScrapyCommand): # contract requests contract_reqs = defaultdict(list) + assert self.crawler_process spider_loader = self.crawler_process.spider_loader + async def start(self: Spider) -> AsyncIterator[Any]: + for request in conman.from_spider(self, result): + yield request + with set_environ(SCRAPY_CHECK="true"): for spidername in args or spider_loader.list(): spidercls = spider_loader.load(spidername) - spidercls.start_requests = lambda s: conman.from_spider(s, result) + spidercls.start = start # type: ignore[method-assign] tested_methods = conman.tested_methods_from_spidercls(spidercls) if opts.list: @@ -92,17 +104,19 @@ class Command(ScrapyCommand): # start checks if opts.list: - for spider, methods in sorted(contract_reqs.items()): - if not methods and not opts.verbose: - continue - print(spider) - for method in sorted(methods): - print(f" * {method}") + print( + "\n".join( + f"{spider}\n" + + "\n".join(f" * {method}" for method in sorted(methods)) + for spider, methods in sorted(contract_reqs.items()) + if methods or opts.verbose + ) + ) else: - start = time.time() + start_time = time.monotonic() self.crawler_process.start() - stop = time.time() + stop = time.monotonic() result.printErrors() - result.printSummary(start, stop) + result.printSummary(start_time, stop) self.exitcode = int(not result.wasSuccessful()) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 2f0f1c7b9..866ba9f6b 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,37 +1,34 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError +if TYPE_CHECKING: + import argparse + class Command(BaseRunSpiderCommand): requires_project = True - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Run a spider" - def run(self, args, opts): + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) < 1: - raise UsageError() - elif len(args) > 1: + raise UsageError + if len(args) > 1: raise UsageError( "running 'scrapy crawl' with more than one spider is not supported" ) spname = args[0] - crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) - - if getattr(crawl_defer, "result", None) is not None and issubclass( - crawl_defer.result.type, Exception - ): + assert self.crawler_process + self.crawler_process.crawl(spname, **opts.spargs) + self.crawler_process.start() + if self.crawler_process.bootstrap_failed: self.exitcode = 1 - else: - self.crawler_process.start() - - if ( - self.crawler_process.bootstrap_failed - or hasattr(self.crawler_process, "has_exception") - and self.crawler_process.has_exception - ): - self.exitcode = 1 diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 03a8ed5c7..cd7c57f28 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -1,40 +1,48 @@ +import argparse import os import sys +from typing import Any, ClassVar from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError +from scrapy.spiderloader import get_spider_loader class Command(ScrapyCommand): requires_project = True - default_settings = {"LOG_ENABLED": False} + requires_crawler_process = False + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return "" - def short_desc(self): + def short_desc(self) -> str: return "Edit spider" - def long_desc(self): + def long_desc(self) -> str: return ( "Edit a spider using the editor defined in the EDITOR environment" " variable or else the EDITOR setting" ) - def _err(self, msg): + def _err(self, msg: str) -> None: sys.stderr.write(msg + os.linesep) self.exitcode = 1 - def run(self, args, opts): + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) != 1: - raise UsageError() + raise UsageError + assert self.settings is not None editor = self.settings["EDITOR"] + spider_loader = get_spider_loader(self.settings) try: - spidercls = self.crawler_process.spider_loader.load(args[0]) + spidercls = spider_loader.load(args[0]) except KeyError: - return self._err(f"Spider not found: {args[0]}") + self._err(f"Spider not found: {args[0]}") + return sfile = sys.modules[spidercls.__module__].__file__ + assert sfile sfile = sfile.replace(".pyc", ".py") - self.exitcode = os.system(f'{editor} "{sfile}"') + self.exitcode = os.system(f'{editor} "{sfile}"') # noqa: S605 diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index cdb7ad4ae..0b8311efb 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,34 +1,39 @@ +from __future__ import annotations + import sys -from argparse import Namespace -from typing import List, Type +from argparse import Namespace # noqa: TC003 +from typing import TYPE_CHECKING, Any from w3lib.url import is_url -from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import DefaultSpider, spidercls_for_request +if TYPE_CHECKING: + from argparse import ArgumentParser + from collections.abc import AsyncIterator + + from scrapy import Spider + class Command(ScrapyCommand): - requires_project = False - - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Fetch a URL using the Scrapy downloader" - def long_desc(self): + def long_desc(self) -> str: return ( "Fetch a URL using the Scrapy downloader and print its content" " to stdout. You may want to use --nolog to disable logging" ) - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: ArgumentParser) -> None: + super().add_options(parser) parser.add_argument("--spider", dest="spider", help="use this spider") parser.add_argument( "--headers", @@ -44,25 +49,26 @@ class Command(ScrapyCommand): help="do not handle HTTP 3xx status codes and print response as-is", ) - def _print_headers(self, headers, prefix): + def _print_headers(self, headers: dict[bytes, list[bytes]], prefix: bytes) -> None: for key, values in headers.items(): for value in values: self._print_bytes(prefix + b" " + key + b": " + value) - def _print_response(self, response, opts): + def _print_response(self, response: Response, opts: Namespace) -> None: if opts.headers: + assert response.request self._print_headers(response.request.headers, b">") print(">") self._print_headers(response.headers, b"<") else: self._print_bytes(response.body) - def _print_bytes(self, bytes_): + def _print_bytes(self, bytes_: bytes) -> None: sys.stdout.buffer.write(bytes_ + b"\n") - def run(self, args: List[str], opts: Namespace) -> None: + def run(self, args: list[str], opts: Namespace) -> None: if len(args) != 1 or not is_url(args[0]): - raise UsageError() + raise UsageError request = Request( args[0], callback=self._print_response, @@ -76,12 +82,18 @@ class Command(ScrapyCommand): else: request.meta["handle_httpstatus_all"] = True - spidercls: Type[Spider] = DefaultSpider + spidercls: type[Spider] = DefaultSpider assert self.crawler_process spider_loader = self.crawler_process.spider_loader if opts.spider: spidercls = spider_loader.load(opts.spider) else: spidercls = spidercls_for_request(spider_loader, request, spidercls) - self.crawler_process.crawl(spidercls, start_requests=lambda: [request]) + + async def start(self: Spider) -> AsyncIterator[Any]: + yield request + + spidercls.start = start # type: ignore[method-assign] + + self.crawler_process.crawl(spidercls) self.crawler_process.start() diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 68cbe8ff6..cc8624fa1 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -1,18 +1,24 @@ +from __future__ import annotations + import os import shutil import string from importlib import import_module from pathlib import Path -from typing import Optional, cast +from typing import TYPE_CHECKING, Any, ClassVar, cast from urllib.parse import urlparse import scrapy from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError +from scrapy.spiderloader import get_spider_loader from scrapy.utils.template import render_templatefile, string_camelcase +if TYPE_CHECKING: + import argparse -def sanitize_module_name(module_name): + +def sanitize_module_name(module_name: str) -> str: """Sanitize the given module name, by replacing dashes and points with underscores and prefixing it with a letter if it doesn't start with one @@ -23,7 +29,7 @@ def sanitize_module_name(module_name): return module_name -def extract_domain(url): +def extract_domain(url: str) -> str: """Extract domain name from URL string""" o = urlparse(url) if o.scheme == "" and o.netloc == "": @@ -31,7 +37,7 @@ def extract_domain(url): return o.netloc -def verify_url_scheme(url): +def verify_url_scheme(url: str) -> str: """Check url for scheme and insert https if none found.""" parsed = urlparse(url) if parsed.scheme == "" and parsed.netloc == "": @@ -40,17 +46,17 @@ def verify_url_scheme(url): class Command(ScrapyCommand): - requires_project = False - default_settings = {"LOG_ENABLED": False} + requires_crawler_process = False + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Generate new spider using pre-defined templates" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-l", "--list", @@ -86,7 +92,8 @@ class Command(ScrapyCommand): help="If the spider already exists, overwrite it with the template", ) - def run(self, args, opts): + def run(self, args: list[str], opts: argparse.Namespace) -> None: + assert self.settings is not None if opts.list: self._list_templates() return @@ -96,7 +103,7 @@ class Command(ScrapyCommand): print(template_file.read_text(encoding="utf-8")) return if len(args) != 2: - raise UsageError() + raise UsageError name, url = args[0:2] url = verify_url_scheme(url) @@ -113,27 +120,45 @@ class Command(ScrapyCommand): if template_file: self._genspider(module, name, url, opts.template, template_file) if opts.edit: - self.exitcode = os.system(f'scrapy edit "{name}"') + self.exitcode = os.system(f'scrapy edit "{name}"') # noqa: S605 - def _genspider(self, module, name, url, template_name, template_file): - """Generate the spider module, based on the given template""" + def _generate_template_variables( + self, + module: str, + name: str, + url: str, + template_name: str, + ) -> dict[str, Any]: + assert self.settings is not None capitalized_module = "".join(s.capitalize() for s in module.split("_")) - domain = extract_domain(url) - tvars = { + return { "project_name": self.settings.get("BOT_NAME"), "ProjectName": string_camelcase(self.settings.get("BOT_NAME")), "module": module, "name": name, "url": url, - "domain": domain, + "domain": extract_domain(url), "classname": f"{capitalized_module}Spider", } + + def _genspider( + self, + module: str, + name: str, + url: str, + template_name: str, + template_file: str | os.PathLike[str], + ) -> None: + """Generate the spider module, based on the given template""" + assert self.settings is not None + tvars = self._generate_template_variables(module, name, url, template_name) if self.settings.get("NEWSPIDER_MODULE"): spiders_module = import_module(self.settings["NEWSPIDER_MODULE"]) + assert spiders_module.__file__ spiders_dir = Path(spiders_module.__file__).parent.resolve() else: spiders_module = None - spiders_dir = Path(".") + spiders_dir = Path() spider_file = f"{spiders_dir / module}.py" shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) @@ -144,21 +169,28 @@ class Command(ScrapyCommand): if spiders_module: print(f"in module:\n {spiders_module.__name__}.{module}") - def _find_template(self, template: str) -> Optional[Path]: + def _find_template(self, template: str) -> Path | None: template_file = Path(self.templates_dir, f"{template}.tmpl") if template_file.exists(): return template_file - print(f"Unable to find template: {template}\n") - print('Use "scrapy genspider --list" to see all available templates.') + print( + f"Unable to find template: {template}\n", + 'Use "scrapy genspider --list" to see all available templates.', + ) return None - def _list_templates(self): - print("Available templates:") - for file in sorted(Path(self.templates_dir).iterdir()): - if file.suffix == ".tmpl": - print(f" {file.stem}") + def _list_templates(self) -> None: + print( + "Available templates:\n", + "\n".join( + f" {file.stem}" + for file in sorted(Path(self.templates_dir).iterdir()) + if file.suffix == ".tmpl" + ), + ) def _spider_exists(self, name: str) -> bool: + assert self.settings is not None if not self.settings.get("NEWSPIDER_MODULE"): # if run as a standalone command and file with same filename already exists path = Path(name + ".py") @@ -167,23 +199,22 @@ class Command(ScrapyCommand): return True return False - assert ( - self.crawler_process is not None - ), "crawler_process must be set before calling run" - + spider_loader = get_spider_loader(self.settings) try: - spidercls = self.crawler_process.spider_loader.load(name) + spidercls = spider_loader.load(name) except KeyError: pass else: # if spider with same name exists - print(f"Spider {name!r} already exists in module:") - print(f" {spidercls.__module__}") + print( + f"Spider {name!r} already exists in module:\n", + f" {spidercls.__module__}", + ) return True # a file with the same name exists in the target directory spiders_module = import_module(self.settings["NEWSPIDER_MODULE"]) - spiders_dir = Path(cast(str, spiders_module.__file__)).parent + spiders_dir = Path(cast("str", spiders_module.__file__)).parent spiders_dir_abs = spiders_dir.resolve() path = spiders_dir_abs / (name + ".py") if path.exists(): @@ -194,6 +225,7 @@ class Command(ScrapyCommand): @property def templates_dir(self) -> str: + assert self.settings is not None return str( Path( self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"), diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 2f5032360..a037f5c9b 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,13 +1,23 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar + from scrapy.commands import ScrapyCommand +from scrapy.spiderloader import get_spider_loader + +if TYPE_CHECKING: + import argparse class Command(ScrapyCommand): requires_project = True - default_settings = {"LOG_ENABLED": False} + requires_crawler_process = False + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} - def short_desc(self): + def short_desc(self) -> str: return "List available spiders" - def run(self, args, opts): - for s in sorted(self.crawler_process.spider_loader.list()): - print(s) + def run(self, args: list[str], opts: argparse.Namespace) -> None: + assert self.settings is not None + spider_loader = get_spider_loader(self.settings) + print("\n".join(sorted(spider_loader.list()))) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index c9f8586d3..2ac65bf3f 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,43 +1,58 @@ +from __future__ import annotations + import functools import inspect import json import logging -from typing import Dict +from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, overload -from itemadapter import ItemAdapter, is_item -from twisted.internet.defer import maybeDeferred +from itemadapter import ItemAdapter +from twisted.internet.defer import Deferred, maybeDeferred from w3lib.url import is_url from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.utils import display from scrapy.utils.asyncgen import collect_asyncgen -from scrapy.utils.defer import aiter_errback, deferred_from_coro +from scrapy.utils.defer import _schedule_coro, aiter_errback, deferred_from_coro from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter from scrapy.utils.spider import spidercls_for_request +if TYPE_CHECKING: + import argparse + from collections.abc import AsyncGenerator, AsyncIterator, Coroutine, Iterable + + from twisted.python.failure import Failure + + from scrapy.http.request import CallbackT + from scrapy.spiders import Spider + + logger = logging.getLogger(__name__) +_T = TypeVar("_T") + class Command(BaseRunSpiderCommand): requires_project = True - spider = None - items: Dict[int, list] = {} - requests: Dict[int, list] = {} + spider: Spider | None = None + items: ClassVar[dict[int, list[Any]]] = {} + requests: ClassVar[dict[int, list[Request]]] = {} + spidercls: type[Spider] | None first_response = None - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Parse URL (using its spider) and print the results" - def add_options(self, parser): - BaseRunSpiderCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "--spider", dest="spider", @@ -106,7 +121,7 @@ class Command(BaseRunSpiderCommand): ) @property - def max_level(self): + def max_level(self) -> int: max_items, max_requests = 0, 0 if self.items: max_items = max(self.items) @@ -114,34 +129,41 @@ class Command(BaseRunSpiderCommand): max_requests = max(self.requests) return max(max_items, max_requests) - def handle_exception(self, _failure): + def handle_exception(self, _failure: Failure) -> None: logger.error( "An error is caught while iterating the async iterable", exc_info=failure_to_exc_info(_failure), ) - def iterate_spider_output(self, result): + @overload + def iterate_spider_output( + self, result: AsyncGenerator[_T] | Coroutine[Any, Any, _T] + ) -> Deferred[_T]: ... + + @overload + def iterate_spider_output(self, result: _T) -> Iterable[Any]: ... + + def iterate_spider_output(self, result: Any) -> Iterable[Any] | Deferred[Any]: + d: Deferred[Any] if inspect.isasyncgen(result): d = deferred_from_coro( collect_asyncgen(aiter_errback(result, self.handle_exception)) ) - d.addCallback(self.iterate_spider_output) - return d + return d.addCallback(self.iterate_spider_output) + d = deferred_from_coro(result) if inspect.iscoroutine(result): - d = deferred_from_coro(result) - d.addCallback(self.iterate_spider_output) - return d - return arg_to_iter(deferred_from_coro(result)) + return d.addCallback(self.iterate_spider_output) + return arg_to_iter(d) - def add_items(self, lvl, new_items): + def add_items(self, lvl: int, new_items: list[Any]) -> None: old_items = self.items.get(lvl, []) self.items[lvl] = old_items + new_items - def add_requests(self, lvl, new_reqs): + def add_requests(self, lvl: int, new_reqs: list[Request]) -> None: old_reqs = self.requests.get(lvl, []) self.requests[lvl] = old_reqs + new_reqs - def print_items(self, lvl=None, colour=True): + def print_items(self, lvl: int | None = None, colour: bool = True) -> None: if lvl is None: items = [item for lst in self.items.values() for item in lst] else: @@ -150,19 +172,18 @@ class Command(BaseRunSpiderCommand): print("# Scraped Items ", "-" * 60) display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour) - def print_requests(self, lvl=None, colour=True): - if lvl is None: - if self.requests: - requests = self.requests[max(self.requests)] - else: - requests = [] - else: + def print_requests(self, lvl: int | None = None, colour: bool = True) -> None: + if lvl is not None: requests = self.requests.get(lvl, []) + elif self.requests: + requests = self.requests[max(self.requests)] + else: + requests = [] print("# Requests ", "-" * 65) display.pprint(requests, colorize=colour) - def print_results(self, opts): + def print_results(self, opts: argparse.Namespace) -> None: colour = not opts.nocolour if opts.verbose: @@ -179,23 +200,38 @@ class Command(BaseRunSpiderCommand): if not opts.nolinks: self.print_requests(colour=colour) - def _get_items_and_requests(self, spider_output, opts, depth, spider, callback): + def _get_items_and_requests( + self, + spider_output: Iterable[Any], + opts: argparse.Namespace, + depth: int, + spider: Spider, + callback: CallbackT, + ) -> tuple[list[Any], list[Request], argparse.Namespace, int, Spider, CallbackT]: items, requests = [], [] for x in spider_output: - if is_item(x): - items.append(x) - elif isinstance(x, Request): + if isinstance(x, Request): requests.append(x) + else: + items.append(x) return items, requests, opts, depth, spider, callback - def run_callback(self, response, callback, cb_kwargs=None): + def run_callback( + self, + response: Response, + callback: CallbackT, + cb_kwargs: dict[str, Any] | None = None, + ) -> Deferred[Any]: cb_kwargs = cb_kwargs or {} - d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) - return d + return maybeDeferred( + self.iterate_spider_output, callback(response, **cb_kwargs) + ) - def get_callback_from_rules(self, spider, response): + def get_callback_from_rules( + self, spider: Spider, response: Response + ) -> CallbackT | str | None: if getattr(spider, "rules", None): - for rule in spider.rules: + for rule in spider.rules: # type: ignore[attr-defined] if rule.link_extractor.matches(response.url): return rule.callback or "parse" else: @@ -204,8 +240,10 @@ class Command(BaseRunSpiderCommand): "please specify a callback to use for parsing", {"spider": spider.name}, ) + return None - def set_spidercls(self, url, opts): + def set_spidercls(self, url: str, opts: argparse.Namespace) -> None: + assert self.crawler_process spider_loader = self.crawler_process.spider_loader if opts.spider: try: @@ -219,26 +257,38 @@ class Command(BaseRunSpiderCommand): if not self.spidercls: logger.error("Unable to find spider for: %(url)s", {"url": url}) - def _start_requests(spider): + async def start(spider: Spider) -> AsyncIterator[Any]: yield self.prepare_request(spider, Request(url), opts) if self.spidercls: - self.spidercls.start_requests = _start_requests + self.spidercls.start = start # type: ignore[assignment,method-assign] - def start_parsing(self, url, opts): + def start_parsing(self, url: str, opts: argparse.Namespace) -> None: + assert self.crawler_process + assert self.spidercls self.crawler_process.crawl(self.spidercls, **opts.spargs) - self.pcrawler = list(self.crawler_process.crawlers)[0] + self.pcrawler = next(iter(self.crawler_process.crawlers)) self.crawler_process.start() if not self.first_response: logger.error("No response downloaded for: %(url)s", {"url": url}) - def scraped_data(self, args): + def scraped_data( + self, + args: tuple[ + list[Any], list[Request], argparse.Namespace, int, Spider, CallbackT + ], + ) -> list[Any]: items, requests, opts, depth, spider, callback = args if opts.pipelines: + assert self.pcrawler.engine itemproc = self.pcrawler.engine.scraper.itemproc - for item in items: - itemproc.process_item(item, spider) + if hasattr(itemproc, "process_item_async"): + for item in items: + _schedule_coro(itemproc.process_item_async(item)) + else: + for item in items: + itemproc.process_item(item, spider) self.add_items(depth, items) self.add_requests(depth, requests) @@ -252,8 +302,14 @@ class Command(BaseRunSpiderCommand): return scraped_data - def _get_callback(self, *, spider, opts, response=None): - cb = None + def _get_callback( + self, + *, + spider: Spider, + opts: argparse.Namespace, + response: Response | None = None, + ) -> CallbackT: + cb: str | CallbackT | None = None if response: cb = response.meta["_callback"] if not cb: @@ -270,6 +326,7 @@ class Command(BaseRunSpiderCommand): cb = "parse" if not callable(cb): + assert cb is not None cb_method = getattr(spider, cb, None) if callable(cb_method): cb = cb_method @@ -277,10 +334,13 @@ class Command(BaseRunSpiderCommand): raise ValueError( f"Cannot find callback {cb!r} in spider: {spider.name}" ) + assert callable(cb) return cb - def prepare_request(self, spider, request, opts): - def callback(response, **cb_kwargs): + def prepare_request( + self, spider: Spider, request: Request, opts: argparse.Namespace + ) -> Request: + def callback(response: Response, **cb_kwargs: Any) -> Deferred[list[Any]]: # memorize first request if not self.first_response: self.first_response = response @@ -288,7 +348,7 @@ class Command(BaseRunSpiderCommand): cb = self._get_callback(spider=spider, opts=opts, response=response) # parse items and requests - depth = response.meta["_depth"] + depth: int = response.meta["_depth"] d = self.run_callback(response, cb, cb_kwargs) d.addCallback(self._get_items_and_requests, opts, depth, spider, callback) @@ -311,13 +371,13 @@ class Command(BaseRunSpiderCommand): request.callback = callback return request - def process_options(self, args, opts): - BaseRunSpiderCommand.process_options(self, args, opts) + def process_options(self, args: list[str], opts: argparse.Namespace) -> None: + super().process_options(args, opts) self.process_request_meta(opts) self.process_request_cb_kwargs(opts) - def process_request_meta(self, opts): + def process_request_meta(self, opts: argparse.Namespace) -> None: if opts.meta: try: opts.meta = json.loads(opts.meta) @@ -326,9 +386,9 @@ class Command(BaseRunSpiderCommand): "Invalid -m/--meta value, pass a valid json string to -m or --meta. " 'Example: --meta=\'{"foo" : "bar"}\'', print_help=False, - ) + ) from None - def process_request_cb_kwargs(self, opts): + def process_request_cb_kwargs(self, opts: argparse.Namespace) -> None: if opts.cbkwargs: try: opts.cbkwargs = json.loads(opts.cbkwargs) @@ -337,14 +397,13 @@ class Command(BaseRunSpiderCommand): "Invalid --cbkwargs value, pass a valid json string to --cbkwargs. " 'Example: --cbkwargs=\'{"foo" : "bar"}\'', print_help=False, - ) + ) from None - def run(self, args, opts): + def run(self, args: list[str], opts: argparse.Namespace) -> None: # parse arguments if not len(args) == 1 or not is_url(args[0]): - raise UsageError() - else: - url = args[0] + raise UsageError + url = args[0] # prepare spidercls self.set_spidercls(url, opts) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 58ed89a81..0b9036457 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,21 +1,27 @@ +from __future__ import annotations + import sys from importlib import import_module -from os import PathLike from pathlib import Path -from types import ModuleType -from typing import Union +from typing import TYPE_CHECKING, Any, ClassVar from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError +from scrapy.spiderloader import DummySpiderLoader from scrapy.utils.spider import iter_spider_classes +if TYPE_CHECKING: + import argparse + from os import PathLike + from types import ModuleType -def _import_file(filepath: Union[str, PathLike]) -> ModuleType: + +def _import_file(filepath: str | PathLike[str]) -> ModuleType: abspath = Path(filepath).resolve() - if abspath.suffix not in (".py", ".pyw"): + if abspath.suffix not in {".py", ".pyw"}: raise ValueError(f"Not a Python source file: {abspath}") dirname = str(abspath.parent) - sys.path = [dirname] + sys.path + sys.path = [dirname, *sys.path] try: module = import_module(abspath.stem) finally: @@ -24,33 +30,35 @@ def _import_file(filepath: Union[str, PathLike]) -> ModuleType: class Command(BaseRunSpiderCommand): - requires_project = False - default_settings = {"SPIDER_LOADER_WARN_ONLY": True} + default_settings: ClassVar[dict[str, Any]] = { + "SPIDER_LOADER_CLASS": DummySpiderLoader + } - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Run a self-contained spider (without creating a project)" - def long_desc(self): + def long_desc(self) -> str: return "Run the spider defined in the given file" - def run(self, args, opts): + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) != 1: - raise UsageError() + raise UsageError filename = Path(args[0]) if not filename.exists(): raise UsageError(f"File not found: {filename}\n") try: module = _import_file(filename) except (ImportError, ValueError) as e: - raise UsageError(f"Unable to load {str(filename)!r}: {e}\n") + raise UsageError(f"Unable to load {str(filename)!r}: {e}\n") from e spclasses = list(iter_spider_classes(module)) if not spclasses: raise UsageError(f"No spider found in file: {filename}\n") spidercls = spclasses.pop() + assert self.crawler_process self.crawler_process.crawl(spidercls, **opts.spargs) self.crawler_process.start() diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 318187204..26a97ccbb 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,21 +1,23 @@ +import argparse import json +from typing import Any, ClassVar from scrapy.commands import ScrapyCommand from scrapy.settings import BaseSettings class Command(ScrapyCommand): - requires_project = False - default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} + requires_crawler_process = False + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return "[options]" - def short_desc(self): + def short_desc(self) -> str: return "Get settings values" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "--get", dest="get", metavar="SETTING", help="print raw setting value" ) @@ -44,8 +46,9 @@ class Command(ScrapyCommand): help="print setting value, interpreted as a list", ) - def run(self, args, opts): - settings = self.crawler_process.settings + def run(self, args: list[str], opts: argparse.Namespace) -> None: + assert self.settings is not None + settings = self.settings if opts.get: s = settings.get(opts.get) if isinstance(s, BaseSettings): diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 12e37babc..19138ffd0 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -3,40 +3,48 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ -from argparse import Namespace -from threading import Thread -from typing import List, Type -from scrapy import Spider +from __future__ import annotations + +import asyncio +from threading import Thread +from typing import TYPE_CHECKING, Any, ClassVar + from scrapy.commands import ScrapyCommand +from scrapy.crawler import AsyncCrawlerProcess, Crawler from scrapy.http import Request from scrapy.shell import Shell +from scrapy.utils.defer import _schedule_coro from scrapy.utils.spider import DefaultSpider, spidercls_for_request from scrapy.utils.url import guess_scheme +if TYPE_CHECKING: + from argparse import ArgumentParser, Namespace + + from scrapy import Spider + class Command(ScrapyCommand): - requires_project = False - default_settings = { + default_settings: ClassVar[dict[str, Any]] = { + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", "KEEP_ALIVE": True, "LOGSTATS_INTERVAL": 0, - "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", } - def syntax(self): + def syntax(self) -> str: return "[url|file]" - def short_desc(self): + def short_desc(self) -> str: return "Interactive scraping console" - def long_desc(self): + def long_desc(self) -> str: return ( "Interactive console for scraping the given url or file. " "Use ./file.html syntax or full path for local file." ) - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-c", dest="code", @@ -51,13 +59,12 @@ class Command(ScrapyCommand): help="do not handle HTTP 3xx status codes and print response as-is", ) - def update_vars(self, vars): + def update_vars(self, vars: dict[str, Any]) -> None: # noqa: A002 """You can use this function to update the Scrapy objects that will be available in the shell """ - pass - def run(self, args: List[str], opts: Namespace) -> None: + def run(self, args: list[str], opts: Namespace) -> None: url = args[0] if args else None if url: # first argument may be a local file @@ -66,7 +73,7 @@ class Command(ScrapyCommand): assert self.crawler_process spider_loader = self.crawler_process.spider_loader - spidercls: Type[Spider] = DefaultSpider + spidercls: type[Spider] = DefaultSpider if opts.spider: spidercls = spider_loader.load(opts.spider) elif url: @@ -78,16 +85,48 @@ class Command(ScrapyCommand): # crawling engine, so the set up in the crawl method won't work crawler = self.crawler_process._create_crawler(spidercls) crawler._apply_settings() - # The Shell class needs a persistent engine in the crawler - crawler.engine = crawler._create_engine() - crawler.engine.start() - - self._start_crawler_thread() - - shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) + loop: asyncio.AbstractEventLoop | None = None + if crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): + self._init_with_reactor(crawler) + else: + self._init_without_reactor(crawler) + loop = self._get_reactorless_loop() + shell = Shell(crawler, update_vars=self.update_vars, code=opts.code, loop=loop) shell.start(url=url, redirect=not opts.no_redirect) - def _start_crawler_thread(self): + def _init_with_reactor(self, crawler: Crawler) -> None: + # Create the engine and run start_async() in the main thread + crawler.engine = crawler._create_engine() + _schedule_coro(crawler.engine.start_async(_start_request_processing=False)) + self._start_crawler_thread() + + def _init_without_reactor(self, crawler: Crawler) -> None: + # Create the engine and run start_async() in the event loop thread + loop = self._get_reactorless_loop() + self._start_crawler_thread() + + async def _init_engine() -> None: + # We may need to wait until some parts of start_async() have + # finished, which may need a special event in the engine and may + # wait until https://github.com/scrapy/scrapy/issues/6916 + crawler.engine = crawler._create_engine() + loop.create_task( + crawler.engine.start_async(_start_request_processing=False) + ) + + future = asyncio.run_coroutine_threadsafe(_init_engine(), loop) + future.result() + + def _get_reactorless_loop(self) -> asyncio.AbstractEventLoop: + assert self.crawler_process + assert isinstance(self.crawler_process, AsyncCrawlerProcess) + loop = self.crawler_process._reactorless_loop + assert loop + return loop + + def _start_crawler_thread(self) -> None: + """Run self.crawler_process.start() in a separate thread.""" + assert self.crawler_process t = Thread( target=self.crawler_process.start, kwargs={"stop_after_crawl": False, "install_signal_handlers": False}, diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index fde609c6f..5661a0a48 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,17 +1,22 @@ -import os +from __future__ import annotations + import re import string from importlib.util import find_spec from pathlib import Path from shutil import copy2, copystat, ignore_patterns, move from stat import S_IWUSR as OWNER_WRITE_PERMISSION +from typing import TYPE_CHECKING, Any, ClassVar import scrapy from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.template import render_templatefile, string_camelcase -TEMPLATES_TO_RENDER = ( +if TYPE_CHECKING: + import argparse + +TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = ( ("scrapy.cfg",), ("${project_name}", "settings.py.tmpl"), ("${project_name}", "items.py.tmpl"), @@ -22,23 +27,23 @@ TEMPLATES_TO_RENDER = ( IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn") -def _make_writable(path): - current_permissions = os.stat(path).st_mode - os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION) +def _make_writable(path: Path) -> None: + current_permissions = path.stat().st_mode + path.chmod(current_permissions | OWNER_WRITE_PERMISSION) class Command(ScrapyCommand): - requires_project = False - default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} + requires_crawler_process = False + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return " [project_dir]" - def short_desc(self): + def short_desc(self) -> str: return "Create new project" - def _is_valid_name(self, project_name): - def _module_exists(module_name): + def _is_valid_name(self, project_name: str) -> bool: + def _module_exists(module_name: str) -> bool: spec = find_spec(module_name) return spec is not None and spec.loader is not None @@ -53,7 +58,7 @@ class Command(ScrapyCommand): return True return False - def _copytree(self, src: Path, dst: Path): + def _copytree(self, src: Path, dst: Path) -> None: """ Since the original function always creates the directory, to resolve the issue a new function had to be created. It's a simple copy and @@ -84,16 +89,13 @@ class Command(ScrapyCommand): copystat(src, dst) _make_writable(dst) - def run(self, args, opts): - if len(args) not in (1, 2): - raise UsageError() + def run(self, args: list[str], opts: argparse.Namespace) -> None: + if len(args) not in {1, 2}: + raise UsageError project_name = args[0] - if len(args) == 2: - project_dir = Path(args[1]) - else: - project_dir = Path(args[0]) + project_dir = Path(args[-1]) if (project_dir / "scrapy.cfg").exists(): self.exitcode = 1 @@ -121,15 +123,16 @@ class Command(ScrapyCommand): ) print( f"New Scrapy project '{project_name}', using template directory " - f"'{self.templates_dir}', created in:" + f"'{self.templates_dir}', created in:\n", + f" {project_dir.resolve()}\n\n", + "You can start your first spider with:\n", + f" cd {project_dir}\n", + " scrapy genspider example example.com", ) - print(f" {project_dir.resolve()}\n") - print("You can start your first spider with:") - print(f" cd {project_dir}") - print(" scrapy genspider example example.com") @property def templates_dir(self) -> str: + assert self.settings is not None return str( Path( self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"), diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 47582866b..1d1985d4e 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -1,19 +1,23 @@ +import argparse +from typing import Any, ClassVar + import scrapy from scrapy.commands import ScrapyCommand -from scrapy.utils.versions import scrapy_components_versions +from scrapy.utils.versions import get_versions class Command(ScrapyCommand): - default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} + requires_crawler_process = False + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return "[-v]" - def short_desc(self): + def short_desc(self) -> str: return "Print Scrapy version" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "--verbose", "-v", @@ -22,9 +26,9 @@ class Command(ScrapyCommand): help="also display twisted/python/platform info (useful for bug reports)", ) - def run(self, args, opts): + def run(self, args: list[str], opts: argparse.Namespace) -> None: if opts.verbose: - versions = scrapy_components_versions() + versions = get_versions() width = max(len(n) for (n, _) in versions) for name, version in versions: print(f"{name:<{width}} : {version}") diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index ebdfa10a8..21679e3aa 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -1,21 +1,28 @@ import argparse +import logging from scrapy.commands import fetch +from scrapy.http import Response, TextResponse from scrapy.utils.response import open_in_browser +logger = logging.getLogger(__name__) + class Command(fetch.Command): - def short_desc(self): + def short_desc(self) -> str: return "Open URL in browser, as seen by Scrapy" - def long_desc(self): + def long_desc(self) -> str: return ( "Fetch a URL using the Scrapy downloader and show its contents in a browser" ) - def add_options(self, parser): + def add_options(self, parser: argparse.ArgumentParser) -> None: super().add_options(parser) parser.add_argument("--headers", help=argparse.SUPPRESS) - def _print_response(self, response, opts): + def _print_response(self, response: Response, opts: argparse.Namespace) -> None: + if not isinstance(response, TextResponse): + logger.error("Cannot view a non-text response.") + return open_in_browser(response) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index d46eb7c51..c0da7dfa4 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -1,32 +1,44 @@ +from __future__ import annotations + import re import sys +from collections.abc import AsyncGenerator, Iterable from functools import wraps from inspect import getmembers from types import CoroutineType -from typing import AsyncGenerator, Dict, Optional, Type -from unittest import TestCase +from typing import TYPE_CHECKING, Any, ClassVar, cast +from unittest import TestCase, TestResult -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.utils.python import get_spec from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + from collections.abc import Callable + + from twisted.python.failure import Failure + + from scrapy import Spider + class Contract: """Abstract class for contracts""" - request_cls: Optional[Type[Request]] = None + request_cls: type[Request] | None = None + name: str - def __init__(self, method, *args): + def __init__(self, method: Callable[..., Any], *args: Any): self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook") self.testcase_post = _create_testcase(method, f"@{self.name} post-hook") - self.args = args + self.args: tuple[Any, ...] = args - def add_pre_hook(self, request, results): + def add_pre_hook(self, request: Request, results: TestResult) -> Request: if hasattr(self, "pre_process"): cb = request.callback + assert cb is not None @wraps(cb) - def wrapper(response, **cb_kwargs): + def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]: try: results.startTest(self.testcase_pre) self.pre_process(response) @@ -37,28 +49,26 @@ class Contract: results.addError(self.testcase_pre, sys.exc_info()) else: results.addSuccess(self.testcase_pre) - finally: - cb_result = cb(response, **cb_kwargs) - if isinstance(cb_result, (AsyncGenerator, CoroutineType)): - raise TypeError("Contracts don't support async callbacks") - return list( # pylint: disable=return-in-finally - iterate_spider_output(cb_result) - ) + cb_result = cb(response, **cb_kwargs) + if isinstance(cb_result, (AsyncGenerator, CoroutineType)): + raise TypeError("Contracts don't support async callbacks") + return list(cast("Iterable[Any]", iterate_spider_output(cb_result))) request.callback = wrapper return request - def add_post_hook(self, request, results): + def add_post_hook(self, request: Request, results: TestResult) -> Request: if hasattr(self, "post_process"): cb = request.callback + assert cb is not None @wraps(cb) - def wrapper(response, **cb_kwargs): + def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") - output = list(iterate_spider_output(cb_result)) + output = list(cast("Iterable[Any]", iterate_spider_output(cb_result))) try: results.startTest(self.testcase_post) self.post_process(output) @@ -69,25 +79,24 @@ class Contract: results.addError(self.testcase_post, sys.exc_info()) else: results.addSuccess(self.testcase_post) - finally: - return output # pylint: disable=return-in-finally + return output request.callback = wrapper return request - def adjust_request_args(self, args): + def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: return args class ContractsManager: - contracts: Dict[str, Contract] = {} + contracts: ClassVar[dict[str, type[Contract]]] = {} - def __init__(self, contracts): + def __init__(self, contracts: Iterable[type[Contract]]): for contract in contracts: self.contracts[contract.name] = contract - def tested_methods_from_spidercls(self, spidercls): + def tested_methods_from_spidercls(self, spidercls: type[Spider]) -> list[str]: is_method = re.compile(r"^\s*@", re.MULTILINE).search methods = [] for key, value in getmembers(spidercls): @@ -96,23 +105,27 @@ class ContractsManager: return methods - def extract_contracts(self, method): - contracts = [] - for line in method.__doc__.split("\n"): - line = line.strip() + def extract_contracts(self, method: Callable[..., Any]) -> list[Contract]: + contracts: list[Contract] = [] + assert method.__doc__ is not None + for line_ in method.__doc__.split("\n"): + line = line_.strip() if line.startswith("@"): - name, args = re.match(r"@(\w+)\s*(.*)", line).groups() + m = re.match(r"@(\w+)\s*(.*)", line) + if m is None: + continue + name, args = m.groups() args = re.split(r"\s+", args) contracts.append(self.contracts[name](method, *args)) return contracts - def from_spider(self, spider, results): - requests = [] + def from_spider(self, spider: Spider, results: TestResult) -> list[Request | None]: + requests: list[Request | None] = [] for method in self.tested_methods_from_spidercls(type(spider)): - bound_method = spider.__getattribute__(method) + bound_method = getattr(spider, method) try: requests.append(self.from_method(bound_method, results)) except Exception: @@ -121,7 +134,9 @@ class ContractsManager: return requests - def from_method(self, method, results): + def from_method( + self, method: Callable[..., Any], results: TestResult + ) -> Request | None: contracts = self.extract_contracts(method) if contracts: request_cls = Request @@ -154,35 +169,39 @@ class ContractsManager: self._clean_req(request, method, results) return request + return None - def _clean_req(self, request, method, results): + def _clean_req( + self, request: Request, method: Callable[..., Any], results: TestResult + ) -> None: """stop the request from returning objects and records any errors""" cb = request.callback + assert cb is not None @wraps(cb) - def cb_wrapper(response, **cb_kwargs): + def cb_wrapper(response: Response, **cb_kwargs: Any) -> None: try: output = cb(response, **cb_kwargs) - output = list(iterate_spider_output(output)) + output = list(cast("Iterable[Any]", iterate_spider_output(output))) except Exception: case = _create_testcase(method, "callback") results.addError(case, sys.exc_info()) - def eb_wrapper(failure): + def eb_wrapper(failure: Failure) -> None: case = _create_testcase(method, "errback") exc_info = failure.type, failure.value, failure.getTracebackObject() - results.addError(case, exc_info) + results.addError(case, exc_info) # type: ignore[arg-type] request.callback = cb_wrapper request.errback = eb_wrapper -def _create_testcase(method, desc): - spider = method.__self__.name +def _create_testcase(method: Callable[..., Any], desc: str) -> TestCase: + spider = method.__self__.name # type: ignore[attr-defined] class ContractTestCase(TestCase): - def __str__(_self): + def __str__(_self) -> str: # pylint: disable=no-self-argument return f"[{spider}] {method.__name__} ({desc})" name = f"{spider}_{method.__name__}" diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index eac702cef..9b42ca36f 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -1,4 +1,7 @@ +from __future__ import annotations + import json +from typing import TYPE_CHECKING, Any, ClassVar from itemadapter import ItemAdapter, is_item @@ -6,6 +9,9 @@ from scrapy.contracts import Contract from scrapy.exceptions import ContractFail from scrapy.http import Request +if TYPE_CHECKING: + from collections.abc import Callable + # contracts class UrlContract(Contract): @@ -15,7 +21,7 @@ class UrlContract(Contract): name = "url" - def adjust_request_args(self, args): + def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: args["url"] = self.args[0] return args @@ -29,11 +35,25 @@ class CallbackKeywordArgumentsContract(Contract): name = "cb_kwargs" - def adjust_request_args(self, args): + def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: args["cb_kwargs"] = json.loads(" ".join(self.args)) return args +class MetadataContract(Contract): + """Contract to set metadata arguments for the request. + The value should be JSON-encoded dictionary, e.g.: + + @meta {"arg1": "some value"} + """ + + name = "meta" + + def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: + args["meta"] = json.loads(" ".join(self.args)) + return args + + class ReturnsContract(Contract): """Contract to check the output of a callback @@ -48,17 +68,17 @@ class ReturnsContract(Contract): """ name = "returns" - object_type_verifiers = { + object_type_verifiers: ClassVar[dict[str | None, Callable[[Any], bool]]] = { "request": lambda x: isinstance(x, Request), "requests": lambda x: isinstance(x, Request), "item": is_item, "items": is_item, } - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) - if len(self.args) not in [1, 2, 3]: + if len(self.args) not in {1, 2, 3}: raise ValueError( f"Incorrect argument quantity: expected 1, 2 or 3, got {len(self.args)}" ) @@ -66,16 +86,16 @@ class ReturnsContract(Contract): self.obj_type_verifier = self.object_type_verifiers[self.obj_name] try: - self.min_bound = int(self.args[1]) + self.min_bound: float = int(self.args[1]) except IndexError: self.min_bound = 1 try: - self.max_bound = int(self.args[2]) + self.max_bound: float = int(self.args[2]) except IndexError: self.max_bound = float("inf") - def post_process(self, output): + def post_process(self, output: list[Any]) -> None: occurrences = 0 for x in output: if self.obj_type_verifier(x): @@ -85,7 +105,7 @@ class ReturnsContract(Contract): if not assertion: if self.min_bound == self.max_bound: - expected = self.min_bound + expected = str(self.min_bound) else: expected = f"{self.min_bound}..{self.max_bound}" @@ -101,7 +121,7 @@ class ScrapesContract(Contract): name = "scrapes" - def post_process(self, output): + def post_process(self, output: list[Any]) -> None: for x in output: if is_item(x): missing = [arg for arg in self.args if arg not in ItemAdapter(x)] diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index c84525160..7c0ee0eec 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,59 +1,74 @@ +from __future__ import annotations + import random from collections import deque +from dataclasses import dataclass, field from datetime import datetime -from time import time -from typing import TYPE_CHECKING, Any, Deque, Dict, Set, Tuple, cast +from time import monotonic +from typing import TYPE_CHECKING, Any -from twisted.internet import task -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, inlineCallbacks +from twisted.python.failure import Failure from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager -from scrapy.http import Response from scrapy.resolver import dnscache -from scrapy.settings import BaseSettings -from scrapy.signalmanager import SignalManager -from scrapy.utils.defer import mustbe_deferred +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + CallLaterResult, + call_later, + create_looping_call, +) +from scrapy.utils.decorators import _warn_spider_arg +from scrapy.utils.defer import ( + _defer_sleep_async, + _schedule_coro, + deferred_from_coro, + maybe_deferred_to_future, +) +from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: + from collections.abc import Generator + + from twisted.internet.task import LoopingCall + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings + from scrapy.signalmanager import SignalManager +@dataclass(slots=True, eq=False) class Slot: """Downloader slot""" - def __init__(self, concurrency: int, delay: float, randomize_delay: bool): - self.concurrency: int = concurrency - self.delay: float = delay - self.randomize_delay: bool = randomize_delay + concurrency: int + delay: float + randomize_delay: bool - self.active: Set[Request] = set() - self.queue: Deque[Tuple[Request, Deferred]] = deque() - self.transferring: Set[Request] = set() - self.lastseen: float = 0 - self.latercall = None + active: set[Request] = field(default_factory=set, init=False, repr=False) + queue: deque[tuple[Request, Deferred[Response]]] = field( + default_factory=deque, init=False, repr=False + ) + transferring: set[Request] = field(default_factory=set, init=False, repr=False) + lastseen: float = field(default=0, init=False, repr=False) + latercall: CallLaterResult | None = field(default=None, init=False, repr=False) def free_transfer_slots(self) -> int: return self.concurrency - len(self.transferring) def download_delay(self) -> float: if self.randomize_delay: - return random.uniform(0.5 * self.delay, 1.5 * self.delay) + return random.uniform(0.5 * self.delay, 1.5 * self.delay) # noqa: S311 return self.delay def close(self) -> None: - if self.latercall and self.latercall.active(): + if self.latercall: self.latercall.cancel() - - def __repr__(self) -> str: - cls_name = self.__class__.__name__ - return ( - f"{cls_name}(concurrency={self.concurrency!r}, " - f"delay={self.delay:.2f}, " - f"randomize_delay={self.randomize_delay!r})" - ) + self.latercall = None def __str__(self) -> str: return ( @@ -67,12 +82,15 @@ class Slot: def _get_concurrency_delay( concurrency: int, spider: Spider, settings: BaseSettings -) -> Tuple[int, float]: +) -> tuple[int, float]: delay: float = settings.getfloat("DOWNLOAD_DELAY") if hasattr(spider, "download_delay"): delay = spider.download_delay - if hasattr(spider, "max_concurrent_requests"): + if hasattr(spider, "max_concurrent_requests"): # pragma: no cover + warn_on_deprecated_spider_attribute( + "max_concurrent_requests", "CONCURRENT_REQUESTS" + ) concurrency = spider.max_concurrent_requests return concurrency, delay @@ -80,12 +98,14 @@ def _get_concurrency_delay( class Downloader: DOWNLOAD_SLOT = "download_slot" + _SLOT_GC_INTERVAL: float = 60.0 # seconds - def __init__(self, crawler: "Crawler"): + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler self.settings: BaseSettings = crawler.settings self.signals: SignalManager = crawler.signals - self.slots: Dict[str, Slot] = {} - self.active: Set[Request] = set() + self.slots: dict[str, Slot] = {} + self.active: set[Request] = set() self.handlers: DownloadHandlers = DownloadHandlers(crawler) self.total_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS") self.domain_concurrency: int = self.settings.getint( @@ -96,32 +116,42 @@ class Downloader: self.middleware: DownloaderMiddlewareManager = ( DownloaderMiddlewareManager.from_crawler(crawler) ) - self._slot_gc_loop: task.LoopingCall = task.LoopingCall(self._slot_gc) - self._slot_gc_loop.start(60) - self.per_slot_settings: Dict[str, Dict[str, Any]] = self.settings.getdict( - "DOWNLOAD_SLOTS", {} + self._slot_gc_loop: AsyncioLoopingCall | LoopingCall | None = None + self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict( + "DOWNLOAD_SLOTS" ) - def fetch(self, request: Request, spider: Spider) -> Deferred: - def _deactivate(response: Response) -> Response: - self.active.remove(request) - return response - + @inlineCallbacks + @_warn_spider_arg + def fetch( + self, request: Request, spider: Spider | None = None + ) -> Generator[Deferred[Any], Any, Response | Request]: self.active.add(request) - dfd = self.middleware.download(self._enqueue_request, request, spider) - return dfd.addBoth(_deactivate) + try: + result: Response | Request = yield ( + deferred_from_coro( + self.middleware.download_async(self._enqueue_request, request) + ) + ) + return result + finally: + self.active.remove(request) def needs_backout(self) -> bool: return len(self.active) >= self.total_concurrency - def _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]: - key = self._get_slot_key(request, spider) + @_warn_spider_arg + def _get_slot( + self, request: Request, spider: Spider | None = None + ) -> tuple[str, Slot]: + key = self.get_slot_key(request) if key not in self.slots: + assert self.crawler.spider slot_settings = self.per_slot_settings.get(key, {}) - conc = ( - self.ip_concurrency if self.ip_concurrency else self.domain_concurrency + conc = self.ip_concurrency or self.domain_concurrency + conc, delay = _get_concurrency_delay( + conc, self.crawler.spider, self.settings ) - conc, delay = _get_concurrency_delay(conc, spider, self.settings) conc, delay = ( slot_settings.get("concurrency", conc), slot_settings.get("delay", delay), @@ -129,12 +159,14 @@ class Downloader: randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot + self._start_slot_gc() return key, self.slots[key] - def _get_slot_key(self, request: Request, spider: Spider) -> str: - if self.DOWNLOAD_SLOT in request.meta: - return cast(str, request.meta[self.DOWNLOAD_SLOT]) + def get_slot_key(self, request: Request) -> str: + meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT) + if meta_slot is not None: + return meta_slot key = urlparse_cached(request).hostname or "" if self.ip_concurrency: @@ -142,93 +174,111 @@ class Downloader: return key - def _enqueue_request(self, request: Request, spider: Spider) -> Deferred: - key, slot = self._get_slot(request, spider) + # passed as download_func into self.middleware.download() in self.fetch() + async def _enqueue_request(self, request: Request) -> Response: + key, slot = self._get_slot(request) request.meta[self.DOWNLOAD_SLOT] = key - - def _deactivate(response: Response) -> Response: - slot.active.remove(request) - return response - slot.active.add(request) self.signals.send_catch_log( - signal=signals.request_reached_downloader, request=request, spider=spider + signal=signals.request_reached_downloader, + request=request, + spider=self.crawler.spider, ) - deferred: Deferred = Deferred().addBoth(_deactivate) - slot.queue.append((request, deferred)) - self._process_queue(spider, slot) - return deferred + d: Deferred[Response] = Deferred() + slot.queue.append((request, d)) + self._process_queue(slot) + try: + return await maybe_deferred_to_future(d) # fired in _wait_for_download() + finally: + slot.active.remove(request) - def _process_queue(self, spider: Spider, slot: Slot) -> None: - from twisted.internet import reactor - - if slot.latercall and slot.latercall.active(): + def _process_queue(self, slot: Slot) -> None: + if slot.latercall: + # block processing until slot.latercall is called return # Delay queue processing if a download_delay is configured - now = time() + now = monotonic() delay = slot.download_delay() if delay: penalty = delay - now + slot.lastseen if penalty > 0: - slot.latercall = reactor.callLater( - penalty, self._process_queue, spider, slot - ) + slot.latercall = call_later(penalty, self._latercall, slot) return # Process enqueued requests if there are free slots to transfer for this slot while slot.queue and slot.free_transfer_slots() > 0: slot.lastseen = now - request, deferred = slot.queue.popleft() - dfd = self._download(slot, request, spider) - dfd.chainDeferred(deferred) + request, queue_dfd = slot.queue.popleft() + _schedule_coro(self._wait_for_download(slot, request, queue_dfd)) # prevent burst if inter-request delays were configured if delay: - self._process_queue(spider, slot) + self._process_queue(slot) break - def _download(self, slot: Slot, request: Request, spider: Spider) -> Deferred: - # The order is very important for the following deferreds. Do not change! + def _latercall(self, slot: Slot) -> None: + slot.latercall = None + self._process_queue(slot) - # 1. Create the download deferred - dfd = mustbe_deferred(self.handlers.download_request, request, spider) - - # 2. Notify response_downloaded listeners about the recent download - # before querying queue for next request - def _downloaded(response: Response) -> Response: + async def _download(self, slot: Slot, request: Request) -> Response: + # The order is very important for the following logic. Do not change! + slot.transferring.add(request) + try: + # 1. Download the response + response: Response = await self.handlers.download_request_async(request) + # 2. Notify response_downloaded listeners about the recent download + # before querying queue for next request self.signals.send_catch_log( signal=signals.response_downloaded, response=response, request=request, - spider=spider, + spider=self.crawler.spider, ) return response - - dfd.addCallback(_downloaded) - - # 3. After response arrives, remove the request from transferring - # state to free up the transferring slot so it can be used by the - # following requests (perhaps those which came from the downloader - # middleware itself) - slot.transferring.add(request) - - def finish_transferring(_: Any) -> Any: + except Exception: + await _defer_sleep_async() + raise + finally: + # 3. After response arrives, remove the request from transferring + # state to free up the transferring slot so it can be used by the + # following requests (perhaps those which came from the downloader + # middleware itself) slot.transferring.remove(request) - self._process_queue(spider, slot) + self._process_queue(slot) self.signals.send_catch_log( - signal=signals.request_left_downloader, request=request, spider=spider + signal=signals.request_left_downloader, + request=request, + spider=self.crawler.spider, ) - return _ - return dfd.addBoth(finish_transferring) + async def _wait_for_download( + self, slot: Slot, request: Request, queue_dfd: Deferred[Response] + ) -> None: + try: + response = await self._download(slot, request) + except Exception: + queue_dfd.errback(Failure()) + else: + queue_dfd.callback(response) # awaited in _enqueue_request() def close(self) -> None: - self._slot_gc_loop.stop() + self._stop_slot_gc() for slot in self.slots.values(): slot.close() def _slot_gc(self, age: float = 60) -> None: - mintime = time() - age + mintime = monotonic() - age for key, slot in list(self.slots.items()): if not slot.active and slot.lastseen + slot.delay < mintime: self.slots.pop(key).close() + + def _start_slot_gc(self) -> None: + if self._slot_gc_loop: + return + self._slot_gc_loop = create_looping_call(self._slot_gc) + self._slot_gc_loop.start(self._SLOT_GC_INTERVAL, now=False) + + def _stop_slot_gc(self) -> None: + if self._slot_gc_loop: + self._slot_gc_loop.stop() + self._slot_gc_loop = None diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index dba4d8cdc..40f09ffb2 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,13 +1,14 @@ +from __future__ import annotations + import warnings -from typing import TYPE_CHECKING, Any, List, Optional +from typing import TYPE_CHECKING, Any, cast from OpenSSL import SSL -from twisted.internet._sslverify import _setAcceptableProtocols from twisted.internet.ssl import ( AcceptableCiphers, CertificateOptions, + TLSVersion, optionsForClientTLS, - platformTrust, ) from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS @@ -15,102 +16,157 @@ from zope.interface.declarations import implementer from zope.interface.verify import verifyObject from scrapy.core.downloader.tls import ( + _TWISTED_VERSION_MAP, DEFAULT_CIPHERS, - ScrapyClientTLSOptions, - openssl_methods, + _openssl_methods, + _ScrapyClientTLSOptions, + _ScrapyClientTLSOptions26, ) -from scrapy.settings import BaseSettings +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils._deps_compat import TWISTED_TLS_NEW_IMPL +from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.misc import build_from_crawler, load_object +from scrapy.utils.ssl import _get_cert_options_version_kwargs, _get_tls_version_limits if TYPE_CHECKING: from twisted.internet._sslverify import ClientTLSOptions + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + @implementer(IPolicyForHTTPS) -class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): - """ - Non-peer-certificate verifying HTTPS context factory +class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): + """Non-peer-certificate verifying HTTPS context factory. - Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD) - which allows TLS protocol negotiation + Uses :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`, + :setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION` + to configure the :class:`~twisted.internet.ssl.CertificateOptions` + instance. - 'A TLS/SSL connection established with [this method] may - understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.' + The purpose of this custom class is to provide a ``creatorForNetloc()`` + method that returns a ``_ScrapyClientTLSOptions`` instance configured based + on TLS settings provided to the factory. """ def __init__( self, - method: int = SSL.SSLv23_METHOD, + method: int | None = SSL.SSLv23_METHOD, # noqa: S503 tls_verbose_logging: bool = False, - tls_ciphers: Optional[str] = None, + tls_ciphers: str | None = None, *args: Any, + verify_certificates: bool = False, + tls_min_version: TLSVersion | None = None, + tls_max_version: TLSVersion | None = None, **kwargs: Any, ): - super().__init__(*args, **kwargs) - self._ssl_method: int = method - self.tls_verbose_logging: bool = tls_verbose_logging + super().__init__(*args, **kwargs) # type: ignore[no-untyped-call] + self._ssl_method: int | None = method + self.tls_min_version: TLSVersion | None = tls_min_version + self.tls_max_version: TLSVersion | None = tls_max_version + self.tls_verbose_logging: bool = tls_verbose_logging # unused self.tls_ciphers: AcceptableCiphers if tls_ciphers: self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers) else: self.tls_ciphers = DEFAULT_CIPHERS + self._verify_certificates = verify_certificates @classmethod - def from_settings( + def from_crawler( cls, - settings: BaseSettings, - method: int = SSL.SSLv23_METHOD, + crawler: Crawler, + method: int | None = SSL.SSLv23_METHOD, # noqa: S503 *args: Any, **kwargs: Any, - ): - tls_verbose_logging: bool = settings.getbool( + ) -> Self: + tls_verbose_logging: bool = crawler.settings.getbool( "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" ) - tls_ciphers: Optional[str] = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] + tls_ciphers: str | None = crawler.settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] + # DOWNLOADER_CLIENT_TLS_METHOD reading and handling should be also moved here + # when the deprecated load_context_factory_from_settings() is removed + tls_min_ver, tls_max_ver = _get_tls_version_limits( + crawler.settings, _TWISTED_VERSION_MAP.__getitem__ + ) + if tls_min_ver or tls_max_ver: + method = None + verify_certificates = crawler.settings.getbool("DOWNLOAD_VERIFY_CERTIFICATES") return cls( # type: ignore[misc] + *args, method=method, tls_verbose_logging=tls_verbose_logging, tls_ciphers=tls_ciphers, - *args, + tls_min_version=tls_min_ver, + tls_max_version=tls_max_ver, + verify_certificates=verify_certificates, **kwargs, ) - def getCertificateOptions(self) -> CertificateOptions: - # setting verify=True will require you to provide CAs - # to verify against; in other words: it's not that simple + # should be removed together with ScrapyClientContextFactory + def getCertificateOptions(self) -> CertificateOptions: # pragma: no cover + return self._get_cert_options() - # backward-compatible SSL/TLS method: - # - # * this will respect `method` attribute in often recommended - # `ScrapyClientContextFactory` subclass - # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133) - # - # * getattr() for `_ssl_method` attribute for context factories - # not calling super().__init__ - return CertificateOptions( - verify=False, - method=getattr(self, "method", getattr(self, "_ssl_method", None)), - fixBrokenPeers=True, - acceptableCiphers=self.tls_ciphers, + def _get_cert_options(self) -> CertificateOptions: + return _ScrapyCertificateOptions(**self._get_cert_options_kwargs()) + + def _get_cert_options_kwargs(self) -> dict[str, Any]: + kwargs: dict[str, Any] = { + "fixBrokenPeers": True, + "acceptableCiphers": self.tls_ciphers, + } + if self.tls_min_version or self.tls_max_version: + kwargs.update( + _get_cert_options_version_kwargs( + self.tls_min_version, self.tls_max_version + ) + ) + # when ScrapyClientContextFactory is removed self._ssl_method can just be None by default + elif self._ssl_method != SSL.SSLv23_METHOD: + kwargs["method"] = self._ssl_method + return kwargs + + # should be removed together with ScrapyClientContextFactory + def getContext( + self, hostname: Any = None, port: Any = None + ) -> SSL.Context: # pragma: no cover + return self._get_context() + + def _get_context(self) -> SSL.Context: + return self._get_cert_options().getContext() + + def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: + if not self._verify_certificates: + # Our options class is needed to skip verification errors + if TWISTED_TLS_NEW_IMPL: + return _ScrapyClientTLSOptions26( + self._get_cert_options()._makeTLSConnection, + hostname.decode("ascii"), + ) + return _ScrapyClientTLSOptions( + hostname.decode("ascii"), # type: ignore[arg-type] + self._get_context(), # type: ignore[arg-type] + ) + # Otherwise use the normal Twisted function. + return optionsForClientTLS( # type: ignore[no-any-return] + hostname=hostname.decode("ascii"), + extraCertificateOptions=self._get_cert_options_kwargs(), ) - # kept for old-style HTTP/1.0 downloader context twisted calls, - # e.g. connectSSL() - def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context: - ctx = self.getCertificateOptions().getContext() - ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT - return ctx - def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": - return ScrapyClientTLSOptions( - hostname.decode("ascii"), - self.getContext(), - verbose_logging=self.tls_verbose_logging, - ) +ScrapyClientContextFactory = create_deprecated_class( + "ScrapyClientContextFactory", + _ScrapyClientContextFactory, + subclass_warn_message="{old} is deprecated.", + instance_warn_message="{cls} is deprecated.", +) @implementer(IPolicyForHTTPS) -class BrowserLikeContextFactory(ScrapyClientContextFactory): +class BrowserLikeContextFactory(_ScrapyClientContextFactory): """ Twisted-recommended context factory for web clients. @@ -123,66 +179,134 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): :meth:`creatorForNetloc` is the same as :class:`~twisted.web.client.BrowserLikePolicyForHTTPS` except this context factory allows setting the TLS/SSL method to use. - - The default OpenSSL method is ``TLS_METHOD`` (also called - ``SSLv23_METHOD``) which allows TLS protocol negotiation. """ - def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": - # trustRoot set to platformTrust() will use the platform's root CAs. - # - # This means that a website like https://www.cacert.org will be rejected - # by default, since CAcert.org CA certificate is seldom shipped. - return optionsForClientTLS( + def __init__(self, *args: Any, **kwargs: Any): + warnings.warn( + "BrowserLikeContextFactory is deprecated." + " You can set DOWNLOAD_VERIFY_CERTIFICATES=True to enable" + " certificate verification instead of using it.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + super().__init__(*args, **kwargs) + + def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: + return optionsForClientTLS( # type: ignore[no-any-return] hostname=hostname.decode("ascii"), - trustRoot=platformTrust(), - extraCertificateOptions={"method": self._ssl_method}, + extraCertificateOptions=self._get_cert_options_kwargs(), ) @implementer(IPolicyForHTTPS) -class AcceptableProtocolsContextFactory: +class _AcceptableProtocolsContextFactory: """Context factory to used to override the acceptable protocols - to set up the [OpenSSL.SSL.Context] for doing NPN and/or ALPN - negotiation. + to set up the :class:`OpenSSL.SSL.Context` for doing ALPN negotiation. + It's a private class for :class:`~.H2DownloadHandler`. + + This class wraps ``creatorForNetloc()`` of another factory class, setting + the acceptable protocols on the :class:`.ClientTLSOptions` instance + returned by it. It's only needed because we support custom factories via + :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`. + + It's a no-op on Twisted 26.4.0+, though using it with custom + factories on those Twisted versions may be not enough for HTTP/2 support. """ - def __init__(self, context_factory: Any, acceptable_protocols: List[bytes]): + # Something needs to call set_alpn_protos() for ALPN to work. + # + # Twisted < 26.4.0 does it in OpenSSLCertificateOptions._makeContext() + # (requires passing acceptableProtocols from the factory to + # OpenSSLCertificateOptions) and in TLSMemoryBIOFactory._createConnection() + # based on H2ClientFactory.acceptableProtocols (too late, it seems). + # + # Newer Twisted does it in OpenSSLCertificateOptions._makeContext() as + # well, and in OpenSSLCertificateOptions._makeTLSConnection() based on + # H2ClientFactory.acceptableProtocols (which now works). + # + # When we drop DOWNLOADER_CLIENTCONTEXTFACTORY it looks like we can replace + # all of this with _ScrapyClientContextFactory.acceptableProtocols. + + def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]): verifyObject(IPolicyForHTTPS, context_factory) self._wrapped_context_factory: Any = context_factory - self._acceptable_protocols: List[bytes] = acceptable_protocols + self._acceptable_protocols: list[bytes] = acceptable_protocols - def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": - options: "ClientTLSOptions" = self._wrapped_context_factory.creatorForNetloc( + def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: + options: ClientTLSOptions = self._wrapped_context_factory.creatorForNetloc( hostname, port ) - _setAcceptableProtocols(options._ctx, self._acceptable_protocols) + if not TWISTED_TLS_NEW_IMPL: + from twisted.internet._sslverify import ( # type: ignore[attr-defined] # noqa: PLC0415 # pylint: disable=no-name-in-module + _setAcceptableProtocols, + ) + + _setAcceptableProtocols(options._ctx, self._acceptable_protocols) # type: ignore[attr-defined] return options -def load_context_factory_from_settings(settings, crawler): - ssl_method = openssl_methods[settings.get("DOWNLOADER_CLIENT_TLS_METHOD")] - context_factory_cls = load_object(settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]) - # try method-aware context factory - try: - context_factory = build_from_crawler( - context_factory_cls, - crawler, - method=ssl_method, - ) - except TypeError: - # use context factory defaults - context_factory = build_from_crawler( - context_factory_cls, - crawler, - ) - msg = ( - f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept " - "a `method` argument (type OpenSSL.SSL method, e.g. " - "OpenSSL.SSL.SSLv23_METHOD) and/or a `tls_verbose_logging` " - "argument and/or a `tls_ciphers` argument. Please, upgrade your " - "context factory class to handle them or ignore them." - ) - warnings.warn(msg) +AcceptableProtocolsContextFactory = create_deprecated_class( + "AcceptableProtocolsContextFactory", + _AcceptableProtocolsContextFactory, + subclass_warn_message="{old} is deprecated.", + instance_warn_message="{cls} is deprecated.", +) - return context_factory + +class _ScrapyCertificateOptions(CertificateOptions): + """A wrapper needed to add flags to the SSL context before it's used.""" + + def _makeContext(self, skipCiphers: bool = False) -> SSL.Context: + if TWISTED_TLS_NEW_IMPL: + ctx = super()._makeContext(skipCiphers) + else: + ctx = super()._makeContext() + ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT + return ctx + + +def _load_context_factory_from_settings(crawler: Crawler) -> IPolicyForHTTPS: + """Create an instance of :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`. + + Also passes values of other relevant settings to the factory class. + """ + tls_method_setting: str = crawler.settings["DOWNLOADER_CLIENT_TLS_METHOD"] + if tls_method_setting != "TLS": + warnings.warn( + "Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is" + " deprecated, please use DOWNLOAD_TLS_MIN_VERSION and/or" + " DOWNLOAD_TLS_MAX_VERSION instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + tls_method = _openssl_methods[tls_method_setting] + if crawler.settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] == "SENTINEL": + context_factory_cls = _ScrapyClientContextFactory + else: # pragma: no cover + warnings.warn( + "The 'DOWNLOADER_CLIENTCONTEXTFACTORY' setting is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + context_factory_cls = load_object( + crawler.settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] + ) + return cast( + "IPolicyForHTTPS", + build_from_crawler( + context_factory_cls, + crawler, + method=tls_method, + ), + ) + + +def load_context_factory_from_settings( + settings: BaseSettings, crawler: Crawler +) -> IPolicyForHTTPS: # pragma: no cover + warnings.warn( + "load_context_factory_from_settings() is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return _load_context_factory_from_settings(crawler) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 416669b7f..fb27cdb8b 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -1,33 +1,67 @@ """Download handlers for different schemes""" -import logging -from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Union, cast +from __future__ import annotations -from twisted.internet import defer -from twisted.internet.defer import Deferred +import inspect +import logging +import warnings +from typing import TYPE_CHECKING, Any, Protocol, cast from scrapy import Request, Spider, signals -from scrapy.exceptions import NotConfigured, NotSupported +from scrapy.exceptions import NotConfigured, NotSupported, ScrapyDeprecationWarning +from scrapy.utils.defer import ( + deferred_from_coro, + ensure_awaitable, + maybe_deferred_to_future, +) from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler, load_object -from scrapy.utils.python import without_none_values +from scrapy.utils.python import global_object_name, without_none_values if TYPE_CHECKING: + from collections.abc import Callable + + from twisted.internet.defer import Deferred + from scrapy.crawler import Crawler + from scrapy.http import Response + logger = logging.getLogger(__name__) +# This is the official API but we temporarily support the old deprecated one: +# * lazy is not mandatory (defaults to True). +# * download_request() can return a Deferred[Response] instead of a coroutine, +# and takes a spider argument in this case. +# * close() can return None or Deferred[None] instead of a coroutine. +# * close() is not mandatory. + + +class DownloadHandlerProtocol(Protocol): + lazy: bool + + async def download_request(self, request: Request) -> Response: ... + + async def close(self) -> None: ... + + class DownloadHandlers: - def __init__(self, crawler: "Crawler"): - self._crawler: "Crawler" = crawler - self._schemes: Dict[ - str, Union[str, Callable] - ] = {} # stores acceptable schemes on instancing - self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes - self._notconfigured: Dict[str, str] = {} # remembers failed handlers - handlers: Dict[str, Union[str, Callable]] = without_none_values( - crawler.settings.getwithbase("DOWNLOAD_HANDLERS") + def __init__(self, crawler: Crawler): + self._crawler: Crawler = crawler + # stores acceptable schemes on instancing + self._schemes: dict[str, str | Callable[..., Any]] = {} + # stores instanced handlers for schemes + self._handlers: dict[str, DownloadHandlerProtocol] = {} + # remembers failed handlers + self._notconfigured: dict[str, str] = {} + # remembers handlers with Deferred-based download_request() + self._old_style_handlers: set[str] = set() + handlers: dict[str, str | Callable[..., Any]] = without_none_values( + cast( + "dict[str, str | Callable[..., Any]]", + crawler.settings.getwithbase("DOWNLOAD_HANDLERS"), + ) ) for scheme, clspath in handlers.items(): self._schemes[scheme] = clspath @@ -35,7 +69,7 @@ class DownloadHandlers: crawler.signals.connect(self._close, signals.engine_stopped) - def _get_handler(self, scheme: str) -> Any: + def _get_handler(self, scheme: str) -> DownloadHandlerProtocol | None: """Lazy-load the downloadhandler for a scheme only on the first request for that scheme. """ @@ -49,12 +83,23 @@ class DownloadHandlers: return self._load_handler(scheme) - def _load_handler(self, scheme: str, skip_lazy: bool = False) -> Any: + def _load_handler( + self, scheme: str, skip_lazy: bool = False + ) -> DownloadHandlerProtocol | None: path = self._schemes[scheme] try: - dhcls = load_object(path) - if skip_lazy and getattr(dhcls, "lazy", True): - return None + dhcls: type[DownloadHandlerProtocol] = load_object(path) + if skip_lazy: + if not hasattr(dhcls, "lazy"): + warnings.warn( + f"{global_object_name(dhcls)} doesn't define a 'lazy' attribute." + f" This is deprecated, please add 'lazy = True' (which is the current" + f" default value) to the class definition.", + category=ScrapyDeprecationWarning, + stacklevel=1, + ) + if getattr(dhcls, "lazy", True): + return None dh = build_from_crawler( dhcls, self._crawler, @@ -71,21 +116,63 @@ class DownloadHandlers: ) self._notconfigured[scheme] = str(ex) return None - else: - self._handlers[scheme] = dh - return dh + self._handlers[scheme] = dh + if not inspect.iscoroutinefunction(dh.download_request): # pragma: no cover + warnings.warn( + f"{global_object_name(dh.download_request)} is not a coroutine function." + f" This is deprecated, please rewrite it to return a coroutine and remove" + f" the 'spider' argument.", + category=ScrapyDeprecationWarning, + stacklevel=1, + ) + self._old_style_handlers.add(scheme) + return dh - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request( + self, request: Request, spider: Spider | None = None + ) -> Deferred[Response]: # pragma: no cover + warnings.warn( + "DownloadHandlers.download_request() is deprecated, use download_request_async() instead", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return deferred_from_coro(self.download_request_async(request)) + + async def download_request_async(self, request: Request) -> Response: scheme = urlparse_cached(request).scheme handler = self._get_handler(scheme) if not handler: raise NotSupported( f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}" ) - return cast(Deferred, handler.download_request(request, spider)) + assert self._crawler.spider + if scheme in self._old_style_handlers: # pragma: no cover + return await maybe_deferred_to_future( + cast( + "Deferred[Response]", + handler.download_request(request, self._crawler.spider), # type: ignore[call-arg] + ) + ) + return await handler.download_request(request) - @defer.inlineCallbacks - def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred, Any, None]: + async def _close(self) -> None: for dh in self._handlers.values(): - if hasattr(dh, "close"): - yield dh.close() + if not hasattr(dh, "close"): # pragma: no cover + warnings.warn( + f"{global_object_name(dh)} doesn't define a close() method." + f" This is deprecated, please add an empty 'async def close()' method.", + category=ScrapyDeprecationWarning, + stacklevel=1, + ) + continue + + if inspect.iscoroutinefunction(dh.close): + await dh.close() + else: # pragma: no cover + warnings.warn( + f"{global_object_name(dh.close)} is not a coroutine function." + f" This is deprecated, please rewrite it to return a coroutine.", + category=ScrapyDeprecationWarning, + stacklevel=1, + ) + await ensure_awaitable(dh.close()) diff --git a/scrapy/core/downloader/handlers/_base_http.py b/scrapy/core/downloader/handlers/_base_http.py new file mode 100644 index 000000000..83d130462 --- /dev/null +++ b/scrapy/core/downloader/handlers/_base_http.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from abc import ABC +from typing import TYPE_CHECKING + +from .base import BaseDownloadHandler + +if TYPE_CHECKING: + from scrapy.crawler import Crawler + + +class BaseHttpDownloadHandler(BaseDownloadHandler, ABC): + """Base class for built-in HTTP download handlers.""" + + def __init__(self, crawler: Crawler): + super().__init__(crawler) + self._default_maxsize: int = crawler.settings.getint("DOWNLOAD_MAXSIZE") + self._default_warnsize: int = crawler.settings.getint("DOWNLOAD_WARNSIZE") + self._fail_on_dataloss: bool = crawler.settings.getbool( + "DOWNLOAD_FAIL_ON_DATALOSS" + ) + self._tls_verbose_logging: bool = crawler.settings.getbool( + "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" + ) + self._fail_on_dataloss_warned: bool = False diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.py new file mode 100644 index 000000000..16b655716 --- /dev/null +++ b/scrapy/core/downloader/handlers/_base_streaming.py @@ -0,0 +1,307 @@ +from __future__ import annotations + +import base64 +import logging +import time +from abc import ABC, abstractmethod +from io import BytesIO +from typing import TYPE_CHECKING, Any, ClassVar, Generic, NoReturn, TypedDict, TypeVar +from urllib.parse import quote, urlsplit + +from scrapy import Request, signals +from scrapy.exceptions import ( + DownloadCancelledError, + NotConfigured, + ResponseDataLossError, +) +from scrapy.utils._download_handlers import ( + check_stop_download, + get_dataloss_msg, + get_maxsize_msg, + get_warnsize_msg, + make_response, + normalize_bind_address, +) +from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.url import add_http_if_no_scheme + +from ._base_http import BaseHttpDownloadHandler + +if TYPE_CHECKING: + from collections.abc import AsyncIterable + from contextlib import AbstractAsyncContextManager + from ipaddress import IPv4Address, IPv6Address + + from _typeshed import SizedBuffer + + # typing.NotRequired requires Python 3.11 + from typing_extensions import NotRequired + + from scrapy.crawler import Crawler + from scrapy.http import Headers, Response + + +logger = logging.getLogger(__name__) + +_ResponseT = TypeVar("_ResponseT") + + +class _BaseResponseArgs(TypedDict): + status: int + url: str + headers: Headers + certificate: NotRequired[Any] + ip_address: NotRequired[IPv4Address | IPv6Address | None] + protocol: str | None + + +class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_ResponseT]): + """A base class for HTTP download handlers that follow the streaming logic flow.""" + + _DEFAULT_CONNECT_TIMEOUT: ClassVar[float] = 10 + experimental: ClassVar[bool] = False + requires_asyncio: ClassVar[bool] = True + # require subclasses to disable proxies explicitly with an explanation + supports_proxies: ClassVar[bool] = True + supports_per_request_bindaddress: ClassVar[bool] = False + + def __init__(self, crawler: Crawler): + if self.requires_asyncio and not is_asyncio_available(): # pragma: no cover + raise NotConfigured( + f"{type(self).__name__} requires the asyncio support. Make" + f" sure that you have either enabled the asyncio Twisted" + f" reactor in the TWISTED_REACTOR setting or disabled the" + f" TWISTED_REACTOR_ENABLED setting. See the asyncio documentation" + f" of Scrapy for more information." + ) + self._check_deps_installed() + super().__init__(crawler) + if self.experimental: + logger.warning( + f"{type(self).__name__} is experimental and is not recommended for production use." + ) + self._bind_address = normalize_bind_address( + crawler.settings.get("DOWNLOAD_BIND_ADDRESS") + ) + self._proxy_auth_encoding: str = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") + # these are useful for many handlers but used in different ways by them + self._pool_size_total: int = crawler.settings.getint("CONCURRENT_REQUESTS") + self._pool_size_per_host: int = crawler.settings.getint( + "CONCURRENT_REQUESTS_PER_DOMAIN" + ) + + @staticmethod + @abstractmethod + def _check_deps_installed() -> None: + """Raise NotConfigured if the required deps are not installed.""" + raise NotImplementedError + + @abstractmethod + def _make_request( + self, request: Request, timeout: float + ) -> AbstractAsyncContextManager[_ResponseT]: + """Return an async context manager yielding the library-specific response. + + Exceptions raised by the library should be reraised as Scrapy-specific ones. + """ + raise NotImplementedError + + @staticmethod + @abstractmethod + def _extract_headers(response: _ResponseT) -> Headers: + """Convert library-specific response headers to a + :class:`~scrapy.http.headers.Headers` object.""" + raise NotImplementedError + + @staticmethod + @abstractmethod + def _build_base_response_args( + response: _ResponseT, request: Request, headers: Headers + ) -> _BaseResponseArgs: + """Build kwargs for :func:`scrapy.utils._download_handlers.make_response`.""" + raise NotImplementedError + + @staticmethod + @abstractmethod + def _iter_body_chunks(response: _ResponseT) -> AsyncIterable[SizedBuffer]: + """Return an async iterable yielding body chunks from the response.""" + raise NotImplementedError + + @staticmethod + @abstractmethod + def _is_dataloss_exception(exc: Exception) -> bool: + """Return True if ``exc`` represents dataloss.""" + raise NotImplementedError + + def _log_tls_info(self, response: _ResponseT, request: Request) -> None: + """Log TLS connection details, if possible.""" + + async def download_request(self, request: Request) -> Response: + if not self.supports_proxies and request.meta.get("proxy"): + raise NotImplementedError(f"{type(self).__name__} doesn't support proxies.") + if not self.supports_per_request_bindaddress and request.meta.get( + "bindaddress" + ): + logger.error( + f"The 'bindaddress' request meta key is not supported by" + f" {type(self).__name__} and will be ignored." + ) + timeout: float = request.meta.get( + "download_timeout", self._DEFAULT_CONNECT_TIMEOUT + ) + start_time = time.monotonic() + async with self._make_request(request, timeout) as response: + request.meta["download_latency"] = time.monotonic() - start_time + return await self._read_response(response, request) + + async def _read_response(self, response: _ResponseT, request: Request) -> Response: + maxsize: int = request.meta.get("download_maxsize", self._default_maxsize) + warnsize: int = request.meta.get("download_warnsize", self._default_warnsize) + + headers = self._extract_headers(response) + content_length = headers.get("Content-Length") + expected_size = int(content_length) if content_length is not None else None + if maxsize and expected_size and expected_size > maxsize: + self._cancel_maxsize(expected_size, maxsize, request, expected=True) + + reached_warnsize = False + if warnsize and expected_size and expected_size > warnsize: + reached_warnsize = True + logger.warning( + get_warnsize_msg(expected_size, warnsize, request, expected=True) + ) + + make_response_base_args = self._build_base_response_args( + response, request, headers + ) + + if self._tls_verbose_logging: + self._log_tls_info(response, request) + + if stop_download := check_stop_download( + signals.headers_received, + self.crawler, + request, + headers=headers, + body_length=expected_size, + ): + return make_response( + **make_response_base_args, + stop_download=stop_download, + ) + + response_body = BytesIO() + bytes_received = 0 + try: + async for chunk in self._iter_body_chunks(response): + response_body.write(chunk) + bytes_received += len(chunk) + + if stop_download := check_stop_download( + signals.bytes_received, self.crawler, request, data=chunk + ): + return make_response( + **make_response_base_args, + body=response_body.getvalue(), + stop_download=stop_download, + ) + + if maxsize and bytes_received > maxsize: + response_body.truncate(0) + self._cancel_maxsize( + bytes_received, maxsize, request, expected=False + ) + + if warnsize and bytes_received > warnsize and not reached_warnsize: + reached_warnsize = True + logger.warning( + get_warnsize_msg( + bytes_received, warnsize, request, expected=False + ) + ) + except Exception as e: + if not self._is_dataloss_exception(e): + raise + fail_on_dataloss: bool = request.meta.get( + "download_fail_on_dataloss", self._fail_on_dataloss + ) + if not fail_on_dataloss: + return make_response( + **make_response_base_args, + body=response_body.getvalue(), + flags=["dataloss"], + ) + if not self._fail_on_dataloss_warned: + logger.warning(get_dataloss_msg(request.url)) + self._fail_on_dataloss_warned = True + raise ResponseDataLossError(str(e)) from e + + return make_response( + **make_response_base_args, + body=response_body.getvalue(), + ) + + def _get_bind_address_host(self) -> str | None: + """Return the host portion of the bind address. + + Needed for handlers that don't support the bind port. + """ + if self._bind_address is None: + return None + host, port = self._bind_address + if port != 0: + logger.warning( + "DOWNLOAD_BIND_ADDRESS specifies a port (%s), but %s does not " + "support binding to a specific local port. Ignoring the port " + "and binding only to %r.", + port, + type(self).__name__, + host, + ) + return host + + @staticmethod + def _cancel_maxsize( + size: int, limit: int, request: Request, *, expected: bool + ) -> NoReturn: + warning_msg = get_maxsize_msg(size, limit, request, expected=expected) + logger.warning(warning_msg) + raise DownloadCancelledError(warning_msg) + + @staticmethod + def _extract_proxy(request: Request) -> tuple[str | None, str | None]: + """Return a tuple of the proxy URL with a scheme and the value of the + Proxy-Authorization header. + + This is useful for handlers that take the proxy headers separately. + """ + proxy: str | None = request.meta.get("proxy") + if not proxy: + return None, None + proxy = add_http_if_no_scheme(proxy) + auth_header: list[bytes] | None = request.headers.pop( + b"Proxy-Authorization", None + ) + return proxy, auth_header[0].decode("ascii") if auth_header else None + + def _extract_proxy_url_with_creds(self, request: Request) -> str | None: + """Return the proxy URL with the userinfo added based on the + Proxy-Authorization header. + + This is useful for handlers that cannot take the proxy headers + separately. + """ + proxy_url, auth_header = self._extract_proxy(request) + if proxy_url is None or auth_header is None: + return proxy_url + scheme, token = auth_header.split(" ", 1) + if scheme != "Basic": + raise ValueError( + f"Expected Basic auth in Proxy-Authorization, got {scheme}" + ) + user, password = ( + base64.b64decode(token).decode(self._proxy_auth_encoding).split(":", 1) + ) + parts = urlsplit(proxy_url) + netloc = f"{quote(user)}:{quote(password)}@{parts.netloc}" + return parts._replace(netloc=netloc).geturl() diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py new file mode 100644 index 000000000..c960fc152 --- /dev/null +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -0,0 +1,226 @@ +"""``httpx``-based HTTP(S) download handler. Currently not recommended for production use.""" + +from __future__ import annotations + +import ipaddress +import ssl +from contextlib import asynccontextmanager +from socket import gaierror +from typing import TYPE_CHECKING, ClassVar + +from scrapy.exceptions import ( + CannotResolveHostError, + DownloadConnectionRefusedError, + DownloadFailedError, + DownloadTimeoutError, + NotConfigured, + UnsupportedURLSchemeError, +) +from scrapy.http import Headers +from scrapy.utils._download_handlers import NullCookieJar +from scrapy.utils.python import _iter_exc_causes +from scrapy.utils.ssl import ( + _log_sslobj_debug_info, + _make_insecure_ssl_ctx, + _make_ssl_context, +) + +from ._base_streaming import BaseStreamingDownloadHandler, _BaseResponseArgs + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from httpcore import AsyncNetworkStream + + from scrapy import Request + from scrapy.crawler import Crawler + + +HAS_SOCKS = HAS_HTTP2 = False + +try: + import httpx +except ImportError: + httpx = None # type: ignore[assignment] +else: + # a small hack to avoid importing these optional extras unconditionally + + DOWNLOAD_FAILED_EXCEPTIONS: tuple[type[BaseException], ...] = ( + httpx.RequestError, + httpx.InvalidURL, + ) + + try: + import h2.exceptions + + HAS_HTTP2 = True + DOWNLOAD_FAILED_EXCEPTIONS += (h2.exceptions.InvalidBodyLengthError,) + except ImportError: # pragma: no cover + pass + + try: + import socksio.exceptions + + HAS_SOCKS = True + DOWNLOAD_FAILED_EXCEPTIONS += (socksio.exceptions.ProtocolError,) + except ImportError: # pragma: no cover + pass + + +if TYPE_CHECKING: + _Base = BaseStreamingDownloadHandler[httpx.Response] +else: + _Base = BaseStreamingDownloadHandler + + +class HttpxDownloadHandler(_Base): + experimental: ClassVar[bool] = True + + def __init__(self, crawler: Crawler): + super().__init__(crawler) + self._verify_certificates: bool = crawler.settings.getbool( + "DOWNLOAD_VERIFY_CERTIFICATES" + ) + self._enable_h2: bool = crawler.settings.getbool("HTTPX_HTTP2_ENABLED") + if self._enable_h2 and not HAS_HTTP2: # pragma: no cover + raise NotConfigured( + f"HTTP/2 support in {type(self).__name__} requires the 'httpx[http2]' extra to be installed." + ) + self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings) + self._bind_host: str | None = self._get_bind_address_host() + self._limits: httpx.Limits = httpx.Limits( + # hard limit on simultaneous connections + max_connections=self._pool_size_total, + # total number of idle connections in the pool (extra ones are closed) + max_keepalive_connections=self._pool_size_total, + ) + + self._default_client: httpx.AsyncClient = self._make_client() + # httpx doesn't support per-request proxies: https://github.com/encode/httpx/discussions/3183, + # so we keep a pool of clients per proxy URL. LRU eviction can be added here if needed. + self._proxy_clients: dict[str, httpx.AsyncClient] = {} + + @staticmethod + def _check_deps_installed() -> None: + if httpx is None: # pragma: no cover + raise NotConfigured( + "HttpxDownloadHandler requires the httpx library to be installed." + ) + + def _make_client(self, proxy_url: str | None = None) -> httpx.AsyncClient: + if proxy_url: + if proxy_url.startswith("https:") and not self._verify_certificates: + proxy_ssl_context = _make_insecure_ssl_ctx() + else: + proxy_ssl_context = None + proxy = httpx.Proxy(proxy_url, ssl_context=proxy_ssl_context) + else: + proxy = None + + client = httpx.AsyncClient( + cookies=NullCookieJar(), + transport=httpx.AsyncHTTPTransport( + verify=self._ssl_context, + local_address=self._bind_host, + http2=self._enable_h2, + limits=self._limits, + trust_env=False, + proxy=proxy, + ), + ) + # https://github.com/encode/httpx/discussions/1566 + for header_name in ("accept", "accept-encoding", "user-agent"): + client.headers.pop(header_name, None) + return client + + def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient: + if proxy_url is None: + return self._default_client + if cached := self._proxy_clients.get(proxy_url): + return cached + client = self._make_client(proxy_url) + self._proxy_clients[proxy_url] = client + return client + + @asynccontextmanager + async def _make_request( + self, request: Request, timeout: float + ) -> AsyncIterator[httpx.Response]: + proxy = self._extract_proxy_url_with_creds(request) + if proxy and proxy.startswith("socks") and not HAS_SOCKS: # pragma: no cover + raise ValueError( + f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed." + ) + client = self._get_client(proxy) + + try: + async with client.stream( + request.method, + request.url, + content=request.body, + headers=request.headers.to_tuple_list(), + timeout=timeout, + ) as response: + yield response + except httpx.TimeoutException as e: + raise DownloadTimeoutError( + f"Getting {request.url} took longer than {timeout} seconds." + ) from e + except httpx.UnsupportedProtocol as e: + raise UnsupportedURLSchemeError(str(e)) from e + except httpx.ConnectError as e: + if any(isinstance(c, gaierror) for c in _iter_exc_causes(e)): + raise CannotResolveHostError(str(e)) from e + raise DownloadConnectionRefusedError(str(e)) from e + except httpx.ProxyError as e: + raise DownloadConnectionRefusedError(str(e)) from e + except DOWNLOAD_FAILED_EXCEPTIONS as e: + raise DownloadFailedError(str(e)) from e + + @staticmethod + def _extract_headers(response: httpx.Response) -> Headers: + return Headers(response.headers.multi_items()) + + @staticmethod + def _build_base_response_args( + response: httpx.Response, + request: Request, + headers: Headers, + ) -> _BaseResponseArgs: + network_stream: AsyncNetworkStream = response.extensions["network_stream"] + server_addr = network_stream.get_extra_info("server_addr") + ip_address = ipaddress.ip_address(server_addr[0]) + ssl_object = network_stream.get_extra_info("ssl_object") + if isinstance(ssl_object, ssl.SSLObject): + cert = ssl_object.getpeercert(binary_form=True) + else: + cert = None + return { + "status": response.status_code, + "url": request.url, + "headers": headers, + "certificate": cert, + "ip_address": ip_address, + "protocol": response.http_version, + } + + @staticmethod + def _iter_body_chunks(response: httpx.Response) -> AsyncIterator[bytes]: + return response.aiter_raw() + + @staticmethod + def _is_dataloss_exception(exc: Exception) -> bool: + return isinstance( + exc, httpx.RemoteProtocolError + ) and "peer closed connection without sending complete message body" in str(exc) + + def _log_tls_info(self, response: httpx.Response, request: Request) -> None: + network_stream: AsyncNetworkStream = response.extensions["network_stream"] + extra_ssl_object = network_stream.get_extra_info("ssl_object") + if isinstance(extra_ssl_object, ssl.SSLObject): + _log_sslobj_debug_info(extra_ssl_object) + + async def close(self) -> None: + await self._default_client.aclose() + for client in self._proxy_clients.values(): + await client.aclose() diff --git a/scrapy/core/downloader/handlers/base.py b/scrapy/core/downloader/handlers/base.py new file mode 100644 index 000000000..999c003b4 --- /dev/null +++ b/scrapy/core/downloader/handlers/base.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy import Request + from scrapy.crawler import Crawler + from scrapy.http import Response + + +class BaseDownloadHandler(ABC): + """Optional base class for download handlers.""" + + lazy: bool = False + + def __init__(self, crawler: Crawler): + self.crawler = crawler + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls(crawler) + + @abstractmethod + async def download_request(self, request: Request) -> Response: + raise NotImplementedError + + async def close(self) -> None: # noqa: B027 + pass diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index 902c48fe2..69f403574 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -1,27 +1,27 @@ -from typing import Any, Dict +from __future__ import annotations + +from typing import TYPE_CHECKING from w3lib.url import parse_data_uri -from scrapy import Request, Spider +from scrapy.core.downloader.handlers.base import BaseDownloadHandler from scrapy.http import Response, TextResponse -from scrapy.utils.decorators import defers from scrapy.utils.response import get_response_class +if TYPE_CHECKING: + from scrapy import Request -class DataURIDownloadHandler: - lazy = False - @defers - def download_request(self, request: Request, spider: Spider) -> Response: +class DataURIDownloadHandler(BaseDownloadHandler): + async def download_request(self, request: Request) -> Response: uri = parse_data_uri(request.url) respcls = get_response_class( body=uri.data, declared_mime_types=(uri.media_type.encode(),), ) - resp_kwargs: Dict[str, Any] = {} if issubclass(respcls, TextResponse) and uri.media_type.split("/")[0] == "text": charset = uri.media_type_parameters.get("charset") - resp_kwargs["encoding"] = charset + return respcls(url=request.url, body=uri.data, encoding=charset) - return respcls(url=request.url, body=uri.data, **resp_kwargs) + return respcls(url=request.url, body=uri.data) diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 46b1cb0f8..8adedd943 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -1,17 +1,22 @@ +from __future__ import annotations + from pathlib import Path +from typing import TYPE_CHECKING from w3lib.url import file_uri_to_path -from scrapy.utils.decorators import defers +from scrapy.core.downloader.handlers.base import BaseDownloadHandler +from scrapy.utils.asyncio import run_in_thread from scrapy.utils.response import get_response_class +if TYPE_CHECKING: + from scrapy import Request + from scrapy.http import Response -class FileDownloadHandler: - lazy = False - @defers - def download_request(self, request, spider): +class FileDownloadHandler(BaseDownloadHandler): + async def download_request(self, request: Request) -> Response: filepath = file_uri_to_path(request.url) - body = Path(filepath).read_bytes() + body = await run_in_thread(Path(filepath).read_bytes) respcls = get_response_class(url=request.url, body=body) return respcls(url=request.url, body=body) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 201ae8f57..d75ee917a 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -28,59 +28,73 @@ In case of status 200 request, response.headers will come with two keys: 'Size' - with size of the downloaded data """ +from __future__ import annotations + import re from io import BytesIO +from pathlib import Path +from typing import TYPE_CHECKING, BinaryIO, ClassVar from urllib.parse import unquote from twisted.internet.protocol import ClientCreator, Protocol -from twisted.protocols.ftp import CommandFailed, FTPClient +from scrapy.core.downloader.handlers.base import BaseDownloadHandler +from scrapy.exceptions import NotConfigured from scrapy.http import Response +from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes from scrapy.utils.response import get_response_class +if TYPE_CHECKING: + from twisted.protocols.ftp import FTPClient + + from scrapy import Request + from scrapy.crawler import Crawler + class ReceivedDataProtocol(Protocol): - def __init__(self, filename=None): - self.__filename = filename - self.body = open(filename, "wb") if filename else BytesIO() - self.size = 0 + def __init__(self, filename: bytes | None = None): + self.__filename: bytes | None = filename + self.body: BinaryIO = ( + Path(filename.decode()).open("wb") if filename else BytesIO() + ) + self.size: int = 0 - def dataReceived(self, data): + def dataReceived(self, data: bytes) -> None: self.body.write(data) self.size += len(data) @property - def filename(self): + def filename(self) -> bytes | None: return self.__filename - def close(self): - self.body.close() if self.filename else self.body.seek(0) + def close(self) -> None: + if self.filename: + self.body.close() + else: + self.body.seek(0) _CODE_RE = re.compile(r"\d+") -class FTPDownloadHandler: - lazy = False - - CODE_MAPPING = { +class FTPDownloadHandler(BaseDownloadHandler): + CODE_MAPPING: ClassVar[dict[str, int]] = { "550": 404, "default": 503, } - def __init__(self, settings): - self.default_user = settings["FTP_USER"] - self.default_password = settings["FTP_PASSWORD"] - self.passive_mode = settings["FTP_PASSIVE_MODE"] + def __init__(self, crawler: Crawler): + if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): + raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.") + super().__init__(crawler) + self.default_user = crawler.settings["FTP_USER"] + self.default_password = crawler.settings["FTP_PASSWORD"] + self.passive_mode = crawler.settings["FTP_PASSIVE_MODE"] - @classmethod - def from_crawler(cls, crawler): - return cls(crawler.settings) - - def download_request(self, request, spider): + async def download_request(self, request: Request) -> Response: from twisted.internet import reactor + from twisted.protocols.ftp import CommandFailed, FTPClient parsed_url = urlparse_cached(request) user = request.meta.get("ftp_user", self.default_user) @@ -91,35 +105,23 @@ class FTPDownloadHandler: creator = ClientCreator( reactor, FTPClient, user, password, passive=passive_mode ) - dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21) - return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path)) - - def gotClient(self, client, request, filepath): - self.client = client - protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename")) - return client.retrieveFile(filepath, protocol).addCallbacks( - callback=self._build_response, - callbackArgs=(request, protocol), - errback=self._failed, - errbackArgs=(request,), + client: FTPClient = await maybe_deferred_to_future( + creator.connectTCP(parsed_url.hostname, parsed_url.port or 21) ) - - def _build_response(self, result, request, protocol): - self.result = result - protocol.close() - headers = {"local filename": protocol.filename or "", "size": protocol.size} - body = to_bytes(protocol.filename or protocol.body.read()) - respcls = get_response_class(url=request.url, body=body) - return respcls(url=request.url, status=200, body=body, headers=headers) - - def _failed(self, result, request): - message = result.getErrorMessage() - if result.type == CommandFailed: - m = _CODE_RE.search(message) - if m: + filepath = unquote(parsed_url.path) + protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename")) + try: + await maybe_deferred_to_future(client.retrieveFile(filepath, protocol)) + except CommandFailed as e: + message = str(e) + if m := _CODE_RE.search(message): ftpcode = m.group() httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"]) - return Response( - url=request.url, status=httpcode, body=to_bytes(message) - ) - raise result.type(result.value) + return Response(url=request.url, status=httpcode, body=message.encode()) + raise + protocol.close() + headers = {"local filename": protocol.filename or b"", "size": protocol.size} + body = protocol.filename or protocol.body.read() + respcls = get_response_class(url=request.url, body=body) + # hints for Headers-related types may need to be fixed to not use AnyStr + return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type] diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 52535bd8b..dc1b1e375 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,4 +1,19 @@ -from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler +# pragma: no file cover +import warnings + from scrapy.core.downloader.handlers.http11 import ( HTTP11DownloadHandler as HTTPDownloadHandler, ) +from scrapy.exceptions import ScrapyDeprecationWarning + +warnings.warn( + "The scrapy.core.downloader.handlers.http module is deprecated," + " please import scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" + " instead of its deprecated alias scrapy.core.downloader.handlers.http.HTTPDownloadHandler", + ScrapyDeprecationWarning, + stacklevel=2, +) + +__all__ = [ + "HTTPDownloadHandler", +] diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py deleted file mode 100644 index b6ac7a251..000000000 --- a/scrapy/core/downloader/handlers/http10.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Download handlers for http and https schemes -""" -from scrapy.utils.misc import build_from_crawler, load_object -from scrapy.utils.python import to_unicode - - -class HTTP10DownloadHandler: - lazy = False - - def __init__(self, settings, crawler=None): - self.HTTPClientFactory = load_object(settings["DOWNLOADER_HTTPCLIENTFACTORY"]) - self.ClientContextFactory = load_object( - settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] - ) - self._settings = settings - self._crawler = crawler - - @classmethod - def from_crawler(cls, crawler): - return cls(crawler.settings, crawler) - - def download_request(self, request, spider): - """Return a deferred for the HTTP download""" - factory = self.HTTPClientFactory(request) - self._connect(factory) - return factory.deferred - - def _connect(self, factory): - from twisted.internet import reactor - - host, port = to_unicode(factory.host), factory.port - if factory.scheme == b"https": - client_context_factory = build_from_crawler( - self.ClientContextFactory, - self._crawler, - ) - return reactor.connectSSL(host, port, factory, client_context_factory) - return reactor.connectTCP(host, port, factory) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index a8677b732..ef509c0cc 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -1,16 +1,21 @@ """Download handlers for http and https schemes""" +from __future__ import annotations + import ipaddress import logging import re from contextlib import suppress +from functools import partial from io import BytesIO -from time import time -from urllib.parse import urldefrag, urlunparse +from time import monotonic +from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast +from urllib.parse import urldefrag, urlparse -from twisted.internet import defer, protocol, ssl +from twisted.internet import ssl +from twisted.internet.defer import Deferred, succeed from twisted.internet.endpoints import TCP4ClientEndpoint -from twisted.internet.error import TimeoutError +from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.python.failure import Failure from twisted.web.client import ( URI, @@ -19,86 +24,144 @@ from twisted.web.client import ( ResponseDone, ResponseFailed, ) +from twisted.web.client import Response as TxResponse from twisted.web.http import PotentialDataLoss, _DataLoss from twisted.web.http_headers import Headers as TxHeaders -from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS, IResponse from zope.interface import implementer -from scrapy import signals -from scrapy.core.downloader.contextfactory import load_context_factory_from_settings -from scrapy.core.downloader.webclient import _parse -from scrapy.exceptions import StopDownload -from scrapy.http import Headers +from scrapy import Request, signals +from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings +from scrapy.exceptions import ( + DownloadCancelledError, + DownloadTimeoutError, + NotConfigured, + ResponseDataLossError, + StopDownload, +) +from scrapy.http import Headers, Response +from scrapy.utils._download_handlers import ( + check_stop_download, + get_dataloss_msg, + get_maxsize_msg, + get_warnsize_msg, + make_response, + normalize_bind_address, + wrap_twisted_exceptions, +) +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode -from scrapy.utils.response import get_response_class +from scrapy.utils.ssl import _log_ssl_conn_debug_info +from scrapy.utils.url import add_http_if_no_scheme + +from ._base_http import BaseHttpDownloadHandler + +if TYPE_CHECKING: + from twisted.internet.base import ReactorBase + from twisted.internet.interfaces import IConsumer + + # typing.NotRequired requires Python 3.11 + from typing_extensions import NotRequired + + from scrapy.crawler import Crawler logger = logging.getLogger(__name__) +_T = TypeVar("_T") -class HTTP11DownloadHandler: - lazy = False - def __init__(self, settings, crawler=None): +class _ResultT(TypedDict): + txresponse: TxResponse + body: NotRequired[bytes] + flags: NotRequired[list[str] | None] + certificate: NotRequired[ssl.Certificate | None] + ip_address: NotRequired[ipaddress.IPv4Address | ipaddress.IPv6Address | None] + stop_download: NotRequired[StopDownload | None] + + +class HTTP11DownloadHandler(BaseHttpDownloadHandler): + def __init__(self, crawler: Crawler): + if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): + raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.") + super().__init__(crawler) self._crawler = crawler from twisted.internet import reactor - self._pool = HTTPConnectionPool(reactor, persistent=True) - self._pool.maxPersistentPerHost = settings.getint( + self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True) + self._pool.maxPersistentPerHost = crawler.settings.getint( "CONCURRENT_REQUESTS_PER_DOMAIN" ) self._pool._factory.noisy = False - self._contextFactory = load_context_factory_from_settings(settings, crawler) - self._default_maxsize = settings.getint("DOWNLOAD_MAXSIZE") - self._default_warnsize = settings.getint("DOWNLOAD_WARNSIZE") - self._fail_on_dataloss = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS") - self._disconnect_timeout = 1 + self._contextFactory: IPolicyForHTTPS = _load_context_factory_from_settings( + crawler + ) + self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS") + self._disconnect_timeout: int = 1 - @classmethod - def from_crawler(cls, crawler): - return cls(crawler.settings, crawler) - - def download_request(self, request, spider): + async def download_request(self, request: Request) -> Response: """Return a deferred for the HTTP download""" - agent = ScrapyAgent( + if hasattr(self._crawler.spider, "download_maxsize"): # pragma: no cover + warn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE") + if hasattr(self._crawler.spider, "download_warnsize"): # pragma: no cover + warn_on_deprecated_spider_attribute( + "download_warnsize", "DOWNLOAD_WARNSIZE" + ) + + agent = _ScrapyAgent( contextFactory=self._contextFactory, + bindAddress=self._bind_address, pool=self._pool, - maxsize=getattr(spider, "download_maxsize", self._default_maxsize), - warnsize=getattr(spider, "download_warnsize", self._default_warnsize), + maxsize=getattr( + self._crawler.spider, "download_maxsize", self._default_maxsize + ), + warnsize=getattr( + self._crawler.spider, "download_warnsize", self._default_warnsize + ), fail_on_dataloss=self._fail_on_dataloss, crawler=self._crawler, + tls_verbose_logging=self._tls_verbose_logging, ) - return agent.download_request(request) + try: + with wrap_twisted_exceptions(): + return await maybe_deferred_to_future(agent.download_request(request)) + except ResponseDataLossError: + if not self._fail_on_dataloss_warned: + logger.warning(get_dataloss_msg(request.url)) + self._fail_on_dataloss_warned = True + raise - def close(self): + async def close(self) -> None: from twisted.internet import reactor - d = self._pool.closeCachedConnections() + d: Deferred[None] = self._pool.closeCachedConnections() # closeCachedConnections will hang on network or server issues, so # we'll manually timeout the deferred. # # Twisted issue addressing this problem can be found here: - # https://twistedmatrix.com/trac/ticket/7738. + # https://github.com/twisted/twisted/issues/7738 # # closeCachedConnections doesn't handle external errbacks, so we'll # issue a callback after `_disconnect_timeout` seconds. - delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, []) + # + # See also https://github.com/scrapy/scrapy/issues/2653 + delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, ()) - def cancel_delayed_call(result): + try: + await maybe_deferred_to_future(d) + finally: if delayed_call.active(): delayed_call.cancel() - return result - - d.addBoth(cancel_delayed_call) - return d class TunnelError(Exception): """An HTTP CONNECT tunnel could not be established by the proxy.""" -class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): +class _TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): """An endpoint that tunnels through proxies to allow HTTPS downloads. To accomplish that, this endpoint sends an HTTP CONNECT to the proxy. The HTTP CONNECT is always sent when using this endpoint, I think this could @@ -115,39 +178,41 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def __init__( self, - reactor, - host, - port, - proxyConf, - contextFactory, - timeout=30, - bindAddress=None, + reactor: ReactorBase, + host: str, + port: int, + proxyConf: tuple[str, int, bytes | None], + contextFactory: IPolicyForHTTPS, + timeout: float = 30, + bindAddress: tuple[str, int] | None = None, ): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) - self._tunnelReadyDeferred = defer.Deferred() - self._tunneledHost = host - self._tunneledPort = port - self._contextFactory = contextFactory - self._connectBuffer = bytearray() + self._tunnelReadyDeferred: Deferred[Protocol] = Deferred() + self._tunneledHost: str = host + self._tunneledPort: int = port + self._contextFactory: IPolicyForHTTPS = contextFactory + self._connectBuffer: bytearray = bytearray() - def requestTunnel(self, protocol): + def requestTunnel(self, protocol: Protocol) -> Protocol: """Asks the proxy to open a tunnel.""" - tunnelReq = tunnel_request_data( + assert protocol.transport + tunnelReq = _tunnel_request_data( self._tunneledHost, self._tunneledPort, self._proxyAuthHeader ) protocol.transport.write(tunnelReq) self._protocolDataReceived = protocol.dataReceived - protocol.dataReceived = self.processProxyResponse + protocol.dataReceived = self.processProxyResponse # type: ignore[method-assign] self._protocol = protocol return protocol - def processProxyResponse(self, rcvd_bytes): + def processProxyResponse(self, data: bytes) -> None: """Processes the response from the proxy. If the tunnel is successfully created, notifies the client that we are ready to send requests. If not raises a TunnelError. """ - self._connectBuffer += rcvd_bytes + assert self._protocol.transport + self._connectBuffer += data # make sure that enough (all) bytes are consumed # and that we've got all HTTP headers (ending with a blank line) # from the proxy so that we don't send those bytes to the TLS layer @@ -155,23 +220,25 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): # see https://github.com/scrapy/scrapy/issues/2491 if b"\r\n\r\n" not in self._connectBuffer: return - self._protocol.dataReceived = self._protocolDataReceived - respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer) + self._protocol.dataReceived = self._protocolDataReceived # type: ignore[method-assign] + respm = _TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer) if respm and int(respm.group("status")) == 200: # set proper Server Name Indication extension - sslOptions = self._contextFactory.creatorForNetloc( - self._tunneledHost, self._tunneledPort + sslOptions = self._contextFactory.creatorForNetloc( # type: ignore[call-arg,misc] + self._tunneledHost, # type: ignore[arg-type] + self._tunneledPort, ) self._protocol.transport.startTLS(sslOptions, self._protocolFactory) self._tunnelReadyDeferred.callback(self._protocol) else: + extra: Any if respm: extra = { "status": int(respm.group("status")), "reason": respm.group("reason").strip(), } else: - extra = rcvd_bytes[: self._truncatedLength] + extra = data[: self._truncatedLength] self._tunnelReadyDeferred.errback( TunnelError( "Could not open CONNECT tunnel with proxy " @@ -179,11 +246,11 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): ) ) - def connectFailed(self, reason): + def connectFailed(self, reason: Failure) -> None: """Propagates the errback to the appropriate deferred.""" self._tunnelReadyDeferred.errback(reason) - def connect(self, protocolFactory): + def connect(self, protocolFactory: Factory) -> Deferred[Protocol]: self._protocolFactory = protocolFactory connectDeferred = super().connect(protocolFactory) connectDeferred.addCallback(self.requestTunnel) @@ -191,16 +258,18 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): return self._tunnelReadyDeferred -def tunnel_request_data(host, port, proxy_auth_header=None): +def _tunnel_request_data( + host: str, port: int, proxy_auth_header: bytes | None = None +) -> bytes: r""" Return binary content of a CONNECT request. >>> from scrapy.utils.python import to_unicode as s - >>> s(tunnel_request_data("example.com", 8080)) + >>> s(_tunnel_request_data("example.com", 8080)) 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n' - >>> s(tunnel_request_data("example.com", 8080, b"123")) + >>> s(_tunnel_request_data("example.com", 8080, b"123")) 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n' - >>> s(tunnel_request_data(b"example.com", "8090")) + >>> s(_tunnel_request_data(b"example.com", "8090")) 'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n' """ host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port)) @@ -212,7 +281,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None): return tunnel_req -class TunnelingAgent(Agent): +class _TunnelingAgent(Agent): """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS downloads. It may look strange that we have chosen to subclass Agent and not ProxyAgent but consider that after the tunnel is opened the proxy is @@ -222,19 +291,20 @@ class TunnelingAgent(Agent): def __init__( self, - reactor, - proxyConf, - contextFactory=None, - connectTimeout=None, - bindAddress=None, - pool=None, + *, + reactor: ReactorBase, + proxyConf: tuple[str, int, bytes | None], + contextFactory: IPolicyForHTTPS, + connectTimeout: float | None = None, + bindAddress: tuple[str, int] | None = None, + pool: HTTPConnectionPool | None = None, ): - super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) - self._proxyConf = proxyConf - self._contextFactory = contextFactory + super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) # type: ignore[no-untyped-call] + self._proxyConf: tuple[str, int, bytes | None] = proxyConf + self._contextFactory: IPolicyForHTTPS = contextFactory - def _getEndpoint(self, uri): - return TunnelingTCP4ClientEndpoint( + def _getEndpoint(self, uri: URI) -> _TunnelingTCP4ClientEndpoint: + return _TunnelingTCP4ClientEndpoint( reactor=self._reactor, host=uri.host, port=uri.port, @@ -245,8 +315,15 @@ class TunnelingAgent(Agent): ) def _requestWithEndpoint( - self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath - ): + self, + key: Any, + endpoint: TCP4ClientEndpoint, + method: bytes, + parsedURI: URI, + headers: TxHeaders | None, + bodyProducer: IBodyProducer | None, + requestPath: bytes, + ) -> Deferred[IResponse]: # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy @@ -262,27 +339,40 @@ class TunnelingAgent(Agent): ) -class ScrapyProxyAgent(Agent): +class _ScrapyProxyAgent(Agent): def __init__( - self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None + self, + reactor: ReactorBase, + proxyURI: bytes, + contextFactory: IPolicyForHTTPS, + connectTimeout: float | None = None, + bindAddress: tuple[str, int] | None = None, + pool: HTTPConnectionPool | None = None, ): - super().__init__( + super().__init__( # type: ignore[no-untyped-call] reactor=reactor, + contextFactory=contextFactory, connectTimeout=connectTimeout, bindAddress=bindAddress, pool=pool, ) - self._proxyURI = URI.fromBytes(proxyURI) + self._proxyURI: URI = URI.fromBytes(proxyURI) - def request(self, method, uri, headers=None, bodyProducer=None): + def request( + self, + method: bytes, + uri: bytes, + headers: TxHeaders | None = None, + bodyProducer: IBodyProducer | None = None, + ) -> Deferred[IResponse]: """ Issue a new request via the configured proxy. """ # Cache *all* connections under the same key, since we are only # connecting to a single destination, the proxy: return self._requestWithEndpoint( - key=("http-proxy", self._proxyURI.host, self._proxyURI.port), - endpoint=self._getEndpoint(self._proxyURI), + key=(b"http-proxy", self._proxyURI.host, self._proxyURI.port), + endpoint=self._getEndpoint(self._proxyURI), # type: ignore[no-untyped-call] method=method, parsedURI=URI.fromBytes(uri), headers=headers, @@ -291,45 +381,53 @@ class ScrapyProxyAgent(Agent): ) -class ScrapyAgent: - _Agent = Agent - _ProxyAgent = ScrapyProxyAgent - _TunnelingAgent = TunnelingAgent - +class _ScrapyAgent: def __init__( self, - contextFactory=None, - connectTimeout=10, - bindAddress=None, - pool=None, - maxsize=0, - warnsize=0, - fail_on_dataloss=True, - crawler=None, + *, + contextFactory: IPolicyForHTTPS, + connectTimeout: float = 10, + bindAddress: str | tuple[str, int] | None = None, + pool: HTTPConnectionPool | None = None, + maxsize: int = 0, + warnsize: int = 0, + fail_on_dataloss: bool = True, + crawler: Crawler, + tls_verbose_logging: bool = False, ): - self._contextFactory = contextFactory - self._connectTimeout = connectTimeout - self._bindAddress = bindAddress - self._pool = pool - self._maxsize = maxsize - self._warnsize = warnsize - self._fail_on_dataloss = fail_on_dataloss - self._txresponse = None - self._crawler = crawler + self._contextFactory: IPolicyForHTTPS = contextFactory + self._connectTimeout: float = connectTimeout + self._bindAddress: str | tuple[str, int] | None = bindAddress + self._pool: HTTPConnectionPool | None = pool + self._maxsize: int = maxsize + self._warnsize: int = warnsize + self._fail_on_dataloss: bool = fail_on_dataloss + self._txresponse: TxResponse | None = None + self._crawler: Crawler = crawler + self._tls_verbose_logging: bool = tls_verbose_logging - def _get_agent(self, request, timeout): + def _get_agent(self, request: Request, timeout: float) -> Agent: from twisted.internet import reactor bindaddress = request.meta.get("bindaddress") or self._bindAddress + bindaddress = normalize_bind_address(bindaddress) proxy = request.meta.get("proxy") if proxy: - proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy) - scheme = _parse(request.url)[0] - proxyHost = to_unicode(proxyHost) - if scheme == b"https": + proxy = add_http_if_no_scheme(proxy) + proxy_parsed = urlparse(proxy) + proxy_host = proxy_parsed.hostname + proxy_port = proxy_parsed.port + if not proxy_port: + proxy_port = 443 if proxy_parsed.scheme == "https" else 80 + if urlparse_cached(request).scheme == "https": + if proxy_parsed.scheme == "https": # pragma: no cover + raise NotImplementedError( + "HTTPS proxies for HTTPS destinations are not supported" + ) + assert proxy_host is not None proxyAuth = request.headers.get(b"Proxy-Authorization", None) - proxyConf = (proxyHost, proxyPort, proxyAuth) - return self._TunnelingAgent( + proxyConf = (proxy_host, proxy_port, proxyAuth) + return _TunnelingAgent( reactor=reactor, proxyConf=proxyConf, contextFactory=self._contextFactory, @@ -337,17 +435,16 @@ class ScrapyAgent: bindAddress=bindaddress, pool=self._pool, ) - proxyScheme = proxyScheme or b"http" - proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, "", "", "")) - return self._ProxyAgent( + return _ScrapyProxyAgent( reactor=reactor, - proxyURI=to_bytes(proxyURI, encoding="ascii"), + proxyURI=to_bytes(proxy, encoding="ascii"), + contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool, ) - return self._Agent( + return Agent( reactor=reactor, contextFactory=self._contextFactory, connectTimeout=timeout, @@ -355,7 +452,7 @@ class ScrapyAgent: pool=self._pool, ) - def download_request(self, request): + def download_request(self, request: Request) -> Deferred[Response]: from twisted.internet import reactor timeout = request.meta.get("download_timeout") or self._connectTimeout @@ -365,27 +462,27 @@ class ScrapyAgent: url = urldefrag(request.url)[0] method = to_bytes(request.method) headers = TxHeaders(request.headers) - if isinstance(agent, self._TunnelingAgent): + if isinstance(agent, _TunnelingAgent): headers.removeHeader(b"Proxy-Authorization") - if request.body: - bodyproducer = _RequestBodyProducer(request.body) - else: - bodyproducer = None - start_time = time() - d = agent.request( - method, to_bytes(url, encoding="ascii"), headers, bodyproducer + bodyproducer = _RequestBodyProducer(request.body) if request.body else None + start_time = monotonic() + d: Deferred[IResponse] = agent.request( + method, + to_bytes(url, encoding="ascii"), + headers, + cast("IBodyProducer", bodyproducer), ) # set download latency d.addCallback(self._cb_latency, request, start_time) # response body is ready to be consumed - d.addCallback(self._cb_bodyready, request) - d.addCallback(self._cb_bodydone, request, url) + d2: Deferred[_ResultT] = d.addCallback(self._cb_bodyready, request) + d3: Deferred[Response] = d2.addCallback(self._cb_bodydone, url) # check download timeout - self._timeout_cl = reactor.callLater(timeout, d.cancel) - d.addBoth(self._cb_timeout, request, url, timeout) - return d + self._timeout_cl = reactor.callLater(timeout, d3.cancel) + d3.addBoth(self._cb_timeout, request, url, timeout) + return d3 - def _cb_timeout(self, result, request, url, timeout): + def _cb_timeout(self, result: _T, request: Request, url: str, timeout: float) -> _T: if self._timeout_cl.active(): self._timeout_cl.cancel() return result @@ -394,90 +491,68 @@ class ScrapyAgent: if self._txresponse: self._txresponse._transport.stopProducing() - raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") + raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.") - def _cb_latency(self, result, request, start_time): - request.meta["download_latency"] = time() - start_time + def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T: + request.meta["download_latency"] = monotonic() - start_time return result @staticmethod - def _headers_from_twisted_response(response): + def _headers_from_twisted_response(response: TxResponse) -> Headers: headers = Headers() if response.length != UNKNOWN_LENGTH: headers[b"Content-Length"] = str(response.length).encode() headers.update(response.headers.getAllRawHeaders()) return headers - def _cb_bodyready(self, txresponse, request): - headers_received_result = self._crawler.signals.send_catch_log( - signal=signals.headers_received, + def _cb_bodyready( + self, txresponse: TxResponse, request: Request + ) -> _ResultT | Deferred[_ResultT]: + if stop_download := check_stop_download( + signals.headers_received, + self._crawler, + request, headers=self._headers_from_twisted_response(txresponse), body_length=txresponse.length, - request=request, - spider=self._crawler.spider, - ) - for handler, result in headers_received_result: - if isinstance(result, Failure) and isinstance(result.value, StopDownload): - logger.debug( - "Download stopped for %(request)s from signal handler %(handler)s", - {"request": request, "handler": handler.__qualname__}, - ) - txresponse._transport.stopProducing() - txresponse._transport.loseConnection() - return { - "txresponse": txresponse, - "body": b"", - "flags": ["download_stopped"], - "certificate": None, - "ip_address": None, - "failure": result if result.value.fail else None, - } - - # deliverBody hangs for responses without body - if txresponse.length == 0: + ): + txresponse._transport.stopProducing() + txresponse._transport.loseConnection() + return { + "txresponse": txresponse, + "stop_download": stop_download, + } + + # deliverBody hangs for responses without body + if cast("int", txresponse.length) == 0: return { "txresponse": txresponse, - "body": b"", - "flags": None, - "certificate": None, - "ip_address": None, } maxsize = request.meta.get("download_maxsize", self._maxsize) warnsize = request.meta.get("download_warnsize", self._warnsize) - expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 + expected_size = ( + cast("int", txresponse.length) + if txresponse.length != UNKNOWN_LENGTH + else -1 + ) fail_on_dataloss = request.meta.get( "download_fail_on_dataloss", self._fail_on_dataloss ) if maxsize and expected_size > maxsize: - warning_msg = ( - "Cancelling download of %(url)s: expected response " - "size (%(size)s) larger than download max size (%(maxsize)s)." + warning_msg = get_maxsize_msg( + expected_size, maxsize, request, expected=True ) - warning_args = { - "url": request.url, - "size": expected_size, - "maxsize": maxsize, - } - - logger.warning(warning_msg, warning_args) - + logger.warning(warning_msg) txresponse._transport.loseConnection() - raise defer.CancelledError(warning_msg % warning_args) + raise DownloadCancelledError(warning_msg) if warnsize and expected_size > warnsize: logger.warning( - "Expected response size (%(size)s) larger than " - "download warn size (%(warnsize)s) in request %(request)s.", - {"size": expected_size, "warnsize": warnsize, "request": request}, + get_warnsize_msg(expected_size, warnsize, request, expected=True) ) - def _cancel(_): - # Abort connection immediately. - txresponse._transport._producer.abortConnection() - - d = defer.Deferred(_cancel) + d: Deferred[_ResultT] = Deferred(partial(self._cancel, txresponse=txresponse)) txresponse.deliverBody( _ResponseReader( finished=d, @@ -487,6 +562,7 @@ class ScrapyAgent: warnsize=warnsize, fail_on_dataloss=fail_on_dataloss, crawler=self._crawler, + tls_verbose_logging=self._tls_verbose_logging, ) ) @@ -495,73 +571,78 @@ class ScrapyAgent: return d - def _cb_bodydone(self, result, request, url): + @staticmethod + def _cancel(_: Any, txresponse: TxResponse) -> None: + # Abort connection immediately. + txresponse._transport._producer.abortConnection() + + def _cb_bodydone(self, result: _ResultT, url: str) -> Response: headers = self._headers_from_twisted_response(result["txresponse"]) - respcls = get_response_class(http_headers=headers, url=url, body=result["body"]) try: version = result["txresponse"].version protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}" except (AttributeError, TypeError, IndexError): protocol = None - response = respcls( + return make_response( url=url, status=int(result["txresponse"].code), headers=headers, - body=result["body"], - flags=result["flags"], - certificate=result["certificate"], - ip_address=result["ip_address"], + body=result.get("body", b""), + flags=result.get("flags"), + certificate=result.get("certificate"), + ip_address=result.get("ip_address"), protocol=protocol, + stop_download=result.get("stop_download"), ) - if result.get("failure"): - result["failure"].value.response = response - return result["failure"] - return response @implementer(IBodyProducer) class _RequestBodyProducer: - def __init__(self, body): + def __init__(self, body: bytes): self.body = body self.length = len(body) - def startProducing(self, consumer): + def startProducing(self, consumer: IConsumer) -> Deferred[None]: consumer.write(self.body) - return defer.succeed(None) + return succeed(None) - def pauseProducing(self): + def pauseProducing(self) -> None: pass - def stopProducing(self): + def stopProducing(self) -> None: pass -class _ResponseReader(protocol.Protocol): +class _ResponseReader(Protocol): def __init__( self, - finished, - txresponse, - request, - maxsize, - warnsize, - fail_on_dataloss, - crawler, + finished: Deferred[_ResultT], + txresponse: TxResponse, + request: Request, + maxsize: int, + warnsize: int, + fail_on_dataloss: bool, + crawler: Crawler, + *, + tls_verbose_logging: bool = False, ): - self._finished = finished - self._txresponse = txresponse - self._request = request - self._bodybuf = BytesIO() - self._maxsize = maxsize - self._warnsize = warnsize - self._fail_on_dataloss = fail_on_dataloss - self._fail_on_dataloss_warned = False - self._reached_warnsize = False - self._bytes_received = 0 - self._certificate = None - self._ip_address = None - self._crawler = crawler + self._finished: Deferred[_ResultT] = finished + self._txresponse: TxResponse = txresponse + self._request: Request = request + self._bodybuf: BytesIO = BytesIO() + self._maxsize: int = maxsize + self._warnsize: int = warnsize + self._fail_on_dataloss: bool = fail_on_dataloss + self._reached_warnsize: bool = False + self._bytes_received: int = 0 + self._certificate: ssl.Certificate | None = None + self._ip_address: ipaddress.IPv4Address | ipaddress.IPv6Address | None = None + self._crawler: Crawler = crawler + self._tls_verbose_logging: bool = tls_verbose_logging - def _finish_response(self, flags=None, failure=None): + def _finish_response( + self, flags: list[str] | None = None, stop_download: StopDownload | None = None + ) -> None: self._finished.callback( { "txresponse": self._txresponse, @@ -569,11 +650,12 @@ class _ResponseReader(protocol.Protocol): "flags": flags, "certificate": self._certificate, "ip_address": self._ip_address, - "failure": failure, + "stop_download": stop_download, } ) - def connectionMade(self): + def connectionMade(self) -> None: + assert self.transport if self._certificate is None: with suppress(AttributeError): self._certificate = ssl.Certificate( @@ -585,40 +667,33 @@ class _ResponseReader(protocol.Protocol): self.transport._producer.getPeer().host ) - def dataReceived(self, bodyBytes): + if self._tls_verbose_logging: + connection = self.transport._producer.getHandle() + hostname = urlparse_cached(self._request).hostname + assert hostname is not None + _log_ssl_conn_debug_info(hostname, connection) + + def dataReceived(self, data: bytes) -> None: # This maybe called several times after cancel was called with buffered data. if self._finished.called: return - self._bodybuf.write(bodyBytes) - self._bytes_received += len(bodyBytes) + assert self.transport + self._bodybuf.write(data) + self._bytes_received += len(data) - bytes_received_result = self._crawler.signals.send_catch_log( - signal=signals.bytes_received, - data=bodyBytes, - request=self._request, - spider=self._crawler.spider, - ) - for handler, result in bytes_received_result: - if isinstance(result, Failure) and isinstance(result.value, StopDownload): - logger.debug( - "Download stopped for %(request)s from signal handler %(handler)s", - {"request": self._request, "handler": handler.__qualname__}, - ) - self.transport.stopProducing() - self.transport.loseConnection() - failure = result if result.value.fail else None - self._finish_response(flags=["download_stopped"], failure=failure) + if stop_download := check_stop_download( + signals.bytes_received, self._crawler, self._request, data=data + ): + self.transport.stopProducing() + self.transport.loseConnection() + self._finish_response(stop_download=stop_download) if self._maxsize and self._bytes_received > self._maxsize: logger.warning( - "Received (%(bytes)s) bytes larger than download " - "max size (%(maxsize)s) in request %(request)s.", - { - "bytes": self._bytes_received, - "maxsize": self._maxsize, - "request": self._request, - }, + get_maxsize_msg( + self._bytes_received, self._maxsize, self._request, expected=False + ) ) # Clear buffer earlier to avoid keeping data in memory for a long time. self._bodybuf.truncate(0) @@ -631,12 +706,12 @@ class _ResponseReader(protocol.Protocol): ): self._reached_warnsize = True logger.warning( - "Received more bytes than download " - "warn size (%(warnsize)s) in request %(request)s.", - {"warnsize": self._warnsize, "request": self._request}, + get_warnsize_msg( + self._bytes_received, self._warnsize, self._request, expected=False + ) ) - def connectionLost(self, reason): + def connectionLost(self, reason: Failure = connectionDone) -> None: if self._finished.called: return @@ -649,19 +724,15 @@ class _ResponseReader(protocol.Protocol): return if reason.check(ResponseFailed) and any( - r.check(_DataLoss) for r in reason.value.reasons + r.check(_DataLoss) + for r in reason.value.reasons # type: ignore[union-attr] ): if not self._fail_on_dataloss: self._finish_response(flags=["dataloss"]) return - if not self._fail_on_dataloss_warned: - logger.warning( - "Got data loss in %s. If you want to process broken " - "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" - " -- This message won't be shown in further requests", - self._txresponse.request.absoluteURI.decode(), - ) - self._fail_on_dataloss_warned = True + exc = ResponseDataLossError() + exc.__cause__ = reason.value + reason = Failure(exc) self._finished.errback(reason) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index b2579362c..f60c58d1b 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,64 +1,78 @@ -from time import time -from typing import Optional, Type, TypeVar +from __future__ import annotations + +from time import monotonic +from typing import TYPE_CHECKING from urllib.parse import urldefrag -from twisted.internet.base import DelayedCall -from twisted.internet.defer import Deferred -from twisted.internet.error import TimeoutError -from twisted.web.client import URI - -from scrapy.core.downloader.contextfactory import load_context_factory_from_settings -from scrapy.core.downloader.webclient import _parse -from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent -from scrapy.crawler import Crawler -from scrapy.http import Request, Response -from scrapy.settings import Settings -from scrapy.spiders import Spider -from scrapy.utils.python import to_bytes - -H2DownloadHandlerOrSubclass = TypeVar( - "H2DownloadHandlerOrSubclass", bound="H2DownloadHandler" +from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings +from scrapy.core.downloader.handlers.base import BaseDownloadHandler +from scrapy.core.http2.agent import H2Agent, H2ConnectionPool +from scrapy.exceptions import ( + DownloadTimeoutError, + NotConfigured, + UnsupportedURLSchemeError, ) +from scrapy.utils._download_handlers import ( + normalize_bind_address, + wrap_twisted_exceptions, +) +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.httpobj import urlparse_cached + +if TYPE_CHECKING: + from twisted.internet.base import DelayedCall + from twisted.internet.defer import Deferred + from twisted.web.iweb import IPolicyForHTTPS + + from scrapy.crawler import Crawler + from scrapy.http import Request, Response + from scrapy.spiders import Spider -class H2DownloadHandler: - def __init__(self, settings: Settings, crawler: Optional[Crawler] = None): +class H2DownloadHandler(BaseDownloadHandler): + lazy = True + + def __init__(self, crawler: Crawler): + if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): + raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.") + super().__init__(crawler) self._crawler = crawler from twisted.internet import reactor - self._pool = H2ConnectionPool(reactor, settings) - self._context_factory = load_context_factory_from_settings(settings, crawler) + self._pool = H2ConnectionPool(reactor, crawler.settings) + self._context_factory = _load_context_factory_from_settings(crawler) + self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS") - @classmethod - def from_crawler( - cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler - ) -> H2DownloadHandlerOrSubclass: - return cls(crawler.settings, crawler) - - def download_request(self, request: Request, spider: Spider) -> Deferred: - agent = ScrapyH2Agent( + async def download_request(self, request: Request) -> Response: + if urlparse_cached(request).scheme == "http": # pragma: no cover + raise UnsupportedURLSchemeError( + f"{type(self).__name__} doesn't support plain HTTP." + ) + agent = _ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, + bind_address=self._bind_address, crawler=self._crawler, ) - return agent.download_request(request, spider) + assert self._crawler.spider + with wrap_twisted_exceptions(): + return await maybe_deferred_to_future( + agent.download_request(request, self._crawler.spider) + ) - def close(self) -> None: + async def close(self) -> None: self._pool.close_connections() -class ScrapyH2Agent: - _Agent = H2Agent - _ProxyAgent = ScrapyProxyH2Agent - +class _ScrapyH2Agent: def __init__( self, - context_factory, + context_factory: IPolicyForHTTPS, pool: H2ConnectionPool, connect_timeout: int = 10, - bind_address: Optional[bytes] = None, - crawler: Optional[Crawler] = None, + bind_address: str | tuple[str, int] | None = None, + crawler: Crawler | None = None, ) -> None: self._context_factory = context_factory self._connect_timeout = connect_timeout @@ -66,30 +80,14 @@ class ScrapyH2Agent: self._pool = pool self._crawler = crawler - def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent: + def _get_agent(self, request: Request, timeout: float | None) -> H2Agent: from twisted.internet import reactor + if request.meta.get("proxy"): # pragma: no cover + raise NotImplementedError(f"{type(self).__name__} doesn't support proxies.") bind_address = request.meta.get("bindaddress") or self._bind_address - proxy = request.meta.get("proxy") - if proxy: - _, _, proxy_host, proxy_port, proxy_params = _parse(proxy) - scheme = _parse(request.url)[0] - - if scheme == b"https": - # ToDo - raise NotImplementedError( - "Tunneling via CONNECT method using HTTP/2.0 is not yet supported" - ) - return self._ProxyAgent( - reactor=reactor, - context_factory=self._context_factory, - proxy_uri=URI.fromBytes(to_bytes(proxy, encoding="ascii")), - connect_timeout=timeout, - bind_address=bind_address, - pool=self._pool, - ) - - return self._Agent( + bind_address = normalize_bind_address(bind_address) + return H2Agent( reactor=reactor, context_factory=self._context_factory, connect_timeout=timeout, @@ -97,13 +95,13 @@ class ScrapyH2Agent: pool=self._pool, ) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: from twisted.internet import reactor timeout = request.meta.get("download_timeout") or self._connect_timeout agent = self._get_agent(request, timeout) - start_time = time() + start_time = monotonic() d = agent.request(request, spider) d.addCallback(self._cb_latency, request, start_time) @@ -115,7 +113,7 @@ class ScrapyH2Agent: def _cb_latency( response: Response, request: Request, start_time: float ) -> Response: - request.meta["download_latency"] = time() - start_time + request.meta["download_latency"] = monotonic() - start_time return response @staticmethod @@ -127,4 +125,4 @@ class ScrapyH2Agent: return response url = urldefrag(request.url)[0] - raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") + raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.") diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 1f7533759..c6660ff48 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,66 +1,53 @@ -from scrapy.core.downloader.handlers.http import HTTPDownloadHandler +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, cast + +from scrapy.core.downloader.handlers.base import BaseDownloadHandler from scrapy.exceptions import NotConfigured from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.misc import build_from_crawler +from scrapy.utils.misc import build_from_crawler, load_object + +if TYPE_CHECKING: + from collections.abc import Mapping + + from scrapy import Request + from scrapy.crawler import Crawler + from scrapy.http import Response -class S3DownloadHandler: - def __init__( - self, - settings, - *, - crawler=None, - aws_access_key_id=None, - aws_secret_access_key=None, - aws_session_token=None, - httpdownloadhandler=HTTPDownloadHandler, - **kw, - ): +class S3DownloadHandler(BaseDownloadHandler): + lazy = True + + def __init__(self, crawler: Crawler): if not is_botocore_available(): raise NotConfigured("missing botocore library") - if not aws_access_key_id: - aws_access_key_id = settings["AWS_ACCESS_KEY_ID"] - if not aws_secret_access_key: - aws_secret_access_key = settings["AWS_SECRET_ACCESS_KEY"] - if not aws_session_token: - aws_session_token = settings["AWS_SESSION_TOKEN"] - - # If no credentials could be found anywhere, - # consider this an anonymous connection request by default; - # unless 'anon' was set explicitly (True/False). - anon = kw.get("anon") - if anon is None and not aws_access_key_id and not aws_secret_access_key: - kw["anon"] = True - self.anon = kw.get("anon") - + super().__init__(crawler) + aws_access_key_id = crawler.settings["AWS_ACCESS_KEY_ID"] + aws_secret_access_key = crawler.settings["AWS_SECRET_ACCESS_KEY"] + aws_session_token = crawler.settings["AWS_SESSION_TOKEN"] + self.anon = not aws_access_key_id and not aws_secret_access_key self._signer = None - import botocore.auth - import botocore.credentials - - kw.pop("anon", None) - if kw: - raise TypeError(f"Unexpected keyword arguments: {kw}") if not self.anon: + import botocore.auth # noqa: PLC0415 + import botocore.credentials # noqa: PLC0415 + SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"] - self._signer = SignerCls( + # botocore.auth.BaseSigner doesn't have an __init__() with args, only subclasses do + self._signer = SignerCls( # type: ignore[call-arg] botocore.credentials.Credentials( aws_access_key_id, aws_secret_access_key, aws_session_token ) ) - _http_handler = build_from_crawler( - httpdownloadhandler, + _http_handler: BaseDownloadHandler = build_from_crawler( + load_object(crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"]), crawler, ) self._download_http = _http_handler.download_request - @classmethod - def from_crawler(cls, crawler, **kwargs): - return cls(crawler.settings, crawler=crawler, **kwargs) - - def download_request(self, request, spider): + async def download_request(self, request: Request) -> Response: p = urlparse_cached(request) scheme = "https" if request.meta.get("is_secure") else "http" bucket = p.hostname @@ -69,14 +56,15 @@ class S3DownloadHandler: if self.anon: request = request.replace(url=url) else: - import botocore.awsrequest + import botocore.awsrequest # noqa: PLC0415 awsrequest = botocore.awsrequest.AWSRequest( method=request.method, url=f"{scheme}://s3.amazonaws.com/{bucket}{path}", - headers=request.headers.to_unicode_dict(), + headers=cast("Mapping[str, Any]", request.headers.to_unicode_dict()), data=request.body, ) + assert self._signer self._signer.add_auth(awsrequest) request = request.replace(url=url, headers=awsrequest.headers.items()) - return self._download_http(request, spider) + return await self._download_http(request) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index dca13c01e..ab74e22a4 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,101 +3,154 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -from typing import Any, Callable, Generator, List, Union, cast -from twisted.internet.defer import Deferred, inlineCallbacks -from twisted.python.failure import Failure +from __future__ import annotations -from scrapy import Spider -from scrapy.exceptions import _InvalidOutput +import warnings +from functools import wraps +from typing import TYPE_CHECKING, Any + +from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput from scrapy.http import Request, Response from scrapy.middleware import MiddlewareManager -from scrapy.settings import BaseSettings from scrapy.utils.conf import build_component_list -from scrapy.utils.defer import deferred_from_coro, mustbe_deferred +from scrapy.utils.defer import ( + _defer_sleep_async, + deferred_from_coro, + ensure_awaitable, + maybe_deferred_to_future, +) +from scrapy.utils.python import global_object_name + +if TYPE_CHECKING: + from collections.abc import Callable, Coroutine + + from twisted.internet.defer import Deferred + + from scrapy import Spider + from scrapy.settings import BaseSettings class DownloaderMiddlewareManager(MiddlewareManager): component_name = "downloader middleware" @classmethod - def _get_mwlist_from_settings(cls, settings: BaseSettings) -> List[Any]: - return build_component_list(settings.getwithbase("DOWNLOADER_MIDDLEWARES")) + def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: + return build_component_list( + settings.get_component_priority_dict_with_base("DOWNLOADER_MIDDLEWARES") + ) def _add_middleware(self, mw: Any) -> None: if hasattr(mw, "process_request"): self.methods["process_request"].append(mw.process_request) + self._check_mw_method_spider_arg(mw.process_request) if hasattr(mw, "process_response"): self.methods["process_response"].appendleft(mw.process_response) + self._check_mw_method_spider_arg(mw.process_response) if hasattr(mw, "process_exception"): self.methods["process_exception"].appendleft(mw.process_exception) + self._check_mw_method_spider_arg(mw.process_exception) def download( - self, download_func: Callable, request: Request, spider: Spider - ) -> Deferred: - @inlineCallbacks - def process_request(request: Request) -> Generator[Deferred, Any, Any]: - for method in self.methods["process_request"]: - method = cast(Callable, method) - response = yield deferred_from_coro( - method(request=request, spider=spider) - ) - if response is not None and not isinstance( - response, (Response, Request) - ): - raise _InvalidOutput( - f"Middleware {method.__qualname__} must return None, Response or " - f"Request, got {response.__class__.__name__}" - ) - if response: - return response - return (yield download_func(request=request, spider=spider)) + self, + download_func: Callable[[Request, Spider], Deferred[Response]], + request: Request, + spider: Spider, + ) -> Deferred[Response | Request]: + warnings.warn( + "DownloaderMiddlewareManager.download() is deprecated, use download_async() instead", + ScrapyDeprecationWarning, + stacklevel=2, + ) - @inlineCallbacks - def process_response( - response: Union[Response, Request] - ) -> Generator[Deferred, Any, Union[Response, Request]]: - if response is None: - raise TypeError("Received None in process_response") - elif isinstance(response, Request): + @wraps(download_func) + async def download_func_wrapped(request: Request) -> Response: + return await maybe_deferred_to_future(download_func(request, spider)) + + self._set_compat_spider(spider) + return deferred_from_coro(self.download_async(download_func_wrapped, request)) + + async def download_async( + self, + download_func: Callable[[Request], Coroutine[Any, Any, Response]], + request: Request, + ) -> Response | Request: + + try: + result: Response | Request = await self._process_request( + request, download_func + ) + except Exception as ex: + await _defer_sleep_async() + # either returns a request or response (which we pass to process_response()) + # or reraises the exception + result = await self._process_exception(ex, request) + return await self._process_response(result, request) + + def _handle_mw_method(self, method: Callable[..., Any], **kwargs: Any) -> Any: + if method in self._mw_methods_requiring_spider: + kwargs["spider"] = self._spider + + return method(**kwargs) + + async def _process_request( + self, + request: Request, + download_func: Callable[[Request], Coroutine[Any, Any, Response]], + ) -> Response | Request: + for method in self.methods["process_request"]: + assert method is not None + response = await ensure_awaitable( + self._handle_mw_method(method, request=request), + _warn=global_object_name(method), + ) + if response is not None and not isinstance(response, (Response, Request)): + raise _InvalidOutput( + f"Middleware {method.__qualname__} must return None, Response or " + f"Request, got {response.__class__.__name__}" + ) + if response: return response + return await download_func(request) - for method in self.methods["process_response"]: - method = cast(Callable, method) - response = yield deferred_from_coro( - method(request=request, response=response, spider=spider) - ) - if not isinstance(response, (Response, Request)): - raise _InvalidOutput( - f"Middleware {method.__qualname__} must return Response or Request, " - f"got {type(response)}" - ) - if isinstance(response, Request): - return response + async def _process_response( + self, response: Response | Request, request: Request + ) -> Response | Request: + if response is None: + raise TypeError("Received None in process_response") + if isinstance(response, Request): return response - @inlineCallbacks - def process_exception( - failure: Failure, - ) -> Generator[Deferred, Any, Union[Failure, Response, Request]]: - exception = failure.value - for method in self.methods["process_exception"]: - method = cast(Callable, method) - response = yield deferred_from_coro( - method(request=request, exception=exception, spider=spider) - ) - if response is not None and not isinstance( - response, (Response, Request) - ): - raise _InvalidOutput( - f"Middleware {method.__qualname__} must return None, Response or " - f"Request, got {type(response)}" - ) - if response: - return response - return failure + for method in self.methods["process_response"]: + assert method is not None + response = await ensure_awaitable( + self._handle_mw_method(method, request=request, response=response), + _warn=global_object_name(method), + ) - deferred = mustbe_deferred(process_request, request) - deferred.addErrback(process_exception) - deferred.addCallback(process_response) - return deferred + if not isinstance(response, (Response, Request)): + raise _InvalidOutput( + f"Middleware {method.__qualname__} must return Response or Request, " + f"got {type(response)}" + ) + if isinstance(response, Request): + return response + return response + + async def _process_exception( + self, exception: Exception, request: Request | Response + ) -> Response | Request: + for method in self.methods["process_exception"]: + assert method is not None + response = await ensure_awaitable( + self._handle_mw_method(method, request=request, exception=exception), + _warn=global_object_name(method), + ) + if response is not None and not isinstance(response, (Response, Request)): + raise _InvalidOutput( + f"Middleware {method.__qualname__} must return None, Response or " + f"Request, got {type(response)}" + ) + if response: + return response + raise exception diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 33cea7263..3b903b39d 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -1,35 +1,74 @@ +from __future__ import annotations + import logging -from typing import Any, Dict +import warnings +from typing import TYPE_CHECKING, Any from OpenSSL import SSL +from service_identity import VerificationError from service_identity.exceptions import CertificateError -from twisted.internet._sslverify import ( - ClientTLSOptions, - VerificationError, - verifyHostname, +from service_identity.hazmat import ( + DNS_ID, + IPAddress_ID, + ServiceID, + verify_service_identity, ) -from twisted.internet.ssl import AcceptableCiphers +from service_identity.pyopenssl import ( + extract_patterns, + verify_hostname, + verify_ip_address, +) +from twisted.internet._sslverify import ClientTLSOptions +from twisted.internet.ssl import AcceptableCiphers, TLSVersion + +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.deprecate import create_deprecated_class + +if TYPE_CHECKING: + from collections.abc import Callable + + from OpenSSL.crypto import X509 + from twisted.protocols.tls import TLSMemoryBIOProtocol -from scrapy.utils.ssl import get_temp_key_info, x509name_to_string logger = logging.getLogger(__name__) -METHOD_TLS = "TLS" -METHOD_TLSv10 = "TLSv1.0" -METHOD_TLSv11 = "TLSv1.1" -METHOD_TLSv12 = "TLSv1.2" - - -openssl_methods: Dict[str, int] = { - METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) - METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only - METHOD_TLSv11: SSL.TLSv1_1_METHOD, # TLS 1.1 only - METHOD_TLSv12: SSL.TLSv1_2_METHOD, # TLS 1.2 only +_openssl_methods: dict[str, int] = { + "TLS": SSL.SSLv23_METHOD, # protocol negotiation (recommended) + "TLSv1.0": SSL.TLSv1_METHOD, # TLS 1.0 only + "TLSv1.1": SSL.TLSv1_1_METHOD, # TLS 1.1 only + "TLSv1.2": SSL.TLSv1_2_METHOD, # TLS 1.2 only } -class ScrapyClientTLSOptions(ClientTLSOptions): +def __getattr__(name: str) -> Any: + deprecated = { + "METHOD_TLS": "TLS", + "METHOD_TLSv10": "TLSv1.0", + "METHOD_TLSv11": "TLSv1.1", + "METHOD_TLSv12": "TLSv1.2", + "openssl_methods": _openssl_methods, + } + if name in deprecated: + warnings.warn( + f"scrapy.core.downloader.tls.{name} is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return deprecated[name] + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +_TWISTED_VERSION_MAP: dict[str, TLSVersion] = { + "TLSv1.0": TLSVersion.TLSv1_0, + "TLSv1.1": TLSVersion.TLSv1_1, + "TLSv1.2": TLSVersion.TLSv1_2, + "TLSv1.3": TLSVersion.TLSv1_3, +} + + +class _ScrapyClientTLSOptions(ClientTLSOptions): """ SSL Client connection creator ignoring certificate verification errors (for genuinely invalid certificates or bugs in verification code). @@ -37,46 +76,29 @@ class ScrapyClientTLSOptions(ClientTLSOptions): Same as Twisted's private _sslverify.ClientTLSOptions, except that VerificationError, CertificateError and ValueError exceptions are caught, so that the connection is not closed, only - logging warnings. Also, HTTPS connection parameters logging is added. - """ + logging warnings. - def __init__(self, hostname: str, ctx: SSL.Context, verbose_logging: bool = False): - super().__init__(hostname, ctx) - self.verbose_logging: bool = verbose_logging + Instances of this class are returned from + :class:`._ScrapyClientContextFactory`. + + This class is used on Twisted older than 26.4.0. + """ def _identityVerifyingInfoCallback( self, connection: SSL.Connection, where: int, ret: Any ) -> None: - if where & SSL.SSL_CB_HANDSHAKE_START: - connection.set_tlsext_host_name(self._hostnameBytes) - elif where & SSL.SSL_CB_HANDSHAKE_DONE: - if self.verbose_logging: - logger.debug( - "SSL connection to %s using protocol %s, cipher %s", - self._hostnameASCII, - connection.get_protocol_version_name(), - connection.get_cipher_name(), - ) - server_cert = connection.get_peer_certificate() - if server_cert: - logger.debug( - 'SSL connection certificate: issuer "%s", subject "%s"', - x509name_to_string(server_cert.get_issuer()), - x509name_to_string(server_cert.get_subject()), - ) - key_info = get_temp_key_info(connection._ssl) - if key_info: - logger.debug("SSL temp key: %s", key_info) - + if where & SSL.SSL_CB_HANDSHAKE_DONE: try: - verifyHostname(connection, self._hostnameASCII) + if self._hostnameIsDnsName: + verify_hostname(connection, self._hostnameASCII) + else: + verify_ip_address(connection, self._hostnameASCII) except (CertificateError, VerificationError) as e: logger.warning( 'Remote certificate is not valid for hostname "%s"; %s', self._hostnameASCII, e, ) - except ValueError as e: logger.warning( "Ignoring error while verifying certificate " @@ -84,6 +106,77 @@ class ScrapyClientTLSOptions(ClientTLSOptions): self._hostnameASCII, e, ) + else: + super()._identityVerifyingInfoCallback(connection, where, ret) # type: ignore[misc] + + +ScrapyClientTLSOptions = create_deprecated_class( + "ScrapyClientTLSOptions", + _ScrapyClientTLSOptions, + subclass_warn_message="{old} is deprecated.", + instance_warn_message="{cls} is deprecated.", +) + + +class _ScrapyClientTLSOptions26(ClientTLSOptions): + """ + SSL Client connection creator ignoring certificate verification errors + (for genuinely invalid certificates or bugs in verification code). + + Same as Twisted's private _sslverify.ClientTLSOptions, + except that VerificationError, CertificateError and ValueError + exceptions are caught, so that the connection is not closed, only + logging warnings. + + Instances of this class are returned from + :class:`._ScrapyClientContextFactory`. + + This class is used on Twisted 26.4.0 and newer. + """ + + def clientConnectionForTLS( + self, tlsProtocol: TLSMemoryBIOProtocol + ) -> SSL.Connection: + """This method is needed to override the verify callback.""" + conn = super().clientConnectionForTLS(tlsProtocol) + callback = self._verifyCB(self._hostnameIsDnsName, self._hostnameASCII) + conn.set_verify(SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, callback) + return conn + + @staticmethod + def _verifyCB( + hostIsDNS: bool, hostnameASCII: str + ) -> Callable[[SSL.Connection, X509, int, int, int], bool]: + svcid: ServiceID = ( + DNS_ID(hostnameASCII) if hostIsDNS else IPAddress_ID(hostnameASCII) + ) + + def verifyCallback( + conn: SSL.Connection, cert: X509, err: int, depth: int, ok: int + ) -> bool: + if depth != 0: + # We are only verifying the leaf certificate. + return True + + try: + verify_service_identity(extract_patterns(cert), [svcid], []) + except (CertificateError, VerificationError) as e: + logger.warning( + 'Remote certificate is not valid for hostname "%s"; %s', + hostnameASCII, + e, + ) + except ValueError as e: + logger.warning( + "Ignoring error while verifying certificate " + 'from host "%s" (exception: %r)', + hostnameASCII, + e, + ) + + return True + + return verifyCallback DEFAULT_CIPHERS: AcceptableCiphers = AcceptableCiphers.fromOpenSSLCipherString( diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py deleted file mode 100644 index 647fc7e43..000000000 --- a/scrapy/core/downloader/webclient.py +++ /dev/null @@ -1,231 +0,0 @@ -import re -from time import time -from typing import Optional, Tuple -from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse - -from twisted.internet import defer -from twisted.internet.protocol import ClientFactory -from twisted.web.http import HTTPClient - -from scrapy import Request -from scrapy.http import Headers -from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes, to_unicode -from scrapy.utils.response import get_response_class - - -def _parsed_url_args(parsed: ParseResult) -> Tuple[bytes, bytes, bytes, int, bytes]: - # Assume parsed is urlparse-d from Request.url, - # which was passed via safe_url_string and is ascii-only. - path_str = urlunparse(("", "", parsed.path or "/", parsed.params, parsed.query, "")) - path = to_bytes(path_str, encoding="ascii") - assert parsed.hostname is not None - host = to_bytes(parsed.hostname, encoding="ascii") - port = parsed.port - scheme = to_bytes(parsed.scheme, encoding="ascii") - netloc = to_bytes(parsed.netloc, encoding="ascii") - if port is None: - port = 443 if scheme == b"https" else 80 - return scheme, netloc, host, port, path - - -def _parse(url: str) -> Tuple[bytes, bytes, bytes, int, bytes]: - """Return tuple of (scheme, netloc, host, port, path), - all in bytes except for port which is int. - Assume url is from Request.url, which was passed via safe_url_string - and is ascii-only. - """ - url = url.strip() - if not re.match(r"^\w+://", url): - url = "//" + url - parsed = urlparse(url) - return _parsed_url_args(parsed) - - -class ScrapyHTTPPageGetter(HTTPClient): - delimiter = b"\n" - - def connectionMade(self): - self.headers = Headers() # bucket for response headers - - # Method command - self.sendCommand(self.factory.method, self.factory.path) - # Headers - for key, values in self.factory.headers.items(): - for value in values: - self.sendHeader(key, value) - self.endHeaders() - # Body - if self.factory.body is not None: - self.transport.write(self.factory.body) - - def lineReceived(self, line): - return HTTPClient.lineReceived(self, line.rstrip()) - - def handleHeader(self, key, value): - self.headers.appendlist(key, value) - - def handleStatus(self, version, status, message): - self.factory.gotStatus(version, status, message) - - def handleEndHeaders(self): - self.factory.gotHeaders(self.headers) - - def connectionLost(self, reason): - self._connection_lost_reason = reason - HTTPClient.connectionLost(self, reason) - self.factory.noPage(reason) - - def handleResponse(self, response): - if self.factory.method.upper() == b"HEAD": - self.factory.page(b"") - elif self.length is not None and self.length > 0: - self.factory.noPage(self._connection_lost_reason) - else: - self.factory.page(response) - self.transport.loseConnection() - - def timeout(self): - self.transport.loseConnection() - - # transport cleanup needed for HTTPS connections - if self.factory.url.startswith(b"https"): - self.transport.stopProducing() - - self.factory.noPage( - defer.TimeoutError( - f"Getting {self.factory.url} took longer " - f"than {self.factory.timeout} seconds." - ) - ) - - -# This class used to inherit from Twisted’s -# twisted.web.client.HTTPClientFactory. When that class was deprecated in -# Twisted (https://github.com/twisted/twisted/pull/643), we merged its -# non-overridden code into this class. -class ScrapyHTTPClientFactory(ClientFactory): - protocol = ScrapyHTTPPageGetter - - waiting = 1 - noisy = False - followRedirect = False - afterFoundGet = False - - def _build_response(self, body, request): - request.meta["download_latency"] = self.headers_time - self.start_time - status = int(self.status) - headers = Headers(self.response_headers) - respcls = get_response_class(http_headers=headers, url=self._url, body=body) - return respcls( - url=self._url, - status=status, - headers=headers, - body=body, - protocol=to_unicode(self.version), - ) - - def _set_connection_attributes(self, request): - parsed = urlparse_cached(request) - self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args( - parsed - ) - proxy = request.meta.get("proxy") - if proxy: - self.scheme, _, self.host, self.port, _ = _parse(proxy) - self.path = self.url - - def __init__(self, request: Request, timeout: float = 180): - self._url: str = urldefrag(request.url)[0] - # converting to bytes to comply to Twisted interface - self.url: bytes = to_bytes(self._url, encoding="ascii") - self.method: bytes = to_bytes(request.method, encoding="ascii") - self.body: Optional[bytes] = request.body or None - self.headers: Headers = Headers(request.headers) - self.response_headers: Optional[Headers] = None - self.timeout: float = request.meta.get("download_timeout") or timeout - self.start_time: float = time() - self.deferred: defer.Deferred = defer.Deferred().addCallback( - self._build_response, request - ) - - # Fixes Twisted 11.1.0+ support as HTTPClientFactory is expected - # to have _disconnectedDeferred. See Twisted r32329. - # As Scrapy implements it's own logic to handle redirects is not - # needed to add the callback _waitForDisconnect. - # Specifically this avoids the AttributeError exception when - # clientConnectionFailed method is called. - self._disconnectedDeferred: defer.Deferred = defer.Deferred() - - self._set_connection_attributes(request) - - # set Host header based on url - self.headers.setdefault("Host", self.netloc) - - # set Content-Length based len of body - if self.body is not None: - self.headers["Content-Length"] = len(self.body) - # just in case a broken http/1.1 decides to keep connection alive - self.headers.setdefault("Connection", "close") - # Content-Length must be specified in POST method even with no body - elif self.method == b"POST": - self.headers["Content-Length"] = 0 - - def __repr__(self) -> str: - return f"<{self.__class__.__name__}: {self._url}>" - - def _cancelTimeout(self, result, timeoutCall): - if timeoutCall.active(): - timeoutCall.cancel() - return result - - def buildProtocol(self, addr): - p = ClientFactory.buildProtocol(self, addr) - p.followRedirect = self.followRedirect - p.afterFoundGet = self.afterFoundGet - if self.timeout: - from twisted.internet import reactor - - timeoutCall = reactor.callLater(self.timeout, p.timeout) - self.deferred.addBoth(self._cancelTimeout, timeoutCall) - return p - - def gotHeaders(self, headers): - self.headers_time = time() - self.response_headers = headers - - def gotStatus(self, version, status, message): - """ - Set the status of the request on us. - @param version: The HTTP version. - @type version: L{bytes} - @param status: The HTTP status code, an integer represented as a - bytestring. - @type status: L{bytes} - @param message: The HTTP status message. - @type message: L{bytes} - """ - self.version, self.status, self.message = version, status, message - - def page(self, page): - if self.waiting: - self.waiting = 0 - self.deferred.callback(page) - - def noPage(self, reason): - if self.waiting: - self.waiting = 0 - self.deferred.errback(reason) - - def clientConnectionFailed(self, _, reason): - """ - When a connection attempt fails, the request cannot be issued. If no - result has yet been provided to the result Deferred, provide the - connection failure reason as an error result. - """ - if self.waiting: - self.waiting = 0 - # If the connection attempt failed, there is nothing more to - # disconnect, so just fire that Deferred now. - self._disconnectedDeferred.callback(None) - self.deferred.errback(reason) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 545cd401f..1033e874f 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -4,61 +4,79 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spider. For more information see docs/topics/architecture.rst """ -import logging -from time import time -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Generator, - Iterable, - Iterator, - Optional, - Set, - Type, - Union, - cast, -) -from twisted.internet.defer import Deferred, inlineCallbacks, succeed -from twisted.internet.task import LoopingCall +from __future__ import annotations + +import asyncio +import contextlib +import logging +import warnings +from functools import partial +from time import time +from traceback import format_exc +from typing import TYPE_CHECKING, Any + +from twisted.internet.defer import CancelledError, Deferred, inlineCallbacks from twisted.python.failure import Failure from scrapy import signals -from scrapy.core.downloader import Downloader +from scrapy.core.scheduler import BaseScheduler from scrapy.core.scraper import Scraper -from scrapy.exceptions import CloseSpider, DontCloseSpider +from scrapy.exceptions import ( + CloseSpider, + DontCloseSpider, + IgnoreRequest, + ScrapyDeprecationWarning, +) from scrapy.http import Request, Response -from scrapy.logformatter import LogFormatter -from scrapy.settings import BaseSettings, Settings -from scrapy.signalmanager import SignalManager -from scrapy.spiders import Spider +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + create_looping_call, + is_asyncio_available, +) +from scrapy.utils.defer import ( + _schedule_coro, + deferred_from_coro, + ensure_awaitable, + maybe_deferred_to_future, +) +from scrapy.utils.deprecate import argument_is_required from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object +from scrapy.utils.python import global_object_name from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: - from scrapy.core.scheduler import BaseScheduler + from collections.abc import AsyncIterator, Callable, Coroutine, Generator + + from twisted.internet.task import LoopingCall + + from scrapy.core.downloader import Downloader from scrapy.crawler import Crawler + from scrapy.logformatter import LogFormatter + from scrapy.settings import BaseSettings, Settings + from scrapy.signalmanager import SignalManager + from scrapy.spiders import Spider + logger = logging.getLogger(__name__) -class Slot: +class _Slot: def __init__( self, - start_requests: Iterable[Request], close_if_idle: bool, - nextcall: CallLaterOnce, - scheduler: "BaseScheduler", + nextcall: CallLaterOnce[None], + scheduler: BaseScheduler, ) -> None: - self.closing: Optional[Deferred] = None - self.inprogress: Set[Request] = set() - self.start_requests: Optional[Iterator[Request]] = iter(start_requests) + self.closing: Deferred[None] | None = None + self.inprogress: set[Request] = set() self.close_if_idle: bool = close_if_idle - self.nextcall: CallLaterOnce = nextcall - self.scheduler: "BaseScheduler" = scheduler - self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule) + self.nextcall: CallLaterOnce[None] = nextcall + self.scheduler: BaseScheduler = scheduler + self.heartbeat: AsyncioLoopingCall | LoopingCall = create_looping_call( + nextcall.schedule + ) def add_request(self, request: Request) -> None: self.inprogress.add(request) @@ -67,10 +85,10 @@ class Slot: self.inprogress.remove(request) self._maybe_fire_closing() - def close(self) -> Deferred: + async def close(self) -> None: self.closing = Deferred() self._maybe_fire_closing() - return self.closing + await maybe_deferred_to_future(self.closing) def _maybe_fire_closing(self) -> None: if self.closing is not None and not self.inprogress: @@ -82,29 +100,60 @@ class Slot: class ExecutionEngine: - def __init__(self, crawler: "Crawler", spider_closed_callback: Callable) -> None: - self.crawler: "Crawler" = crawler + _SLOT_HEARTBEAT_INTERVAL: float = 5.0 + + def __init__( + self, + crawler: Crawler, + spider_closed_callback: Callable[ + [Spider], Coroutine[Any, Any, None] | Deferred[None] | None + ], + ) -> None: + self.crawler: Crawler = crawler self.settings: Settings = crawler.settings self.signals: SignalManager = crawler.signals assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter - self.slot: Optional[Slot] = None - self.spider: Optional[Spider] = None + self._slot: _Slot | None = None + self.spider: Spider | None = None self.running: bool = False + self._starting: bool = False + self._stopping: bool = False self.paused: bool = False - self.scheduler_cls: Type["BaseScheduler"] = self._get_scheduler_class( - crawler.settings - ) - downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"]) - self.downloader: Downloader = downloader_cls(crawler) - self.scraper = Scraper(crawler) - self._spider_closed_callback: Callable = spider_closed_callback - self.start_time: Optional[float] = None + self._spider_closed_callback: Callable[ + [Spider], Coroutine[Any, Any, None] | Deferred[None] | None + ] = spider_closed_callback + self.start_time: float | None = None + self._start: AsyncIterator[Any] | None = None + self._closewait: Deferred[None] | None = None + self._start_request_processing_awaitable: ( + asyncio.Future[None] | Deferred[None] | None + ) = None + downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) + try: + self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( + crawler.settings + ) + self.downloader: Downloader = downloader_cls(crawler) + self._downloader_fetch_needs_spider: bool = argument_is_required( + self.downloader.fetch, "spider" + ) + if self._downloader_fetch_needs_spider: + warnings.warn( + f"The fetch() method of {global_object_name(downloader_cls)} requires a spider argument," + f" this is deprecated and the argument will not be passed in future Scrapy versions.", + ScrapyDeprecationWarning, + stacklevel=2, + ) - def _get_scheduler_class(self, settings: BaseSettings) -> Type["BaseScheduler"]: - from scrapy.core.scheduler import BaseScheduler + self.scraper: Scraper = Scraper(crawler) + except Exception: + if hasattr(self, "downloader"): + self.downloader.close() + raise - scheduler_cls: Type = load_object(settings["SCHEDULER"]) + def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: + scheduler_cls: type[BaseScheduler] = load_object(settings["SCHEDULER"]) if not issubclass(scheduler_cls, BaseScheduler): raise TypeError( f"The provided scheduler class ({settings['SCHEDULER']})" @@ -112,48 +161,104 @@ class ExecutionEngine: ) return scheduler_cls - @inlineCallbacks - def start(self) -> Generator[Deferred, Any, None]: - if self.running: + def start( + self, _start_request_processing: bool = True + ) -> Deferred[None]: # pragma: no cover + warnings.warn( + "ExecutionEngine.start() is deprecated, use start_async() instead", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return deferred_from_coro( + self.start_async(_start_request_processing=_start_request_processing) + ) + + async def start_async(self, *, _start_request_processing: bool = True) -> None: + """Start the execution engine. + + .. versionadded:: 2.14 + """ + if self._starting: raise RuntimeError("Engine already running") self.start_time = time() - yield self.signals.send_catch_log_deferred(signal=signals.engine_started) + self._starting = True + await self.signals.send_catch_log_async(signal=signals.engine_started) + if self._stopping: + # band-aid until https://github.com/scrapy/scrapy/issues/6916 + return + if _start_request_processing and self.spider is None: + # require an opened spider when not run in scrapy shell + return self.running = True - self._closewait: Deferred = Deferred() - yield self._closewait + self._closewait = Deferred() + if _start_request_processing: + coro = self._start_request_processing() + if is_asyncio_available(): + # not wrapping in a Deferred here to avoid https://github.com/twisted/twisted/issues/12470 + # (can happen when this is cancelled, e.g. in test_close_during_start_iteration()) + self._start_request_processing_awaitable = asyncio.ensure_future(coro) + else: + self._start_request_processing_awaitable = Deferred.fromCoroutine(coro) + with contextlib.suppress(asyncio.exceptions.CancelledError): + await maybe_deferred_to_future(self._closewait) - def stop(self) -> Deferred: - """Gracefully stop the execution engine""" + def stop(self) -> Deferred[None]: # pragma: no cover + warnings.warn( + "ExecutionEngine.stop() is deprecated, use stop_async() instead", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return deferred_from_coro(self.stop_async()) - @inlineCallbacks - def _finish_stopping_engine(_: Any) -> Generator[Deferred, Any, None]: - yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) - self._closewait.callback(None) + async def stop_async(self) -> None: + """Gracefully stop the execution engine. - if not self.running: + .. versionadded:: 2.14 + """ + + if not self._starting: raise RuntimeError("Engine not running") - self.running = False - dfd = ( - self.close_spider(self.spider, reason="shutdown") - if self.spider is not None - else succeed(None) - ) - return dfd.addBoth(_finish_stopping_engine) + self.running = self._starting = False + self._stopping = True + if self._start_request_processing_awaitable is not None: + if ( + not is_asyncio_available() + or self._start_request_processing_awaitable + is not asyncio.current_task() + ): + # If using the asyncio loop and stop_async() was called from + # start() itself, we can't cancel it, and _start_request_processing() + # will exit via the self.running check. + self._start_request_processing_awaitable.cancel() + self._start_request_processing_awaitable = None + if self.spider is not None: + await self.close_spider_async(reason="shutdown") + await self.signals.send_catch_log_async(signal=signals.engine_stopped) + if self._closewait: + self._closewait.callback(None) - def close(self) -> Deferred: + def close(self) -> Deferred[None]: # pragma: no cover + warnings.warn( + "ExecutionEngine.close() is deprecated, use close_async() instead", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return deferred_from_coro(self.close_async()) + + async def close_async(self) -> None: """ Gracefully close the execution engine. If it has already been started, stop it. In all cases, close the spider and the downloader. """ if self.running: - return self.stop() # will also close spider and downloader - if self.spider is not None: - return self.close_spider( - self.spider, reason="shutdown" + await self.stop_async() # will also close spider and downloader + elif self.spider is not None: + await self.close_spider_async( + reason="shutdown" ) # will also close downloader - self.downloader.close() - return succeed(None) + elif hasattr(self, "downloader"): + self.downloader.close() def pause(self) -> None: self.paused = True @@ -161,58 +266,107 @@ class ExecutionEngine: def unpause(self) -> None: self.paused = False - def _next_request(self) -> None: - if self.slot is None: + async def _process_start_next(self) -> None: + """Processes the next item or request from Spider.start(). + + If a request, it is scheduled. If an item, it is sent to item + pipelines. + """ + assert self._start is not None + try: + item_or_request = await anext(self._start) + except StopAsyncIteration: + self._start = None + except Exception as exception: + self._start = None + exception_traceback = format_exc() + logger.error( + f"Error while reading start items and requests: {exception}.\n{exception_traceback}", + exc_info=True, + ) + else: + if not self.spider: + return # spider already closed + if isinstance(item_or_request, Request): + self.crawl(item_or_request) + else: + assert self._slot is not None + _schedule_coro( + self.scraper.start_itemproc_async(item_or_request, response=None) + ) + self._slot.nextcall.schedule() + + async def _start_request_processing(self) -> None: + """Starts consuming Spider.start() output and sending scheduled + requests.""" + # Starts the processing of scheduled requests, as well as a periodic + # call to that processing method for scenarios where the scheduler + # reports having pending requests but returns none. + try: + assert self._slot is not None # typing + self._slot.nextcall.schedule() + self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) + + while self._start and self.spider and self.running: + await self._process_start_next() + if not self.needs_backout(): + # Give room for the outcome of self._process_start_next() to be + # processed before continuing with the next iteration. + self._slot.nextcall.schedule() + await self._slot.nextcall.wait() + except (asyncio.exceptions.CancelledError, CancelledError): + # self.stop_async() has cancelled us, nothing to do + return + except Exception: + # an error happened, log it and stop the engine + self._start_request_processing_awaitable = None + logger.error( + "Error while processing requests from start()", + exc_info=True, + extra={"spider": self.spider}, + ) + await self.stop_async() + + def _start_scheduled_requests(self) -> None: + if self._slot is None or self._slot.closing is not None or self.paused: return - assert self.spider is not None # typing + while not self.needs_backout(): + if not self._start_scheduled_request(): + break - if self.paused: - return None - - while ( - not self._needs_backout() - and self._next_request_from_scheduler() is not None - ): - pass - - if self.slot.start_requests is not None and not self._needs_backout(): - try: - request = next(self.slot.start_requests) - except StopIteration: - self.slot.start_requests = None - except Exception: - self.slot.start_requests = None - logger.error( - "Error while obtaining start requests", - exc_info=True, - extra={"spider": self.spider}, - ) - else: - self.crawl(request) - - if self.spider_is_idle() and self.slot.close_if_idle: + if self.spider_is_idle() and self._slot.close_if_idle: self._spider_idle() - def _needs_backout(self) -> bool: - assert self.slot is not None # typing + def needs_backout(self) -> bool: + """Returns ``True`` if no more requests can be sent at the moment, or + ``False`` otherwise. + + See :ref:`start-requests-lazy` for an example. + """ assert self.scraper.slot is not None # typing return ( not self.running - or bool(self.slot.closing) + or not self._slot + or bool(self._slot.closing) or self.downloader.needs_backout() or self.scraper.slot.needs_backout() ) - def _next_request_from_scheduler(self) -> Optional[Deferred]: - assert self.slot is not None # typing + def _remove_request(self, _: Any, request: Request) -> None: + assert self._slot + self._slot.remove_request(request) + + def _start_scheduled_request(self) -> bool: + assert self._slot is not None # typing assert self.spider is not None # typing - request = self.slot.scheduler.next_request() + request = self._slot.scheduler.next_request() if request is None: - return None + self.signals.send_catch_log(signals.scheduler_empty) + return False - d = self._download(request) + d: Deferred[Response | Request] = self._download(request) d.addBoth(self._handle_downloader_output, request) d.addErrback( lambda f: logger.info( @@ -222,34 +376,29 @@ class ExecutionEngine: ) ) - def _remove_request(_: Any) -> None: - assert self.slot - self.slot.remove_request(request) - - d.addBoth(_remove_request) - d.addErrback( + d2: Deferred[None] = d.addBoth(partial(self._remove_request, request=request)) + d2.addErrback( lambda f: logger.info( "Error while removing request from slot", exc_info=failure_to_exc_info(f), extra={"spider": self.spider}, ) ) - slot = self.slot - d.addBoth(lambda _: slot.nextcall.schedule()) - d.addErrback( + slot = self._slot + d2.addBoth(lambda _: slot.nextcall.schedule()) + d2.addErrback( lambda f: logger.info( "Error while scheduling new request", exc_info=failure_to_exc_info(f), extra={"spider": self.spider}, ) ) - return d + return True + @inlineCallbacks def _handle_downloader_output( - self, result: Union[Request, Response, Failure], request: Request - ) -> Optional[Deferred]: - assert self.spider is not None # typing - + self, result: Request | Response | Failure, request: Request + ) -> Generator[Deferred[Any], Any, None]: if not isinstance(result, (Request, Response, Failure)): raise TypeError( f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}" @@ -258,66 +407,94 @@ class ExecutionEngine: # downloader middleware can return requests (for example, redirects) if isinstance(result, Request): self.crawl(result) - return None + return - d = self.scraper.enqueue_scrape(result, request, self.spider) - d.addErrback( - lambda f: logger.error( - "Error while enqueuing downloader output", - exc_info=failure_to_exc_info(f), + try: + yield self.scraper.enqueue_scrape(result, request) + except Exception: + assert self.spider is not None + logger.error( + "Error while enqueuing scrape", + exc_info=True, extra={"spider": self.spider}, ) - ) - return d def spider_is_idle(self) -> bool: - if self.slot is None: + if self._slot is None: raise RuntimeError("Engine slot not assigned") if not self.scraper.slot.is_idle(): # type: ignore[union-attr] return False if self.downloader.active: # downloader has pending requests return False - if self.slot.start_requests is not None: # not all start requests are handled + if self._start is not None: # not all start requests are handled return False - if self.slot.scheduler.has_pending_requests(): - return False - return True + return not self._slot.scheduler.has_pending_requests() def crawl(self, request: Request) -> None: """Inject the request into the spider <-> downloader pipeline""" if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - self._schedule_request(request, self.spider) - self.slot.nextcall.schedule() # type: ignore[union-attr] + self._schedule_request(request) + self._slot.nextcall.schedule() # type: ignore[union-attr] - def _schedule_request(self, request: Request, spider: Spider) -> None: - self.signals.send_catch_log( - signals.request_scheduled, request=request, spider=spider + def _schedule_request(self, request: Request) -> None: + request_scheduled_result = self.signals.send_catch_log( + signals.request_scheduled, + request=request, + spider=self.spider, + dont_log=IgnoreRequest, ) - if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] + for _, result in request_scheduled_result: + if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest): + return + if not self._slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log( - signals.request_dropped, request=request, spider=spider + signals.request_dropped, request=request, spider=self.spider ) - def download(self, request: Request) -> Deferred: + def download(self, request: Request) -> Deferred[Response]: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" + warnings.warn( + "ExecutionEngine.download() is deprecated, use download_async() instead", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return deferred_from_coro(self.download_async(request)) + + async def download_async(self, request: Request) -> Response: + """Return a coroutine which fires with a Response as result. + + Only downloader middlewares are applied. + + .. versionadded:: 2.14 + """ if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - return self._download(request).addBoth(self._downloaded, request) + try: + response_or_request = await maybe_deferred_to_future( + self._download(request) + ) + finally: + assert self._slot is not None + self._slot.remove_request(request) + if isinstance(response_or_request, Request): + return await self.download_async(response_or_request) + return response_or_request - def _downloaded( - self, result: Union[Response, Request, Failure], request: Request - ) -> Union[Deferred, Response, Failure]: - assert self.slot is not None # typing - self.slot.remove_request(request) - return self.download(result) if isinstance(result, Request) else result + @inlineCallbacks + def _download( + self, request: Request + ) -> Generator[Deferred[Any], Any, Response | Request]: + assert self._slot is not None # typing + assert self.spider is not None - def _download(self, request: Request) -> Deferred: - assert self.slot is not None # typing - - self.slot.add_request(request) - - def _on_success(result: Union[Response, Request]) -> Union[Response, Request]: + self._slot.add_request(request) + try: + result: Response | Request + if self._downloader_fetch_needs_spider: + result = yield self.downloader.fetch(request, self.spider) + else: + result = yield self.downloader.fetch(request) if not isinstance(result, (Response, Request)): raise TypeError( f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}" @@ -325,7 +502,6 @@ class ExecutionEngine: if isinstance(result, Response): if result.request is None: result.request = request - assert self.spider is not None logkws = self.logformatter.crawled(result.request, result, self.spider) if logkws is not None: logger.log( @@ -338,40 +514,48 @@ class ExecutionEngine: spider=self.spider, ) return result + finally: + self._slot.nextcall.schedule() - def _on_complete(_: Any) -> Any: - assert self.slot is not None - self.slot.nextcall.schedule() - return _ - - assert self.spider is not None - dwld = self.downloader.fetch(request, self.spider) - dwld.addCallbacks(_on_success) - dwld.addBoth(_on_complete) - return dwld - - @inlineCallbacks def open_spider( - self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True - ) -> Generator[Deferred, Any, None]: - if self.slot is not None: - raise RuntimeError(f"No free spider slot when opening {spider.name!r}") - logger.info("Spider opened", extra={"spider": spider}) - nextcall = CallLaterOnce(self._next_request) - scheduler = build_from_crawler(self.scheduler_cls, self.crawler) - start_requests = yield self.scraper.spidermw.process_start_requests( - start_requests, spider + self, spider: Spider, close_if_idle: bool = True + ) -> Deferred[None]: # pragma: no cover + warnings.warn( + "ExecutionEngine.open_spider() is deprecated, use open_spider_async() instead", + ScrapyDeprecationWarning, + stacklevel=2, ) - self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) - self.spider = spider - if hasattr(scheduler, "open"): - yield scheduler.open(spider) - yield self.scraper.open_spider(spider) + return deferred_from_coro(self.open_spider_async(close_if_idle=close_if_idle)) + + async def open_spider_async(self, *, close_if_idle: bool = True) -> None: + assert self.crawler.spider + if self._slot is not None: + raise RuntimeError( + f"No free spider slot when opening {self.crawler.spider.name!r}" + ) + logger.info("Spider opened", extra={"spider": self.crawler.spider}) + self.spider = self.crawler.spider + nextcall = CallLaterOnce(self._start_scheduled_requests) + scheduler = build_from_crawler(self.scheduler_cls, self.crawler) + self._slot = _Slot(close_if_idle, nextcall, scheduler) + self._start = await self.scraper.spidermw.process_start() + if hasattr(scheduler, "open") and (d := scheduler.open(self.crawler.spider)): + await maybe_deferred_to_future(d) + await self.scraper.open_spider_async() assert self.crawler.stats - self.crawler.stats.open_spider(spider) - yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) - self.slot.nextcall.schedule() - self.slot.heartbeat.start(5) + if argument_is_required(self.crawler.stats.open_spider, "spider"): + warnings.warn( + f"The open_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument," + f" this is deprecated and the argument will not be passed in future Scrapy versions.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + self.crawler.stats.open_spider(spider=self.crawler.spider) + else: + self.crawler.stats.open_spider() + await self.signals.send_catch_log_async( + signals.spider_opened, spider=self.crawler.spider + ) def _spider_idle(self) -> None: """ @@ -392,74 +576,111 @@ class ExecutionEngine: if isinstance(x, Failure) and isinstance(x.value, ex) } if DontCloseSpider in detected_ex: - return None + return if self.spider_is_idle(): ex = detected_ex.get(CloseSpider, CloseSpider(reason="finished")) assert isinstance(ex, CloseSpider) # typing - self.close_spider(self.spider, reason=ex.reason) + _schedule_coro(self.close_spider_async(reason=ex.reason)) - def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred: - """Close (cancel) spider and clear all its outstanding requests""" - if self.slot is None: + def close_spider( + self, spider: Spider, reason: str = "cancelled" + ) -> Deferred[None]: # pragma: no cover + warnings.warn( + "ExecutionEngine.close_spider() is deprecated, use close_spider_async() instead", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return deferred_from_coro(self.close_spider_async(reason=reason)) + + async def close_spider_async(self, *, reason: str = "cancelled") -> None: # noqa: PLR0912 + """Close (cancel) spider and clear all its outstanding requests. + + .. versionadded:: 2.14 + """ + if self.spider is None: + raise RuntimeError("Spider not opened") + + if self._slot is None: raise RuntimeError("Engine slot not assigned") - if self.slot.closing is not None: - return self.slot.closing + if self._slot.closing is not None: + await maybe_deferred_to_future(self._slot.closing) + return + + spider = self.spider logger.info( "Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider} ) - dfd = self.slot.close() + try: + await self._slot.close() + except Exception: + logger.error("Slot close failure", exc_info=True, extra={"spider": spider}) - def log_failure(msg: str) -> Callable: - def errback(failure: Failure) -> None: + try: + self.downloader.close() + except Exception: + logger.error( + "Downloader close failure", exc_info=True, extra={"spider": spider} + ) + + try: + await self.scraper.close_spider_async() + except Exception: + logger.error( + "Scraper close failure", exc_info=True, extra={"spider": spider} + ) + + if hasattr(self._slot.scheduler, "close"): + try: + if (d := self._slot.scheduler.close(reason)) is not None: + await maybe_deferred_to_future(d) + except Exception: logger.error( - msg, exc_info=failure_to_exc_info(failure), extra={"spider": spider} + "Scheduler close failure", exc_info=True, extra={"spider": spider} ) - return errback - - dfd.addBoth(lambda _: self.downloader.close()) - dfd.addErrback(log_failure("Downloader close failure")) - - dfd.addBoth(lambda _: self.scraper.close_spider(spider)) - dfd.addErrback(log_failure("Scraper close failure")) - - if hasattr(self.slot.scheduler, "close"): - dfd.addBoth(lambda _: cast(Slot, self.slot).scheduler.close(reason)) - dfd.addErrback(log_failure("Scheduler close failure")) - - dfd.addBoth( - lambda _: self.signals.send_catch_log_deferred( + try: + await self.signals.send_catch_log_async( signal=signals.spider_closed, spider=spider, reason=reason, ) - ) - dfd.addErrback(log_failure("Error while sending spider_close signal")) - - def close_stats(_: Any) -> None: - assert self.crawler.stats - self.crawler.stats.close_spider(spider, reason=reason) - - dfd.addBoth(close_stats) - dfd.addErrback(log_failure("Stats close failure")) - - dfd.addBoth( - lambda _: logger.info( - "Spider closed (%(reason)s)", - {"reason": reason}, + except Exception: + logger.error( + "Error while sending spider_close signal", + exc_info=True, extra={"spider": spider}, ) + + assert self.crawler.stats + try: + if argument_is_required(self.crawler.stats.close_spider, "spider"): + warnings.warn( + f"The close_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument," + f" this is deprecated and the argument will not be passed in future Scrapy versions.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + self.crawler.stats.close_spider( + spider=self.crawler.spider, reason=reason + ) + else: + self.crawler.stats.close_spider(reason=reason) + except Exception: + logger.error("Stats close failure") + + logger.info( + "Spider closed (%(reason)s)", + {"reason": reason}, + extra={"spider": spider}, ) - dfd.addBoth(lambda _: setattr(self, "slot", None)) - dfd.addErrback(log_failure("Error while unassigning slot")) + self._slot = None + self.spider = None - dfd.addBoth(lambda _: setattr(self, "spider", None)) - dfd.addErrback(log_failure("Error while unassigning spider")) - - dfd.addBoth(lambda _: self._spider_closed_callback(spider)) - - return dfd + try: + await ensure_awaitable(self._spider_closed_callback(spider)) + except Exception: + logger.error("Error running spider_closed_callback") diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 215ea9716..7137a0f2b 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,10 +1,10 @@ +from __future__ import annotations + from collections import deque -from typing import Deque, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING from twisted.internet import defer -from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred -from twisted.internet.endpoints import HostnameEndpoint from twisted.python.failure import Failure from twisted.web.client import ( URI, @@ -14,11 +14,19 @@ from twisted.web.client import ( ) from twisted.web.error import SchemeNotSupported -from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory +from scrapy.core.downloader.contextfactory import _AcceptableProtocolsContextFactory from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol -from scrapy.http.request import Request -from scrapy.settings import Settings -from scrapy.spiders import Spider + +if TYPE_CHECKING: + from twisted.internet.base import ReactorBase + from twisted.internet.endpoints import HostnameEndpoint + + from scrapy.http import Request, Response + from scrapy.settings import Settings + from scrapy.spiders import Spider + + +ConnectionKeyT = tuple[bytes, bytes, int] class H2ConnectionPool: @@ -28,19 +36,25 @@ class H2ConnectionPool: # Store a dictionary which is used to get the respective # H2ClientProtocolInstance using the key as Tuple(scheme, hostname, port) - self._connections: Dict[Tuple, H2ClientProtocol] = {} + self._connections: dict[ConnectionKeyT, H2ClientProtocol] = {} # Save all requests that arrive before the connection is established - self._pending_requests: Dict[Tuple, Deque[Deferred]] = {} + self._pending_requests: dict[ + ConnectionKeyT, deque[Deferred[H2ClientProtocol]] + ] = {} + + self._tls_verbose_logging: bool = settings.getbool( + "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" + ) def get_connection( - self, key: Tuple, uri: URI, endpoint: HostnameEndpoint - ) -> Deferred: + self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint + ) -> Deferred[H2ClientProtocol]: if key in self._pending_requests: # Received a request while connecting to remote # Create a deferred which will fire with the H2ClientProtocol # instance - d: Deferred = Deferred() + d: Deferred[H2ClientProtocol] = Deferred() self._pending_requests[key].append(d) return d @@ -54,22 +68,29 @@ class H2ConnectionPool: return self._new_connection(key, uri, endpoint) def _new_connection( - self, key: Tuple, uri: URI, endpoint: HostnameEndpoint - ) -> Deferred: + self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint + ) -> Deferred[H2ClientProtocol]: self._pending_requests[key] = deque() - conn_lost_deferred: Deferred = Deferred() + conn_lost_deferred: Deferred[list[BaseException]] = Deferred() conn_lost_deferred.addCallback(self._remove_connection, key) - factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) + factory = H2ClientFactory( + uri, + self.settings, + conn_lost_deferred, + tls_verbose_logging=self._tls_verbose_logging, + ) conn_d = endpoint.connect(factory) conn_d.addCallback(self.put_connection, key) - d: Deferred = Deferred() + d: Deferred[H2ClientProtocol] = Deferred() self._pending_requests[key].append(d) return d - def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol: + def put_connection( + self, conn: H2ClientProtocol, key: ConnectionKeyT + ) -> H2ClientProtocol: self._connections[key] = conn # Now as we have established a proper HTTP/2 connection @@ -81,7 +102,9 @@ class H2ConnectionPool: return conn - def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None: + def _remove_connection( + self, errors: list[BaseException], key: ConnectionKeyT + ) -> None: self._connections.pop(key) # Call the errback of all the pending requests for this connection @@ -106,30 +129,30 @@ class H2Agent: self, reactor: ReactorBase, pool: H2ConnectionPool, - context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, - bind_address: Optional[bytes] = None, + context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), # noqa: B008 + connect_timeout: float | None = None, + bind_address: tuple[str, int] | None = None, ) -> None: self._reactor = reactor self._pool = pool - self._context_factory = AcceptableProtocolsContextFactory( + self._context_factory = _AcceptableProtocolsContextFactory( context_factory, acceptable_protocols=[b"h2"] ) self.endpoint_factory = _StandardEndpointFactory( self._reactor, self._context_factory, connect_timeout, bind_address ) - def get_endpoint(self, uri: URI): - return self.endpoint_factory.endpointForURI(uri) + def get_endpoint(self, uri: URI) -> HostnameEndpoint: + return self.endpoint_factory.endpointForURI(uri) # type: ignore[no-any-return] - def get_key(self, uri: URI) -> Tuple: + def get_key(self, uri: URI) -> ConnectionKeyT: """ Arguments: uri - URI obtained directly from request URL """ return uri.scheme, uri.host, uri.port - def request(self, request: Request, spider: Spider) -> Deferred: + def request(self, request: Request, spider: Spider) -> Deferred[Response]: uri = URI.fromBytes(bytes(request.url, encoding="utf-8")) try: endpoint = self.get_endpoint(uri) @@ -137,33 +160,8 @@ class H2Agent: return defer.fail(Failure()) key = self.get_key(uri) - d = self._pool.get_connection(key, uri, endpoint) - d.addCallback(lambda conn: conn.request(request, spider)) - return d - - -class ScrapyProxyH2Agent(H2Agent): - def __init__( - self, - reactor: ReactorBase, - proxy_uri: URI, - pool: H2ConnectionPool, - context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, - bind_address: Optional[bytes] = None, - ) -> None: - super().__init__( - reactor=reactor, - pool=pool, - context_factory=context_factory, - connect_timeout=connect_timeout, - bind_address=bind_address, + d: Deferred[H2ClientProtocol] = self._pool.get_connection(key, uri, endpoint) + d2: Deferred[Response] = d.addCallback( + lambda conn: conn.request(request, spider) ) - self._proxy_uri = proxy_uri - - def get_endpoint(self, uri: URI): - return self.endpoint_factory.endpointForURI(self._proxy_uri) - - def get_key(self, uri: URI) -> Tuple: - """We use the proxy uri instead of uri obtained from request url""" - return "http-proxy", self._proxy_uri.host, self._proxy_uri.port + return d2 diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index bc8da50d7..39703f976 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,9 +1,10 @@ +from __future__ import annotations + import ipaddress import itertools import logging from collections import deque -from ipaddress import IPv4Address, IPv6Address -from typing import Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, cast from h2.config import H2Configuration from h2.connection import H2Connection @@ -20,20 +21,32 @@ from h2.events import ( WindowUpdated, ) from h2.exceptions import FrameTooLargeError, H2Error -from twisted.internet.defer import Deferred -from twisted.internet.error import TimeoutError -from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory +from twisted.internet.interfaces import ( + IAddress, + IHandshakeListener, + IProtocolNegotiationFactory, +) from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin -from twisted.python.failure import Failure -from twisted.web.client import URI from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason -from scrapy.http import Request -from scrapy.settings import Settings -from scrapy.spiders import Spider +from scrapy.exceptions import DownloadTimeoutError +from scrapy.http import Request, Response +from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute +from scrapy.utils.ssl import _log_ssl_conn_debug_info + +if TYPE_CHECKING: + from ipaddress import IPv4Address, IPv6Address + + from twisted.internet.defer import Deferred + from twisted.python.failure import Failure + from twisted.web.client import URI + + from scrapy.settings import Settings + from scrapy.spiders import Spider + logger = logging.getLogger(__name__) @@ -52,7 +65,7 @@ class InvalidNegotiatedProtocol(H2Error): class RemoteTerminatedConnection(H2Error): def __init__( self, - remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], + remote_ip_address: IPv4Address | IPv6Address | None, event: ConnectionTerminated, ) -> None: self.remote_ip_address = remote_ip_address @@ -63,9 +76,7 @@ class RemoteTerminatedConnection(H2Error): class MethodNotAllowed405(H2Error): - def __init__( - self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]] - ) -> None: + def __init__(self, remote_ip_address: IPv4Address | IPv6Address | None) -> None: self.remote_ip_address = remote_ip_address def __str__(self) -> str: @@ -77,7 +88,12 @@ class H2ClientProtocol(Protocol, TimeoutMixin): IDLE_TIMEOUT = 240 def __init__( - self, uri: URI, settings: Settings, conn_lost_deferred: Deferred + self, + uri: URI, + settings: Settings, + conn_lost_deferred: Deferred[list[BaseException]], + *, + tls_verbose_logging: bool = False, ) -> None: """ Arguments: @@ -87,8 +103,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin): settings -- Scrapy project settings conn_lost_deferred -- Deferred fires with the reason: Failure to notify that connection was lost + tls_verbose_logging -- Whether to log TLS details """ - self._conn_lost_deferred = conn_lost_deferred + self._conn_lost_deferred: Deferred[list[BaseException]] = conn_lost_deferred + self._tls_verbose_logging: bool = tls_verbose_logging config = H2Configuration(client_side=True, header_encoding="utf-8") self.conn = H2Connection(config=config) @@ -99,19 +117,19 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._stream_id_generator = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs - self.streams: Dict[int, Stream] = {} + self.streams: dict[int, Stream] = {} # If requests are received before connection is made we keep # all requests in a pool and send them as the connection is made - self._pending_request_stream_pool: deque = deque() + self._pending_request_stream_pool: deque[Stream] = deque() # Save an instance of errors raised which lead to losing the connection # We pass these instances to the streams ResponseFailed() failure - self._conn_lost_errors: List[BaseException] = [] + self._conn_lost_errors: list[BaseException] = [] # Some meta data of this connection # initialized when connection is successfully made - self.metadata: Dict = { + self.metadata: dict[str, Any] = { # Peer certificate instance "certificate": None, # Address of the server we are connected to which @@ -179,6 +197,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def _new_stream(self, request: Request, spider: Spider) -> Stream: """Instantiates a new Stream object""" + if hasattr(spider, "download_maxsize"): # pragma: no cover + warn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE") + if hasattr(spider, "download_warnsize"): # pragma: no cover + warn_on_deprecated_spider_attribute( + "download_warnsize", "DOWNLOAD_WARNSIZE" + ) + stream = Stream( stream_id=next(self._stream_id_generator), request=request, @@ -199,19 +224,19 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """ assert self.transport is not None # typing # Reset the idle timeout as connection is still actively sending data - self.resetTimeout() + self.resetTimeout() # type: ignore[no-untyped-call] data = self.conn.data_to_send() self.transport.write(data) - def request(self, request: Request, spider: Spider) -> Deferred: + def request(self, request: Request, spider: Spider) -> Deferred[Response]: if not isinstance(request, Request): raise TypeError( f"Expected scrapy.http.Request, received {request.__class__.__qualname__}" ) stream = self._new_stream(request, spider) - d = stream.get_response() + d: Deferred[Response] = stream.get_response() # Add the stream to the request pool self._pending_request_stream_pool.append(stream) @@ -226,7 +251,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): sending some data now: we should open with the connection preamble. """ # Initialize the timeout - self.setTimeout(self.IDLE_TIMEOUT) + self.setTimeout(self.IDLE_TIMEOUT) # type: ignore[no-untyped-call] assert self.transport is not None # typing destination = self.transport.getPeer() @@ -236,10 +261,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.conn.initiate_connection() self._write_to_transport() - def _lose_connection_with_error(self, errors: List[BaseException]) -> None: + def _lose_connection_with_error(self, errors: list[BaseException]) -> None: """Helper function to lose the connection with the error sent as a reason""" - self._conn_lost_errors += errors + self._conn_lost_errors.extend(errors) assert self.transport is not None # typing self.transport.loseConnection() @@ -257,6 +282,11 @@ class H2ClientProtocol(Protocol, TimeoutMixin): [InvalidNegotiatedProtocol(self.transport.negotiatedProtocol)] ) + if self._tls_verbose_logging: + connection = self.transport.getHandle() + hostname = self.metadata["uri"].host.decode("ascii") + _log_ssl_conn_debug_info(hostname, connection) + def _check_received_data(self, data: bytes) -> None: """Checks for edge cases where the connection to remote fails without raising an appropriate H2Error @@ -269,7 +299,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def dataReceived(self, data: bytes) -> None: # Reset the idle timeout as connection is still actively receiving data - self.resetTimeout() + self.resetTimeout() # type: ignore[no-untyped-call] try: self._check_received_data(data) @@ -279,7 +309,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): if isinstance(e, FrameTooLargeError): # hyper-h2 does not drop the connection in this scenario, we # need to abort the connection manually. - self._conn_lost_errors += [e] + self._conn_lost_errors.append(e) assert self.transport is not None # typing self.transport.abortConnection() return @@ -293,7 +323,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def timeoutConnection(self) -> None: """Called when the connection times out. - We lose the connection with TimeoutError""" + We lose the connection with DownloadTimeoutError""" # Check whether there are open streams. If there are, we're going to # want to use the error code PROTOCOL_ERROR. If there aren't, use @@ -310,7 +340,11 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._write_to_transport() self._lose_connection_with_error( - [TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")] + [ + DownloadTimeoutError( + f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s" + ) + ] ) def connectionLost(self, reason: Failure = connectionDone) -> None: @@ -318,7 +352,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): No need to write anything to transport here. """ # Cancel the timeout if not done yet - self.setTimeout(None) + self.setTimeout(None) # type: ignore[no-untyped-call] # Notify the connection pool instance such that no new requests are # sent over current connection @@ -339,7 +373,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._pending_request_stream_pool.clear() self.conn.close_connection() - def _handle_events(self, events: List[Event]) -> None: + def _handle_events(self, events: list[Event]) -> None: """Private method which acts as a bridge between the events received from the HTTP/2 data and IH2EventsHandler @@ -384,7 +418,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): except KeyError: pass # We ignore server-initiated events else: - stream.receive_headers(event.headers) + stream.receive_headers(cast("list[tuple[str, str]]", event.headers)) def settings_acknowledged(self, event: SettingsAcknowledged) -> None: self.metadata["settings_acknowledged"] = True @@ -425,14 +459,25 @@ class H2ClientProtocol(Protocol, TimeoutMixin): @implementer(IProtocolNegotiationFactory) class H2ClientFactory(Factory): def __init__( - self, uri: URI, settings: Settings, conn_lost_deferred: Deferred + self, + uri: URI, + settings: Settings, + conn_lost_deferred: Deferred[list[BaseException]], + *, + tls_verbose_logging: bool = False, ) -> None: self.uri = uri self.settings = settings self.conn_lost_deferred = conn_lost_deferred + self.tls_verbose_logging = tls_verbose_logging - def buildProtocol(self, addr) -> H2ClientProtocol: - return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred) + def buildProtocol(self, addr: IAddress) -> H2ClientProtocol: + return H2ClientProtocol( + self.uri, + self.settings, + self.conn_lost_deferred, + tls_verbose_logging=self.tls_verbose_logging, + ) - def acceptableProtocols(self) -> List[bytes]: + def acceptableProtocols(self) -> list[bytes]: return [PROTOCOL_NAME] diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 50d853aba..4d072c555 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,23 +1,31 @@ +from __future__ import annotations + import logging from enum import Enum from io import BytesIO -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple -from urllib.parse import urlparse +from typing import TYPE_CHECKING, Any from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError -from hpack import HeaderTuple -from twisted.internet.defer import CancelledError, Deferred +from twisted.internet.defer import Deferred from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -from scrapy.http import Request +from scrapy.exceptions import DownloadCancelledError from scrapy.http.headers import Headers -from scrapy.utils.response import get_response_class +from scrapy.utils._download_handlers import ( + get_maxsize_msg, + get_warnsize_msg, + make_response, +) +from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: + from collections.abc import Sequence + from scrapy.core.http2.protocol import H2ClientProtocol + from scrapy.http import Request, Response logger = logging.getLogger(__name__) @@ -71,6 +79,9 @@ class StreamCloseReason(Enum): # As a result sending this request will the end the connection INVALID_HOSTNAME = 7 + # Actual response body size is more than allowed limit + MAXSIZE_EXCEEDED_ACTUAL = 8 + class Stream: """Represents a single HTTP/2 Stream. @@ -87,7 +98,7 @@ class Stream: self, stream_id: int, request: Request, - protocol: "H2ClientProtocol", + protocol: H2ClientProtocol, download_maxsize: int = 0, download_warnsize: int = 0, ) -> None: @@ -99,7 +110,7 @@ class Stream: """ self.stream_id: int = stream_id self._request: Request = request - self._protocol: "H2ClientProtocol" = protocol + self._protocol: H2ClientProtocol = protocol self._download_maxsize = self._request.meta.get( "download_maxsize", download_maxsize @@ -110,18 +121,18 @@ class Stream: # Metadata of an HTTP/2 connection stream # initialized when stream is instantiated - self.metadata: Dict = { - "request_content_length": 0 - if self._request.body is None - else len(self._request.body), + self.metadata: dict[str, Any] = { + "request_content_length": ( + 0 if self._request.body is None else len(self._request.body) + ), # Flag to keep track whether the stream has initiated the request "request_sent": False, # Flag to track whether we have logged about exceeding download warnsize "reached_warnsize": False, # Each time we send a data frame, we will decrease value by the amount send. - "remaining_content_length": 0 - if self._request.body is None - else len(self._request.body), + "remaining_content_length": ( + 0 if self._request.body is None else len(self._request.body) + ), # Flag to keep track whether client (self) have closed this stream "stream_closed_local": False, # Flag to keep track whether the server has closed the stream @@ -131,7 +142,7 @@ class Stream: # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response: Dict = { + self._response: dict[str, Any] = { # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. "body": BytesIO(), @@ -139,19 +150,21 @@ class Stream: # flow control window "flow_controlled_size": 0, # Headers received after sending the request - "headers": Headers({}), + "headers": Headers(), + # Response status code + "status": None, } - def _cancel(_) -> None: - # Close this stream as gracefully as possible - # If the associated request is initiated we reset this stream - # else we directly call close() method - if self.metadata["request_sent"]: - self.reset_stream(StreamCloseReason.CANCELLED) - else: - self.close(StreamCloseReason.CANCELLED) + self._deferred_response: Deferred[Response] = Deferred(self._cancel) - self._deferred_response: Deferred = Deferred(_cancel) + def _cancel(self, _: Any) -> None: + # Close this stream as gracefully as possible + # If the associated request is initiated we reset this stream + # else we directly call close() method + if self.metadata["request_sent"]: + self.reset_stream(StreamCloseReason.CANCELLED) + else: + self.close(StreamCloseReason.CANCELLED) def __repr__(self) -> str: return f"Stream(id={self.stream_id!r})" @@ -177,7 +190,7 @@ class Stream: and not self.metadata["reached_warnsize"] ) - def get_response(self) -> Deferred: + def get_response(self) -> Deferred[Response]: """Simply return a Deferred which fires when response from the asynchronous request is available """ @@ -185,16 +198,16 @@ class Stream: def check_request_url(self) -> bool: # Make sure that we are sending the request to the correct URL - url = urlparse(self._request.url) + url = urlparse_cached(self._request) return ( url.netloc == str(self._protocol.metadata["uri"].host, "utf-8") or url.netloc == str(self._protocol.metadata["uri"].netloc, "utf-8") or url.netloc - == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' + == f"{self._protocol.metadata['ip_address']}:{self._protocol.metadata['uri'].port}" ) - def _get_request_headers(self) -> List[Tuple[str, str]]: - url = urlparse(self._request.url) + def _get_request_headers(self) -> list[tuple[str, str]]: + url = urlparse_cached(self._request) path = url.path if url.query: @@ -330,14 +343,16 @@ class Stream: self._download_maxsize and self._response["flow_controlled_size"] > self._download_maxsize ): - self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) + self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED_ACTUAL) return if self._log_warnsize: self.metadata["reached_warnsize"] = True - warning_msg = ( - f'Received more ({self._response["flow_controlled_size"]}) bytes than download ' - f"warn size ({self._download_warnsize}) in request {self._request}" + warning_msg = get_warnsize_msg( + self._response["flow_controlled_size"], + self._download_warnsize, + self._request, + expected=False, ) logger.warning(warning_msg) @@ -346,9 +361,13 @@ class Stream: self._response["flow_controlled_size"], self.stream_id ) - def receive_headers(self, headers: List[HeaderTuple]) -> None: + def receive_headers(self, headers: list[tuple[str, str]]) -> None: for name, value in headers: - self._response["headers"].appendlist(name, value) + if name == ":status": + # it's a pseudo-header + self._response["status"] = int(value) + else: + self._response["headers"].appendlist(name, value) # Check if we exceed the allowed max data size which can be received expected_size = int(self._response["headers"].get(b"Content-Length", -1)) @@ -358,9 +377,8 @@ class Stream: if self._log_warnsize: self.metadata["reached_warnsize"] = True - warning_msg = ( - f"Expected response size ({expected_size}) larger than " - f"download warn size ({self._download_warnsize}) in request {self._request}" + warning_msg = get_warnsize_msg( + expected_size, self._download_warnsize, self._request, expected=True ) logger.warning(warning_msg) @@ -379,7 +397,7 @@ class Stream: def close( self, reason: StreamCloseReason, - errors: Optional[List[BaseException]] = None, + errors: Sequence[BaseException] | None = None, from_protocol: bool = False, ) -> None: """Based on the reason sent we will handle each case.""" @@ -393,7 +411,7 @@ class Stream: # Have default value of errors as an empty list as # some cases can add a list of exceptions - errors = errors or [] + errors = errors or () if not from_protocol: self._protocol.pop_stream(self.stream_id) @@ -408,18 +426,23 @@ class Stream: # As we immediately cancel the request when maxsize is exceeded while # receiving DATA_FRAME's when we have received the headers (not # having Content-Length) - if reason is StreamCloseReason.MAXSIZE_EXCEEDED: + if reason in { + StreamCloseReason.MAXSIZE_EXCEEDED, + StreamCloseReason.MAXSIZE_EXCEEDED_ACTUAL, + }: expected_size = int( self._response["headers"].get( b"Content-Length", self._response["flow_controlled_size"] ) ) - error_msg = ( - f"Cancelling download of {self._request.url}: received response " - f"size ({expected_size}) larger than download max size ({self._download_maxsize})" + error_msg = get_maxsize_msg( + expected_size, + self._download_maxsize, + self._request, + expected=reason == StreamCloseReason.MAXSIZE_EXCEEDED, ) logger.error(error_msg) - self._deferred_response.errback(CancelledError(error_msg)) + self._deferred_response.errback(DownloadCancelledError(error_msg)) elif reason is StreamCloseReason.ENDED: self._fire_response_deferred() @@ -434,7 +457,8 @@ class Stream: # There maybe no :status in headers, we make # HTTP Status Code: 499 - Client Closed Request - self._response["headers"][":status"] = "499" + if self._response["status"] is None: + self._response["status"] = 499 self._fire_response_deferred() elif reason is StreamCloseReason.RESET: @@ -442,7 +466,7 @@ class Stream: ResponseFailed( [ Failure( - f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM', + f"Remote peer {self._protocol.metadata['ip_address']} sent RST_STREAM", ProtocolError, ) ] @@ -453,7 +477,7 @@ class Stream: self._deferred_response.errback(ResponseFailed(errors)) elif reason is StreamCloseReason.INACTIVE: - errors.insert(0, InactiveStreamClosed(self._request)) + errors = (InactiveStreamClosed(self._request), *errors) self._deferred_response.errback(ResponseFailed(errors)) else: @@ -462,7 +486,7 @@ class Stream: InvalidHostname( self._request, str(self._protocol.metadata["uri"].host, "utf-8"), - f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}', + f"{self._protocol.metadata['ip_address']}:{self._protocol.metadata['uri'].port}", ) ) @@ -471,22 +495,13 @@ class Stream: and fires the response deferred callback with the generated response instance""" - body = self._response["body"].getvalue() - response_cls = get_response_class( - http_headers=self._response["headers"], + response = make_response( url=self._request.url, - body=body, - ) - - response = response_cls( - url=self._request.url, - status=int(self._response["headers"][":status"]), + status=self._response["status"], headers=self._response["headers"], - body=body, - request=self._request, + body=self._response["body"].getvalue(), certificate=self._protocol.metadata["certificate"], ip_address=self._protocol.metadata["ip_address"], protocol="h2", ) - self._deferred_response.callback(response) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index f41b83a67..7217da942 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,22 +4,28 @@ import json import logging from abc import abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, cast +from typing import TYPE_CHECKING, Any, cast -from twisted.internet.defer import Deferred +# working around https://github.com/sphinx-doc/sphinx/issues/10400 +from twisted.internet.defer import Deferred # noqa: TC002 -from scrapy.crawler import Crawler -from scrapy.dupefilters import BaseDupeFilter -from scrapy.http.request import Request -from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector +from scrapy.spiders import Spider # noqa: TC001 from scrapy.utils.job import job_dir from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: + # requires queuelib >= 1.6.2 + from queuelib.queue import BaseQueue + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.dupefilters import BaseDupeFilter + from scrapy.http.request import Request + from scrapy.pqueues import ScrapyPriorityQueue + from scrapy.statscollectors import StatsCollector + logger = logging.getLogger(__name__) @@ -44,18 +50,17 @@ class BaseSchedulerMeta(type): class BaseScheduler(metaclass=BaseSchedulerMeta): - """ - The scheduler component is responsible for storing requests received from - the engine, and feeding them back upon request (also to the engine). + """The scheduler component is responsible for storing requests received + from the engine, and feeding them back upon request (also to the engine). The original sources of said requests are: - * Spider: ``start_requests`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks + * Spider: ``start`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks * Spider middleware: ``process_spider_output`` and ``process_spider_exception`` methods * Downloader middleware: ``process_request``, ``process_response`` and ``process_exception`` methods The order in which the scheduler returns its stored requests (via the ``next_request`` method) - plays a great part in determining the order in which those requests are downloaded. + plays a great part in determining the order in which those requests are downloaded. See :ref:`request-order`. The methods defined in this class constitute the minimal interface that the Scrapy engine will interact with. """ @@ -67,7 +72,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): """ return cls() - def open(self, spider: Spider) -> Optional[Deferred]: + def open(self, spider: Spider) -> Deferred[None] | None: """ Called when the spider is opened by the engine. It receives the spider instance as argument and it's useful to execute initialization code. @@ -75,9 +80,8 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): :param spider: the spider object for the current crawl :type spider: :class:`~scrapy.spiders.Spider` """ - pass - def close(self, reason: str) -> Optional[Deferred]: + def close(self, reason: str) -> Deferred[None] | None: """ Called when the spider is closed by the engine. It receives the reason why the crawl finished as argument and it's useful to execute cleaning code. @@ -85,14 +89,13 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): :param reason: a string which describes the reason why the spider was closed :type reason: :class:`str` """ - pass @abstractmethod def has_pending_requests(self) -> bool: """ ``True`` if the scheduler has enqueued requests, ``False`` otherwise """ - raise NotImplementedError() + raise NotImplementedError @abstractmethod def enqueue_request(self, request: Request) -> bool: @@ -106,100 +109,149 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): For reference, the default Scrapy scheduler returns ``False`` when the request is rejected by the dupefilter. """ - raise NotImplementedError() + raise NotImplementedError @abstractmethod - def next_request(self) -> Optional[Request]: + def next_request(self) -> Request | None: """ - Return the next :class:`~scrapy.http.Request` to be processed, or ``None`` + Return the next :class:`~scrapy.Request` to be processed, or ``None`` to indicate that there are no requests to be considered ready at the moment. Returning ``None`` implies that no request from the scheduler will be sent to the downloader in the current reactor cycle. The engine will continue calling ``next_request`` until ``has_pending_requests`` is ``False``. """ - raise NotImplementedError() - - -SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler") + raise NotImplementedError class Scheduler(BaseScheduler): + r"""Default scheduler. + + Requests are stored into priority queues + (:setting:`SCHEDULER_PRIORITY_QUEUE`) that sort requests by + :attr:`~scrapy.http.Request.priority`. + + By default, a single, memory-based priority queue is used for all requests. + When using :setting:`JOBDIR`, a disk-based priority queue is also created, + and only unserializable requests are stored in the memory-based priority + queue. For a given priority value, requests in memory take precedence over + requests in disk. + + Each priority queue stores requests in separate internal queues, one per + priority value. The memory priority queue uses + :setting:`SCHEDULER_MEMORY_QUEUE` queues, while the disk priority queue + uses :setting:`SCHEDULER_DISK_QUEUE` queues. The internal queues determine + :ref:`request order ` when requests have the same priority. + :ref:`Start requests ` are stored into separate internal + queues by default, and :ref:`ordered differently `. + + Duplicate requests are filtered out with an instance of + :setting:`DUPEFILTER_CLASS`. + + .. _request-order: + + Request order + ============= + + With default settings, pending requests are stored in a LIFO_ queue + (:ref:`except for start requests `). As a result, + crawling happens in `DFO order`_, which is usually the most convenient + crawl order. However, you can enforce :ref:`BFO ` or :ref:`a custom + order ` (:ref:`except for the first few requests + `). + + .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) + .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search + + .. _start-request-order: + + Start request order + ------------------- + + :ref:`Start requests ` are sent in the order they are + yielded from :meth:`~scrapy.Spider.start`, and given the same + :attr:`~scrapy.http.Request.priority`, other requests take precedence over + start requests. + + You can set :setting:`SCHEDULER_START_MEMORY_QUEUE` and + :setting:`SCHEDULER_START_DISK_QUEUE` to ``None`` to handle start requests + the same as other requests when it comes to order and priority. + + + .. _bfo: + + Crawling in BFO order + --------------------- + + If you do want to crawl in `BFO order`_, you can do it by setting the + following :ref:`settings `: + + | :setting:`DEPTH_PRIORITY` = ``1`` + | :setting:`SCHEDULER_DISK_QUEUE` = + ``"scrapy.squeues.PickleFifoDiskQueue"`` + | :setting:`SCHEDULER_MEMORY_QUEUE` = ``"scrapy.squeues.FifoMemoryQueue"`` + + .. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search + + + .. _custom-request-order: + + Crawling in a custom order + -------------------------- + + You can manually set :attr:`~scrapy.http.Request.priority` on requests to + force a specific request order. + + + .. _concurrency-v-order: + + Concurrency affects order + ------------------------- + + While pending requests are below the configured values of + :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` + or :setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent + concurrently. + + As a result, the first few requests of a crawl may not follow the desired + order. Lowering those settings to ``1`` enforces the desired order except + for the very first request, but it significantly slows down the crawl as a + whole. + + Job directory contents + ====================== + + .. warning:: The files that this class generates in the :ref:`job directory + ` are an implementation detail, and may change without a + warning in a future version of Scrapy. Do not rely on the following + information for anything other than debugging purposes. + + When using :setting:`JOBDIR`, this scheduler class: + + - Creates a directory named ``requests.queue`` inside the :ref:`job + directory `, meant to keep track of all requests stored in + the scheduler (i.e. not downloaded yet). + + - Generates inside that directory an ``active.json`` file with a JSON + representation of the state (``startprios``) of + :setting:`SCHEDULER_PRIORITY_QUEUE`. + + The file is generated whenever the job stops (cleanly) and is loaded + when resuming the job. + + - Instantiates the configured :setting:`SCHEDULER_PRIORITY_QUEUE` with + ``requests.queue/`` as persistence directory (*key*) and + :setting:`SCHEDULER_DISK_QUEUE` as *downstream_queue_cls*. The priority + queue may create additional files and directories inside that + directory, directly or though instances of + :setting:`SCHEDULER_DISK_QUEUE`. + + This scheduler class also uses the configured :setting:`DUPEFILTER_CLASS`, + which may also write data inside the job directory. """ - Default Scrapy scheduler. This implementation also handles duplication - filtering via the :setting:`dupefilter `. - - This scheduler stores requests into several priority queues (defined by the - :setting:`SCHEDULER_PRIORITY_QUEUE` setting). In turn, said priority queues - are backed by either memory or disk based queues (respectively defined by the - :setting:`SCHEDULER_MEMORY_QUEUE` and :setting:`SCHEDULER_DISK_QUEUE` settings). - - Request prioritization is almost entirely delegated to the priority queue. The only - prioritization performed by this scheduler is using the disk-based queue if present - (i.e. if the :setting:`JOBDIR` setting is defined) and falling back to the memory-based - queue if a serialization error occurs. If the disk queue is not present, the memory one - is used directly. - - :param dupefilter: An object responsible for checking and filtering duplicate requests. - The value for the :setting:`DUPEFILTER_CLASS` setting is used by default. - :type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar: - any class that implements the `BaseDupeFilter` interface - - :param jobdir: The path of a directory to be used for persisting the crawl's state. - The value for the :setting:`JOBDIR` setting is used by default. - See :ref:`topics-jobs`. - :type jobdir: :class:`str` or ``None`` - - :param dqclass: A class to be used as persistent request queue. - The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default. - :type dqclass: class - - :param mqclass: A class to be used as non-persistent request queue. - The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default. - :type mqclass: class - - :param logunser: A boolean that indicates whether or not unserializable requests should be logged. - The value for the :setting:`SCHEDULER_DEBUG` setting is used by default. - :type logunser: bool - - :param stats: A stats collector object to record stats about the request scheduling process. - The value for the :setting:`STATS_CLASS` setting is used by default. - :type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar: - any class that implements the `StatsCollector` interface - - :param pqclass: A class to be used as priority queue for requests. - The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default. - :type pqclass: class - - :param crawler: The crawler object corresponding to the current crawl. - :type crawler: :class:`scrapy.crawler.Crawler` - """ - - def __init__( - self, - dupefilter: BaseDupeFilter, - jobdir: Optional[str] = None, - dqclass=None, - mqclass=None, - logunser: bool = False, - stats: Optional[StatsCollector] = None, - pqclass=None, - crawler: Optional[Crawler] = None, - ): - self.df: BaseDupeFilter = dupefilter - self.dqdir: Optional[str] = self._dqdir(jobdir) - self.pqclass = pqclass - self.dqclass = dqclass - self.mqclass = mqclass - self.logunser: bool = logunser - self.stats: Optional[StatsCollector] = stats - self.crawler: Optional[Crawler] = crawler @classmethod - def from_crawler(cls: Type[SchedulerTV], crawler: Crawler) -> SchedulerTV: - """ - Factory method, initializes the scheduler with arguments taken from the crawl settings - """ + def from_crawler(cls, crawler: Crawler) -> Self: dupefilter_cls = load_object(crawler.settings["DUPEFILTER_CLASS"]) return cls( dupefilter=build_from_crawler(dupefilter_cls, crawler), @@ -212,21 +264,93 @@ class Scheduler(BaseScheduler): crawler=crawler, ) + def __init__( + self, + dupefilter: BaseDupeFilter, + jobdir: str | None = None, + dqclass: type[BaseQueue] | None = None, + mqclass: type[BaseQueue] | None = None, + logunser: bool = False, + stats: StatsCollector | None = None, + pqclass: type[ScrapyPriorityQueue] | None = None, + crawler: Crawler | None = None, + ): + """Initialize the scheduler. + + :param dupefilter: An object responsible for checking and filtering duplicate requests. + The value for the :setting:`DUPEFILTER_CLASS` setting is used by default. + :type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar: + any class that implements the `BaseDupeFilter` interface + + :param jobdir: The path of a directory to be used for persisting the crawl's state. + The value for the :setting:`JOBDIR` setting is used by default. + See :ref:`topics-jobs`. + :type jobdir: :class:`str` or ``None`` + + :param dqclass: A class to be used as persistent request queue. + The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default. + :type dqclass: class + + :param mqclass: A class to be used as non-persistent request queue. + The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default. + :type mqclass: class + + :param logunser: A boolean that indicates whether or not unserializable requests should be logged. + The value for the :setting:`SCHEDULER_DEBUG` setting is used by default. + :type logunser: bool + + :param stats: A stats collector object to record stats about the request scheduling process. + The value for the :setting:`STATS_CLASS` setting is used by default. + :type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar: + any class that implements the `StatsCollector` interface + + :param pqclass: A class to be used as priority queue for requests. + The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default. + :type pqclass: class + + :param crawler: The crawler object corresponding to the current crawl. + :type crawler: :class:`scrapy.crawler.Crawler` + """ + self.df: BaseDupeFilter = dupefilter + self.dqdir: str | None = self._dqdir(jobdir) + self.pqclass: type[ScrapyPriorityQueue] | None = pqclass + self.dqclass: type[BaseQueue] | None = dqclass + self.mqclass: type[BaseQueue] | None = mqclass + self.logunser: bool = logunser + self.stats: StatsCollector | None = stats + self.crawler: Crawler | None = crawler + self._sdqclass: type[BaseQueue] | None = self._get_start_queue_cls( + crawler, "DISK" + ) + self._smqclass: type[BaseQueue] | None = self._get_start_queue_cls( + crawler, "MEMORY" + ) + + def _get_start_queue_cls( + self, crawler: Crawler | None, queue: str + ) -> type[BaseQueue] | None: + if crawler is None: + return None + cls = crawler.settings[f"SCHEDULER_START_{queue}_QUEUE"] + if not cls: + return None + return cast("type[BaseQueue]", load_object(cls)) + def has_pending_requests(self) -> bool: return len(self) > 0 - def open(self, spider: Spider) -> Optional[Deferred]: + def open(self, spider: Spider) -> Deferred[None] | None: """ (1) initialize the memory queue (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory (3) return the result of the dupefilter's ``open`` method """ - self.spider = spider - self.mqs = self._mq() - self.dqs = self._dq() if self.dqdir else None + self.spider: Spider = spider + self.mqs: ScrapyPriorityQueue = self._mq() + self.dqs: ScrapyPriorityQueue | None = self._dq() if self.dqdir else None return self.df.open() - def close(self, reason: str) -> Optional[Deferred]: + def close(self, reason: str) -> Deferred[None] | None: """ (1) dump pending requests to disk if there is a disk queue (2) return the result of the dupefilter's ``close`` method @@ -253,32 +377,32 @@ class Scheduler(BaseScheduler): dqok = self._dqpush(request) assert self.stats is not None if dqok: - self.stats.inc_value("scheduler/enqueued/disk", spider=self.spider) + self.stats.inc_value("scheduler/enqueued/disk") else: self._mqpush(request) - self.stats.inc_value("scheduler/enqueued/memory", spider=self.spider) - self.stats.inc_value("scheduler/enqueued", spider=self.spider) + self.stats.inc_value("scheduler/enqueued/memory") + self.stats.inc_value("scheduler/enqueued") return True - def next_request(self) -> Optional[Request]: + def next_request(self) -> Request | None: """ - Return a :class:`~scrapy.http.Request` object from the memory queue, + Return a :class:`~scrapy.Request` object from the memory queue, falling back to the disk queue if the memory queue is empty. Return ``None`` if there are no more enqueued requests. Increment the appropriate stats, such as: ``scheduler/dequeued``, ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``. """ - request: Optional[Request] = self.mqs.pop() + request: Request | None = self.mqs.pop() assert self.stats is not None if request is not None: - self.stats.inc_value("scheduler/dequeued/memory", spider=self.spider) + self.stats.inc_value("scheduler/dequeued/memory") else: request = self._dqpop() if request is not None: - self.stats.inc_value("scheduler/dequeued/disk", spider=self.spider) + self.stats.inc_value("scheduler/dequeued/disk") if request is not None: - self.stats.inc_value("scheduler/dequeued", spider=self.spider) + self.stats.inc_value("scheduler/dequeued") return request def __len__(self) -> int: @@ -307,31 +431,35 @@ class Scheduler(BaseScheduler): ) self.logunser = False assert self.stats is not None - self.stats.inc_value("scheduler/unserializable", spider=self.spider) + self.stats.inc_value("scheduler/unserializable") return False - else: - return True + return True def _mqpush(self, request: Request) -> None: self.mqs.push(request) - def _dqpop(self) -> Optional[Request]: + def _dqpop(self) -> Request | None: if self.dqs is not None: return self.dqs.pop() return None - def _mq(self): + def _mq(self) -> ScrapyPriorityQueue: """Create a new priority queue instance, with in-memory storage""" + assert self.crawler + assert self.pqclass return build_from_crawler( self.pqclass, self.crawler, downstream_queue_cls=self.mqclass, key="", + start_queue_cls=self._smqclass, ) - def _dq(self): + def _dq(self) -> ScrapyPriorityQueue: """Create a new priority queue instance, with disk storage""" + assert self.crawler assert self.dqdir + assert self.pqclass state = self._read_dqs_state(self.dqdir) q = build_from_crawler( self.pqclass, @@ -339,6 +467,7 @@ class Scheduler(BaseScheduler): downstream_queue_cls=self.dqclass, key=self.dqdir, startprios=state, + start_queue_cls=self._sdqclass, ) if q: logger.info( @@ -348,7 +477,7 @@ class Scheduler(BaseScheduler): ) return q - def _dqdir(self, jobdir: Optional[str]) -> Optional[str]: + def _dqdir(self, jobdir: str | None) -> str | None: """Return a folder name to keep disk queue state at""" if jobdir: dqdir = Path(jobdir, "requests.queue") @@ -357,13 +486,13 @@ class Scheduler(BaseScheduler): return str(dqdir) return None - def _read_dqs_state(self, dqdir: str) -> list: + def _read_dqs_state(self, dqdir: str) -> Any: path = Path(dqdir, "active.json") if not path.exists(): - return [] + return () with path.open(encoding="utf-8") as f: - return cast(list, json.load(f)) + return json.load(f) - def _write_dqs_state(self, dqdir: str, state: list) -> None: + def _write_dqs_state(self, dqdir: str, state: Any) -> None: with Path(dqdir, "active.json").open("w", encoding="utf-8") as f: json.dump(state, f) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 8fb16b8a9..e756d27eb 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -1,74 +1,78 @@ """This module implements the Scraper component which parses responses and extracts information from them""" + from __future__ import annotations import logging +import warnings from collections import deque -from typing import ( - TYPE_CHECKING, - Any, - AsyncGenerator, - AsyncIterable, - Deque, - Generator, - Iterable, - Optional, - Set, - Tuple, - Type, - Union, -) +from collections.abc import AsyncIterator +from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar -from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from scrapy import Spider, signals from scrapy.core.spidermw import SpiderMiddlewareManager -from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest +from scrapy.exceptions import ( + CloseSpider, + DropItem, + IgnoreRequest, + ScrapyDeprecationWarning, +) from scrapy.http import Request, Response -from scrapy.logformatter import LogFormatter from scrapy.pipelines import ItemPipelineManager -from scrapy.signalmanager import SignalManager +from scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available from scrapy.utils.defer import ( + _defer_sleep_async, + _schedule_coro, aiter_errback, - defer_fail, - defer_succeed, + deferred_from_coro, + ensure_awaitable, iter_errback, + maybe_deferred_to_future, parallel, parallel_async, ) +from scrapy.utils.deprecate import method_is_overridden from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import load_object, warn_on_generator_with_return_value +from scrapy.utils.python import global_object_name from scrapy.utils.spider import iterate_spider_output if TYPE_CHECKING: + from collections.abc import Generator, Iterable + from scrapy.crawler import Crawler - - -QueueTuple = Tuple[Union[Response, Failure], Request, Deferred] + from scrapy.logformatter import LogFormatter + from scrapy.signalmanager import SignalManager logger = logging.getLogger(__name__) +_T = TypeVar("_T") +QueueTuple: TypeAlias = tuple[Response | Failure, Request, Deferred[None]] + + class Slot: """Scraper slot (one per running spider)""" MIN_RESPONSE_SIZE = 1024 def __init__(self, max_active_size: int = 5000000): - self.max_active_size = max_active_size - self.queue: Deque[QueueTuple] = deque() - self.active: Set[Request] = set() + self.max_active_size: int = max_active_size + self.queue: deque[QueueTuple] = deque() + self.active: set[Request] = set() self.active_size: int = 0 - self.itemproc_size: int = 0 - self.closing: Optional[Deferred] = None + self.itemproc_size: int = 0 # just for scrapy.utils.engine.get_engine_status() + self.closing: Deferred[Spider] | None = None def add_response_request( - self, result: Union[Response, Failure], request: Request - ) -> Deferred: - deferred: Deferred = Deferred() + self, result: Response | Failure, request: Request + ) -> Deferred[None]: + # this Deferred will be awaited in enqueue_scrape() + deferred: Deferred[None] = Deferred() self.queue.append((result, request, deferred)) if isinstance(result, Response): self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE) @@ -77,13 +81,11 @@ class Slot: return deferred def next_response_request_deferred(self) -> QueueTuple: - response, request, deferred = self.queue.popleft() + result, request, deferred = self.queue.popleft() self.active.add(request) - return response, request, deferred + return result, request, deferred - def finish_response( - self, result: Union[Response, Failure], request: Request - ) -> None: + def finish_response(self, result: Response | Failure, request: Request) -> None: self.active.remove(request) if isinstance(result, Response): self.active_size -= max(len(result.body), self.MIN_RESPONSE_SIZE) @@ -99,295 +101,442 @@ class Slot: class Scraper: def __init__(self, crawler: Crawler) -> None: - self.slot: Optional[Slot] = None + self.slot: Slot | None = None self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler( crawler ) - itemproc_cls: Type[ItemPipelineManager] = load_object( + itemproc_cls: type[ItemPipelineManager] = load_object( crawler.settings["ITEM_PROCESSOR"] ) self.itemproc: ItemPipelineManager = itemproc_cls.from_crawler(crawler) + self._itemproc_has_async: dict[str, bool] = {} + for method in [ + "open_spider", + "close_spider", + "process_item", + ]: + self._check_deprecated_itemproc_method(method) + self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS") self.crawler: Crawler = crawler self.signals: SignalManager = crawler.signals assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter - @inlineCallbacks - def open_spider(self, spider: Spider) -> Generator[Deferred, Any, None]: - """Open the given spider for scraping and allocate resources for it""" - self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE")) - yield self.itemproc.open_spider(spider) + def _check_deprecated_itemproc_method(self, method: str) -> None: + itemproc_cls = type(self.itemproc) + if not hasattr(self.itemproc, "process_item_async"): + warnings.warn( + f"{global_object_name(itemproc_cls)} doesn't define a {method}_async() method," + f" this is deprecated and the method will be required in future Scrapy versions.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + self._itemproc_has_async[method] = False + elif ( + issubclass(itemproc_cls, ItemPipelineManager) + and method_is_overridden(itemproc_cls, ItemPipelineManager, method) + and not method_is_overridden( + itemproc_cls, ItemPipelineManager, f"{method}_async" + ) + ): + warnings.warn( + f"{global_object_name(itemproc_cls)} overrides {method}() but doesn't override {method}_async()." + f" This is deprecated. {method}() will be used, but in future Scrapy versions {method}_async() will be used instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + self._itemproc_has_async[method] = False + else: + self._itemproc_has_async[method] = True - def close_spider(self, spider: Spider) -> Deferred: - """Close a spider being scraped and release its resources""" + def open_spider( + self, spider: Spider | None = None + ) -> Deferred[None]: # pragma: no cover + warnings.warn( + "Scraper.open_spider() is deprecated, use open_spider_async() instead", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return deferred_from_coro(self.open_spider_async()) + + async def open_spider_async(self) -> None: + """Open the spider for scraping and allocate resources for it. + + .. versionadded:: 2.14 + """ + self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE")) + if not self.crawler.spider: + raise RuntimeError( + "Scraper.open_spider() called before Crawler.spider is set." + ) + if self._itemproc_has_async["open_spider"]: + await self.itemproc.open_spider_async() + else: + await maybe_deferred_to_future( + self.itemproc.open_spider(self.crawler.spider) + ) + + def close_spider(self) -> Deferred[None]: # pragma: no cover + warnings.warn( + "Scraper.close_spider() is deprecated, use close_spider_async() instead", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return deferred_from_coro(self.close_spider_async()) + + async def close_spider_async(self) -> None: + """Close the spider being scraped and release its resources. + + .. versionadded:: 2.14 + """ if self.slot is None: raise RuntimeError("Scraper slot not assigned") self.slot.closing = Deferred() - self.slot.closing.addCallback(self.itemproc.close_spider) - self._check_if_closing(spider) - return self.slot.closing + self._check_if_closing() + await maybe_deferred_to_future(self.slot.closing) + if self._itemproc_has_async["close_spider"]: + await self.itemproc.close_spider_async() + else: + assert self.crawler.spider + await maybe_deferred_to_future( + self.itemproc.close_spider(self.crawler.spider) + ) def is_idle(self) -> bool: """Return True if there isn't any more spiders to process""" return not self.slot - def _check_if_closing(self, spider: Spider) -> None: + def _check_if_closing(self) -> None: assert self.slot is not None # typing if self.slot.closing and self.slot.is_idle(): - self.slot.closing.callback(spider) + assert self.crawler.spider + self.slot.closing.callback(self.crawler.spider) + @inlineCallbacks def enqueue_scrape( - self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred: + self, result: Response | Failure, request: Request + ) -> Generator[Deferred[Any], Any, None]: if self.slot is None: raise RuntimeError("Scraper slot not assigned") dfd = self.slot.add_response_request(result, request) - - def finish_scraping(_: Any) -> Any: - assert self.slot is not None - self.slot.finish_response(result, request) - self._check_if_closing(spider) - self._scrape_next(spider) - return _ - - dfd.addBoth(finish_scraping) - dfd.addErrback( - lambda f: logger.error( + self._scrape_next() + try: + yield dfd # fired in _wait_for_processing() + except Exception: + logger.error( "Scraper bug processing %(request)s", {"request": request}, - exc_info=failure_to_exc_info(f), - extra={"spider": spider}, + exc_info=True, + extra={"spider": self.crawler.spider}, ) - ) - self._scrape_next(spider) - return dfd + finally: + self.slot.finish_response(result, request) + self._check_if_closing() + self._scrape_next() - def _scrape_next(self, spider: Spider) -> None: + def _scrape_next(self) -> None: assert self.slot is not None # typing while self.slot.queue: - response, request, deferred = self.slot.next_response_request_deferred() - self._scrape(response, request, spider).chainDeferred(deferred) + result, request, queue_dfd = self.slot.next_response_request_deferred() + _schedule_coro(self._wait_for_processing(result, request, queue_dfd)) - def _scrape( - self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred: - """ - Handle the downloaded response or failure through the spider callback/errback - """ + async def _scrape(self, result: Response | Failure, request: Request) -> None: + """Handle the downloaded response or failure through the spider callback/errback.""" if not isinstance(result, (Response, Failure)): raise TypeError( f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}" ) - dfd = self._scrape2( - result, request, spider - ) # returns spider's processed output - dfd.addErrback(self.handle_spider_error, request, result, spider) - dfd.addCallback(self.handle_spider_output, request, result, spider) - return dfd - def _scrape2( - self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred: - """ - Handle the different cases of request's result been a Response or a Failure - """ + output: Iterable[Any] | AsyncIterator[Any] if isinstance(result, Response): - return self.spidermw.scrape_response( - self.call_spider, result, request, spider - ) - # else result is a Failure - dfd = self.call_spider(result, request, spider) - return dfd.addErrback(self._log_download_errors, result, request, spider) + try: + # call the spider middlewares and the request callback with the response + output = await self.spidermw.scrape_response_async( + self.call_spider_async, result, request + ) + except Exception: + self.handle_spider_error(Failure(), request, result) + else: + await self.handle_spider_output_async(output, request, result) + return + + try: + # call the request errback with the downloader error + output = await self.call_spider_async(result, request) + except Exception as spider_exc: + # the errback didn't silence the exception + assert self.crawler.spider + if not result.check(IgnoreRequest): + logkws = self.logformatter.download_error( + result, request, self.crawler.spider + ) + logger.log( + *logformatter_adapter(logkws), + extra={"spider": self.crawler.spider}, + exc_info=failure_to_exc_info(result), + ) + if spider_exc is not result.value: + # the errback raised a different exception, handle it + self.handle_spider_error(Failure(), request, result) + else: + await self.handle_spider_output_async(output, request, result) + + async def _wait_for_processing( + self, result: Response | Failure, request: Request, queue_dfd: Deferred[None] + ) -> None: + try: + await self._scrape(result, request) + except Exception: + queue_dfd.errback(Failure()) + else: + queue_dfd.callback(None) # awaited in enqueue_scrape() def call_spider( - self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred: + self, result: Response | Failure, request: Request, spider: Spider | None = None + ) -> Deferred[Iterable[Any] | AsyncIterator[Any]]: # pragma: no cover + warnings.warn( + "Scraper.call_spider() is deprecated, use call_spider_async() instead", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return deferred_from_coro(self.call_spider_async(result, request)) + + async def call_spider_async( + self, result: Response | Failure, request: Request + ) -> Iterable[Any] | AsyncIterator[Any]: + """Call the request callback or errback with the response or failure. + + .. versionadded:: 2.13 + """ + await _defer_sleep_async() + assert self.crawler.spider if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request assert result.request - callback = result.request.callback or spider._parse - warn_on_generator_with_return_value(spider, callback) - dfd = defer_succeed(result) - dfd.addCallbacks( - callback=callback, callbackKeywords=result.request.cb_kwargs - ) + callback = result.request.callback or self.crawler.spider._parse + warn_on_generator_with_return_value(self.crawler.spider, callback) + output = callback(result, **result.request.cb_kwargs) + if isinstance(output, Deferred): + warnings.warn( + f"{callback} returned a Deferred." + f" Returning Deferreds from spider callbacks is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) else: # result is a Failure # TODO: properly type adding this attribute to a Failure result.request = request # type: ignore[attr-defined] - dfd = defer_fail(result) - if request.errback: - warn_on_generator_with_return_value(spider, request.errback) - dfd.addErrback(request.errback) - return dfd.addCallback(iterate_spider_output) + if not request.errback: + result.raiseException() + warn_on_generator_with_return_value(self.crawler.spider, request.errback) + output = request.errback(result) + if isinstance(output, Failure): + output.raiseException() + # else the errback returned actual output (like a callback), + # which needs to be passed to iterate_spider_output() + if isinstance(output, Deferred): + warnings.warn( + f"{request.errback} returned a Deferred." + f" Returning Deferreds from spider errbacks is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return await ensure_awaitable(iterate_spider_output(output)) def handle_spider_error( self, _failure: Failure, request: Request, - response: Union[Response, Failure], - spider: Spider, + response: Response | Failure, ) -> None: + """Handle an exception raised by a spider callback or errback.""" + assert self.crawler.spider exc = _failure.value if isinstance(exc, CloseSpider): assert self.crawler.engine is not None # typing - self.crawler.engine.close_spider(spider, exc.reason or "cancelled") + _schedule_coro( + self.crawler.engine.close_spider_async(reason=exc.reason or "cancelled") + ) return - logkws = self.logformatter.spider_error(_failure, request, response, spider) + logkws = self.logformatter.spider_error( + _failure, request, response, self.crawler.spider + ) logger.log( *logformatter_adapter(logkws), exc_info=failure_to_exc_info(_failure), - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) self.signals.send_catch_log( signal=signals.spider_error, failure=_failure, response=response, - spider=spider, + spider=self.crawler.spider, ) assert self.crawler.stats + self.crawler.stats.inc_value("spider_exceptions/count") self.crawler.stats.inc_value( - f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider + f"spider_exceptions/{_failure.value.__class__.__name__}" ) def handle_spider_output( self, - result: Union[Iterable, AsyncIterable], + result: Iterable[_T] | AsyncIterator[_T], request: Request, - response: Union[Response, Failure], - spider: Spider, - ) -> Deferred: - if not result: - return defer_succeed(None) - it: Union[Generator, AsyncGenerator] - if isinstance(result, AsyncIterable): - it = aiter_errback( - result, self.handle_spider_error, request, response, spider + response: Response | Failure, + ) -> Deferred[None]: # pragma: no cover + """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" + warnings.warn( + "Scraper.handle_spider_output() is deprecated, use handle_spider_output_async() instead", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return deferred_from_coro( + self.handle_spider_output_async(result, request, response) + ) + + async def handle_spider_output_async( + self, + result: Iterable[_T] | AsyncIterator[_T], + request: Request, + response: Response | Failure, + ) -> None: + """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel. + + .. versionadded:: 2.13 + """ + it: Iterable[_T] | AsyncIterator[_T] + if is_asyncio_available(): + if isinstance(result, AsyncIterator): + it = aiter_errback(result, self.handle_spider_error, request, response) + else: + it = iter_errback(result, self.handle_spider_error, request, response) + await _parallel_asyncio( + it, self.concurrent_items, self._process_spidermw_output_async, response ) - dfd = parallel_async( + return + if isinstance(result, AsyncIterator): + it = aiter_errback(result, self.handle_spider_error, request, response) + await maybe_deferred_to_future( + parallel_async( + it, + self.concurrent_items, + self._process_spidermw_output, + response, + ) + ) + return + it = iter_errback(result, self.handle_spider_error, request, response) + await maybe_deferred_to_future( + parallel( it, self.concurrent_items, self._process_spidermw_output, - request, response, - spider, ) - else: - it = iter_errback( - result, self.handle_spider_error, request, response, spider - ) - dfd = parallel( - it, - self.concurrent_items, - self._process_spidermw_output, - request, - response, - spider, - ) - return dfd + ) def _process_spidermw_output( - self, output: Any, request: Request, response: Response, spider: Spider - ) -> Optional[Deferred]: + self, output: Any, response: Response | Failure + ) -> Deferred[None]: """Process each Request/Item (given in the output parameter) returned - from the given spider + from the given spider. + + Items are sent to the item pipelines, requests are scheduled. + """ + return deferred_from_coro(self._process_spidermw_output_async(output, response)) + + async def _process_spidermw_output_async( + self, output: Any, response: Response | Failure + ) -> None: + """Process each Request/Item (given in the output parameter) returned + from the given spider. + + Items are sent to the item pipelines, requests are scheduled. """ - assert self.slot is not None # typing if isinstance(output, Request): assert self.crawler.engine is not None # typing self.crawler.engine.crawl(request=output) - elif is_item(output): - self.slot.itemproc_size += 1 - dfd = self.itemproc.process_item(output, spider) - dfd.addBoth(self._itemproc_finished, output, response, spider) - return dfd - elif output is None: - pass - else: - typename = type(output).__name__ - logger.error( - "Spider must return request, item, or None, got %(typename)r in %(request)s", - {"request": request, "typename": typename}, - extra={"spider": spider}, - ) - return None + return + if output is not None: + await self.start_itemproc_async(output, response=response) - def _log_download_errors( - self, - spider_failure: Failure, - download_failure: Failure, - request: Request, - spider: Spider, - ) -> Union[Failure, None]: - """Log and silence errors that come from the engine (typically download - errors that got propagated thru here). + def start_itemproc( + self, item: Any, *, response: Response | Failure | None + ) -> Deferred[None]: # pragma: no cover + """Send *item* to the item pipelines for processing. - spider_failure: the value passed into the errback of self.call_spider() - download_failure: the value passed into _scrape2() from - ExecutionEngine._handle_downloader_output() as "result" + *response* is the source of the item data. If the item does not come + from response data, e.g. it was hard-coded, set it to ``None``. """ - if not download_failure.check(IgnoreRequest): - if download_failure.frames: - logkws = self.logformatter.download_error( - download_failure, request, spider - ) - logger.log( - *logformatter_adapter(logkws), - extra={"spider": spider}, - exc_info=failure_to_exc_info(download_failure), - ) - else: - errmsg = download_failure.getErrorMessage() - if errmsg: - logkws = self.logformatter.download_error( - download_failure, request, spider, errmsg - ) - logger.log( - *logformatter_adapter(logkws), - extra={"spider": spider}, - ) + warnings.warn( + "Scraper.start_itemproc() is deprecated, use start_itemproc_async() instead", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return deferred_from_coro(self.start_itemproc_async(item, response=response)) - if spider_failure is not download_failure: - return spider_failure - return None + async def start_itemproc_async( + self, item: Any, *, response: Response | Failure | None + ) -> None: + """Send *item* to the item pipelines for processing. - def _itemproc_finished( - self, output: Any, item: Any, response: Response, spider: Spider - ) -> Deferred: - """ItemProcessor finished for the given ``item`` and returned ``output``""" + *response* is the source of the item data. If the item does not come + from response data, e.g. it was hard-coded, set it to ``None``. + + .. versionadded:: 2.14 + """ assert self.slot is not None # typing - self.slot.itemproc_size -= 1 - if isinstance(output, Failure): - ex = output.value - if isinstance(ex, DropItem): - logkws = self.logformatter.dropped(item, ex, response, spider) - if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) - return self.signals.send_catch_log_deferred( - signal=signals.item_dropped, - item=item, - response=response, - spider=spider, - exception=output.value, + assert self.crawler.spider is not None # typing + self.slot.itemproc_size += 1 + try: + if self._itemproc_has_async["process_item"]: + output = await self.itemproc.process_item_async(item) + else: + output = await maybe_deferred_to_future( + self.itemproc.process_item(item, self.crawler.spider) ) - assert ex - logkws = self.logformatter.item_error(item, ex, response, spider) + except DropItem as ex: + logkws = self.logformatter.dropped(item, ex, response, self.crawler.spider) + if logkws is not None: + logger.log( + *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} + ) + await self.signals.send_catch_log_async( + signal=signals.item_dropped, + item=item, + response=response, + spider=self.crawler.spider, + exception=ex, + ) + except Exception as ex: + logkws = self.logformatter.item_error( + item, ex, response, self.crawler.spider + ) logger.log( *logformatter_adapter(logkws), - extra={"spider": spider}, - exc_info=failure_to_exc_info(output), + extra={"spider": self.crawler.spider}, + exc_info=True, ) - return self.signals.send_catch_log_deferred( + await self.signals.send_catch_log_async( signal=signals.item_error, item=item, response=response, - spider=spider, - failure=output, + spider=self.crawler.spider, + failure=Failure(), ) - logkws = self.logformatter.scraped(output, response, spider) - if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) - return self.signals.send_catch_log_deferred( - signal=signals.item_scraped, item=output, response=response, spider=spider - ) + else: + logkws = self.logformatter.scraped(output, response, self.crawler.spider) + if logkws is not None: + logger.log( + *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} + ) + await self.signals.send_catch_log_async( + signal=signals.item_scraped, + item=output, + response=response, + spider=self.crawler.spider, + ) + finally: + self.slot.itemproc_size -= 1 diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 031a0be36..fbc6f2530 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,343 +3,285 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ -import logging -from inspect import isasyncgenfunction, iscoroutine -from itertools import islice -from typing import ( - Any, - AsyncGenerator, - AsyncIterable, - Callable, - Generator, - Iterable, - List, - Optional, - Tuple, - Union, - cast, -) -from twisted.internet.defer import Deferred, inlineCallbacks +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator, Callable, Coroutine, Iterable +from functools import wraps +from inspect import isasyncgenfunction +from itertools import islice +from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar +from warnings import warn + from twisted.python.failure import Failure from scrapy import Request, Spider -from scrapy.exceptions import _InvalidOutput +from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput from scrapy.http import Response from scrapy.middleware import MiddlewareManager -from scrapy.settings import BaseSettings -from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen +from scrapy.utils.asyncgen import as_async_generator from scrapy.utils.conf import build_component_list from scrapy.utils.defer import ( - deferred_f_from_coro_f, + _defer_sleep_async, deferred_from_coro, maybe_deferred_to_future, - mustbe_deferred, ) -from scrapy.utils.python import MutableAsyncChain, MutableChain +from scrapy.utils.python import MutableAsyncChain, global_object_name + +if TYPE_CHECKING: + from twisted.internet.defer import Deferred + + from scrapy.settings import BaseSettings + logger = logging.getLogger(__name__) -ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] - - -def _isiterable(o: Any) -> bool: - return isinstance(o, (Iterable, AsyncIterable)) +_T = TypeVar("_T") +ScrapeFunc: TypeAlias = Callable[ + [Response | Failure, Request], + Coroutine[Any, Any, Iterable[_T] | AsyncIterator[_T]], +] class SpiderMiddlewareManager(MiddlewareManager): component_name = "spider middleware" - def __init__(self, *middlewares: Any): - super().__init__(*middlewares) - self.downgrade_warning_done = False - @classmethod - def _get_mwlist_from_settings(cls, settings: BaseSettings) -> List[Any]: - return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES")) + def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: + return build_component_list( + settings.get_component_priority_dict_with_base("SPIDER_MIDDLEWARES") + ) def _add_middleware(self, mw: Any) -> None: - super()._add_middleware(mw) if hasattr(mw, "process_spider_input"): self.methods["process_spider_input"].append(mw.process_spider_input) - if hasattr(mw, "process_start_requests"): - self.methods["process_start_requests"].appendleft(mw.process_start_requests) - process_spider_output = self._get_async_method_pair(mw, "process_spider_output") + self._check_mw_method_spider_arg(mw.process_spider_input) + + if hasattr(mw, "process_start"): + self.methods["process_start"].appendleft(mw.process_start) + + process_spider_output = self._get_process_spider_output(mw) self.methods["process_spider_output"].appendleft(process_spider_output) + if process_spider_output is not None: + self._check_mw_method_spider_arg(process_spider_output) + process_spider_exception = getattr(mw, "process_spider_exception", None) self.methods["process_spider_exception"].appendleft(process_spider_exception) + if process_spider_exception is not None: + self._check_mw_method_spider_arg(process_spider_exception) - def _process_spider_input( + async def _process_spider_input( self, - scrape_func: ScrapeFunc, + scrape_func: ScrapeFunc[_T], response: Response, request: Request, - spider: Spider, - ) -> Any: + ) -> Iterable[_T] | AsyncIterator[_T]: for method in self.methods["process_spider_input"]: - method = cast(Callable, method) + assert method is not None try: - result = method(response=response, spider=spider) + if method in self._mw_methods_requiring_spider: + result = method(response=response, spider=self._spider) + else: + result = method(response=response) if result is not None: msg = ( - f"{method.__qualname__} must return None " + f"{global_object_name(method)} must return None " f"or raise an exception, got {type(result)}" ) raise _InvalidOutput(msg) except _InvalidOutput: raise except Exception: - return scrape_func(Failure(), request, spider) - return scrape_func(response, request, spider) + return await scrape_func(Failure(), request) + return await scrape_func(response, request) - def _evaluate_iterable( + async def _evaluate_iterable( self, response: Response, - spider: Spider, - iterable: Union[Iterable, AsyncIterable], + iterable: AsyncIterator[_T], exception_processor_index: int, - recover_to: Union[MutableChain, MutableAsyncChain], - ) -> Union[Generator, AsyncGenerator]: - def process_sync(iterable: Iterable) -> Generator: - try: - yield from iterable - except Exception as ex: - exception_result = self._process_spider_exception( - response, spider, Failure(ex), exception_processor_index - ) - if isinstance(exception_result, Failure): - raise - recover_to.extend(exception_result) - - async def process_async(iterable: AsyncIterable) -> AsyncGenerator: - try: - async for r in iterable: - yield r - except Exception as ex: - exception_result = self._process_spider_exception( - response, spider, Failure(ex), exception_processor_index - ) - if isinstance(exception_result, Failure): - raise - recover_to.extend(exception_result) - - if isinstance(iterable, AsyncIterable): - return process_async(iterable) - return process_sync(iterable) + recover_to: MutableAsyncChain[_T], + ) -> AsyncIterator[_T]: + try: + async for r in iterable: + yield r + except Exception as ex: + exception_result: MutableAsyncChain[_T] = self._process_spider_exception( + response, ex, exception_processor_index + ) + recover_to.extend(exception_result) def _process_spider_exception( self, response: Response, - spider: Spider, - _failure: Failure, + exception: Exception, start_index: int = 0, - ) -> Union[Failure, MutableChain]: - exception = _failure.value + ) -> MutableAsyncChain[_T]: # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): - return _failure + raise exception method_list = islice( self.methods["process_spider_exception"], start_index, None ) for method_index, method in enumerate(method_list, start=start_index): if method is None: continue - method = cast(Callable, method) - result = method(response=response, exception=exception, spider=spider) - if _isiterable(result): + if method in self._mw_methods_requiring_spider: + result = method( + response=response, exception=exception, spider=self._spider + ) + else: + result = method(response=response, exception=exception) + if isinstance(result, (Iterable, AsyncIterator)): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned - dfd: Deferred = self._process_spider_output( - response, spider, result, method_index + 1 - ) - # _process_spider_output() returns a Deferred only because of downgrading so this can be - # simplified when downgrading is removed. - if dfd.called: - # the result is available immediately if _process_spider_output didn't do downgrading - return cast(MutableChain, dfd.result) - # we forbid waiting here because otherwise we would need to return a deferred from - # _process_spider_exception too, which complicates the architecture - msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" - raise _InvalidOutput(msg) - elif result is None: + if isinstance(result, Iterable): + result = as_async_generator(result) + return self._process_spider_output(response, result, method_index + 1) + if result is None: continue - else: - msg = ( - f"{method.__qualname__} must return None " - f"or an iterable, got {type(result)}" - ) - raise _InvalidOutput(msg) - return _failure + msg = ( + f"{global_object_name(method)} must return None " + f"or an iterable, got {type(result)}" + ) + raise _InvalidOutput(msg) + raise exception - # This method cannot be made async def, as _process_spider_exception relies on the Deferred result - # being available immediately which doesn't work when it's a wrapped coroutine. - # It also needs @inlineCallbacks only because of downgrading so it can be removed when downgrading is removed. - @inlineCallbacks def _process_spider_output( self, response: Response, - spider: Spider, - result: Union[Iterable, AsyncIterable], + result: AsyncIterator[_T], start_index: int = 0, - ) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]: + ) -> MutableAsyncChain[_T]: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - recovered: Union[MutableChain, MutableAsyncChain] - last_result_is_async = isinstance(result, AsyncIterable) - if last_result_is_async: - recovered = MutableAsyncChain() - else: - recovered = MutableChain() - - # There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async. - # 1. def foo. Sync iterables are passed as is, async ones are downgraded. - # 2. async def foo. Sync iterables are upgraded, async ones are passed as is. - # 3. def foo + async def foo_async. Iterables are passed to the respective method. - # Storing methods and method tuples in the same list is weird but we should be able to roll this back - # when we drop this compatibility feature. - + recovered: MutableAsyncChain[_T] = MutableAsyncChain() method_list = islice(self.methods["process_spider_output"], start_index, None) - for method_index, method_pair in enumerate(method_list, start=start_index): - if method_pair is None: + for method_index, method in enumerate(method_list, start=start_index): + if method is None: continue - need_upgrade = need_downgrade = False - if isinstance(method_pair, tuple): - # This tuple handling is only needed until _async compatibility methods are removed. - method_sync, method_async = method_pair - method = method_async if last_result_is_async else method_sync + if method in self._mw_methods_requiring_spider: + result = method(response=response, result=result, spider=self._spider) else: - method = method_pair - if not last_result_is_async and isasyncgenfunction(method): - need_upgrade = True - elif last_result_is_async and not isasyncgenfunction(method): - need_downgrade = True - try: - if need_upgrade: - # Iterable -> AsyncIterable - result = as_async_generator(result) - elif need_downgrade: - if not self.downgrade_warning_done: - logger.warning( - f"Async iterable passed to {method.__qualname__} " - f"was downgraded to a non-async one" - ) - self.downgrade_warning_done = True - assert isinstance(result, AsyncIterable) - # AsyncIterable -> Iterable - result = yield deferred_from_coro(collect_asyncgen(result)) - if isinstance(recovered, AsyncIterable): - recovered_collected = yield deferred_from_coro( - collect_asyncgen(recovered) - ) - recovered = MutableChain(recovered_collected) - # might fail directly if the output value is not a generator - result = method(response=response, result=result, spider=spider) - except Exception as ex: - exception_result = self._process_spider_exception( - response, spider, Failure(ex), method_index + 1 - ) - if isinstance(exception_result, Failure): - raise - return exception_result - if _isiterable(result): - result = self._evaluate_iterable( - response, spider, result, method_index + 1, recovered - ) - else: - if iscoroutine(result): - result.close() # Silence warning about not awaiting - msg = ( - f"{method.__qualname__} must be an asynchronous " - f"generator (i.e. use yield)" - ) - else: - msg = ( - f"{method.__qualname__} must return an iterable, got " - f"{type(result)}" - ) - raise _InvalidOutput(msg) - last_result_is_async = isinstance(result, AsyncIterable) - - if last_result_is_async: - return MutableAsyncChain(result, recovered) - return MutableChain(result, recovered) # type: ignore[arg-type] + result = method(response=response, result=result) + result = self._evaluate_iterable( + response, result, method_index + 1, recovered + ) + return MutableAsyncChain(result, recovered) async def _process_callback_output( - self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable] - ) -> Union[MutableChain, MutableAsyncChain]: - recovered: Union[MutableChain, MutableAsyncChain] - if isinstance(result, AsyncIterable): - recovered = MutableAsyncChain() - else: - recovered = MutableChain() - result = self._evaluate_iterable(response, spider, result, 0, recovered) - result = await maybe_deferred_to_future( - self._process_spider_output(response, spider, result) - ) - if isinstance(result, AsyncIterable): - return MutableAsyncChain(result, recovered) - if isinstance(recovered, AsyncIterable): - recovered_collected = await collect_asyncgen(recovered) - recovered = MutableChain(recovered_collected) - return MutableChain(result, recovered) + self, response: Response, result: AsyncIterator[_T] + ) -> MutableAsyncChain[_T]: + recovered: MutableAsyncChain[_T] = MutableAsyncChain() + result = self._evaluate_iterable(response, result, 0, recovered) + result = self._process_spider_output(response, result) + return MutableAsyncChain(result, recovered) def scrape_response( self, - scrape_func: ScrapeFunc, + scrape_func: Callable[ + [Response | Failure, Request], + Deferred[Iterable[_T] | AsyncIterator[_T]], + ], response: Response, request: Request, spider: Spider, - ) -> Deferred: - async def process_callback_output( - result: Union[Iterable, AsyncIterable] - ) -> Union[MutableChain, MutableAsyncChain]: - return await self._process_callback_output(response, spider, result) - - def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]: - return self._process_spider_exception(response, spider, _failure) - - dfd = mustbe_deferred( - self._process_spider_input, scrape_func, response, request, spider + ) -> Deferred[MutableAsyncChain[_T]]: # pragma: no cover + warn( + "SpiderMiddlewareManager.scrape_response() is deprecated, use scrape_response_async() instead", + ScrapyDeprecationWarning, + stacklevel=2, ) - dfd.addCallbacks( - callback=deferred_f_from_coro_f(process_callback_output), - errback=process_spider_exception, - ) - return dfd - def process_start_requests( - self, start_requests: Iterable[Request], spider: Spider - ) -> Deferred: - return self._process_chain("process_start_requests", start_requests, spider) + @wraps(scrape_func) + async def scrape_func_wrapped( + response: Response | Failure, request: Request + ) -> Iterable[_T] | AsyncIterator[_T]: + return await maybe_deferred_to_future(scrape_func(response, request)) + + self._set_compat_spider(spider) + return deferred_from_coro( + self.scrape_response_async(scrape_func_wrapped, response, request) + ) + + async def scrape_response_async( + self, + scrape_func: ScrapeFunc[_T], + response: Response, + request: Request, + ) -> MutableAsyncChain[_T]: + if not self.crawler: + raise RuntimeError( + "scrape_response_async() called on a SpiderMiddlewareManager" + " instance created without a crawler." + ) + try: + it: Iterable[_T] | AsyncIterator[_T] = await self._process_spider_input( + scrape_func, response, request + ) + ait = it if isinstance(it, AsyncIterator) else as_async_generator(it) + return await self._process_callback_output(response, ait) + except Exception as ex: + await _defer_sleep_async() + return self._process_spider_exception(response, ex) + + async def process_start( + self, spider: Spider | None = None + ) -> AsyncIterator[Any] | None: + if spider: + if self.crawler: + msg = ( + "Passing a spider argument to SpiderMiddlewareManager.process_start() is deprecated" + " and the passed value is ignored." + ) + else: + msg = ( + "Passing a spider argument to SpiderMiddlewareManager.process_start() is deprecated," + " SpiderMiddlewareManager should be instantiated with a Crawler instance instead." + ) + warn(msg, category=ScrapyDeprecationWarning, stacklevel=2) + self._set_compat_spider(spider) + start = self._spider.start() + return await self._process_chain("process_start", start) # This method is only needed until _async compatibility methods are removed. @staticmethod - def _get_async_method_pair( - mw: Any, methodname: str - ) -> Union[None, Callable, Tuple[Callable, Callable]]: - normal_method: Optional[Callable] = getattr(mw, methodname, None) - methodname_async = methodname + "_async" - async_method: Optional[Callable] = getattr(mw, methodname_async, None) + def _get_process_spider_output(mw: Any) -> Callable[..., Any] | None: + normal_method: Callable[..., Any] | None = getattr( + mw, "process_spider_output", None + ) + async_method: Callable[..., Any] | None = getattr( + mw, "process_spider_output_async", None + ) if not async_method: + if normal_method and not isasyncgenfunction(normal_method): + raise TypeError( + f"Middleware {global_object_name(mw.__class__)} doesn't support" + f" asynchronous spider output. Its process_spider_output() method" + f" should be an async generator function or it should additionally" + f" define a process_spider_output_async() method." + ) return normal_method if not normal_method: logger.error( - f"Middleware {mw.__qualname__} has {methodname_async} " - f"without {methodname}, skipping this method." + f"Middleware {global_object_name(mw.__class__)} has" + f" process_spider_output_async() without process_spider_output()," + f" skipping this method. Please rename it to process_spider_output()." ) return None if not isasyncgenfunction(async_method): logger.error( - f"{async_method.__qualname__} is not " + f"{global_object_name(async_method)} is not " f"an async generator function, skipping this method." ) return normal_method if isasyncgenfunction(normal_method): logger.error( - f"{normal_method.__qualname__} is an async " - f"generator function while {methodname_async} exists, " - f"skipping both methods." + f"{global_object_name(normal_method)} is an async " + f"generator function while process_spider_output_async() exists, " + f"skipping both methods. Please remove process_spider_output_async()." ) return None - return normal_method, async_method + return async_method diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1db9ace28..8e3ae7879 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,38 +1,27 @@ from __future__ import annotations +import asyncio +import contextlib import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union, cast +from abc import ABC, abstractmethod +from functools import partial +from typing import TYPE_CHECKING, Any, TypeVar -from twisted.internet.defer import ( - Deferred, - DeferredList, - inlineCallbacks, - maybeDeferred, -) +from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks -try: - # zope >= 5.0 only supports MultipleInvalid - from zope.interface.exceptions import MultipleInvalid -except ImportError: - MultipleInvalid = None - -from zope.interface.verify import verifyClass - -from scrapy import Spider, signals +from scrapy import Spider from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extension import ExtensionManager -from scrapy.interfaces import ISpiderLoader -from scrapy.logformatter import LogFormatter -from scrapy.settings import BaseSettings, Settings, overridden_settings +from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings from scrapy.signalmanager import SignalManager -from scrapy.statscollectors import StatsCollector +from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader +from scrapy.utils.defer import deferred_from_coro from scrapy.utils.log import ( - LogCounterHandler, configure_logging, get_scrapy_root_handler, install_scrapy_root_handler, @@ -42,24 +31,34 @@ from scrapy.utils.log import ( from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.reactor import ( + _asyncio_reactor_path, install_reactor, is_asyncio_reactor_installed, + is_reactor_installed, + set_asyncio_event_loop, verify_installed_asyncio_event_loop, verify_installed_reactor, ) +from scrapy.utils.reactorless import install_reactor_import_hook if TYPE_CHECKING: - from scrapy.utils.request import RequestFingerprinter + from collections.abc import Awaitable, Generator, Iterable + + from scrapy.logformatter import LogFormatter + from scrapy.statscollectors import StatsCollector + from scrapy.utils.request import RequestFingerprinterProtocol logger = logging.getLogger(__name__) +_T = TypeVar("_T") + class Crawler: def __init__( self, - spidercls: Type[Spider], - settings: Union[None, Dict[str, Any], Settings] = None, + spidercls: type[Spider], + settings: dict[str, Any] | Settings | None = None, init_reactor: bool = False, ): if isinstance(spidercls, Spider): @@ -68,7 +67,7 @@ class Crawler: if isinstance(settings, dict) or settings is None: settings = Settings(settings) - self.spidercls: Type[Spider] = spidercls + self.spidercls: type[Spider] = spidercls self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) self._update_root_log_handler() @@ -80,12 +79,12 @@ class Crawler: self.crawling: bool = False self._started: bool = False - self.extensions: Optional[ExtensionManager] = None - self.stats: Optional[StatsCollector] = None - self.logformatter: Optional[LogFormatter] = None - self.request_fingerprinter: Optional[RequestFingerprinter] = None - self.spider: Optional[Spider] = None - self.engine: Optional[ExecutionEngine] = None + self.extensions: ExtensionManager | None = None + self.stats: StatsCollector | None = None + self.logformatter: LogFormatter | None = None + self.request_fingerprinter: RequestFingerprinterProtocol | None = None + self.spider: Spider | None = None + self.engine: ExecutionEngine | None = None def _update_root_log_handler(self) -> None: if get_scrapy_root_handler() is not None: @@ -99,14 +98,7 @@ class Crawler: self.addons.load_settings(self.settings) self.stats = load_object(self.settings["STATS_CLASS"])(self) - handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL")) - logging.root.addHandler(handler) - # lambda is assigned to Crawler attribute because this way it is not - # garbage collected after leaving the scope - self.__remove_handler = lambda: logging.root.removeHandler(handler) - self.signals.connect(self.__remove_handler, signals.engine_stopped) - - lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) + lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) self.logformatter = lf_cls.from_crawler(self) self.request_fingerprinter = build_from_crawler( @@ -114,20 +106,42 @@ class Crawler: self, ) - reactor_class: str = self.settings["TWISTED_REACTOR"] - event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"] - if self._init_reactor: - # this needs to be done after the spider settings are merged, - # but before something imports twisted.internet.reactor + use_reactor = self.settings.getbool("TWISTED_REACTOR_ENABLED") + if use_reactor: + # We either install a reactor or expect one to be installed. + reactor_class: str = self.settings["TWISTED_REACTOR"] + event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"] + if self._init_reactor: + # We need to install a reactor. + # This needs to be done after the spider settings are merged, + # but before something imports twisted.internet.reactor. + if reactor_class: + # Install a specific reactor. + install_reactor(reactor_class, event_loop) + else: + # Install the default one. + from twisted.internet import reactor # noqa: F401 + elif not is_reactor_installed(): + # We need a reactor to be already installed. + raise RuntimeError( + "We expected a Twisted reactor to be installed but it isn't." + ) if reactor_class: - install_reactor(reactor_class, event_loop) - else: - from twisted.internet import reactor # noqa: F401 - log_reactor_info() - if reactor_class: - verify_installed_reactor(reactor_class) - if is_asyncio_reactor_installed() and event_loop: - verify_installed_asyncio_event_loop(event_loop) + # We need to check that the correct reactor is installed. + verify_installed_reactor(reactor_class) + if is_asyncio_reactor_installed() and event_loop: + verify_installed_asyncio_event_loop(event_loop) + + if self._init_reactor or reactor_class: + log_reactor_info() + else: + # We expect a reactor to not be installed. + if is_reactor_installed(): + raise RuntimeError( + "TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed." + ) + logger.debug("Not using a Twisted reactor") + self._apply_reactorless_default_settings() self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() @@ -137,15 +151,36 @@ class Crawler: "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} ) + def _apply_reactorless_default_settings(self) -> None: + """Change some setting defaults when not using a Twisted reactor. + + Some settings need different defaults when using and not using a + reactor, but as we can't put this logic into default_settings.py we + change them here when the reactor is not used. + """ + self.settings.set("TELNETCONSOLE_ENABLED", False, priority="default") + for scheme in ("http", "https"): + self.settings["DOWNLOAD_HANDLERS_BASE"][scheme] = ( + "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler" + ) + self.settings["DOWNLOAD_HANDLERS_BASE"]["ftp"] = None + + # Cannot use @deferred_f_from_coro_f because that relies on the reactor + # being installed already, which is done within _apply_settings(), inside + # this method. @inlineCallbacks - def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]: + def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred[Any], Any, None]: + """Start the crawler by instantiating its spider class with the given + *args* and *kwargs* arguments, while setting the execution engine in + motion. Should be called only once. + + Return a deferred that is fired when the crawl is finished. + """ if self.crawling: raise RuntimeError("Crawling already taking place") if self._started: - warnings.warn( - "Running Crawler.crawl() more than once is deprecated.", - ScrapyDeprecationWarning, - stacklevel=2, + raise RuntimeError( + "Cannot run Crawler.crawl() more than once on the same instance." ) self.crawling = self._started = True @@ -154,41 +189,98 @@ class Crawler: self._apply_settings() self._update_root_log_handler() self.engine = self._create_engine() - start_requests = iter(self.spider.start_requests()) - yield self.engine.open_spider(self.spider, start_requests) - yield maybeDeferred(self.engine.start) + yield deferred_from_coro(self.engine.open_spider_async()) + yield deferred_from_coro(self.engine.start_async()) except Exception: self.crawling = False if self.engine is not None: - yield self.engine.close() + yield deferred_from_coro(self.engine.close_async()) + raise + + async def crawl_async(self, *args: Any, **kwargs: Any) -> None: + """Start the crawler by instantiating its spider class with the given + *args* and *kwargs* arguments, while setting the execution engine in + motion. Should be called only once. + + .. versionadded:: 2.14 + + Complete when the crawl is finished. + """ + if self.crawling: + raise RuntimeError("Crawling already taking place") + if self._started: + raise RuntimeError( + "Cannot run Crawler.crawl_async() more than once on the same instance." + ) + self.crawling = self._started = True + + try: + self.spider = self._create_spider(*args, **kwargs) + self._apply_settings() + self._update_root_log_handler() + self.engine = self._create_engine() + await self.engine.open_spider_async() + await self.engine.start_async() + except Exception: + self.crawling = False + if self.engine is not None: + await self.engine.close_async() raise def _create_spider(self, *args: Any, **kwargs: Any) -> Spider: return self.spidercls.from_crawler(self, *args, **kwargs) def _create_engine(self) -> ExecutionEngine: - return ExecutionEngine(self, lambda _: self.stop()) + return ExecutionEngine(self, lambda _: self.stop_async()) - @inlineCallbacks - def stop(self) -> Generator[Deferred, Any, None]: - """Starts a graceful stop of the crawler and returns a deferred that is + def stop(self) -> Deferred[None]: + """Start a graceful stop of the crawler and return a deferred that is fired when the crawler is stopped.""" + warnings.warn( + "Crawler.stop() is deprecated, use stop_async() instead", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return deferred_from_coro(self.stop_async()) + + async def stop_async(self) -> None: + """Start a graceful stop of the crawler and complete when the crawler is stopped. + + .. versionadded:: 2.14 + """ if self.crawling: self.crawling = False assert self.engine - yield maybeDeferred(self.engine.stop) + if self.engine.running: + await self.engine.stop_async() @staticmethod - def _get_component(component_class, components): + def _get_component( + component_class: type[_T], components: Iterable[Any] + ) -> _T | None: for component in components: if isinstance(component, component_class): return component return None - def get_addon(self, cls): + def get_addon(self, cls: type[_T]) -> _T | None: + """Return the run-time instance of an :ref:`add-on ` of + the specified class or a subclass, or ``None`` if none is found. + + .. versionadded:: 2.12 + """ return self._get_component(cls, self.addons.addons) - def get_downloader_middleware(self, cls): + def get_downloader_middleware(self, cls: type[_T]) -> _T | None: + """Return the run-time instance of a :ref:`downloader middleware + ` of the specified class or a subclass, + or ``None`` if none is found. + + .. versionadded:: 2.12 + + This method can only be called after the crawl engine has been created, + e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. + """ if not self.engine: raise RuntimeError( "Crawler.get_downloader_middleware() can only be called after " @@ -196,7 +288,17 @@ class Crawler: ) return self._get_component(cls, self.engine.downloader.middleware.middlewares) - def get_extension(self, cls): + def get_extension(self, cls: type[_T]) -> _T | None: + """Return the run-time instance of an :ref:`extension + ` of the specified class or a subclass, + or ``None`` if none is found. + + .. versionadded:: 2.12 + + This method can only be called after the extension manager has been + created, e.g. at signals :signal:`engine_started` or + :signal:`spider_opened`. + """ if not self.extensions: raise RuntimeError( "Crawler.get_extension() can only be called after the " @@ -204,7 +306,16 @@ class Crawler: ) return self._get_component(cls, self.extensions.middlewares) - def get_item_pipeline(self, cls): + def get_item_pipeline(self, cls: type[_T]) -> _T | None: + """Return the run-time instance of a :ref:`item pipeline + ` of the specified class or a subclass, or + ``None`` if none is found. + + .. versionadded:: 2.12 + + This method can only be called after the crawl engine has been created, + e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. + """ if not self.engine: raise RuntimeError( "Crawler.get_item_pipeline() can only be called after the " @@ -212,7 +323,16 @@ class Crawler: ) return self._get_component(cls, self.engine.scraper.itemproc.middlewares) - def get_spider_middleware(self, cls): + def get_spider_middleware(self, cls: type[_T]) -> _T | None: + """Return the run-time instance of a :ref:`spider middleware + ` of the specified class or a subclass, or + ``None`` if none is found. + + .. versionadded:: 2.12 + + This method can only be called after the crawl engine has been created, + e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. + """ if not self.engine: raise RuntimeError( "Crawler.get_spider_middleware() can only be called after the " @@ -221,7 +341,60 @@ class Crawler: return self._get_component(cls, self.engine.scraper.spidermw.middlewares) -class CrawlerRunner: +class CrawlerRunnerBase(ABC): + def __init__(self, settings: dict[str, Any] | Settings | None = None): + if isinstance(settings, dict) or settings is None: + settings = Settings(settings) + AddonManager.load_pre_crawler_settings(settings) + self.settings: Settings = settings + self.spider_loader: SpiderLoaderProtocol = get_spider_loader(settings) + self._crawlers: set[Crawler] = set() + self.bootstrap_failed = False + + @property + def crawlers(self) -> set[Crawler]: + """Set of :class:`crawlers ` started by + :meth:`crawl` and managed by this class.""" + return self._crawlers + + def create_crawler( + self, crawler_or_spidercls: type[Spider] | str | Crawler + ) -> Crawler: + """ + Return a :class:`~scrapy.crawler.Crawler` object. + + * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. + * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler + is constructed for it. + * If ``crawler_or_spidercls`` is a string, this function finds + a spider with this name in a Scrapy project (using spider loader), + then creates a Crawler instance for it. + """ + if isinstance(crawler_or_spidercls, Spider): + raise ValueError( + "The crawler_or_spidercls argument cannot be a spider object, " + "it must be a spider class (or a Crawler object)" + ) + if isinstance(crawler_or_spidercls, Crawler): + return crawler_or_spidercls + return self._create_crawler(crawler_or_spidercls) + + def _create_crawler(self, spidercls: str | type[Spider]) -> Crawler: + if isinstance(spidercls, str): + spidercls = self.spider_loader.load(spidercls) + return Crawler(spidercls, self.settings) + + @abstractmethod + def crawl( + self, + crawler_or_spidercls: type[Spider] | str | Crawler, + *args: Any, + **kwargs: Any, + ) -> Awaitable[None]: + raise NotImplementedError + + +class CrawlerRunner(CrawlerRunnerBase): """ This is a convenient helper class that keeps track of, manages and runs crawlers inside an already setup :mod:`~twisted.internet.reactor`. @@ -232,37 +405,25 @@ class CrawlerRunner: This class shouldn't be needed (since Scrapy is responsible of using it accordingly) unless writing scripts that manually handle the crawling process. See :ref:`run-from-script` for an example. + + This class provides Deferred-based APIs. Use :class:`AsyncCrawlerRunner` + for modern coroutine APIs. """ - crawlers = property( - lambda self: self._crawlers, - doc="Set of :class:`crawlers ` started by " - ":meth:`crawl` and managed by this class.", - ) - - @staticmethod - def _get_spider_loader(settings: BaseSettings): - """Get SpiderLoader instance from settings""" - cls_path = settings.get("SPIDER_LOADER_CLASS") - loader_cls = load_object(cls_path) - verifyClass(ISpiderLoader, loader_cls) - return loader_cls.from_settings(settings.frozencopy()) - - def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None): - if isinstance(settings, dict) or settings is None: - settings = Settings(settings) - self.settings = settings - self.spider_loader = self._get_spider_loader(settings) - self._crawlers: Set[Crawler] = set() - self._active: Set[Deferred] = set() - self.bootstrap_failed = False + def __init__(self, settings: dict[str, Any] | Settings | None = None): + super().__init__(settings) + if not self.settings.getbool("TWISTED_REACTOR_ENABLED"): + raise RuntimeError( + f"{type(self).__name__} doesn't support TWISTED_REACTOR_ENABLED=False." + ) + self._active: set[Deferred[None]] = set() def crawl( self, - crawler_or_spidercls: Union[Type[Spider], str, Crawler], + crawler_or_spidercls: type[Spider] | str | Crawler, *args: Any, **kwargs: Any, - ) -> Deferred: + ) -> Deferred[None]: """ Run a crawler with the provided arguments. @@ -292,57 +453,34 @@ class CrawlerRunner: crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) - def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred: + @inlineCallbacks + def _crawl( + self, crawler: Crawler, *args: Any, **kwargs: Any + ) -> Generator[Deferred[Any], Any, None]: self.crawlers.add(crawler) d = crawler.crawl(*args, **kwargs) self._active.add(d) - - def _done(result: Any) -> Any: + failed = False + try: + yield d + except Exception: + failed = True + raise + finally: self.crawlers.discard(crawler) self._active.discard(d) - self.bootstrap_failed |= not getattr(crawler, "spider", None) - return result + self.bootstrap_failed |= not getattr(crawler, "spider", None) or failed - return d.addBoth(_done) - - def create_crawler( - self, crawler_or_spidercls: Union[Type[Spider], str, Crawler] - ) -> Crawler: - """ - Return a :class:`~scrapy.crawler.Crawler` object. - - * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. - * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler - is constructed for it. - * If ``crawler_or_spidercls`` is a string, this function finds - a spider with this name in a Scrapy project (using spider loader), - then creates a Crawler instance for it. - """ - if isinstance(crawler_or_spidercls, Spider): - raise ValueError( - "The crawler_or_spidercls argument cannot be a spider object, " - "it must be a spider class (or a Crawler object)" - ) - if isinstance(crawler_or_spidercls, Crawler): - return crawler_or_spidercls - return self._create_crawler(crawler_or_spidercls) - - def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler: - if isinstance(spidercls, str): - spidercls = self.spider_loader.load(spidercls) - # temporary cast until self.spider_loader is typed - return Crawler(cast(Type[Spider], spidercls), self.settings) - - def stop(self) -> Deferred: + def stop(self) -> Deferred[Any]: """ Stops simultaneously all the crawling jobs taking place. Returns a deferred that is fired when they all have ended. """ - return DeferredList([c.stop() for c in list(self.crawlers)]) + return DeferredList(deferred_from_coro(c.stop_async()) for c in self.crawlers) @inlineCallbacks - def join(self) -> Generator[Deferred, Any, None]: + def join(self) -> Generator[Deferred[Any], Any, None]: """ join() @@ -353,7 +491,233 @@ class CrawlerRunner: yield DeferredList(self._active) -class CrawlerProcess(CrawlerRunner): +class AsyncCrawlerRunner(CrawlerRunnerBase): + """ + This is a convenient helper class that keeps track of, manages and runs + crawlers inside an already setup :mod:`~twisted.internet.reactor` or + asyncio event loop. + + The AsyncCrawlerRunner object must be instantiated with a + :class:`~scrapy.settings.Settings` object. + + When the :setting:`TWISTED_REACTOR_ENABLED` setting is set to ``True``, + this class requires a reactor to be installed and uses it, otherwise it + requires a reactor to not be installed but requires an asyncio event loop + to be installed and uses it. + + This class shouldn't be needed (since Scrapy is responsible of using it + accordingly) unless writing scripts that manually handle the crawling + process. See :ref:`run-from-script` for an example. + + This class provides coroutine APIs. It requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` when used + with a reactor. + """ + + def __init__(self, settings: dict[str, Any] | Settings | None = None): + super().__init__(settings) + self._active: set[asyncio.Task[None]] = set() + + def crawl( + self, + crawler_or_spidercls: type[Spider] | str | Crawler, + *args: Any, + **kwargs: Any, + ) -> asyncio.Task[None]: + """ + Run a crawler with the provided arguments. + + It will call the given Crawler's :meth:`~Crawler.crawl` method, while + keeping track of it so it can be stopped later. + + If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler` + instance, this method will try to create one using this parameter as + the spider class given to it. + + Returns a :class:`~asyncio.Task` object which completes when the + crawling is finished. + + :param crawler_or_spidercls: already created crawler, or a spider class + or spider's name inside the project to create it + :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance, + :class:`~scrapy.spiders.Spider` subclass or string + + :param args: arguments to initialize the spider + + :param kwargs: keyword arguments to initialize the spider + """ + if isinstance(crawler_or_spidercls, Spider): + raise ValueError( + "The crawler_or_spidercls argument cannot be a spider object, " + "it must be a spider class (or a Crawler object)" + ) + if self.settings.getbool("TWISTED_REACTOR_ENABLED"): + if not is_reactor_installed(): + raise RuntimeError( + "We expected a Twisted reactor to be installed but it isn't." + ) + if not is_asyncio_reactor_installed(): + raise RuntimeError( + f"When TWISTED_REACTOR_ENABLED is True, {type(self).__name__} " + f"requires that the installed Twisted reactor is " + f'"twisted.internet.asyncioreactor.AsyncioSelectorReactor".' + ) + elif is_reactor_installed(): + raise RuntimeError( + "TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed." + ) + crawler = self.create_crawler(crawler_or_spidercls) + return self._crawl(crawler, *args, **kwargs) + + async def _crawl_and_track( + self, crawler: Crawler, *args: Any, **kwargs: Any + ) -> None: + try: + await crawler.crawl_async(*args, **kwargs) + except Exception: + self.bootstrap_failed = True + raise # re-raise so asyncio still logs it to stderr naturally + + def _done(self, task: asyncio.Task[None], crawler: Crawler) -> None: + self._active.discard(task) + self.crawlers.discard(crawler) + self.bootstrap_failed |= not getattr(crawler, "spider", None) + + def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> asyncio.Task[None]: + # At this point the asyncio loop has been installed either by the user + # or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()). + loop = asyncio.get_event_loop() + self.crawlers.add(crawler) + + task = loop.create_task(self._crawl_and_track(crawler, *args, **kwargs)) + self._active.add(task) + task.add_done_callback(partial(self._done, crawler=crawler)) + + return task + + async def stop(self) -> None: + """ + Stops simultaneously all the crawling jobs taking place. + + Completes when they all have ended. + """ + if self.crawlers: + await asyncio.wait( + [asyncio.create_task(c.stop_async()) for c in self.crawlers] + ) + + async def join(self) -> None: + """ + Completes when all managed :attr:`crawlers` have completed their + executions. + """ + while self._active: + await asyncio.wait(self._active) + + +class CrawlerProcessBase(CrawlerRunnerBase): + def __init__( + self, + settings: dict[str, Any] | Settings | None = None, + install_root_handler: bool = True, + ): + super().__init__(settings) + configure_logging(self.settings, install_root_handler) + log_scrapy_info(self.settings) + + @abstractmethod + def start( + self, stop_after_crawl: bool = True, install_signal_handlers: bool = True + ) -> None: + raise NotImplementedError + + def _signal_shutdown(self, signum: int, _: Any) -> None: + from twisted.internet import reactor + + install_shutdown_handlers(self._signal_kill) + self._log_shutdown(signum) + reactor.callFromThread(self._graceful_stop_reactor) + + def _signal_kill(self, signum: int, _: Any) -> None: + from twisted.internet import reactor + + install_shutdown_handlers(signal.SIG_IGN) + self._log_kill(signum) + reactor.callFromThread(self._stop_reactor) + + @staticmethod + def _log_shutdown(signum: int) -> None: + signame = signal_names[signum] + logger.info( + "Received %(signame)s, shutting down gracefully. Send again to force ", + {"signame": signame}, + ) + + @staticmethod + def _log_kill(signum: int) -> None: + signame = signal_names[signum] + logger.info( + "Received %(signame)s twice, forcing unclean shutdown", {"signame": signame} + ) + + def _setup_reactor(self, install_signal_handlers: bool) -> None: + from twisted.internet import reactor + + dns_priority = self.settings.getpriority("DNS_RESOLVER") or 0 + default_priority = SETTINGS_PRIORITIES["default"] + + if dns_priority > default_priority: + warnings.warn( + "The DNS_RESOLVER setting is deprecated, please use " + "TWISTED_DNS_RESOLVER instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + + twisted_dns_priority = ( + self.settings.getpriority("TWISTED_DNS_RESOLVER") or 0 + ) + if twisted_dns_priority > dns_priority: + resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"] + else: + resolver_cls_path = self.settings["DNS_RESOLVER"] + else: + resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"] + + resolver_class = load_object(resolver_cls_path) + + # We pass self, which is CrawlerProcess, instead of Crawler here, + # which works because the default resolvers only use crawler.settings. + resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[call-overload] + resolver.install_on_reactor() + tp = reactor.getThreadPool() + tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) + reactor.addSystemEventTrigger("before", "shutdown", self._stop_dfd) + if install_signal_handlers: + reactor.addSystemEventTrigger( + "after", "startup", install_shutdown_handlers, self._signal_shutdown + ) + + @abstractmethod + def _stop_dfd(self) -> Deferred[Any]: + raise NotImplementedError + + @inlineCallbacks + def _graceful_stop_reactor(self) -> Generator[Deferred[Any], Any, None]: + try: + yield self._stop_dfd() + finally: + self._stop_reactor() + + def _stop_reactor(self, _: Any = None) -> None: + from twisted.internet import reactor + + # raised if already stopped or in shutdown stage + with contextlib.suppress(RuntimeError): + reactor.stop() + + +class CrawlerProcess(CrawlerProcessBase, CrawlerRunner): """ A class to run multiple scrapy crawlers in a process simultaneously. @@ -375,56 +739,37 @@ class CrawlerProcess(CrawlerRunner): This class shouldn't be needed (since Scrapy is responsible of using it accordingly) unless writing scripts that manually handle the crawling process. See :ref:`run-from-script` for an example. + + This class provides Deferred-based APIs. Use :class:`AsyncCrawlerProcess` + for modern coroutine APIs. """ def __init__( self, - settings: Union[Dict[str, Any], Settings, None] = None, + settings: dict[str, Any] | Settings | None = None, install_root_handler: bool = True, ): - super().__init__(settings) - configure_logging(self.settings, install_root_handler) - log_scrapy_info(self.settings) - self._initialized_reactor = False + super().__init__(settings, install_root_handler) + self._initialized_reactor: bool = False + logger.debug("Using CrawlerProcess") - def _signal_shutdown(self, signum: int, _: Any) -> None: - from twisted.internet import reactor - - install_shutdown_handlers(self._signal_kill) - signame = signal_names[signum] - logger.info( - "Received %(signame)s, shutting down gracefully. Send again to force ", - {"signame": signame}, - ) - reactor.callFromThread(self._graceful_stop_reactor) - - def _signal_kill(self, signum: int, _: Any) -> None: - from twisted.internet import reactor - - install_shutdown_handlers(signal.SIG_IGN) - signame = signal_names[signum] - logger.info( - "Received %(signame)s twice, forcing unclean shutdown", {"signame": signame} - ) - reactor.callFromThread(self._stop_reactor) - - def _create_crawler(self, spidercls: Union[Type[Spider], str]) -> Crawler: + def _create_crawler(self, spidercls: type[Spider] | str) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) init_reactor = not self._initialized_reactor self._initialized_reactor = True - # temporary cast until self.spider_loader is typed - return Crawler( - cast(Type[Spider], spidercls), self.settings, init_reactor=init_reactor - ) + return Crawler(spidercls, self.settings, init_reactor=init_reactor) + + def _stop_dfd(self) -> Deferred[Any]: + return self.stop() def start( self, stop_after_crawl: bool = True, install_signal_handlers: bool = True ) -> None: """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool - size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache - based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. + size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS + resolver based on :setting:`DNSCACHE_ENABLED`. If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. @@ -444,27 +789,262 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) - resolver_class = load_object(self.settings["DNS_RESOLVER"]) - resolver = build_from_crawler(resolver_class, self, reactor=reactor) - resolver.install_on_reactor() - tp = reactor.getThreadPool() - tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) - reactor.addSystemEventTrigger("before", "shutdown", self.stop) - if install_signal_handlers: - reactor.addSystemEventTrigger( - "after", "startup", install_shutdown_handlers, self._signal_shutdown - ) + self._setup_reactor(install_signal_handlers) reactor.run(installSignalHandlers=install_signal_handlers) # blocking call - def _graceful_stop_reactor(self) -> Deferred: - d = self.stop() - d.addBoth(self._stop_reactor) - return d - def _stop_reactor(self, _: Any = None) -> None: - from twisted.internet import reactor +class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): + """ + A class to run multiple scrapy crawlers in a process simultaneously. + + This class extends :class:`~scrapy.crawler.AsyncCrawlerRunner` by adding support + for starting a :mod:`~twisted.internet.reactor` and handling shutdown + signals, like the keyboard interrupt command Ctrl-C. It also configures + top-level logging. + + This utility should be a better fit than + :class:`~scrapy.crawler.AsyncCrawlerRunner` if you aren't running another + :mod:`~twisted.internet.reactor` within your application. + + The AsyncCrawlerProcess object must be instantiated with a + :class:`~scrapy.settings.Settings` object. + + When the :setting:`TWISTED_REACTOR_ENABLED` setting is set to ``True``, + this class installs a reactor and uses it, otherwise it requires a reactor + to not be installed but installs an asyncio event loop and uses it. + + :param install_root_handler: whether to install root logging handler + (default: True) + + This class shouldn't be needed (since Scrapy is responsible of using it + accordingly) unless writing scripts that manually handle the crawling + process. See :ref:`run-from-script` for an example. + + This class provides coroutine APIs. It requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` when used + with a reactor. + """ + + def __init__( + self, + settings: dict[str, Any] | Settings | None = None, + install_root_handler: bool = True, + ): + super().__init__(settings, install_root_handler) + logger.debug("Using AsyncCrawlerProcess") + self._reactorless_loop: asyncio.AbstractEventLoop | None = None + # We want the asyncio event loop to be installed early, so that it's + # always the correct one. And as we do that, we can also install the + # reactor here. + # The ASYNCIO_EVENT_LOOP setting cannot be overridden by add-ons and + # spiders when using AsyncCrawlerProcess. + loop_path = self.settings["ASYNCIO_EVENT_LOOP"] + if not self.settings.getbool("TWISTED_REACTOR_ENABLED"): + if is_reactor_installed(): + raise RuntimeError( + "TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed." + ) + self._reactorless_loop = set_asyncio_event_loop(loop_path) + install_reactor_import_hook() + elif is_reactor_installed(): + # The user could install a reactor before this class is instantiated. + # We need to make sure the reactor is the correct one and the loop + # type matches the setting. + verify_installed_reactor(_asyncio_reactor_path) + if loop_path: + verify_installed_asyncio_event_loop(loop_path) + else: + install_reactor(_asyncio_reactor_path, loop_path) + self._initialized_reactor = True + self._reactorless_main_task: asyncio.Future[None] | None = None + + def _stop_dfd(self) -> Deferred[Any]: + return deferred_from_coro(self.stop()) + + def start( + self, stop_after_crawl: bool = True, install_signal_handlers: bool = True + ) -> None: + """ + This method starts a :mod:`~twisted.internet.reactor` or an asyncio + event loop, depending on the value of the + :setting:`TWISTED_REACTOR_ENABLED` setting. + + When using a reactor it adjusts its pool size to + :setting:`REACTOR_THREADPOOL_MAXSIZE` and installs a DNS resolver based + on :setting:`DNSCACHE_ENABLED`. + + If ``stop_after_crawl`` is True, the reactor will be stopped after all + crawlers have finished, using :meth:`join`. + + :param bool stop_after_crawl: stop or not the reactor when all + crawlers have finished + + :param bool install_signal_handlers: whether to install the OS signal + handlers from Twisted and Scrapy (default: True) + """ + + if not self.settings.getbool("TWISTED_REACTOR_ENABLED"): + self._start_asyncio(stop_after_crawl, install_signal_handlers) + else: + self._start_twisted(stop_after_crawl, install_signal_handlers) + + def _start_asyncio( + self, stop_after_crawl: bool, install_signal_handlers: bool + ) -> None: + # We cannot use asyncio.run() here, because we can't let it handle the + # loop lifetime: _crawl() needs a loop (which we create in __init__()), + # because crawl() returns a Task. + # So we reproduce a part of asyncio.runners.Runner that is useful to us. + + # Normal workflow: + # 1. _start_asyncio() creates a task for self.join() and calls _run_loop() + # 2. _run_loop() calls loop.run_until_complete(main_task) + # 3. Crawling tasks start and finish + # 4. join() completes, loop.run_until_complete() and thus _run_loop() return + # 5. _start_asyncio() calls _close_loop() + # 6. _close_loop() does finalization and calls loop.close() + + # Normal workflow with stop_after_crawl=False: + # 1. _start_asyncio() creates a simple future and calls _run_loop() + # 2. _run_loop() calls loop.run_until_complete(main_task) + # 3. Crawling tasks start and finish + # 4. _run_loop() blocks until the loop is stopped externally or the + # future is cancelled via Ctrl-C + # 5. (after _run_loop() returns) _start_asyncio() calls _close_loop() + # 6. _close_loop() does finalization and calls loop.close() + + # Workflow with Ctrl-C pressed once: + # 1. While loop.run_until_complete() blocks, _signal_shutdown_reactorless() + # is called + # 2. _signal_shutdown_reactorless() calls _shutdown_graceful_reactorless() + # (via call_soon_threadsafe()) + # 3. _shutdown_graceful_reactorless() calls stop() + # 4. For stop_after_crawl=True: crawl tasks finish, join() completes, + # loop.run_until_complete() and thus _run_loop() return + # For stop_after_crawl=False: _shutdown_graceful_reactorless() waits + # for crawl tasks via join(), then cancels the main task, + # loop.run_until_complete() raises CancelledError, _run_loop() returns + # 5. _start_asyncio() calls _close_loop() + # 6. _close_loop() does finalization and calls loop.close() + + # Workflow with Ctrl-C pressed twice: + # 1. While loop.run_until_complete() blocks, _signal_shutdown_reactorless() + # is called + # 2. _signal_shutdown_reactorless() calls _shutdown_graceful_reactorless() + # (via call_soon_threadsafe()) and installs _signal_kill_reactorless() + # as the next handler + # 3. Before _shutdown_graceful_reactorless() completes, + # _signal_kill_reactorless() is called + # 4. _signal_kill_reactorless() cancels the main task + # (via call_soon_threadsafe()) + # 5. loop.run_until_complete() raises CancelledError, _run_loop() returns + # 6. _start_asyncio() calls _close_loop() + # 7. _close_loop() cancels all pending tasks (including + # _shutdown_graceful_reactorless()), does finalization and calls loop.close() + + loop = self._reactorless_loop + assert loop + + if stop_after_crawl: + self._reactorless_main_task = loop.create_task(self.join()) + else: + self._reactorless_main_task = loop.create_future() + self._stop_after_crawl = stop_after_crawl try: - reactor.stop() - except RuntimeError: # raised if already stopped or in shutdown stage + self._run_loop(install_signal_handlers) # blocking call + except asyncio.CancelledError: pass + finally: + self._close_loop() + + def _run_loop(self, install_signal_handlers: bool) -> None: + # similar to asyncio.runners.Runner.run() + if install_signal_handlers: + install_shutdown_handlers(self._signal_shutdown_reactorless) + assert self._reactorless_loop + assert self._reactorless_main_task + self._reactorless_loop.run_until_complete(self._reactorless_main_task) + + def _close_loop(self) -> None: + # Similar to asyncio.runners.Runner.close() + loop = self._reactorless_loop + assert loop + try: + self._cancel_all_tasks(loop) + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.run_until_complete(loop.shutdown_default_executor()) + finally: + self._reactorless_main_task = None + asyncio.set_event_loop(None) + loop.close() + self._reactorless_loop = None + + @staticmethod + def _cancel_all_tasks(loop: asyncio.AbstractEventLoop) -> None: + # copy of asyncio.runners._cancel_all_tasks() + to_cancel = asyncio.all_tasks(loop) + if not to_cancel: + return + + for task in to_cancel: + task.cancel() + + loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True)) + + for task in to_cancel: + if task.cancelled(): + continue + if task.exception() is not None: + loop.call_exception_handler( + { + "message": "unhandled exception during AsyncCrawlerProcess shutdown", + "exception": task.exception(), + "task": task, + } + ) + + def _signal_shutdown_reactorless(self, signum: int, _: Any) -> None: + install_shutdown_handlers(self._signal_kill_reactorless) + self._log_shutdown(signum) + if (loop := self._reactorless_loop) is None: + return + + loop.call_soon_threadsafe(self._create_shutdown_task) + + def _create_shutdown_task(self) -> None: + assert self._reactorless_loop + coro = self._shutdown_graceful_reactorless() + try: + self._reactorless_loop.create_task(coro) + except RuntimeError: + coro.close() + + async def _shutdown_graceful_reactorless(self) -> None: + await self.stop() + if not self._stop_after_crawl: + # wait until crawl tasks finish and cancel the future + await self.join() + if self._reactorless_main_task and not self._reactorless_main_task.done(): + self._reactorless_main_task.cancel() + + def _signal_kill_reactorless(self, signum: int, _: Any) -> None: + install_shutdown_handlers(signal.SIG_IGN) + self._log_kill(signum) + if (loop := self._reactorless_loop) is None: + return + if (task := self._reactorless_main_task) is not None: + loop.call_soon_threadsafe(task.cancel) + + def _start_twisted( + self, stop_after_crawl: bool, install_signal_handlers: bool + ) -> None: + from twisted.internet import reactor + + if stop_after_crawl: + loop = asyncio.get_event_loop() + join_task = loop.create_task(self.join()) + join_task.add_done_callback(self._stop_reactor) + + self._setup_reactor(install_signal_handlers) + reactor.run(installSignalHandlers=install_signal_handlers) # blocking call diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py deleted file mode 100644 index 0e757e4be..000000000 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ /dev/null @@ -1,107 +0,0 @@ -from __future__ import annotations - -import logging -import re -from typing import TYPE_CHECKING, Union - -from w3lib import html - -from scrapy import Request, Spider -from scrapy.crawler import Crawler -from scrapy.exceptions import NotConfigured -from scrapy.http import HtmlResponse, Response -from scrapy.settings import BaseSettings - -if TYPE_CHECKING: - # typing.Self requires Python 3.11 - from typing_extensions import Self - -logger = logging.getLogger(__name__) - - -class AjaxCrawlMiddleware: - """ - Handle 'AJAX crawlable' pages marked as crawlable via meta tag. - For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. - """ - - def __init__(self, settings: BaseSettings): - if not settings.getbool("AJAXCRAWL_ENABLED"): - raise NotConfigured - - # XXX: Google parses at least first 100k bytes; scrapy's redirect - # middleware parses first 4k. 4k turns out to be insufficient - # for this middleware, and parsing 100k could be slow. - # We use something in between (32K) by default. - self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE", 32768) - - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - return cls(crawler.settings) - - def process_response( - self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: - if not isinstance(response, HtmlResponse) or response.status != 200: - return response - - if request.method != "GET": - # other HTTP methods are either not safe or don't have a body - return response - - if "ajax_crawlable" in request.meta: # prevent loops - return response - - if not self._has_ajax_crawlable_variant(response): - return response - - # scrapy already handles #! links properly - ajax_crawl_request = request.replace(url=request.url + "#!") - logger.debug( - "Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s", - {"ajax_crawl_request": ajax_crawl_request, "request": request}, - extra={"spider": spider}, - ) - - ajax_crawl_request.meta["ajax_crawlable"] = True - return ajax_crawl_request - - def _has_ajax_crawlable_variant(self, response: Response) -> bool: - """ - Return True if a page without hash fragment could be "AJAX crawlable" - according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. - """ - body = response.text[: self.lookup_bytes] - return _has_ajaxcrawlable_meta(body) - - -# XXX: move it to w3lib? -_ajax_crawlable_re: re.Pattern[str] = re.compile( - r'' -) - - -def _has_ajaxcrawlable_meta(text: str) -> bool: - """ - >>> _has_ajaxcrawlable_meta('') - True - >>> _has_ajaxcrawlable_meta("") - True - >>> _has_ajaxcrawlable_meta('') - False - >>> _has_ajaxcrawlable_meta('') - False - """ - - # Stripping scripts and comments is slow (about 20x slower than - # just checking if a string is in text); this is a quick fail-fast - # path that should work for most pages. - if "fragment" not in text: - return False - if "content" not in text: - return False - - text = html.remove_tags_with_content(text, ("script", "noscript")) - text = html.replace_entities(text) - text = html.remove_comments(text) - return _ajax_crawlable_re.search(text) is not None diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 85781efd6..cd8c2abca 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -2,37 +2,34 @@ from __future__ import annotations import logging from collections import defaultdict -from http.cookiejar import Cookie -from typing import ( - TYPE_CHECKING, - Any, - DefaultDict, - Dict, - Iterable, - Optional, - Sequence, - Union, -) +from typing import TYPE_CHECKING, Any from tldextract import TLDExtract -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + from http.cookiejar import Cookie + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http.request import VerboseCookie + logger = logging.getLogger(__name__) _split_domain = TLDExtract(include_psl_private_domains=True) +_UNSET = object() def _is_public_domain(domain: str) -> bool: @@ -43,23 +40,26 @@ def _is_public_domain(domain: str) -> bool: class CookiesMiddleware: """This middleware enables working with sites that need cookies""" + crawler: Crawler + def __init__(self, debug: bool = False): - self.jars: DefaultDict[Any, CookieJar] = defaultdict(CookieJar) + self.jars: defaultdict[Any, CookieJar] = defaultdict(CookieJar) self.debug: bool = debug @classmethod def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("COOKIES_ENABLED"): raise NotConfigured - return cls(crawler.settings.getbool("COOKIES_DEBUG")) + o = cls(crawler.settings.getbool("COOKIES_DEBUG")) + o.crawler = crawler + return o def _process_cookies( self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request ) -> None: for cookie in cookies: cookie_domain = cookie.domain - if cookie_domain.startswith("."): - cookie_domain = cookie_domain[1:] + cookie_domain = cookie_domain.removeprefix(".") hostname = urlparse_cached(request).hostname assert hostname is not None @@ -72,9 +72,10 @@ class CookiesMiddleware: jar.set_cookie_if_ok(cookie, request) + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + self, request: Request, spider: Spider | None = None + ) -> Request | Response | None: if request.meta.get("dont_merge_cookies", False): return None @@ -86,12 +87,13 @@ class CookiesMiddleware: # set Cookie header request.headers.pop("Cookie", None) jar.add_cookie_header(request) - self._debug_cookie(request, spider) + self._debug_cookie(request) return None + @_warn_spider_arg def process_response( - self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + self, request: Request, response: Response, spider: Spider | None = None + ) -> Request | Response: if request.meta.get("dont_merge_cookies", False): return response @@ -101,11 +103,11 @@ class CookiesMiddleware: cookies = jar.make_cookies(response, request) self._process_cookies(cookies, jar=jar, request=request) - self._debug_set_cookie(response, spider) + self._debug_set_cookie(response) return response - def _debug_cookie(self, request: Request, spider: Spider) -> None: + def _debug_cookie(self, request: Request) -> None: if self.debug: cl = [ to_unicode(c, errors="replace") @@ -114,9 +116,9 @@ class CookiesMiddleware: if cl: cookies = "\n".join(f"Cookie: {c}\n" for c in cl) msg = f"Sending cookies to: {request}\n{cookies}" - logger.debug(msg, extra={"spider": spider}) + logger.debug(msg, extra={"spider": self.crawler.spider}) - def _debug_set_cookie(self, response: Response, spider: Spider) -> None: + def _debug_set_cookie(self, response: Response) -> None: if self.debug: cl = [ to_unicode(c, errors="replace") @@ -125,37 +127,46 @@ class CookiesMiddleware: if cl: cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl) msg = f"Received cookies from: {response}\n{cookies}" - logger.debug(msg, extra={"spider": spider}) + logger.debug(msg, extra={"spider": self.crawler.spider}) - def _format_cookie(self, cookie: Dict[str, Any], request: Request) -> Optional[str]: + def _format_cookie(self, cookie: VerboseCookie, request: Request) -> str | None: """ Given a dict consisting of cookie components, return its string representation. Decode from bytes if necessary. """ decoded = {} + flags = set() for key in ("name", "value", "path", "domain"): - if cookie.get(key) is None: - if key in ("name", "value"): + value = cookie.get(key) + if value is None: + if key in {"name", "value"}: msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)" logger.warning(msg) return None continue - if isinstance(cookie[key], (bool, float, int, str)): - decoded[key] = str(cookie[key]) + if isinstance(value, (bool, float, int, str)): + decoded[key] = str(value) else: + assert isinstance(value, bytes) try: - decoded[key] = cookie[key].decode("utf8") + decoded[key] = value.decode("utf8") except UnicodeDecodeError: logger.warning( "Non UTF-8 encoded cookie found in request %s: %s", request, cookie, ) - decoded[key] = cookie[key].decode("latin1", errors="replace") - + decoded[key] = value.decode("latin1", errors="replace") + for flag in ("secure",): + value = cookie.get(flag, _UNSET) + if value is _UNSET or not value: + continue + flags.add(flag) cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}" for key, value in decoded.items(): # path, domain cookie_str += f"; {key.capitalize()}={value}" + for flag in flags: # secure + cookie_str += f"; {flag.capitalize()}" return cookie_str def _get_request_cookies( @@ -165,12 +176,14 @@ class CookiesMiddleware: Extract cookies from the Request.cookies attribute """ if not request.cookies: - return [] - cookies: Iterable[Dict[str, Any]] + return () + cookies: Iterable[VerboseCookie] if isinstance(request.cookies, dict): - cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) + cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items()) else: cookies = request.cookies + for cookie in cookies: + cookie.setdefault("secure", urlparse_cached(request).scheme == "https") formatted = filter(None, (self._format_cookie(c, request) for c in cookies)) response = Response(request.url, headers={"Set-Cookie": formatted}) return jar.make_cookies(response, request) diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 8aec37cf1..7c940a078 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -3,32 +3,38 @@ DefaultHeaders downloader middleware See documentation in docs/topics/downloader-middleware.rst """ + from __future__ import annotations -from typing import TYPE_CHECKING, Iterable, Tuple, Union +from typing import TYPE_CHECKING -from scrapy import Request, Spider -from scrapy.crawler import Crawler -from scrapy.http import Response +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.python import without_none_values if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + class DefaultHeadersMiddleware: - def __init__(self, headers: Iterable[Tuple[str, str]]): - self._headers: Iterable[Tuple[str, str]] = headers + def __init__(self, headers: Iterable[tuple[str, str]]): + self._headers: Iterable[tuple[str, str]] = headers @classmethod def from_crawler(cls, crawler: Crawler) -> Self: headers = without_none_values(crawler.settings["DEFAULT_REQUEST_HEADERS"]) return cls(headers.items()) + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + self, request: Request, spider: Spider | None = None + ) -> Request | Response | None: for k, v in self._headers: request.headers.setdefault(k, v) return None diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index 1c904c05b..bccfb230c 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -3,18 +3,22 @@ Download timeout middleware See documentation in docs/topics/downloader-middleware.rst """ + from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler -from scrapy.http import Response +from scrapy.utils.decorators import _warn_spider_arg +from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + class DownloadTimeoutMiddleware: def __init__(self, timeout: float = 180): @@ -27,11 +31,14 @@ class DownloadTimeoutMiddleware: return o def spider_opened(self, spider: Spider) -> None: + if hasattr(spider, "download_timeout"): # pragma: no cover + warn_on_deprecated_spider_attribute("download_timeout", "DOWNLOAD_TIMEOUT") self._timeout = getattr(spider, "download_timeout", self._timeout) + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + self, request: Request, spider: Spider | None = None + ) -> Request | Response | None: if self._timeout: request.meta.setdefault("download_timeout", self._timeout) return None diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 63490a37a..c28c93d4e 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -6,19 +6,21 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from w3lib.http import basic_auth_header from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler -from scrapy.http import Response +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.url import url_is_from_any_domain if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + class HttpAuthMiddleware: """Set Basic HTTP Authorization header @@ -37,11 +39,15 @@ class HttpAuthMiddleware: self.auth = basic_auth_header(usr, pwd) self.domain = spider.http_auth_domain # type: ignore[attr-defined] + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + self, request: Request, spider: Spider | None = None + ) -> Request | Response | None: auth = getattr(self, "auth", None) - if auth and b"Authorization" not in request.headers: - if not self.domain or url_is_from_any_domain(request.url, [self.domain]): - request.headers[b"Authorization"] = auth + if ( + auth + and b"Authorization" not in request.headers + and (not self.domain or url_is_from_any_domain(request.url, [self.domain])) + ): + request.headers[b"Authorization"] = auth return None diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 971473403..c6c811809 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,49 +1,46 @@ from __future__ import annotations from email.utils import formatdate -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING -from twisted.internet import defer -from twisted.internet.error import ( - ConnectError, - ConnectionDone, - ConnectionLost, - ConnectionRefusedError, - DNSLookupError, - TCPTimedOutError, - TimeoutError, -) -from twisted.web.client import ResponseFailed +from twisted.internet.error import ConnectError, ConnectionDone, ConnectionLost from scrapy import signals -from scrapy.crawler import Crawler -from scrapy.exceptions import IgnoreRequest, NotConfigured -from scrapy.http.request import Request -from scrapy.http.response import Response -from scrapy.settings import Settings -from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector +from scrapy.exceptions import ( + DownloadConnectionRefusedError, + DownloadFailedError, + DownloadTimeoutError, + IgnoreRequest, + NotConfigured, +) +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.misc import load_object if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http.request import Request + from scrapy.http.response import Response + from scrapy.settings import Settings + from scrapy.spiders import Spider + from scrapy.statscollectors import StatsCollector + class HttpCacheMiddleware: DOWNLOAD_EXCEPTIONS = ( - defer.TimeoutError, - TimeoutError, - DNSLookupError, - ConnectionRefusedError, ConnectionDone, ConnectError, ConnectionLost, - TCPTimedOutError, - ResponseFailed, OSError, + DownloadTimeoutError, + DownloadConnectionRefusedError, + DownloadFailedError, ) + crawler: Crawler + def __init__(self, settings: Settings, stats: StatsCollector) -> None: if not settings.getbool("HTTPCACHE_ENABLED"): raise NotConfigured @@ -58,6 +55,7 @@ class HttpCacheMiddleware: o = cls(crawler.settings, crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) + o.crawler = crawler return o def spider_opened(self, spider: Spider) -> None: @@ -66,9 +64,10 @@ class HttpCacheMiddleware: def spider_closed(self, spider: Spider) -> None: self.storage.close_spider(spider) + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + self, request: Request, spider: Spider | None = None + ) -> Request | Response | None: if request.meta.get("dont_cache", False): return None @@ -78,20 +77,20 @@ class HttpCacheMiddleware: return None # Look for cached response and check if expired - cachedresponse: Optional[Response] = self.storage.retrieve_response( - spider, request + cachedresponse: Response | None = self.storage.retrieve_response( + self.crawler.spider, request ) if cachedresponse is None: - self.stats.inc_value("httpcache/miss", spider=spider) + self.stats.inc_value("httpcache/miss") if self.ignore_missing: - self.stats.inc_value("httpcache/ignore", spider=spider) + self.stats.inc_value("httpcache/ignore") raise IgnoreRequest(f"Ignored request not in cache: {request}") return None # first time request # Return cached response only if not expired cachedresponse.flags.append("cached") if self.policy.is_cached_response_fresh(cachedresponse, request): - self.stats.inc_value("httpcache/hit", spider=spider) + self.stats.inc_value("httpcache/hit") return cachedresponse # Keep a reference to cached response to avoid a second cache lookup on @@ -100,14 +99,15 @@ class HttpCacheMiddleware: return None + @_warn_spider_arg def process_response( - self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + self, request: Request, response: Response, spider: Spider | None = None + ) -> Request | Response: if request.meta.get("dont_cache", False): return response # Skip cached responses and uncacheable requests - if "cached" in response.flags or "_dont_cache" in request.meta: + if "_dont_cache" in request.meta or "cached" in response.flags: request.meta.pop("_dont_cache", None) return response @@ -117,40 +117,35 @@ class HttpCacheMiddleware: response.headers["Date"] = formatdate(usegmt=True) # Do not validate first-hand responses - cachedresponse: Optional[Response] = request.meta.pop("cached_response", None) + cachedresponse: Response | None = request.meta.pop("cached_response", None) if cachedresponse is None: - self.stats.inc_value("httpcache/firsthand", spider=spider) - self._cache_response(spider, response, request, cachedresponse) + self.stats.inc_value("httpcache/firsthand") + self._cache_response(response, request) return response if self.policy.is_cached_response_valid(cachedresponse, response, request): - self.stats.inc_value("httpcache/revalidate", spider=spider) + self.stats.inc_value("httpcache/revalidate") return cachedresponse - self.stats.inc_value("httpcache/invalidate", spider=spider) - self._cache_response(spider, response, request, cachedresponse) + self.stats.inc_value("httpcache/invalidate") + self._cache_response(response, request) return response + @_warn_spider_arg def process_exception( - self, request: Request, exception: Exception, spider: Spider - ) -> Union[Request, Response, None]: - cachedresponse: Optional[Response] = request.meta.pop("cached_response", None) + self, request: Request, exception: Exception, spider: Spider | None = None + ) -> Request | Response | None: + cachedresponse: Response | None = request.meta.pop("cached_response", None) if cachedresponse is not None and isinstance( exception, self.DOWNLOAD_EXCEPTIONS ): - self.stats.inc_value("httpcache/errorrecovery", spider=spider) + self.stats.inc_value("httpcache/errorrecovery") return cachedresponse return None - def _cache_response( - self, - spider: Spider, - response: Response, - request: Request, - cachedresponse: Optional[Response], - ) -> None: + def _cache_response(self, response: Response, request: Request) -> None: if self.policy.should_cache_response(response, request): - self.stats.inc_value("httpcache/store", spider=spider) - self.storage.store_response(spider, request, response) + self.stats.inc_value("httpcache/store") + self.storage.store_response(self.crawler.spider, request, response) else: - self.stats.inc_value("httpcache/uncacheable", spider=spider) + self.stats.inc_value("httpcache/uncacheable") diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 0e53087ca..2417c6881 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,14 +1,21 @@ from __future__ import annotations -import io -import zlib -from typing import TYPE_CHECKING, List, Optional, Union +import warnings +from itertools import chain +from logging import getLogger +from typing import TYPE_CHECKING, Any -from scrapy import Request, Spider -from scrapy.crawler import Crawler -from scrapy.exceptions import NotConfigured +from scrapy import Request, Spider, signals +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Response, TextResponse -from scrapy.statscollectors import StatsCollector +from scrapy.utils._compression import ( + _DecompressionMaxSizeExceeded, + _inflate, + _unbrotli, + _unzstd, +) +from scrapy.utils.decorators import _warn_spider_arg +from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute from scrapy.utils.gz import gunzip from scrapy.utils.response import get_response_class @@ -16,99 +23,189 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self -ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"] + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + +logger = getLogger(__name__) + +ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate"] try: - import brotli - - ACCEPTED_ENCODINGS.append(b"br") + try: + import brotli + except ImportError: + import brotlicffi as brotli except ImportError: pass +else: + try: + brotli.Decompressor.can_accept_more_data # noqa: B018 + except AttributeError: # pragma: no cover + warnings.warn( + "You have brotli installed. But 'br' encoding support now requires " + "brotli's or brotlicffi's version >= 1.2.0. Please upgrade " + "brotli/brotlicffi to make Scrapy decode 'br' encoded responses.", + stacklevel=2, + ) + else: + ACCEPTED_ENCODINGS.append(b"br") try: - import zstandard - + import zstandard # noqa: F401 +except ImportError: + pass +else: ACCEPTED_ENCODINGS.append(b"zstd") -except ImportError: - pass class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be - sent/received from web sites""" + sent/received from websites""" - def __init__(self, stats: Optional[StatsCollector] = None): - self.stats = stats + def __init__( + self, + stats: StatsCollector | None = None, + *, + crawler: Crawler | None = None, + ): + if not crawler: + self.stats = stats + self._max_size = 1073741824 + self._warn_size = 33554432 + return + self.stats = crawler.stats + self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") + self._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE") + crawler.signals.connect(self.open_spider, signals.spider_opened) @classmethod def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("COMPRESSION_ENABLED"): raise NotConfigured - return cls(stats=crawler.stats) + return cls(crawler=crawler) + def open_spider(self, spider: Spider) -> None: + if hasattr(spider, "download_maxsize"): + warn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE") + self._max_size = spider.download_maxsize + if hasattr(spider, "download_warnsize"): + warn_on_deprecated_spider_attribute( + "download_warnsize", "DOWNLOAD_WARNSIZE" + ) + self._warn_size = spider.download_warnsize + + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + self, request: Request, spider: Spider | None = None + ) -> Request | Response | None: request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS)) return None + @_warn_spider_arg def process_response( - self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: - if ( - request.method == "HEAD" - or not isinstance(response, Response) - or "Content-Encoding" not in response.headers - ): + self, request: Request, response: Response, spider: Spider | None = None + ) -> Request | Response: + if request.method == "HEAD": return response - header_list = response.headers.getlist("Content-Encoding") - encodings = [item.strip() for item in b",".join(header_list).split(b",")] - if not encodings: - return response - while encodings: - encoding = encodings.pop() - decoded_body = self._decode(response.body, encoding.lower()) - if encodings: - response.headers["Content-Encoding"] = b",".join(encodings) - else: - del response.headers["Content-Encoding"] - respcls = get_response_class( - http_headers=response.headers, - url=response.url, - body=decoded_body, - ) - kwargs = dict(cls=respcls, body=decoded_body) - if issubclass(respcls, TextResponse): - # Force recalculating the encoding based on the new, - # decoded (uncompressed) body. - kwargs["encoding"] = None - response = response.replace(**kwargs) - if self.stats: - self.stats.inc_value( - "httpcompression/response_bytes", len(decoded_body), spider=spider - ) - self.stats.inc_value("httpcompression/response_count", spider=spider) + if isinstance(response, Response): + content_encoding = response.headers.getlist("Content-Encoding") + if content_encoding: + max_size = request.meta.get("download_maxsize", self._max_size) + warn_size = request.meta.get("download_warnsize", self._warn_size) + try: + decoded_body, content_encoding = self._handle_encoding( + response.body, content_encoding, max_size + ) + except _DecompressionMaxSizeExceeded as e: + raise IgnoreRequest( + f"Ignored response {response} because its body " + f"({len(response.body)} B compressed, " + f"{e.decompressed_size} B decompressed so far) exceeded " + f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression." + ) from e + if len(response.body) < warn_size <= len(decoded_body): + logger.warning( + f"{response} body size after decompression " + f"({len(decoded_body)} B) is larger than the " + f"download warning size ({warn_size} B)." + ) + if content_encoding: + self._warn_unknown_encoding(response, content_encoding) + response.headers["Content-Encoding"] = content_encoding + if self.stats: + self.stats.inc_value( + "httpcompression/response_bytes", + len(decoded_body), + ) + self.stats.inc_value("httpcompression/response_count") + respcls = get_response_class( + http_headers=response.headers, + url=response.url, + body=decoded_body, + ) + kwargs: dict[str, Any] = {"body": decoded_body} + if issubclass(respcls, TextResponse): + # force recalculating the encoding until we make sure the + # responsetypes guessing is reliable + kwargs["encoding"] = None + response = response.replace(cls=respcls, **kwargs) + if not content_encoding: + del response.headers["Content-Encoding"] + return response - def _decode(self, body: bytes, encoding: bytes) -> bytes: - if encoding == b"gzip" or encoding == b"x-gzip": - body = gunzip(body) + def _handle_encoding( + self, body: bytes, content_encoding: list[bytes], max_size: int + ) -> tuple[bytes, list[bytes]]: + to_decode, to_keep = self._split_encodings(content_encoding) + for encoding in to_decode: + body = self._decode(body, encoding, max_size) + return body, to_keep + @staticmethod + def _split_encodings( + content_encoding: list[bytes], + ) -> tuple[list[bytes], list[bytes]]: + supported_encodings = {*ACCEPTED_ENCODINGS, b"x-gzip"} + to_keep: list[bytes] = [ + encoding.strip().lower() + for encoding in chain.from_iterable( + encodings.split(b",") for encodings in content_encoding + ) + ] + to_decode: list[bytes] = [] + while to_keep: + encoding = to_keep.pop() + if encoding not in supported_encodings: + to_keep.append(encoding) + return to_decode, to_keep + to_decode.append(encoding) + return to_decode, to_keep + + @staticmethod + def _decode(body: bytes, encoding: bytes, max_size: int) -> bytes: + if encoding in {b"gzip", b"x-gzip"}: + return gunzip(body, max_size=max_size) if encoding == b"deflate": - try: - body = zlib.decompress(body) - except zlib.error: - # ugly hack to work with raw deflate content that may - # be sent by microsoft servers. For more information, see: - # http://carsten.codimi.de/gzip.yaws/ - # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx - # http://www.gzip.org/zlib/zlib_faq.html#faq38 - body = zlib.decompress(body, -15) - if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS: - body = brotli.decompress(body) - if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS: - # Using its streaming API since its simple API could handle only cases - # where there is content size data embedded in the frame - reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body)) - body = reader.read() - return body + return _inflate(body, max_size=max_size) + if encoding == b"br": + return _unbrotli(body, max_size=max_size) + if encoding == b"zstd": + return _unzstd(body, max_size=max_size) + # shouldn't be reached + return body # pragma: no cover + + def _warn_unknown_encoding( + self, response: Response, encodings: list[bytes] + ) -> None: + encodings_str = b",".join(encodings).decode() + msg = ( + f"{self.__class__.__name__} cannot decode the response for {response.url} " + f"from unsupported encoding(s) '{encodings_str}'." + ) + if b"br" in encodings: + msg += " You need to install brotli or brotlicffi >= 1.2.0 to decode 'br'." + if b"zstd" in encodings: + msg += " You need to install zstandard to decode 'zstd'." + logger.warning(msg) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 335896ac1..e034dced1 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,7 +1,7 @@ from __future__ import annotations import base64 -from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union +from typing import TYPE_CHECKING from urllib.parse import unquote, urlunparse from urllib.request import ( # type: ignore[attr-defined] _parse_proxy, @@ -9,10 +9,8 @@ from urllib.request import ( # type: ignore[attr-defined] proxy_bypass, ) -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes @@ -20,11 +18,15 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + class HttpProxyMiddleware: - def __init__(self, auth_encoding: Optional[str] = "latin-1"): - self.auth_encoding: Optional[str] = auth_encoding - self.proxies: Dict[str, Tuple[Optional[bytes], str]] = {} + def __init__(self, auth_encoding: str | None = "latin-1"): + self.auth_encoding: str | None = auth_encoding + self.proxies: dict[str, tuple[bytes | None, str]] = {} for type_, url in getproxies().items(): try: self.proxies[type_] = self._get_proxy(url, type_) @@ -37,7 +39,7 @@ class HttpProxyMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("HTTPPROXY_ENABLED"): raise NotConfigured - auth_encoding: Optional[str] = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") + auth_encoding: str | None = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") return cls(auth_encoding) def _basic_auth_header(self, username: str, password: str) -> bytes: @@ -46,40 +48,45 @@ class HttpProxyMiddleware: ) return base64.b64encode(user_pass) - def _get_proxy(self, url: str, orig_type: str) -> Tuple[Optional[bytes], str]: + def _get_proxy(self, url: str, orig_type: str) -> tuple[bytes | None, str]: proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", "")) - if user: - creds = self._basic_auth_header(user, password) - else: - creds = None + creds = self._basic_auth_header(user, password) if user else None return creds, proxy_url + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: - creds, proxy_url = None, None + self, request: Request, spider: Spider | None = None + ) -> Request | Response | None: + creds, proxy_url, scheme = None, None, None if "proxy" in request.meta: if request.meta["proxy"] is not None: creds, proxy_url = self._get_proxy(request.meta["proxy"], "") elif self.proxies: parsed = urlparse_cached(request) - scheme = parsed.scheme + _scheme = parsed.scheme if ( # 'no_proxy' is only supported by http schemes - scheme not in ("http", "https") + _scheme not in {"http", "https"} or (parsed.hostname and not proxy_bypass(parsed.hostname)) - ) and scheme in self.proxies: + ) and _scheme in self.proxies: + scheme = _scheme creds, proxy_url = self.proxies[scheme] - self._set_proxy_and_creds(request, proxy_url, creds) + self._set_proxy_and_creds(request, proxy_url, creds, scheme) return None def _set_proxy_and_creds( - self, request: Request, proxy_url: Optional[str], creds: Optional[bytes] + self, + request: Request, + proxy_url: str | None, + creds: bytes | None, + scheme: str | None, ) -> None: + if scheme: + request.meta["_scheme_proxy"] = True if proxy_url: request.meta["proxy"] = proxy_url elif request.meta.get("proxy") is not None: diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py similarity index 61% rename from scrapy/spidermiddlewares/offsite.py rename to scrapy/downloadermiddlewares/offsite.py index a5214702d..10f19bacc 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -1,68 +1,68 @@ -""" -Offsite Spider Middleware - -See documentation in docs/topics/spider-middleware.rst -""" from __future__ import annotations import logging import re import warnings -from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable, Set +from typing import TYPE_CHECKING -from scrapy import Spider, signals -from scrapy.crawler import Crawler -from scrapy.http import Request, Response -from scrapy.statscollectors import StatsCollector +from scrapy import Request, Spider, signals +from scrapy.exceptions import IgnoreRequest +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + logger = logging.getLogger(__name__) class OffsiteMiddleware: + crawler: Crawler + def __init__(self, stats: StatsCollector): - self.stats: StatsCollector = stats + self.stats = stats + self.domains_seen: set[str] = set() @classmethod def from_crawler(cls, crawler: Crawler) -> Self: assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled) + o.crawler = crawler return o - def process_spider_output( - self, response: Response, result: Iterable[Any], spider: Spider - ) -> Iterable[Any]: - return (r for r in result if self._filter(r, spider)) + def spider_opened(self, spider: Spider) -> None: + self.host_regex: re.Pattern[str] = self.get_host_regex(spider) - async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: - async for r in result: - if self._filter(r, spider): - yield r + def request_scheduled(self, request: Request, spider: Spider) -> None: + self.process_request(request) - def _filter(self, request: Any, spider: Spider) -> bool: - if not isinstance(request, Request): - return True - if request.dont_filter or self.should_follow(request, spider): - return True + @_warn_spider_arg + def process_request(self, request: Request, spider: Spider | None = None) -> None: + assert self.crawler.spider + if ( + request.dont_filter + or request.meta.get("allow_offsite") + or self.should_follow(request, self.crawler.spider) + ): + return domain = urlparse_cached(request).hostname if domain and domain not in self.domains_seen: self.domains_seen.add(domain) logger.debug( "Filtered offsite request to %(domain)r: %(request)s", {"domain": domain, "request": request}, - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) - self.stats.inc_value("offsite/domains", spider=spider) - self.stats.inc_value("offsite/filtered", spider=spider) - return False + self.stats.inc_value("offsite/domains") + self.stats.inc_value("offsite/filtered") + raise IgnoreRequest def should_follow(self, request: Request, spider: Spider) -> bool: regex = self.host_regex @@ -86,26 +86,14 @@ class OffsiteMiddleware: "allowed_domains accepts only domains, not URLs. " f"Ignoring URL entry {domain} in allowed_domains." ) - warnings.warn(message, URLWarning) + warnings.warn(message, stacklevel=2) elif port_pattern.search(domain): message = ( "allowed_domains accepts only domains without ports. " f"Ignoring entry {domain} in allowed_domains." ) - warnings.warn(message, PortWarning) + warnings.warn(message, stacklevel=2) else: domains.append(re.escape(domain)) - regex = rf'^(.*\.)?({"|".join(domains)})$' + regex = rf"^(.*\.)?({'|'.join(domains)})$" return re.compile(regex) - - def spider_opened(self, spider: Spider) -> None: - self.host_regex: re.Pattern[str] = self.get_host_regex(spider) - self.domains_seen: Set[str] = set() - - -class URLWarning(Warning): - pass - - -class PortWarning(Warning): - pass diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 814b1a561..821f41699 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -1,43 +1,34 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, List, Union, cast +from typing import TYPE_CHECKING, Any, cast from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string -from scrapy import Request, Spider -from scrapy.crawler import Crawler +from scrapy import signals from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import HtmlResponse, Response -from scrapy.settings import BaseSettings +from scrapy.spidermiddlewares.referer import RefererMiddleware +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import global_object_name from scrapy.utils.response import get_meta_refresh if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) -def _build_redirect_request( - source_request: Request, *, url: str, **kwargs: Any -) -> Request: - redirect_request = source_request.replace( - url=url, - **kwargs, - cookies=None, - ) - if "Cookie" in redirect_request.headers: - source_request_netloc = urlparse_cached(source_request).netloc - redirect_request_netloc = urlparse_cached(redirect_request).netloc - if source_request_netloc != redirect_request_netloc: - del redirect_request.headers["Cookie"] - return redirect_request - - class BaseRedirectMiddleware: + crawler: Crawler enabled_setting: str = "REDIRECT_ENABLED" def __init__(self, settings: BaseSettings): @@ -46,52 +37,161 @@ class BaseRedirectMiddleware: self.max_redirect_times: int = settings.getint("REDIRECT_MAX_TIMES") self.priority_adjust: int = settings.getint("REDIRECT_PRIORITY_ADJUST") + self._referer_spider_middleware: RefererMiddleware | None = None @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - return cls(crawler.settings) + o = cls(crawler.settings) + o.crawler = crawler + crawler.signals.connect(o._engine_started, signal=signals.engine_started) + return o - def _redirect( - self, redirected: Request, request: Request, spider: Spider, reason: Any - ) -> Request: + def handle_referer(self, request: Request, response: Response) -> None: + """Remove, modify or keep the Referer header of *request* based on the + *response* that triggered *request*. + + By default, this method finds a run-time instance of + scrapy.spidermiddlewares.referer.RefererMiddleware (or of a subclass) + and uses it to set the right Referer header. + + Override this method if you use a different Scrapy component to handle + Referer headers, of if you want to use a custom logic to set the + Referer header on redirects. + """ + request.headers.pop("Referer", None) + if not self._referer_spider_middleware: + return + self._referer_spider_middleware.get_processed_request(request, response) + + def _engine_started(self) -> None: + self._referer_spider_middleware = self.crawler.get_spider_middleware( + RefererMiddleware + ) + if self._referer_spider_middleware: + return + redirect_cls = global_object_name(self.__class__) + referer_cls = global_object_name(RefererMiddleware) + if self.__class__ in {RedirectMiddleware, MetaRefreshMiddleware}: + replacement = ( + f"replace {redirect_cls} with a subclass that overrides the " + f"handle_referer() method" + ) + else: + replacement = ( + f"or edit {redirect_cls} (if defined in your code base) to " + f"override the handle_referer() method, or replace " + f"{redirect_cls} with a subclass that overrides the " + f"handle_referer() method." + ) + logger.warning( + f"{redirect_cls} found no {referer_cls} instance to handle " + f"Referer header handling, so the Referer header will be removed " + f"on redirects. To set a Referer header on redirects, enable " + f"{referer_cls} (or a subclass), or {replacement}.", + ) + + def _redirect(self, redirected: Request, request: Request, reason: Any) -> Request: ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times) redirects = request.meta.get("redirect_times", 0) + 1 if ttl and redirects <= self.max_redirect_times: redirected.meta["redirect_times"] = redirects redirected.meta["redirect_ttl"] = ttl - 1 - redirected.meta["redirect_urls"] = request.meta.get("redirect_urls", []) + [ - request.url + redirected.meta["redirect_urls"] = [ + *request.meta.get("redirect_urls", []), + request.url, + ] + redirected.meta["redirect_reasons"] = [ + *request.meta.get("redirect_reasons", []), + reason, ] - redirected.meta["redirect_reasons"] = request.meta.get( - "redirect_reasons", [] - ) + [reason] redirected.dont_filter = request.dont_filter redirected.priority = request.priority + self.priority_adjust logger.debug( "Redirecting (%(reason)s) to %(redirected)s from %(request)s", {"reason": reason, "redirected": redirected, "request": request}, - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) return redirected logger.debug( "Discarding %(request)s: max redirections reached", {"request": request}, - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) raise IgnoreRequest("max redirections reached") - def _redirect_request_using_get( - self, request: Request, redirect_url: str + def _build_redirect_request( + self, source_request: Request, response: Response, *, url: str, **kwargs: Any ) -> Request: - redirect_request = _build_redirect_request( + redirect_request = source_request.replace( + url=url, + **kwargs, + cls=None, + cookies=None, + ) + if "_scheme_proxy" in redirect_request.meta: + source_request_scheme = urlparse_cached(source_request).scheme + redirect_request_scheme = urlparse_cached(redirect_request).scheme + if source_request_scheme != redirect_request_scheme: + redirect_request.meta.pop("_scheme_proxy") + redirect_request.meta.pop("proxy", None) + redirect_request.meta.pop("_auth_proxy", None) + redirect_request.headers.pop(b"Proxy-Authorization", None) + + has_cookie_header = "Cookie" in redirect_request.headers + has_authorization_header = "Authorization" in redirect_request.headers + if has_cookie_header or has_authorization_header: + default_ports = {"http": 80, "https": 443} + + parsed_source_request = urlparse_cached(source_request) + source_scheme, source_host, source_port = ( + parsed_source_request.scheme, + parsed_source_request.hostname, + parsed_source_request.port + or default_ports.get(parsed_source_request.scheme), + ) + + parsed_redirect_request = urlparse_cached(redirect_request) + redirect_scheme, redirect_host, redirect_port = ( + parsed_redirect_request.scheme, + parsed_redirect_request.hostname, + parsed_redirect_request.port + or default_ports.get(parsed_redirect_request.scheme), + ) + + if has_cookie_header and ( + redirect_scheme not in {source_scheme, "https"} + or source_host != redirect_host + ): + del redirect_request.headers["Cookie"] + + # https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name + if has_authorization_header and ( + source_scheme != redirect_scheme + or source_host != redirect_host + or source_port != redirect_port + ): + del redirect_request.headers["Authorization"] + + self.handle_referer(redirect_request, response) + + return redirect_request + + def _redirect_request_using_get( + self, request: Request, response: Response, redirect_url: str + ) -> Request: + redirect_request = self._build_redirect_request( request, + response, url=redirect_url, method="GET", body="", ) redirect_request.headers.pop("Content-Type", None) redirect_request.headers.pop("Content-Length", None) + redirect_request.headers.pop("Content-Encoding", None) + redirect_request.headers.pop("Content-Language", None) + redirect_request.headers.pop("Content-Location", None) return redirect_request @@ -101,35 +201,53 @@ class RedirectMiddleware(BaseRedirectMiddleware): and meta-refresh html tag. """ + @_warn_spider_arg def process_response( - self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + self, request: Request, response: Response, spider: Spider | None = None + ) -> Request | Response: if ( request.meta.get("dont_redirect", False) - or response.status in getattr(spider, "handle_httpstatus_list", []) - or response.status in request.meta.get("handle_httpstatus_list", []) + or response.status + in getattr(self.crawler.spider, "handle_httpstatus_list", ()) + or response.status in request.meta.get("handle_httpstatus_list", ()) or request.meta.get("handle_httpstatus_all", False) ): return response - allowed_status = (301, 302, 303, 307, 308) - if "Location" not in response.headers or response.status not in allowed_status: + if "Location" not in response.headers or response.status not in { + 301, + 302, + 303, + 307, + 308, + }: return response assert response.headers["Location"] is not None location = safe_url_string(response.headers["Location"]) if response.headers["Location"].startswith(b"//"): - request_scheme = urlparse(request.url).scheme + request_scheme = urlparse_cached(request).scheme location = request_scheme + "://" + location.lstrip("/") redirected_url = urljoin(request.url, location) - if response.status in (301, 307, 308) or request.method == "HEAD": - redirected = _build_redirect_request(request, url=redirected_url) - return self._redirect(redirected, request, spider, response.status) + if not urlparse(redirected_url).fragment: + fragment = urlparse_cached(request).fragment + if fragment: + redirected_url = urljoin(redirected_url, f"#{fragment}") - redirected = self._redirect_request_using_get(request, redirected_url) - return self._redirect(redirected, request, spider, response.status) + redirected = self._build_redirect_request(request, response, url=redirected_url) + if urlparse_cached(redirected).scheme not in {"http", "https"}: + return response + + if (response.status in {301, 302} and request.method == "POST") or ( + response.status == 303 and request.method not in {"GET", "HEAD"} + ): + redirected = self._redirect_request_using_get( + request, response, redirected_url + ) + + return self._redirect(redirected, request, response.status) class MetaRefreshMiddleware(BaseRedirectMiddleware): @@ -137,22 +255,27 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): def __init__(self, settings: BaseSettings): super().__init__(settings) - self._ignore_tags: List[str] = settings.getlist("METAREFRESH_IGNORE_TAGS") + self._ignore_tags: list[str] = settings.getlist("METAREFRESH_IGNORE_TAGS") self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY") + @_warn_spider_arg def process_response( - self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + self, request: Request, response: Response, spider: Spider | None = None + ) -> Request | Response: if ( request.meta.get("dont_redirect", False) or request.method == "HEAD" or not isinstance(response, HtmlResponse) + or urlparse_cached(request).scheme not in {"http", "https"} ): return response interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags) - if url and cast(float, interval) < self._maxdelay: - redirected = self._redirect_request_using_get(request, url) - return self._redirect(redirected, request, spider, "meta refresh") - + if not url: + return response + redirected = self._redirect_request_using_get(request, response, url) + if urlparse_cached(redirected).scheme not in {"http", "https"}: + return response + if cast("float", interval) < self._maxdelay: + return self._redirect(redirected, request, "meta refresh") return response diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 3c494de78..5f125cae4 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -7,20 +7,16 @@ RETRY_TIMES - how many times to retry a failed page RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, -once the spider has finished crawling all regular (non failed) pages. +once the spider has finished crawling all regular (non-failed) pages. """ + from __future__ import annotations -import warnings -from logging import Logger, getLogger -from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union +from logging import Logger, getLevelName, getLogger +from typing import TYPE_CHECKING -from scrapy.crawler import Crawler -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.http import Response -from scrapy.http.request import Request -from scrapy.settings import BaseSettings, Settings -from scrapy.spiders import Spider +from scrapy.exceptions import NotConfigured +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.misc import load_object from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message @@ -29,54 +25,43 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + import scrapy + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.http.request import Request + from scrapy.settings import BaseSettings + + retry_logger = getLogger(__name__) -def backwards_compatibility_getattr(self: Any, name: str) -> Tuple[Any, ...]: - if name == "EXCEPTIONS_TO_RETRY": - warnings.warn( - "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. " - "Use the RETRY_EXCEPTIONS setting instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - return tuple( - load_object(x) if isinstance(x, str) else x - for x in Settings().getlist("RETRY_EXCEPTIONS") - ) - raise AttributeError( - f"{self.__class__.__name__!r} object has no attribute {name!r}" - ) - - -class BackwardsCompatibilityMetaclass(type): - __getattr__ = backwards_compatibility_getattr - - def get_retry_request( request: Request, *, - spider: Spider, - reason: Union[str, Exception, Type[Exception]] = "unspecified", - max_retry_times: Optional[int] = None, - priority_adjust: Optional[int] = None, + spider: scrapy.Spider, + reason: str | Exception | type[Exception] = "unspecified", + max_retry_times: int | None = None, + priority_adjust: int | None = None, logger: Logger = retry_logger, + give_up_log_level: int | str | None = None, stats_base_key: str = "retry", -) -> Optional[Request]: +) -> Request | None: """ Returns a new :class:`~scrapy.Request` object to retry the specified request, or ``None`` if retries of the specified request have been exhausted. For example, in a :class:`~scrapy.Spider` callback, you could use it as - follows:: + follows: + + .. code-block:: python def parse(self, response): if not response.text: new_request_or_none = get_retry_request( response.request, spider=self, - reason='empty', + reason="empty", ) return new_request_or_none @@ -100,6 +85,10 @@ def get_retry_request( *logger* is the logging.Logger object to be used when logging messages + *give_up_log_level* is the :ref:`logging level ` used for the + message logged when a request exceeds its retries. See + :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details. + *stats_base_key* is a string to be used as the base key for the retry-related job stats """ @@ -132,71 +121,85 @@ def get_retry_request( stats.inc_value(f"{stats_base_key}/count") stats.inc_value(f"{stats_base_key}/reason_count/{reason}") return new_request + if give_up_log_level is None: + give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"] + if isinstance(give_up_log_level, str): + level = getLevelName(give_up_log_level) + if not isinstance(level, int): + raise ValueError(f"Invalid give-up log level: {give_up_log_level!r}") + give_up_log_level = level stats.inc_value(f"{stats_base_key}/max_reached") - logger.error( - "Gave up retrying %(request)s (failed %(retry_times)d times): " "%(reason)s", + logger.log( + give_up_log_level, + "Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s", {"request": request, "retry_times": retry_times, "reason": reason}, extra={"spider": spider}, ) return None -class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): +class RetryMiddleware: + crawler: Crawler + def __init__(self, settings: BaseSettings): if not settings.getbool("RETRY_ENABLED"): raise NotConfigured self.max_retry_times = settings.getint("RETRY_TIMES") - self.retry_http_codes = set( - int(x) for x in settings.getlist("RETRY_HTTP_CODES") - ) + self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")} self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") - - try: - self.exceptions_to_retry = self.__getattribute__("EXCEPTIONS_TO_RETRY") - except AttributeError: - # If EXCEPTIONS_TO_RETRY is not "overridden" - self.exceptions_to_retry = tuple( - load_object(x) if isinstance(x, str) else x - for x in settings.getlist("RETRY_EXCEPTIONS") - ) + self.give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"] + self.exceptions_to_retry = tuple( + load_object(x) if isinstance(x, str) else x + for x in settings.getlist("RETRY_EXCEPTIONS") + ) @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - return cls(crawler.settings) + o = cls(crawler.settings) + o.crawler = crawler + return o + @_warn_spider_arg def process_response( - self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + self, + request: Request, + response: Response, + spider: scrapy.Spider | None = None, + ) -> Request | Response: if request.meta.get("dont_retry", False): return response if response.status in self.retry_http_codes: reason = response_status_message(response.status) - return self._retry(request, reason, spider) or response + return self._retry(request, reason) or response return response + @_warn_spider_arg def process_exception( - self, request: Request, exception: Exception, spider: Spider - ) -> Union[Request, Response, None]: + self, + request: Request, + exception: Exception, + spider: scrapy.Spider | None = None, + ) -> Request | Response | None: if isinstance(exception, self.exceptions_to_retry) and not request.meta.get( "dont_retry", False ): - return self._retry(request, exception, spider) + return self._retry(request, exception) return None def _retry( - self, - request: Request, - reason: Union[str, Exception, Type[Exception]], - spider: Spider, - ) -> Optional[Request]: + self, request: Request, reason: str | Exception | type[Exception] + ) -> Request | None: max_retry_times = request.meta.get("max_retry_times", self.max_retry_times) priority_adjust = request.meta.get("priority_adjust", self.priority_adjust) + give_up_log_level = request.meta.get( + "give_up_log_level", self.give_up_log_level + ) + assert self.crawler.spider return get_retry_request( request, reason=reason, - spider=spider, + spider=self.crawler.spider, max_retry_times=max_retry_times, priority_adjust=priority_adjust, + give_up_log_level=give_up_log_level, ) - - __getattr__ = backwards_compatibility_getattr diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 6a0ecb7bf..7d0c17884 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -7,25 +7,26 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING -from twisted.internet.defer import Deferred, maybeDeferred -from twisted.python.failure import Failure +from twisted.internet.defer import Deferred -from scrapy import Spider -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.robotstxt import RobotParser +from scrapy.utils.decorators import _warn_spider_arg +from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import load_object if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.robotstxt import RobotParser + logger = logging.getLogger(__name__) @@ -36,12 +37,10 @@ class RobotsTxtMiddleware: def __init__(self, crawler: Crawler): if not crawler.settings.getbool("ROBOTSTXT_OBEY"): raise NotConfigured - self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy") - self._robotstxt_useragent: Optional[str] = crawler.settings.get( - "ROBOTSTXT_USER_AGENT", None - ) + self._default_useragent: str = crawler.settings["USER_AGENT"] + self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"] self.crawler: Crawler = crawler - self._parsers: Dict[str, Union[RobotParser, Deferred, None]] = {} + self._parsers: dict[str, RobotParser | Deferred[RobotParser | None] | None] = {} self._parserimpl: RobotParser = load_object( crawler.settings.get("ROBOTSTXT_PARSER") ) @@ -53,22 +52,22 @@ class RobotsTxtMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]: - if request.meta.get("dont_obey_robotstxt"): - return None - if request.url.startswith("data:") or request.url.startswith("file:"): - return None - d: Deferred = maybeDeferred(self.robot_parser, request, spider) - d.addCallback(self.process_request_2, request, spider) - return d - - def process_request_2( - self, rp: Optional[RobotParser], request: Request, spider: Spider + @_warn_spider_arg + async def process_request( + self, request: Request, spider: Spider | None = None ) -> None: + if request.meta.get("dont_obey_robotstxt"): + return + if request.url.startswith("data:") or request.url.startswith("file:"): + return + rp = await self.robot_parser(request) + self.process_request_2(rp, request) + + def process_request_2(self, rp: RobotParser | None, request: Request) -> None: if rp is None: return - useragent: Union[str, bytes, None] = self._robotstxt_useragent + useragent: str | bytes | None = self._robotstxt_useragent if not useragent: useragent = request.headers.get(b"User-Agent", self._default_useragent) assert useragent is not None @@ -76,15 +75,13 @@ class RobotsTxtMiddleware: logger.debug( "Forbidden by robots.txt: %(request)s", {"request": request}, - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/forbidden") raise IgnoreRequest("Forbidden by robots.txt") - def robot_parser( - self, request: Request, spider: Spider - ) -> Union[RobotParser, Deferred, None]: + async def robot_parser(self, request: Request) -> RobotParser | None: url = urlparse_cached(request) netloc = url.netloc @@ -99,35 +96,26 @@ class RobotsTxtMiddleware: ) assert self.crawler.engine assert self.crawler.stats - dfd = self.crawler.engine.download(robotsreq) - dfd.addCallback(self._parse_robots, netloc, spider) - dfd.addErrback(self._logerror, robotsreq, spider) - dfd.addErrback(self._robots_error, netloc) + try: + resp = await self.crawler.engine.download_async(robotsreq) + self._parse_robots(resp, netloc) + except Exception as e: + if not isinstance(e, IgnoreRequest): + logger.error( + "Error downloading %(request)s: %(f_exception)s", + {"request": request, "f_exception": e}, + exc_info=True, + extra={"spider": self.crawler.spider}, + ) + self._robots_error(e, netloc) self.crawler.stats.inc_value("robotstxt/request_count") parser = self._parsers[netloc] if isinstance(parser, Deferred): - d: Deferred = Deferred() - - def cb(result: Any) -> Any: - d.callback(result) - return result - - parser.addCallback(cb) - return d + return await maybe_deferred_to_future(parser) return parser - def _logerror(self, failure: Failure, request: Request, spider: Spider) -> Failure: - if failure.type is not IgnoreRequest: - logger.error( - "Error downloading %(request)s: %(f_exception)s", - {"request": request, "f_exception": failure.value}, - exc_info=failure_to_exc_info(failure), - extra={"spider": spider}, - ) - return failure - - def _parse_robots(self, response: Response, netloc: str, spider: Spider) -> None: + def _parse_robots(self, response: Response, netloc: str) -> None: assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/response_count") self.crawler.stats.inc_value( @@ -139,9 +127,9 @@ class RobotsTxtMiddleware: self._parsers[netloc] = rp rp_dfd.callback(rp) - def _robots_error(self, failure: Failure, netloc: str) -> None: - if failure.type is not IgnoreRequest: - key = f"robotstxt/exception_count/{failure.type}" + def _robots_error(self, exc: Exception, netloc: str) -> None: + if not isinstance(exc, IgnoreRequest): + key = f"robotstxt/exception_count/{type(exc)}" assert self.crawler.stats self.crawler.stats.inc_value(key) rp_dfd = self._parsers[netloc] diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index df30e8ca4..bafa931de 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,14 +1,11 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, Union +from typing import TYPE_CHECKING from twisted.web import http -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response -from scrapy.statscollectors import StatsCollector +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.request import request_httprepr @@ -16,8 +13,15 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.statscollectors import StatsCollector -def get_header_size(headers: Dict[str, Union[list, tuple]]) -> int: + +def get_header_size( + headers: dict[str, list[str | bytes] | tuple[str | bytes, ...]], +) -> int: size = 0 for key, value in headers.items(): if isinstance(value, (list, tuple)): @@ -42,24 +46,22 @@ class DownloaderStats: assert crawler.stats return cls(crawler.stats) + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: - self.stats.inc_value("downloader/request_count", spider=spider) - self.stats.inc_value( - f"downloader/request_method_count/{request.method}", spider=spider - ) + self, request: Request, spider: Spider | None = None + ) -> Request | Response | None: + self.stats.inc_value("downloader/request_count") + self.stats.inc_value(f"downloader/request_method_count/{request.method}") reqlen = len(request_httprepr(request)) - self.stats.inc_value("downloader/request_bytes", reqlen, spider=spider) + self.stats.inc_value("downloader/request_bytes", reqlen) return None + @_warn_spider_arg def process_response( - self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: - self.stats.inc_value("downloader/response_count", spider=spider) - self.stats.inc_value( - f"downloader/response_status_count/{response.status}", spider=spider - ) + self, request: Request, response: Response, spider: Spider | None = None + ) -> Request | Response: + self.stats.inc_value("downloader/response_count") + self.stats.inc_value(f"downloader/response_status_count/{response.status}") reslen = ( len(response.body) + get_header_size(response.headers) @@ -67,15 +69,14 @@ class DownloaderStats: + 4 ) # response.body + b"\r\n"+ response.header + b"\r\n" + response.status - self.stats.inc_value("downloader/response_bytes", reslen, spider=spider) + self.stats.inc_value("downloader/response_bytes", reslen) return response + @_warn_spider_arg def process_exception( - self, request: Request, exception: Exception, spider: Spider - ) -> Union[Request, Response, None]: + self, request: Request, exception: Exception, spider: Spider | None = None + ) -> Request | Response | None: ex_class = global_object_name(exception.__class__) - self.stats.inc_value("downloader/exception_count", spider=spider) - self.stats.inc_value( - f"downloader/exception_type_count/{ex_class}", spider=spider - ) + self.stats.inc_value("downloader/exception_count") + self.stats.inc_value(f"downloader/exception_type_count/{ex_class}") return None diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index 92f1ec897..61f84b518 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -2,16 +2,19 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler -from scrapy.http import Response +from scrapy.utils.decorators import _warn_spider_arg +from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + class UserAgentMiddleware: """This middleware allows spiders to override the user_agent""" @@ -26,11 +29,15 @@ class UserAgentMiddleware: return o def spider_opened(self, spider: Spider) -> None: + if hasattr(spider, "user_agent"): # pragma: no cover + warn_on_deprecated_spider_attribute("user_agent", "USER_AGENT") + self.user_agent = getattr(spider, "user_agent", self.user_agent) + @_warn_spider_arg def process_request( - self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + self, request: Request, spider: Spider | None = None + ) -> Request | Response | None: if self.user_agent: request.headers.setdefault(b"User-Agent", self.user_agent) return None diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index dd2420e98..36fb0f97d 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -2,13 +2,10 @@ from __future__ import annotations import logging from pathlib import Path -from typing import TYPE_CHECKING, Optional, Set +from typing import TYPE_CHECKING +from warnings import warn -from twisted.internet.defer import Deferred - -from scrapy.http.request import Request -from scrapy.settings import BaseSettings -from scrapy.spiders import Spider +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.job import job_dir from scrapy.utils.request import ( RequestFingerprinter, @@ -17,69 +14,92 @@ from scrapy.utils.request import ( ) if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import Request + from scrapy.spiders import Spider class BaseDupeFilter: + """Dummy duplicate request filtering class (:setting:`DUPEFILTER_CLASS`) + that does not filter out any request.""" + @classmethod - def from_settings(cls, settings: BaseSettings) -> Self: + def from_crawler(cls, crawler: Crawler) -> Self: return cls() def request_seen(self, request: Request) -> bool: return False - def open(self) -> Optional[Deferred]: + def open(self) -> Deferred[None] | None: pass - def close(self, reason: str) -> Optional[Deferred]: + def close(self, reason: str) -> Deferred[None] | None: pass def log(self, request: Request, spider: Spider) -> None: """Log that a request has been filtered""" - pass + warn( + "Calling BaseDupeFilter.log() is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) class RFPDupeFilter(BaseDupeFilter): - """Request Fingerprint duplicates filter""" + """Duplicate request filtering class (:setting:`DUPEFILTER_CLASS`) that + filters out requests with the canonical + (:func:`w3lib.url.canonicalize_url`) :attr:`~scrapy.http.Request.url`, + :attr:`~scrapy.http.Request.method` and :attr:`~scrapy.http.Request.body`. + + Job directory contents + ====================== + + .. warning:: The files that this class generates in the :ref:`job directory + ` are an implementation detail, and may change without a + warning in a future version of Scrapy. Do not rely on the following + information for anything other than debugging purposes. + + When using :setting:`JOBDIR`, seen fingerprints are tracked in a file named + ``requests.seen`` in the :ref:`job directory `, which contains 1 + request fingerprint per line. + """ def __init__( self, - path: Optional[str] = None, + path: str | None = None, debug: bool = False, *, - fingerprinter: Optional[RequestFingerprinterProtocol] = None, + fingerprinter: RequestFingerprinterProtocol | None = None, ) -> None: self.file = None self.fingerprinter: RequestFingerprinterProtocol = ( fingerprinter or RequestFingerprinter() ) - self.fingerprints: Set[str] = set() + self.fingerprints: set[str] = set() self.logdupes = True self.debug = debug self.logger = logging.getLogger(__name__) if path: - self.file = Path(path, "requests.seen").open("a+", encoding="utf-8") + # line-by-line writing, see: https://github.com/scrapy/scrapy/issues/6019 + self.file = Path(path, "requests.seen").open( + "a+", buffering=1, encoding="utf-8" + ) + self.file.reconfigure(write_through=True) self.file.seek(0) self.fingerprints.update(x.rstrip() for x in self.file) - @classmethod - def from_settings( - cls, - settings: BaseSettings, - *, - fingerprinter: Optional[RequestFingerprinterProtocol] = None, - ) -> Self: - debug = settings.getbool("DUPEFILTER_DEBUG") - return cls(job_dir(settings), debug, fingerprinter=fingerprinter) - @classmethod def from_crawler(cls, crawler: Crawler) -> Self: assert crawler.request_fingerprinter - return cls.from_settings( - crawler.settings, + debug = crawler.settings.getbool("DUPEFILTER_DEBUG") + return cls( + job_dir(crawler.settings), + debug, fingerprinter=crawler.request_fingerprinter, ) @@ -93,6 +113,7 @@ class RFPDupeFilter(BaseDupeFilter): return False def request_fingerprint(self, request: Request) -> str: + """Returns a string that uniquely identifies the specified request.""" return self.fingerprinter.fingerprint(request).hex() def close(self, reason: str) -> None: @@ -114,4 +135,4 @@ class RFPDupeFilter(BaseDupeFilter): self.logdupes = False assert spider.crawler.stats - spider.crawler.stats.inc_value("dupefilter/filtered", spider=spider) + spider.crawler.stats.inc_value("dupefilter/filtered") diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 6d188c489..204132973 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -4,7 +4,13 @@ Scrapy core exceptions These exceptions are documented in docs/topics/exceptions.rst. Please don't add new exceptions here without documenting them there. """ -from typing import Any + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from scrapy.http import Response # Internal @@ -12,8 +18,6 @@ from typing import Any class NotConfigured(Exception): """Indicates a missing configuration situation""" - pass - class _InvalidOutput(TypeError): """ @@ -21,8 +25,6 @@ class _InvalidOutput(TypeError): Internal and undocumented, it should not be raised or caught by user code. """ - pass - # HTTP and crawling @@ -34,8 +36,6 @@ class IgnoreRequest(Exception): class DontCloseSpider(Exception): """Request the spider not to be closed yet""" - pass - class CloseSpider(Exception): """Raise this from callbacks to request the spider to be closed""" @@ -52,25 +52,55 @@ class StopDownload(Exception): should be handled by the request errback. Note that 'fail' is a keyword-only argument. """ + response: Response | None + def __init__(self, *, fail: bool = True): super().__init__() self.fail = fail +class DownloadConnectionRefusedError(Exception): + """Indicates that a connection was refused by the server.""" + + +class CannotResolveHostError(Exception): + """Indicates that the provided hostname cannot be resolved.""" + + +class DownloadTimeoutError(Exception): + """Indicates that a request download has timed out.""" + + +class DownloadCancelledError(Exception): + """Indicates that a request download was cancelled.""" + + +class DownloadFailedError(Exception): + """Indicates that a request download has failed.""" + + +class ResponseDataLossError(Exception): + """Indicates that Scrapy couldn't get a complete response.""" + + +class UnsupportedURLSchemeError(Exception): + """Indicates that the URL scheme is not supported.""" + + # Items class DropItem(Exception): """Drop item from the item pipeline""" - pass + def __init__(self, message: str, log_level: str | None = None): + super().__init__(message) + self.log_level = log_level class NotSupported(Exception): """Indicates a feature or method is not supported""" - pass - # Commands @@ -88,10 +118,6 @@ class ScrapyDeprecationWarning(Warning): DeprecationWarning is silenced on Python 2.7+ """ - pass - class ContractFail(AssertionError): """Error raised in case of a failing contract""" - - pass diff --git a/scrapy/exporters.py b/scrapy/exporters.py index f85f1dad8..e18f1e6ed 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -2,63 +2,78 @@ Item Exporters are used to export/serialize items into different formats. """ +from __future__ import annotations + import csv -import io import marshal import pickle import pprint -from collections.abc import Mapping +from abc import ABC, abstractmethod +from collections.abc import Callable, Iterable, Mapping +from io import BytesIO, TextIOWrapper +from typing import TYPE_CHECKING, Any from xml.sax.saxutils import XMLGenerator +from xml.sax.xmlreader import AttributesImpl from itemadapter import ItemAdapter, is_item -from scrapy.item import Item +from scrapy.item import Field, Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder +if TYPE_CHECKING: + from json import JSONEncoder + __all__ = [ "BaseItemExporter", - "PprintItemExporter", - "PickleItemExporter", "CsvItemExporter", - "XmlItemExporter", - "JsonLinesItemExporter", "JsonItemExporter", + "JsonLinesItemExporter", "MarshalItemExporter", + "PickleItemExporter", + "PprintItemExporter", + "XmlItemExporter", ] -class BaseItemExporter: - def __init__(self, *, dont_fail=False, **kwargs): - self._kwargs = kwargs +class BaseItemExporter(ABC): + def __init__(self, *, dont_fail: bool = False, **kwargs: Any): + self._kwargs: dict[str, Any] = kwargs self._configure(kwargs, dont_fail=dont_fail) - def _configure(self, options, dont_fail=False): + def _configure(self, options: dict[str, Any], dont_fail: bool = False) -> None: """Configure the exporter by popping options from the ``options`` dict. If dont_fail is set, it won't raise an exception on unexpected options (useful for using with keyword arguments in subclasses ``__init__`` methods) """ - self.encoding = options.pop("encoding", None) - self.fields_to_export = options.pop("fields_to_export", None) - self.export_empty_fields = options.pop("export_empty_fields", False) - self.indent = options.pop("indent", None) + self.encoding: str | None = options.pop("encoding", None) + self.fields_to_export: Mapping[str, str] | Iterable[str] | None = options.pop( + "fields_to_export", None + ) + self.export_empty_fields: bool = options.pop("export_empty_fields", False) + self.indent: int | None = options.pop("indent", None) if not dont_fail and options: raise TypeError(f"Unexpected options: {', '.join(options.keys())}") - def export_item(self, item): + @abstractmethod + def export_item(self, item: Any) -> None: raise NotImplementedError - def serialize_field(self, field, name, value): - serializer = field.get("serializer", lambda x: x) + def serialize_field( + self, field: Mapping[str, Any] | Field, name: str, value: Any + ) -> Any: + serializer: Callable[[Any], Any] = field.get("serializer", lambda x: x) return serializer(value) - def start_exporting(self): + def start_exporting(self) -> None: # noqa: B027 pass - def finish_exporting(self): + def finish_exporting(self) -> None: # noqa: B027 pass - def _get_serialized_fields(self, item, default_value=None, include_empty=None): + def _get_serialized_fields( + self, item: Any, default_value: Any = None, include_empty: bool | None = None + ) -> Iterable[tuple[str, Any]]: """Return the fields to export as an iterable of tuples (name, serialized_value) """ @@ -68,10 +83,7 @@ class BaseItemExporter: include_empty = self.export_empty_fields if self.fields_to_export is None: - if include_empty: - field_iter = item.field_names() - else: - field_iter = item.keys() + field_iter = item.field_names() if include_empty else item.keys() elif isinstance(self.fields_to_export, Mapping): if include_empty: field_iter = self.fields_to_export.items() @@ -79,11 +91,10 @@ class BaseItemExporter: field_iter = ( (x, y) for x, y in self.fields_to_export.items() if x in item ) + elif include_empty: + field_iter = self.fields_to_export else: - if include_empty: - field_iter = self.fields_to_export - else: - field_iter = (x for x in self.fields_to_export if x in item) + field_iter = (x for x in self.fields_to_export if x in item) for field_name in field_iter: if isinstance(field_name, str): @@ -100,22 +111,22 @@ class BaseItemExporter: class JsonLinesItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(dont_fail=True, **kwargs) - self.file = file + self.file: BytesIO = file self._kwargs.setdefault("ensure_ascii", not self.encoding) - self.encoder = ScrapyJSONEncoder(**self._kwargs) + self.encoder: JSONEncoder = ScrapyJSONEncoder(**self._kwargs) - def export_item(self, item): + def export_item(self, item: Any) -> None: itemdict = dict(self._get_serialized_fields(item)) data = self.encoder.encode(itemdict) + "\n" self.file.write(to_bytes(data, self.encoding)) class JsonItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(dont_fail=True, **kwargs) - self.file = file + self.file: BytesIO = file # there is a small difference between the behaviour or JsonItemExporter.indent # and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent # the addition of newlines everywhere @@ -127,26 +138,26 @@ class JsonItemExporter(BaseItemExporter): self.encoder = ScrapyJSONEncoder(**self._kwargs) self.first_item = True - def _beautify_newline(self): + def _beautify_newline(self) -> None: if self.indent is not None: self.file.write(b"\n") - def _add_comma_after_first(self): + def _add_comma_after_first(self) -> None: if self.first_item: self.first_item = False else: self.file.write(b",") self._beautify_newline() - def start_exporting(self): + def start_exporting(self) -> None: self.file.write(b"[") self._beautify_newline() - def finish_exporting(self): + def finish_exporting(self) -> None: self._beautify_newline() self.file.write(b"]") - def export_item(self, item): + def export_item(self, item: Any) -> None: itemdict = dict(self._get_serialized_fields(item)) data = to_bytes(self.encoder.encode(itemdict), self.encoding) self._add_comma_after_first() @@ -154,7 +165,7 @@ class JsonItemExporter(BaseItemExporter): class XmlItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): self.item_element = kwargs.pop("item_element", "item") self.root_element = kwargs.pop("root_element", "items") super().__init__(**kwargs) @@ -162,22 +173,22 @@ class XmlItemExporter(BaseItemExporter): self.encoding = "utf-8" self.xg = XMLGenerator(file, encoding=self.encoding) - def _beautify_newline(self, new_item=False): + def _beautify_newline(self, new_item: bool = False) -> None: if self.indent is not None and (self.indent > 0 or new_item): self.xg.characters("\n") - def _beautify_indent(self, depth=1): + def _beautify_indent(self, depth: int = 1) -> None: if self.indent: self.xg.characters(" " * self.indent * depth) - def start_exporting(self): + def start_exporting(self) -> None: self.xg.startDocument() - self.xg.startElement(self.root_element, {}) + self.xg.startElement(self.root_element, AttributesImpl({})) self._beautify_newline(new_item=True) - def export_item(self, item): + def export_item(self, item: Any) -> None: self._beautify_indent(depth=1) - self.xg.startElement(self.item_element, {}) + self.xg.startElement(self.item_element, AttributesImpl({})) self._beautify_newline() for name, value in self._get_serialized_fields(item, default_value=""): self._export_xml_field(name, value, depth=2) @@ -185,13 +196,13 @@ class XmlItemExporter(BaseItemExporter): self.xg.endElement(self.item_element) self._beautify_newline(new_item=True) - def finish_exporting(self): + def finish_exporting(self) -> None: self.xg.endElement(self.root_element) self.xg.endDocument() - def _export_xml_field(self, name, serialized_value, depth): + def _export_xml_field(self, name: str, serialized_value: Any, depth: int) -> None: self._beautify_indent(depth=depth) - self.xg.startElement(name, {}) + self.xg.startElement(name, AttributesImpl({})) if hasattr(serialized_value, "items"): self._beautify_newline() for subname, value in serialized_value.items(): @@ -213,17 +224,17 @@ class XmlItemExporter(BaseItemExporter): class CsvItemExporter(BaseItemExporter): def __init__( self, - file, - include_headers_line=True, - join_multivalued=",", - errors=None, - **kwargs, + file: BytesIO, + include_headers_line: bool = True, + join_multivalued: str = ",", + errors: str | None = None, + **kwargs: Any, ): super().__init__(dont_fail=True, **kwargs) if not self.encoding: self.encoding = "utf-8" self.include_headers_line = include_headers_line - self.stream = io.TextIOWrapper( + self.stream = TextIOWrapper( file, line_buffering=False, write_through=True, @@ -235,11 +246,13 @@ class CsvItemExporter(BaseItemExporter): self._headers_not_written = True self._join_multivalued = join_multivalued - def serialize_field(self, field, name, value): - serializer = field.get("serializer", self._join_if_needed) + def serialize_field( + self, field: Mapping[str, Any] | Field, name: str, value: Any + ) -> Any: + serializer: Callable[[Any], Any] = field.get("serializer", self._join_if_needed) return serializer(value) - def _join_if_needed(self, value): + def _join_if_needed(self, value: Any) -> Any: if isinstance(value, (list, tuple)): try: return self._join_multivalued.join(value) @@ -247,7 +260,7 @@ class CsvItemExporter(BaseItemExporter): pass return value - def export_item(self, item): + def export_item(self, item: Any) -> None: if self._headers_not_written: self._headers_not_written = False self._write_headers_and_set_fields_to_export(item) @@ -256,36 +269,38 @@ class CsvItemExporter(BaseItemExporter): values = list(self._build_row(x for _, x in fields)) self.csv_writer.writerow(values) - def finish_exporting(self): + def finish_exporting(self) -> None: self.stream.detach() # Avoid closing the wrapped file. - def _build_row(self, values): + def _build_row(self, values: Iterable[Any]) -> Iterable[Any]: for s in values: try: yield to_unicode(s, self.encoding) except TypeError: yield s - def _write_headers_and_set_fields_to_export(self, item): + def _write_headers_and_set_fields_to_export(self, item: Any) -> None: if self.include_headers_line: if not self.fields_to_export: # use declared field names, or keys if the item is a dict self.fields_to_export = ItemAdapter(item).field_names() + fields: Iterable[str] if isinstance(self.fields_to_export, Mapping): fields = self.fields_to_export.values() else: + assert self.fields_to_export fields = self.fields_to_export row = list(self._build_row(fields)) self.csv_writer.writerow(row) class PickleItemExporter(BaseItemExporter): - def __init__(self, file, protocol=4, **kwargs): + def __init__(self, file: BytesIO, protocol: int = 4, **kwargs: Any): super().__init__(**kwargs) - self.file = file - self.protocol = protocol + self.file: BytesIO = file + self.protocol: int = protocol - def export_item(self, item): + def export_item(self, item: Any) -> None: d = dict(self._get_serialized_fields(item)) pickle.dump(d, self.file, self.protocol) @@ -299,20 +314,20 @@ class MarshalItemExporter(BaseItemExporter): opened in binary mode, a :class:`~io.BytesIO` object, etc) """ - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(**kwargs) - self.file = file + self.file: BytesIO = file - def export_item(self, item): + def export_item(self, item: Any) -> None: marshal.dump(dict(self._get_serialized_fields(item)), self.file) class PprintItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(**kwargs) - self.file = file + self.file: BytesIO = file - def export_item(self, item): + def export_item(self, item: Any) -> None: itemdict = dict(self._get_serialized_fields(item)) self.file.write(to_bytes(pprint.pformat(itemdict) + "\n")) @@ -327,30 +342,34 @@ class PythonItemExporter(BaseItemExporter): .. _msgpack: https://pypi.org/project/msgpack/ """ - def _configure(self, options, dont_fail=False): + def _configure(self, options: dict[str, Any], dont_fail: bool = False) -> None: super()._configure(options, dont_fail) if not self.encoding: self.encoding = "utf-8" - def serialize_field(self, field, name, value): - serializer = field.get("serializer", self._serialize_value) + def serialize_field( + self, field: Mapping[str, Any] | Field, name: str, value: Any + ) -> Any: + serializer: Callable[[Any], Any] = field.get( + "serializer", self._serialize_value + ) return serializer(value) - def _serialize_value(self, value): + def _serialize_value(self, value: Any) -> Any: if isinstance(value, Item): return self.export_item(value) + if isinstance(value, (str, bytes)): + return to_unicode(value, encoding=self.encoding) if is_item(value): return dict(self._serialize_item(value)) if is_listlike(value): return [self._serialize_value(v) for v in value] - if isinstance(value, (str, bytes)): - return to_unicode(value, encoding=self.encoding) return value - def _serialize_item(self, item): + def _serialize_item(self, item: Any) -> Iterable[tuple[str | bytes, Any]]: for key, value in ItemAdapter(item).items(): yield key, self._serialize_value(value) - def export_item(self, item): - result = dict(self._get_serialized_fields(item)) + def export_item(self, item: Any) -> dict[str | bytes, Any]: # type: ignore[override] + result: dict[str | bytes, Any] = dict(self._get_serialized_fields(item)) return result diff --git a/scrapy/extension.py b/scrapy/extension.py index 4e365cfa1..05cce326b 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -3,13 +3,23 @@ The Extension Manager See documentation in docs/topics/extensions.rst """ + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list +if TYPE_CHECKING: + from scrapy.settings import Settings + class ExtensionManager(MiddlewareManager): component_name = "extension" @classmethod - def _get_mwlist_from_settings(cls, settings): - return build_component_list(settings.getwithbase("EXTENSIONS")) + def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]: + return build_component_list( + settings.get_component_priority_dict_with_base("EXTENSIONS") + ) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 4307b4170..a4362b182 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -4,93 +4,127 @@ conditions are met. See documentation in docs/topics/extensions.rst """ +from __future__ import annotations + import logging from collections import defaultdict +from typing import TYPE_CHECKING, Any -from scrapy import signals +from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + CallLaterResult, + call_later, + create_looping_call, +) +from scrapy.utils.defer import _schedule_coro + +if TYPE_CHECKING: + from twisted.internet.task import LoopingCall + from twisted.python.failure import Failure + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.http import Response + logger = logging.getLogger(__name__) class CloseSpider: - def __init__(self, crawler): - self.crawler = crawler + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler - self.close_on = { + # for CLOSESPIDER_TIMEOUT + self.task: CallLaterResult | None = None + + # for CLOSESPIDER_TIMEOUT_NO_ITEM + self.task_no_item: AsyncioLoopingCall | LoopingCall | None = None + + self.close_on: dict[str, Any] = { "timeout": crawler.settings.getfloat("CLOSESPIDER_TIMEOUT"), "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"), "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"), "errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"), "timeout_no_item": crawler.settings.getint("CLOSESPIDER_TIMEOUT_NO_ITEM"), + "pagecount_no_item": crawler.settings.getint( + "CLOSESPIDER_PAGECOUNT_NO_ITEM" + ), } if not any(self.close_on.values()): raise NotConfigured - self.counter = defaultdict(int) + self.counter: defaultdict[str, int] = defaultdict(int) if self.close_on.get("errorcount"): crawler.signals.connect(self.error_count, signal=signals.spider_error) - if self.close_on.get("pagecount"): + if self.close_on.get("pagecount") or self.close_on.get("pagecount_no_item"): crawler.signals.connect(self.page_count, signal=signals.response_received) if self.close_on.get("timeout"): crawler.signals.connect(self.spider_opened, signal=signals.spider_opened) - if self.close_on.get("itemcount"): + if self.close_on.get("itemcount") or self.close_on.get("pagecount_no_item"): crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) if self.close_on.get("timeout_no_item"): - self.timeout_no_item = self.close_on["timeout_no_item"] - self.items_in_period = 0 + self.timeout_no_item: int = self.close_on["timeout_no_item"] + self.items_in_period: int = 0 crawler.signals.connect( self.spider_opened_no_item, signal=signals.spider_opened ) crawler.signals.connect( self.item_scraped_no_item, signal=signals.item_scraped ) + crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def error_count(self, failure, response, spider): + def error_count(self, failure: Failure, response: Response, spider: Spider) -> None: self.counter["errorcount"] += 1 if self.counter["errorcount"] == self.close_on["errorcount"]: - self.crawler.engine.close_spider(spider, "closespider_errorcount") + self._close_spider("closespider_errorcount") - def page_count(self, response, request, spider): + def page_count(self, response: Response, request: Request, spider: Spider) -> None: self.counter["pagecount"] += 1 + self.counter["pagecount_since_last_item"] += 1 if self.counter["pagecount"] == self.close_on["pagecount"]: - self.crawler.engine.close_spider(spider, "closespider_pagecount") + self._close_spider("closespider_pagecount") + return + if self.close_on["pagecount_no_item"] and ( + self.counter["pagecount_since_last_item"] + >= self.close_on["pagecount_no_item"] + ): + self._close_spider("closespider_pagecount_no_item") - def spider_opened(self, spider): - from twisted.internet import reactor - - self.task = reactor.callLater( - self.close_on["timeout"], - self.crawler.engine.close_spider, - spider, - reason="closespider_timeout", + def spider_opened(self, spider: Spider) -> None: + assert self.crawler.engine + self.task = call_later( + self.close_on["timeout"], self._close_spider, "closespider_timeout" ) - def item_scraped(self, item, spider): + def item_scraped(self, item: Any, spider: Spider) -> None: self.counter["itemcount"] += 1 + self.counter["pagecount_since_last_item"] = 0 if self.counter["itemcount"] == self.close_on["itemcount"]: - self.crawler.engine.close_spider(spider, "closespider_itemcount") + self._close_spider("closespider_itemcount") - def spider_closed(self, spider): - task = getattr(self, "task", False) - if task and task.active(): - task.cancel() + def spider_closed(self, spider: Spider) -> None: + if self.task: + self.task.cancel() + self.task = None - task_no_item = getattr(self, "task_no_item", False) - if task_no_item and task_no_item.running: - task_no_item.stop() + if self.task_no_item: + if self.task_no_item.running: + self.task_no_item.stop() + self.task_no_item = None - def spider_opened_no_item(self, spider): - from twisted.internet import task - - self.task_no_item = task.LoopingCall(self._count_items_produced, spider) + def spider_opened_no_item(self, spider: Spider) -> None: + self.task_no_item = create_looping_call(self._count_items_produced) self.task_no_item.start(self.timeout_no_item, now=False) logger.info( @@ -98,10 +132,10 @@ class CloseSpider: f"{self.timeout_no_item} seconds." ) - def item_scraped_no_item(self, item, spider): + def item_scraped_no_item(self, item: Any, spider: Spider) -> None: self.items_in_period += 1 - def _count_items_produced(self, spider): + def _count_items_produced(self) -> None: if self.items_in_period >= 1: self.items_in_period = 0 else: @@ -109,4 +143,8 @@ class CloseSpider: f"Closing spider since no items were produced in the last " f"{self.timeout_no_item} seconds." ) - self.crawler.engine.close_spider(spider, "closespider_timeout_no_item") + self._close_spider("closespider_timeout_no_item") + + def _close_spider(self, reason: str) -> None: + assert self.crawler.engine + _schedule_coro(self.crawler.engine.close_spider_async(reason=reason)) diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 302a615f2..6a5e55992 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -1,18 +1,32 @@ """ Extension for collecting core stats like items scraped and start/finish times """ -from datetime import datetime, timezone -from scrapy import signals +from __future__ import annotations + +from datetime import datetime, timezone +from time import monotonic +from typing import TYPE_CHECKING, Any + +from scrapy import Spider, signals + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector class CoreStats: - def __init__(self, stats): - self.stats = stats - self.start_time = None + def __init__(self, stats: StatsCollector): + self.stats: StatsCollector = stats + self.start_time: datetime | None = None + self._start_time_mono: float | None = None @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: + assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) @@ -21,27 +35,27 @@ class CoreStats: crawler.signals.connect(o.response_received, signal=signals.response_received) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: self.start_time = datetime.now(tz=timezone.utc) - self.stats.set_value("start_time", self.start_time, spider=spider) + self._start_time_mono = monotonic() + self.stats.set_value("start_time", self.start_time) - def spider_closed(self, spider, reason): - finish_time = datetime.now(tz=timezone.utc) - elapsed_time = finish_time - self.start_time - elapsed_time_seconds = elapsed_time.total_seconds() - self.stats.set_value( - "elapsed_time_seconds", elapsed_time_seconds, spider=spider - ) - self.stats.set_value("finish_time", finish_time, spider=spider) - self.stats.set_value("finish_reason", reason, spider=spider) + def spider_closed(self, spider: Spider, reason: str) -> None: + assert self.start_time is not None + assert self._start_time_mono is not None + finish_time, finish_time_mono = datetime.now(tz=timezone.utc), monotonic() + elapsed_time_seconds = finish_time_mono - self._start_time_mono + self.stats.set_value("elapsed_time_seconds", elapsed_time_seconds) + self.stats.set_value("finish_time", finish_time) + self.stats.set_value("finish_reason", reason) - def item_scraped(self, item, spider): - self.stats.inc_value("item_scraped_count", spider=spider) + def item_scraped(self, item: Any, spider: Spider) -> None: + self.stats.inc_value("item_scraped_count") - def response_received(self, spider): - self.stats.inc_value("response_received_count", spider=spider) + def response_received(self, spider: Spider) -> None: + self.stats.inc_value("response_received_count") - def item_dropped(self, item, spider, exception): + def item_dropped(self, item: Any, spider: Spider, exception: BaseException) -> None: reason = exception.__class__.__name__ - self.stats.inc_value("item_dropped_count", spider=spider) - self.stats.inc_value(f"item_dropped_reasons_count/{reason}", spider=spider) + self.stats.inc_value("item_dropped_count") + self.stats.inc_value(f"item_dropped_reasons_count/{reason}") diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 1b6c7777f..5def7509e 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -4,34 +4,48 @@ Extensions for debugging Scrapy See documentation in docs/topics/extensions.rst """ +from __future__ import annotations + +import contextlib import logging import signal import sys import threading import traceback from pdb import Pdb +from typing import TYPE_CHECKING from scrapy.utils.engine import format_engine_status from scrapy.utils.trackref import format_live_refs +if TYPE_CHECKING: + from types import FrameType + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) class StackTraceDump: - def __init__(self, crawler=None): - self.crawler = crawler + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler try: - signal.signal(signal.SIGUSR2, self.dump_stacktrace) - signal.signal(signal.SIGQUIT, self.dump_stacktrace) + signal.signal(signal.SIGUSR2, self.dump_stacktrace) # type: ignore[attr-defined,unused-ignore] + signal.signal(signal.SIGQUIT, self.dump_stacktrace) # type: ignore[attr-defined,unused-ignore] except AttributeError: # win32 platforms don't support SIGUSR signals pass @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def dump_stacktrace(self, signum, frame): + def dump_stacktrace(self, signum: int, frame: FrameType | None) -> None: + assert self.crawler.engine log_args = { "stackdumps": self._thread_stacks(), "enginestatus": format_engine_status(self.crawler.engine), @@ -44,8 +58,8 @@ class StackTraceDump: extra={"crawler": self.crawler}, ) - def _thread_stacks(self): - id2name = dict((th.ident, th.name) for th in threading.enumerate()) + def _thread_stacks(self) -> str: + id2name = {th.ident: th.name for th in threading.enumerate()} dumps = "" for id_, frame in sys._current_frames().items(): name = id2name.get(id_, "") @@ -55,12 +69,11 @@ class StackTraceDump: class Debugger: - def __init__(self): - try: - signal.signal(signal.SIGUSR2, self._enter_debugger) - except AttributeError: - # win32 platforms don't support SIGUSR signals - pass + def __init__(self) -> None: + # win32 platforms don't support SIGUSR signals + with contextlib.suppress(AttributeError): + signal.signal(signal.SIGUSR2, self._enter_debugger) # type: ignore[attr-defined,unused-ignore] - def _enter_debugger(self, signum, frame): + def _enter_debugger(self, signum: int, frame: FrameType | None) -> None: + assert frame Pdb().set_trace(frame.f_back) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e5e363b52..8029f85c9 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -4,46 +4,52 @@ Feed Exports extension See documentation in docs/topics/feed-exports.rst """ +from __future__ import annotations + +import asyncio +import contextlib import logging import re import sys import warnings +from abc import ABC, abstractmethod +from collections.abc import Callable, Coroutine from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union +from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, cast from urllib.parse import unquote, urlparse -from twisted.internet import defer, threads -from twisted.internet.defer import DeferredList +from twisted.internet.defer import Deferred, DeferredList from w3lib.url import file_uri_to_path from zope.interface import Interface, implementer from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager -from scrapy.utils.boto import is_botocore_available +from scrapy.utils.asyncio import is_asyncio_available, run_in_thread from scrapy.utils.conf import feed_complete_default_values_from_settings -from scrapy.utils.defer import maybe_deferred_to_future -from scrapy.utils.deprecate import create_deprecated_class +from scrapy.utils.defer import deferred_from_coro, ensure_awaitable from scrapy.utils.ftp import ftp_store_file -from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values +if TYPE_CHECKING: + from _typeshed import OpenBinaryMode + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.exporters import BaseItemExporter + from scrapy.settings import BaseSettings, Settings + + logger = logging.getLogger(__name__) -try: - import boto3 # noqa: F401 - - IS_BOTO3_AVAILABLE = True -except ImportError: - IS_BOTO3_AVAILABLE = False - - -def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): - kwargs["feed_options"] = feed_options - return builder(*preargs, uri, *args, **kwargs) +UriParamsCallableT: TypeAlias = Callable[ + [dict[str, Any], Spider], dict[str, Any] | None +] class ItemFilter: @@ -55,10 +61,10 @@ class ItemFilter: :type feed_options: dict """ - feed_options: Optional[dict] - item_classes: Tuple + feed_options: dict[str, Any] | None + item_classes: tuple[type, ...] - def __init__(self, feed_options: Optional[dict]) -> None: + def __init__(self, feed_options: dict[str, Any] | None) -> None: self.feed_options = feed_options if feed_options is not None: self.item_classes = tuple( @@ -66,7 +72,7 @@ class ItemFilter: for item_class in feed_options.get("item_classes") or () ) else: - self.item_classes = tuple() + self.item_classes = () def accepts(self, item: Any) -> bool: """ @@ -82,43 +88,67 @@ class ItemFilter: return True # accept all items by default -class IFeedStorage(Interface): +class IFeedStorage(Interface): # type: ignore[misc] """Interface that all Feed Storages must implement""" - def __init__(uri, *, feed_options=None): + # pylint: disable=no-self-argument + + def __init__(uri, *, feed_options=None): # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called """Initialize the storage with the parameters given in the URI and the feed-specific options (see :setting:`FEEDS`)""" - def open(spider): + def open(spider): # type: ignore[no-untyped-def] """Open the storage for the given spider. It must return a file-like object that will be used for the exporters""" - def store(file): + def store(file): # type: ignore[no-untyped-def] + """Store the given file stream""" + + +class FeedStorageProtocol(Protocol): + """Reimplementation of ``IFeedStorage`` that can be used in type hints.""" + + def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None): + """Initialize the storage with the parameters given in the URI and the + feed-specific options (see :setting:`FEEDS`)""" + + def open(self, spider: Spider) -> IO[bytes]: + """Open the storage for the given spider. It must return a file-like + object that will be used for the exporters""" + + def store(self, file: IO[bytes]) -> Deferred[None] | None: """Store the given file stream""" @implementer(IFeedStorage) -class BlockingFeedStorage: - def open(self, spider): +class BlockingFeedStorage(ABC): + def open(self, spider: Spider) -> IO[bytes]: path = spider.crawler.settings["FEED_TEMPDIR"] if path and not Path(path).is_dir(): raise OSError("Not a Directory: " + str(path)) return NamedTemporaryFile(prefix="feed-", dir=path) - def store(self, file): - return threads.deferToThread(self._store_in_thread, file) + def store(self, file: IO[bytes]) -> Deferred[None] | None: + return deferred_from_coro(run_in_thread(self._store_in_thread, file)) - def _store_in_thread(self, file): + @abstractmethod + def _store_in_thread(self, file: IO[bytes]) -> None: raise NotImplementedError @implementer(IFeedStorage) class StdoutFeedStorage: - def __init__(self, uri, _stdout=None, *, feed_options=None): + def __init__( + self, + uri: str, + _stdout: IO[bytes] | None = None, + *, + feed_options: dict[str, Any] | None = None, + ): if not _stdout: _stdout = sys.stdout.buffer - self._stdout = _stdout + self._stdout: IO[bytes] = _stdout if feed_options and feed_options.get("overwrite", False) is True: logger.warning( "Standard output (stdout) storage does not support " @@ -127,87 +157,70 @@ class StdoutFeedStorage: "it to False." ) - def open(self, spider): + def open(self, spider: Spider) -> IO[bytes]: return self._stdout - def store(self, file): + def store(self, file: IO[bytes]) -> Deferred[None] | None: pass @implementer(IFeedStorage) class FileFeedStorage: - def __init__(self, uri, *, feed_options=None): - self.path = file_uri_to_path(uri) + def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None): + self.path: str = file_uri_to_path(uri) if uri.startswith("file:") else uri feed_options = feed_options or {} - self.write_mode = "wb" if feed_options.get("overwrite", False) else "ab" + self.write_mode: OpenBinaryMode = ( + "wb" if feed_options.get("overwrite", False) else "ab" + ) - def open(self, spider) -> IO[Any]: + def open(self, spider: Spider) -> IO[bytes]: dirname = Path(self.path).parent if dirname and not dirname.exists(): dirname.mkdir(parents=True) return Path(self.path).open(self.write_mode) - def store(self, file): + def store(self, file: IO[bytes]) -> Deferred[None] | None: file.close() + return None class S3FeedStorage(BlockingFeedStorage): def __init__( self, - uri, - access_key=None, - secret_key=None, - acl=None, - endpoint_url=None, + uri: str, + access_key: str | None = None, + secret_key: str | None = None, + acl: str | None = None, + endpoint_url: str | None = None, *, - feed_options=None, - session_token=None, - region_name=None, + feed_options: dict[str, Any] | None = None, + session_token: str | None = None, + region_name: str | None = None, ): - if not is_botocore_available(): - raise NotConfigured("missing botocore library") + try: + import boto3.session # noqa: PLC0415 + except ImportError: + raise NotConfigured("missing boto3 library") from None u = urlparse(uri) - self.bucketname = u.hostname - self.access_key = u.username or access_key - self.secret_key = u.password or secret_key - self.session_token = session_token - self.keyname = u.path[1:] # remove first "/" - self.acl = acl - self.endpoint_url = endpoint_url - self.region_name = region_name + assert u.hostname + self.bucketname: str = u.hostname + self.access_key: str | None = u.username or access_key + self.secret_key: str | None = u.password or secret_key + self.session_token: str | None = session_token + self.keyname: str = u.path[1:] # remove first "/" + self.acl: str | None = acl + self.endpoint_url: str | None = endpoint_url + self.region_name: str | None = region_name - if IS_BOTO3_AVAILABLE: - import boto3.session - - session = boto3.session.Session() - - self.s3_client = session.client( - "s3", - aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key, - aws_session_token=self.session_token, - endpoint_url=self.endpoint_url, - region_name=self.region_name, - ) - else: - warnings.warn( - "`botocore` usage has been deprecated for S3 feed " - "export, please use `boto3` to avoid problems", - category=ScrapyDeprecationWarning, - ) - - import botocore.session - - session = botocore.session.get_session() - - self.s3_client = session.create_client( - "s3", - aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key, - aws_session_token=self.session_token, - endpoint_url=self.endpoint_url, - region_name=self.region_name, - ) + boto3_session = boto3.session.Session() + self.s3_client = boto3_session.client( + "s3", + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, + endpoint_url=self.endpoint_url, + region_name=self.region_name, + ) if feed_options and feed_options.get("overwrite", True) is False: logger.warning( @@ -217,9 +230,14 @@ class S3FeedStorage(BlockingFeedStorage): ) @classmethod - def from_crawler(cls, crawler, uri, *, feed_options=None): - return build_storage( - cls, + def from_crawler( + cls, + crawler: Crawler, + uri: str, + *, + feed_options: dict[str, Any] | None = None, + ) -> Self: + return cls( uri, access_key=crawler.settings["AWS_ACCESS_KEY_ID"], secret_key=crawler.settings["AWS_SECRET_ACCESS_KEY"], @@ -230,45 +248,73 @@ class S3FeedStorage(BlockingFeedStorage): feed_options=feed_options, ) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) - if IS_BOTO3_AVAILABLE: - kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} + if self.acl: self.s3_client.upload_fileobj( - Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs + Bucket=self.bucketname, + Key=self.keyname, + Fileobj=file, + ExtraArgs={"ACL": self.acl}, ) else: - kwargs = {"ACL": self.acl} if self.acl else {} - self.s3_client.put_object( - Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs + self.s3_client.upload_fileobj( + Bucket=self.bucketname, + Key=self.keyname, + Fileobj=file, ) file.close() class GCSFeedStorage(BlockingFeedStorage): - def __init__(self, uri, project_id, acl): - self.project_id = project_id - self.acl = acl + def __init__( + self, + uri: str, + project_id: str | None, + acl: str | None, + *, + feed_options: dict[str, Any] | None = None, + ): + self.project_id: str | None = project_id + self.acl: str | None = acl u = urlparse(uri) - self.bucket_name = u.hostname - self.blob_name = u.path[1:] # remove first "/" + assert u.hostname + self.bucket_name: str = u.hostname + self.blob_name: str = u.path[1:] # remove first "/" + + if feed_options and feed_options.get("overwrite", True) is False: + logger.warning( + "GCS does not support appending to files. To " + "suppress this warning, remove the overwrite " + "option from your FEEDS setting or set it to True." + ) @classmethod - def from_crawler(cls, crawler, uri): + def from_crawler( + cls, + crawler: Crawler, + uri: str, + *, + feed_options: dict[str, Any] | None = None, + ) -> Self: return cls( uri, crawler.settings["GCS_PROJECT_ID"], crawler.settings["FEED_STORAGE_GCS_ACL"] or None, + feed_options=feed_options, ) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) - from google.cloud.storage import Client + try: + from google.cloud.storage import Client # noqa: PLC0415 - client = Client(project=self.project_id) - bucket = client.get_bucket(self.bucket_name) - blob = bucket.blob(self.blob_name) - blob.upload_from_file(file, predefined_acl=self.acl) + client = Client(project=self.project_id) + bucket = client.get_bucket(self.bucket_name) + blob = bucket.blob(self.blob_name) + blob.upload_from_file(file, predefined_acl=self.acl) + finally: + file.close() class FTPFeedStorage(BlockingFeedStorage): @@ -277,7 +323,7 @@ class FTPFeedStorage(BlockingFeedStorage): uri: str, use_active_mode: bool = False, *, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: dict[str, Any] | None = None, ): u = urlparse(uri) if not u.hostname: @@ -291,15 +337,20 @@ class FTPFeedStorage(BlockingFeedStorage): self.overwrite: bool = not feed_options or feed_options.get("overwrite", True) @classmethod - def from_crawler(cls, crawler, uri, *, feed_options=None): - return build_storage( - cls, + def from_crawler( + cls, + crawler: Crawler, + uri: str, + *, + feed_options: dict[str, Any] | None = None, + ) -> Self: + return cls( uri, - crawler.settings.getbool("FEED_STORAGE_FTP_ACTIVE"), + use_active_mode=crawler.settings.getbool("FEED_STORAGE_FTP_ACTIVE"), feed_options=feed_options, ) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: ftp_store_file( path=self.path, file=file, @@ -315,50 +366,55 @@ class FTPFeedStorage(BlockingFeedStorage): class FeedSlot: def __init__( self, - storage, - uri, - format, - store_empty, - batch_id, - uri_template, - filter, - feed_options, - spider, - exporters, - settings, - crawler, + storage: FeedStorageProtocol, + uri: str, + format: str, # noqa: A002 + store_empty: bool, + batch_id: int, + uri_template: str, + filter: ItemFilter, # noqa: A002 + feed_options: dict[str, Any], + spider: Spider, + exporters: dict[str, type[BaseItemExporter]], + settings: BaseSettings, + crawler: Crawler, ): - self.file = None - self.exporter = None - self.storage = storage + self.file: IO[bytes] | None = None + self.exporter: BaseItemExporter | None = None + self.storage: FeedStorageProtocol = storage # feed params - self.batch_id = batch_id - self.format = format - self.store_empty = store_empty - self.uri_template = uri_template - self.uri = uri - self.filter = filter + self.batch_id: int = batch_id + self.format: str = format + self.store_empty: bool = store_empty + self.uri_template: str = uri_template + self.uri: str = uri + self.filter: ItemFilter = filter # exporter params - self.feed_options = feed_options - self.spider = spider - self.exporters = exporters - self.settings = settings - self.crawler = crawler + self.feed_options: dict[str, Any] = feed_options + self.spider: Spider = spider + self.exporters: dict[str, type[BaseItemExporter]] = exporters + self.settings: BaseSettings = settings + self.crawler: Crawler = crawler # flags - self.itemcount = 0 - self._exporting = False - self._fileloaded = False + self.itemcount: int = 0 + self._exporting: bool = False + self._fileloaded: bool = False - def start_exporting(self): + def start_exporting(self) -> None: if not self._fileloaded: self.file = self.storage.open(self.spider) if "postprocessing" in self.feed_options: - self.file = PostProcessingManager( - self.feed_options["postprocessing"], self.file, self.feed_options + self.file = cast( + "IO[bytes]", + PostProcessingManager( + self.feed_options["postprocessing"], + self.file, + self.feed_options, + ), ) self.exporter = self._get_exporter( file=self.file, - format=self.feed_options["format"], + format_=self.feed_options["format"], fields_to_export=self.feed_options["fields"], encoding=self.feed_options["encoding"], indent=self.feed_options["indent"], @@ -367,44 +423,40 @@ class FeedSlot: self._fileloaded = True if not self._exporting: + assert self.exporter self.exporter.start_exporting() self._exporting = True - def _get_instance(self, objcls, *args, **kwargs): - return build_from_crawler(objcls, self.crawler, *args, **kwargs) + def _get_exporter( + self, file: IO[bytes], format_: str, *args: Any, **kwargs: Any + ) -> BaseItemExporter: + return build_from_crawler( + self.exporters[format_], self.crawler, file, *args, **kwargs + ) - def _get_exporter(self, file, format, *args, **kwargs): - return self._get_instance(self.exporters[format], file, *args, **kwargs) - - def finish_exporting(self): + def finish_exporting(self) -> None: if self._exporting: + assert self.exporter self.exporter.finish_exporting() self._exporting = False -_FeedSlot = create_deprecated_class( - name="_FeedSlot", - new_class=FeedSlot, -) - - class FeedExporter: - _pending_deferreds: List[defer.Deferred] = [] - @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: exporter = cls(crawler) crawler.signals.connect(exporter.open_spider, signals.spider_opened) crawler.signals.connect(exporter.close_spider, signals.spider_closed) crawler.signals.connect(exporter.item_scraped, signals.item_scraped) return exporter - def __init__(self, crawler): - self.crawler = crawler - self.settings = crawler.settings + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler + self.settings: Settings = crawler.settings self.feeds = {} - self.slots = [] - self.filters = {} + self.slots: list[FeedSlot] = [] + self.filters: dict[str, ItemFilter] = {} + self._pending_close_coros: list[Coroutine[Any, Any, None]] = [] if not self.settings["FEEDS"] and not self.settings["FEED_URI"]: raise NotConfigured @@ -420,7 +472,7 @@ class FeedExporter: uri = self.settings["FEED_URI"] # handle pathlib.Path objects uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri() - feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")} + feed_options = {"format": self.settings["FEED_FORMAT"]} self.feeds[uri] = feed_complete_default_values_from_settings( feed_options, self.settings ) @@ -428,16 +480,24 @@ class FeedExporter: # End: Backward compatibility for FEED_URI and FEED_FORMAT settings # 'FEEDS' setting takes precedence over 'FEED_URI' - for uri, feed_options in self.settings.getdict("FEEDS").items(): + for settings_uri, feed_options in self.settings.getdict("FEEDS").items(): # handle pathlib.Path objects - uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri() + uri = ( + str(settings_uri) + if not isinstance(settings_uri, Path) + else settings_uri.absolute().as_uri() + ) self.feeds[uri] = feed_complete_default_values_from_settings( feed_options, self.settings ) self.filters[uri] = self._load_filter(feed_options) - self.storages = self._load_components("FEED_STORAGES") - self.exporters = self._load_components("FEED_EXPORTERS") + self.storages: dict[str, type[FeedStorageProtocol]] = self._load_components( + "FEED_STORAGES" + ) + self.exporters: dict[str, type[BaseItemExporter]] = self._load_components( + "FEED_EXPORTERS" + ) for uri, feed_options in self.feeds.items(): if not self._storage_supported(uri, feed_options): raise NotConfigured @@ -446,7 +506,7 @@ class FeedExporter: if not self._exporter_supported(feed_options["format"]): raise NotConfigured - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> None: for uri, feed_options in self.feeds.items(): uri_params = self._get_uri_params(spider, feed_options["uri_params"]) self.slots.append( @@ -459,25 +519,33 @@ class FeedExporter: ) ) - async def close_spider(self, spider): - for slot in self.slots: - self._close_slot(slot, spider) - - # Await all deferreds - if self._pending_deferreds: - await maybe_deferred_to_future(DeferredList(self._pending_deferreds)) - - # Send FEED_EXPORTER_CLOSED signal - await maybe_deferred_to_future( - self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed) + async def close_spider(self, spider: Spider) -> None: + self._pending_close_coros.extend( + self._close_slot(slot, spider) for slot in self.slots ) - def _close_slot(self, slot, spider): - def get_file(slot_): - if isinstance(slot_.file, PostProcessingManager): - slot_.file.close() - return slot_.file.file - return slot_.file + if self._pending_close_coros: + if is_asyncio_available(): + await asyncio.wait( + [asyncio.create_task(coro) for coro in self._pending_close_coros] + ) + else: + await DeferredList( + deferred_from_coro(coro) for coro in self._pending_close_coros + ) + + # Send FEED_EXPORTER_CLOSED signal + await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed) + + @staticmethod + def _get_file(slot_: FeedSlot) -> IO[bytes]: + assert slot_.file + if isinstance(slot_.file, PostProcessingManager): + slot_.file.close() + return slot_.file.file + return slot_.file + + async def _close_slot(self, slot: FeedSlot, spider: Spider) -> None: if slot.itemcount: # Normal case @@ -488,41 +556,37 @@ class FeedExporter: slot.finish_exporting() else: # In this case, the file is not stored, so no processing is required. - return None + return logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" - d = defer.maybeDeferred(slot.storage.store, get_file(slot)) - - d.addCallback( - self._handle_store_success, logmsg, spider, type(slot.storage).__name__ - ) - d.addErrback( - self._handle_store_error, logmsg, spider, type(slot.storage).__name__ - ) - self._pending_deferreds.append(d) - d.addCallback( - lambda _: self.crawler.signals.send_catch_log_deferred( - signals.feed_slot_closed, slot=slot + slot_type = type(slot.storage).__name__ + assert self.crawler.stats + try: + await ensure_awaitable(slot.storage.store(self._get_file(slot))) + except Exception: + logger.error( + "Error storing %s", + logmsg, + exc_info=True, + extra={"spider": spider}, ) + self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") + else: + logger.info("Stored %s", logmsg, extra={"spider": spider}) + self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}") + + await self.crawler.signals.send_catch_log_async( + signals.feed_slot_closed, slot=slot ) - d.addBoth(lambda _: self._pending_deferreds.remove(d)) - return d - - def _handle_store_error(self, f, logmsg, spider, slot_type): - logger.error( - "Error storing %s", - logmsg, - exc_info=failure_to_exc_info(f), - extra={"spider": spider}, - ) - self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") - - def _handle_store_success(self, f, logmsg, spider, slot_type): - logger.info("Stored %s", logmsg, extra={"spider": spider}) - self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}") - - def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template): + def _start_new_batch( + self, + batch_id: int, + uri: str, + feed_options: dict[str, Any], + spider: Spider, + uri_template: str, + ) -> FeedSlot: """ Redirect the output data stream to a new file. Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified @@ -533,7 +597,7 @@ class FeedExporter: :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri """ storage = self._get_storage(uri, feed_options) - slot = FeedSlot( + return FeedSlot( storage=storage, uri=uri, format=feed_options["format"], @@ -545,11 +609,10 @@ class FeedExporter: spider=spider, exporters=self.exporters, settings=self.settings, - crawler=getattr(self, "crawler", None), + crawler=self.crawler, ) - return slot - def item_scraped(self, item, spider): + def item_scraped(self, item: Any, spider: Spider) -> None: slots = [] for slot in self.slots: if not slot.filter.accepts(item): @@ -559,6 +622,7 @@ class FeedExporter: continue slot.start_exporting() + assert slot.exporter slot.exporter.export_item(item) slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one @@ -569,7 +633,7 @@ class FeedExporter: uri_params = self._get_uri_params( spider, self.feeds[slot.uri_template]["uri_params"], slot ) - self._close_slot(slot, spider) + self._pending_close_coros.append(self._close_slot(slot, spider)) slots.append( self._start_new_batch( batch_id=slot.batch_id + 1, @@ -583,22 +647,23 @@ class FeedExporter: slots.append(slot) self.slots = slots - def _load_components(self, setting_prefix): - conf = without_none_values(self.settings.getwithbase(setting_prefix)) + def _load_components(self, setting_prefix: str) -> dict[str, Any]: + conf = without_none_values( + cast("dict[str, str]", self.settings.getwithbase(setting_prefix)) + ) d = {} for k, v in conf.items(): - try: + with contextlib.suppress(NotConfigured): d[k] = load_object(v) - except NotConfigured: - pass return d - def _exporter_supported(self, format): - if format in self.exporters: + def _exporter_supported(self, format_: str) -> bool: + if format_ in self.exporters: return True - logger.error("Unknown feed format: %(format)s", {"format": format}) + logger.error("Unknown feed format: %(format)s", {"format": format_}) + return False - def _settings_are_valid(self): + def _settings_are_valid(self) -> bool: """ If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain %(batch_time)s or %(batch_id)d to distinguish different files of partial output @@ -616,7 +681,7 @@ class FeedExporter: return False return True - def _storage_supported(self, uri, feed_options): + def _storage_supported(self, uri: str, feed_options: dict[str, Any]) -> bool: scheme = urlparse(uri).scheme if scheme in self.storages or PureWindowsPath(uri).drive: try: @@ -624,53 +689,33 @@ class FeedExporter: return True except NotConfigured as e: logger.error( - "Disabled feed storage scheme: %(scheme)s. " "Reason: %(reason)s", + "Disabled feed storage scheme: %(scheme)s. Reason: %(reason)s", {"scheme": scheme, "reason": str(e)}, ) else: logger.error("Unknown feed storage scheme: %(scheme)s", {"scheme": scheme}) + return False - def _get_storage(self, uri, feed_options): - """Fork of create_instance specific to feed storage classes - - It supports not passing the *feed_options* parameters to classes that - do not support it, and issuing a deprecation warning instead. - """ - feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"]) - crawler = getattr(self, "crawler", None) - - def build_instance(builder, *preargs): - return build_storage( - builder, uri, feed_options=feed_options, preargs=preargs - ) - - if crawler and hasattr(feedcls, "from_crawler"): - instance = build_instance(feedcls.from_crawler, crawler) - method_name = "from_crawler" - elif hasattr(feedcls, "from_settings"): - instance = build_instance(feedcls.from_settings, self.settings) - method_name = "from_settings" - else: - instance = build_instance(feedcls) - method_name = "__new__" - if instance is None: - raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None") - return instance + def _get_storage( + self, uri: str, feed_options: dict[str, Any] + ) -> FeedStorageProtocol: + """Build a storage object for the specified *uri* with the specified + *feed_options*.""" + cls = self.storages.get(urlparse(uri).scheme, self.storages["file"]) + return build_from_crawler(cls, self.crawler, uri, feed_options=feed_options) def _get_uri_params( self, spider: Spider, - uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]], - slot: Optional[FeedSlot] = None, - ) -> dict: - params = {} - for k in dir(spider): - params[k] = getattr(spider, k) + uri_params_function: str | UriParamsCallableT | None, + slot: FeedSlot | None = None, + ) -> dict[str, Any]: + params = {k: getattr(spider, k) for k in dir(spider)} utc_now = datetime.now(tz=timezone.utc) params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-") params["batch_time"] = utc_now.isoformat().replace(":", "-") params["batch_id"] = slot.batch_id + 1 if slot is not None else 1 - uripar_function = ( + uripar_function: UriParamsCallableT = ( load_object(uri_params_function) if uri_params_function else lambda params, _: params @@ -678,7 +723,9 @@ class FeedExporter: new_params = uripar_function(params, spider) return new_params if new_params is not None else params - def _load_filter(self, feed_options): + def _load_filter(self, feed_options: dict[str, Any]) -> ItemFilter: # load the item filter if declared else load the default filter class - item_filter_class = load_object(feed_options.get("item_filter", ItemFilter)) + item_filter_class: type[ItemFilter] = load_object( + feed_options.get("item_filter", ItemFilter) + ) return item_filter_class(feed_options) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 243d289e6..3fb9d37e9 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import gzip import logging import pickle @@ -5,56 +7,73 @@ from email.utils import mktime_tz, parsedate_tz from importlib import import_module from pathlib import Path from time import time +from typing import IO, TYPE_CHECKING, Any, Concatenate, cast from weakref import WeakKeyDictionary from w3lib.http import headers_dict_to_raw, headers_raw_to_dict from scrapy.http import Headers, Response -from scrapy.http.request import Request -from scrapy.spiders import Spider from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.response import get_response_class +if TYPE_CHECKING: + import os + from collections.abc import Callable + from types import ModuleType + + from scrapy.http.request import Request + from scrapy.settings import BaseSettings + from scrapy.spiders import Spider + from scrapy.utils.request import RequestFingerprinterProtocol + + logger = logging.getLogger(__name__) class DummyPolicy: - def __init__(self, settings): - self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") - self.ignore_http_codes = [ + def __init__(self, settings: BaseSettings): + self.ignore_schemes: list[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") + self.ignore_http_codes: list[int] = [ int(x) for x in settings.getlist("HTTPCACHE_IGNORE_HTTP_CODES") ] - def should_cache_request(self, request): + def should_cache_request(self, request: Request) -> bool: return urlparse_cached(request).scheme not in self.ignore_schemes - def should_cache_response(self, response, request): + def should_cache_response(self, response: Response, request: Request) -> bool: return response.status not in self.ignore_http_codes - def is_cached_response_fresh(self, cachedresponse, request): + def is_cached_response_fresh( + self, cachedresponse: Response, request: Request + ) -> bool: return True - def is_cached_response_valid(self, cachedresponse, response, request): + def is_cached_response_valid( + self, cachedresponse: Response, response: Response, request: Request + ) -> bool: return True class RFC2616Policy: MAXAGE = 3600 * 24 * 365 # one year - def __init__(self, settings): - self.always_store = settings.getbool("HTTPCACHE_ALWAYS_STORE") - self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") - self._cc_parsed = WeakKeyDictionary() - self.ignore_response_cache_controls = [ + def __init__(self, settings: BaseSettings): + self.always_store: bool = settings.getbool("HTTPCACHE_ALWAYS_STORE") + self.ignore_schemes: list[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") + self._cc_parsed: WeakKeyDictionary[ + Request | Response, dict[bytes, bytes | None] + ] = WeakKeyDictionary() + self.ignore_response_cache_controls: list[bytes] = [ to_bytes(cc) for cc in settings.getlist("HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS") ] - def _parse_cachecontrol(self, r): + def _parse_cachecontrol(self, r: Request | Response) -> dict[bytes, bytes | None]: if r not in self._cc_parsed: cch = r.headers.get(b"Cache-Control", b"") + assert cch is not None parsed = parse_cachecontrol(cch) if isinstance(r, Response): for key in self.ignore_response_cache_controls: @@ -62,17 +81,14 @@ class RFC2616Policy: self._cc_parsed[r] = parsed return self._cc_parsed[r] - def should_cache_request(self, request): + def should_cache_request(self, request: Request) -> bool: if urlparse_cached(request).scheme in self.ignore_schemes: return False cc = self._parse_cachecontrol(request) # obey user-agent directive "Cache-Control: no-store" - if b"no-store" in cc: - return False - # Any other is eligible for caching - return True + return b"no-store" not in cc - def should_cache_response(self, response, request): + def should_cache_response(self, response: Response, request: Request) -> bool: # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 # Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 # Status code 206 is not included because cache can not deal with partial contents @@ -90,17 +106,19 @@ class RFC2616Policy: if b"max-age" in cc or b"Expires" in response.headers: return True # Firefox fallbacks this statuses to one year expiration if none is set - if response.status in (300, 301, 308): + if response.status in {300, 301, 308}: return True # Other statuses without expiration requires at least one validator - if response.status in (200, 203, 401): + if response.status in {200, 203, 401}: return b"Last-Modified" in response.headers or b"ETag" in response.headers # Any other is probably not eligible for caching # Makes no sense to cache responses that does not contain expiration # info and can not be revalidated return False - def is_cached_response_fresh(self, cachedresponse, request): + def is_cached_response_fresh( + self, cachedresponse: Response, request: Request + ) -> bool: cc = self._parse_cachecontrol(cachedresponse) ccreq = self._parse_cachecontrol(request) if b"no-cache" in cc or b"no-cache" in ccreq: @@ -141,7 +159,9 @@ class RFC2616Policy: self._set_conditional_validators(request, cachedresponse) return False - def is_cached_response_valid(self, cachedresponse, response, request): + def is_cached_response_valid( + self, cachedresponse: Response, response: Response, request: Request + ) -> bool: # Use the cached response if the new response is a server error, # as long as the old response didn't specify must-revalidate. if response.status >= 500: @@ -152,7 +172,9 @@ class RFC2616Policy: # Use the cached response if the server says it hasn't changed. return response.status == 304 - def _set_conditional_validators(self, request, cachedresponse): + def _set_conditional_validators( + self, request: Request, cachedresponse: Response + ) -> None: if b"Last-Modified" in cachedresponse.headers: request.headers[b"If-Modified-Since"] = cachedresponse.headers[ b"Last-Modified" @@ -161,13 +183,15 @@ class RFC2616Policy: if b"ETag" in cachedresponse.headers: request.headers[b"If-None-Match"] = cachedresponse.headers[b"ETag"] - def _get_max_age(self, cc): + def _get_max_age(self, cc: dict[bytes, bytes | None]) -> int | None: try: - return max(0, int(cc[b"max-age"])) + return max(0, int(cc[b"max-age"])) # type: ignore[arg-type] except (KeyError, ValueError): return None - def _compute_freshness_lifetime(self, response, request, now): + def _compute_freshness_lifetime( + self, response: Response, request: Request, now: float + ) -> float: # Reference nsHttpResponseHead::ComputeFreshnessLifetime # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#706 cc = self._parse_cachecontrol(response) @@ -192,16 +216,18 @@ class RFC2616Policy: return (date - lastmodified) / 10 # This request can be cached indefinitely - if response.status in (300, 301, 308): + if response.status in {300, 301, 308}: return self.MAXAGE # Insufficient information to compute freshness lifetime return 0 - def _compute_current_age(self, response, request, now): + def _compute_current_age( + self, response: Response, request: Request, now: float + ) -> float: # Reference nsHttpResponseHead::ComputeCurrentAge # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#658 - currentage = 0 + currentage: float = 0 # If Date header is not set we assume it is a fast connection, and # clock is in sync with the server date = rfc1123_to_epoch(response.headers.get(b"Date")) or now @@ -210,7 +236,7 @@ class RFC2616Policy: if b"Age" in response.headers: try: - age = int(response.headers[b"Age"]) + age = int(response.headers[b"Age"]) # type: ignore[arg-type] currentage = max(currentage, age) except ValueError: pass @@ -219,13 +245,13 @@ class RFC2616Policy: class DbmCacheStorage: - def __init__(self, settings): - self.cachedir = data_path(settings["HTTPCACHE_DIR"], createdir=True) - self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS") - self.dbmodule = import_module(settings["HTTPCACHE_DBM_MODULE"]) - self.db = None + def __init__(self, settings: BaseSettings): + self.cachedir: str = data_path(settings["HTTPCACHE_DIR"], createdir=True) + self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") + self.dbmodule: ModuleType = import_module(settings["HTTPCACHE_DBM_MODULE"]) + self.db: Any = None # the real type is private - def open_spider(self, spider: Spider): + def open_spider(self, spider: Spider) -> None: dbpath = Path(self.cachedir, f"{spider.name}.db") self.db = self.dbmodule.open(str(dbpath), "c") @@ -235,24 +261,28 @@ class DbmCacheStorage: extra={"spider": spider}, ) - self._fingerprinter = spider.crawler.request_fingerprinter + assert spider.crawler.request_fingerprinter + self._fingerprinter: RequestFingerprinterProtocol = ( + spider.crawler.request_fingerprinter + ) - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> None: self.db.close() - def retrieve_response(self, spider, request): + def retrieve_response(self, spider: Spider, request: Request) -> Response | None: data = self._read_data(spider, request) if data is None: - return # not cached + return None # not cached url = data["url"] status = data["status"] headers = Headers(data["headers"]) body = data["body"] respcls = get_response_class(http_headers=headers, url=url, body=body) - response = respcls(url=url, headers=headers, status=status, body=body) - return response + return respcls(url=url, headers=headers, status=status, body=body) - def store_response(self, spider, request, response): + def store_response( + self, spider: Spider, request: Request, response: Response + ) -> None: key = self._fingerprinter.fingerprint(request).hex() data = { "status": response.status, @@ -263,28 +293,33 @@ class DbmCacheStorage: self.db[f"{key}_data"] = pickle.dumps(data, protocol=4) self.db[f"{key}_time"] = str(time()) - def _read_data(self, spider, request): + def _read_data(self, spider: Spider, request: Request) -> dict[str, Any] | None: key = self._fingerprinter.fingerprint(request).hex() db = self.db tkey = f"{key}_time" if tkey not in db: - return # not found + return None # not found ts = db[tkey] if 0 < self.expiration_secs < time() - float(ts): - return # expired + return None # expired - return pickle.loads(db[f"{key}_data"]) + return cast("dict[str, Any]", pickle.loads(db[f"{key}_data"])) # noqa: S301 class FilesystemCacheStorage: - def __init__(self, settings): - self.cachedir = data_path(settings["HTTPCACHE_DIR"]) - self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS") - self.use_gzip = settings.getbool("HTTPCACHE_GZIP") - self._open = gzip.open if self.use_gzip else open + def __init__(self, settings: BaseSettings): + self.cachedir: str = data_path(settings["HTTPCACHE_DIR"]) + self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") + self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP") + # https://github.com/python/mypy/issues/10740 + self._open: Callable[ + Concatenate[str | os.PathLike[str], str, ...], IO[bytes] + ] = ( + gzip.open if self.use_gzip else open # type: ignore[assignment] + ) - def open_spider(self, spider: Spider): + def open_spider(self, spider: Spider) -> None: logger.debug( "Using filesystem cache storage in %(cachedir)s", {"cachedir": self.cachedir}, @@ -294,27 +329,28 @@ class FilesystemCacheStorage: assert spider.crawler.request_fingerprinter self._fingerprinter = spider.crawler.request_fingerprinter - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> None: pass - def retrieve_response(self, spider: Spider, request: Request): + def retrieve_response(self, spider: Spider, request: Request) -> Response | None: """Return response if present in cache, or None otherwise.""" metadata = self._read_meta(spider, request) if metadata is None: - return # not cached + return None # not cached rpath = Path(self._get_request_path(spider, request)) with self._open(rpath / "response_body", "rb") as f: body = f.read() with self._open(rpath / "response_headers", "rb") as f: rawheaders = f.read() - url = metadata.get("response_url") + url = metadata["response_url"] status = metadata["status"] headers = Headers(headers_raw_to_dict(rawheaders)) respcls = get_response_class(http_headers=headers, url=url, body=body) - response = respcls(url=url, headers=headers, status=status, body=body) - return response + return respcls(url=url, headers=headers, status=status, body=body) - def store_response(self, spider: Spider, request: Request, response): + def store_response( + self, spider: Spider, request: Request, response: Response + ) -> None: """Store the given response in the cache.""" rpath = Path(self._get_request_path(spider, request)) if not rpath.exists(): @@ -343,19 +379,19 @@ class FilesystemCacheStorage: key = self._fingerprinter.fingerprint(request).hex() return str(Path(self.cachedir, spider.name, key[0:2], key)) - def _read_meta(self, spider: Spider, request: Request): + def _read_meta(self, spider: Spider, request: Request) -> dict[str, Any] | None: rpath = Path(self._get_request_path(spider, request)) metapath = rpath / "pickled_meta" if not metapath.exists(): - return # not found + return None # not found mtime = metapath.stat().st_mtime if 0 < self.expiration_secs < time() - mtime: - return # expired + return None # expired with self._open(metapath, "rb") as f: - return pickle.load(f) + return cast("dict[str, Any]", pickle.load(f)) # noqa: S301 -def parse_cachecontrol(header): +def parse_cachecontrol(header: bytes) -> dict[bytes, bytes | None]: """Parse Cache-Control header https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 @@ -375,9 +411,9 @@ def parse_cachecontrol(header): return directives -def rfc1123_to_epoch(date_str): +def rfc1123_to_epoch(date_str: str | bytes | None) -> int | None: try: - date_str = to_unicode(date_str, encoding="ascii") - return mktime_tz(parsedate_tz(date_str)) + date_str = to_unicode(date_str, encoding="ascii") # type: ignore[arg-type] + return mktime_tz(parsedate_tz(date_str)) # type: ignore[arg-type] except Exception: return None diff --git a/scrapy/extensions/logcount.py b/scrapy/extensions/logcount.py new file mode 100644 index 000000000..e6d51a7d8 --- /dev/null +++ b/scrapy/extensions/logcount.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from scrapy import Spider, signals +from scrapy.utils.log import LogCounterHandler + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + + +logger = logging.getLogger(__name__) + + +class LogCount: + """Install a log handler that counts log messages by level. + + The handler installed is :class:`scrapy.utils.log.LogCounterHandler`. + The counts are stored in stats as ``log_count/``. + + .. versionadded:: 2.14 + """ + + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler + self.handler: LogCounterHandler | None = None + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + o = cls(crawler) + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) + return o + + def spider_opened(self, spider: Spider) -> None: + self.handler = LogCounterHandler( + self.crawler, level=self.crawler.settings.get("LOG_LEVEL") + ) + logging.root.addHandler(self.handler) + + def spider_closed(self, spider: Spider, reason: str) -> None: + if self.handler: + logging.root.removeHandler(self.handler) + self.handler = None diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 78874a6db..3d7674905 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -1,58 +1,100 @@ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING -from twisted.internet import task - -from scrapy import signals +from scrapy import Spider, signals from scrapy.exceptions import NotConfigured +from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call + +if TYPE_CHECKING: + from twisted.internet.task import LoopingCall + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + logger = logging.getLogger(__name__) class LogStats: - """Log basic scraping stats periodically""" + """Log basic scraping stats periodically like: + * RPM - Requests per Minute + * IPM - Items per Minute + """ - def __init__(self, stats, interval=60.0): - self.stats = stats - self.interval = interval - self.multiplier = 60.0 / self.interval - self.task = None + def __init__(self, stats: StatsCollector, interval: float = 60.0): + self.stats: StatsCollector = stats + self.interval: float = interval + self.multiplier: float = 60.0 / self.interval + self.task: AsyncioLoopingCall | LoopingCall | None = None @classmethod - def from_crawler(cls, crawler): - interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") + def from_crawler(cls, crawler: Crawler) -> Self: + interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL") if not interval: raise NotConfigured + assert crawler.stats o = cls(crawler.stats, interval) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_opened(self, spider): - self.pagesprev = 0 - self.itemsprev = 0 + def spider_opened(self, spider: Spider) -> None: + self.pagesprev: int = 0 + self.itemsprev: int = 0 - self.task = task.LoopingCall(self.log, spider) + self.task = create_looping_call(self.log, spider) self.task.start(self.interval) - def log(self, spider): - items = self.stats.get_value("item_scraped_count", 0) - pages = self.stats.get_value("response_received_count", 0) - irate = (items - self.itemsprev) * self.multiplier - prate = (pages - self.pagesprev) * self.multiplier - self.pagesprev, self.itemsprev = pages, items + def log(self, spider: Spider) -> None: + self.calculate_stats() msg = ( "Crawled %(pages)d pages (at %(pagerate)d pages/min), " "scraped %(items)d items (at %(itemrate)d items/min)" ) log_args = { - "pages": pages, - "pagerate": prate, - "items": items, - "itemrate": irate, + "pages": self.pages, + "pagerate": self.prate, + "items": self.items, + "itemrate": self.irate, } logger.info(msg, log_args, extra={"spider": spider}) - def spider_closed(self, spider, reason): + def calculate_stats(self) -> None: + self.items: int = self.stats.get_value("item_scraped_count", 0) + self.pages: int = self.stats.get_value("response_received_count", 0) + self.irate: float = (self.items - self.itemsprev) * self.multiplier + self.prate: float = (self.pages - self.pagesprev) * self.multiplier + self.pagesprev, self.itemsprev = self.pages, self.items + + def spider_closed(self, spider: Spider, reason: str) -> None: if self.task and self.task.running: self.task.stop() + + rpm_final, ipm_final = self.calculate_final_stats(spider) + self.stats.set_value("responses_per_minute", rpm_final) + self.stats.set_value("items_per_minute", ipm_final) + + def calculate_final_stats( + self, spider: Spider + ) -> tuple[None, None] | tuple[float, float]: + start_time = self.stats.get_value("start_time") + finish_time = self.stats.get_value("finish_time") + + if not start_time or not finish_time: + return None, None + + mins_elapsed = (finish_time - start_time).seconds / 60 + + if mins_elapsed == 0: + return None, None + + items = self.stats.get_value("item_scraped_count", 0) + pages = self.stats.get_value("response_received_count", 0) + + return (pages / mins_elapsed), (items / mins_elapsed) diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index 03ede0681..1fde6b296 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -4,33 +4,40 @@ MemoryDebugger extension See documentation in docs/topics/extensions.rst """ -import gc +from __future__ import annotations -from scrapy import signals +import gc +from typing import TYPE_CHECKING + +from scrapy import Spider, signals from scrapy.exceptions import NotConfigured from scrapy.utils.trackref import live_refs +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + class MemoryDebugger: - def __init__(self, stats): - self.stats = stats + def __init__(self, stats: StatsCollector): + self.stats: StatsCollector = stats @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("MEMDEBUG_ENABLED"): raise NotConfigured + assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_closed(self, spider, reason): + def spider_closed(self, spider: Spider, reason: str) -> None: gc.collect() - self.stats.set_value( - "memdebug/gc_garbage_count", len(gc.garbage), spider=spider - ) + self.stats.set_value("memdebug/gc_garbage_count", len(gc.garbage)) for cls, wdict in live_refs.items(): if not wdict: continue - self.stats.set_value( - f"memdebug/live_refs/{cls.__name__}", len(wdict), spider=spider - ) + self.stats.set_value(f"memdebug/live_refs/{cls.__name__}", len(wdict)) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index ca766c938..01af02ba3 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -3,79 +3,107 @@ MemoryUsage extension See documentation in docs/topics/extensions.rst """ + +from __future__ import annotations + import logging import socket import sys +import warnings from importlib import import_module from pprint import pformat - -from twisted.internet import task +from typing import TYPE_CHECKING from scrapy import signals -from scrapy.exceptions import NotConfigured -from scrapy.mail import MailSender +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call +from scrapy.utils.defer import _schedule_coro from scrapy.utils.engine import get_engine_status +if TYPE_CHECKING: + from twisted.internet.task import LoopingCall + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) class MemoryUsage: - def __init__(self, crawler): + def __init__(self, crawler: Crawler): if not crawler.settings.getbool("MEMUSAGE_ENABLED"): raise NotConfigured try: # stdlib's resource module is only available on unix platforms. self.resource = import_module("resource") - except ImportError: - raise NotConfigured + except ImportError as exc: + raise NotConfigured from exc - self.crawler = crawler - self.warned = False - self.notify_mails = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") - self.limit = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 - self.warning = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 - self.check_interval = crawler.settings.getfloat( + self.crawler: Crawler = crawler + self.warned: bool = False + self.notify_mails: list[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") + if self.notify_mails: # pragma: no cover + from scrapy.mail import MailSender # noqa: PLC0415 + + warnings.warn( + "The 'MEMUSAGE_NOTIFY_MAIL' setting is deprecated and will be removed " + "in a future release. Please use the 'memusage_warning_reached' and 'spider_closed' " + "signals to implement custom notifications.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self.mail = MailSender.from_crawler(crawler) + + self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 + self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 + self.check_interval: float = crawler.settings.getfloat( "MEMUSAGE_CHECK_INTERVAL_SECONDS" ) - self.mail = MailSender.from_settings(crawler.settings) crawler.signals.connect(self.engine_started, signal=signals.engine_started) crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def get_virtual_size(self): - size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss + def get_virtual_size(self) -> int: + size: int = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss if sys.platform != "darwin": # on macOS ru_maxrss is in bytes, on Linux it is in KB size *= 1024 return size - def engine_started(self): + def engine_started(self) -> None: + assert self.crawler.stats self.crawler.stats.set_value("memusage/startup", self.get_virtual_size()) - self.tasks = [] - tsk = task.LoopingCall(self.update) + self.tasks: list[AsyncioLoopingCall | LoopingCall] = [] + tsk = create_looping_call(self.update) self.tasks.append(tsk) tsk.start(self.check_interval, now=True) if self.limit: - tsk = task.LoopingCall(self._check_limit) + tsk = create_looping_call(self._check_limit) self.tasks.append(tsk) tsk.start(self.check_interval, now=True) if self.warning: - tsk = task.LoopingCall(self._check_warning) + tsk = create_looping_call(self._check_warning) self.tasks.append(tsk) tsk.start(self.check_interval, now=True) - def engine_stopped(self): + def engine_stopped(self) -> None: for tsk in self.tasks: if tsk.running: tsk.stop() - def update(self): + def update(self) -> None: + assert self.crawler.stats self.crawler.stats.max_value("memusage/max", self.get_virtual_size()) - def _check_limit(self): + def _check_limit(self) -> None: + assert self.crawler.engine + assert self.crawler.stats peak_mem_usage = self.get_virtual_size() if peak_mem_usage > self.limit: self.crawler.stats.set_value("memusage/limit_reached", 1) @@ -94,22 +122,24 @@ class MemoryUsage: self.crawler.stats.set_value("memusage/limit_notified", 1) if self.crawler.engine.spider is not None: - self.crawler.engine.close_spider( - self.crawler.engine.spider, "memusage_exceeded" + _schedule_coro( + self.crawler.engine.close_spider_async(reason="memusage_exceeded") ) else: - self.crawler.stop() + _schedule_coro(self.crawler.stop_async()) else: logger.info( "Peak memory usage is %(virtualsize)dMiB", {"virtualsize": peak_mem_usage / 1024 / 1024}, ) - def _check_warning(self): + def _check_warning(self) -> None: if self.warned: # warn only once return + assert self.crawler.stats if self.get_virtual_size() > self.warning: self.crawler.stats.set_value("memusage/warning_reached", 1) + self.crawler.signals.send_catch_log(signal=signals.memusage_warning_reached) mem = self.warning / 1024 / 1024 logger.warning( "Memory usage reached %(memusage)dMiB", @@ -125,8 +155,10 @@ class MemoryUsage: self.crawler.stats.set_value("memusage/warning_notified", 1) self.warned = True - def _send_report(self, rcpts, subject): + def _send_report(self, rcpts: list[str], subject: str) -> None: """send notification mail with some additional useful info""" + assert self.crawler.engine + assert self.crawler.stats stats = self.crawler.stats s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n" s += f"Maximum memory usage : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n" diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 2d557f123..cd35c8165 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -1,12 +1,27 @@ +from __future__ import annotations + import logging from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any -from twisted.internet import task - -from scrapy import signals +from scrapy import Spider, signals from scrapy.exceptions import NotConfigured +from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call from scrapy.utils.serialize import ScrapyJSONEncoder +if TYPE_CHECKING: + from collections.abc import Sequence + from json import JSONEncoder + + from twisted.internet.task import LoopingCall + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) @@ -15,32 +30,42 @@ class PeriodicLog: def __init__( self, - stats, - interval=60.0, - ext_stats={}, - ext_delta={}, - ext_timing_enabled=False, + stats: StatsCollector, + interval: float = 60.0, + ext_stats: dict[str, Any] | None = None, + ext_delta: dict[str, Any] | None = None, + ext_timing_enabled: bool = False, ): - self.stats = stats - self.interval = interval - self.multiplier = 60.0 / self.interval - self.task = None - self.encoder = ScrapyJSONEncoder(sort_keys=True, indent=4) - self.ext_stats_enabled = bool(ext_stats) - self.ext_stats_include = ext_stats.get("include", []) - self.ext_stats_exclude = ext_stats.get("exclude", []) - self.ext_delta_enabled = bool(ext_delta) - self.ext_delta_include = ext_delta.get("include", []) - self.ext_delta_exclude = ext_delta.get("exclude", []) - self.ext_timing_enabled = ext_timing_enabled + self.stats: StatsCollector = stats + self.interval: float = interval + self.multiplier: float = 60.0 / self.interval + self.task: AsyncioLoopingCall | LoopingCall | None = None + self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) + self.ext_stats_enabled: bool = bool(ext_stats) + self.ext_stats_include: Sequence[str] = ( + ext_stats.get("include", ()) if ext_stats else () + ) + self.ext_stats_exclude: Sequence[str] = ( + ext_stats.get("exclude", ()) if ext_stats else () + ) + self.ext_delta_enabled: bool = bool(ext_delta) + self.ext_delta_include: Sequence[str] = ( + ext_delta.get("include", ()) if ext_delta else () + ) + self.ext_delta_exclude: Sequence[str] = ( + ext_delta.get("exclude", ()) if ext_delta else () + ) + self.ext_timing_enabled: bool = ext_timing_enabled @classmethod - def from_crawler(cls, crawler): - interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") + def from_crawler(cls, crawler: Crawler) -> Self: + interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL") if not interval: raise NotConfigured try: - ext_stats = crawler.settings.getdict("PERIODIC_LOG_STATS") + ext_stats: dict[str, Any] | None = crawler.settings.getdict( + "PERIODIC_LOG_STATS" + ) except (TypeError, ValueError): ext_stats = ( {"enabled": True} @@ -48,7 +73,9 @@ class PeriodicLog: else None ) try: - ext_delta = crawler.settings.getdict("PERIODIC_LOG_DELTA") + ext_delta: dict[str, Any] | None = crawler.settings.getdict( + "PERIODIC_LOG_DELTA" + ) except (TypeError, ValueError): ext_delta = ( {"enabled": True} @@ -56,11 +83,14 @@ class PeriodicLog: else None ) - ext_timing_enabled = crawler.settings.getbool( - "PERIODIC_LOG_TIMING_ENABLED", False + ext_timing_enabled: bool = crawler.settings.getbool( + "PERIODIC_LOG_TIMING_ENABLED" ) if not (ext_stats or ext_delta or ext_timing_enabled): raise NotConfigured + assert crawler.stats + assert ext_stats is not None + assert ext_delta is not None o = cls( crawler.stats, interval, @@ -72,16 +102,16 @@ class PeriodicLog: crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_opened(self, spider): - self.time_prev = datetime.now(tz=timezone.utc) - self.delta_prev = {} - self.stats_prev = {} + def spider_opened(self, spider: Spider) -> None: + self.time_prev: datetime = datetime.now(tz=timezone.utc) + self.delta_prev: dict[str, int | float] = {} + self.stats_prev: dict[str, int | float] = {} - self.task = task.LoopingCall(self.log) + self.task = create_looping_call(self.log) self.task.start(self.interval) - def log(self): - data = {} + def log(self) -> None: + data: dict[str, Any] = {} if self.ext_timing_enabled: data.update(self.log_timing()) if self.ext_delta_enabled: @@ -90,8 +120,8 @@ class PeriodicLog: data.update(self.log_crawler_stats()) logger.info(self.encoder.encode(data)) - def log_delta(self): - num_stats = { + def log_delta(self) -> dict[str, Any]: + num_stats: dict[str, int | float] = { k: v for k, v in self.stats._stats.items() if isinstance(v, (int, float)) @@ -101,7 +131,7 @@ class PeriodicLog: self.delta_prev = num_stats return {"delta": delta} - def log_timing(self): + def log_timing(self) -> dict[str, Any]: now = datetime.now(tz=timezone.utc) time = { "log_interval": self.interval, @@ -113,7 +143,7 @@ class PeriodicLog: self.time_prev = now return {"time": time} - def log_crawler_stats(self): + def log_crawler_stats(self) -> dict[str, Any]: stats = { k: v for k, v in self.stats._stats.items() @@ -121,7 +151,9 @@ class PeriodicLog: } return {"stats": stats} - def param_allowed(self, stat_name, include, exclude): + def param_allowed( + self, stat_name: str, include: Sequence[str], exclude: Sequence[str] + ) -> bool: if not include and not exclude: return True for p in exclude: @@ -129,12 +161,9 @@ class PeriodicLog: return False if exclude and not include: return True - for p in include: - if p in stat_name: - return True - return False + return any(p in stat_name for p in include) - def spider_closed(self, spider, reason): + def spider_closed(self, spider: Spider, reason: str) -> None: self.log() if self.task and self.task.running: self.task.stop() diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index 79e3b1656..5828ae52e 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -1,11 +1,12 @@ """ Extension for processing data before they are exported to feeds. """ + from bz2 import BZ2File from gzip import GzipFile from io import IOBase from lzma import LZMAFile -from typing import Any, BinaryIO, Dict, List +from typing import IO, Any, BinaryIO, cast from scrapy.utils.misc import load_object @@ -23,7 +24,7 @@ class GzipPlugin: See :py:class:`gzip.GzipFile` for more info about parameters. """ - def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + def __init__(self, file: BinaryIO, feed_options: dict[str, Any]) -> None: self.file = file self.feed_options = feed_options compress_level = self.feed_options.get("gzip_compresslevel", 9) @@ -42,7 +43,6 @@ class GzipPlugin: def close(self) -> None: self.gzipfile.close() - self.file.close() class Bz2Plugin: @@ -56,7 +56,7 @@ class Bz2Plugin: See :py:class:`bz2.BZ2File` for more info about parameters. """ - def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + def __init__(self, file: BinaryIO, feed_options: dict[str, Any]) -> None: self.file = file self.feed_options = feed_options compress_level = self.feed_options.get("bz2_compresslevel", 9) @@ -69,7 +69,6 @@ class Bz2Plugin: def close(self) -> None: self.bz2file.close() - self.file.close() class LZMAPlugin: @@ -89,18 +88,18 @@ class LZMAPlugin: See :py:class:`lzma.LZMAFile` for more info about parameters. """ - def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + def __init__(self, file: BinaryIO, feed_options: dict[str, Any]) -> None: self.file = file self.feed_options = feed_options - format = self.feed_options.get("lzma_format") + format_ = self.feed_options.get("lzma_format") check = self.feed_options.get("lzma_check", -1) preset = self.feed_options.get("lzma_preset") filters = self.feed_options.get("lzma_filters") self.lzmafile = LZMAFile( filename=self.file, mode="wb", - format=format, + format=format_, check=check, preset=preset, filters=filters, @@ -111,7 +110,6 @@ class LZMAPlugin: def close(self) -> None: self.lzmafile.close() - self.file.close() # io.IOBase is subclassed here, so that exporters can use the PostProcessingManager @@ -128,7 +126,7 @@ class PostProcessingManager(IOBase): """ def __init__( - self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any] + self, plugins: list[Any], file: IO[bytes], feed_options: dict[str, Any] ) -> None: self.plugins = self._load_plugins(plugins) self.file = file @@ -144,7 +142,7 @@ class PostProcessingManager(IOBase): :return: returns number of bytes written :rtype: int """ - return self.head_plugin.write(data) + return cast("int", self.head_plugin.write(data)) def tell(self) -> int: return self.file.tell() @@ -158,9 +156,8 @@ class PostProcessingManager(IOBase): def writable(self) -> bool: return True - def _load_plugins(self, plugins: List[Any]) -> List[Any]: - plugins = [load_object(plugin) for plugin in plugins] - return plugins + def _load_plugins(self, plugins: list[Any]) -> list[Any]: + return [load_object(plugin) for plugin in plugins] def _get_head_plugin(self) -> Any: prev = self.file diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 929a3be70..7b8756572 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -1,19 +1,28 @@ +from __future__ import annotations + import pickle from pathlib import Path +from typing import TYPE_CHECKING -from scrapy import signals +from scrapy import Spider, signals from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + class SpiderState: """Store and load spider state during a scraping job""" - def __init__(self, jobdir=None): - self.jobdir = jobdir + def __init__(self, jobdir: str | None = None): + self.jobdir: str | None = jobdir @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: jobdir = job_dir(crawler.settings) if not jobdir: raise NotConfigured @@ -23,18 +32,20 @@ class SpiderState: crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened) return obj - def spider_closed(self, spider): + def spider_closed(self, spider: Spider) -> None: if self.jobdir: with Path(self.statefn).open("wb") as f: + assert hasattr(spider, "state") # set in spider_opened pickle.dump(spider.state, f, protocol=4) - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: if self.jobdir and Path(self.statefn).exists(): with Path(self.statefn).open("rb") as f: - spider.state = pickle.load(f) + spider.state = pickle.load(f) # type: ignore[attr-defined] # noqa: S301 else: - spider.state = {} + spider.state = {} # type: ignore[attr-defined] @property def statefn(self) -> str: + assert self.jobdir return str(Path(self.jobdir, "spider.state")) diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 58610c25e..f05595806 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -4,29 +4,51 @@ StatsMailer extension sends an email when a spider finishes scraping. Use STATSMAILER_RCPTS setting to enable and give the recipient mail address """ -from scrapy import signals -from scrapy.exceptions import NotConfigured +from __future__ import annotations + +import warnings +from typing import TYPE_CHECKING + +from scrapy import Spider, signals +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.mail import MailSender +if TYPE_CHECKING: + from twisted.internet.defer import Deferred + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + +warnings.warn( + "The scrapy.extensions.statsmailer module is deprecated and will be " + "removed in a future release.", + stacklevel=2, + category=ScrapyDeprecationWarning, +) + class StatsMailer: - def __init__(self, stats, recipients, mail): - self.stats = stats - self.recipients = recipients - self.mail = mail + def __init__(self, stats: StatsCollector, recipients: list[str], mail: MailSender): + self.stats: StatsCollector = stats + self.recipients: list[str] = recipients + self.mail: MailSender = mail @classmethod - def from_crawler(cls, crawler): - recipients = crawler.settings.getlist("STATSMAILER_RCPTS") + def from_crawler(cls, crawler: Crawler) -> Self: + recipients: list[str] = crawler.settings.getlist("STATSMAILER_RCPTS") if not recipients: raise NotConfigured - mail = MailSender.from_settings(crawler.settings) + mail: MailSender = MailSender.from_crawler(crawler) + assert crawler.stats o = cls(crawler.stats, recipients, mail) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_closed(self, spider): - spider_stats = self.stats.get_stats(spider) + def spider_closed(self, spider: Spider) -> Deferred[None] | None: + spider_stats = self.stats.get_stats() body = "Global stats\n\n" body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items()) body += f"\n\n{spider.name} stats\n\n" diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index c92b7f5fe..3be24c53f 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -4,30 +4,34 @@ Scrapy Telnet Console extension See documentation in docs/topics/telnetconsole.rst """ +from __future__ import annotations + import binascii import logging import os import pprint -import traceback +from typing import TYPE_CHECKING, Any +from twisted.conch import telnet +from twisted.conch.insults import insults from twisted.internet import protocol - -try: - from twisted.conch import manhole, telnet - from twisted.conch.insults import insults - - TWISTED_CONCH_AVAILABLE = True -except (ImportError, SyntaxError): - _TWISTED_CONCH_TRACEBACK = traceback.format_exc() - TWISTED_CONCH_AVAILABLE = False +from twisted.internet.defer import fail, succeed from scrapy import signals from scrapy.exceptions import NotConfigured -from scrapy.utils.decorators import defers from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp from scrapy.utils.trackref import print_live_refs +if TYPE_CHECKING: + from twisted.internet.tcp import Port + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) # signal to update telnet variables @@ -36,22 +40,24 @@ update_telnet_vars = object() class TelnetConsole(protocol.ServerFactory): - def __init__(self, crawler): + def __init__(self, crawler: Crawler): if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"): raise NotConfigured - if not TWISTED_CONCH_AVAILABLE: + + if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): raise NotConfigured( - "TELNETCONSOLE_ENABLED setting is True but required twisted " - "modules failed to import:\n" + _TWISTED_CONCH_TRACEBACK + "The TelnetConsole extension requires a Twisted reactor." + " You can set the TELNETCONSOLE_ENABLED setting to False to remove this warning." ) - self.crawler = crawler - self.noisy = False - self.portrange = [ + + self.crawler: Crawler = crawler + self.noisy: bool = False + self.portrange: list[int] = [ int(x) for x in crawler.settings.getlist("TELNETCONSOLE_PORT") ] - self.host = crawler.settings["TELNETCONSOLE_HOST"] - self.username = crawler.settings["TELNETCONSOLE_USERNAME"] - self.password = crawler.settings["TELNETCONSOLE_PASSWORD"] + self.host: str = crawler.settings["TELNETCONSOLE_HOST"] + self.username: str = crawler.settings["TELNETCONSOLE_USERNAME"] + self.password: str = crawler.settings["TELNETCONSOLE_PASSWORD"] if not self.password: self.password = binascii.hexlify(os.urandom(8)).decode("utf8") @@ -61,11 +67,11 @@ class TelnetConsole(protocol.ServerFactory): self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def start_listening(self): - self.port = listen_tcp(self.portrange, self.host, self) + def start_listening(self) -> None: + self.port: Port = listen_tcp(self.portrange, self.host, self) h = self.port.getHost() logger.info( "Telnet console listening on %(host)s:%(port)d", @@ -73,34 +79,35 @@ class TelnetConsole(protocol.ServerFactory): extra={"crawler": self.crawler}, ) - def stop_listening(self): + def stop_listening(self) -> None: self.port.stopListening() - def protocol(self): + def protocol(self) -> telnet.TelnetTransport: class Portal: """An implementation of IPortal""" - @defers - def login(self_, credentials, mind, *interfaces): + def login(self_, credentials, mind, *interfaces): # type: ignore[no-untyped-def] # pylint: disable=no-self-argument if not ( credentials.username == self.username.encode("utf8") and credentials.checkPassword(self.password.encode("utf8")) ): - raise ValueError("Invalid credentials") + return fail(ValueError("Invalid credentials")) + + from twisted.conch import manhole protocol = telnet.TelnetBootstrapProtocol( insults.ServerProtocol, manhole.Manhole, self._get_telnet_vars() ) - return (interfaces[0], protocol, lambda: None) + return succeed((interfaces[0], protocol, lambda: None)) return telnet.TelnetTransport(telnet.AuthenticatingTelnetProtocol, Portal()) - def _get_telnet_vars(self): + def _get_telnet_vars(self) -> dict[str, Any]: # Note: if you add entries here also update topics/telnetconsole.rst - telnet_vars = { + assert self.crawler.engine + telnet_vars: dict[str, Any] = { "engine": self.crawler.engine, "spider": self.crawler.engine.spider, - "slot": self.crawler.engine.slot, "crawler": self.crawler, "extensions": self.crawler.extensions, "stats": self.crawler.stats, diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 396800775..cdb0671ae 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -1,51 +1,74 @@ -import logging +from __future__ import annotations -from scrapy import signals +import logging +from typing import TYPE_CHECKING + +from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.core.downloader import Slot + from scrapy.crawler import Crawler + from scrapy.http import Response + + logger = logging.getLogger(__name__) class AutoThrottle: - def __init__(self, crawler): - self.crawler = crawler + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler if not crawler.settings.getbool("AUTOTHROTTLE_ENABLED"): raise NotConfigured - self.debug = crawler.settings.getbool("AUTOTHROTTLE_DEBUG") - self.target_concurrency = crawler.settings.getfloat( + self.debug: bool = crawler.settings.getbool("AUTOTHROTTLE_DEBUG") + self.target_concurrency: float = crawler.settings.getfloat( "AUTOTHROTTLE_TARGET_CONCURRENCY" ) + if self.target_concurrency <= 0.0: + raise NotConfigured( + f"AUTOTHROTTLE_TARGET_CONCURRENCY " + f"({self.target_concurrency!r}) must be higher than 0." + ) crawler.signals.connect(self._spider_opened, signal=signals.spider_opened) crawler.signals.connect( self._response_downloaded, signal=signals.response_downloaded ) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def _spider_opened(self, spider): + def _spider_opened(self, spider: Spider) -> None: self.mindelay = self._min_delay(spider) self.maxdelay = self._max_delay(spider) - spider.download_delay = self._start_delay(spider) + spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined] - def _min_delay(self, spider): + def _min_delay(self, spider: Spider) -> float: s = self.crawler.settings return getattr(spider, "download_delay", s.getfloat("DOWNLOAD_DELAY")) - def _max_delay(self, spider): + def _max_delay(self, spider: Spider) -> float: return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY") - def _start_delay(self, spider): + def _start_delay(self, spider: Spider) -> float: return max( self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") ) - def _response_downloaded(self, response, request, spider): + def _response_downloaded( + self, response: Response, request: Request, spider: Spider + ) -> None: key, slot = self._get_slot(request, spider) latency = request.meta.get("download_latency") - if latency is None or slot is None: + if ( + latency is None + or slot is None + or request.meta.get("autothrottle_dont_adjust_delay", False) is True + ): return olddelay = slot.delay @@ -69,11 +92,16 @@ class AutoThrottle: extra={"spider": spider}, ) - def _get_slot(self, request, spider): - key = request.meta.get("download_slot") + def _get_slot( + self, request: Request, spider: Spider + ) -> tuple[str | None, Slot | None]: + key: str | None = request.meta.get("download_slot") + if key is None: + return None, None + assert self.crawler.engine return key, self.crawler.engine.downloader.slots.get(key) - def _adjust_delay(self, slot, latency, response): + def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None: """Define delay adjustment policy""" # If a server needs `latency` seconds to respond then diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py index d0b726bad..e20e894ae 100644 --- a/scrapy/http/__init__.py +++ b/scrapy/http/__init__.py @@ -5,9 +5,11 @@ Use this module (instead of the more specific ones) when importing Headers, Request and Response outside this module. """ +from warnings import catch_warnings, filterwarnings + +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http.headers import Headers from scrapy.http.request import Request -from scrapy.http.request.form import FormRequest from scrapy.http.request.json_request import JsonRequest from scrapy.http.request.rpc import XmlRpcRequest from scrapy.http.response import Response @@ -15,3 +17,29 @@ from scrapy.http.response.html import HtmlResponse from scrapy.http.response.json import JsonResponse from scrapy.http.response.text import TextResponse from scrapy.http.response.xml import XmlResponse +from scrapy.utils.deprecate import create_deprecated_class + +with catch_warnings(): + filterwarnings("ignore", category=ScrapyDeprecationWarning) + + from scrapy.http.request.form import FormRequest as _FormRequest + + FormRequest = create_deprecated_class( + name="FormRequest", + new_class=_FormRequest, + subclass_warn_message="{cls} inherits from deprecated class {old}, use the form2request library instead.", + instance_warn_message="{cls} is deprecated, use the form2request library instead.", + ) + +__all__ = [ + "FormRequest", + "Headers", + "HtmlResponse", + "JsonRequest", + "JsonResponse", + "Request", + "Response", + "TextResponse", + "XmlResponse", + "XmlRpcRequest", +] diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 2595f328e..599f20947 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,36 +1,49 @@ +from __future__ import annotations + import re import time -from http.cookiejar import Cookie +from http.cookiejar import Cookie, CookiePolicy, DefaultCookiePolicy from http.cookiejar import CookieJar as _CookieJar -from http.cookiejar import DefaultCookiePolicy -from typing import Sequence +from typing import TYPE_CHECKING, Any, cast -from scrapy import Request -from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + from collections.abc import Iterator, Sequence + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy import Request + from scrapy.http import Response + + # Defined in the http.cookiejar module, but undocumented: # https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527 IPV4_RE = re.compile(r"\.\d+$", re.ASCII) class CookieJar: - def __init__(self, policy=None, check_expired_frequency=10000): - self.policy = policy or DefaultCookiePolicy() - self.jar = _CookieJar(self.policy) - self.jar._cookies_lock = _DummyLock() - self.check_expired_frequency = check_expired_frequency - self.processed = 0 + def __init__( + self, + policy: CookiePolicy | None = None, + check_expired_frequency: int = 10000, + ): + self.policy: CookiePolicy = policy or DefaultCookiePolicy() + self.jar: _CookieJar = _CookieJar(self.policy) + self.jar._cookies_lock = _DummyLock() # type: ignore[attr-defined] + self.check_expired_frequency: int = check_expired_frequency + self.processed: int = 0 - def extract_cookies(self, response, request): + def extract_cookies(self, response: Response, request: Request) -> None: wreq = WrappedRequest(request) wrsp = WrappedResponse(response) - return self.jar.extract_cookies(wrsp, wreq) + self.jar.extract_cookies(wrsp, wreq) # type: ignore[arg-type] def add_cookie_header(self, request: Request) -> None: wreq = WrappedRequest(request) - self.policy._now = self.jar._now = int(time.time()) + self.policy._now = self.jar._now = int(time.time()) # type: ignore[attr-defined] # the cookiejar implementation iterates through all domains # instead we restrict to potential matches on the domain @@ -41,19 +54,18 @@ class CookieJar: if not IPV4_RE.search(req_host): hosts = potential_domain_matches(req_host) if "." not in req_host: - hosts += [req_host + ".local"] + hosts.append(req_host + ".local") else: hosts = [req_host] cookies = [] for host in hosts: - if host in self.jar._cookies: - cookies += self.jar._cookies_for_domain(host, wreq) + if host in self.jar._cookies: # type: ignore[attr-defined] + cookies.extend(self.jar._cookies_for_domain(host, wreq)) # type: ignore[attr-defined] - attrs = self.jar._cookie_attrs(cookies) - if attrs: - if not wreq.has_header("Cookie"): - wreq.add_unredirected_header("Cookie", "; ".join(attrs)) + attrs = self.jar._cookie_attrs(cookies) # type: ignore[attr-defined] + if attrs and not wreq.has_header("Cookie"): + wreq.add_unredirected_header("Cookie", "; ".join(attrs)) self.processed += 1 if self.processed % self.check_expired_frequency == 0: @@ -61,37 +73,42 @@ class CookieJar: self.jar.clear_expired_cookies() @property - def _cookies(self): - return self.jar._cookies + def _cookies(self) -> dict[str, dict[str, dict[str, Cookie]]]: + return self.jar._cookies # type: ignore[attr-defined,no-any-return] - def clear_session_cookies(self, *args, **kwargs): - return self.jar.clear_session_cookies(*args, **kwargs) + def clear_session_cookies(self) -> None: + return self.jar.clear_session_cookies() - def clear(self, domain=None, path=None, name=None): - return self.jar.clear(domain, path, name) + def clear( + self, + domain: str | None = None, + path: str | None = None, + name: str | None = None, + ) -> None: + self.jar.clear(domain, path, name) - def __iter__(self): + def __iter__(self) -> Iterator[Cookie]: return iter(self.jar) - def __len__(self): + def __len__(self) -> int: return len(self.jar) - def set_policy(self, pol): - return self.jar.set_policy(pol) + def set_policy(self, pol: CookiePolicy) -> None: + self.jar.set_policy(pol) def make_cookies(self, response: Response, request: Request) -> Sequence[Cookie]: wreq = WrappedRequest(request) wrsp = WrappedResponse(response) - return self.jar.make_cookies(wrsp, wreq) + return self.jar.make_cookies(wrsp, wreq) # type: ignore[arg-type] - def set_cookie(self, cookie): + def set_cookie(self, cookie: Cookie) -> None: self.jar.set_cookie(cookie) def set_cookie_if_ok(self, cookie: Cookie, request: Request) -> None: - self.jar.set_cookie_if_ok(cookie, WrappedRequest(request)) + self.jar.set_cookie_if_ok(cookie, WrappedRequest(request)) # type: ignore[arg-type] -def potential_domain_matches(domain): +def potential_domain_matches(domain: str) -> list[str]: """Potential domain matches for a cookie >>> potential_domain_matches('www.example.com') @@ -111,10 +128,10 @@ def potential_domain_matches(domain): class _DummyLock: - def acquire(self): + def acquire(self) -> None: pass - def release(self): + def release(self) -> None: pass @@ -124,19 +141,19 @@ class WrappedRequest: see http://docs.python.org/library/urllib2.html#urllib2.Request """ - def __init__(self, request): + def __init__(self, request: Request): self.request = request - def get_full_url(self): + def get_full_url(self) -> str: return self.request.url - def get_host(self): + def get_host(self) -> str: return urlparse_cached(self.request).netloc - def get_type(self): + def get_type(self) -> str: return urlparse_cached(self.request).scheme - def is_unverifiable(self): + def is_unverifiable(self) -> bool: """Unverifiable should indicate whether the request is unverifiable, as defined by RFC 2965. It defaults to False. An unverifiable request is one whose URL the user did not have the @@ -144,35 +161,36 @@ class WrappedRequest: HTML document, and the user had no option to approve the automatic fetching of the image, this should be true. """ - return self.request.meta.get("is_unverifiable", False) + return cast("bool", self.request.meta.get("is_unverifiable", False)) @property - def full_url(self): + def full_url(self) -> str: return self.get_full_url() @property - def host(self): + def host(self) -> str: return self.get_host() @property - def type(self): + def type(self) -> str: return self.get_type() @property - def unverifiable(self): + def unverifiable(self) -> bool: return self.is_unverifiable() @property - def origin_req_host(self): - return urlparse_cached(self.request).hostname + def origin_req_host(self) -> str: + return cast("str", urlparse_cached(self.request).hostname) - def has_header(self, name): + def has_header(self, name: str) -> bool: return name in self.request.headers - def get_header(self, name, default=None): - return to_unicode(self.request.headers.get(name, default), errors="replace") + def get_header(self, name: str, default: str | None = None) -> str | None: + value = self.request.headers.get(name, default) + return to_unicode(value, errors="replace") if value is not None else None - def header_items(self): + def header_items(self) -> list[tuple[str, list[str]]]: return [ ( to_unicode(k, errors="replace"), @@ -181,18 +199,18 @@ class WrappedRequest: for k, v in self.request.headers.items() ] - def add_unredirected_header(self, name, value): + def add_unredirected_header(self, name: str, value: str) -> None: self.request.headers.appendlist(name, value) class WrappedResponse: - def __init__(self, response): + def __init__(self, response: Response): self.response = response - def info(self): + def info(self) -> Self: return self - def get_all(self, name, default=None): + def get_all(self, name: str, default: Any = None) -> list[str]: return [ to_unicode(v, errors="replace") for v in self.response.headers.getlist(name) ] diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 21eb9fb73..34d4ec6f2 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,18 +1,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Dict, - Iterable, - List, - Optional, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, AnyStr, TypeAlias, cast from w3lib.http import headers_dict_to_raw @@ -20,11 +9,13 @@ from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self -_RawValueT = Union[bytes, str, int] +_RawValue: TypeAlias = bytes | str | int # isn't fully compatible typing-wise with either dict or CaselessDict, @@ -34,17 +25,17 @@ class Headers(CaselessDict): def __init__( self, - seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, encoding: str = "utf-8", ): self.encoding: str = encoding super().__init__(seq) def update( # type: ignore[override] - self, seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]]] + self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] ) -> None: seq = seq.items() if isinstance(seq, Mapping) else seq - iseq: Dict[bytes, List[bytes]] = {} + iseq: dict[bytes, list[bytes]] = {} for k, v in seq: iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v)) super().update(iseq) @@ -53,9 +44,9 @@ class Headers(CaselessDict): """Normalize key to bytes""" return self._tobytes(key.title()) - def normvalue(self, value: Union[_RawValueT, Iterable[_RawValueT]]) -> List[bytes]: + def normvalue(self, value: _RawValue | Iterable[_RawValue]) -> list[bytes]: """Normalize values to bytes""" - _value: Iterable[_RawValueT] + _value: Iterable[_RawValue] if value is None: _value = [] elif isinstance(value, (str, bytes)): @@ -67,7 +58,7 @@ class Headers(CaselessDict): return [self._tobytes(x) for x in _value] - def _tobytes(self, x: _RawValueT) -> bytes: + def _tobytes(self, x: _RawValue) -> bytes: if isinstance(x, bytes): return x if isinstance(x, str): @@ -76,48 +67,50 @@ class Headers(CaselessDict): return str(x).encode(self.encoding) raise TypeError(f"Unsupported value type: {type(x)}") - def __getitem__(self, key: AnyStr) -> Optional[bytes]: + def __getitem__(self, key: AnyStr) -> bytes | None: try: - return cast(List[bytes], super().__getitem__(key))[-1] + return cast("list[bytes]", super().__getitem__(key))[-1] except IndexError: return None - def get(self, key: AnyStr, def_val: Any = None) -> Optional[bytes]: + def get(self, key: AnyStr, def_val: Any = None) -> bytes | None: try: - return cast(List[bytes], super().get(key, def_val))[-1] + return cast("list[bytes]", super().get(key, def_val))[-1] except IndexError: return None - def getlist(self, key: AnyStr, def_val: Any = None) -> List[bytes]: + def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]: try: - return cast(List[bytes], super().__getitem__(key)) + return cast("list[bytes]", super().__getitem__(key)) except KeyError: if def_val is not None: return self.normvalue(def_val) return [] - def setlist(self, key: AnyStr, list_: Iterable[_RawValueT]) -> None: + def setlist(self, key: AnyStr, list_: Iterable[_RawValue]) -> None: self[key] = list_ def setlistdefault( - self, key: AnyStr, default_list: Iterable[_RawValueT] = () + self, key: AnyStr, default_list: Iterable[_RawValue] = () ) -> Any: return self.setdefault(key, default_list) - def appendlist(self, key: AnyStr, value: Iterable[_RawValueT]) -> None: + def appendlist(self, key: AnyStr, value: Iterable[_RawValue]) -> None: lst = self.getlist(key) lst.extend(self.normvalue(value)) self[key] = lst - def items(self) -> Iterable[Tuple[bytes, List[bytes]]]: # type: ignore[override] + def items(self) -> Iterable[tuple[bytes, list[bytes]]]: # type: ignore[override] return ((k, self.getlist(k)) for k in self.keys()) - def values(self) -> List[Optional[bytes]]: # type: ignore[override] - return [self[k] for k in self.keys()] + def values(self) -> list[bytes | None]: # type: ignore[override] + return [ + self[k] + for k in self.keys() # pylint: disable=consider-using-dict-items + ] def to_string(self) -> bytes: - # cast() can be removed if the headers_dict_to_raw() hint is improved - return cast(bytes, headers_dict_to_raw(self)) + return headers_dict_to_raw(self) def to_unicode_dict(self) -> CaseInsensitiveDict: """Return headers as a CaseInsensitiveDict with str keys @@ -131,6 +124,17 @@ class Headers(CaselessDict): for key, value in self.items() ) + def to_tuple_list(self) -> list[tuple[str, str]]: + """Return headers as a list of ``(key, value)`` tuples. + + Multiple values are represented as multiple tuples with the same key. + """ + return [ + (key.decode(self.encoding), value.decode(self.encoding)) + for key, values in self.items() + for value in values + ] + def __copy__(self) -> Self: return self.__class__(self) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a1c5a5e51..7db648a45 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,39 +4,62 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ + +from __future__ import annotations + import inspect from typing import ( + TYPE_CHECKING, Any, AnyStr, - Callable, - Dict, - Iterable, - List, - Mapping, + Concatenate, NoReturn, - Optional, - Tuple, - Type, + TypeAlias, + TypedDict, TypeVar, - Union, - cast, + overload, ) from w3lib.url import safe_url_string -import scrapy +# a workaround for the docs "more than one target found" problem +import scrapy # noqa: TC001 from scrapy.http.headers import Headers from scrapy.utils.curl import curl_to_request_kwargs from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref -from scrapy.utils.url import escape_ajax + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Mapping + + from twisted.python.failure import Failure + + # typing.NotRequired and typing.Self require Python 3.11 + from typing_extensions import NotRequired, Self + + # circular import + from scrapy.http import Response + + CallbackT: TypeAlias = Callable[Concatenate[Response, ...], Any] + + +class VerboseCookie(TypedDict): + name: str | bytes + value: str | bytes | bool | float | int + domain: NotRequired[str | bytes] + path: NotRequired[str | bytes] + secure: NotRequired[bool] + + +CookiesT: TypeAlias = dict[str, str] | list[VerboseCookie] + RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn: """When assigned to the ``callback`` parameter of - :class:`~scrapy.http.Request`, it indicates that the request is not meant + :class:`~scrapy.Request`, it indicates that the request is not meant to have a spider callback at all. For example: @@ -60,54 +83,76 @@ def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn: class Request(object_ref): """Represents an HTTP request, which is usually generated in a Spider and - executed by the Downloader, thus generating a :class:`Response`. + executed by the Downloader, thus generating a :class:`~scrapy.http.Response`. """ - attributes: Tuple[str, ...] = ( + __attrs_and_slots = ("callback", "dont_filter", "errback", "method", "priority") + attributes: tuple[str, ...] = ( "url", - "callback", - "method", "headers", "body", "cookies", "meta", "encoding", - "priority", - "dont_filter", - "errback", "flags", "cb_kwargs", + *__attrs_and_slots, ) """A tuple of :class:`str` objects containing the name of all public attributes of the class that are also keyword parameters of the - ``__init__`` method. + ``__init__()`` method. - Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and + Currently used by :meth:`.Request.replace`, :meth:`.Request.to_dict` and :func:`~scrapy.utils.request.request_from_dict`. """ + __slots__ = ( + "__weakref__", + "_body", + "_cb_kwargs", + "_cookies", + "_encoding", + "_flags", + "_headers", + "_meta", + "_url", + *__attrs_and_slots, + ) + del __attrs_and_slots + def __init__( self, url: str, - callback: Optional[Callable] = None, + callback: CallbackT | None = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, - body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, - meta: Optional[Dict[str, Any]] = None, + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + body: bytes | str | None = None, + cookies: CookiesT | None = None, + meta: dict[str, Any] | None = None, encoding: str = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, - flags: Optional[List[str]] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, + errback: Callable[[Failure], Any] | None = None, + flags: list[str] | None = None, + cb_kwargs: dict[str, Any] | None = None, ) -> None: self._encoding: str = encoding # this one has to be set first self.method: str = str(method).upper() + self._meta: dict[str, Any] | None = dict(meta) if meta else None self._set_url(url) self._set_body(body) if not isinstance(priority, int): raise TypeError(f"Request priority not an integer: {priority!r}") + + #: Default: ``0`` + #: + #: Value that the :ref:`scheduler ` may use for + #: request prioritization. + #: + #: Built-in schedulers prioritize requests with a higher priority + #: value. + #: + #: Negative values are allowed. self.priority: int = priority if not (callable(callback) or callback is None): @@ -116,27 +161,89 @@ class Request(object_ref): ) if not (callable(errback) or errback is None): raise TypeError(f"errback must be a callable, got {type(errback).__name__}") - self.callback: Optional[Callable] = callback - self.errback: Optional[Callable] = errback - self.cookies: Union[dict, List[dict]] = cookies or {} - self.headers: Headers = Headers(headers or {}, encoding=encoding) + #: :class:`~collections.abc.Callable` to parse the + #: :class:`~scrapy.http.Response` to this request once received. + #: + #: The callable must expect the response as its first parameter, and + #: support any additional keyword arguments set through + #: :attr:`cb_kwargs`. + #: + #: In addition to an arbitrary callable, the following values are also + #: supported: + #: + #: - ``None`` (default), which indicates that the + #: :meth:`~scrapy.Spider.parse` method of the spider must be used. + #: + #: - :func:`~scrapy.http.request.NO_CALLBACK`. + #: + #: If an unhandled exception is raised during request or response + #: processing, i.e. by a :ref:`spider middleware + #: `, :ref:`downloader middleware + #: ` or download handler + #: (:setting:`DOWNLOAD_HANDLERS`), :attr:`errback` is called instead. + #: + #: .. tip:: + #: :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware` + #: raises exceptions for non-2xx responses by default, sending them + #: to the :attr:`errback` instead. + #: + #: .. seealso:: + #: :ref:`topics-request-response-ref-request-callback-arguments` + self.callback: CallbackT | None = callback + + #: :class:`~collections.abc.Callable` to handle exceptions raised + #: during request or response processing. + #: + #: The callable must expect a :exc:`~twisted.python.failure.Failure` as + #: its first parameter. + #: + #: .. seealso:: :ref:`topics-request-response-ref-errbacks` + self.errback: Callable[[Failure], Any] | None = errback + + self._cookies: CookiesT | None = cookies or None + self._headers: Headers | None = ( + Headers(headers, encoding=encoding) if headers else None + ) + + #: Whether this request may be filtered out by :ref:`components + #: ` that support filtering out requests (``False``, + #: default), or those components should not filter out this request + #: (``True``). + #: + #: The following built-in components check this attribute: + #: + #: - The :ref:`scheduler ` uses it to skip + #: duplicate request filtering (see + #: :setting:`DUPEFILTER_CLASS`). When set to ``True``, the + #: request is not checked against the duplicate filter, + #: allowing requests that would otherwise be considered duplicates + #: to be scheduled multiple times. + #: - :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` + #: uses it to allow requests to domains not in + #: :attr:`~scrapy.Spider.allowed_domains`. To skip only the offsite + #: filter without affecting other components, consider using the + #: :reqmeta:`allow_offsite` request meta key instead. + #: + #: Third-party components may also use this attribute to decide whether + #: to filter out a request. + #: + #: When defining the start URLs of a spider through + #: :attr:`~scrapy.Spider.start_urls`, this attribute is enabled by + #: default. See :meth:`~scrapy.Spider.start`. self.dont_filter: bool = dont_filter - self._meta: Optional[Dict[str, Any]] = dict(meta) if meta else None - self._cb_kwargs: Optional[Dict[str, Any]] = ( - dict(cb_kwargs) if cb_kwargs else None - ) - self.flags: List[str] = [] if flags is None else list(flags) + self._cb_kwargs: dict[str, Any] | None = dict(cb_kwargs) if cb_kwargs else None + self._flags: list[str] | None = list(flags) if flags else None @property - def cb_kwargs(self) -> Dict[str, Any]: + def cb_kwargs(self) -> dict[str, Any]: if self._cb_kwargs is None: self._cb_kwargs = {} return self._cb_kwargs @property - def meta(self) -> Dict[str, Any]: + def meta(self) -> dict[str, Any]: if self._meta is None: self._meta = {} return self._meta @@ -145,12 +252,17 @@ class Request(object_ref): def url(self) -> str: return self._url + def _url_is_verbatim(self) -> bool: + return bool(self._meta and self._meta.get("verbatim_url")) + def _set_url(self, url: str) -> None: if not isinstance(url, str): raise TypeError(f"Request url must be str, got {type(url).__name__}") - s = safe_url_string(url, self.encoding) - self._url = escape_ajax(s) + if self._url_is_verbatim(): + self._url = url + else: + self._url = safe_url_string(url, self.encoding) if ( "://" not in self._url @@ -163,35 +275,83 @@ class Request(object_ref): def body(self) -> bytes: return self._body - def _set_body(self, body: Optional[Union[str, bytes]]) -> None: - self._body = b"" if body is None else to_bytes(body, self.encoding) + def _set_body(self, body: str | bytes | None) -> None: + self._body = b"" if not body else to_bytes(body, self.encoding) @property def encoding(self) -> str: return self._encoding + @property + def flags(self) -> list[str]: + if self._flags is None: + self._flags = [] + return self._flags + + @flags.setter + def flags(self, value: list[str] | None) -> None: + self._flags = value + + @property + def cookies(self) -> CookiesT: + if self._cookies is None: + self._cookies = {} + return self._cookies + + @cookies.setter + def cookies(self, value: CookiesT | None) -> None: + self._cookies = value + + @property + def headers(self) -> Headers: + if self._headers is None: + self._headers = Headers(encoding=self.encoding) + return self._headers + + @headers.setter + def headers( + self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None + ) -> None: + if isinstance(value, Headers): + self._headers = value + else: + self._headers = ( + Headers(value, encoding=self.encoding) if value is not None else None + ) + def __repr__(self) -> str: return f"<{self.method} {self.url}>" - def copy(self) -> "Request": + def copy(self) -> Self: return self.replace() - def replace(self, *args: Any, **kwargs: Any) -> "Request": + @overload + def replace( + self, *args: Any, cls: type[RequestTypeVar], **kwargs: Any + ) -> RequestTypeVar: ... + + @overload + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... + + def replace( + self, *args: Any, cls: type[Request] | None = None, **kwargs: Any + ) -> Request: """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) - cls = kwargs.pop("cls", self.__class__) - return cast(Request, cls(*args, **kwargs)) + if cls is None: + cls = self.__class__ + return cls(*args, **kwargs) @classmethod def from_curl( - cls: Type[RequestTypeVar], + cls, curl_command: str, ignore_unknown_options: bool = True, **kwargs: Any, - ) -> RequestTypeVar: + ) -> Self: """Create a Request object from a string containing a `cURL - `_ command. It populates the HTTP method, the + `_ command. It populates the HTTP method, the URL, the headers, the cookies and the body. It accepts the same arguments as the :class:`Request` class, taking preference and overriding the values of the same arguments contained in the cURL @@ -201,7 +361,7 @@ class Request(object_ref): finding unknown options call this method by passing ``ignore_unknown_options=False``. - .. caution:: Using :meth:`from_curl` from :class:`~scrapy.http.Request` + .. caution:: Using :meth:`from_curl` from :class:`~scrapy.Request` subclasses, such as :class:`~scrapy.http.JsonRequest`, or :class:`~scrapy.http.XmlRpcRequest`, as well as having :ref:`downloader middlewares ` @@ -212,7 +372,7 @@ class Request(object_ref): :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`, or :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`, - may modify the :class:`~scrapy.http.Request` object. + may modify the :class:`~scrapy.Request` object. To translate a cURL command into a Scrapy request, you may use `curl2scrapy `_. @@ -221,7 +381,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]: + def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. @@ -231,12 +391,16 @@ class Request(object_ref): """ d = { "url": self.url, # urls are safe (safe_string_url) - "callback": _find_method(spider, self.callback) - if callable(self.callback) - else self.callback, - "errback": _find_method(spider, self.errback) - if callable(self.errback) - else self.errback, + "callback": ( + _find_method(spider, self.callback) + if callable(self.callback) + else self.callback + ), + "errback": ( + _find_method(spider, self.errback) + if callable(self.errback) + else self.errback + ), "headers": dict(self.headers), } for attr in self.attributes: @@ -246,7 +410,7 @@ class Request(object_ref): return d -def _find_method(obj: Any, func: Callable) -> str: +def _find_method(obj: Any, func: Callable[..., Any]) -> str: """Helper function for Request.to_dict""" # Only instance methods contain ``__func__`` if obj and hasattr(func, "__func__"): diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 1bba4f2a4..4da595b22 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -7,35 +7,47 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit +from warnings import warn -from lxml.html import ( - FormElement, - HTMLParser, - InputElement, - MultipleSelectOptions, - SelectElement, - TextareaElement, -) -from parsel.selector import create_root_node +from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http.request import Request -from scrapy.http.response.text import TextResponse from scrapy.utils.python import is_listlike, to_bytes if TYPE_CHECKING: # typing.Self requires Python 3.11 + from lxml.html import ( + FormElement, + InputElement, + MultipleSelectOptions, + SelectElement, + TextareaElement, + ) from typing_extensions import Self + from scrapy.http.response.text import TextResponse -FormdataKVType = Tuple[str, Union[str, Iterable[str]]] -FormdataType = Optional[Union[dict, List[FormdataKVType]]] +warn( + "The entire scrapy.http.request.form module is deprecated. Use the " + "form2request library instead.", + ScrapyDeprecationWarning, + stacklevel=2, +) + +FormdataVType: TypeAlias = str | Iterable[str] +FormdataKVType: TypeAlias = tuple[str, FormdataVType] +FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None class FormRequest(Request): - valid_form_methods = ["GET", "POST"] + __slots__ = () + + valid_form_methods: ClassVar[list[str]] = ["GET", "POST"] def __init__( self, *args: Any, formdata: FormdataType = None, **kwargs: Any @@ -62,21 +74,19 @@ class FormRequest(Request): def from_response( cls, response: TextResponse, - formname: Optional[str] = None, - formid: Optional[str] = None, + formname: str | None = None, + formid: str | None = None, formnumber: int = 0, formdata: FormdataType = None, - clickdata: Optional[dict] = None, + clickdata: dict[str, str | int] | None = None, dont_click: bool = False, - formxpath: Optional[str] = None, - formcss: Optional[str] = None, + formxpath: str | None = None, + formcss: str | None = None, **kwargs: Any, ) -> Self: kwargs.setdefault("encoding", response.encoding) if formcss is not None: - from parsel.csstranslator import HTMLTranslator - formxpath = HTMLTranslator().css_to_xpath(formcss) form = _get_form(response, formname, formid, formnumber, formxpath) @@ -92,7 +102,7 @@ class FormRequest(Request): return cls(url=url, method=method, formdata=formdata, **kwargs) -def _get_form_url(form: FormElement, url: Optional[str]) -> str: +def _get_form_url(form: FormElement, url: str | None) -> str: assert form.base_url is not None # typing if url is None: action = form.get("action") @@ -106,20 +116,20 @@ def _urlencode(seq: Iterable[FormdataKVType], enc: str) -> str: values = [ (to_bytes(k, enc), to_bytes(v, enc)) for k, vs in seq - for v in (cast(Iterable[str], vs) if is_listlike(vs) else [cast(str, vs)]) + for v in (vs if is_listlike(vs) else [cast("str", vs)]) ] return urlencode(values, doseq=True) def _get_form( response: TextResponse, - formname: Optional[str], - formid: Optional[str], + formname: str | None, + formid: str | None, formnumber: int, - formxpath: Optional[str], + formxpath: str | None, ) -> FormElement: """Find the wanted form element within the given response.""" - root = create_root_node(response.text, HTMLParser, base_url=response.base_url) + root = response.selector.root forms = root.xpath("//form") if not forms: raise ValueError(f"No element found in {response}") @@ -127,12 +137,12 @@ def _get_form( if formname is not None: f = root.xpath(f'//form[@name="{formname}"]') if f: - return cast(FormElement, f[0]) + return cast("FormElement", f[0]) if formid is not None: f = root.xpath(f'//form[@id="{formid}"]') if f: - return cast(FormElement, f[0]) + return cast("FormElement", f[0]) # Get form element from xpath, if not found, go up if formxpath is not None: @@ -141,7 +151,7 @@ def _get_form( el = nodes[0] while True: if el.tag == "form": - return cast(FormElement, el) + return cast("FormElement", el) el = el.getparent() if el is None: break @@ -151,22 +161,21 @@ def _get_form( try: form = forms[formnumber] except IndexError: - raise IndexError(f"Form number {formnumber} not found in {response}") - else: - return cast(FormElement, form) + raise IndexError(f"Form number {formnumber} not found in {response}") from None + return cast("FormElement", form) def _get_inputs( form: FormElement, formdata: FormdataType, dont_click: bool, - clickdata: Optional[dict], -) -> List[FormdataKVType]: + clickdata: dict[str, str | int] | None, +) -> list[FormdataKVType]: """Return a list of key-value pairs for the inputs found in the given form.""" try: formdata_keys = dict(formdata or ()).keys() except (ValueError, TypeError): - raise ValueError("formdata should be a dict or iterable of tuples") + raise ValueError("formdata should be a dict or iterable of tuples") from None if not formdata: formdata = [] @@ -179,7 +188,7 @@ def _get_inputs( ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', namespaces={"re": "http://exslt.org/regular-expressions"}, ) - values: List[FormdataKVType] = [ + values: list[FormdataKVType] = [ (k, "" if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata_keys @@ -187,29 +196,27 @@ def _get_inputs( if not dont_click: clickable = _get_clickable(clickdata, form) - if clickable and clickable[0] not in formdata and not clickable[0] is None: + if clickable and clickable[0] not in formdata and clickable[0] is not None: values.append(clickable) - if isinstance(formdata, dict): - formdata = formdata.items() # type: ignore[assignment] - - values.extend((k, v) for k, v in formdata if v is not None) + formdata_items = formdata.items() if isinstance(formdata, dict) else formdata + values.extend((k, v) for k, v in formdata_items if v is not None) return values def _value( - ele: Union[InputElement, SelectElement, TextareaElement] -) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: + ele: InputElement | SelectElement | TextareaElement, +) -> tuple[str | None, str | MultipleSelectOptions | None]: n = ele.name v = ele.value if ele.tag == "select": - return _select_value(cast(SelectElement, ele), n, v) + return _select_value(cast("SelectElement", ele), n, v) return n, v def _select_value( - ele: SelectElement, n: Optional[str], v: Union[None, str, MultipleSelectOptions] -) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: + ele: SelectElement, n: str | None, v: str | MultipleSelectOptions | None +) -> tuple[str | None, str | MultipleSelectOptions | None]: multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected @@ -220,8 +227,8 @@ def _select_value( def _get_clickable( - clickdata: Optional[dict], form: FormElement -) -> Optional[Tuple[str, str]]: + clickdata: dict[str, str | int] | None, form: FormElement +) -> tuple[str, str] | None: """ Returns the clickable element specified in clickdata, if the latter is given. If not, it returns the first @@ -247,12 +254,13 @@ def _get_clickable( # because that uniquely identifies the element nr = clickdata.get("nr", None) if nr is not None: + assert isinstance(nr, int) try: el = list(form.inputs)[nr] except IndexError: pass else: - return (el.get("name"), el.get("value") or "") + return (cast("str", el.get("name")), el.get("value") or "") # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such @@ -265,5 +273,4 @@ def _get_clickable( f"Multiple elements found ({el!r}) matching the " f"criteria in clickdata: {clickdata!r}" ) - else: - raise ValueError(f"No clickable element matching clickdata: {clickdata!r}") + raise ValueError(f"No clickable element matching clickdata: {clickdata!r}") diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 1dd9e6c87..13fee06d8 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -5,30 +5,40 @@ This module implements the JsonRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ +from __future__ import annotations + import copy import json import warnings -from typing import Any, Optional, Tuple +from typing import TYPE_CHECKING, Any, overload -from scrapy.http.request import Request +from scrapy.http.request import Request, RequestTypeVar + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class JsonRequest(Request): - attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) + __slots__ = ("_dumps_kwargs",) + + attributes: tuple[str, ...] = (*Request.attributes, "dumps_kwargs") def __init__( - self, *args: Any, dumps_kwargs: Optional[dict] = None, **kwargs: Any + self, *args: Any, dumps_kwargs: dict[str, Any] | None = None, **kwargs: Any ) -> None: dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} dumps_kwargs.setdefault("sort_keys", True) - self._dumps_kwargs = dumps_kwargs + self._dumps_kwargs: dict[str, Any] = dumps_kwargs - body_passed = kwargs.get("body", None) is not None - data = kwargs.pop("data", None) - data_passed = data is not None + body_passed = kwargs.get("body") is not None + data: Any = kwargs.pop("data", None) + data_passed: bool = data is not None if body_passed and data_passed: - warnings.warn("Both body and data passed. data will be ignored") + warnings.warn( + "Both body and data passed. data will be ignored", stacklevel=2 + ) elif not body_passed and data_passed: kwargs["body"] = self._dumps(data) if "method" not in kwargs: @@ -41,21 +51,33 @@ class JsonRequest(Request): ) @property - def dumps_kwargs(self) -> dict: + def dumps_kwargs(self) -> dict[str, Any]: return self._dumps_kwargs - def replace(self, *args: Any, **kwargs: Any) -> Request: - body_passed = kwargs.get("body", None) is not None - data = kwargs.pop("data", None) - data_passed = data is not None + @overload + def replace( + self, *args: Any, cls: type[RequestTypeVar], **kwargs: Any + ) -> RequestTypeVar: ... + + @overload + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... + + def replace( + self, *args: Any, cls: type[Request] | None = None, **kwargs: Any + ) -> Request: + body_passed = kwargs.get("body") is not None + data: Any = kwargs.pop("data", None) + data_passed: bool = data is not None if body_passed and data_passed: - warnings.warn("Both body and data passed. data will be ignored") + warnings.warn( + "Both body and data passed. data will be ignored", stacklevel=2 + ) elif not body_passed and data_passed: kwargs["body"] = self._dumps(data) - return super().replace(*args, **kwargs) + return super().replace(*args, cls=cls, **kwargs) - def _dumps(self, data: dict) -> str: + def _dumps(self, data: Any) -> str: """Convert to JSON""" return json.dumps(data, **self._dumps_kwargs) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index bde860a66..c68c8aabd 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -4,19 +4,28 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ + +from __future__ import annotations + import xmlrpc.client as xmlrpclib -from typing import Any, Optional +from typing import Any + +import defusedxml.xmlrpc from scrapy.http.request import Request from scrapy.utils.python import get_func_args +defusedxml.xmlrpc.monkey_patch() + DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): - def __init__(self, *args: Any, encoding: Optional[str] = None, **kwargs: Any): + __slots__ = () + + def __init__(self, *args: Any, encoding: str | None = None, **kwargs: Any): if "body" not in kwargs and "params" in kwargs: - kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs) + kw = {k: kwargs.pop(k) for k in DUMPS_ARGS if k in kwargs} kwargs["body"] = xmlrpclib.dumps(**kw) # spec defines that requests must use POST method diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 6eae3e8b3..09b1c8b32 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,28 +4,12 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ + from __future__ import annotations -from ipaddress import IPv4Address, IPv6Address -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Callable, - Dict, - Generator, - Iterable, - List, - Mapping, - Optional, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, overload from urllib.parse import urljoin -from twisted.internet.ssl import Certificate - from scrapy.exceptions import NotSupported from scrapy.http.headers import Headers from scrapy.http.request import Request @@ -33,73 +17,97 @@ from scrapy.link import Link from scrapy.utils.trackref import object_ref if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Mapping + from ipaddress import IPv4Address, IPv6Address + + from twisted.python.failure import Failure + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.http.request import CallbackT, CookiesT from scrapy.selector import SelectorList +ResponseTypeVar = TypeVar("ResponseTypeVar", bound="Response") + + class Response(object_ref): """An object that represents an HTTP response, which is usually downloaded (by the Downloader) and fed to the Spiders for processing. """ - attributes: Tuple[str, ...] = ( - "url", + __attrs_and_slots = ( "status", - "headers", - "body", - "flags", "request", "certificate", "ip_address", "protocol", ) + attributes: tuple[str, ...] = ( + "url", + "headers", + "body", + "flags", + *__attrs_and_slots, + ) """A tuple of :class:`str` objects containing the name of all public attributes of the class that are also keyword parameters of the - ``__init__`` method. + ``__init__()`` method. Currently used by :meth:`Response.replace`. """ + __slots__ = ( + "__weakref__", + "_url", + "_body", + "_headers", + "_flags", + *__attrs_and_slots, + ) + del __attrs_and_slots + def __init__( self, url: str, status: int = 200, - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, body: bytes = b"", - flags: Optional[List[str]] = None, - request: Optional[Request] = None, - certificate: Optional[Certificate] = None, - ip_address: Union[IPv4Address, IPv6Address, None] = None, - protocol: Optional[str] = None, + flags: list[str] | None = None, + request: Request | None = None, + certificate: Any = None, + ip_address: IPv4Address | IPv6Address | None = None, + protocol: str | None = None, ): - self.headers: Headers = Headers(headers or {}) + self._headers: Headers | None = Headers(headers) if headers else None self.status: int = int(status) self._set_body(body) self._set_url(url) - self.request: Optional[Request] = request - self.flags: List[str] = [] if flags is None else list(flags) - self.certificate: Optional[Certificate] = certificate - self.ip_address: Union[IPv4Address, IPv6Address, None] = ip_address - self.protocol: Optional[str] = protocol + self.request: Request | None = request + self._flags: list[str] | None = list(flags) if flags else None + self.certificate: Any = certificate + self.ip_address: IPv4Address | IPv6Address | None = ip_address + self.protocol: str | None = protocol @property - def cb_kwargs(self) -> Dict[str, Any]: + def cb_kwargs(self) -> dict[str, Any]: try: return self.request.cb_kwargs # type: ignore[union-attr] except AttributeError: raise AttributeError( "Response.cb_kwargs not available, this response " "is not tied to any request" - ) + ) from None @property - def meta(self) -> Dict[str, Any]: + def meta(self) -> dict[str, Any]: try: return self.request.meta # type: ignore[union-attr] except AttributeError: raise AttributeError( - "Response.meta not available, this response " - "is not tied to any request" - ) + "Response.meta not available, this response is not tied to any request" + ) from None @property def url(self) -> str: @@ -110,14 +118,14 @@ class Response(object_ref): self._url: str = url else: raise TypeError( - f"{type(self).__name__} url must be str, " f"got {type(url).__name__}" + f"{type(self).__name__} url must be str, got {type(url).__name__}" ) @property def body(self) -> bytes: return self._body - def _set_body(self, body: Optional[bytes]) -> None: + def _set_body(self, body: bytes | None) -> None: if body is None: self._body = b"" elif not isinstance(body, bytes): @@ -129,19 +137,55 @@ class Response(object_ref): else: self._body = body + @property + def headers(self) -> Headers: + if self._headers is None: + self._headers = Headers() + return self._headers + + @headers.setter + def headers( + self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None + ) -> None: + if isinstance(value, Headers): + self._headers = value + else: + self._headers = Headers(value) if value is not None else None + + @property + def flags(self) -> list[str]: + if self._flags is None: + self._flags = [] + return self._flags + + @flags.setter + def flags(self, value: list[str] | None) -> None: + self._flags = value + def __repr__(self) -> str: return f"<{self.status} {self.url}>" - def copy(self) -> Response: + def copy(self) -> Self: """Return a copy of this Response""" return self.replace() - def replace(self, *args: Any, **kwargs: Any) -> Response: + @overload + def replace( + self, *args: Any, cls: type[ResponseTypeVar], **kwargs: Any + ) -> ResponseTypeVar: ... + + @overload + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... + + def replace( + self, *args: Any, cls: type[Response] | None = None, **kwargs: Any + ) -> Response: """Create a new Response with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) - cls = kwargs.pop("cls", self.__class__) - return cast(Response, cls(*args, **kwargs)) + if cls is None: + cls = self.__class__ + return cls(*args, **kwargs) def urljoin(self, url: str) -> str: """Join this Response's url with a possible relative url to form an @@ -175,32 +219,29 @@ class Response(object_ref): def follow( self, - url: Union[str, Link], - callback: Optional[Callable] = None, + url: str | Link, + callback: CallbackT | None = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, - body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, - meta: Optional[Dict[str, Any]] = None, - encoding: Optional[str] = "utf-8", + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + body: bytes | str | None = None, + cookies: CookiesT | None = None, + meta: dict[str, Any] | None = None, + encoding: str | None = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, - flags: Optional[List[str]] = None, + errback: Callable[[Failure], Any] | None = None, + cb_kwargs: dict[str, Any] | None = None, + flags: list[str] | None = None, ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. - It accepts the same arguments as ``Request.__init__`` method, - but ``url`` can be a relative URL or a ``scrapy.link.Link`` object, + It accepts the same arguments as ``Request.__init__()`` method, + but ``url`` can be a relative URL or a :class:`~scrapy.link.Link` object, not only an absolute URL. :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow` method which supports selectors in addition to absolute/relative URLs and Link objects. - - .. versionadded:: 2.0 - The *flags* parameter. """ if encoding is None: raise ValueError("encoding can't be None") @@ -228,25 +269,23 @@ class Response(object_ref): def follow_all( self, - urls: Iterable[Union[str, Link]], - callback: Optional[Callable] = None, + urls: Iterable[str | Link], + callback: CallbackT | None = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, - body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, - meta: Optional[Dict[str, Any]] = None, - encoding: Optional[str] = "utf-8", + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + body: bytes | str | None = None, + cookies: CookiesT | None = None, + meta: dict[str, Any] | None = None, + encoding: str | None = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, - flags: Optional[List[str]] = None, - ) -> Generator[Request, None, None]: + errback: Callable[[Failure], Any] | None = None, + cb_kwargs: dict[str, Any] | None = None, + flags: list[str] | None = None, + ) -> Iterable[Request]: """ - .. versionadded:: 2.0 - Return an iterable of :class:`~.Request` instances to follow all links - in ``urls``. It accepts the same arguments as ``Request.__init__`` method, + in ``urls``. It accepts the same arguments as ``Request.__init__()`` method, but elements of ``urls`` can be relative URLs or :class:`~scrapy.link.Link` objects, not only absolute URLs. diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index 7eed052c2..70c08c11d 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -9,4 +9,4 @@ from scrapy.http.response.text import TextResponse class HtmlResponse(TextResponse): - pass + __slots__ = () diff --git a/scrapy/http/response/json.py b/scrapy/http/response/json.py index 219691094..0428dde6e 100644 --- a/scrapy/http/response/json.py +++ b/scrapy/http/response/json.py @@ -9,4 +9,4 @@ from scrapy.http.response.text import TextResponse class JsonResponse(TextResponse): - pass + __slots__ = () diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 937a4d745..695de5482 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -4,25 +4,12 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ + from __future__ import annotations import json from contextlib import suppress -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Callable, - Dict, - Generator, - Iterable, - List, - Mapping, - Optional, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, AnyStr, cast from urllib.parse import urljoin import parsel @@ -35,32 +22,44 @@ from w3lib.encoding import ( ) from w3lib.html import get_base_url, strip_html5_whitespace -from scrapy.http import Request from scrapy.http.response import Response -from scrapy.link import Link from scrapy.utils.python import memoizemethod_noargs, to_unicode if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Mapping + + from twisted.python.failure import Failure + + from scrapy.http.request import CallbackT, CookiesT, Request + from scrapy.link import Link from scrapy.selector import Selector, SelectorList + _NONE = object() class TextResponse(Response): _DEFAULT_ENCODING = "ascii" - _cached_decoded_json = _NONE - attributes: Tuple[str, ...] = Response.attributes + ("encoding",) + attributes: tuple[str, ...] = (*Response.attributes, "encoding") + __slots__ = ( + "_cached_benc", + "_cached_decoded_json", + "_cached_selector", + "_cached_ubody", + "_encoding", + ) def __init__(self, *args: Any, **kwargs: Any): - self._encoding: Optional[str] = kwargs.pop("encoding", None) - self._cached_base_url: Optional[str] = None - self._cached_benc: Optional[str] = None - self._cached_ubody: Optional[str] = None - self._cached_selector: Optional[Selector] = None + self._encoding: str | None = kwargs.pop("encoding", None) + self._cached_base_url: str | None = None + self._cached_benc: str | None = None + self._cached_ubody: str | None = None + self._cached_selector: Selector | None = None + self._cached_decoded_json: object = _NONE super().__init__(*args, **kwargs) - def _set_body(self, body: Union[str, bytes, None]) -> None: + def _set_body(self, body: str | bytes | None) -> None: self._body: bytes = b"" # used by encoding detection if isinstance(body, str): if self._encoding is None: @@ -77,7 +76,7 @@ class TextResponse(Response): def encoding(self) -> str: return self._declared_encoding() or self._body_inferred_encoding() - def _declared_encoding(self) -> Optional[str]: + def _declared_encoding(self) -> str | None: return ( self._encoding or self._bom_encoding() @@ -86,11 +85,7 @@ class TextResponse(Response): ) def json(self) -> Any: - """ - .. versionadded:: 2.2 - - Deserialize a JSON document to a Python object. - """ + """Deserialize a JSON document to a Python object.""" if self._cached_decoded_json is _NONE: self._cached_decoded_json = json.loads(self.body) return self._cached_decoded_json @@ -129,14 +124,15 @@ class TextResponse(Response): return urljoin(self.base_url, url) @memoizemethod_noargs - def _headers_encoding(self) -> Optional[str]: - content_type = cast(bytes, self.headers.get(b"Content-Type", b"")) + def _headers_encoding(self) -> str | None: + content_type = self.headers.get(b"Content-Type") or b"" return http_content_type_encoding(to_unicode(content_type, encoding="latin-1")) def _body_inferred_encoding(self) -> str: if self._cached_benc is None: content_type = to_unicode( - cast(bytes, self.headers.get(b"Content-Type", b"")), encoding="latin-1" + cast("bytes", self.headers.get(b"Content-Type", b"")), + encoding="latin-1", ) benc, ubody = html_to_unicode( content_type, @@ -148,7 +144,7 @@ class TextResponse(Response): self._cached_ubody = ubody return self._cached_benc - def _auto_detect_fun(self, text: bytes) -> Optional[str]: + def _auto_detect_fun(self, text: bytes) -> str | None: for enc in (self._DEFAULT_ENCODING, "utf-8", "cp1252"): try: text.decode(enc) @@ -158,68 +154,62 @@ class TextResponse(Response): return None @memoizemethod_noargs - def _body_declared_encoding(self) -> Optional[str]: + def _body_declared_encoding(self) -> str | None: return html_body_declared_encoding(self.body) @memoizemethod_noargs - def _bom_encoding(self) -> Optional[str]: + def _bom_encoding(self) -> str | None: return read_bom(self.body)[0] @property def selector(self) -> Selector: - from scrapy.selector import Selector - if self._cached_selector is None: + # circular import + from scrapy.selector import Selector # noqa: PLC0415 + self._cached_selector = Selector(self) return self._cached_selector def jmespath(self, query: str, **kwargs: Any) -> SelectorList: - from scrapy.selector import SelectorList - - if not hasattr(self.selector, "jmespath"): # type: ignore[attr-defined] + if not hasattr(self.selector, "jmespath"): raise AttributeError( "Please install parsel >= 1.8.1 to get jmespath support" ) - - return cast(SelectorList, self.selector.jmespath(query, **kwargs)) # type: ignore[attr-defined] + return cast("SelectorList", self.selector.jmespath(query, **kwargs)) def xpath(self, query: str, **kwargs: Any) -> SelectorList: - from scrapy.selector import SelectorList - - return cast(SelectorList, self.selector.xpath(query, **kwargs)) + return cast("SelectorList", self.selector.xpath(query, **kwargs)) def css(self, query: str) -> SelectorList: - from scrapy.selector import SelectorList - - return cast(SelectorList, self.selector.css(query)) + return cast("SelectorList", self.selector.css(query)) def follow( self, - url: Union[str, Link, parsel.Selector], - callback: Optional[Callable] = None, + url: str | Link | parsel.Selector, + callback: CallbackT | None = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, - body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, - meta: Optional[Dict[str, Any]] = None, - encoding: Optional[str] = None, + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + body: bytes | str | None = None, + cookies: CookiesT | None = None, + meta: dict[str, Any] | None = None, + encoding: str | None = None, priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, - flags: Optional[List[str]] = None, + errback: Callable[[Failure], Any] | None = None, + cb_kwargs: dict[str, Any] | None = None, + flags: list[str] | None = None, ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. - It accepts the same arguments as ``Request.__init__`` method, + It accepts the same arguments as ``Request.__init__()`` method, but ``url`` can be not only an absolute URL, but also * a relative URL * a :class:`~scrapy.link.Link` object, e.g. the result of :ref:`topics-link-extractors` - * a :class:`~scrapy.selector.Selector` object for a ```` or ```` element, e.g. + * a :class:`~scrapy.Selector` object for a ```` or ```` element, e.g. ``response.css('a.my_link')[0]`` - * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g. + * an attribute :class:`~scrapy.Selector` (not SelectorList), e.g. ``response.css('a::attr(href)')[0]`` or ``response.xpath('//img/@src')[0]`` @@ -248,39 +238,39 @@ class TextResponse(Response): def follow_all( self, - urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None, - callback: Optional[Callable] = None, + urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None, + callback: CallbackT | None = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, - body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, - meta: Optional[Dict[str, Any]] = None, - encoding: Optional[str] = None, + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + body: bytes | str | None = None, + cookies: CookiesT | None = None, + meta: dict[str, Any] | None = None, + encoding: str | None = None, priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, - flags: Optional[List[str]] = None, - css: Optional[str] = None, - xpath: Optional[str] = None, - ) -> Generator[Request, None, None]: + errback: Callable[[Failure], Any] | None = None, + cb_kwargs: dict[str, Any] | None = None, + flags: list[str] | None = None, + css: str | None = None, + xpath: str | None = None, + ) -> Iterable[Request]: """ A generator that produces :class:`~.Request` instances to follow all links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s - ``__init__`` method, except that each ``urls`` element does not need to be + ``__init__()`` method, except that each ``urls`` element does not need to be an absolute URL, it can be any of the following: * a relative URL * a :class:`~scrapy.link.Link` object, e.g. the result of :ref:`topics-link-extractors` - * a :class:`~scrapy.selector.Selector` object for a ```` or ```` element, e.g. + * a :class:`~scrapy.Selector` object for a ```` or ```` element, e.g. ``response.css('a.my_link')[0]`` - * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g. + * an attribute :class:`~scrapy.Selector` (not SelectorList), e.g. ``response.css('a::attr(href)')[0]`` or ``response.xpath('//img/@src')[0]`` In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction - within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted). + within the ``follow_all()`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted). Note that when passing a ``SelectorList`` as argument for the ``urls`` parameter or using the ``css`` or ``xpath`` parameters, this method will not produce requests for @@ -304,7 +294,7 @@ class TextResponse(Response): with suppress(_InvalidSelector): urls.append(_url_from_selector(sel)) return super().follow_all( - urls=cast(Iterable[Union[str, Link]], urls), + urls=cast("Iterable[str | Link]", urls), callback=callback, method=method, headers=headers, @@ -332,9 +322,9 @@ def _url_from_selector(sel: parsel.Selector) -> str: return strip_html5_whitespace(sel.root) if not hasattr(sel.root, "tag"): raise _InvalidSelector(f"Unsupported selector: {sel}") - if sel.root.tag not in ("a", "link"): + if sel.root.tag not in {"a", "link"}: raise _InvalidSelector( - "Only and elements are supported; " f"got <{sel.root.tag}>" + f"Only and elements are supported; got <{sel.root.tag}>" ) href = sel.root.get("href") if href is None: diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.py index abf474a2f..6d9c4cb73 100644 --- a/scrapy/http/response/xml.py +++ b/scrapy/http/response/xml.py @@ -9,4 +9,4 @@ from scrapy.http.response.text import TextResponse class XmlResponse(TextResponse): - pass + __slots__ = () diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index 9a2c5f170..b4f1d9394 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -1,3 +1,5 @@ +# pylint: disable=no-method-argument,no-self-argument + from zope.interface import Interface diff --git a/scrapy/item.py b/scrapy/item.py index d3eb90b7b..1cc0ae584 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -4,16 +4,24 @@ Scrapy Item See documentation in docs/topics/item.rst """ +from __future__ import annotations + from abc import ABCMeta from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat -from typing import Dict +from typing import TYPE_CHECKING, Any, NoReturn from scrapy.utils.trackref import object_ref +if TYPE_CHECKING: + from collections.abc import Iterator, KeysView -class Field(dict): + # typing.Self requires Python 3.11 + from typing_extensions import Self + + +class Field(dict[str, Any]): """Container of field metadata""" @@ -23,7 +31,9 @@ class ItemMeta(ABCMeta): .. _metaclass: https://realpython.com/python-metaclasses """ - def __new__(mcs, class_name, bases, attrs): + def __new__( + mcs, class_name: str, bases: tuple[type, ...], attrs: dict[str, Any] + ) -> ItemMeta: classcell = attrs.pop("__classcell__", None) new_bases = tuple(base._class for base in bases if hasattr(base, "_class")) _class = super().__new__(mcs, "x_" + class_name, new_bases, attrs) @@ -44,17 +54,14 @@ class ItemMeta(ABCMeta): return super().__new__(mcs, class_name, bases, new_attrs) -class Item(MutableMapping, object_ref, metaclass=ItemMeta): - """ - Base class for scraped items. +class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta): + """Base class for scraped items. - In Scrapy, an object is considered an ``item`` if it is an instance of either - :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a - spider callback is evaluated, only instances of :class:`Item` or - :class:`dict` are passed to :ref:`item pipelines `. - - If you need instances of a custom class to be considered items by Scrapy, - you must inherit from either :class:`Item` or :class:`dict`. + In Scrapy, an object is considered an ``item`` if it's supported by the + `itemadapter`_ library. For example, when the output of a spider callback + is evaluated, only such objects are passed to :ref:`item pipelines + `. :class:`Item` is one of the classes supported by + `itemadapter`_ by default. Items must declare :class:`Field` attributes, which are processed and stored in the ``fields`` attribute. This restricts the set of allowed field names @@ -65,55 +72,61 @@ class Item(MutableMapping, object_ref, metaclass=ItemMeta): Unlike instances of :class:`dict`, instances of :class:`Item` may be :ref:`tracked ` to debug memory leaks. + + .. _itemadapter: https://github.com/scrapy/itemadapter """ - fields: Dict[str, Field] + #: A dictionary containing *all declared fields* for this Item, not only + #: those populated. The keys are the field names and the values are the + #: :class:`Field` objects used in the :ref:`Item declaration + #: `. + fields: dict[str, Field] - def __init__(self, *args, **kwargs): - self._values = {} + def __init__(self, *args: Any, **kwargs: Any): + self._values: dict[str, Any] = {} if args or kwargs: # avoid creating dict for most common case for k, v in dict(*args, **kwargs).items(): self[k] = v - def __getitem__(self, key): + def __getitem__(self, key: str) -> Any: return self._values[key] - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: Any) -> None: if key in self.fields: self._values[key] = value else: raise KeyError(f"{self.__class__.__name__} does not support field: {key}") - def __delitem__(self, key): + def __delitem__(self, key: str) -> None: del self._values[key] - def __getattr__(self, name): + def __getattr__(self, name: str) -> NoReturn: if name in self.fields: raise AttributeError(f"Use item[{name!r}] to get field value") raise AttributeError(name) - def __setattr__(self, name, value): + def __setattr__(self, name: str, value: Any) -> None: if not name.startswith("_"): raise AttributeError(f"Use item[{name!r}] = {value!r} to set field value") super().__setattr__(name, value) - def __len__(self): + def __len__(self) -> int: return len(self._values) - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self._values) __hash__ = object_ref.__hash__ - def keys(self): + def keys(self) -> KeysView[str]: return self._values.keys() - def __repr__(self): + def __repr__(self) -> str: return pformat(dict(self)) - def copy(self): + def copy(self) -> Self: return self.__class__(self) - def deepcopy(self): + def deepcopy(self) -> Self: """Return a :func:`~copy.deepcopy` of this item.""" return deepcopy(self) diff --git a/scrapy/link.py b/scrapy/link.py index 0868ae5ef..9c272ab2f 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,7 +4,6 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ -from typing import Any class Link: @@ -25,7 +24,7 @@ class Link: of the anchor tag. """ - __slots__ = ["url", "text", "fragment", "nofollow"] + __slots__ = ["fragment", "nofollow", "text", "url"] def __init__( self, url: str, text: str = "", fragment: str = "", nofollow: bool = False @@ -38,7 +37,7 @@ class Link: self.fragment: str = fragment self.nofollow: bool = nofollow - def __eq__(self, other: Any) -> bool: + def __eq__(self, other: object) -> bool: if not isinstance(other, Link): raise NotImplementedError return ( diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 3774430a7..b39859f7b 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -5,7 +5,14 @@ This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ -import re + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable + from re import Pattern # common file extensions that are not followed if they occur in links IGNORED_EXTENSIONS = [ @@ -109,16 +116,18 @@ IGNORED_EXTENSIONS = [ ] -_re_type = type(re.compile("", 0)) - - -def _matches(url, regexs): +def _matches(url: str, regexs: Iterable[Pattern[str]]) -> bool: return any(r.search(url) for r in regexs) -def _is_valid_url(url): +def _is_valid_url(url: str) -> bool: return url.split("://", 1)[0] in {"http", "https", "file", "ftp"} # Top-level imports from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor + +__all__ = [ + "IGNORED_EXTENSIONS", + "LinkExtractor", +] diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 7d634b5c9..9ad0f48a3 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,9 +1,15 @@ """ Link extractor based on lxml.html """ + +from __future__ import annotations + import logging import operator +import re +from collections.abc import Callable, Iterable from functools import partial +from typing import TYPE_CHECKING, Any, TypeAlias, cast from urllib.parse import urljoin, urlparse from lxml import etree @@ -12,17 +18,18 @@ from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string from scrapy.link import Link -from scrapy.linkextractors import ( - IGNORED_EXTENSIONS, - _is_valid_url, - _matches, - _re_type, - re, -) +from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain +if TYPE_CHECKING: + from lxml.html import HtmlElement + + from scrapy import Selector + from scrapy.http import TextResponse + + logger = logging.getLogger(__name__) # from lxml/src/lxml/html/__init__.py @@ -31,41 +38,59 @@ XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml" _collect_string_content = etree.XPath("string()") -def _nons(tag): - if isinstance(tag, str): - if tag[0] == "{" and tag[1 : len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE: - return tag.split("}")[-1] +def _nons(tag: Any) -> Any: + if ( + isinstance(tag, str) + and tag[0] == "{" + and tag[1 : len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE + ): + return tag.split("}")[-1] return tag -def _identity(x): +def _identity(x: Any) -> Any: return x -def _canonicalize_link_url(link): +def _canonicalize_link_url(link: Link) -> str: return canonicalize_url(link.url, keep_fragments=True) class LxmlParserLinkExtractor: def __init__( self, - tag="a", - attr="href", - process=None, - unique=False, - strip=True, - canonicalized=False, + tag: str | Callable[[str], bool] = "a", + attr: str | Callable[[str], bool] = "href", + process: Callable[[Any], Any] | None = None, + unique: bool = False, + strip: bool = True, + canonicalized: bool = False, ): - self.scan_tag = tag if callable(tag) else partial(operator.eq, tag) - self.scan_attr = attr if callable(attr) else partial(operator.eq, attr) - self.process_attr = process if callable(process) else _identity - self.unique = unique - self.strip = strip - self.link_key = ( - operator.attrgetter("url") if canonicalized else _canonicalize_link_url + # mypy doesn't infer types for operator.* and also for partial() + self.scan_tag: Callable[[str], bool] = ( + tag + if callable(tag) + else cast("Callable[[str], bool]", partial(operator.eq, tag)) + ) + self.scan_attr: Callable[[str], bool] = ( + attr + if callable(attr) + else cast("Callable[[str], bool]", partial(operator.eq, attr)) + ) + self.process_attr: Callable[[Any], Any] = ( + process if callable(process) else _identity + ) + self.unique: bool = unique + self.strip: bool = strip + self.link_key: Callable[[Link], str] = ( + cast("Callable[[Link], str]", operator.attrgetter("url")) + if canonicalized + else _canonicalize_link_url ) - def _iter_links(self, document): + def _iter_links( + self, document: HtmlElement + ) -> Iterable[tuple[HtmlElement, str, str]]: for el in document.iter(etree.Element): if not self.scan_tag(_nons(el.tag)): continue @@ -73,17 +98,23 @@ class LxmlParserLinkExtractor: for attrib in attribs: if not self.scan_attr(attrib): continue - yield (el, attrib, attribs[attrib]) + yield el, attrib, attribs[attrib] - def _extract_links(self, selector, response_url, response_encoding, base_url): - links = [] + def _extract_links( + self, + selector: Selector, + response_url: str, + response_encoding: str, + base_url: str, + ) -> list[Link]: + links: list[Link] = [] # hacky way to get the underlying lxml parsed document - for el, attr, attr_val in self._iter_links(selector.root): + for el, _, attr_val in self._iter_links(selector.root): # pseudo lxml.html.HtmlElement.make_links_absolute(base_url) try: if self.strip: - attr_val = strip_html5_whitespace(attr_val) - attr_val = urljoin(base_url, attr_val) + attr_val = strip_html5_whitespace(attr_val) # noqa: PLW2901 this is intended + attr_val = urljoin(base_url, attr_val) # noqa: PLW2901 except ValueError: continue # skipping bogus links else: @@ -106,7 +137,7 @@ class LxmlParserLinkExtractor: links.append(link) return self._deduplicate_if_needed(links) - def extract_links(self, response): + def extract_links(self, response: TextResponse) -> list[Link]: return self._extract_links( response.selector, response.url, @@ -114,38 +145,42 @@ class LxmlParserLinkExtractor: response.base_url, ) - def _process_links(self, links): + def _process_links(self, links: list[Link]) -> list[Link]: """Normalize and filter extracted links The subclass should override it if necessary """ return self._deduplicate_if_needed(links) - def _deduplicate_if_needed(self, links): + def _deduplicate_if_needed(self, links: list[Link]) -> list[Link]: if self.unique: return unique_list(links, key=self.link_key) return links +_Regex: TypeAlias = str | re.Pattern[str] +_RegexOrSeveral: TypeAlias = _Regex | Iterable[_Regex] + + class LxmlLinkExtractor: _csstranslator = HTMLTranslator() def __init__( self, - allow=(), - deny=(), - allow_domains=(), - deny_domains=(), - restrict_xpaths=(), - tags=("a", "area"), - attrs=("href",), - canonicalize=False, - unique=True, - process_value=None, - deny_extensions=None, - restrict_css=(), - strip=True, - restrict_text=None, + allow: _RegexOrSeveral = (), + deny: _RegexOrSeveral = (), + allow_domains: str | Iterable[str] = (), + deny_domains: str | Iterable[str] = (), + restrict_xpaths: str | Iterable[str] = (), + tags: str | Iterable[str] = ("a", "area"), + attrs: str | Iterable[str] = ("href",), + canonicalize: bool = False, + unique: bool = True, + process_value: Callable[[Any], Any] | None = None, + deny_extensions: str | Iterable[str] | None = None, + restrict_css: str | Iterable[str] = (), + strip: bool = True, + restrict_text: _RegexOrSeveral | None = None, ): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) self.link_extractor = LxmlParserLinkExtractor( @@ -154,33 +189,33 @@ class LxmlLinkExtractor: unique=unique, process=process_value, strip=strip, - canonicalized=canonicalize, + canonicalized=not canonicalize, ) - self.allow_res = [ - x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow) - ] - self.deny_res = [ - x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny) - ] + self.allow_res: list[re.Pattern[str]] = self._compile_regexes(allow) + self.deny_res: list[re.Pattern[str]] = self._compile_regexes(deny) - self.allow_domains = set(arg_to_iter(allow_domains)) - self.deny_domains = set(arg_to_iter(deny_domains)) + self.allow_domains: set[str] = set(arg_to_iter(allow_domains)) + self.deny_domains: set[str] = set(arg_to_iter(deny_domains)) - self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) + self.restrict_xpaths: tuple[str, ...] = tuple(arg_to_iter(restrict_xpaths)) self.restrict_xpaths += tuple( map(self._csstranslator.css_to_xpath, arg_to_iter(restrict_css)) ) if deny_extensions is None: deny_extensions = IGNORED_EXTENSIONS - self.canonicalize = canonicalize - self.deny_extensions = {"." + e for e in arg_to_iter(deny_extensions)} - self.restrict_text = [ - x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(restrict_text) + self.canonicalize: bool = canonicalize + self.deny_extensions: set[str] = {"." + e for e in arg_to_iter(deny_extensions)} + self.restrict_text: list[re.Pattern[str]] = self._compile_regexes(restrict_text) + + @staticmethod + def _compile_regexes(value: _RegexOrSeveral | None) -> list[re.Pattern[str]]: + return [ + x if isinstance(x, re.Pattern) else re.compile(x) + for x in arg_to_iter(value) ] - def _link_allowed(self, link): + def _link_allowed(self, link: Link) -> bool: if not _is_valid_url(link.url): return False if self.allow_res and not _matches(link.url, self.allow_res): @@ -198,11 +233,9 @@ class LxmlLinkExtractor: parsed_url, self.deny_extensions ): return False - if self.restrict_text and not _matches(link.text, self.restrict_text): - return False - return True + return not self.restrict_text or _matches(link.text, self.restrict_text) - def matches(self, url): + def matches(self, url: str) -> bool: if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): return False if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): @@ -213,21 +246,20 @@ class LxmlLinkExtractor: if self.allow_res else [True] ) - denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] + denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else () return any(allowed) and not any(denied) - def _process_links(self, links): + def _process_links(self, links: list[Link]) -> list[Link]: links = [x for x in links if self._link_allowed(x)] if self.canonicalize: for link in links: link.url = canonicalize_url(link.url) - links = self.link_extractor._process_links(links) - return links + return self.link_extractor._process_links(links) - def _extract_links(self, *args, **kwargs): + def _extract_links(self, *args: Any, **kwargs: Any) -> list[Link]: return self.link_extractor._extract_links(*args, **kwargs) - def extract_links(self, response): + def extract_links(self, response: TextResponse) -> list[Link]: """Returns a list of :class:`~scrapy.link.Link` objects from the specified :class:`response `. @@ -253,5 +285,5 @@ class LxmlLinkExtractor: ) all_links.extend(self._process_links(links)) if self.link_extractor.unique: - return unique_list(all_links) + return unique_list(all_links, key=self.link_extractor.link_key) return all_links diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 1042a3d48..2f5c0343b 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -3,11 +3,19 @@ Item Loader See documentation in docs/topics/loaders.rst """ + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + import itemloaders from scrapy.item import Item from scrapy.selector import Selector +if TYPE_CHECKING: + from scrapy.http import TextResponse + class ItemLoader(itemloaders.ItemLoader): """ @@ -24,7 +32,7 @@ class ItemLoader(itemloaders.ItemLoader): :param selector: The selector to extract data from, when using the :meth:`add_xpath`, :meth:`add_css`, :meth:`replace_xpath`, or :meth:`replace_css` method. - :type selector: :class:`~scrapy.selector.Selector` object + :type selector: :class:`~scrapy.Selector` object :param response: The response used to construct the selector using the :attr:`default_selector_class`, unless the selector argument is given, @@ -71,7 +79,7 @@ class ItemLoader(itemloaders.ItemLoader): .. attribute:: selector - The :class:`~scrapy.selector.Selector` object to extract data from. + The :class:`~scrapy.Selector` object to extract data from. It's either the selector given in the ``__init__`` method or one created from the response given in the ``__init__`` method using the :attr:`default_selector_class`. This attribute is meant to be @@ -81,7 +89,14 @@ class ItemLoader(itemloaders.ItemLoader): default_item_class: type = Item default_selector_class = Selector - def __init__(self, item=None, selector=None, response=None, parent=None, **context): + def __init__( + self, + item: Any = None, + selector: Selector | None = None, + response: TextResponse | None = None, + parent: itemloaders.ItemLoader | None = None, + **context: Any, + ): if selector is None and response is not None: try: selector = self.default_selector_class(response) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index d720b2f38..a50064e08 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -2,12 +2,14 @@ from __future__ import annotations import logging import os -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, TypedDict from twisted.python.failure import Failure -from scrapy import Request, Spider -from scrapy.http import Response +# working around https://github.com/sphinx-doc/sphinx/issues/10400 +from scrapy import Request, Spider # noqa: TC001 +from scrapy.http import Response # noqa: TC001 +from scrapy.utils.python import global_object_name from scrapy.utils.request import referer_str if TYPE_CHECKING: @@ -26,6 +28,12 @@ DOWNLOADERRORMSG_SHORT = "Error downloading %(request)s" DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s" +class LogFormatterResult(TypedDict): + level: int + msg: str + args: dict[str, Any] | tuple[Any, ...] + + class LogFormatter: """Class for generating log messages for different actions. @@ -64,10 +72,12 @@ class LogFormatter: } """ - def crawled(self, request: Request, response: Response, spider: Spider) -> dict: + def crawled( + self, request: Request, response: Response, spider: Spider + ) -> LogFormatterResult: """Logs a message when the crawler finds a webpage.""" - request_flags = f" {str(request.flags)}" if request.flags else "" - response_flags = f" {str(response.flags)}" if response.flags else "" + request_flags = f" {request.flags!s}" if request.flags else "" + response_flags = f" {response.flags!s}" if response.flags else "" return { "level": logging.DEBUG, "msg": CRAWLEDMSG, @@ -83,11 +93,13 @@ class LogFormatter: } def scraped( - self, item: Any, response: Union[Response, Failure], spider: Spider - ) -> dict: + self, item: Any, response: Response | Failure | None, spider: Spider + ) -> LogFormatterResult: """Logs a message when an item is scraped by a spider.""" src: Any - if isinstance(response, Failure): + if response is None: + src = f"{global_object_name(spider.__class__)}.start" + elif isinstance(response, Failure): src = response.getErrorMessage() else: src = response @@ -101,11 +113,19 @@ class LogFormatter: } def dropped( - self, item: Any, exception: BaseException, response: Response, spider: Spider - ) -> dict: + self, + item: Any, + exception: BaseException, + response: Response | Failure | None, + spider: Spider, + ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" + if (level := getattr(exception, "log_level", None)) is None: + level = spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] + if isinstance(level, str): + level = getattr(logging, level) return { - "level": logging.WARNING, + "level": level, "msg": DROPPEDMSG, "args": { "exception": exception, @@ -114,12 +134,14 @@ class LogFormatter: } def item_error( - self, item: Any, exception: BaseException, response: Response, spider: Spider - ) -> dict: + self, + item: Any, + exception: BaseException, + response: Response | Failure | None, + spider: Spider, + ) -> LogFormatterResult: """Logs a message when an item causes an error while it is passing through the item pipeline. - - .. versionadded:: 2.0 """ return { "level": logging.ERROR, @@ -133,13 +155,10 @@ class LogFormatter: self, failure: Failure, request: Request, - response: Union[Response, Failure], + response: Response | Failure, spider: Spider, - ) -> dict: - """Logs an error message from a spider. - - .. versionadded:: 2.0 - """ + ) -> LogFormatterResult: + """Logs an error message from a spider.""" return { "level": logging.ERROR, "msg": SPIDERERRORMSG, @@ -154,14 +173,12 @@ class LogFormatter: failure: Failure, request: Request, spider: Spider, - errmsg: Optional[str] = None, - ) -> dict: + errmsg: str | None = None, + ) -> LogFormatterResult: """Logs a download error message from a spider (typically coming from the engine). - - .. versionadded:: 2.0 """ - args: Dict[str, Any] = {"request": request} + args: dict[str, Any] = {"request": request} if errmsg: msg = DOWNLOADERRORMSG_LONG args["errmsg"] = errmsg diff --git a/scrapy/mail.py b/scrapy/mail.py index 237327451..fbd11ad1d 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -3,7 +3,11 @@ Mail sending helpers See documentation in docs/topics/email.rst """ + +from __future__ import annotations + import logging +import warnings from email import encoders as Encoders from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart @@ -11,13 +15,28 @@ from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO +from typing import IO, TYPE_CHECKING, Any -from twisted import version as twisted_version -from twisted.internet import defer, ssl -from twisted.python.versions import Version +from twisted.internet import ssl +from twisted.internet.defer import Deferred +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes +from scrapy.utils.reactorless import is_reactorless + +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + + # imports twisted.internet.reactor + from twisted.mail.smtp import ESMTPSenderFactory + from twisted.python.failure import Failure + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) @@ -27,35 +46,46 @@ logger = logging.getLogger(__name__) COMMASPACE = ", " -def _to_bytes_or_none(text): +def _to_bytes_or_none(text: str | bytes | None) -> bytes | None: if text is None: return None return to_bytes(text) +warnings.warn( + "The scrapy.mail module is deprecated and will be removed in a future release. " + "Please use a dedicated Python mail library instead.", + stacklevel=2, + category=ScrapyDeprecationWarning, +) + + class MailSender: def __init__( self, - smtphost="localhost", - mailfrom="scrapy@localhost", - smtpuser=None, - smtppass=None, - smtpport=25, - smtptls=False, - smtpssl=False, - debug=False, + smtphost: str = "localhost", + mailfrom: str = "scrapy@localhost", + smtpuser: str | None = None, + smtppass: str | None = None, + smtpport: int = 25, + smtptls: bool = False, + smtpssl: bool = False, + debug: bool = False, ): - self.smtphost = smtphost - self.smtpport = smtpport - self.smtpuser = _to_bytes_or_none(smtpuser) - self.smtppass = _to_bytes_or_none(smtppass) - self.smtptls = smtptls - self.smtpssl = smtpssl - self.mailfrom = mailfrom - self.debug = debug + if is_reactorless(): # pragma: no cover + raise RuntimeError(f"{type(self).__name__} requires a Twisted reactor.") + self.smtphost: str = smtphost + self.smtpport: int = smtpport + self.smtpuser: bytes | None = _to_bytes_or_none(smtpuser) + self.smtppass: bytes | None = _to_bytes_or_none(smtppass) + self.smtptls: bool = smtptls + self.smtpssl: bool = smtpssl + self.mailfrom: str = mailfrom + self.debug: bool = debug @classmethod - def from_settings(cls, settings): + def from_crawler(cls, crawler: Crawler) -> Self: # pragma: no cover + settings = crawler.settings return cls( smtphost=settings["MAIL_HOST"], mailfrom=settings["MAIL_FROM"], @@ -68,21 +98,20 @@ class MailSender: def send( self, - to, - subject, - body, - cc=None, - attachs=(), - mimetype="text/plain", - charset=None, - _callback=None, - ): + to: str | list[str], + subject: str, + body: str, + cc: str | list[str] | None = None, + attachs: Sequence[tuple[str, str, IO[Any]]] = (), + mimetype: str = "text/plain", + charset: str | None = None, + _callback: Callable[..., None] | None = None, + ) -> Deferred[None] | None: from twisted.internet import reactor - if attachs: - msg = MIMEMultipart() - else: - msg = MIMENonMultipart(*mimetype.split("/", 1)) + msg: MIMEBase = ( + MIMEMultipart() if attachs else MIMENonMultipart(*mimetype.split("/", 1)) + ) to = list(arg_to_iter(to)) cc = list(arg_to_iter(cc)) @@ -100,8 +129,8 @@ class MailSender: if charset: msg.set_charset(charset) msg.attach(MIMEText(body, "plain", charset or "us-ascii")) - for attach_name, mimetype, f in attachs: - part = MIMEBase(*mimetype.split("/")) + for attach_name, attach_mimetype, f in attachs: + part = MIMEBase(*attach_mimetype.split("/")) part.set_payload(f.read()) Encoders.encode_base64(part) part.add_header( @@ -125,19 +154,19 @@ class MailSender: "mailattachs": len(attachs), }, ) - return + return None - dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8")) - dfd.addCallbacks( - callback=self._sent_ok, - errback=self._sent_failed, - callbackArgs=[to, cc, subject, len(attachs)], - errbackArgs=[to, cc, subject, len(attachs)], + dfd: Deferred[Any] = self._sendmail( + rcpts, msg.as_string().encode(charset or "utf-8") ) + dfd.addCallback(self._sent_ok, to, cc, subject, len(attachs)) + dfd.addErrback(self._sent_failed, to, cc, subject, len(attachs)) reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd) return dfd - def _sent_ok(self, result, to, cc, subject, nattachs): + def _sent_ok( + self, result: Any, to: list[str], cc: list[str], subject: str, nattachs: int + ) -> None: logger.info( "Mail sent OK: To=%(mailto)s Cc=%(mailcc)s " 'Subject="%(mailsubject)s" Attachs=%(mailattachs)d', @@ -149,7 +178,14 @@ class MailSender: }, ) - def _sent_failed(self, failure, to, cc, subject, nattachs): + def _sent_failed( + self, + failure: Failure, + to: list[str], + cc: list[str], + subject: str, + nattachs: int, + ) -> Failure: errstr = str(failure.value) logger.error( "Unable to send mail: To=%(mailto)s Cc=%(mailcc)s " @@ -165,13 +201,13 @@ class MailSender: ) return failure - def _sendmail(self, to_addrs, msg): + def _sendmail(self, to_addrs: list[str], msg: bytes) -> Deferred[Any]: from twisted.internet import reactor - msg = BytesIO(msg) - d = defer.Deferred() + msg_io = BytesIO(msg) + d: Deferred[Any] = Deferred() - factory = self._create_sender_factory(to_addrs, msg, d) + factory = self._create_sender_factory(to_addrs, msg_io, d) if self.smtpssl: reactor.connectSSL( @@ -182,19 +218,19 @@ class MailSender: return d - def _create_sender_factory(self, to_addrs, msg, d): - from twisted.mail.smtp import ESMTPSenderFactory + def _create_sender_factory( + self, to_addrs: list[str], msg: IO[bytes], d: Deferred[Any] + ) -> ESMTPSenderFactory: + # imports twisted.internet.reactor + from twisted.mail.smtp import ESMTPSenderFactory # noqa: PLC0415 - factory_keywords = { + factory_keywords: dict[str, Any] = { "heloFallback": True, "requireAuthentication": False, "requireTransportSecurity": self.smtptls, + "hostname": self.smtphost, } - # Newer versions of twisted require the hostname to use STARTTLS - if twisted_version >= Version("twisted", 21, 2, 0): - factory_keywords["hostname"] = self.smtphost - factory = ESMTPSenderFactory( self.smtpuser, self.smtppass, @@ -202,7 +238,7 @@ class MailSender: to_addrs, msg, d, - **factory_keywords + **factory_keywords, ) factory.noisy = False return factory diff --git a/scrapy/middleware.py b/scrapy/middleware.py index f60c726f9..37ec0f583 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -2,72 +2,97 @@ from __future__ import annotations import logging import pprint +import warnings +from abc import ABC, abstractmethod from collections import defaultdict, deque -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Deque, - Dict, - Iterable, - List, - Optional, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, cast -from twisted.internet.defer import Deferred - -from scrapy import Spider -from scrapy.exceptions import NotConfigured -from scrapy.settings import Settings -from scrapy.utils.defer import process_chain, process_parallel -from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.utils.defer import ensure_awaitable +from scrapy.utils.deprecate import argument_is_required +from scrapy.utils.misc import build_from_crawler, load_object +from scrapy.utils.python import global_object_name if TYPE_CHECKING: + from collections.abc import Callable, Iterable + + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider from scrapy.crawler import Crawler + from scrapy.settings import Settings logger = logging.getLogger(__name__) +_T = TypeVar("_T") +_P = ParamSpec("_P") -class MiddlewareManager: + +class MiddlewareManager(ABC): """Base class for implementing middleware managers""" - component_name = "foo middleware" + component_name: str + _compat_spider: Spider | None = None - def __init__(self, *middlewares: Any) -> None: - self.middlewares = middlewares + def __init__(self, *middlewares: Any, crawler: Crawler | None = None) -> None: + self.crawler: Crawler | None = crawler + if crawler is None: + warnings.warn( + f"MiddlewareManager.__init__() was called without the crawler argument" + f" when creating {global_object_name(self.__class__)}." + f" This is deprecated and the argument will be required in future Scrapy versions.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self.middlewares: tuple[Any, ...] = middlewares # Only process_spider_output and process_spider_exception can be None. - # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed. - self.methods: Dict[ - str, Deque[Union[None, Callable, Tuple[Callable, Callable]]] - ] = defaultdict(deque) + self.methods: dict[str, deque[Callable[..., Any] | None]] = defaultdict(deque) + self._mw_methods_requiring_spider: set[Callable[..., Any]] = set() for mw in middlewares: self._add_middleware(mw) + @property + def _spider(self) -> Spider: + if self.crawler is not None: + if self.crawler.spider is None: + raise ValueError( + f"{type(self).__name__} needs to access self.crawler.spider but it is None." + ) + return self.crawler.spider + if self._compat_spider is not None: + return self._compat_spider + raise ValueError(f"{type(self).__name__} has no known Spider instance.") + + def _set_compat_spider(self, spider: Spider | None) -> None: + if spider is None or self.crawler is not None: + return + # printing a deprecation warning is the caller's responsibility + if self._compat_spider is None: + self._compat_spider = spider + elif self._compat_spider is not spider: + raise RuntimeError( + f"Different instances of Spider were passed to {type(self).__name__}:" + f" {self._compat_spider} and {spider}" + ) + @classmethod - def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: + @abstractmethod + def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]: raise NotImplementedError @classmethod - def from_settings( - cls, settings: Settings, crawler: Optional[Crawler] = None - ) -> Self: - mwlist = cls._get_mwlist_from_settings(settings) + def from_crawler(cls, crawler: Crawler) -> Self: + mwlist = cls._get_mwlist_from_settings(crawler.settings) middlewares = [] enabled = [] for clspath in mwlist: try: mwcls = load_object(clspath) - if crawler is not None: - mw = build_from_crawler(mwcls, crawler) - else: - mw = build_from_settings(mwcls, settings) + mw = build_from_crawler(mwcls, crawler) middlewares.append(mw) enabled.append(clspath) except NotConfigured as e: @@ -86,28 +111,55 @@ class MiddlewareManager: }, extra={"crawler": crawler}, ) - return cls(*middlewares) + return cls(*middlewares, crawler=crawler) - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - return cls.from_settings(crawler.settings, crawler) + def _add_middleware(self, mw: Any) -> None: # noqa: B027 + pass - def _add_middleware(self, mw: Any) -> None: - if hasattr(mw, "open_spider"): - self.methods["open_spider"].append(mw.open_spider) - if hasattr(mw, "close_spider"): - self.methods["close_spider"].appendleft(mw.close_spider) + def _check_mw_method_spider_arg(self, method: Callable[..., Any]) -> None: + if argument_is_required(method, "spider"): + warnings.warn( + f"{method.__qualname__}() requires a spider argument," + f" this is deprecated and the argument will not be passed in future Scrapy versions." + f" If you need to access the spider instance you can save the crawler instance" + f" passed to from_crawler() and use its spider attribute.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self._mw_methods_requiring_spider.add(method) - def _process_parallel(self, methodname: str, obj: Any, *args: Any) -> Deferred: - methods = cast(Iterable[Callable], self.methods[methodname]) - return process_parallel(methods, obj, *args) + async def _process_chain( + self, + methodname: str, + obj: _T, + *args: Any, + add_spider: bool = False, + always_add_spider: bool = False, + warn_deferred: bool = False, + ) -> _T: + methods = cast( + "Iterable[Callable[Concatenate[_T, _P], _T]]", self.methods[methodname] + ) + for method in methods: + warn = global_object_name(method) if warn_deferred else None + if always_add_spider or ( + add_spider and method in self._mw_methods_requiring_spider + ): + obj = await ensure_awaitable( + method(obj, *(*args, self._spider)), _warn=warn + ) + else: + obj = await ensure_awaitable(method(obj, *args), _warn=warn) + return obj - def _process_chain(self, methodname: str, obj: Any, *args: Any) -> Deferred: - methods = cast(Iterable[Callable], self.methods[methodname]) - return process_chain(methods, obj, *args) + def open_spider(self, spider: Spider) -> Deferred[list[None]]: # pragma: no cover + raise NotImplementedError( + "MiddlewareManager.open_spider() is no longer implemented" + " and will be removed in a future Scrapy version." + ) - def open_spider(self, spider: Spider) -> Deferred: - return self._process_parallel("open_spider", spider) - - def close_spider(self, spider: Spider) -> Deferred: - return self._process_parallel("close_spider", spider) + def close_spider(self, spider: Spider) -> Deferred[list[None]]: # pragma: no cover + raise NotImplementedError( + "MiddlewareManager.close_spider() is no longer implemented" + " and will be removed in a future Scrapy version." + ) diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index c97d71fb6..84fb5f85e 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -3,29 +3,140 @@ Item pipeline See documentation in docs/item-pipeline.rst """ -from typing import Any, List -from twisted.internet.defer import Deferred +from __future__ import annotations -from scrapy import Spider +import asyncio +import warnings +from typing import TYPE_CHECKING, Any, cast + +from twisted.internet.defer import Deferred, DeferredList, FirstError + +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.middleware import MiddlewareManager +from scrapy.utils.asyncio import is_asyncio_available from scrapy.utils.conf import build_component_list -from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.defer import _maybeDeferred_coro, deferred_from_coro, ensure_awaitable +from scrapy.utils.python import global_object_name + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable, Coroutine, Iterable + + from twisted.python.failure import Failure + + from scrapy import Spider + from scrapy.settings import Settings class ItemPipelineManager(MiddlewareManager): component_name = "item pipeline" @classmethod - def _get_mwlist_from_settings(cls, settings) -> List[Any]: - return build_component_list(settings.getwithbase("ITEM_PIPELINES")) + def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]: + return build_component_list( + settings.get_component_priority_dict_with_base("ITEM_PIPELINES") + ) - def _add_middleware(self, pipe: Any) -> None: - super()._add_middleware(pipe) - if hasattr(pipe, "process_item"): - self.methods["process_item"].append( - deferred_f_from_coro_f(pipe.process_item) - ) + def _add_middleware(self, mw: Any) -> None: + if hasattr(mw, "open_spider"): + self.methods["open_spider"].append(mw.open_spider) + self._check_mw_method_spider_arg(mw.open_spider) + if hasattr(mw, "close_spider"): + self.methods["close_spider"].appendleft(mw.close_spider) + self._check_mw_method_spider_arg(mw.close_spider) + if hasattr(mw, "process_item"): + self.methods["process_item"].append(mw.process_item) + self._check_mw_method_spider_arg(mw.process_item) - def process_item(self, item: Any, spider: Spider) -> Deferred: - return self._process_chain("process_item", item, spider) + def process_item(self, item: Any, spider: Spider) -> Deferred[Any]: + warnings.warn( + f"{global_object_name(type(self))}.process_item() is deprecated, use process_item_async() instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self._set_compat_spider(spider) + return deferred_from_coro(self.process_item_async(item)) + + async def process_item_async(self, item: Any) -> Any: + return await self._process_chain( + "process_item", item, add_spider=True, warn_deferred=True + ) + + def _get_dfd( + self, + method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None], + ) -> Deferred[None]: + if method in self._mw_methods_requiring_spider: + return _maybeDeferred_coro(method, True, self._spider) + return _maybeDeferred_coro(method, True) + + @staticmethod + def _eb(failure: Failure) -> Failure: + assert isinstance(failure.value, FirstError) + return failure.value.subFailure + + def _process_parallel_dfd(self, methodname: str) -> Deferred[list[None]]: + methods = cast( + "Iterable[Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None]]", + self.methods[methodname], + ) + + dfds = [self._get_dfd(m) for m in methods] + d: Deferred[list[tuple[bool, None]]] = DeferredList( + dfds, fireOnOneErrback=True, consumeErrors=True + ) + d2: Deferred[list[None]] = d.addCallback(lambda r: [x[1] for x in r]) + + d2.addErrback(self._eb) + return d2 + + def get_awaitable( + self, + method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None], + ) -> Awaitable[None]: + if method in self._mw_methods_requiring_spider: + result = method(self._spider) + else: + result = method() + return ensure_awaitable(result, _warn=global_object_name(method)) + + async def _process_parallel_asyncio(self, methodname: str) -> list[None]: + methods = cast( + "Iterable[Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None]]", + self.methods[methodname], + ) + if not methods: + return [] + + awaitables = [self.get_awaitable(m) for m in methods] + await asyncio.gather(*awaitables) + return [None for _ in methods] + + async def _process_parallel(self, methodname: str) -> list[None]: + if is_asyncio_available(): + return await self._process_parallel_asyncio(methodname) + return await self._process_parallel_dfd(methodname) + + def open_spider(self, spider: Spider) -> Deferred[list[None]]: + warnings.warn( + f"{global_object_name(type(self))}.open_spider() is deprecated, use open_spider_async() instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self._set_compat_spider(spider) + return deferred_from_coro(self._process_parallel("open_spider")) + + async def open_spider_async(self) -> None: + await self._process_parallel("open_spider") + + def close_spider(self, spider: Spider) -> Deferred[list[None]]: + warnings.warn( + f"{global_object_name(type(self))}.close_spider() is deprecated, use close_spider_async() instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self._set_compat_spider(spider) + return deferred_from_coro(self._process_parallel("close_spider")) + + async def close_spider_async(self) -> None: + await self._process_parallel("close_spider") diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 73064ad10..0066fd38f 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -3,65 +3,130 @@ Files Pipeline See documentation in topics/media-pipeline.rst """ + +from __future__ import annotations + import base64 import functools import hashlib import logging import mimetypes import time +import warnings from collections import defaultdict from contextlib import suppress from ftplib import FTP from io import BytesIO -from os import PathLike from pathlib import Path -from typing import DefaultDict, Optional, Set, Union +from typing import IO, TYPE_CHECKING, Any, ClassVar, NoReturn, Protocol, TypedDict, cast from urllib.parse import urlparse from itemadapter import ItemAdapter -from twisted.internet import defer, threads +from twisted.internet.defer import Deferred, maybeDeferred -from scrapy.exceptions import IgnoreRequest, NotConfigured -from scrapy.http import Request +from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning +from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.media import MediaPipeline -from scrapy.settings import Settings +from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline +from scrapy.utils.asyncio import run_in_thread from scrapy.utils.boto import is_botocore_available from scrapy.utils.datatypes import CaseInsensitiveDict +from scrapy.utils.defer import deferred_from_coro, ensure_awaitable from scrapy.utils.ftp import ftp_store_file +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info -from scrapy.utils.misc import md5sum from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str +if TYPE_CHECKING: + from collections.abc import Awaitable + from os import PathLike + + from twisted.python.failure import Failure + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) -def _to_string(path: Union[str, PathLike]) -> str: +def _to_string(path: str | PathLike[str]) -> str: return str(path) # convert a Path object to string +def _md5sum(file: IO[bytes]) -> str: + """Calculate the md5 checksum of a file-like object without reading its + whole content in memory. + + >>> from io import BytesIO + >>> _md5sum(BytesIO(b'file content to hash')) + '784406af91dd5a54fbb9c84c2236595a' + """ + m = hashlib.md5() # noqa: S324 + while True: + d = file.read(8096) + if not d: + break + m.update(d) + return m.hexdigest() + + class FileException(Exception): """General media error exception""" +class StatInfo(TypedDict, total=False): + checksum: str + last_modified: float + + +class FilesStoreProtocol(Protocol): + def __init__(self, basedir: str): ... + + def persist_file( + self, + path: str, + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + ) -> Deferred[Any] | None: ... + + def stat_file( + self, path: str, info: MediaPipeline.SpiderInfo + ) -> StatInfo | Deferred[StatInfo]: ... + + class FSFilesStore: - def __init__(self, basedir: Union[str, PathLike]): + def __init__(self, basedir: str | PathLike[str]): basedir = _to_string(basedir) if "://" in basedir: basedir = basedir.split("://", 1)[1] - self.basedir = basedir + self.basedir: str = basedir self._mkdir(Path(self.basedir)) - self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set) + self.created_directories: defaultdict[MediaPipeline.SpiderInfo, set[str]] = ( + defaultdict(set) + ) def persist_file( - self, path: Union[str, PathLike], buf, info, meta=None, headers=None - ): + self, + path: str | PathLike[str], + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + ) -> None: absolute_path = self._get_filesystem_path(path) self._mkdir(absolute_path.parent, info) absolute_path.write_bytes(buf.getvalue()) - def stat_file(self, path: Union[str, PathLike], info): + def stat_file( + self, path: str | PathLike[str], info: MediaPipeline.SpiderInfo + ) -> StatInfo: absolute_path = self._get_filesystem_path(path) try: last_modified = absolute_path.stat().st_mtime @@ -69,16 +134,18 @@ class FSFilesStore: return {} with absolute_path.open("rb") as f: - checksum = md5sum(f) + checksum = _md5sum(f) return {"last_modified": last_modified, "checksum": checksum} - def _get_filesystem_path(self, path: Union[str, PathLike]) -> Path: + def _get_filesystem_path(self, path: str | PathLike[str]) -> Path: path_comps = _to_string(path).split("/") return Path(self.basedir, *path_comps) - def _mkdir(self, dirname: Path, domain: Optional[str] = None): - seen = self.created_directories[domain] if domain else set() + def _mkdir( + self, dirname: Path, domain: MediaPipeline.SpiderInfo | None = None + ) -> None: + seen: set[str] = self.created_directories[domain] if domain else set() if str(dirname) not in seen: if not dirname.exists(): dirname.mkdir(parents=True) @@ -94,15 +161,15 @@ class S3FilesStore: AWS_USE_SSL = None AWS_VERIFY = None - POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings - HEADERS = { + POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler() + HEADERS: ClassVar[dict[str, str]] = { "Cache-Control": "max-age=172800", } - def __init__(self, uri): + def __init__(self, uri: str): if not is_botocore_available(): raise NotConfigured("missing botocore library") - import botocore.session + import botocore.session # noqa: PLC0415 session = botocore.session.get_session() self.s3_client = session.create_client( @@ -119,39 +186,56 @@ class S3FilesStore: raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'") self.bucket, self.prefix = uri[5:].split("/", 1) - def stat_file(self, path, info): - def _onsuccess(boto_key): - checksum = boto_key["ETag"].strip('"') - last_modified = boto_key["LastModified"] - modified_stamp = time.mktime(last_modified.timetuple()) - return {"checksum": checksum, "last_modified": modified_stamp} + @staticmethod + def _onsuccess(boto_key: dict[str, Any]) -> StatInfo: + checksum = boto_key["ETag"].strip('"') + last_modified = boto_key["LastModified"] + modified_stamp = time.mktime(last_modified.timetuple()) + return {"checksum": checksum, "last_modified": modified_stamp} - return self._get_boto_key(path).addCallback(_onsuccess) + def stat_file( + self, path: str, info: MediaPipeline.SpiderInfo + ) -> Deferred[StatInfo]: - def _get_boto_key(self, path): + return self._get_boto_key(path).addCallback(self._onsuccess) + + def _get_boto_key(self, path: str) -> Deferred[dict[str, Any]]: key_name = f"{self.prefix}{path}" - return threads.deferToThread( - self.s3_client.head_object, Bucket=self.bucket, Key=key_name + return deferred_from_coro( + run_in_thread( + self.s3_client.head_object, # type: ignore[attr-defined] + Bucket=self.bucket, + Key=key_name, + ) ) - def persist_file(self, path, buf, info, meta=None, headers=None): + def persist_file( + self, + path: str, + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + ) -> Deferred[Any]: """Upload file to S3 storage""" key_name = f"{self.prefix}{path}" buf.seek(0) extra = self._headers_to_botocore_kwargs(self.HEADERS) if headers: extra.update(self._headers_to_botocore_kwargs(headers)) - return threads.deferToThread( - self.s3_client.put_object, - Bucket=self.bucket, - Key=key_name, - Body=buf, - Metadata={k: str(v) for k, v in (meta or {}).items()}, - ACL=self.POLICY, - **extra, + return deferred_from_coro( + run_in_thread( + self.s3_client.put_object, # type: ignore[attr-defined] + Bucket=self.bucket, + Key=key_name, + Body=buf, + Metadata={k: str(v) for k, v in meta.items()} if meta else {}, + ACL=self.POLICY, + **extra, + ) ) - def _headers_to_botocore_kwargs(self, headers): + def _headers_to_botocore_kwargs(self, headers: dict[str, Any]) -> dict[str, Any]: """Convert headers to botocore keyword arguments.""" # This is required while we need to support both boto and botocore. mapping = CaseInsensitiveDict( @@ -183,14 +267,15 @@ class S3FilesStore: "X-Amz-Website-Redirect-Location": "WebsiteRedirectLocation", } ) - extra = {} + extra: dict[str, Any] = {} for key, value in headers.items(): try: kwarg = mapping[key] except KeyError: - raise TypeError(f'Header "{key}" is not supported by botocore') - else: - extra[kwarg] = value + raise TypeError( + f'Header "{key}" is not supported by botocore' + ) from None + extra[kwarg] = value return extra @@ -200,16 +285,16 @@ class GCSFilesStore: CACHE_CONTROL = "max-age=172800" # The bucket's default object ACL will be applied to the object. - # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. + # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_crawler(). POLICY = None - def __init__(self, uri): - from google.cloud import storage + def __init__(self, uri: str): + from google.cloud import storage # noqa: PLC0415 client = storage.Client(project=self.GCS_PROJECT_ID) bucket, prefix = uri[5:].split("/", 1) self.bucket = client.bucket(bucket) - self.prefix = prefix + self.prefix: str = prefix permissions = self.bucket.test_iam_permissions( ["storage.objects.get", "storage.objects.create"] ) @@ -225,87 +310,116 @@ class GCSFilesStore: {"bucket": bucket}, ) - def stat_file(self, path, info): - def _onsuccess(blob): - if blob: - checksum = base64.b64decode(blob.md5_hash).hex() - last_modified = time.mktime(blob.updated.timetuple()) - return {"checksum": checksum, "last_modified": last_modified} - return {} + @staticmethod + def _onsuccess(blob: Any) -> StatInfo: + if blob: + checksum = base64.b64decode(blob.md5_hash).hex() + last_modified = time.mktime(blob.updated.timetuple()) + return {"checksum": checksum, "last_modified": last_modified} + return {} + + def stat_file( + self, path: str, info: MediaPipeline.SpiderInfo + ) -> Deferred[StatInfo]: blob_path = self._get_blob_path(path) - return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback( - _onsuccess + d: Deferred[Any] = deferred_from_coro( + run_in_thread(self.bucket.get_blob, blob_path) ) + return d.addCallback(self._onsuccess) - def _get_content_type(self, headers): + def _get_content_type(self, headers: dict[str, str] | None) -> str: if headers and "Content-Type" in headers: return headers["Content-Type"] return "application/octet-stream" - def _get_blob_path(self, path): + def _get_blob_path(self, path: str) -> str: return self.prefix + path - def persist_file(self, path, buf, info, meta=None, headers=None): + def persist_file( + self, + path: str, + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + ) -> Deferred[Any]: blob_path = self._get_blob_path(path) blob = self.bucket.blob(blob_path) blob.cache_control = self.CACHE_CONTROL - blob.metadata = {k: str(v) for k, v in (meta or {}).items()} - return threads.deferToThread( - blob.upload_from_string, - data=buf.getvalue(), - content_type=self._get_content_type(headers), - predefined_acl=self.POLICY, + blob.metadata = {k: str(v) for k, v in meta.items()} if meta else {} + return deferred_from_coro( + run_in_thread( + blob.upload_from_string, + data=buf.getvalue(), + content_type=self._get_content_type(headers), + predefined_acl=self.POLICY, + ) ) class FTPFilesStore: - FTP_USERNAME = None - FTP_PASSWORD = None - USE_ACTIVE_MODE = None + FTP_USERNAME: str | None = None + FTP_PASSWORD: str | None = None + USE_ACTIVE_MODE: bool | None = None - def __init__(self, uri): + def __init__(self, uri: str): if not uri.startswith("ftp://"): raise ValueError(f"Incorrect URI scheme in {uri}, expected 'ftp'") u = urlparse(uri) - self.port = u.port - self.host = u.hostname + assert u.port + assert u.hostname + self.port: int = u.port + self.host: str = u.hostname self.port = int(u.port or 21) - self.username = u.username or self.FTP_USERNAME - self.password = u.password or self.FTP_PASSWORD - self.basedir = u.path.rstrip("/") + assert self.FTP_USERNAME + assert self.FTP_PASSWORD + self.username: str = u.username or self.FTP_USERNAME + self.password: str = u.password or self.FTP_PASSWORD + self.basedir: str = u.path.rstrip("/") - def persist_file(self, path, buf, info, meta=None, headers=None): + def persist_file( + self, + path: str, + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + ) -> Deferred[Any]: path = f"{self.basedir}/{path}" - return threads.deferToThread( - ftp_store_file, - path=path, - file=buf, - host=self.host, - port=self.port, - username=self.username, - password=self.password, - use_active_mode=self.USE_ACTIVE_MODE, + return deferred_from_coro( + run_in_thread( + ftp_store_file, + path=path, + file=buf, + host=self.host, + port=self.port, + username=self.username, + password=self.password, + use_active_mode=bool(self.USE_ACTIVE_MODE), + ) ) - def stat_file(self, path, info): - def _stat_file(path): - try: - ftp = FTP() + def _stat_file(self, path: str) -> StatInfo: + try: + with FTP() as ftp: ftp.connect(self.host, self.port) ftp.login(self.username, self.password) if self.USE_ACTIVE_MODE: ftp.set_pasv(False) file_path = f"{self.basedir}/{path}" last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip()) - m = hashlib.md5() + m = hashlib.md5() # noqa: S324 ftp.retrbinary(f"RETR {file_path}", m.update) - return {"last_modified": last_modified, "checksum": m.hexdigest()} - # The file doesn't exist - except Exception: - return {} + return {"last_modified": last_modified, "checksum": m.hexdigest()} + # The file doesn't exist + except Exception: + return {} - return threads.deferToThread(_stat_file, path) + def stat_file( + self, path: str, info: MediaPipeline.SpiderInfo + ) -> Deferred[StatInfo]: + return deferred_from_coro(run_in_thread(self._stat_file, path)) class FilesPipeline(MediaPipeline): @@ -327,49 +441,76 @@ class FilesPipeline(MediaPipeline): """ - MEDIA_NAME = "file" - EXPIRES = 90 - STORE_SCHEMES = { + MEDIA_NAME: str = "file" + EXPIRES: int = 90 + STORE_SCHEMES: ClassVar[dict[str, type[FilesStoreProtocol]]] = { "": FSFilesStore, "file": FSFilesStore, "s3": S3FilesStore, "gs": GCSFilesStore, "ftp": FTPFilesStore, } - DEFAULT_FILES_URLS_FIELD = "file_urls" - DEFAULT_FILES_RESULT_FIELD = "files" + DEFAULT_FILES_URLS_FIELD: str = "file_urls" + DEFAULT_FILES_RESULT_FIELD: str = "files" def __init__( - self, store_uri: Union[str, PathLike], download_func=None, settings=None + self, + store_uri: str | PathLike[str], + download_func: None = None, + *, + crawler: Crawler, ): - store_uri = _to_string(store_uri) - if not store_uri: - raise NotConfigured + if download_func is not None: # pragma: no cover + warnings.warn( + "The download_func argument of FilesPipeline.__init__() is ignored" + " and will be removed in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) - if isinstance(settings, dict) or settings is None: - settings = Settings(settings) + if not (store_uri and (store_uri := _to_string(store_uri))): + from scrapy.pipelines.images import ImagesPipeline # noqa: PLC0415 + + setting_name = ( + "IMAGES_STORE" if isinstance(self, ImagesPipeline) else "FILES_STORE" + ) + raise NotConfigured( + f"{setting_name} setting must be set to a valid path (not empty) " + f"to enable {self.__class__.__name__}." + ) + + settings = crawler.settings cls_name = "FilesPipeline" - self.store = self._get_store(store_uri) + self.store: FilesStoreProtocol = self._get_store(store_uri) resolve = functools.partial( self._key_for_pipe, base_class_name=cls_name, settings=settings ) - self.expires = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES) + self.expires: int = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES) if not hasattr(self, "FILES_URLS_FIELD"): self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELD if not hasattr(self, "FILES_RESULT_FIELD"): self.FILES_RESULT_FIELD = self.DEFAULT_FILES_RESULT_FIELD - self.files_urls_field = settings.get( + self.files_urls_field: str = settings.get( resolve("FILES_URLS_FIELD"), self.FILES_URLS_FIELD ) - self.files_result_field = settings.get( + self.files_result_field: str = settings.get( resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD ) - super().__init__(download_func=download_func, settings=settings) + super().__init__(crawler=crawler) @classmethod - def from_settings(cls, settings): - s3store = cls.STORE_SCHEMES["s3"] + def from_crawler(cls, crawler: Crawler) -> Self: + settings = crawler.settings + cls._update_stores(settings) + store_uri = settings["FILES_STORE"] + return cls(store_uri, crawler=crawler) + + @classmethod + def _update_stores(cls, settings: BaseSettings) -> None: + s3store: type[S3FilesStore] = cast( + "type[S3FilesStore]", cls.STORE_SCHEMES["s3"] + ) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] @@ -379,70 +520,83 @@ class FilesPipeline(MediaPipeline): s3store.AWS_VERIFY = settings["AWS_VERIFY"] s3store.POLICY = settings["FILES_STORE_S3_ACL"] - gcs_store = cls.STORE_SCHEMES["gs"] + gcs_store: type[GCSFilesStore] = cast( + "type[GCSFilesStore]", cls.STORE_SCHEMES["gs"] + ) gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] gcs_store.POLICY = settings["FILES_STORE_GCS_ACL"] or None - ftp_store = cls.STORE_SCHEMES["ftp"] + ftp_store: type[FTPFilesStore] = cast( + "type[FTPFilesStore]", cls.STORE_SCHEMES["ftp"] + ) ftp_store.FTP_USERNAME = settings["FTP_USER"] ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE") - store_uri = settings["FILES_STORE"] - return cls(store_uri, settings=settings) - - def _get_store(self, uri: str): - if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir - scheme = "file" - else: - scheme = urlparse(uri).scheme + def _get_store(self, uri: str) -> FilesStoreProtocol: + # to support win32 paths like: C:\\some\dir + scheme = "file" if Path(uri).is_absolute() else urlparse(uri).scheme store_cls = self.STORE_SCHEMES[scheme] return store_cls(uri) - def media_to_download(self, request, info, *, item=None): - def _onsuccess(result): - if not result: - return # returning None force download + def _onsuccess( + self, + result: StatInfo, + request: Request, + info: MediaPipeline.SpiderInfo, + path: str, + ) -> FileInfo | None: + if not result: + return None # returning None force download - last_modified = result.get("last_modified", None) - if not last_modified: - return # returning None force download + last_modified = result.get("last_modified", None) + if not last_modified: + return None # returning None force download - age_seconds = time.time() - last_modified - age_days = age_seconds / 60 / 60 / 24 - if age_days > self.expires: - return # returning None force download + age_seconds = time.time() - last_modified + age_days = age_seconds / 60 / 60 / 24 + if age_days > self.expires: + return None # returning None force download - referer = referer_str(request) - logger.debug( - "File (uptodate): Downloaded %(medianame)s from %(request)s " - "referred in <%(referer)s>", - {"medianame": self.MEDIA_NAME, "request": request, "referer": referer}, - extra={"spider": info.spider}, - ) - self.inc_stats(info.spider, "uptodate") + referer = referer_str(request) + logger.debug( + "File (uptodate): Downloaded %(medianame)s from %(request)s " + "referred in <%(referer)s>", + {"medianame": self.MEDIA_NAME, "request": request, "referer": referer}, + extra={"spider": info.spider}, + ) + self.inc_stats("uptodate") - checksum = result.get("checksum", None) - return { - "url": request.url, - "path": path, - "checksum": checksum, - "status": "uptodate", - } + checksum = result.get("checksum", None) + return { + "url": request.url, + "path": path, + "checksum": checksum, + "status": "uptodate", + } + def media_to_download( + self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None + ) -> Deferred[FileInfo | None] | None: path = self.file_path(request, info=info, item=item) - dfd = defer.maybeDeferred(self.store.stat_file, path, info) - dfd.addCallbacks(_onsuccess, lambda _: None) - dfd.addErrback( + # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type + dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[call-overload] + dfd2: Deferred[FileInfo | None] = dfd.addCallback( + functools.partial(self._onsuccess, request=request, info=info, path=path) + ) + dfd2.addErrback(lambda _: None) + dfd2.addErrback( lambda f: logger.error( self.__class__.__name__ + ".store.stat_file", exc_info=failure_to_exc_info(f), extra={"spider": info.spider}, ) ) - return dfd + return dfd2 - def media_failed(self, failure, request, info): + def media_failed( + self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo + ) -> NoReturn: if not isinstance(failure.value, IgnoreRequest): referer = referer_str(request) logger.warning( @@ -459,7 +613,14 @@ class FilesPipeline(MediaPipeline): raise FileException - def media_downloaded(self, response, request, info, *, item=None): + async def media_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> FileInfo: referer = referer_str(request) if response.status != 200: @@ -487,11 +648,13 @@ class FilesPipeline(MediaPipeline): {"status": status, "request": request, "referer": referer}, extra={"spider": info.spider}, ) - self.inc_stats(info.spider, status) + self.inc_stats(status) try: path = self.file_path(request, response=response, info=info, item=item) - checksum = self.file_downloaded(response, request, info, item=item) + checksum: str = await ensure_awaitable( + self.file_downloaded(response, request, info, item=item) + ) except FileException as exc: logger.warning( "File (error): Error processing file from %(request)s " @@ -509,7 +672,7 @@ class FilesPipeline(MediaPipeline): exc_info=True, extra={"spider": info.spider}, ) - raise FileException(str(exc)) + raise FileException(str(exc)) from exc return { "url": request.url, @@ -518,36 +681,77 @@ class FilesPipeline(MediaPipeline): "status": status, } - def inc_stats(self, spider, status): - spider.crawler.stats.inc_value("file_count", spider=spider) - spider.crawler.stats.inc_value(f"file_status_count/{status}", spider=spider) + def inc_stats(self, status: str) -> None: + assert self.crawler.stats + self.crawler.stats.inc_value("file_count") + self.crawler.stats.inc_value(f"file_status_count/{status}") - # Overridable Interface - def get_media_requests(self, item, info): - urls = ItemAdapter(item).get(self.files_urls_field, []) - return [Request(u, callback=NO_CALLBACK) for u in urls] - - def file_downloaded(self, response, request, info, *, item=None): + async def _file_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> str: path = self.file_path(request, response=response, info=info, item=item) buf = BytesIO(response.body) - checksum = md5sum(buf) + checksum = _md5sum(buf) buf.seek(0) - self.store.persist_file(path, buf, info) + await ensure_awaitable(self.store.persist_file(path, buf, info)) return checksum - def item_completed(self, results, item, info): + # Overridable Interface + def get_media_requests( + self, item: Any, info: MediaPipeline.SpiderInfo + ) -> list[Request]: + urls = ItemAdapter(item).get(self.files_urls_field, []) + if not isinstance(urls, list): + raise TypeError( + f"{self.files_urls_field} must be a list of URLs, got {type(urls).__name__}. " + ) + return [Request(u, callback=NO_CALLBACK) for u in urls] + + def file_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> str | Awaitable[str]: + return self._file_downloaded(response, request, info, item=item) + + def item_completed( + self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo + ) -> Any: with suppress(KeyError): ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None, *, item=None): - media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() - media_ext = Path(request.url).suffix + def file_path( + self, + request: Request, + response: Response | None = None, + info: MediaPipeline.SpiderInfo | None = None, + *, + item: Any = None, + ) -> str: + media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324 + + # clean it up and look at the path first + parsed_url = urlparse_cached(request) + media_ext = Path(parsed_url.path).suffix + + # if path has no extension look at the raw URL + if media_ext not in mimetypes.types_map: + media_ext = Path(request.url).suffix + # Handles empty and wild extensions by trying to guess the # mime type then extension or default to empty string otherwise if media_ext not in mimetypes.types_map: media_ext = "" media_type = mimetypes.guess_type(request.url)[0] if media_type: - media_ext = mimetypes.guess_extension(media_type) + media_ext = cast("str", mimetypes.guess_extension(media_type)) return f"full/{media_guid}{media_ext}" diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 02c4b1361..83d04e6ca 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -3,37 +3,36 @@ Images Pipeline See documentation in topics/media-pipeline.rst """ + +from __future__ import annotations + import functools import hashlib import warnings from contextlib import suppress from io import BytesIO -from os import PathLike -from typing import Dict, Tuple, Union +from typing import TYPE_CHECKING, Any, ClassVar from itemadapter import ItemAdapter -from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning -from scrapy.http import Request +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.files import FileException, FilesPipeline +from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum +from scrapy.utils.defer import ensure_awaitable +from scrapy.utils.python import to_bytes -# TODO: from scrapy.pipelines.media import MediaPipeline -from scrapy.settings import Settings -from scrapy.utils.misc import md5sum -from scrapy.utils.python import get_func_args, to_bytes +if TYPE_CHECKING: + from collections.abc import Iterable + from os import PathLike + from PIL import Image -class NoimagesDrop(DropItem): - """Product with no images exception""" + # typing.Self requires Python 3.11 + from typing_extensions import Self - def __init__(self, *args, **kwargs): - warnings.warn( - "The NoimagesDrop class is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - super().__init__(*args, **kwargs) + from scrapy.crawler import Crawler + from scrapy.pipelines.media import FileInfoOrError, MediaPipeline class ImageException(FileException): @@ -43,106 +42,129 @@ class ImageException(FileException): class ImagesPipeline(FilesPipeline): """Abstract pipeline that implement the image thumbnail generation logic""" - MEDIA_NAME = "image" + MEDIA_NAME: str = "image" # Uppercase attributes kept for backward compatibility with code that subclasses # ImagesPipeline. They may be overridden by settings. - MIN_WIDTH = 0 - MIN_HEIGHT = 0 - EXPIRES = 90 - THUMBS: Dict[str, Tuple[int, int]] = {} + MIN_WIDTH: int = 0 + MIN_HEIGHT: int = 0 + EXPIRES: int = 90 + THUMBS: ClassVar[dict[str, tuple[int, int]]] = {} DEFAULT_IMAGES_URLS_FIELD = "image_urls" DEFAULT_IMAGES_RESULT_FIELD = "images" def __init__( - self, store_uri: Union[str, PathLike], download_func=None, settings=None + self, + store_uri: str | PathLike[str], + download_func: None = None, + *, + crawler: Crawler, ): - try: - from PIL import Image - - self._Image = Image - except ImportError: - raise NotConfigured( - "ImagesPipeline requires installing Pillow 4.0.0 or later" + if download_func is not None: # pragma: no cover + warnings.warn( + "The download_func argument of ImagesPipeline.__init__() is ignored" + " and will be removed in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, ) - super().__init__(store_uri, settings=settings, download_func=download_func) + try: + from PIL import Image, ImageOps # noqa: PLC0415 - if isinstance(settings, dict) or settings is None: - settings = Settings(settings) + self._Image = Image + self._ImageOps = ImageOps + except ImportError: + raise NotConfigured( + "ImagesPipeline requires installing Pillow 8.3.2 or later" + ) from None + super().__init__(store_uri, crawler=crawler) + + settings = crawler.settings resolve = functools.partial( self._key_for_pipe, base_class_name="ImagesPipeline", settings=settings, ) - self.expires = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES) + self.expires: int = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES) if not hasattr(self, "IMAGES_RESULT_FIELD"): - self.IMAGES_RESULT_FIELD = self.DEFAULT_IMAGES_RESULT_FIELD + self.IMAGES_RESULT_FIELD: str = self.DEFAULT_IMAGES_RESULT_FIELD if not hasattr(self, "IMAGES_URLS_FIELD"): - self.IMAGES_URLS_FIELD = self.DEFAULT_IMAGES_URLS_FIELD + self.IMAGES_URLS_FIELD: str = self.DEFAULT_IMAGES_URLS_FIELD - self.images_urls_field = settings.get( + self.images_urls_field: str = settings.get( resolve("IMAGES_URLS_FIELD"), self.IMAGES_URLS_FIELD ) - self.images_result_field = settings.get( + self.images_result_field: str = settings.get( resolve("IMAGES_RESULT_FIELD"), self.IMAGES_RESULT_FIELD ) - self.min_width = settings.getint(resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH) - self.min_height = settings.getint(resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT) - self.thumbs = settings.get(resolve("IMAGES_THUMBS"), self.THUMBS) - - self._deprecated_convert_image = None + self.min_width: int = settings.getint( + resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH + ) + self.min_height: int = settings.getint( + resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT + ) + self.thumbs: dict[str, tuple[int, int]] = settings.get( + resolve("IMAGES_THUMBS"), self.THUMBS + ) @classmethod - def from_settings(cls, settings): - s3store = cls.STORE_SCHEMES["s3"] - s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] - s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] - s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] - s3store.AWS_ENDPOINT_URL = settings["AWS_ENDPOINT_URL"] - s3store.AWS_REGION_NAME = settings["AWS_REGION_NAME"] - s3store.AWS_USE_SSL = settings["AWS_USE_SSL"] - s3store.AWS_VERIFY = settings["AWS_VERIFY"] - s3store.POLICY = settings["IMAGES_STORE_S3_ACL"] - - gcs_store = cls.STORE_SCHEMES["gs"] - gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] - gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None - - ftp_store = cls.STORE_SCHEMES["ftp"] - ftp_store.FTP_USERNAME = settings["FTP_USER"] - ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] - ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE") - + def from_crawler(cls, crawler: Crawler) -> Self: + settings = crawler.settings + cls._update_stores(settings) store_uri = settings["IMAGES_STORE"] - return cls(store_uri, settings=settings) + return cls(store_uri, crawler=crawler) - def file_downloaded(self, response, request, info, *, item=None): - return self.image_downloaded(response, request, info, item=item) + async def file_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> str: + return await self.image_downloaded(response, request, info, item=item) - def image_downloaded(self, response, request, info, *, item=None): - checksum = None + async def image_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> str: + checksum: str | None = None for path, image, buf in self.get_images(response, request, info, item=item): if checksum is None: buf.seek(0) - checksum = md5sum(buf) + checksum = _md5sum(buf) width, height = image.size - self.store.persist_file( - path, - buf, - info, - meta={"width": width, "height": height}, - headers={"Content-Type": "image/jpeg"}, + await ensure_awaitable( + self.store.persist_file( + path, + buf, + info, + meta={"width": width, "height": height}, + headers={"Content-Type": "image/jpeg"}, + ) ) + assert checksum is not None return checksum - def get_images(self, response, request, info, *, item=None): + def get_images( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> Iterable[tuple[str, Image.Image, BytesIO]]: path = self.file_path(request, response=response, info=info, item=item) orig_image = self._Image.open(BytesIO(response.body)) + transposed_image = self._ImageOps.exif_transpose(orig_image) - width, height = orig_image.size + width, height = transposed_image.size if width < self.min_width or height < self.min_height: raise ImageException( "Image too small " @@ -150,45 +172,26 @@ class ImagesPipeline(FilesPipeline): f"{self.min_width}x{self.min_height})" ) - if self._deprecated_convert_image is None: - self._deprecated_convert_image = "response_body" not in get_func_args( - self.convert_image - ) - if self._deprecated_convert_image: - warnings.warn( - f"{self.__class__.__name__}.convert_image() method overridden in a deprecated way, " - "overridden method does not accept response_body argument.", - category=ScrapyDeprecationWarning, - ) - - if self._deprecated_convert_image: - image, buf = self.convert_image(orig_image) - else: - image, buf = self.convert_image( - orig_image, response_body=BytesIO(response.body) - ) + image, buf = self.convert_image( + transposed_image, response_body=BytesIO(response.body) + ) yield path, image, buf for thumb_id, size in self.thumbs.items(): thumb_path = self.thumb_path( request, thumb_id, response=response, info=info, item=item ) - if self._deprecated_convert_image: - thumb_image, thumb_buf = self.convert_image(image, size) - else: - thumb_image, thumb_buf = self.convert_image(image, size, buf) + thumb_image, thumb_buf = self.convert_image(image, size, response_body=buf) yield thumb_path, thumb_image, thumb_buf - def convert_image(self, image, size=None, response_body=None): - if response_body is None: - warnings.warn( - f"{self.__class__.__name__}.convert_image() method called in a deprecated way, " - "method called without response_body argument.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - - if image.format in ("PNG", "WEBP") and image.mode == "RGBA": + def convert_image( + self, + image: Image.Image, + size: tuple[int, int] | None = None, + *, + response_body: BytesIO, + ) -> tuple[Image.Image, BytesIO]: + if image.format in {"PNG", "WEBP"} and image.mode == "RGBA": background = self._Image.new("RGBA", image.size, (255, 255, 255)) background.paste(image, image) image = background.convert("RGB") @@ -208,28 +211,51 @@ class ImagesPipeline(FilesPipeline): # when updating the minimum requirements for Pillow. resampling_filter = self._Image.Resampling.LANCZOS except AttributeError: - resampling_filter = self._Image.ANTIALIAS + resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined] image.thumbnail(size, resampling_filter) - elif response_body is not None and image.format == "JPEG": + elif image.format == "JPEG": return image, response_body buf = BytesIO() image.save(buf, "JPEG") return image, buf - def get_media_requests(self, item, info): + def get_media_requests( + self, item: Any, info: MediaPipeline.SpiderInfo + ) -> list[Request]: urls = ItemAdapter(item).get(self.images_urls_field, []) + if not isinstance(urls, list): + raise TypeError( + f"{self.images_urls_field} must be a list of URLs, got {type(urls).__name__}. " + ) return [Request(u, callback=NO_CALLBACK) for u in urls] - def item_completed(self, results, item, info): + def item_completed( + self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo + ) -> Any: with suppress(KeyError): ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None, *, item=None): - image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() + def file_path( + self, + request: Request, + response: Response | None = None, + info: MediaPipeline.SpiderInfo | None = None, + *, + item: Any = None, + ) -> str: + image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324 return f"full/{image_guid}.jpg" - def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None): - thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() + def thumb_path( + self, + request: Request, + thumb_id: str, + response: Response | None = None, + info: MediaPipeline.SpiderInfo | None = None, + *, + item: Any = None, + ) -> str: + thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324 return f"thumbs/{thumb_id}/{thumb_guid}.jpg" diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index fc156ab41..5b5d2dcb2 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,163 +1,238 @@ +from __future__ import annotations + +import asyncio import functools import logging +import warnings +from abc import ABC, abstractmethod from collections import defaultdict +from typing import TYPE_CHECKING, Any, Literal, TypeAlias, TypedDict, cast from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure -from scrapy.http.request import NO_CALLBACK -from scrapy.settings import Settings +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.http.request import NO_CALLBACK, Request +from scrapy.utils._deps_compat import TWISTED_FAILURE_HAS_STACK +from scrapy.utils.asyncio import call_later, is_asyncio_available from scrapy.utils.datatypes import SequenceExclude -from scrapy.utils.defer import defer_result, mustbe_deferred +from scrapy.utils.decorators import _warn_spider_arg +from scrapy.utils.defer import ( + _DEFER_DELAY, + _defer_sleep_async, + deferred_from_coro, + ensure_awaitable, + maybe_deferred_to_future, +) from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter +from scrapy.utils.python import global_object_name + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import Settings + from scrapy.utils.request import RequestFingerprinterProtocol + + +class FileInfo(TypedDict): + url: str + path: str + checksum: str | None + status: str + + +FileInfoOrError: TypeAlias = ( + tuple[Literal[True], FileInfo] | tuple[Literal[False], Failure] +) logger = logging.getLogger(__name__) -def _DUMMY_CALLBACK(response): - return response - - -class MediaPipeline: - LOG_FAILED_RESULTS = True +class MediaPipeline(ABC): + LOG_FAILED_RESULTS: bool = True class SpiderInfo: - def __init__(self, spider): - self.spider = spider - self.downloading = set() - self.downloaded = {} - self.waiting = defaultdict(list) + def __init__(self, spider: Spider): + self.spider: Spider = spider + self.downloading: set[bytes] = set() + self.downloaded: dict[bytes, FileInfo | Failure] = {} + self.waiting: defaultdict[bytes, list[Deferred[FileInfo]]] = defaultdict( + list + ) - def __init__(self, download_func=None, settings=None): - self.download_func = download_func - self._expects_item = {} + def __init__( + self, + download_func: None = None, + *, + crawler: Crawler, + ): + if download_func is not None: # pragma: no cover + warnings.warn( + "The download_func argument of MediaPipeline.__init__() is ignored" + " and will be removed in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self.crawler: Crawler = crawler + assert crawler.request_fingerprinter + self._fingerprinter: RequestFingerprinterProtocol = ( + crawler.request_fingerprinter + ) - if isinstance(settings, dict) or settings is None: - settings = Settings(settings) + settings = crawler.settings resolve = functools.partial( self._key_for_pipe, base_class_name="MediaPipeline", settings=settings ) - self.allow_redirects = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False) + self.allow_redirects: bool = settings.getbool( + resolve("MEDIA_ALLOW_REDIRECTS"), False + ) self._handle_statuses(self.allow_redirects) - def _handle_statuses(self, allow_redirects): + def _handle_statuses(self, allow_redirects: bool) -> None: self.handle_httpstatus_list = None if allow_redirects: self.handle_httpstatus_list = SequenceExclude(range(300, 400)) - def _key_for_pipe(self, key, base_class_name=None, settings=None): - """ - >>> MediaPipeline()._key_for_pipe("IMAGES") - 'IMAGES' - >>> class MyPipe(MediaPipeline): - ... pass - >>> MyPipe()._key_for_pipe("IMAGES", base_class_name="MediaPipeline") - 'MYPIPE_IMAGES' - """ + def _key_for_pipe( + self, + key: str, + base_class_name: str | None = None, + settings: Settings | None = None, + ) -> str: class_name = self.__class__.__name__ formatted_key = f"{class_name.upper()}_{key}" if ( not base_class_name or class_name == base_class_name - or settings - and not settings.get(formatted_key) + or (settings and not settings.get(formatted_key)) ): return key return formatted_key @classmethod - def from_crawler(cls, crawler): - try: - pipe = cls.from_settings(crawler.settings) - except AttributeError: - pipe = cls() - pipe.crawler = crawler - pipe._fingerprinter = crawler.request_fingerprinter - return pipe + def from_crawler(cls, crawler: Crawler) -> Self: + return cls(crawler=crawler) - def open_spider(self, spider): - self.spiderinfo = self.SpiderInfo(spider) + @_warn_spider_arg + def open_spider(self, spider: Spider | None = None) -> None: + assert self.crawler.spider + self.spiderinfo = self.SpiderInfo(self.crawler.spider) - def process_item(self, item, spider): + @_warn_spider_arg + async def process_item(self, item: Any, spider: Spider | None = None) -> Any: info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) - dlist = [self._process_request(r, info, item) for r in requests] - dfd = DeferredList(dlist, consumeErrors=True) - return dfd.addCallback(self.item_completed, item, info) + coros = [self._process_request(r, info, item) for r in requests] + results: list[FileInfoOrError] = [] + if coros: + if is_asyncio_available(): + results_asyncio = await asyncio.gather(*coros, return_exceptions=True) + for res in results_asyncio: + if isinstance(res, BaseException): + results.append((False, Failure(res))) + else: + results.append((True, res)) + else: + results = await cast( + "Deferred[list[FileInfoOrError]]", + DeferredList( + (deferred_from_coro(coro) for coro in coros), consumeErrors=True + ), + ) + return self.item_completed(results, item, info) - def _process_request(self, request, info, item): + async def _process_request( + self, request: Request, info: SpiderInfo, item: Any + ) -> FileInfo: fp = self._fingerprinter.fingerprint(request) - if not request.callback or request.callback is NO_CALLBACK: - cb = _DUMMY_CALLBACK - else: - cb = request.callback - eb = request.errback + + eb: Callable[[Failure], FileInfo] | None = request.errback request.callback = NO_CALLBACK request.errback = None # Return cached result if request was already seen if fp in info.downloaded: - return defer_result(info.downloaded[fp]).addCallbacks(cb, eb) + await _defer_sleep_async() + cached_result = info.downloaded[fp] + if isinstance(cached_result, Failure): + if eb: + return eb(cached_result) + cached_result.raiseException() + return cached_result # Otherwise, wait for result - wad = Deferred().addCallbacks(cb, eb) + wad: Deferred[FileInfo] = Deferred() + if eb: + wad.addErrback(eb) info.waiting[fp].append(wad) # Check if request is downloading right now to avoid doing it twice if fp in info.downloading: - return wad + return await maybe_deferred_to_future(wad) # Download request checking media_to_download hook output first info.downloading.add(fp) - dfd = mustbe_deferred(self.media_to_download, request, info, item=item) - dfd.addCallback(self._check_media_to_download, request, info, item=item) - dfd.addErrback(self._log_exception) - dfd.addBoth(self._cache_result_and_execute_waiters, fp, info) - return dfd.addBoth(lambda _: wad) # it must return wad at last + await _defer_sleep_async() + result: FileInfo | Failure + try: + file_info: FileInfo | None = await ensure_awaitable( + self.media_to_download(request, info, item=item) + ) + if file_info: + # got a result without downloading + result = file_info + else: + # download the result + result = await self._check_media_to_download(request, info, item=item) + except Exception: + result = Failure() + logger.exception(result) + self._cache_result_and_execute_waiters(result, fp, info) + return await maybe_deferred_to_future(wad) # it must return wad at last - def _log_exception(self, result): - logger.exception(result) - return result - - def _modify_media_request(self, request): + def _modify_media_request(self, request: Request) -> None: if self.handle_httpstatus_list: request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list else: request.meta["handle_httpstatus_all"] = True - def _check_media_to_download(self, result, request, info, item): - if result is not None: - return result - if self.download_func: - # this ugly code was left only to support tests. TODO: remove - dfd = mustbe_deferred(self.download_func, request, info.spider) - dfd.addCallbacks( - callback=self.media_downloaded, - callbackArgs=(request, info), - callbackKeywords={"item": item}, - errback=self.media_failed, - errbackArgs=(request, info), - ) - else: + async def _check_media_to_download( + self, request: Request, info: SpiderInfo, item: Any + ) -> FileInfo: + try: self._modify_media_request(request) - dfd = self.crawler.engine.download(request) - dfd.addCallbacks( - callback=self.media_downloaded, - callbackArgs=(request, info), - callbackKeywords={"item": item}, - errback=self.media_failed, - errbackArgs=(request, info), + assert self.crawler.engine + response = await self.crawler.engine.download_async(request) + return await ensure_awaitable( + self.media_downloaded(response, request, info, item=item) ) - return dfd + except Exception: + failure = self.media_failed(Failure(), request, info) + if isinstance(failure, Failure): + warnings.warn( + f"{global_object_name(self.media_failed)} returned a Failure instance." + f" This is deprecated, please raise an exception instead, e.g. via failure.raiseException().", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + failure.raiseException() - def _cache_result_and_execute_waiters(self, result, fp, info): + def _cache_result_and_execute_waiters( + self, result: FileInfo | Failure, fp: bytes, info: SpiderInfo + ) -> None: if isinstance(result, Failure): # minimize cached information for failure result.cleanFailure() - result.frames = [] - result.stack = None - + result.frames.clear() + if TWISTED_FAILURE_HAS_STACK: + result.stack.clear() # This code fixes a memory leak by avoiding to keep references to # the Request and Response objects on the Media Pipeline cache. # @@ -176,40 +251,59 @@ class MediaPipeline: # To avoid keeping references to the Response and therefore Request # objects on the Media Pipeline cache, we should wipe the context of # the encapsulated exception when it is a StopIteration instance - # - # This problem does not occur in Python 2.7 since we don't have - # Exception Chaining (https://www.python.org/dev/peps/pep-3134/). context = getattr(result.value, "__context__", None) if isinstance(context, StopIteration): - setattr(result.value, "__context__", None) + assert result.value is not None + result.value.__context__ = None info.downloading.remove(fp) info.downloaded[fp] = result # cache result for wad in info.waiting.pop(fp): - defer_result(result).chainDeferred(wad) + if isinstance(result, Failure): + call_later(_DEFER_DELAY, wad.errback, result) + else: + call_later(_DEFER_DELAY, wad.callback, result) # Overridable Interface - def media_to_download(self, request, info, *, item=None): + @abstractmethod + def media_to_download( + self, request: Request, info: SpiderInfo, *, item: Any = None + ) -> Deferred[FileInfo | None] | None: """Check request before starting download""" - pass + raise NotImplementedError - def get_media_requests(self, item, info): + @abstractmethod + def get_media_requests(self, item: Any, info: SpiderInfo) -> list[Request]: """Returns the media requests to download""" - pass + raise NotImplementedError - def media_downloaded(self, response, request, info, *, item=None): + @abstractmethod + def media_downloaded( + self, + response: Response, + request: Request, + info: SpiderInfo, + *, + item: Any = None, + ) -> FileInfo | Awaitable[FileInfo]: """Handler for success downloads""" - return response + raise NotImplementedError - def media_failed(self, failure, request, info): + @abstractmethod + def media_failed( + self, failure: Failure, request: Request, info: SpiderInfo + ) -> Failure: """Handler for failed downloads""" - return failure + raise NotImplementedError - def item_completed(self, results, item, info): + def item_completed( + self, results: list[FileInfoOrError], item: Any, info: SpiderInfo + ) -> Any: """Called per item when all media requests has been processed""" if self.LOG_FAILED_RESULTS: for ok, value in results: if not ok: + assert isinstance(value, Failure) logger.error( "%(class)s found errors processing %(item)s", {"class": self.__class__.__name__, "item": item}, @@ -218,6 +312,14 @@ class MediaPipeline: ) return item - def file_path(self, request, response=None, info=None, *, item=None): + @abstractmethod + def file_path( + self, + request: Request, + response: Response | None = None, + info: SpiderInfo | None = None, + *, + item: Any = None, + ) -> str: """Returns the path where downloaded media should be stored""" - pass + raise NotImplementedError diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index b62d2fe58..0ad0b5d78 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,12 +1,25 @@ +from __future__ import annotations + import hashlib import logging +from typing import TYPE_CHECKING, Protocol, cast from scrapy.utils.misc import build_from_crawler +if TYPE_CHECKING: + from collections.abc import Iterable + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy import Request + from scrapy.core.downloader import Downloader + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) -def _path_safe(text): +def _path_safe(text: str) -> str: """ Return a filesystem-safe version of a string ``text`` @@ -20,22 +33,35 @@ def _path_safe(text): pathable_slot = "".join([c if c.isalnum() or c in "-._" else "_" for c in text]) # as we replace some letters we can get collision for different slots # add we add unique part - unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() - return "-".join([pathable_slot, unique_slot]) + unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # noqa: S324 + return f"{pathable_slot}-{unique_slot}" + + +class QueueProtocol(Protocol): + """Protocol for downstream queues of ``ScrapyPriorityQueue``.""" + + def push(self, request: Request) -> None: ... + + def pop(self) -> Request | None: ... + + def close(self) -> None: ... + + def __len__(self) -> int: ... class ScrapyPriorityQueue: """A priority queue implemented using multiple internal queues (typically, - FIFO queues). It uses one internal queue for each priority value. The internal - queue must implement the following methods: + FIFO queues). It uses one internal queue for each priority value. The + internal queue must implement the following methods: * push(obj) * pop() * close() * __len__() - Optionally, the queue could provide a ``peek`` method, that should return the - next object to be returned by ``pop``, but without removing it from the queue. + Optionally, the queue could provide a ``peek`` method, that should return + the next object to be returned by ``pop``, but without removing it from the + queue. ``__init__`` method of ScrapyPriorityQueue receives a downstream_queue_cls argument, which is a class used to instantiate a new (internal) queue when @@ -48,61 +74,153 @@ class ScrapyPriorityQueue: previously closed leaving some priority buckets non-empty, those priorities should be passed in startprios. + Disk persistence + ================ + + .. warning:: The files that this class generates on disk are an + implementation detail, and may change without a warning in a future + version of Scrapy. Do not rely on the following information for + anything other than debugging purposes. + + When a component instantiates this class with a non-empty *key* argument, + *key* is used as a persistence directory. + + For every request enqueued, this class checks: + + - Whether the request is a :ref:`start request ` or not. + + - The :data:`~scrapy.Request.priority` of the request. + + For each combination of the above seen, this class creates an instance of + *downstream_queue_cls* with *key* set to a subdirectory of the persistence + directory, named as the request priority (e.g. ``1``), with an ``s`` suffix + in case of a start request (e.g. ``1s``). """ @classmethod - def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()): - return cls(crawler, downstream_queue_cls, key, startprios) + def from_crawler( + cls, + crawler: Crawler, + downstream_queue_cls: type[QueueProtocol], + key: str, + startprios: Iterable[int] = (), + *, + start_queue_cls: type[QueueProtocol] | None = None, + ) -> Self: + return cls( + crawler, + downstream_queue_cls, + key, + startprios, + start_queue_cls=start_queue_cls, + ) - def __init__(self, crawler, downstream_queue_cls, key, startprios=()): - self.crawler = crawler - self.downstream_queue_cls = downstream_queue_cls - self.key = key - self.queues = {} - self.curprio = None + def __init__( + self, + crawler: Crawler, + downstream_queue_cls: type[QueueProtocol], + key: str, + startprios: Iterable[int] = (), + *, + start_queue_cls: type[QueueProtocol] | None = None, + ): + self.crawler: Crawler = crawler + self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls + self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls + self.key: str = key + self.queues: dict[int, QueueProtocol] = {} + self._start_queues: dict[int, QueueProtocol] = {} + self.curprio: int | None = None self.init_prios(startprios) - def init_prios(self, startprios): + def init_prios(self, startprios: Iterable[int]) -> None: if not startprios: return for priority in startprios: - self.queues[priority] = self.qfactory(priority) + q = self.qfactory(priority) + if q: + self.queues[priority] = q + if self._start_queue_cls: + q = self._sqfactory(priority) + if q: + self._start_queues[priority] = q self.curprio = min(startprios) - def qfactory(self, key): + def qfactory(self, key: int) -> QueueProtocol: return build_from_crawler( self.downstream_queue_cls, self.crawler, self.key + "/" + str(key), ) - def priority(self, request): + def _sqfactory(self, key: int) -> QueueProtocol: + assert self._start_queue_cls is not None + return build_from_crawler( + self._start_queue_cls, + self.crawler, + f"{self.key}/{key}s", + ) + + def priority(self, request: Request) -> int: return -request.priority - def push(self, request): + def push(self, request: Request) -> None: priority = self.priority(request) - if priority not in self.queues: - self.queues[priority] = self.qfactory(priority) - q = self.queues[priority] + is_start_request = request.meta.get("is_start_request", False) + if is_start_request and self._start_queue_cls: + if priority not in self._start_queues: + self._start_queues[priority] = self._sqfactory(priority) + q = self._start_queues[priority] + else: + if priority not in self.queues: + self.queues[priority] = self.qfactory(priority) + q = self.queues[priority] q.push(request) # this may fail (eg. serialization error) if self.curprio is None or priority < self.curprio: self.curprio = priority - def pop(self): - if self.curprio is None: - return - q = self.queues[self.curprio] - m = q.pop() - if not q: - del self.queues[self.curprio] - q.close() - prios = [p for p, q in self.queues.items() if q] - self.curprio = min(prios) if prios else None - return m + def pop(self) -> Request | None: + while self.curprio is not None: + try: + q = self.queues[self.curprio] + except KeyError: + pass + else: + m = q.pop() + if not q: + del self.queues[self.curprio] + q.close() + if not self._start_queues: + self._update_curprio() + return m + if self._start_queues: + try: + q = self._start_queues[self.curprio] + except KeyError: + self._update_curprio() + else: + m = q.pop() + if not q: + del self._start_queues[self.curprio] + q.close() + self._update_curprio() + return m + else: + self._update_curprio() + return None - def peek(self): + def _update_curprio(self) -> None: + prios = { + p + for queues in (self.queues, self._start_queues) + for p, q in queues.items() + if q + } + self.curprio = min(prios) if prios else None + + def peek(self) -> Request | None: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -111,31 +229,45 @@ class ScrapyPriorityQueue: """ if self.curprio is None: return None - queue = self.queues[self.curprio] - return queue.peek() + try: + queue = self._start_queues[self.curprio] + except KeyError: + queue = self.queues[self.curprio] + # Protocols can't declare optional members + return cast("Request", queue.peek()) # type: ignore[attr-defined] - def close(self): - active = [] - for p, q in self.queues.items(): - active.append(p) - q.close() - return active + def close(self) -> list[int]: + active: set[int] = set() + for queues in (self.queues, self._start_queues): + for p, q in queues.items(): + active.add(p) + q.close() + return list(active) - def __len__(self): - return sum(len(x) for x in self.queues.values()) if self.queues else 0 + def __len__(self) -> int: + return ( + sum( + len(x) + for queues in (self.queues, self._start_queues) + for x in queues.values() + ) + if self.queues or self._start_queues + else 0 + ) class DownloaderInterface: - def __init__(self, crawler): - self.downloader = crawler.engine.downloader + def __init__(self, crawler: Crawler): + assert crawler.engine + self.downloader: Downloader = crawler.engine.downloader - def stats(self, possible_slots): + def stats(self, possible_slots: Iterable[str]) -> list[tuple[int, str]]: return [(self._active_downloads(slot), slot) for slot in possible_slots] - def get_slot_key(self, request): - return self.downloader._get_slot_key(request, None) + def get_slot_key(self, request: Request) -> str: + return self.downloader.get_slot_key(request) - def _active_downloads(self, slot): + def _active_downloads(self, slot: str) -> int: """Return a number of requests in a Downloader for a given slot""" if slot not in self.downloader.slots: return 0 @@ -146,13 +278,55 @@ class DownloaderAwarePriorityQueue: """PriorityQueue which takes Downloader activity into account: domains (slots) with the least amount of active downloads are dequeued first. + + Disk persistence + ================ + + .. warning:: The files that this class generates on disk are an + implementation detail, and may change without a warning in a future + version of Scrapy. Do not rely on the following information for + anything other than debugging purposes. + + When a component instantiates this class with a non-empty *key* argument, + *key* is used as a persistence directory, and inside that directory this + class creates a subdirectory per download slot (domain). + + Those subdirectories are named after the corresponding download slot, with + path-unsafe characters replaced by underscores and an MD5 hash suffix to + avoid collisions. + + For each download slot, this class creates an instance of + :class:`ScrapyPriorityQueue` with the download slot subdirectory as *key* + and its own *downstream_queue_cls*. """ @classmethod - def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()): - return cls(crawler, downstream_queue_cls, key, startprios) + def from_crawler( + cls, + crawler: Crawler, + downstream_queue_cls: type[QueueProtocol], + key: str, + startprios: dict[str, Iterable[int]] | None = None, + *, + start_queue_cls: type[QueueProtocol] | None = None, + ) -> Self: + return cls( + crawler, + downstream_queue_cls, + key, + startprios, + start_queue_cls=start_queue_cls, + ) - def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()): + def __init__( + self, + crawler: Crawler, + downstream_queue_cls: type[QueueProtocol], + key: str, + slot_startprios: dict[str, Iterable[int]] | None = None, + *, + start_queue_cls: type[QueueProtocol] | None = None, + ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP' @@ -163,50 +337,83 @@ class DownloaderAwarePriorityQueue: "DownloaderAwarePriorityQueue accepts " "``slot_startprios`` as a dict; " f"{slot_startprios.__class__!r} instance " - "is passed. Most likely, it means the state is" + "is passed. Most likely, it means the state is " "created by an incompatible priority queue. " "Only a crawl started with the same priority " "queue class can be resumed." ) - self._downloader_interface = DownloaderInterface(crawler) - self.downstream_queue_cls = downstream_queue_cls - self.key = key - self.crawler = crawler + self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler) + self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls + self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls + self.key: str = key + self.crawler: Crawler = crawler - self.pqueues = {} # slot -> priority queue - for slot, startprios in (slot_startprios or {}).items(): - self.pqueues[slot] = self.pqfactory(slot, startprios) + self.pqueues: dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue + self._last_selected_slot: str | None = None + if slot_startprios: + for slot, startprios in slot_startprios.items(): + self.pqueues[slot] = self.pqfactory(slot, startprios) - def pqfactory(self, slot, startprios=()): + def _next_slot(self, stats: list[tuple[int, str]], *, update_state: bool) -> str: + last = self._last_selected_slot + min_active: int | None = None + best_slot: str | None = None + best_slot_after_last: str | None = None + for active, slot in stats: + if min_active is None or active < min_active: + min_active = active + best_slot = slot + best_slot_after_last = None + if last is not None and slot > last: + best_slot_after_last = slot + elif active == min_active: + if best_slot is None or slot < best_slot: + best_slot = slot + if ( + last is not None + and slot > last + and (best_slot_after_last is None or slot < best_slot_after_last) + ): + best_slot_after_last = slot + assert best_slot is not None + slot = best_slot_after_last if best_slot_after_last is not None else best_slot + if update_state: + self._last_selected_slot = slot + return slot + + def pqfactory( + self, slot: str, startprios: Iterable[int] = () + ) -> ScrapyPriorityQueue: return ScrapyPriorityQueue( self.crawler, self.downstream_queue_cls, self.key + "/" + _path_safe(slot), startprios, + start_queue_cls=self._start_queue_cls, ) - def pop(self): + def pop(self) -> Request | None: stats = self._downloader_interface.stats(self.pqueues) if not stats: - return + return None - slot = min(stats)[1] + slot = self._next_slot(stats, update_state=True) queue = self.pqueues[slot] request = queue.pop() if len(queue) == 0: del self.pqueues[slot] return request - def push(self, request): + def push(self, request: Request) -> None: slot = self._downloader_interface.get_slot_key(request) if slot not in self.pqueues: self.pqueues[slot] = self.pqfactory(slot) queue = self.pqueues[slot] queue.push(request) - def peek(self): + def peek(self) -> Request | None: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -216,17 +423,17 @@ class DownloaderAwarePriorityQueue: stats = self._downloader_interface.stats(self.pqueues) if not stats: return None - slot = min(stats)[1] + slot = self._next_slot(stats, update_state=False) queue = self.pqueues[slot] return queue.peek() - def close(self): + def close(self) -> dict[str, list[int]]: active = {slot: queue.close() for slot, queue in self.pqueues.items()} self.pqueues.clear() return active - def __len__(self): + def __len__(self) -> int: return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 - def __contains__(self, slot): + def __contains__(self, slot: str) -> bool: return slot in self.pqueues diff --git a/scrapy/resolver.py b/scrapy/resolver.py index e2e8beff4..f5f00ab0f 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,8 +1,11 @@ -from typing import Any +from __future__ import annotations + +from typing import TYPE_CHECKING, Any from twisted.internet import defer -from twisted.internet.base import ThreadedResolver +from twisted.internet.base import ReactorBase, ThreadedResolver from twisted.internet.interfaces import ( + IAddress, IHostnameResolver, IHostResolution, IResolutionReceiver, @@ -12,6 +15,16 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache +if TYPE_CHECKING: + from collections.abc import Sequence + + from twisted.internet.defer import Deferred + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + # TODO: cache misses dnscache: LocalCache[str, Any] = LocalCache(10000) @@ -22,65 +35,66 @@ class CachingThreadedResolver(ThreadedResolver): Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests. """ - def __init__(self, reactor, cache_size, timeout): + def __init__(self, reactor: ReactorBase, cache_size: int, timeout: float): super().__init__(reactor) dnscache.limit = cache_size self.timeout = timeout @classmethod - def from_crawler(cls, crawler, reactor): + def from_crawler(cls, crawler: Crawler, reactor: ReactorBase) -> Self: if crawler.settings.getbool("DNSCACHE_ENABLED"): cache_size = crawler.settings.getint("DNSCACHE_SIZE") else: cache_size = 0 return cls(reactor, cache_size, crawler.settings.getfloat("DNS_TIMEOUT")) - def install_on_reactor(self): + def install_on_reactor(self) -> None: self.reactor.installResolver(self) - def getHostByName(self, name: str, timeout=None): + def getHostByName(self, name: str, timeout: Sequence[int] = ()) -> Deferred[str]: if name in dnscache: return defer.succeed(dnscache[name]) # in Twisted<=16.6, getHostByName() is always called with # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple), # so the input argument above is simply overridden # to enforce Scrapy's DNS_TIMEOUT setting's value - timeout = (self.timeout,) + # The timeout arg is typed as Sequence[int] but supports floats. + timeout = (self.timeout,) # type: ignore[assignment] d = super().getHostByName(name, timeout) if dnscache.limit: d.addCallback(self._cache_result, name) return d - def _cache_result(self, result, name): + def _cache_result(self, result: Any, name: str) -> Any: dnscache[name] = result return result @implementer(IHostResolution) class HostResolution: - def __init__(self, name): - self.name = name + def __init__(self, name: str): + self.name: str = name - def cancel(self): - raise NotImplementedError() + def cancel(self) -> None: + raise NotImplementedError @provider(IResolutionReceiver) class _CachingResolutionReceiver: - def __init__(self, resolutionReceiver, hostName): - self.resolutionReceiver = resolutionReceiver - self.hostName = hostName - self.addresses = [] + def __init__(self, resolutionReceiver: IResolutionReceiver, hostName: str): + self.resolutionReceiver: IResolutionReceiver = resolutionReceiver + self.hostName: str = hostName + self.addresses: list[IAddress] = [] - def resolutionBegan(self, resolution): + def resolutionBegan(self, resolution: IHostResolution) -> None: self.resolutionReceiver.resolutionBegan(resolution) self.resolution = resolution - def addressResolved(self, address): + def addressResolved(self, address: IAddress) -> None: self.resolutionReceiver.addressResolved(address) self.addresses.append(address) - def resolutionComplete(self): + def resolutionComplete(self) -> None: self.resolutionReceiver.resolutionComplete() if self.addresses: dnscache[self.hostName] = self.addresses @@ -93,30 +107,30 @@ class CachingHostnameResolver: does not support setting a timeout value for DNS requests. """ - def __init__(self, reactor, cache_size): - self.reactor = reactor - self.original_resolver = reactor.nameResolver + def __init__(self, reactor: ReactorBase, cache_size: int): + self.reactor: ReactorBase = reactor + self.original_resolver: IHostnameResolver = reactor.nameResolver dnscache.limit = cache_size @classmethod - def from_crawler(cls, crawler, reactor): + def from_crawler(cls, crawler: Crawler, reactor: ReactorBase) -> Self: if crawler.settings.getbool("DNSCACHE_ENABLED"): cache_size = crawler.settings.getint("DNSCACHE_SIZE") else: cache_size = 0 return cls(reactor, cache_size) - def install_on_reactor(self): + def install_on_reactor(self) -> None: self.reactor.installNameResolver(self) def resolveHostName( self, - resolutionReceiver, + resolutionReceiver: IResolutionReceiver, hostName: str, - portNumber=0, - addressTypes=None, - transportSemantics="TCP", - ): + portNumber: int = 0, + addressTypes: Sequence[type[IAddress]] | None = None, + transportSemantics: str = "TCP", + ) -> IHostResolution: try: addresses = dnscache[hostName] except KeyError: @@ -127,9 +141,8 @@ class CachingHostnameResolver: addressTypes, transportSemantics, ) - else: - resolutionReceiver.resolutionBegan(HostResolution(hostName)) - for addr in addresses: - resolutionReceiver.addressResolved(addr) - resolutionReceiver.resolutionComplete() - return resolutionReceiver + resolutionReceiver.resolutionBegan(HostResolution(hostName)) + for addr in addresses: + resolutionReceiver.addressResolved(addr) + resolutionReceiver.resolutionComplete() + return resolutionReceiver diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 08696c584..66cc53b17 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -2,7 +2,10 @@ This module implements a class which returns the appropriate Response class based on different criteria. """ -from typing import Mapping, Optional, Type, Union + +from __future__ import annotations + +from typing import TYPE_CHECKING, ClassVar from warnings import warn from scrapy.exceptions import ScrapyDeprecationWarning @@ -20,9 +23,13 @@ warn( stacklevel=2, ) +if TYPE_CHECKING: + from collections.abc import Mapping + from mimetypes import MimeTypes + class ResponseTypes: - CLASSES = { + CLASSES: ClassVar[dict[str, str]] = { "text/html": "scrapy.http.HtmlResponse", "application/atom+xml": "scrapy.http.XmlResponse", "application/rdf+xml": "scrapy.http.XmlResponse", @@ -39,24 +46,24 @@ class ResponseTypes: "text/*": "scrapy.http.TextResponse", } - def __init__(self): - self.classes = {} - self.mimetypes = _MIME_TYPES + def __init__(self) -> None: + self.classes: dict[str, type[Response]] = {} + self.mimetypes: MimeTypes = _MIME_TYPES for mimetype, cls in self.CLASSES.items(): self.classes[mimetype] = load_object(cls) - def from_mimetype(self, mimetype: str) -> Type[Response]: + def from_mimetype(self, mimetype: str) -> type[Response]: """Return the most appropriate Response class for the given mimetype""" if mimetype is None: return Response if mimetype in self.classes: return self.classes[mimetype] - basetype = f"{mimetype.split('/')[0]}/*" + basetype = f"{mimetype.split('/', maxsplit=1)[0]}/*" return self.classes.get(basetype, Response) def from_content_type( - self, content_type: Union[str, bytes], content_encoding: Optional[bytes] = None - ) -> Type[Response]: + self, content_type: str | bytes, content_encoding: bytes | None = None + ) -> type[Response]: """Return the most appropriate Response class from an HTTP Content-Type header""" if content_encoding: @@ -67,8 +74,8 @@ class ResponseTypes: return self.from_mimetype(mimetype) def from_content_disposition( - self, content_disposition: Union[str, bytes] - ) -> Type[Response]: + self, content_disposition: str | bytes + ) -> type[Response]: try: filename = ( to_unicode(content_disposition, encoding="latin-1", errors="replace") @@ -80,7 +87,7 @@ class ResponseTypes: except IndexError: return Response - def from_headers(self, headers: Mapping[bytes, bytes]) -> Type[Response]: + def from_headers(self, headers: Mapping[bytes, bytes]) -> type[Response]: """Return the most appropriate Response class by looking at the HTTP headers""" cls = Response @@ -93,14 +100,14 @@ class ResponseTypes: cls = self.from_content_disposition(headers[b"Content-Disposition"]) return cls - def from_filename(self, filename: str) -> Type[Response]: + def from_filename(self, filename: str) -> type[Response]: """Return the most appropriate Response class from a file name""" mimetype, encoding = self.mimetypes.guess_type(filename) if mimetype and not encoding: return self.from_mimetype(mimetype) return Response - def from_body(self, body: bytes) -> Type[Response]: + def from_body(self, body: bytes) -> type[Response]: """Try to guess the appropriate response based on the body content. This method is a bit magic and could be improved in the future, but it's not meant to be used except for special cases where response types @@ -120,11 +127,11 @@ class ResponseTypes: def from_args( self, - headers: Optional[Mapping[bytes, bytes]] = None, - url: Optional[str] = None, - filename: Optional[str] = None, - body: Optional[bytes] = None, - ) -> Type[Response]: + headers: Mapping[bytes, bytes] | None = None, + url: str | None = None, + filename: str | None = None, + body: bytes | None = None, + ) -> type[Response]: """Guess the most appropriate Response class based on the given arguments.""" cls = Response diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 6ea2bfd97..0c64ea5a5 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -3,27 +3,32 @@ from __future__ import annotations import logging import sys from abc import ABCMeta, abstractmethod -from typing import TYPE_CHECKING, Union -from warnings import warn +from typing import TYPE_CHECKING, cast +from urllib.robotparser import RobotFileParser + +from protego import Protego -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import to_unicode if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) -def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): +def decode_robotstxt( + robotstxt_body: bytes, spider: Spider | None, to_native_str_type: bool = False +) -> str: try: if to_native_str_type: - robotstxt_body = to_unicode(robotstxt_body) + body_decoded = to_unicode(robotstxt_body) else: - robotstxt_body = robotstxt_body.decode("utf-8") + body_decoded = robotstxt_body.decode("utf-8-sig", errors="ignore") except UnicodeDecodeError: # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it. # Switch to 'allow all' state. @@ -33,8 +38,8 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): exc_info=sys.exc_info(), extra={"spider": spider}, ) - robotstxt_body = "" - return robotstxt_body + body_decoded = "" + return body_decoded class RobotParser(metaclass=ABCMeta): @@ -50,10 +55,9 @@ class RobotParser(metaclass=ABCMeta): :param robotstxt_body: content of a robots.txt_ file. :type robotstxt_body: bytes """ - pass @abstractmethod - def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: """Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``. :param url: Absolute URL @@ -62,86 +66,58 @@ class RobotParser(metaclass=ABCMeta): :param user_agent: User agent :type user_agent: str or bytes """ - pass class PythonRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): - from urllib.robotparser import RobotFileParser - - self.spider = spider - robotstxt_body = decode_robotstxt( - robotstxt_body, spider, to_native_str_type=True - ) - self.rp = RobotFileParser() - self.rp.parse(robotstxt_body.splitlines()) + def __init__(self, robotstxt_body: bytes, spider: Spider | None): + self.spider: Spider | None = spider + body_decoded = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True) + self.rp: RobotFileParser = RobotFileParser() + self.rp.parse(body_decoded.splitlines()) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider - o = cls(robotstxt_body, spider) - return o + return cls(robotstxt_body, spider) - def allowed(self, url, user_agent): + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(user_agent, url) -class ReppyRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): - warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2) - from reppy.robots import Robots - - self.spider = spider - self.rp = Robots.parse("", robotstxt_body) - - @classmethod - def from_crawler(cls, crawler, robotstxt_body): - spider = None if not crawler else crawler.spider - o = cls(robotstxt_body, spider) - return o - - def allowed(self, url, user_agent): - return self.rp.allowed(url, user_agent) - - class RerpRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): - from robotexclusionrulesparser import RobotExclusionRulesParser + def __init__(self, robotstxt_body: bytes, spider: Spider | None): + from robotexclusionrulesparser import RobotExclusionRulesParser # noqa: PLC0415 - self.spider = spider - self.rp = RobotExclusionRulesParser() - robotstxt_body = decode_robotstxt(robotstxt_body, spider) - self.rp.parse(robotstxt_body) + self.spider: Spider | None = spider + self.rp: RobotExclusionRulesParser = RobotExclusionRulesParser() + body_decoded = decode_robotstxt(robotstxt_body, spider) + self.rp.parse(body_decoded) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider - o = cls(robotstxt_body, spider) - return o + return cls(robotstxt_body, spider) - def allowed(self, url, user_agent): + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) - return self.rp.is_allowed(user_agent, url) + return cast("bool", self.rp.is_allowed(user_agent, url)) class ProtegoRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): - from protego import Protego - - self.spider = spider - robotstxt_body = decode_robotstxt(robotstxt_body, spider) - self.rp = Protego.parse(robotstxt_body) + def __init__(self, robotstxt_body: bytes, spider: Spider | None): + self.spider: Spider | None = spider + body_decoded = decode_robotstxt(robotstxt_body, spider) + self.rp = Protego.parse(body_decoded) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider - o = cls(robotstxt_body, spider) - return o + return cls(robotstxt_body, spider) - def allowed(self, url, user_agent): + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(url, user_agent) diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index 85c500d66..7cfa3c364 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -4,3 +4,8 @@ Selectors # top-level imports from scrapy.selector.unified import Selector, SelectorList + +__all__ = [ + "Selector", + "SelectorList", +] diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 863fb6032..99b22aca9 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -1,12 +1,16 @@ """ XPath selectors based on lxml """ -from typing import Any, Optional, Type, Union + +from __future__ import annotations + +from typing import Any from parsel import Selector as _ParselSelector from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.utils.python import to_bytes +from scrapy.utils.response import get_base_url from scrapy.utils.trackref import object_ref __all__ = ["Selector", "SelectorList"] @@ -14,14 +18,14 @@ __all__ = ["Selector", "SelectorList"] _NOT_SET = object() -def _st(response: Optional[TextResponse], st: Optional[str]) -> str: +def _st(response: TextResponse | None, st: str | None) -> str: if st is None: return "xml" if isinstance(response, XmlResponse) else "html" return st -def _response_from_text(text: Union[str, bytes], st: Optional[str]) -> TextResponse: - rt: Type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse +def _response_from_text(text: str | bytes, st: str | None) -> TextResponse: + rt: type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8")) @@ -45,7 +49,7 @@ class Selector(_ParselSelector, object_ref): ``response`` isn't available. Using ``text`` and ``response`` together is undefined behavior. - ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` + ``type`` defines the selector type, it can be ``"html"``, ``"xml"``, ``"json"`` or ``None`` (default). If ``type`` is ``None``, the selector automatically chooses the best type @@ -57,6 +61,7 @@ class Selector(_ParselSelector, object_ref): * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type + * ``"json"`` for :class:`~scrapy.http.TextResponse` type * ``"html"`` for anything else Otherwise, if ``type`` is set, the selector type will be forced and no @@ -68,16 +73,15 @@ class Selector(_ParselSelector, object_ref): def __init__( self, - response: Optional[TextResponse] = None, - text: Optional[str] = None, - type: Optional[str] = None, - root: Optional[Any] = _NOT_SET, + response: TextResponse | None = None, + text: str | None = None, + type: str | None = None, # noqa: A002 + root: Any | None = _NOT_SET, **kwargs: Any, ): if response is not None and text is not None: raise ValueError( - f"{self.__class__.__name__}.__init__() received " - "both response and text" + f"{self.__class__.__name__}.__init__() received both response and text" ) st = _st(response, type) @@ -87,7 +91,7 @@ class Selector(_ParselSelector, object_ref): if response is not None: text = response.text - kwargs.setdefault("base_url", response.url) + kwargs.setdefault("base_url", get_base_url(response)) self.response = response diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b5d8fdb12..d4671fd35 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -2,41 +2,37 @@ from __future__ import annotations import copy import json +import warnings +from collections.abc import Iterable, Iterator, Mapping, MutableMapping from importlib import import_module +from logging import getLogger from pprint import pformat -from types import ModuleType -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterable, - Iterator, - List, - Mapping, - MutableMapping, - Optional, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, TypeAlias, cast +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import default_settings +from scrapy.utils.misc import load_object +from scrapy.utils.python import global_object_name + +logger = getLogger(__name__) # The key types are restricted in BaseSettings._get_key() to ones supported by JSON, # see https://github.com/scrapy/scrapy/issues/5383. -_SettingsKeyT = Union[bool, float, int, str, None] +_SettingsKey: TypeAlias = bool | float | int | str | None if TYPE_CHECKING: + from types import ModuleType + # https://github.com/python/typing/issues/445#issuecomment-1131458824 from _typeshed import SupportsItems # typing.Self requires Python 3.11 from typing_extensions import Self - _SettingsInputT = Union[SupportsItems[_SettingsKeyT, Any], str, None] + _SettingsInput: TypeAlias = SupportsItems[_SettingsKey, Any] | str | None -SETTINGS_PRIORITIES: Dict[str, int] = { +SETTINGS_PRIORITIES: dict[str, int] = { "default": 0, "command": 10, "addon": 15, @@ -46,7 +42,7 @@ SETTINGS_PRIORITIES: Dict[str, int] = { } -def get_settings_priority(priority: Union[int, str]) -> int: +def get_settings_priority(priority: int | str) -> int: """ Small helper function that looks up a given string priority in the :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its @@ -58,7 +54,6 @@ def get_settings_priority(priority: Union[int, str]) -> int: class SettingsAttribute: - """Class for storing data related to settings attributes. This class is intended for internal usage, you should try Settings class @@ -85,7 +80,7 @@ class SettingsAttribute: return f"" -class BaseSettings(MutableMapping[_SettingsKeyT, Any]): +class BaseSettings(MutableMapping[_SettingsKey, Any]): """ Instances of this class behave like dictionaries, but store priorities along with their ``(key, value)`` pairs, and can be frozen (i.e. marked @@ -109,15 +104,13 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): __default = object() - def __init__( - self, values: _SettingsInputT = None, priority: Union[int, str] = "project" - ): + def __init__(self, values: _SettingsInput = None, priority: int | str = "project"): self.frozen: bool = False - self.attributes: dict[_SettingsKeyT, SettingsAttribute] = {} + self.attributes: dict[_SettingsKey, SettingsAttribute] = {} if values: self.update(values, priority) - def __getitem__(self, opt_name: _SettingsKeyT) -> Any: + def __getitem__(self, opt_name: _SettingsKey) -> Any: if opt_name not in self: return None return self.attributes[opt_name].value @@ -125,7 +118,32 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def __contains__(self, name: Any) -> bool: return name in self.attributes - def get(self, name: _SettingsKeyT, default: Any = None) -> Any: + def add_to_list(self, name: _SettingsKey, item: Any) -> None: + """Append *item* to the :class:`list` setting with the specified *name* + if *item* is not already in that list. + + This change is applied regardless of the priority of the *name* + setting. The setting priority is not affected by this change either. + """ + value: list[str] = self.getlist(name) + if item not in value: + self.set(name, [*value, item], self.getpriority(name) or 0) + + def remove_from_list(self, name: _SettingsKey, item: Any) -> None: + """Remove *item* from the :class:`list` setting with the specified + *name*. + + If *item* is missing, raise :exc:`ValueError`. + + This change is applied regardless of the priority of the *name* + setting. The setting priority is not affected by this change either. + """ + value: list[str] = self.getlist(name) + if item not in value: + raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).") + self.set(name, [v for v in value if v != item], self.getpriority(name) or 0) + + def get(self, name: _SettingsKey, default: Any = None) -> Any: # pylint: disable=arguments-renamed """ Get a setting value without affecting its original type. @@ -135,9 +153,26 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): :param default: the value to return if no setting is found :type default: object """ + if name == "CONCURRENT_REQUESTS_PER_IP" and ( + isinstance(self[name], int) and self[name] != 0 + ): + warnings.warn( + "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + + if name == "DNS_RESOLVER": + warnings.warn( + "The DNS_RESOLVER setting is deprecated, please use " + "TWISTED_DNS_RESOLVER instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return self[name] if self[name] is not None else default - def getbool(self, name: _SettingsKeyT, default: bool = False) -> bool: + def getbool(self, name: _SettingsKey, default: bool = False) -> bool: """ Get a setting value as a boolean. @@ -157,17 +192,17 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): try: return bool(int(got)) except ValueError: - if got in ("True", "true"): + if got in {"True", "true"}: return True - if got in ("False", "false"): + if got in {"False", "false"}: return False raise ValueError( "Supported values for boolean settings " "are 0/1, True/False, '0'/'1', " "'True'/'False' and 'true'/'false'" - ) + ) from None - def getint(self, name: _SettingsKeyT, default: int = 0) -> int: + def getint(self, name: _SettingsKey, default: int = 0) -> int: """ Get a setting value as an int. @@ -179,7 +214,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ return int(self.get(name, default)) - def getfloat(self, name: _SettingsKeyT, default: float = 0.0) -> float: + def getfloat(self, name: _SettingsKey, default: float = 0.0) -> float: """ Get a setting value as a float. @@ -192,11 +227,12 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return float(self.get(name, default)) def getlist( - self, name: _SettingsKeyT, default: Optional[List[Any]] = None - ) -> List[Any]: + self, name: _SettingsKey, default: list[Any] | None = None + ) -> list[Any]: """ - Get a setting value as a list. If the setting original type is a list, a - copy of it will be returned. If it's a string it will be split by ",". + Get a setting value as a list. If the setting original type is a list, + a copy of it will be returned. If it's a string it will be split by + ",". If it is an empty string, an empty list will be returned. For example, settings populated through environment variables set to ``'one,two'`` will return a list ['one', 'two'] when using this method. @@ -208,13 +244,15 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): :type default: object """ value = self.get(name, default or []) + if not value: + return [] if isinstance(value, str): value = value.split(",") return list(value) def getdict( - self, name: _SettingsKeyT, default: Optional[Dict[Any, Any]] = None - ) -> Dict[Any, Any]: + self, name: _SettingsKey, default: dict[Any, Any] | None = None + ) -> dict[Any, Any]: """ Get a setting value as a dictionary. If the setting original type is a dictionary, a copy of it will be returned. If it is a string it will be @@ -237,9 +275,9 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def getdictorlist( self, - name: _SettingsKeyT, - default: Union[Dict[Any, Any], List[Any], Tuple[Any], None] = None, - ) -> Union[Dict[Any, Any], List[Any]]: + name: _SettingsKey, + default: dict[Any, Any] | list[Any] | tuple[Any] | None = None, + ) -> dict[Any, Any] | list[Any]: """Get a setting value as either a :class:`dict` or a :class:`list`. If the setting is already a dict or a list, a copy of it will be @@ -267,19 +305,32 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): if isinstance(value, str): try: value_loaded = json.loads(value) - assert isinstance(value_loaded, (dict, list)) + if not isinstance(value_loaded, (dict, list)): + raise ValueError( + f"JSON string for setting '{name}' must evaluate to a dict or list, " + f"got {type(value_loaded).__name__}: {value_loaded!r}" + ) return value_loaded except ValueError: return value.split(",") if isinstance(value, tuple): return list(value) - assert isinstance(value, (dict, list)) + if not isinstance(value, (dict, list)): + raise ValueError( + f"Setting '{name}' must be a dict, list, tuple, or string, " + f"got {type(value).__name__}: {value!r}" + ) return copy.deepcopy(value) - def getwithbase(self, name: _SettingsKeyT) -> "BaseSettings": - """Get a composition of a dictionary-like setting and its `_BASE` + def getwithbase(self, name: _SettingsKey) -> BaseSettings: + """Get a composition of a dictionary-like setting and its ``_BASE`` counterpart. + Use + :meth:`~scrapy.settings.BaseSettings.get_component_priority_dict_with_base` + instead if the setting is a :ref:`component priority dictionary + `. + :param name: name of the dictionary-like setting :type name: str """ @@ -290,7 +341,55 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): compbs.update(self[name]) return compbs - def getpriority(self, name: _SettingsKeyT) -> Optional[int]: + def get_component_priority_dict_with_base(self, name: _SettingsKey) -> BaseSettings: + """Get a composition of a component priority dictionary setting and + its ``_BASE`` counterpart. + + Keys are resolved to their import path for deduplication and then + restored to their latest input representation. + + :param name: name of the component priority dictionary setting + :type name: str + """ + if not isinstance(name, str): + raise ValueError(f"Base setting key must be a string, got {name}") + + normalized_keys = {} + obj_keys = set() + + def track_loaded_key(k: Any) -> None: + if k not in obj_keys: + obj_keys.add(k) + return + logger.warning( + f"Setting {name} contains multiple keys that refer to the " + f"same object: {global_object_name(k)}. Only the last one will " + f"be kept." + ) + + def normalize_key(key: Any) -> Any: + try: + loaded_key = load_object(key) + except (NameError, TypeError, ValueError): + loaded_key = key + else: + import_path = global_object_name(loaded_key) + normalized_keys[import_path] = key + key = import_path + track_loaded_key(loaded_key) + return key + + def restore_key(k: str) -> Any: + return normalized_keys.get(k, k) + + result = dict(self[name + "_BASE"] or {}) + override = {normalize_key(k): v for k, v in (self[name] or {}).items()} + result.update(override) + return BaseSettings( + {restore_key(k): v for k, v in result.items() if v is not None} + ) + + def getpriority(self, name: _SettingsKey) -> int | None: """ Return the current numerical priority value of a setting, or ``None`` if the given ``name`` does not exist. @@ -310,14 +409,55 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): stored. """ if len(self) > 0: - return max(cast(int, self.getpriority(name)) for name in self) + return max(cast("int", self.getpriority(name)) for name in self) return get_settings_priority("default") - def __setitem__(self, name: _SettingsKeyT, value: Any) -> None: + def replace_in_component_priority_dict( + self, + name: _SettingsKey, + old_cls: type, + new_cls: type, + priority: int | None = None, + ) -> None: + """Replace *old_cls* with *new_cls* in the *name* :ref:`component + priority dictionary `. + + If *old_cls* is missing, or has :data:`None` as value, :exc:`KeyError` + is raised. + + If *old_cls* was present as an import string, even more than once, + those keys are dropped and replaced by *new_cls*. + + If *priority* is specified, that is the value assigned to *new_cls* in + the component priority dictionary. Otherwise, the value of *old_cls* is + used. If *old_cls* was present multiple times (possible with import + strings) with different values, the value assigned to *new_cls* is one + of them, with no guarantee about which one it is. + + This change is applied regardless of the priority of the *name* + setting. The setting priority is not affected by this change either. + """ + component_priority_dict = self.getdict(name) + old_priority = None + for cls_or_path in tuple(component_priority_dict): + if load_object(cls_or_path) != old_cls: + continue + if (old_priority := component_priority_dict.pop(cls_or_path)) is None: + break + if old_priority is None: + raise KeyError( + f"{old_cls} not found in the {name} setting ({component_priority_dict!r})." + ) + component_priority_dict[new_cls] = ( + old_priority if priority is None else priority + ) + self.set(name, component_priority_dict, priority=self.getpriority(name) or 0) + + def __setitem__(self, name: _SettingsKey, value: Any) -> None: self.set(name, value) def set( - self, name: _SettingsKeyT, value: Any, priority: Union[int, str] = "project" + self, name: _SettingsKey, value: Any, priority: int | str = "project" ) -> None: """ Store a key/value attribute with a given priority. @@ -346,11 +486,35 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): else: self.attributes[name].set(value, priority) - def setdefault( + def set_in_component_priority_dict( + self, name: _SettingsKey, cls: type, priority: int | None + ) -> None: + """Set the *cls* component in the *name* :ref:`component priority + dictionary ` setting with *priority*. + + If *cls* already exists, its value is updated. + + If *cls* was present as an import string, even more than once, those + keys are dropped and replaced by *cls*. + + This change is applied regardless of the priority of the *name* + setting. The setting priority is not affected by this change either. + """ + component_priority_dict = self.getdict(name) + for cls_or_path in tuple(component_priority_dict): + if not isinstance(cls_or_path, str): + continue + _cls = load_object(cls_or_path) + if _cls == cls: + del component_priority_dict[cls_or_path] + component_priority_dict[cls] = priority + self.set(name, component_priority_dict, self.getpriority(name) or 0) + + def setdefault( # pylint: disable=arguments-renamed self, - name: _SettingsKeyT, + name: _SettingsKey, default: Any = None, - priority: Union[int, str] = "project", + priority: int | str = "project", ) -> Any: if name not in self: self.set(name, default, priority) @@ -358,13 +522,29 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return self.attributes[name].value - def setdict( - self, values: _SettingsInputT, priority: Union[int, str] = "project" + def setdefault_in_component_priority_dict( + self, name: _SettingsKey, cls: type, priority: int | None ) -> None: + """Set the *cls* component in the *name* :ref:`component priority + dictionary ` setting with *priority* + if not already defined (even as an import string). + + If *cls* is not already defined, it is set regardless of the priority + of the *name* setting. The setting priority is not affected by this + change either. + """ + component_priority_dict = self.getdict(name) + for cls_or_path in tuple(component_priority_dict): + if load_object(cls_or_path) == cls: + return + component_priority_dict[cls] = priority + self.set(name, component_priority_dict, self.getpriority(name) or 0) + + def setdict(self, values: _SettingsInput, priority: int | str = "project") -> None: self.update(values, priority) def setmodule( - self, module: Union[ModuleType, str], priority: Union[int, str] = "project" + self, module: ModuleType | str, priority: int | str = "project" ) -> None: """ Store settings from a module with a given priority. @@ -388,7 +568,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): self.set(key, getattr(module, key), priority) # BaseSettings.update() doesn't support all inputs that MutableMapping.update() supports - def update(self, values: _SettingsInputT, priority: Union[int, str] = "project") -> None: # type: ignore[override] + def update(self, values: _SettingsInput, priority: int | str = "project") -> None: # type: ignore[override] """ Store key/value pairs with a given priority. @@ -412,26 +592,24 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ self._assert_mutability() if isinstance(values, str): - values = cast(dict, json.loads(values)) + values = cast("dict[_SettingsKey, Any]", json.loads(values)) if values is not None: if isinstance(values, BaseSettings): for name, value in values.items(): - self.set(name, value, cast(int, values.getpriority(name))) + self.set(name, value, cast("int", values.getpriority(name))) else: for name, value in values.items(): self.set(name, value, priority) - def delete( - self, name: _SettingsKeyT, priority: Union[int, str] = "project" - ) -> None: + def delete(self, name: _SettingsKey, priority: int | str = "project") -> None: if name not in self: raise KeyError(name) self._assert_mutability() priority = get_settings_priority(priority) - if priority >= cast(int, self.getpriority(name)): + if priority >= cast("int", self.getpriority(name)): del self.attributes[name] - def __delitem__(self, name: _SettingsKeyT) -> None: + def __delitem__(self, name: _SettingsKey) -> None: self._assert_mutability() del self.attributes[name] @@ -439,7 +617,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): if self.frozen: raise TypeError("Trying to modify an immutable Settings object") - def copy(self) -> "Self": + def copy(self) -> Self: """ Make a deep copy of current settings. @@ -461,7 +639,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ self.frozen = True - def frozencopy(self) -> "Self": + def frozencopy(self) -> Self: """ Return an immutable copy of the current settings. @@ -471,26 +649,26 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): copy.freeze() return copy - def __iter__(self) -> Iterator[_SettingsKeyT]: + def __iter__(self) -> Iterator[_SettingsKey]: return iter(self.attributes) def __len__(self) -> int: return len(self.attributes) - def _to_dict(self) -> Dict[_SettingsKeyT, Any]: + def _to_dict(self) -> dict[_SettingsKey, Any]: return { self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) for k, v in self.items() } - def _get_key(self, key_value: Any) -> _SettingsKeyT: + def _get_key(self, key_value: Any) -> _SettingsKey: return ( key_value if isinstance(key_value, (bool, float, int, str, type(None))) else str(key_value) ) - def copy_to_dict(self) -> Dict[_SettingsKeyT, Any]: + def copy_to_dict(self) -> dict[_SettingsKey, Any]: """ Make a copy of current settings and convert to a dict. @@ -513,17 +691,15 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): else: p.text(pformat(self.copy_to_dict())) - def pop(self, name: _SettingsKeyT, default: Any = __default) -> Any: + def pop(self, name: _SettingsKey, default: Any = __default) -> Any: # pylint: disable=arguments-renamed try: value = self.attributes[name].value except KeyError: if default is self.__default: raise - return default - else: - self.__delitem__(name) - return value + del self[name] + return value class Settings(BaseSettings): @@ -537,9 +713,7 @@ class Settings(BaseSettings): described on :ref:`topics-settings-ref` already populated. """ - def __init__( - self, values: _SettingsInputT = None, priority: Union[int, str] = "project" - ): + def __init__(self, values: _SettingsInput = None, priority: int | str = "project"): # Do not pass kwarg values here. We don't want to promote user-defined # dicts, and we want to update, not replace, default dicts with the # values given by the user @@ -553,7 +727,7 @@ class Settings(BaseSettings): self.update(values, priority) -def iter_default_settings() -> Iterable[Tuple[str, Any]]: +def iter_default_settings() -> Iterable[tuple[str, Any]]: """Return the default settings as an iterator of (name, value) tuples""" for name in dir(default_settings): if name.isupper(): @@ -561,8 +735,8 @@ def iter_default_settings() -> Iterable[Tuple[str, Any]]: def overridden_settings( - settings: Mapping[_SettingsKeyT, Any] -) -> Iterable[Tuple[str, Any]]: + settings: Mapping[_SettingsKey, Any], +) -> Iterable[tuple[str, Any]]: """Return an iterable of the settings that have been overridden""" for name, defvalue in iter_default_settings(): value = settings[name] diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 49ab1b5ef..7d5612026 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -1,25 +1,205 @@ -""" -This module contains the default values for all settings used by Scrapy. +"""This module contains the default values for all settings used by Scrapy. For more information about these settings you can read the settings documentation in docs/topics/settings.rst Scrapy developers, if you add a setting here remember to: -* add it in alphabetical order +* add it in alphabetical order, with the exception that enabling flags and + other high-level settings for a group should come first in their group + and pairs like host/port and user/password should be in the usual order * group similar settings without leaving blank lines * add its documentation to the available settings documentation (docs/topics/settings.rst) - """ import sys from importlib import import_module from pathlib import Path +from typing import Any + +__all__ = [ + "ADDONS", + "AJAXCRAWL_ENABLED", + "AJAXCRAWL_MAXSIZE", + "ASYNCIO_EVENT_LOOP", + "AUTOTHROTTLE_DEBUG", + "AUTOTHROTTLE_ENABLED", + "AUTOTHROTTLE_MAX_DELAY", + "AUTOTHROTTLE_START_DELAY", + "AUTOTHROTTLE_TARGET_CONCURRENCY", + "BOT_NAME", + "CLOSESPIDER_ERRORCOUNT", + "CLOSESPIDER_ITEMCOUNT", + "CLOSESPIDER_PAGECOUNT", + "CLOSESPIDER_TIMEOUT", + "COMMANDS_MODULE", + "COMPRESSION_ENABLED", + "CONCURRENT_ITEMS", + "CONCURRENT_REQUESTS", + "CONCURRENT_REQUESTS_PER_DOMAIN", + "COOKIES_DEBUG", + "COOKIES_ENABLED", + "CRAWLSPIDER_FOLLOW_LINKS", + "DEFAULT_DROPITEM_LOG_LEVEL", + "DEFAULT_ITEM_CLASS", + "DEFAULT_REQUEST_HEADERS", + "DEPTH_LIMIT", + "DEPTH_PRIORITY", + "DEPTH_STATS_VERBOSE", + "DNSCACHE_ENABLED", + "DNSCACHE_SIZE", + "DNS_TIMEOUT", + "DOWNLOADER", + "DOWNLOADER_CLIENTCONTEXTFACTORY", + "DOWNLOADER_CLIENT_TLS_CIPHERS", + "DOWNLOADER_CLIENT_TLS_METHOD", + "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING", + "DOWNLOADER_MIDDLEWARES", + "DOWNLOADER_MIDDLEWARES_BASE", + "DOWNLOADER_STATS", + "DOWNLOAD_BIND_ADDRESS", + "DOWNLOAD_DELAY", + "DOWNLOAD_FAIL_ON_DATALOSS", + "DOWNLOAD_HANDLERS", + "DOWNLOAD_HANDLERS_BASE", + "DOWNLOAD_MAXSIZE", + "DOWNLOAD_TIMEOUT", + "DOWNLOAD_TLS_MAX_VERSION", + "DOWNLOAD_TLS_MIN_VERSION", + "DOWNLOAD_WARNSIZE", + "DUPEFILTER_CLASS", + "EDITOR", + "EXTENSIONS", + "EXTENSIONS_BASE", + "FEEDS", + "FEED_EXPORTERS", + "FEED_EXPORTERS_BASE", + "FEED_EXPORT_BATCH_ITEM_COUNT", + "FEED_EXPORT_ENCODING", + "FEED_EXPORT_FIELDS", + "FEED_EXPORT_INDENT", + "FEED_FORMAT", + "FEED_STORAGES", + "FEED_STORAGES_BASE", + "FEED_STORAGE_FTP_ACTIVE", + "FEED_STORAGE_GCS_ACL", + "FEED_STORAGE_S3_ACL", + "FEED_STORE_EMPTY", + "FEED_TEMPDIR", + "FEED_URI_PARAMS", + "FILES_STORE_GCS_ACL", + "FILES_STORE_S3_ACL", + "FORCE_CRAWLER_PROCESS", + "FTP_PASSIVE_MODE", + "FTP_PASSWORD", + "FTP_USER", + "GCS_PROJECT_ID", + "HTTPCACHE_ALWAYS_STORE", + "HTTPCACHE_DBM_MODULE", + "HTTPCACHE_DIR", + "HTTPCACHE_ENABLED", + "HTTPCACHE_EXPIRATION_SECS", + "HTTPCACHE_GZIP", + "HTTPCACHE_IGNORE_HTTP_CODES", + "HTTPCACHE_IGNORE_MISSING", + "HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS", + "HTTPCACHE_IGNORE_SCHEMES", + "HTTPCACHE_POLICY", + "HTTPCACHE_STORAGE", + "HTTPPROXY_AUTH_ENCODING", + "HTTPPROXY_ENABLED", + "HTTPX_HTTP2_ENABLED", + "IMAGES_STORE_GCS_ACL", + "IMAGES_STORE_S3_ACL", + "ITEM_PIPELINES", + "ITEM_PIPELINES_BASE", + "ITEM_PROCESSOR", + "JOBDIR", + "LOGSTATS_INTERVAL", + "LOG_DATEFORMAT", + "LOG_ENABLED", + "LOG_ENCODING", + "LOG_FILE", + "LOG_FILE_APPEND", + "LOG_FORMAT", + "LOG_FORMATTER", + "LOG_LEVEL", + "LOG_SHORT_NAMES", + "LOG_STDOUT", + "LOG_VERSIONS", + "MAIL_FROM", + "MAIL_HOST", + "MAIL_PASS", + "MAIL_PORT", + "MAIL_USER", + "MEMDEBUG_ENABLED", + "MEMDEBUG_NOTIFY", + "MEMUSAGE_CHECK_INTERVAL_SECONDS", + "MEMUSAGE_ENABLED", + "MEMUSAGE_LIMIT_MB", + "MEMUSAGE_NOTIFY_MAIL", + "MEMUSAGE_WARNING_MB", + "METAREFRESH_ENABLED", + "METAREFRESH_IGNORE_TAGS", + "METAREFRESH_MAXDELAY", + "NEWSPIDER_MODULE", + "PERIODIC_LOG_DELTA", + "PERIODIC_LOG_STATS", + "PERIODIC_LOG_TIMING_ENABLED", + "RANDOMIZE_DOWNLOAD_DELAY", + "REACTOR_THREADPOOL_MAXSIZE", + "REDIRECT_ENABLED", + "REDIRECT_MAX_TIMES", + "REDIRECT_PRIORITY_ADJUST", + "REFERER_ENABLED", + "REFERRER_POLICY", + "REQUEST_FINGERPRINTER_CLASS", + "RETRY_ENABLED", + "RETRY_EXCEPTIONS", + "RETRY_GIVE_UP_LOG_LEVEL", + "RETRY_HTTP_CODES", + "RETRY_PRIORITY_ADJUST", + "RETRY_TIMES", + "ROBOTSTXT_OBEY", + "ROBOTSTXT_PARSER", + "ROBOTSTXT_USER_AGENT", + "SCHEDULER", + "SCHEDULER_DEBUG", + "SCHEDULER_DISK_QUEUE", + "SCHEDULER_MEMORY_QUEUE", + "SCHEDULER_PRIORITY_QUEUE", + "SCHEDULER_START_DISK_QUEUE", + "SCHEDULER_START_MEMORY_QUEUE", + "SCRAPER_SLOT_MAX_ACTIVE_SIZE", + "SPIDER_CONTRACTS", + "SPIDER_CONTRACTS_BASE", + "SPIDER_LOADER_CLASS", + "SPIDER_LOADER_WARN_ONLY", + "SPIDER_MIDDLEWARES", + "SPIDER_MIDDLEWARES_BASE", + "SPIDER_MODULES", + "STATSMAILER_RCPTS", + "STATS_CLASS", + "STATS_DUMP", + "TELNETCONSOLE_ENABLED", + "TELNETCONSOLE_HOST", + "TELNETCONSOLE_PASSWORD", + "TELNETCONSOLE_PORT", + "TELNETCONSOLE_USERNAME", + "TEMPLATES_DIR", + "TWISTED_DNS_RESOLVER", + "TWISTED_REACTOR", + "TWISTED_REACTOR_ENABLED", + "URLLENGTH_LIMIT", + "USER_AGENT", + "WARN_ON_GENERATOR_RETURN_VALUE", +] ADDONS = {} AJAXCRAWL_ENABLED = False +AJAXCRAWL_MAXSIZE = 32768 ASYNCIO_EVENT_LOOP = None @@ -31,10 +211,10 @@ AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 BOT_NAME = "scrapybot" -CLOSESPIDER_TIMEOUT = 0 -CLOSESPIDER_PAGECOUNT = 0 -CLOSESPIDER_ITEMCOUNT = 0 CLOSESPIDER_ERRORCOUNT = 0 +CLOSESPIDER_ITEMCOUNT = 0 +CLOSESPIDER_PAGECOUNT = 0 +CLOSESPIDER_TIMEOUT = 0 COMMANDS_MODULE = "" @@ -44,11 +224,14 @@ CONCURRENT_ITEMS = 100 CONCURRENT_REQUESTS = 16 CONCURRENT_REQUESTS_PER_DOMAIN = 8 -CONCURRENT_REQUESTS_PER_IP = 0 COOKIES_ENABLED = True COOKIES_DEBUG = False +CRAWLSPIDER_FOLLOW_LINKS = True + +DEFAULT_DROPITEM_LOG_LEVEL = "WARNING" + DEFAULT_ITEM_CLASS = "scrapy.item.Item" DEFAULT_REQUEST_HEADERS = { @@ -57,57 +240,57 @@ DEFAULT_REQUEST_HEADERS = { } DEPTH_LIMIT = 0 -DEPTH_STATS_VERBOSE = False DEPTH_PRIORITY = 0 +DEPTH_STATS_VERBOSE = False DNSCACHE_ENABLED = True DNSCACHE_SIZE = 10000 DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver" DNS_TIMEOUT = 60 +DOWNLOAD_BIND_ADDRESS = None + DOWNLOAD_DELAY = 0 +DOWNLOAD_FAIL_ON_DATALOSS = True + DOWNLOAD_HANDLERS = {} DOWNLOAD_HANDLERS_BASE = { "data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler", "file": "scrapy.core.downloader.handlers.file.FileDownloadHandler", - "http": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", - "https": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", + "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", + "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "s3": "scrapy.core.downloader.handlers.s3.S3DownloadHandler", "ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler", } -DOWNLOAD_TIMEOUT = 180 # 3mins - DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m -DOWNLOAD_FAIL_ON_DATALOSS = True +DOWNLOAD_TIMEOUT = 180 # 3mins + +DOWNLOAD_TLS_MAX_VERSION = None +DOWNLOAD_TLS_MIN_VERSION = None + +DOWNLOAD_VERIFY_CERTIFICATES = False DOWNLOADER = "scrapy.core.downloader.Downloader" -DOWNLOADER_HTTPCLIENTFACTORY = ( - "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory" -) -DOWNLOADER_CLIENTCONTEXTFACTORY = ( - "scrapy.core.downloader.contextfactory.ScrapyClientContextFactory" -) +DOWNLOADER_CLIENTCONTEXTFACTORY = "SENTINEL" DOWNLOADER_CLIENT_TLS_CIPHERS = "DEFAULT" -# Use highest TLS/SSL protocol version supported by the platform, also allowing negotiation: DOWNLOADER_CLIENT_TLS_METHOD = "TLS" DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False DOWNLOADER_MIDDLEWARES = {} - DOWNLOADER_MIDDLEWARES_BASE = { # Engine side + "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50, "scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware": 100, "scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware": 300, "scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware": 350, "scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400, "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500, "scrapy.downloadermiddlewares.retry.RetryMiddleware": 550, - "scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware": 560, "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580, "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590, "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600, @@ -127,9 +310,9 @@ if sys.platform == "win32": EDITOR = "%s -m idlelib.idle" EXTENSIONS = {} - EXTENSIONS_BASE = { "scrapy.extensions.corestats.CoreStats": 0, + "scrapy.extensions.logcount.LogCount": 0, "scrapy.extensions.telnet.TelnetConsole": 0, "scrapy.extensions.memusage.MemoryUsage": 0, "scrapy.extensions.memdebug.MemoryDebugger": 0, @@ -140,22 +323,11 @@ EXTENSIONS_BASE = { "scrapy.extensions.throttle.AutoThrottle": 0, } -FEED_TEMPDIR = None FEEDS = {} -FEED_URI_PARAMS = None # a function to extend uri arguments -FEED_STORE_EMPTY = True +FEED_EXPORT_BATCH_ITEM_COUNT = 0 FEED_EXPORT_ENCODING = None FEED_EXPORT_FIELDS = None -FEED_STORAGES = {} -FEED_STORAGES_BASE = { - "": "scrapy.extensions.feedexport.FileFeedStorage", - "file": "scrapy.extensions.feedexport.FileFeedStorage", - "ftp": "scrapy.extensions.feedexport.FTPFeedStorage", - "gs": "scrapy.extensions.feedexport.GCSFeedStorage", - "s3": "scrapy.extensions.feedexport.S3FeedStorage", - "stdout": "scrapy.extensions.feedexport.StdoutFeedStorage", -} -FEED_EXPORT_BATCH_ITEM_COUNT = 0 +FEED_EXPORT_INDENT = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { "json": "scrapy.exporters.JsonItemExporter", @@ -167,79 +339,104 @@ FEED_EXPORTERS_BASE = { "marshal": "scrapy.exporters.MarshalItemExporter", "pickle": "scrapy.exporters.PickleItemExporter", } -FEED_EXPORT_INDENT = 0 - +FEED_FORMAT = "jsonlines" +FEED_STORE_EMPTY = True +FEED_STORAGES = {} +FEED_STORAGES_BASE = { + "": "scrapy.extensions.feedexport.FileFeedStorage", + "file": "scrapy.extensions.feedexport.FileFeedStorage", + "ftp": "scrapy.extensions.feedexport.FTPFeedStorage", + "gs": "scrapy.extensions.feedexport.GCSFeedStorage", + "s3": "scrapy.extensions.feedexport.S3FeedStorage", + "stdout": "scrapy.extensions.feedexport.StdoutFeedStorage", +} FEED_STORAGE_FTP_ACTIVE = False FEED_STORAGE_GCS_ACL = "" FEED_STORAGE_S3_ACL = "" +FEED_TEMPDIR = None +FEED_URI_PARAMS = None # a function to extend uri arguments -FILES_STORE_S3_ACL = "private" FILES_STORE_GCS_ACL = "" +FILES_STORE_S3_ACL = "private" + +FORCE_CRAWLER_PROCESS = False -FTP_USER = "anonymous" -FTP_PASSWORD = "guest" FTP_PASSIVE_MODE = True +FTP_USER = "anonymous" +FTP_PASSWORD = "guest" # noqa: S105 GCS_PROJECT_ID = None HTTPCACHE_ENABLED = False -HTTPCACHE_DIR = "httpcache" -HTTPCACHE_IGNORE_MISSING = False -HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" -HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_ALWAYS_STORE = False -HTTPCACHE_IGNORE_HTTP_CODES = [] -HTTPCACHE_IGNORE_SCHEMES = ["file"] -HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = [] HTTPCACHE_DBM_MODULE = "dbm" -HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy" +HTTPCACHE_DIR = "httpcache" +HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_GZIP = False +HTTPCACHE_IGNORE_HTTP_CODES = [] +HTTPCACHE_IGNORE_MISSING = False +HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = [] +HTTPCACHE_IGNORE_SCHEMES = ["file"] +HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy" +HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = "latin-1" -IMAGES_STORE_S3_ACL = "private" -IMAGES_STORE_GCS_ACL = "" +HTTPX_HTTP2_ENABLED = False -ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager" +IMAGES_STORE_GCS_ACL = "" +IMAGES_STORE_S3_ACL = "private" ITEM_PIPELINES = {} ITEM_PIPELINES_BASE = {} +ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager" + JOBDIR = None LOG_ENABLED = True -LOG_ENCODING = "utf-8" -LOG_FORMATTER = "scrapy.logformatter.LogFormatter" -LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s" LOG_DATEFORMAT = "%Y-%m-%d %H:%M:%S" -LOG_STDOUT = False -LOG_LEVEL = "DEBUG" +LOG_ENCODING = "utf-8" LOG_FILE = None LOG_FILE_APPEND = True +LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s" +LOG_FORMATTER = "scrapy.logformatter.LogFormatter" +LOG_LEVEL = "DEBUG" LOG_SHORT_NAMES = False - -SCHEDULER_DEBUG = False +LOG_STDOUT = False +LOG_VERSIONS = [ + "lxml", + "libxml2", + "cssselect", + "parsel", + "w3lib", + "Twisted", + "Python", + "pyOpenSSL", + "cryptography", + "Platform", +] LOGSTATS_INTERVAL = 60.0 +MAIL_FROM = "scrapy@localhost" MAIL_HOST = "localhost" MAIL_PORT = 25 -MAIL_FROM = "scrapy@localhost" -MAIL_PASS = None MAIL_USER = None +MAIL_PASS = None MEMDEBUG_ENABLED = False # enable memory debugging MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown -MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0 MEMUSAGE_ENABLED = True +MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0 MEMUSAGE_LIMIT_MB = 0 MEMUSAGE_NOTIFY_MAIL = [] MEMUSAGE_WARNING_MB = 0 METAREFRESH_ENABLED = True -METAREFRESH_IGNORE_TAGS = [] +METAREFRESH_IGNORE_TAGS = ["noscript"] METAREFRESH_MAXDELAY = 100 NEWSPIDER_MODULE = "" @@ -260,48 +457,58 @@ REFERER_ENABLED = True REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy" REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter" -REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL" RETRY_ENABLED = True -RETRY_TIMES = 2 # initial response + 2 retries = 3 requests -RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] -RETRY_PRIORITY_ADJUST = -1 RETRY_EXCEPTIONS = [ - "twisted.internet.defer.TimeoutError", - "twisted.internet.error.TimeoutError", - "twisted.internet.error.DNSLookupError", - "twisted.internet.error.ConnectionRefusedError", + "scrapy.exceptions.CannotResolveHostError", + "scrapy.exceptions.DownloadConnectionRefusedError", + "scrapy.exceptions.DownloadFailedError", + "scrapy.exceptions.DownloadTimeoutError", + "scrapy.exceptions.ResponseDataLossError", "twisted.internet.error.ConnectionDone", "twisted.internet.error.ConnectError", "twisted.internet.error.ConnectionLost", - "twisted.internet.error.TCPTimedOutError", - "twisted.web.client.ResponseFailed", # OSError is raised by the HttpCompression middleware when trying to # decompress an empty response OSError, "scrapy.core.downloader.handlers.http11.TunnelError", ] +RETRY_GIVE_UP_LOG_LEVEL = "ERROR" +RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] +RETRY_PRIORITY_ADJUST = -1 +RETRY_TIMES = 2 # initial response + 2 retries = 3 requests ROBOTSTXT_OBEY = False ROBOTSTXT_PARSER = "scrapy.robotstxt.ProtegoRobotParser" ROBOTSTXT_USER_AGENT = None SCHEDULER = "scrapy.core.scheduler.Scheduler" +SCHEDULER_DEBUG = False SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleLifoDiskQueue" SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.LifoMemoryQueue" -SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue" +SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.DownloaderAwarePriorityQueue" +SCHEDULER_START_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue" +SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000 +SPIDER_CONTRACTS = {} +SPIDER_CONTRACTS_BASE = { + "scrapy.contracts.default.UrlContract": 1, + "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1, + "scrapy.contracts.default.MetadataContract": 1, + "scrapy.contracts.default.ReturnsContract": 2, + "scrapy.contracts.default.ScrapesContract": 3, +} + SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader" SPIDER_LOADER_WARN_ONLY = False SPIDER_MIDDLEWARES = {} - SPIDER_MIDDLEWARES_BASE = { # Engine side + "scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25, "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50, - "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": 500, "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, @@ -315,24 +522,37 @@ STATS_DUMP = True STATSMAILER_RCPTS = [] -TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve()) - -URLLENGTH_LIMIT = 2083 - -USER_AGENT = f'Scrapy/{import_module("scrapy").__version__} (+https://scrapy.org)' - TELNETCONSOLE_ENABLED = 1 -TELNETCONSOLE_PORT = [6023, 6073] TELNETCONSOLE_HOST = "127.0.0.1" +TELNETCONSOLE_PORT = [6023, 6073] TELNETCONSOLE_USERNAME = "scrapy" TELNETCONSOLE_PASSWORD = None -TWISTED_REACTOR = None +TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve()) -SPIDER_CONTRACTS = {} -SPIDER_CONTRACTS_BASE = { - "scrapy.contracts.default.UrlContract": 1, - "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1, - "scrapy.contracts.default.ReturnsContract": 2, - "scrapy.contracts.default.ScrapesContract": 3, -} +TWISTED_DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver" + +TWISTED_REACTOR_ENABLED = True +TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" + +URLLENGTH_LIMIT = 2083 + +USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)" + +WARN_ON_GENERATOR_RETURN_VALUE = True + + +def __getattr__(name: str) -> Any: + if name == "CONCURRENT_REQUESTS_PER_IP": + import warnings # noqa: PLC0415 + + from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415 + + warnings.warn( + "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return 0 + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/scrapy/shell.py b/scrapy/shell.py index bb3b1461c..44e542470 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -3,40 +3,156 @@ See documentation in docs/topics/shell.rst """ + +from __future__ import annotations + +import asyncio +import contextlib import os import signal +import warnings +from typing import TYPE_CHECKING, Any from itemadapter import is_item -from twisted.internet import defer, threads +from twisted.internet import threads +from twisted.internet.defer import Deferred from twisted.python import threadable from w3lib.url import any_to_uri +import scrapy from scrapy.crawler import Crawler -from scrapy.exceptions import IgnoreRequest +from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.conf import get_config from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console from scrapy.utils.datatypes import SequenceExclude +from scrapy.utils.defer import ( + _schedule_coro, + deferred_f_from_coro_f, + maybe_deferred_to_future, +) from scrapy.utils.misc import load_object from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop from scrapy.utils.response import open_in_browser +if TYPE_CHECKING: + from collections.abc import Callable + +# Hopefully temporary architecture notes +# +# The Shell class is always instantiated in the "main" thread. There are two +# official ways to use it: +# 1. scrapy.commands.shell, which makes a secondary thread and calls +# CrawlerProcess.start() in it, which runs a reactor there. +# 2. scrapy.shell.inspect_response(), which just creates Shell() in the current +# thread. +# +# Shell._inthread is True when this class is run in a thread separate from the +# reactor, e.g. the 1st way (in other words, the reactor is in a secondary +# thread). +# Shell._inthread is False when this class is run in the same thread as the +# reactor, e.g. the 2nd way. +# The only thing that differs is availability of fetch() (it needs the +# reactor to be in a separate thread: the shell sends the request to +# the reactor and waits for the result synchronously). +# +# Thus the only thing Shell needs an event loop for is fetch(). More machinery +# is used for it to work. In chronological order: +# 1. scrapy.commands.shell.Command.run() creates a crawler and an engine, then +# calls +# _schedule_coro(crawler.engine.start_async(_start_request_processing=False)), +# which initializes the engine but doesn't start processing of requests. +# 2. scrapy.commands.shell.Command.run() calls crawler_process.start() in a +# thread which starts a reactor in that thread. +# 3. When fetch() is called, it prepares a request and calls Shell._schedule() +# in the reactor thread (via threads.blockingCallFromThread()). +# 4. Shell._schedule() calls Shell._open_spider() (on the first call). +# 5. Shell._open_spider() calls engine.open_spider_async(close_if_idle=False) +# and engine._start_request_processing(). +# 6. Shell._schedule() calls engine.crawl(request), scheduling the request. +# 7. Shell._schedule() via _request_deferred() waits until the request callback +# is called. When it's called, the response becomes available. +# +# In the reactorless mode this is slightly different, the engine initialization +# happens in the event loop thread as many things need either a reactor or a +# running event loop. +# +# Side note: it should be possible to remove _request_deferred() by using +# engine.download() instead of engine.schedule(), losing the usual stuff like +# spider middlewares (none of which should be important). +# +# Other architecture problems: +# * scrapy.cmdline.execute() creates an AsyncCrawlerProcess instance which +# immediately installs a reactor (which is maybe not thread-specific?) or an +# event loop (which *is* thread-specific, so the main thread will always have +# a (not running) loop installed. +# * scrapy.commands.shell.Command.run() calls _schedule_coro() in the main +# thread, and various engine init code also calls similar things, +# conceptually this shouldn't work (and doesn't in the reactorless mode, so +# there the initialization is moved to the event loop thread). +# * The engine has several code paths specifically for the shell, and the shell +# uses several private members of the engine and of AsyncCrawlerProcess. + class Shell: - relevant_classes = (Crawler, Spider, Request, Response, Settings) + relevant_classes: tuple[type, ...] = (Crawler, Spider, Request, Response, Settings) - def __init__(self, crawler, update_vars=None, code=None): - self.crawler = crawler - self.update_vars = update_vars or (lambda x: None) - self.item_class = load_object(crawler.settings["DEFAULT_ITEM_CLASS"]) - self.spider = None - self.inthread = not threadable.isInIOThread() - self.code = code - self.vars = {} + def __init__( + self, + crawler: Crawler, + update_vars: Callable[[dict[str, Any]], None] | None = None, + code: str | None = None, + *, + loop: asyncio.AbstractEventLoop | None = None, + ): + self._use_reactor = crawler.settings.getbool("TWISTED_REACTOR_ENABLED") + if not self._use_reactor and not loop: # pragma: no cover + raise RuntimeError( + "Shell needs the crawler loop reference when TWISTED_REACTOR_ENABLED=False." + ) + self._loop = loop + self.crawler: Crawler = crawler + self.update_vars: Callable[[dict[str, Any]], None] = update_vars or ( + lambda x: None + ) + self.item_class: type = load_object(crawler.settings["DEFAULT_ITEM_CLASS"]) + self.spider: Spider | None = None + if self._use_reactor: + self._inthread: bool = not threadable.isInIOThread() + else: + try: + # in case there is also a running loop in the main thread + current_loop = asyncio.get_running_loop() + self._inthread = current_loop is not self._loop + except RuntimeError: + self._inthread = True + self.code: str | None = code + self.vars: dict[str, Any] = {} - def start(self, url=None, request=None, response=None, spider=None, redirect=True): + @property + def inthread(self) -> bool: # pragma: no cover + warnings.warn( + "Shell.inthread is deprecated, use Shell.fetch_available instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return self._inthread + + @property + def fetch_available(self) -> bool: + """Whether fetch() can be used.""" + return self._inthread + + def start( + self, + url: str | None = None, + request: Request | None = None, + response: Response | None = None, + spider: Spider | None = None, + redirect: bool = True, + ) -> None: # disable accidental Ctrl-C key press from shutting down the engine signal.signal(signal.SIGINT, signal.SIG_IGN) if url: @@ -49,17 +165,15 @@ class Shell: else: self.populate_vars() if self.code: - print(eval(self.code, globals(), self.vars)) + print(eval(self.code, globals(), self.vars)) # noqa: S307 else: - """ - Detect interactive shell setting in scrapy.cfg - e.g.: ~/.config/scrapy.cfg or ~/.scrapy.cfg - [settings] - # shell can be one of ipython, bpython or python; - # to be used as the interactive python console, if available. - # (default is ipython, fallbacks in the order listed above) - shell = python - """ + # Detect interactive shell setting in scrapy.cfg + # e.g.: ~/.config/scrapy.cfg or ~/.scrapy.cfg + # [settings] + # # shell can be one of ipython, bpython or python; + # # to be used as the interactive python console, if available. + # # (default is ipython, fallbacks in the order listed above) + # shell = python cfg = get_config() section, option = "settings", "shell" env = os.environ.get("SCRAPY_PYTHON_SHELL") @@ -76,32 +190,40 @@ class Shell: self.vars, shells=shells, banner=self.vars.pop("banner", "") ) - def _schedule(self, request, spider): - if is_asyncio_reactor_installed(): + async def _schedule(self, request: Request, spider: Spider | None) -> Response: + """Send the request to the engine, wait for the result. + + Runs in the reactor thread. + """ + if self._use_reactor and is_asyncio_reactor_installed(): # set the asyncio event loop for the current thread event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"] set_asyncio_event_loop(event_loop_path) - spider = self._open_spider(request, spider) - d = _request_deferred(request) - d.addCallback(lambda x: (x, spider)) + if not self.spider: + await self._open_spider(spider) + assert self.crawler.engine is not None + # send the request to the engine self.crawler.engine.crawl(request) - return d - - def _open_spider(self, request, spider): - if self.spider: - return self.spider + # this will fire when the request callback runs (via the callback hijacking in _request_deferred()) + return await maybe_deferred_to_future(_request_deferred(request)) + async def _open_spider(self, spider: Spider | None) -> None: if spider is None: spider = self.crawler.spider or self.crawler._create_spider() self.crawler.spider = spider - self.crawler.engine.open_spider(spider, close_if_idle=False) + assert self.crawler.engine + await self.crawler.engine.open_spider_async(close_if_idle=False) + _schedule_coro(self.crawler.engine._start_request_processing()) self.spider = spider - return spider - - def fetch(self, request_or_url, spider=None, redirect=True, **kwargs): - from twisted.internet import reactor + def fetch( + self, + request_or_url: Request | str, + spider: Spider | None = None, + redirect: bool = True, + **kwargs: Any, + ) -> None: if isinstance(request_or_url, Request): request = request_or_url else: @@ -113,18 +235,32 @@ class Shell: ) else: request.meta["handle_httpstatus_all"] = True - response = None - try: - response, spider = threads.blockingCallFromThread( - reactor, self._schedule, request, spider - ) - except IgnoreRequest: - pass - self.populate_vars(response, request, spider) + response: Response | None = None + if self._use_reactor: + from twisted.internet import reactor - def populate_vars(self, response=None, request=None, spider=None): - import scrapy + with contextlib.suppress(IgnoreRequest): + response = threads.blockingCallFromThread( + reactor, + deferred_f_from_coro_f(self._schedule), # type: ignore[arg-type] + request, + spider, + ) + else: + assert self._loop + with contextlib.suppress(IgnoreRequest): + future = asyncio.run_coroutine_threadsafe( + self._schedule(request, spider), self._loop + ) + response = future.result() + self.populate_vars(response, request, self.spider) + def populate_vars( + self, + response: Response | None = None, + request: Request | None = None, + spider: Spider | None = None, + ) -> None: self.vars["scrapy"] = scrapy self.vars["crawler"] = self.crawler self.vars["item"] = self.item_class() @@ -132,7 +268,7 @@ class Shell: self.vars["spider"] = spider self.vars["request"] = request self.vars["response"] = response - if self.inthread: + if self.fetch_available: self.vars["fetch"] = self.fetch self.vars["view"] = open_in_browser self.vars["shelp"] = self.print_help @@ -140,10 +276,10 @@ class Shell: if not self.code: self.vars["banner"] = self.get_help() - def print_help(self): + def print_help(self) -> None: print(self.get_help()) - def get_help(self): + def get_help(self) -> str: b = [] b.append("Available Scrapy objects:") b.append( @@ -153,7 +289,7 @@ class Shell: if self._is_relevant(v): b.append(f" {k:<10} {v}") b.append("Useful shortcuts:") - if self.inthread: + if self.fetch_available: b.append( " fetch(url[, redirect=True]) " "Fetch URL and update local objects (by default, redirects are followed)" @@ -165,22 +301,26 @@ class Shell: b.append(" shelp() Shell help (print this help)") b.append(" view(response) View response in a browser") - return "\n".join(f"[s] {line}" for line in b) + return "\n".join(f"[s] {line}" for line in b) + "\n" - def _is_relevant(self, value): + def _is_relevant(self, value: Any) -> bool: return isinstance(value, self.relevant_classes) or is_item(value) -def inspect_response(response, spider): +def inspect_response(response: Response, spider: Spider) -> None: """Open a shell to inspect the given response""" # Shell.start removes the SIGINT handler, so save it and re-add it after # the shell has closed sigint_handler = signal.getsignal(signal.SIGINT) - Shell(spider.crawler).start(response=response, spider=spider) + if not spider.crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): + loop = asyncio.get_running_loop() + else: + loop = None + Shell(spider.crawler, loop=loop).start(response=response, spider=spider) signal.signal(signal.SIGINT, sigint_handler) -def _request_deferred(request): +def _request_deferred(request: Request) -> Deferred[Any]: """Wrap a request inside a Deferred. This function is harmful, do not use it until you know what you are doing. @@ -194,15 +334,17 @@ def _request_deferred(request): request_callback = request.callback request_errback = request.errback - def _restore_callbacks(result): + def _restore_callbacks(result: Any) -> Any: request.callback = request_callback request.errback = request_errback return result - d = defer.Deferred() + d: Deferred[Any] = Deferred() d.addBoth(_restore_callbacks) if request.callback: - d.addCallbacks(request.callback, request.errback) + d.addCallback(request.callback) + if request.errback: + d.addErrback(request.errback) request.callback, request.errback = d.callback, d.errback return d diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index f6df191d8..ca3b5b7dd 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,9 +1,14 @@ -from typing import Any, List, Tuple +from __future__ import annotations + +import warnings +from typing import Any from pydispatch import dispatcher from twisted.internet.defer import Deferred +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils import signal as _signal +from scrapy.utils.defer import maybe_deferred_to_future class SignalManager: @@ -36,7 +41,7 @@ class SignalManager: kwargs.setdefault("sender", self.sender) dispatcher.disconnect(receiver, signal, **kwargs) - def send_catch_log(self, signal: Any, **kwargs: Any) -> List[Tuple[Any, Any]]: + def send_catch_log(self, signal: Any, **kwargs: Any) -> list[tuple[Any, Any]]: """ Send a signal, catch exceptions and log them. @@ -46,19 +51,45 @@ class SignalManager: kwargs.setdefault("sender", self.sender) return _signal.send_catch_log(signal, **kwargs) - def send_catch_log_deferred(self, signal: Any, **kwargs: Any) -> Deferred: + def send_catch_log_deferred( + self, signal: Any, **kwargs: Any + ) -> Deferred[list[tuple[Any, Any]]]: # pragma: no cover """ - Like :meth:`send_catch_log` but supports returning - :class:`~twisted.internet.defer.Deferred` objects from signal handlers. + Like :meth:`send_catch_log` but supports :ref:`asynchronous signal + handlers `. Returns a Deferred that gets fired once all signal handlers - deferreds were fired. Send a signal, catch exceptions and log them. + have finished. Send a signal, catch exceptions and log them. The keyword arguments are passed to the signal handlers (connected through the :meth:`connect` method). """ kwargs.setdefault("sender", self.sender) - return _signal.send_catch_log_deferred(signal, **kwargs) + warnings.warn( + "send_catch_log_deferred() is deprecated, use send_catch_log_async() instead", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return _signal._send_catch_log_deferred(signal, **kwargs) + + async def send_catch_log_async( + self, signal: Any, **kwargs: Any + ) -> list[tuple[Any, Any]]: + """ + Like :meth:`send_catch_log` but supports :ref:`asynchronous signal + handlers `. + + Returns a coroutine that completes once all signal handlers + have finished. Send a signal, catch exceptions and log them. + + The keyword arguments are passed to the signal handlers (connected + through the :meth:`connect` method). + + .. versionadded:: 2.14 + """ + # note that this returns exceptions instead of Failures in the second tuple member + kwargs.setdefault("sender", self.sender) + return await _signal.send_catch_log_async(signal, **kwargs) def disconnect_all(self, signal: Any, **kwargs: Any) -> None: """ @@ -69,3 +100,17 @@ class SignalManager: """ kwargs.setdefault("sender", self.sender) _signal.disconnect_all(signal, **kwargs) + + async def wait_for(self, signal: Any) -> None: + """Await the next *signal*. + + See :ref:`start-requests-lazy` for an example. + """ + d: Deferred[None] = Deferred() + + def handle() -> None: + self.disconnect(handle, signal) + d.callback(None) + + self.connect(handle, signal) + await maybe_deferred_to_future(d) diff --git a/scrapy/signals.py b/scrapy/signals.py index 0090f1c8b..972f4fd60 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -7,10 +7,12 @@ signals here without documenting them there. engine_started = object() engine_stopped = object() +scheduler_empty = object() spider_opened = object() spider_idle = object() spider_closed = object() spider_error = object() +memusage_warning_reached = object() request_scheduled = object() request_dropped = object() request_reached_downloader = object() @@ -24,12 +26,3 @@ item_dropped = object() item_error = object() feed_slot_closed = object() feed_exporter_closed = object() - -# for backward compatibility -stats_spider_opened = spider_opened -stats_spider_closing = spider_closed -stats_spider_closed = spider_closed - -item_passed = item_scraped - -request_received = request_scheduled diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index d855c962c..8c980fd46 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -3,21 +3,49 @@ from __future__ import annotations import traceback import warnings from collections import defaultdict -from types import ModuleType -from typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Type +from typing import TYPE_CHECKING, Protocol, cast from zope.interface import implementer +from zope.interface.verify import verifyClass -from scrapy import Request, Spider from scrapy.interfaces import ISpiderLoader -from scrapy.settings import BaseSettings -from scrapy.utils.misc import walk_modules +from scrapy.utils.misc import load_object, walk_modules_iter from scrapy.utils.spider import iter_spider_classes if TYPE_CHECKING: + from types import ModuleType + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.settings import BaseSettings + + +def get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol: + """Get SpiderLoader instance from settings""" + cls_path = settings.get("SPIDER_LOADER_CLASS") + loader_cls = load_object(cls_path) + verifyClass(ISpiderLoader, loader_cls) + return cast("SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy())) + + +class SpiderLoaderProtocol(Protocol): + @classmethod + def from_settings(cls, settings: BaseSettings) -> Self: + """Return an instance of the class for the given settings""" + + def load(self, spider_name: str) -> type[Spider]: + """Return the Spider class for the given spider name. If the spider + name is not found, it must raise a KeyError.""" + + def list(self) -> list[str]: + """Return a list with the names of all spiders available in the + project""" + + def find_by_request(self, request: Request) -> __builtins__.list[str]: + """Return the list of spiders names that can handle the given request""" + @implementer(ISpiderLoader) class SpiderLoader: @@ -27,10 +55,10 @@ class SpiderLoader: """ def __init__(self, settings: BaseSettings): - self.spider_modules: List[str] = settings.getlist("SPIDER_MODULES") + self.spider_modules: list[str] = settings.getlist("SPIDER_MODULES") self.warn_only: bool = settings.getbool("SPIDER_LOADER_WARN_ONLY") - self._spiders: Dict[str, Type[Spider]] = {} - self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list) + self._spiders: dict[str, type[Spider]] = {} + self._found: defaultdict[str, list[tuple[str, str]]] = defaultdict(list) self._load_all_spiders() def _check_name_duplicates(self) -> None: @@ -49,6 +77,7 @@ class SpiderLoader: warnings.warn( "There are several spiders with the same name:\n\n" f"{dupes_string}\n\n This can cause unexpected behavior.", + stacklevel=2, category=UserWarning, ) @@ -60,14 +89,15 @@ class SpiderLoader: def _load_all_spiders(self) -> None: for name in self.spider_modules: try: - for module in walk_modules(name): + for module in walk_modules_iter(name): self._load_spiders(module) - except ImportError: + except (ImportError, SyntaxError): if self.warn_only: warnings.warn( f"\n{traceback.format_exc()}Could not load spiders " f"from module '{name}'. " "See above traceback for details.", + stacklevel=2, category=RuntimeWarning, ) else: @@ -78,7 +108,7 @@ class SpiderLoader: def from_settings(cls, settings: BaseSettings) -> Self: return cls(settings) - def load(self, spider_name: str) -> Type[Spider]: + def load(self, spider_name: str) -> type[Spider]: """ Return the Spider class for the given spider name. If the spider name is not found, raise a KeyError. @@ -86,9 +116,9 @@ class SpiderLoader: try: return self._spiders[spider_name] except KeyError: - raise KeyError(f"Spider not found: {spider_name}") + raise KeyError(f"Spider not found: {spider_name}") from None - def find_by_request(self, request: Request) -> List[str]: + def find_by_request(self, request: Request) -> list[str]: """ Return the list of spider names that can handle the given request. """ @@ -96,8 +126,26 @@ class SpiderLoader: name for name, cls in self._spiders.items() if cls.handles_request(request) ] - def list(self) -> List[str]: + def list(self) -> list[str]: """ Return a list with the names of all spiders available in the project. """ return list(self._spiders.keys()) + + +@implementer(ISpiderLoader) +class DummySpiderLoader: + """A dummy spider loader that does not load any spiders.""" + + @classmethod + def from_settings(cls, settings: BaseSettings) -> Self: + return cls() + + def load(self, spider_name: str) -> type[Spider]: + raise KeyError("DummySpiderLoader doesn't load any spiders") + + def list(self) -> list[str]: + return [] + + def find_by_request(self, request: Request) -> __builtins__.list[str]: + return [] diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py new file mode 100644 index 000000000..e09f2d10e --- /dev/null +++ b/scrapy/spidermiddlewares/base.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from scrapy import Request, Spider +from scrapy.utils.decorators import _warn_spider_arg + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterable + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.http import Response + + +class BaseSpiderMiddleware: + """Optional base class for spider middlewares. + + .. versionadded:: 2.13 + + This class provides helper methods for asynchronous + ``process_spider_output()`` and ``process_start()`` methods. Middlewares + that don't have either of these methods don't need to use this class. + + You can override the + :meth:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware.get_processed_request` + method to add processing code for requests and the + :meth:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware.get_processed_item` + method to add processing code for items. These methods take a single + request or item from the spider output iterable and return a request or + item (the same or a new one), or ``None`` to remove this request or item + from the processing. + """ + + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls(crawler) + + async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]: + async for o in start: + if (o := self._get_processed(o, None)) is not None: + yield o + + @_warn_spider_arg + def process_spider_output( + self, response: Response, result: Iterable[Any], spider: Spider | None = None + ) -> Iterable[Any]: + for o in result: + if (o := self._get_processed(o, response)) is not None: + yield o + + @_warn_spider_arg + async def process_spider_output_async( + self, + response: Response, + result: AsyncIterator[Any], + spider: Spider | None = None, + ) -> AsyncIterator[Any]: + async for o in result: + if (o := self._get_processed(o, response)) is not None: + yield o + + def _get_processed(self, o: Any, response: Response | None) -> Any: + if isinstance(o, Request): + return self.get_processed_request(o, response) + return self.get_processed_item(o, response) + + def get_processed_request( + self, request: Request, response: Response | None + ) -> Request | None: + """Return a processed request from the spider output. + + This method is called with a single request from the start seeds or the + spider output. It should return the same or a different request, or + ``None`` to ignore it. + + :param request: the input request + :type request: :class:`~scrapy.Request` object + + :param response: the response being processed + :type response: :class:`~scrapy.http.Response` object or ``None`` for + start seeds + + :return: the processed request or ``None`` + """ + return request + + def get_processed_item(self, item: Any, response: Response | None) -> Any: + """Return a processed item from the spider output. + + This method is called with a single item from the start seeds or the + spider output. It should return the same or a different item, or + ``None`` to ignore it. + + :param item: the input item + :type item: item object + + :param response: the response being processed + :type response: :class:`~scrapy.http.Response` object or ``None`` for + start seeds + + :return: the processed item or ``None`` + """ + return item diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 1e96654e2..9c387530c 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -7,22 +7,30 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable +from typing import TYPE_CHECKING, Any -from scrapy import Spider -from scrapy.crawler import Crawler -from scrapy.http import Request, Response -from scrapy.statscollectors import StatsCollector +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware +from scrapy.utils.decorators import _warn_spider_arg if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.http import Request, Response + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) -class DepthMiddleware: - def __init__( +class DepthMiddleware(BaseSpiderMiddleware): + crawler: Crawler + + def __init__( # pylint: disable=super-init-not-called self, maxdepth: int, stats: StatsCollector, @@ -41,32 +49,41 @@ class DepthMiddleware: verbose = settings.getbool("DEPTH_STATS_VERBOSE") prio = settings.getint("DEPTH_PRIORITY") assert crawler.stats - return cls(maxdepth, crawler.stats, verbose, prio) + o = cls(maxdepth, crawler.stats, verbose, prio) + o.crawler = crawler + return o + @_warn_spider_arg def process_spider_output( - self, response: Response, result: Iterable[Any], spider: Spider + self, response: Response, result: Iterable[Any], spider: Spider | None = None ) -> Iterable[Any]: - self._init_depth(response, spider) - return (r for r in result if self._filter(r, response, spider)) + self._init_depth(response) + yield from super().process_spider_output(response, result) + @_warn_spider_arg async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: - self._init_depth(response, spider) - async for r in result: - if self._filter(r, response, spider): - yield r + self, + response: Response, + result: AsyncIterator[Any], + spider: Spider | None = None, + ) -> AsyncIterator[Any]: + self._init_depth(response) + async for o in super().process_spider_output_async(response, result): + yield o - def _init_depth(self, response: Response, spider: Spider) -> None: + def _init_depth(self, response: Response) -> None: # base case (depth=0) if "depth" not in response.meta: response.meta["depth"] = 0 if self.verbose_stats: - self.stats.inc_value("request_depth_count/0", spider=spider) + self.stats.inc_value("request_depth_count/0") - def _filter(self, request: Any, response: Response, spider: Spider) -> bool: - if not isinstance(request, Request): - return True + def get_processed_request( + self, request: Request, response: Response | None + ) -> Request | None: + if response is None: + # start requests + return request depth = response.meta["depth"] + 1 request.meta["depth"] = depth if self.prio: @@ -75,10 +92,10 @@ class DepthMiddleware: logger.debug( "Ignoring link (depth > %(maxdepth)d): %(requrl)s ", {"maxdepth": self.maxdepth, "requrl": request.url}, - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) - return False + return None if self.verbose_stats: - self.stats.inc_value(f"request_depth_count/{depth}", spider=spider) - self.stats.max_value("request_depth_max", depth, spider=spider) - return True + self.stats.inc_value(f"request_depth_count/{depth}") + self.stats.max_value("request_depth_max", depth) + return request diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 94450b35b..94b6dfbb5 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -3,21 +3,27 @@ HttpError Spider Middleware See documentation in docs/topics/spider-middleware.rst """ + from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Iterable, List, Optional +from typing import TYPE_CHECKING, Any -from scrapy import Spider -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest -from scrapy.http import Response -from scrapy.settings import BaseSettings +from scrapy.utils.decorators import _warn_spider_arg if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) @@ -30,17 +36,24 @@ class HttpError(IgnoreRequest): class HttpErrorMiddleware: - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - return cls(crawler.settings) + crawler: Crawler def __init__(self, settings: BaseSettings): self.handle_httpstatus_all: bool = settings.getbool("HTTPERROR_ALLOW_ALL") - self.handle_httpstatus_list: List[int] = settings.getlist( + self.handle_httpstatus_list: list[int] = settings.getlist( "HTTPERROR_ALLOWED_CODES" ) - def process_spider_input(self, response: Response, spider: Spider) -> None: + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + o = cls(crawler.settings) + o.crawler = crawler + return o + + @_warn_spider_arg + def process_spider_input( + self, response: Response, spider: Spider | None = None + ) -> None: if 200 <= response.status < 300: # common case return meta = response.meta @@ -52,25 +65,28 @@ class HttpErrorMiddleware: return else: allowed_statuses = getattr( - spider, "handle_httpstatus_list", self.handle_httpstatus_list + self.crawler.spider, + "handle_httpstatus_list", + self.handle_httpstatus_list, ) if response.status in allowed_statuses: return raise HttpError(response, "Ignoring non-200 response") + @_warn_spider_arg def process_spider_exception( - self, response: Response, exception: Exception, spider: Spider - ) -> Optional[Iterable[Any]]: + self, response: Response, exception: Exception, spider: Spider | None = None + ) -> Iterable[Any] | None: if isinstance(exception, HttpError): - assert spider.crawler.stats - spider.crawler.stats.inc_value("httperror/response_ignored_count") - spider.crawler.stats.inc_value( + assert self.crawler.stats + self.crawler.stats.inc_value("httperror/response_ignored_count") + self.crawler.stats.inc_value( f"httperror/response_ignored_status_count/{response.status}" ) logger.info( "Ignoring response %(response)r: HTTP status code is not handled or not allowed", {"response": response}, - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) - return [] + return () return None diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a29e0ebb5..6c5acf0de 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,39 +2,34 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ + from __future__ import annotations import warnings -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - Dict, - Iterable, - Optional, - Tuple, - Type, - Union, - cast, -) +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, cast from urllib.parse import urlparse +from warnings import warn -from w3lib.url import safe_url_string - -from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response -from scrapy.settings import BaseSettings +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware from scrapy.utils.misc import load_object -from scrapy.utils.python import to_unicode +from scrapy.utils.python import _looks_like_import_path, to_unicode from scrapy.utils.url import strip_url if TYPE_CHECKING: # typing.Self requires Python 3.11 - from typing_extensions import Self + from typing_extensions import Self, TypedDict, Unpack -LOCAL_SCHEMES: Tuple[str, ...] = ( + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + class _PolicyKwargs(TypedDict, total=False): + resp_or_url: Response | str + + +LOCAL_SCHEMES: tuple[str, ...] = ( "about", "blob", "data", @@ -52,24 +47,27 @@ POLICY_UNSAFE_URL = "unsafe-url" POLICY_SCRAPY_DEFAULT = "scrapy-default" -class ReferrerPolicy: - NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES +class ReferrerPolicy(ABC): + """Abstract base class for referrer policies.""" + + NOREFERRER_SCHEMES: tuple[str, ...] = LOCAL_SCHEMES name: str - def referrer(self, response_url: str, request_url: str) -> Optional[str]: - raise NotImplementedError() + @abstractmethod + def referrer(self, response_url: str, request_url: str) -> str | None: + raise NotImplementedError - def stripped_referrer(self, url: str) -> Optional[str]: + def stripped_referrer(self, url: str) -> str | None: if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: return self.strip_url(url) return None - def origin_referrer(self, url: str) -> Optional[str]: + def origin_referrer(self, url: str) -> str | None: if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: return self.origin(url) return None - def strip_url(self, url: str, origin_only: bool = False) -> Optional[str]: + def strip_url(self, url: str, origin_only: bool = False) -> str | None: """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -93,19 +91,19 @@ class ReferrerPolicy: origin_only=origin_only, ) - def origin(self, url: str) -> Optional[str]: + def origin(self, url: str) -> str | None: """Return serialized origin (scheme, host, path) for a request or response URL.""" return self.strip_url(url, origin_only=True) def potentially_trustworthy(self, url: str) -> bool: # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy parsed_url = urlparse(url) - if parsed_url.scheme in ("data",): + if parsed_url.scheme == "data": return False return self.tls_protected(url) def tls_protected(self, url: str) -> bool: - return urlparse(url).scheme in ("https", "ftps") + return urlparse(url).scheme in {"https", "ftps"} class NoReferrerPolicy(ReferrerPolicy): @@ -119,7 +117,7 @@ class NoReferrerPolicy(ReferrerPolicy): name: str = POLICY_NO_REFERRER - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: return None @@ -140,7 +138,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: if not self.tls_protected(response_url) or self.tls_protected(request_url): return self.stripped_referrer(response_url) return None @@ -159,7 +157,7 @@ class SameOriginPolicy(ReferrerPolicy): name: str = POLICY_SAME_ORIGIN - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: if self.origin(response_url) == self.origin(request_url): return self.stripped_referrer(response_url) return None @@ -177,7 +175,7 @@ class OriginPolicy(ReferrerPolicy): name: str = POLICY_ORIGIN - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: return self.origin_referrer(response_url) @@ -197,12 +195,11 @@ class StrictOriginPolicy(ReferrerPolicy): name: str = POLICY_STRICT_ORIGIN - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: if ( self.tls_protected(response_url) and self.potentially_trustworthy(request_url) - or not self.tls_protected(response_url) - ): + ) or not self.tls_protected(response_url): return self.origin_referrer(response_url) return None @@ -221,7 +218,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) @@ -248,15 +245,14 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) if ( self.tls_protected(response_url) and self.potentially_trustworthy(request_url) - or not self.tls_protected(response_url) - ): + ) or not self.tls_protected(response_url): return self.origin_referrer(response_url) return None @@ -277,7 +273,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): name: str = POLICY_UNSAFE_URL - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: return self.stripped_referrer(response_url) @@ -288,133 +284,156 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): using ``file://`` or ``s3://`` scheme. """ - NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ("file", "s3") + NOREFERRER_SCHEMES: tuple[str, ...] = (*LOCAL_SCHEMES, "file", "s3") name: str = POLICY_SCRAPY_DEFAULT -_policy_classes: Dict[str, Type[ReferrerPolicy]] = { - p.name: p - for p in ( - NoReferrerPolicy, - NoReferrerWhenDowngradePolicy, - SameOriginPolicy, - OriginPolicy, - StrictOriginPolicy, - OriginWhenCrossOriginPolicy, - StrictOriginWhenCrossOriginPolicy, - UnsafeUrlPolicy, - DefaultReferrerPolicy, - ) -} - -# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string -_policy_classes[""] = NoReferrerWhenDowngradePolicy - - -def _load_policy_class( - policy: str, warning_only: bool = False -) -> Optional[Type[ReferrerPolicy]]: - """ - Expect a string for the path to the policy class, - otherwise try to interpret the string as a standard value - from https://www.w3.org/TR/referrer-policy/#referrer-policies - """ - try: - return cast(Type[ReferrerPolicy], load_object(policy)) - except ValueError: - try: - return _policy_classes[policy.lower()] - except KeyError: - msg = f"Could not load referrer policy {policy!r}" - if not warning_only: - raise RuntimeError(msg) +class RefererMiddleware(BaseSpiderMiddleware): + def __init__(self, settings: BaseSettings | None = None): # pylint: disable=super-init-not-called + self.default_policy: type[ReferrerPolicy] = DefaultReferrerPolicy + self.policies: dict[str, type[ReferrerPolicy]] = { + p.name: p + for p in ( + NoReferrerPolicy, + NoReferrerWhenDowngradePolicy, + SameOriginPolicy, + OriginPolicy, + StrictOriginPolicy, + OriginWhenCrossOriginPolicy, + StrictOriginWhenCrossOriginPolicy, + UnsafeUrlPolicy, + DefaultReferrerPolicy, + ) + } + # Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string + self.policies[""] = NoReferrerWhenDowngradePolicy + if settings is None: + return + setting_policies = settings.getdict("REFERRER_POLICIES") + for policy_name, policy_class_import_path in setting_policies.items(): + if policy_class_import_path is None: + del self.policies[policy_name] else: - warnings.warn(msg, RuntimeWarning) - return None - - -class RefererMiddleware: - def __init__(self, settings: Optional[BaseSettings] = None): - self.default_policy: Type[ReferrerPolicy] = DefaultReferrerPolicy - if settings is not None: - settings_policy = _load_policy_class(settings.get("REFERRER_POLICY")) - assert settings_policy - self.default_policy = settings_policy + self.policies[policy_name] = load_object(policy_class_import_path) + settings_policy = self._load_policy_class( + settings.get("REFERRER_POLICY"), allow_import_path=True + ) + assert settings_policy + self.default_policy = settings_policy @classmethod def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("REFERER_ENABLED"): raise NotConfigured - mw = cls(crawler.settings) - - # Note: this hook is a bit of a hack to intercept redirections - crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled) - - return mw + return cls(crawler.settings) def policy( - self, resp_or_url: Union[Response, str], request: Request + self, + response: Response | str | None = None, + request: Request | None = None, + **kwargs: Unpack[_PolicyKwargs], ) -> ReferrerPolicy: - """ - Determine Referrer-Policy to use from a parent Response (or URL), - and a Request to be sent. + """Return the referrer policy to use for *request* based on *request* + meta, *response* and settings. - if a valid policy is set in Request meta, it is used. - - if the policy is set in meta but is wrong (e.g. a typo error), - the policy from settings is used - - if the policy is not set in Request meta, - but there is a Referrer-policy header in the parent response, - it is used if valid + - if the policy is set in meta but is wrong (e.g. a typo error), the + policy from settings is used + - if the policy is not set in Request meta, but there is a + Referrer-Policy header in the parent response, it is used if valid - otherwise, the policy from settings is used. """ + if "resp_or_url" in kwargs: + if response is not None: + raise TypeError("Cannot pass both 'response' and 'resp_or_url'") + response = kwargs.pop("resp_or_url") + warn( + "Passing 'resp_or_url' is deprecated, use 'response' instead.", + DeprecationWarning, + stacklevel=2, + ) + if response is None: + raise TypeError("Missing required argument: 'response'") + if request is None: + raise TypeError("Missing required argument: 'request'") + if isinstance(response, str): + warn( + "Passing a response URL to RefererMiddleware.policy() instead " + "of a Response object is deprecated.", + DeprecationWarning, + stacklevel=2, + ) + allow_import_path = True policy_name = request.meta.get("referrer_policy") - if policy_name is None: - if isinstance(resp_or_url, Response): - policy_header = resp_or_url.headers.get("Referrer-Policy") - if policy_header is not None: - policy_name = to_unicode(policy_header.decode("latin1")) + if policy_name is None and isinstance(response, Response): + policy_header = response.headers.get("Referrer-Policy") + if policy_header is not None: + policy_name = to_unicode(policy_header.decode("latin1")) + allow_import_path = False if policy_name is None: return self.default_policy() - - cls = _load_policy_class(policy_name, warning_only=True) + cls = self._load_policy_class( + policy_name, warning_only=True, allow_import_path=allow_import_path + ) return cls() if cls else self.default_policy() - def process_spider_output( - self, response: Response, result: Iterable[Any], spider: Spider - ) -> Iterable[Any]: - return (self._set_referer(r, response) for r in result) + def _load_policy_class( + self, + policy: str, + warning_only: bool = False, + *, + allow_import_path: bool = False, + ) -> type[ReferrerPolicy] | None: + """Load the :class:`ReferrerPolicy` class to use for *policy*. - async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: - async for r in result: - yield self._set_referer(r, response) + *policy* may be any of the following: - def _set_referer(self, r: Any, response: Response) -> Any: - if isinstance(r, Request): - referrer = self.policy(response, r).referrer(response.url, r.url) - if referrer is not None: - r.headers.setdefault("Referer", referrer) - return r + - A standard policy name, e.g. ``"no-referrer"``, + ``"origin-when-cross-origin"``, etc. - def request_scheduled(self, request: Request, spider: Spider) -> None: - # check redirected request to patch "Referer" header if necessary - redirected_urls = request.meta.get("redirect_urls", []) - if redirected_urls: - request_referrer = request.headers.get("Referer") - # we don't patch the referrer value if there is none - if request_referrer is not None: - # the request's referrer header value acts as a surrogate - # for the parent response URL - # - # Note: if the 3xx response contained a Referrer-Policy header, - # the information is not available using this hook - parent_url = safe_url_string(request_referrer) - policy_referrer = self.policy(parent_url, request).referrer( - parent_url, request.url - ) - if policy_referrer != request_referrer.decode("latin1"): - if policy_referrer is None: - request.headers.pop("Referer") - else: - request.headers["Referer"] = policy_referrer + - The special ``"scrapy-default"`` policy. + + - The import path of a :class:`ReferrerPolicy` subclass, e.g. + ``"scrapy.spidermiddlewares.referer.NoReferrerPolicy"`` or + ``"myproject.policies.CustomReferrerPolicy"``. + + If *warning_only* is ``False`` (default) and *policy* cannot be turned + into a :class:`ReferrerPolicy` subclass, a :exc:`RuntimeError` is + raised. If *warning_only* is ``True``, a warning is logged and ``None`` + is returned instead. + + If *allow_import_path* is ``False`` (default), import paths are not + allowed, resulting in :exc:`RuntimeError` or ``None``. If ``True``, + they are allowed. Use ``True`` only if you trust the source of the + *policy* value. + """ + if allow_import_path: + try: + return cast("type[ReferrerPolicy]", load_object(policy)) + except ValueError: + pass + policy_names = [ + policy_name.strip() for policy_name in policy.lower().split(",") + ] + # https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header + for policy_name in policy_names[::-1]: + if policy_name in self.policies: + return self.policies[policy_name] + msg = f"Could not load referrer policy {policy!r}" + if not allow_import_path and _looks_like_import_path(policy): + msg += " (import paths from the response Referrer-Policy header are not allowed)" + if not warning_only: + raise RuntimeError(msg) + warnings.warn(msg, RuntimeWarning, stacklevel=2) + return None + + def get_processed_request( + self, request: Request, response: Response | None + ) -> Request | None: + if response is None: + # start requests + return request + referrer = self.policy(response, request).referrer(response.url, request.url) + if referrer is not None: + request.headers.setdefault("Referer", referrer) + return request diff --git a/scrapy/spidermiddlewares/start.py b/scrapy/spidermiddlewares/start.py new file mode 100644 index 000000000..5d76b60d2 --- /dev/null +++ b/scrapy/spidermiddlewares/start.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from .base import BaseSpiderMiddleware + +if TYPE_CHECKING: + from scrapy.http import Request + from scrapy.http.response import Response + + +class StartSpiderMiddleware(BaseSpiderMiddleware): + """Set :reqmeta:`is_start_request`. + + .. reqmeta:: is_start_request + + is_start_request + ---------------- + + :attr:`~scrapy.Request.meta` key that is set to ``True`` in :ref:`start + requests `, allowing you to tell start requests apart from + other requests, e.g. in :ref:`downloader middlewares + `. + """ + + def get_processed_request( + self, request: Request, response: Response | None + ) -> Request | None: + if response is None: + request.meta.setdefault("is_start_request", True) + return request diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index e2aa554a7..f325ce7a0 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -7,53 +7,47 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable +from typing import TYPE_CHECKING -from scrapy import Spider from scrapy.exceptions import NotConfigured -from scrapy.http import Request, Response -from scrapy.settings import BaseSettings +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Request, Response + + logger = logging.getLogger(__name__) -class UrlLengthMiddleware: - def __init__(self, maxlength: int): +class UrlLengthMiddleware(BaseSpiderMiddleware): + crawler: Crawler + + def __init__(self, maxlength: int): # pylint: disable=super-init-not-called self.maxlength: int = maxlength @classmethod - def from_settings(cls, settings: BaseSettings) -> Self: - maxlength = settings.getint("URLLENGTH_LIMIT") + def from_crawler(cls, crawler: Crawler) -> Self: + maxlength = crawler.settings.getint("URLLENGTH_LIMIT") if not maxlength: raise NotConfigured - return cls(maxlength) + o = cls(maxlength) + o.crawler = crawler + return o - def process_spider_output( - self, response: Response, result: Iterable[Any], spider: Spider - ) -> Iterable[Any]: - return (r for r in result if self._filter(r, spider)) - - async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: - async for r in result: - if self._filter(r, spider): - yield r - - def _filter(self, request: Any, spider: Spider) -> bool: - if isinstance(request, Request) and len(request.url) > self.maxlength: - logger.info( - "Ignoring link (url length > %(maxlength)d): %(url)s ", - {"maxlength": self.maxlength, "url": request.url}, - extra={"spider": spider}, - ) - assert spider.crawler.stats - spider.crawler.stats.inc_value( - "urllength/request_ignored_count", spider=spider - ) - return False - return True + def get_processed_request( + self, request: Request, response: Response | None + ) -> Request | None: + if len(request.url) <= self.maxlength: + return request + logger.info( + "Ignoring link (url length > %(maxlength)d): %(url)s ", + {"maxlength": self.maxlength, "url": request.url}, + extra={"spider": self.crawler.spider}, + ) + assert self.crawler.stats + self.crawler.stats.inc_value("urllength/request_ignored_count") + return None diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index e16d71727..299a5d43f 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -3,12 +3,11 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ + from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union, cast - -from twisted.internet.defer import Deferred +from typing import TYPE_CHECKING, Any, cast from scrapy import signals from scrapy.http import Request, Response @@ -16,34 +15,49 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self from scrapy.crawler import Crawler - from scrapy.settings import BaseSettings + from scrapy.http.request import CallbackT + from scrapy.settings import BaseSettings, _SettingsKey + from scrapy.utils.log import SpiderLoggerAdapter class Spider(object_ref): - """Base class for scrapy spiders. All spiders must inherit from this - class. + """Base class that any spider must subclass. + + It provides a default :meth:`start` implementation that sends + requests based on the :attr:`start_urls` class attribute and calls the + :meth:`parse` method for each response. """ name: str - custom_settings: Optional[dict] = None + custom_settings: dict[_SettingsKey, Any] | None = None - def __init__(self, name: Optional[str] = None, **kwargs: Any): + #: Start URLs. See :meth:`start`. + start_urls: list[str] + + def __init__(self, name: str | None = None, **kwargs: Any): if name is not None: - self.name = name + self.name: str = name elif not getattr(self, "name", None): raise ValueError(f"{type(self).__name__} must have a name") self.__dict__.update(kwargs) if not hasattr(self, "start_urls"): - self.start_urls: List[str] = [] + self.start_urls: list[str] = [] @property - def logger(self) -> logging.LoggerAdapter: + def logger(self) -> SpiderLoggerAdapter: + # circular import + from scrapy.utils.log import SpiderLoggerAdapter # noqa: PLC0415 + logger = logging.getLogger(self.name) - return logging.LoggerAdapter(logger, {"spider": self}) + return SpiderLoggerAdapter(logger, {"spider": self}) def log(self, message: Any, level: int = logging.DEBUG, **kw: Any) -> None: """Log the given message at the given log level @@ -61,11 +75,56 @@ class Spider(object_ref): return spider def _set_crawler(self, crawler: Crawler) -> None: - self.crawler = crawler - self.settings = crawler.settings + self.crawler: Crawler = crawler + self.settings: BaseSettings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) - def start_requests(self) -> Iterable[Request]: + async def start(self) -> AsyncIterator[Any]: + """Yield the initial :class:`~scrapy.Request` objects to send. + + .. versionadded:: 2.13 + + For example: + + .. code-block:: python + + from scrapy import Request, Spider + + + class MySpider(Spider): + name = "myspider" + + async def start(self): + yield Request("https://toscrape.com/") + + The default implementation reads URLs from :attr:`start_urls` and + yields a request for each with :attr:`~scrapy.Request.dont_filter` + enabled. It is functionally equivalent to: + + .. code-block:: python + + async def start(self): + for url in self.start_urls: + yield Request(url, dont_filter=True) + + You can also yield :ref:`items `. For example: + + .. code-block:: python + + async def start(self): + yield {"foo": "bar"} + + To write spiders that work on Scrapy versions lower than 2.13, + define also a synchronous ``start_requests()`` method that returns an + iterable. For example: + + .. code-block:: python + + def start_requests(self): + yield Request("https://toscrape.com/") + + .. seealso:: :ref:`start-requests` + """ if not self.start_urls and hasattr(self, "start_url"): raise AttributeError( "Crawling could not start: 'start_urls' not found " @@ -78,10 +137,14 @@ class Spider(object_ref): def _parse(self, response: Response, **kwargs: Any) -> Any: return self.parse(response, **kwargs) - def parse(self, response: Response, **kwargs: Any) -> Any: - raise NotImplementedError( - f"{self.__class__.__name__}.parse callback is not defined" - ) + if TYPE_CHECKING: + parse: CallbackT + else: + + def parse(self, response: Response, **kwargs: Any) -> Any: + raise NotImplementedError( + f"{self.__class__.__name__}.parse callback is not defined" + ) @classmethod def update_settings(cls, settings: BaseSettings) -> None: @@ -92,10 +155,10 @@ class Spider(object_ref): return url_is_from_spider(request.url, cls) @staticmethod - def close(spider: Spider, reason: str) -> Union[Deferred, None]: + def close(spider: Spider, reason: str) -> Deferred[None] | None: closed = getattr(spider, "closed", None) if callable(closed): - return cast(Union[Deferred, None], closed(reason)) + return cast("Deferred[None] | None", closed(reason)) return None def __repr__(self) -> str: @@ -106,3 +169,12 @@ class Spider(object_ref): from scrapy.spiders.crawl import CrawlSpider, Rule from scrapy.spiders.feed import CSVFeedSpider, XMLFeedSpider from scrapy.spiders.sitemap import SitemapSpider + +__all__ = [ + "CSVFeedSpider", + "CrawlSpider", + "Rule", + "SitemapSpider", + "Spider", + "XMLFeedSpider", +] diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index ebb4f5984..e14279b64 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -1,33 +1,59 @@ """ This modules implements the CrawlSpider which is the recommended spider to use -for scraping typical web sites that requires crawling pages. +for scraping typical websites that requires crawling pages. See documentation in docs/topics/spiders.rst """ +from __future__ import annotations + import copy -from typing import AsyncIterable, Awaitable, Sequence +import warnings +from collections.abc import AsyncIterator, Awaitable, Callable +from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast from scrapy.http import HtmlResponse, Request, Response +from scrapy.link import Link from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen +from scrapy.utils.deprecate import method_is_overridden +from scrapy.utils.python import global_object_name from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + from collections.abc import Iterable, Sequence -def _identity(x): + from twisted.python.failure import Failure + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.http.request import CallbackT + + +_T = TypeVar("_T") +ProcessLinksT: TypeAlias = Callable[[list[Link]], list[Link]] +ProcessRequestT: TypeAlias = Callable[[Request, Response], Request | None] + + +def _identity(x: _T) -> _T: return x -def _identity_process_request(request, response): +def _identity_process_request(request: Request, response: Response) -> Request | None: return request -def _get_method(method, spider): +def _get_method( + method: Callable[..., Any] | str | None, spider: Spider +) -> Callable[..., Any] | None: if callable(method): return method if isinstance(method, str): return getattr(spider, method, None) + return None _default_link_extractor = LinkExtractor() @@ -36,87 +62,120 @@ _default_link_extractor = LinkExtractor() class Rule: def __init__( self, - link_extractor=None, - callback=None, - cb_kwargs=None, - follow=None, - process_links=None, - process_request=None, - errback=None, + link_extractor: LinkExtractor | None = None, + callback: CallbackT | str | None = None, + cb_kwargs: dict[str, Any] | None = None, + follow: bool | None = None, + process_links: ProcessLinksT | str | None = None, + process_request: ProcessRequestT | str | None = None, + errback: Callable[[Failure], Any] | str | None = None, ): - self.link_extractor = link_extractor or _default_link_extractor - self.callback = callback - self.errback = errback - self.cb_kwargs = cb_kwargs or {} - self.process_links = process_links or _identity - self.process_request = process_request or _identity_process_request - self.follow = follow if follow is not None else not callback + self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor + self.callback: CallbackT | str | None = callback + self.errback: Callable[[Failure], Any] | str | None = errback + self.cb_kwargs: dict[str, Any] = cb_kwargs or {} + self.process_links: ProcessLinksT | str = process_links or _identity + self.process_request: ProcessRequestT | str = ( + process_request or _identity_process_request + ) + self.follow: bool = follow if follow is not None else not callback - def _compile(self, spider): - self.callback = _get_method(self.callback, spider) - self.errback = _get_method(self.errback, spider) - self.process_links = _get_method(self.process_links, spider) - self.process_request = _get_method(self.process_request, spider) + def _compile(self, spider: Spider) -> None: + # this replaces method names with methods and we can't express this in type hints + self.callback = cast("CallbackT", _get_method(self.callback, spider)) + self.errback = cast( + "Callable[[Failure], Any]", _get_method(self.errback, spider) + ) + self.process_links = cast( + "ProcessLinksT", _get_method(self.process_links, spider) + ) + self.process_request = cast( + "ProcessRequestT", _get_method(self.process_request, spider) + ) class CrawlSpider(Spider): rules: Sequence[Rule] = () + _rules: list[Rule] + _follow_links: bool - def __init__(self, *a, **kw): + def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) self._compile_rules() + if method_is_overridden(self.__class__, CrawlSpider, "_parse_response"): + warnings.warn( + "The CrawlSpider._parse_response method, which the " + f"{global_object_name(self.__class__)} class overrides, is " + "deprecated: it will be removed in future Scrapy releases. " + "Please override the CrawlSpider.parse_with_rules method " + "instead.", + stacklevel=2, + ) - def _parse(self, response, **kwargs): - return self._parse_response( + def _parse(self, response: Response, **kwargs: Any) -> Any: + return self.parse_with_rules( response=response, callback=self.parse_start_url, cb_kwargs=kwargs, follow=True, ) - def parse_start_url(self, response, **kwargs): - return [] + def parse_start_url(self, response: Response, **kwargs: Any) -> Any: + return () - def process_results(self, response: Response, results: list): + def process_results( + self, response: Response, results: Iterable[Any] + ) -> Iterable[Any]: return results - def _build_request(self, rule_index, link): + def _build_request(self, rule_index: int, link: Link) -> Request: return Request( url=link.url, callback=self._callback, errback=self._errback, - meta=dict(rule=rule_index, link_text=link.text), + meta={"rule": rule_index, "link_text": link.text}, ) - def _requests_to_follow(self, response): + def _requests_to_follow(self, response: Response) -> Iterable[Request | None]: if not isinstance(response, HtmlResponse): return - seen = set() + seen: set[Link] = set() for rule_index, rule in enumerate(self._rules): - links = [ + links: list[Link] = [ lnk for lnk in rule.link_extractor.extract_links(response) if lnk not in seen ] - for link in rule.process_links(links): + for link in cast("ProcessLinksT", rule.process_links)(links): seen.add(link) request = self._build_request(rule_index, link) - yield rule.process_request(request, response) + yield cast("ProcessRequestT", rule.process_request)(request, response) - def _callback(self, response, **cb_kwargs): - rule = self._rules[response.meta["rule"]] - return self._parse_response( - response, rule.callback, {**rule.cb_kwargs, **cb_kwargs}, rule.follow + def _callback(self, response: Response, **cb_kwargs: Any) -> Any: + rule = self._rules[cast("int", response.meta["rule"])] + return self.parse_with_rules( + response, + cast("CallbackT", rule.callback), + {**rule.cb_kwargs, **cb_kwargs}, + rule.follow, ) - def _errback(self, failure): - rule = self._rules[failure.request.meta["rule"]] - return self._handle_failure(failure, rule.errback) + def _errback(self, failure: Failure) -> Iterable[Any]: + rule = self._rules[cast("int", failure.request.meta["rule"])] # type: ignore[attr-defined] + return self._handle_failure( + failure, cast("Callable[[Failure], Any]", rule.errback) + ) - async def _parse_response(self, response, callback, cb_kwargs, follow=True): + async def parse_with_rules( + self, + response: Response, + callback: CallbackT | None, + cb_kwargs: dict[str, Any], + follow: bool = True, + ) -> AsyncIterator[Any]: if callback: cb_res = callback(response, **cb_kwargs) or () - if isinstance(cb_res, AsyncIterable): + if isinstance(cb_res, AsyncIterator): cb_res = await collect_asyncgen(cb_res) elif isinstance(cb_res, Awaitable): cb_res = await cb_res @@ -128,21 +187,37 @@ class CrawlSpider(Spider): for request_or_item in self._requests_to_follow(response): yield request_or_item - def _handle_failure(self, failure, errback): + def _parse_response( + self, + response: Response, + callback: CallbackT | None, + cb_kwargs: dict[str, Any], + follow: bool = True, + ) -> AsyncIterator[Any]: + warnings.warn( + "The CrawlSpider._parse_response method is deprecated: " + "it will be removed in future Scrapy releases. " + "Please use the CrawlSpider.parse_with_rules method instead.", + stacklevel=2, + ) + return self.parse_with_rules(response, callback, cb_kwargs, follow) + + def _handle_failure( + self, failure: Failure, errback: Callable[[Failure], Any] | None + ) -> Iterable[Any]: if errback: results = errback(failure) or () yield from iterate_spider_output(results) - def _compile_rules(self): + def _compile_rules(self) -> None: self._rules = [] for rule in self.rules: - self._rules.append(copy.copy(rule)) - self._rules[-1]._compile(self) + copied_rule = copy.copy(rule) + copied_rule._compile(self) + self._rules.append(copied_rule) @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = super().from_crawler(crawler, *args, **kwargs) - spider._follow_links = crawler.settings.getbool( - "CRAWLSPIDER_FOLLOW_LINKS", True - ) + spider._follow_links = crawler.settings.getbool("CRAWLSPIDER_FOLLOW_LINKS") return spider diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 47827e442..395183613 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -4,12 +4,21 @@ for scraping from an XML feed. See documentation in docs/topics/spiders.rst """ + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + from scrapy.exceptions import NotConfigured, NotSupported +from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.spiders import Spider -from scrapy.utils.iterators import csviter, xmliter +from scrapy.utils.iterators import csviter, xmliter_lxml from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + class XMLFeedSpider(Spider): """ @@ -21,11 +30,13 @@ class XMLFeedSpider(Spider): use iternodes, since it's a faster and cleaner. """ - iterator = "iternodes" - itertag = "item" - namespaces = () + iterator: str = "iternodes" + itertag: str = "item" + namespaces: Sequence[tuple[str, str]] = () - def process_results(self, response, results): + def process_results( + self, response: Response, results: Iterable[Any] + ) -> Iterable[Any]: """This overridable method is called for each result (item or request) returned by the spider, and it's intended to perform any last time processing required before returning the results to the framework core, @@ -35,20 +46,20 @@ class XMLFeedSpider(Spider): """ return results - def adapt_response(self, response): + def adapt_response(self, response: Response) -> Response: """You can override this function in order to make any changes you want to into the feed before parsing it. This function must return a response. """ return response - def parse_node(self, response, selector): + def parse_node(self, response: Response, selector: Selector) -> Any: """This method must be overridden with your custom spider functionality""" if hasattr(self, "parse_item"): # backward compatibility return self.parse_item(response, selector) raise NotImplementedError - def parse_nodes(self, response, nodes): + def parse_nodes(self, response: Response, nodes: Iterable[Selector]) -> Any: """This method is called for the nodes matching the provided tag name (itertag). Receives the response and an Selector for each node. Overriding this method is mandatory. Otherwise, you spider won't work. @@ -60,20 +71,25 @@ class XMLFeedSpider(Spider): ret = iterate_spider_output(self.parse_node(response, selector)) yield from self.process_results(response, ret) - def _parse(self, response, **kwargs): + def _parse(self, response: Response, **kwargs: Any) -> Any: if not hasattr(self, "parse_node"): raise NotConfigured( "You must define parse_node method in order to scrape this XML feed" ) response = self.adapt_response(response) + nodes: Iterable[Selector] if self.iterator == "iternodes": nodes = self._iternodes(response) elif self.iterator == "xml": + if not isinstance(response, TextResponse): + raise ValueError("Response content isn't text") selector = Selector(response, type="xml") self._register_namespaces(selector) nodes = selector.xpath(f"//{self.itertag}") elif self.iterator == "html": + if not isinstance(response, TextResponse): + raise ValueError("Response content isn't text") selector = Selector(response, type="html") self._register_namespaces(selector) nodes = selector.xpath(f"//{self.itertag}") @@ -82,12 +98,12 @@ class XMLFeedSpider(Spider): return self.parse_nodes(response, nodes) - def _iternodes(self, response): - for node in xmliter(response, self.itertag): + def _iternodes(self, response: Response) -> Iterable[Selector]: + for node in xmliter_lxml(response, self.itertag): self._register_namespaces(node) yield node - def _register_namespaces(self, selector): + def _register_namespaces(self, selector: Selector) -> None: for prefix, uri in self.namespaces: selector.register_namespace(prefix, uri) @@ -101,27 +117,29 @@ class CSVFeedSpider(Spider): and the file's headers. """ - delimiter = ( + delimiter: str | None = ( None # When this is None, python's csv module's default delimiter is used ) - quotechar = ( + quotechar: str | None = ( None # When this is None, python's csv module's default quotechar is used ) - headers = None + headers: list[str] | None = None - def process_results(self, response, results): + def process_results( + self, response: Response, results: Iterable[Any] + ) -> Iterable[Any]: """This method has the same purpose as the one in XMLFeedSpider""" return results - def adapt_response(self, response): + def adapt_response(self, response: Response) -> Response: """This method has the same purpose as the one in XMLFeedSpider""" return response - def parse_row(self, response, row): + def parse_row(self, response: Response, row: dict[str, str]) -> Any: """This method must be overridden with your custom spider functionality""" raise NotImplementedError - def parse_rows(self, response): + def parse_rows(self, response: Response) -> Any: """Receives a response and a dict (representing each row) with a key for each provided (or detected) header of the CSV file. This spider also gives the opportunity to override adapt_response and @@ -134,7 +152,7 @@ class CSVFeedSpider(Spider): ret = iterate_spider_output(self.parse_row(response, row)) yield from self.process_results(response, ret) - def _parse(self, response, **kwargs): + def _parse(self, response: Response, **kwargs: Any) -> Any: if not hasattr(self, "parse_row"): raise NotConfigured( "You must define parse_row method in order to scrape this CSV feed" diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py deleted file mode 100644 index 3cb215b0f..000000000 --- a/scrapy/spiders/init.py +++ /dev/null @@ -1,31 +0,0 @@ -from scrapy.spiders import Spider -from scrapy.utils.spider import iterate_spider_output - - -class InitSpider(Spider): - """Base Spider with initialization facilities""" - - def start_requests(self): - self._postinit_reqs = super().start_requests() - return iterate_spider_output(self.init_request()) - - def initialized(self, response=None): - """This method must be set as the callback of your last initialization - request. See self.init_request() docstring for more info. - """ - return self.__dict__.pop("_postinit_reqs") - - def init_request(self): - """This function should return one initialization request, with the - self.initialized method as callback. When the self.initialized method - is called this spider is considered initialized. If you need to perform - several requests for initializing your spider, you can do so by using - different callbacks. The only requirement is that the final callback - (of the last initialization request) must be self.initialized. - - The default implementation calls self.initialized immediately, and - means that no initialization is needed. This method should be - overridden only when you need to perform requests to initialize your - spider - """ - return self.initialized() diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 974665fe0..2a80b8d24 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -1,76 +1,141 @@ +from __future__ import annotations + import logging import re -from scrapy.http import Request, XmlResponse +# Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation +from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003 +from typing import TYPE_CHECKING, Any, cast + +from scrapy.http import Request, Response, XmlResponse from scrapy.spiders import Spider +from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.gz import gunzip, gzip_magic_number from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.http.request import CallbackT + logger = logging.getLogger(__name__) class SitemapSpider(Spider): - sitemap_urls = () - sitemap_rules = [("", "parse")] - sitemap_follow = [""] - sitemap_alternate_links = False + sitemap_urls: Sequence[str] = () + sitemap_rules: Sequence[tuple[re.Pattern[str] | str, str | CallbackT]] = [ + ("", "parse") + ] + sitemap_follow: Sequence[re.Pattern[str] | str] = [""] + sitemap_alternate_links: bool = False + _max_size: int + _warn_size: int - def __init__(self, *a, **kw): + @classmethod + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: + spider = super().from_crawler(crawler, *args, **kwargs) + spider._max_size = getattr( + spider, "download_maxsize", spider.settings.getint("DOWNLOAD_MAXSIZE") + ) + spider._warn_size = getattr( + spider, "download_warnsize", spider.settings.getint("DOWNLOAD_WARNSIZE") + ) + return spider + + def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) - self._cbs = [] + self._cbs: list[tuple[re.Pattern[str], CallbackT]] = [] for r, c in self.sitemap_rules: if isinstance(c, str): - c = getattr(self, c) + c = cast("CallbackT", getattr(self, c)) # noqa: PLW2901 self._cbs.append((regex(r), c)) - self._follow = [regex(x) for x in self.sitemap_follow] + self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] - def start_requests(self): + async def start(self) -> AsyncIterator[Any]: for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) - def sitemap_filter(self, entries): + def sitemap_filter( + self, entries: Iterable[dict[str, Any]] + ) -> Iterable[dict[str, Any]]: """This method can be used to filter sitemap entries by their attributes, for example, you can filter locs with lastmod greater than a given date (see docs). """ yield from entries - def _parse_sitemap(self, response): + def _parse_sitemap(self, response: Response) -> Iterable[Request]: if response.url.endswith("/robots.txt"): - for url in sitemap_urls_from_robots(response.text, base_url=response.url): - yield Request(url, callback=self._parse_sitemap) - else: - body = self._get_sitemap_body(response) - if body is None: - logger.warning( - "Ignoring invalid sitemap: %(response)s", - {"response": response}, - extra={"spider": self}, - ) - return + urls = list(sitemap_urls_from_robots(response.body, base_url=response.url)) + return (Request(url, callback=self._parse_sitemap) for url in urls) - s = Sitemap(body) - it = self.sitemap_filter(s) + body = self._get_sitemap_body(response) + if not body: + logger.warning( + "Ignoring invalid sitemap: %(response)s", + {"response": response}, + extra={"spider": self}, + ) + return () - if s.type == "sitemapindex": - for loc in iterloc(it, self.sitemap_alternate_links): - if any(x.search(loc) for x in self._follow): - yield Request(loc, callback=self._parse_sitemap) - elif s.type == "urlset": - for loc in iterloc(it, self.sitemap_alternate_links): - for r, c in self._cbs: - if r.search(loc): - yield Request(loc, callback=c) - break + s = Sitemap(body) - def _get_sitemap_body(self, response): + if s.type == "sitemapindex": + urls = list(self._get_urls_from_sitemapindex(self.sitemap_filter(s))) + return (Request(loc, callback=self._parse_sitemap) for loc in urls) + + if s.type == "urlset": + url_callback_pairs = list( + self._get_urls_and_callbacks_from_urlset(self.sitemap_filter(s)) + ) + return (Request(loc, callback=c) for loc, c in url_callback_pairs) + + logger.warning( + "Ignoring invalid sitemap: %(response)s", + {"response": response}, + extra={"spider": self}, + ) + + return () + + def _get_urls_from_sitemapindex( + self, it: Iterable[dict[str, Any]] + ) -> Iterable[str]: + for loc in iterloc(it, self.sitemap_alternate_links): + if any(x.search(loc) for x in self._follow): + yield loc + + def _get_urls_and_callbacks_from_urlset( + self, it: Iterable[dict[str, Any]] + ) -> Iterable[tuple[str, CallbackT]]: + for loc in iterloc(it, self.sitemap_alternate_links): + for r, c in self._cbs: + if r.search(loc): + yield loc, c + break + + def _get_sitemap_body(self, response: Response) -> bytes | None: """Return the sitemap body contained in the given response, or None if the response is not a sitemap. """ if isinstance(response, XmlResponse): return response.body if gzip_magic_number(response): - return gunzip(response.body) + uncompressed_size = len(response.body) + max_size = response.meta.get("download_maxsize", self._max_size) + warn_size = response.meta.get("download_warnsize", self._warn_size) + try: + body = gunzip(response.body, max_size=max_size) + except _DecompressionMaxSizeExceeded: + return None + if uncompressed_size < warn_size <= len(body): + logger.warning( + f"{response} body size after decompression ({len(body)} B) " + f"is larger than the download warning size ({warn_size} B)." + ) + return body # actual gzipped sitemap files are decompressed above ; # if we are here (response body is not gzipped) # and have a response for .xml.gz, @@ -82,18 +147,20 @@ class SitemapSpider(Spider): # in other word, here, we have plain XML if response.url.endswith(".xml") or response.url.endswith(".xml.gz"): return response.body + return None -def regex(x): +def regex(x: re.Pattern[str] | str) -> re.Pattern[str]: if isinstance(x, str): return re.compile(x) return x -def iterloc(it, alt=False): +def iterloc(it: Iterable[dict[str, Any]], alt: bool = False) -> Iterable[str]: for d in it: - yield d["loc"] + if loc := d["loc"]: + yield loc # Also consider alternate URLs (xhtml:link rel="alternate") - if alt and "alternate" in d: - yield from d["alternate"] + if alt and (alt_list := d.get("alternate")): + yield from alt_list diff --git a/scrapy/squeues.py b/scrapy/squeues.py index f665ad88c..e06e38e23 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -2,20 +2,31 @@ Scheduler queues """ +from __future__ import annotations + import marshal import pickle -from os import PathLike from pathlib import Path -from typing import Union +from typing import TYPE_CHECKING, Any from queuelib import queue from scrapy.utils.request import request_from_dict +if TYPE_CHECKING: + from collections.abc import Callable + from os import PathLike -def _with_mkdir(queue_class): - class DirectoriesCreated(queue_class): - def __init__(self, path: Union[str, PathLike], *args, **kwargs): + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy import Request + from scrapy.crawler import Crawler + + +def _with_mkdir(queue_class: type[queue.BaseQueue]) -> type[queue.BaseQueue]: + class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc] + def __init__(self, path: str | PathLike[str], *args: Any, **kwargs: Any): dirname = Path(path).parent if not dirname.exists(): dirname.mkdir(parents=True, exist_ok=True) @@ -24,18 +35,23 @@ def _with_mkdir(queue_class): return DirectoriesCreated -def _serializable_queue(queue_class, serialize, deserialize): - class SerializableQueue(queue_class): - def push(self, obj): +def _serializable_queue( + queue_class: type[queue.BaseQueue], + serialize: Callable[[Any], bytes], + deserialize: Callable[[bytes], Any], +) -> type[queue.BaseQueue]: + class SerializableQueue(queue_class): # type: ignore[valid-type,misc] + def push(self, obj: Any) -> None: s = serialize(obj) super().push(s) - def pop(self): + def pop(self) -> Any | None: s = super().pop() if s: return deserialize(s) + return None - def peek(self): + def peek(self) -> Any | None: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -50,31 +66,36 @@ def _serializable_queue(queue_class, serialize, deserialize): ) from ex if s: return deserialize(s) + return None return SerializableQueue -def _scrapy_serialization_queue(queue_class): - class ScrapyRequestQueue(queue_class): - def __init__(self, crawler, key): +def _scrapy_serialization_queue( + queue_class: type[queue.BaseQueue], +) -> type[queue.BaseQueue]: + class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc] + def __init__(self, crawler: Crawler, key: str): self.spider = crawler.spider super().__init__(key) @classmethod - def from_crawler(cls, crawler, key, *args, **kwargs): + def from_crawler( + cls, crawler: Crawler, key: str, *args: Any, **kwargs: Any + ) -> Self: return cls(crawler, key) - def push(self, request): - request = request.to_dict(spider=self.spider) - return super().push(request) + def push(self, request: Request) -> None: + request_dict = request.to_dict(spider=self.spider) + super().push(request_dict) - def pop(self): + def pop(self) -> Request | None: request = super().pop() if not request: return None return request_from_dict(request, spider=self.spider) - def peek(self): + def peek(self) -> Request | None: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -89,13 +110,15 @@ def _scrapy_serialization_queue(queue_class): return ScrapyRequestQueue -def _scrapy_non_serialization_queue(queue_class): - class ScrapyRequestQueue(queue_class): +def _scrapy_non_serialization_queue( + queue_class: type[queue.BaseQueue], +) -> type[queue.BaseQueue]: + class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc] @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: return cls() - def peek(self): + def peek(self) -> Any | None: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -113,7 +136,7 @@ def _scrapy_non_serialization_queue(queue_class): return ScrapyRequestQueue -def _pickle_serialize(obj): +def _pickle_serialize(obj: Any) -> bytes: try: return pickle.dumps(obj, protocol=4) # Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s) @@ -122,17 +145,26 @@ def _pickle_serialize(obj): raise ValueError(str(e)) from e +# queue.*Queue aren't subclasses of queue.BaseQueue _PickleFifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads + _with_mkdir(queue.FifoDiskQueue), # type: ignore[arg-type] + _pickle_serialize, + pickle.loads, ) _PickleLifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads + _with_mkdir(queue.LifoDiskQueue), # type: ignore[arg-type] + _pickle_serialize, + pickle.loads, ) _MarshalFifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads + _with_mkdir(queue.FifoDiskQueue), # type: ignore[arg-type] + marshal.dumps, + marshal.loads, ) _MarshalLifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads + _with_mkdir(queue.LifoDiskQueue), # type: ignore[arg-type] + marshal.dumps, + marshal.loads, ) # public queue classes @@ -140,5 +172,5 @@ PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQu PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue) MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue) MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue) -FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) -LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) +FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) # type: ignore[arg-type] +LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) # type: ignore[arg-type] diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 15193aac5..1f9894519 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -1,98 +1,134 @@ """ Scrapy extension for collecting scraping stats """ + +from __future__ import annotations + import logging import pprint -from typing import TYPE_CHECKING, Any, Dict, Optional +from typing import TYPE_CHECKING, Any -from scrapy import Spider +from scrapy.utils.decorators import _warn_spider_arg if TYPE_CHECKING: + from scrapy import Spider from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) -StatsT = Dict[str, Any] +StatsT = dict[str, Any] class StatsCollector: - def __init__(self, crawler: "Crawler"): + def __init__(self, crawler: Crawler): self._dump: bool = crawler.settings.getbool("STATS_DUMP") self._stats: StatsT = {} + self._crawler: Crawler = crawler + + def __getattribute__(self, name: str) -> Any: + cached_name = f"_cached_{name}" + try: + return super().__getattribute__(cached_name) + except AttributeError: + pass + + original_attr = super().__getattribute__(name) + + if name in { + "get_value", + "get_stats", + "set_value", + "set_stats", + "inc_value", + "max_value", + "min_value", + "clear_stats", + "open_spider", + "close_spider", + } and callable(original_attr): + wrapped = _warn_spider_arg(original_attr) + setattr(self, cached_name, wrapped) + return wrapped + + return original_attr def get_value( - self, key: str, default: Any = None, spider: Optional[Spider] = None + self, key: str, default: Any = None, spider: Spider | None = None ) -> Any: return self._stats.get(key, default) - def get_stats(self, spider: Optional[Spider] = None) -> StatsT: + def get_stats(self, spider: Spider | None = None) -> StatsT: return self._stats - def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def set_value(self, key: str, value: Any, spider: Spider | None = None) -> None: self._stats[key] = value - def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None: + def set_stats(self, stats: StatsT, spider: Spider | None = None) -> None: self._stats = stats def inc_value( - self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None + self, key: str, count: int = 1, start: int = 0, spider: Spider | None = None ) -> None: d = self._stats d[key] = d.setdefault(key, start) + count - def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def max_value(self, key: str, value: Any, spider: Spider | None = None) -> None: self._stats[key] = max(self._stats.setdefault(key, value), value) - def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def min_value(self, key: str, value: Any, spider: Spider | None = None) -> None: self._stats[key] = min(self._stats.setdefault(key, value), value) - def clear_stats(self, spider: Optional[Spider] = None) -> None: + def clear_stats(self, spider: Spider | None = None) -> None: self._stats.clear() - def open_spider(self, spider: Spider) -> None: + def open_spider(self, spider: Spider | None = None) -> None: pass - def close_spider(self, spider: Spider, reason: str) -> None: + def close_spider( + self, spider: Spider | None = None, reason: str | None = None + ) -> None: if self._dump: logger.info( "Dumping Scrapy stats:\n" + pprint.pformat(self._stats), - extra={"spider": spider}, + extra={"spider": self._crawler.spider}, ) - self._persist_stats(self._stats, spider) + self._persist_stats(self._stats) - def _persist_stats(self, stats: StatsT, spider: Spider) -> None: + def _persist_stats(self, stats: StatsT) -> None: pass class MemoryStatsCollector(StatsCollector): - def __init__(self, crawler: "Crawler"): + def __init__(self, crawler: Crawler): super().__init__(crawler) - self.spider_stats: Dict[str, StatsT] = {} + self.spider_stats: dict[str, StatsT] = {} - def _persist_stats(self, stats: StatsT, spider: Spider) -> None: - self.spider_stats[spider.name] = stats + def _persist_stats(self, stats: StatsT) -> None: + if self._crawler.spider: + self.spider_stats[self._crawler.spider.name] = stats class DummyStatsCollector(StatsCollector): def get_value( - self, key: str, default: Any = None, spider: Optional[Spider] = None + self, key: str, default: Any = None, spider: Spider | None = None ) -> Any: return default - def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def set_value(self, key: str, value: Any, spider: Spider | None = None) -> None: pass - def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None: + def set_stats(self, stats: StatsT, spider: Spider | None = None) -> None: pass def inc_value( - self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None + self, key: str, count: int = 1, start: int = 0, spider: Spider | None = None ) -> None: pass - def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def max_value(self, key: str, value: Any, spider: Spider | None = None) -> None: pass - def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def min_value(self, key: str, value: Any, spider: Spider | None = None) -> None: pass diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl index 88a18331c..e7d525f36 100644 --- a/scrapy/templates/project/module/items.py.tmpl +++ b/scrapy/templates/project/module/items.py.tmpl @@ -3,10 +3,11 @@ # See documentation in: # https://docs.scrapy.org/en/latest/topics/items.html -import scrapy +from dataclasses import dataclass -class ${ProjectName}Item(scrapy.Item): +@dataclass +class ${ProjectName}Item: # define the fields for your item here like: - # name = scrapy.Field() + # name: str | None = None pass diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 8c9a86dce..a0b5910e0 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -6,7 +6,7 @@ from scrapy import signals # useful for handling different item types with a single interface -from itemadapter import is_item, ItemAdapter +from itemadapter import ItemAdapter class ${ProjectName}SpiderMiddleware: @@ -43,14 +43,11 @@ class ${ProjectName}SpiderMiddleware: # Should return either None or an iterable of Request or item objects. pass - def process_start_requests(self, start_requests, spider): - # Called with the start requests of the spider, and works - # similarly to the process_spider_output() method, except - # that it doesn’t have a response associated. - - # Must return only requests (not items). - for r in start_requests: - yield r + async def process_start(self, start): + # Called with an async iterator over the spider start() method or the + # matching method of an earlier spider middleware. + async for item_or_request in start: + yield item_or_request def spider_opened(self, spider): spider.logger.info("Spider opened: %s" % spider.name) diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index b4779e555..0432a7231 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -12,6 +12,8 @@ BOT_NAME = "$project_name" SPIDER_MODULES = ["$project_name.spiders"] NEWSPIDER_MODULE = "$project_name.spiders" +ADDONS = {} + # Crawl responsibly by identifying yourself (and your website) on the user-agent #USER_AGENT = "$project_name (+http://www.yourdomain.com)" @@ -19,16 +21,10 @@ NEWSPIDER_MODULE = "$project_name.spiders" # Obey robots.txt rules ROBOTSTXT_OBEY = True -# Configure maximum concurrent requests performed by Scrapy (default: 16) -#CONCURRENT_REQUESTS = 32 - -# Configure a delay for requests for the same website (default: 0) -# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay -# See also autothrottle settings and docs -#DOWNLOAD_DELAY = 3 -# The download delay setting will honor only one of: -#CONCURRENT_REQUESTS_PER_DOMAIN = 16 -#CONCURRENT_REQUESTS_PER_IP = 16 +# Concurrency and throttling settings +#CONCURRENT_REQUESTS = 16 +CONCURRENT_REQUESTS_PER_DOMAIN = 1 +DOWNLOAD_DELAY = 1 # Disable cookies (enabled by default) #COOKIES_ENABLED = False @@ -88,5 +84,4 @@ ROBOTSTXT_OBEY = True #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" # Set settings whose default value is deprecated to a future-proof value -TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py new file mode 100644 index 000000000..4767c29f2 --- /dev/null +++ b/scrapy/utils/_compression.py @@ -0,0 +1,90 @@ +import contextlib +import zlib +from io import BytesIO + +with contextlib.suppress(ImportError): + try: + import brotli + except ImportError: + import brotlicffi as brotli + +with contextlib.suppress(ImportError): + import zstandard + + +_CHUNK_SIZE = 65536 # 64 KiB + + +class _DecompressionMaxSizeExceeded(ValueError): + def __init__(self, decompressed_size: int, max_size: int) -> None: + self.decompressed_size = decompressed_size + self.max_size = max_size + + def __str__(self) -> str: + return ( + f"The number of bytes decompressed so far " + f"({self.decompressed_size} B) exceeded the specified maximum " + f"({self.max_size} B)." + ) + + +def _check_max_size(decompressed_size: int, max_size: int) -> None: + if max_size and decompressed_size > max_size: + raise _DecompressionMaxSizeExceeded(decompressed_size, max_size) + + +def _inflate(data: bytes, *, max_size: int = 0) -> bytes: + decompressor = zlib.decompressobj() + try: + first_chunk = decompressor.decompress(data, max_length=_CHUNK_SIZE) + except zlib.error: + # to work with raw deflate content that may be sent by microsoft servers. + decompressor = zlib.decompressobj(wbits=-15) + first_chunk = decompressor.decompress(data, max_length=_CHUNK_SIZE) + decompressed_size = len(first_chunk) + _check_max_size(decompressed_size, max_size) + output_stream = BytesIO() + output_stream.write(first_chunk) + while decompressor.unconsumed_tail: + output_chunk = decompressor.decompress( + decompressor.unconsumed_tail, max_length=_CHUNK_SIZE + ) + decompressed_size += len(output_chunk) + _check_max_size(decompressed_size, max_size) + output_stream.write(output_chunk) + if tail := decompressor.flush(): + decompressed_size += len(tail) + _check_max_size(decompressed_size, max_size) + output_stream.write(tail) + return output_stream.getvalue() + + +def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: + decompressor = brotli.Decompressor() + first_chunk = decompressor.process(data, output_buffer_limit=_CHUNK_SIZE) + decompressed_size = len(first_chunk) + _check_max_size(decompressed_size, max_size) + output_stream = BytesIO() + output_stream.write(first_chunk) + while not decompressor.is_finished(): + output_chunk = decompressor.process(b"", output_buffer_limit=_CHUNK_SIZE) + if not output_chunk: + break + decompressed_size += len(output_chunk) + _check_max_size(decompressed_size, max_size) + output_stream.write(output_chunk) + return output_stream.getvalue() + + +def _unzstd(data: bytes, *, max_size: int = 0) -> bytes: + decompressor = zstandard.ZstdDecompressor() + stream_reader = decompressor.stream_reader(BytesIO(data)) + output_stream = BytesIO() + output_chunk = b"." + decompressed_size = 0 + while output_chunk: + output_chunk = stream_reader.read(_CHUNK_SIZE) + decompressed_size += len(output_chunk) + _check_max_size(decompressed_size, max_size) + output_stream.write(output_chunk) + return output_stream.getvalue() diff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.py new file mode 100644 index 000000000..fad7e6f6b --- /dev/null +++ b/scrapy/utils/_deps_compat.py @@ -0,0 +1,16 @@ +from OpenSSL import __version__ as PYOPENSSL_VERSION_STRING +from packaging.version import Version +from twisted import version as TWISTED_VERSION +from twisted.python.versions import Version as TxVersion + +TWISTED_FAILURE_HAS_STACK = TWISTED_VERSION < TxVersion("twisted", 24, 10, 0) +# changes to private _sslverify code, https://github.com/twisted/twisted/pull/12506 +TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0) +# lowerMaximumSecurityTo off-by-1, https://github.com/twisted/twisted/issues/10232 +TWISTED_TLS_LIMITS_OFFBY1 = TWISTED_VERSION < TxVersion("twisted", 26, 4, 0) + +PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING) +# SSL.Context.use_certificate() wants an X509 object, SSL.Context.use_privatekey() wants a PKey object +PYOPENSSL_WANTS_X509_PKEY = PYOPENSSL_VERSION < Version("24.3.0") +# SSL.Context.set_cipher_list() creates a temporary connection, making the context immutable +PYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0") diff --git a/scrapy/utils/_download_handlers.py b/scrapy/utils/_download_handlers.py new file mode 100644 index 000000000..9dec51e4b --- /dev/null +++ b/scrapy/utils/_download_handlers.py @@ -0,0 +1,153 @@ +"""Utils for built-in HTTP download handlers.""" + +from __future__ import annotations + +from contextlib import contextmanager +from http.cookiejar import CookieJar +from typing import TYPE_CHECKING, Any + +from twisted.internet.defer import CancelledError +from twisted.internet.error import ConnectionRefusedError as TxConnectionRefusedError +from twisted.internet.error import DNSLookupError +from twisted.internet.error import TimeoutError as TxTimeoutError +from twisted.python.failure import Failure +from twisted.web.client import ResponseFailed +from twisted.web.error import SchemeNotSupported + +from scrapy.exceptions import ( + CannotResolveHostError, + DownloadCancelledError, + DownloadConnectionRefusedError, + DownloadFailedError, + DownloadTimeoutError, + StopDownload, + UnsupportedURLSchemeError, +) +from scrapy.utils.log import logger +from scrapy.utils.response import get_response_class + +if TYPE_CHECKING: + from collections.abc import Iterator + from http.client import HTTPResponse + from http.cookiejar import Cookie + from ipaddress import IPv4Address, IPv6Address + from urllib.request import Request as ULRequest + + from scrapy import Request + from scrapy.crawler import Crawler + from scrapy.http import Headers, Response + + +class NullCookieJar(CookieJar): # pragma: no cover + """A CookieJar that rejects all cookies.""" + + def extract_cookies(self, response: HTTPResponse, request: ULRequest) -> None: + pass + + def set_cookie(self, cookie: Cookie) -> None: + pass + + +@contextmanager +def wrap_twisted_exceptions() -> Iterator[None]: + """Context manager that wraps Twisted exceptions into Scrapy exceptions.""" + try: + yield + except SchemeNotSupported as e: + raise UnsupportedURLSchemeError(str(e)) from e + except CancelledError as e: + raise DownloadCancelledError(str(e)) from e + except TxConnectionRefusedError as e: + raise DownloadConnectionRefusedError(str(e)) from e + except DNSLookupError as e: + raise CannotResolveHostError(str(e)) from e + except ResponseFailed as e: + raise DownloadFailedError(str(e)) from e + except TxTimeoutError as e: + raise DownloadTimeoutError(str(e)) from e + + +def check_stop_download( + signal: object, crawler: Crawler, request: Request, **kwargs: Any +) -> StopDownload | None: + """Send the given signal and check if any of its handlers raised + :exc:`~scrapy.exceptions.StopDownload`. + + Return the raised exception or ``None``. + """ + signal_result = crawler.signals.send_catch_log( + signal=signal, + request=request, + spider=crawler.spider, + **kwargs, + ) + for handler, result in signal_result: + if isinstance(result, Failure) and isinstance(result.value, StopDownload): + logger.debug( + f"Download stopped for {request} from signal handler {handler.__qualname__}" + ) + return result.value + + return None + + +def make_response( + url: str, + status: int, + headers: Headers, + body: bytes = b"", + flags: list[str] | None = None, + certificate: Any = None, + ip_address: IPv4Address | IPv6Address | None = None, + protocol: str | None = None, + stop_download: StopDownload | None = None, +) -> Response: + respcls = get_response_class(http_headers=headers, url=url, body=body) + response = respcls( + url=url, + status=status, + headers=headers, + body=body, + flags=flags, + certificate=certificate, + ip_address=ip_address, + protocol=protocol, + ) + if stop_download: + response.flags.append("download_stopped") + if stop_download.fail: + stop_download.response = response + raise stop_download + return response + + +def get_maxsize_msg(size: int, limit: int, request: Request, *, expected: bool) -> str: + prefix = "Expected to receive" if expected else "Received" + return ( + f"{prefix} {size} bytes which is larger than download " + f"max size ({limit}) in request {request}." + ) + + +def get_warnsize_msg(size: int, limit: int, request: Request, *, expected: bool) -> str: + prefix = "Expected to receive" if expected else "Received" + return ( + f"{prefix} {size} bytes which is larger than download " + f"warn size ({limit}) in request {request}." + ) + + +def get_dataloss_msg(url: str) -> str: + return ( + f"Got data loss in {url}. If you want to process broken " + f"responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" + f" -- This message won't be shown in further requests" + ) + + +def normalize_bind_address( + value: str | tuple[str, int] | None, +) -> tuple[str, int] | None: + if isinstance(value, str): + return (value, 0) + return value diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 0505db343..6d96a41f5 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,16 +1,20 @@ -from typing import AsyncGenerator, AsyncIterable, Iterable, Union +from __future__ import annotations + +from collections.abc import AsyncGenerator, AsyncIterator, Iterable +from typing import TypeVar + +_T = TypeVar("_T") -async def collect_asyncgen(result: AsyncIterable) -> list: - results = [] - async for x in result: - results.append(x) - return results +async def collect_asyncgen(result: AsyncIterator[_T]) -> list[_T]: + return [x async for x in result] -async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerator: +async def as_async_generator( + it: Iterable[_T] | AsyncIterator[_T], +) -> AsyncGenerator[_T]: """Wraps an iterable (sync or async) into an async generator.""" - if isinstance(it, AsyncIterable): + if isinstance(it, AsyncIterator): async for r in it: yield r else: diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py new file mode 100644 index 000000000..44604c0fe --- /dev/null +++ b/scrapy/utils/asyncio.py @@ -0,0 +1,313 @@ +"""Utilities related to asyncio and its support in Scrapy.""" + +from __future__ import annotations + +import asyncio +import logging +import time +from collections.abc import AsyncIterator, Callable, Coroutine, Iterable +from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar + +from twisted.internet.defer import Deferred +from twisted.internet.task import LoopingCall +from twisted.internet.threads import deferToThread + +from scrapy.utils.asyncgen import as_async_generator +from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed + +if TYPE_CHECKING: + from twisted.internet.base import DelayedCall + + # typing.Self, typing.TypeVarTuple and typing.Unpack require Python 3.11 + from typing_extensions import Self, TypeVarTuple, Unpack + + _Ts = TypeVarTuple("_Ts") + + +_T = TypeVar("_T") +_P = ParamSpec("_P") + + +logger = logging.getLogger(__name__) + + +def is_asyncio_available() -> bool: + """Check if it's possible to call asyncio code that relies on the asyncio event loop. + + .. versionadded:: 2.14 + + This function returns ``True`` if there is a running asyncio event loop. If + there is no such loop, it returns ``True`` if the Twisted reactor that is + installed is + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, returns + ``False`` if a different reactor is installed, and raises a + :exc:`RuntimeError` if no reactor is installed. + + Code that doesn't directly require a Twisted reactor should use this + function while code that requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` should use + :func:`~scrapy.utils.reactor.is_asyncio_reactor_installed`. + + When this returns ``True``, an asyncio loop is installed and used by + Scrapy. It's possible to call functions that require it, such as + :func:`asyncio.sleep`, and await on :class:`asyncio.Future` objects in + Scrapy-related code. + + When this returns ``False``, a non-asyncio Twisted reactor is installed. + It's not possible to use asyncio features that require an asyncio event + loop or await on :class:`asyncio.Future` objects in Scrapy-related code, + but it's possible to await on :class:`~twisted.internet.defer.Deferred` + objects. + + .. note:: As this function uses :func:`asyncio.get_running_loop()`, it will + only detect the event loop if called in the same thread and from the + code that runs inside that loop (this shouldn't be a problem when + calling it from code such as spiders and Scrapy components, if Scrapy + is run using one of the supported ways). + + .. versionchanged:: 2.15.0 + This function now also returns ``True`` if there is a running asyncio + loop, even if no Twisted reactor is installed. + """ + + # Check if there is a running asyncio loop. + # Can't easily check for an installed but not running one, and if we + # checked that there could be false positives due to some 3rd-party code + # installing it as a side effect (e.g. by calling get_event_loop()). + try: + asyncio.get_running_loop() + except RuntimeError: + pass + else: + return True + + # Check if there is an installed asyncio reactor (it doesn't need to be + # running). + if not is_reactor_installed(): + raise RuntimeError( + "is_asyncio_available() called without an installed reactor" + " or running asyncio loop." + ) + + return is_asyncio_reactor_installed() + + +async def _parallel_asyncio( + iterable: Iterable[_T] | AsyncIterator[_T], + count: int, + callable_: Callable[Concatenate[_T, _P], Coroutine[Any, Any, None]], + *args: _P.args, + **kwargs: _P.kwargs, +) -> None: + """Execute a callable over the objects in the given iterable, in parallel, + using no more than ``count`` concurrent calls. + + This function is only used in + :meth:`scrapy.core.scraper.Scraper.handle_spider_output_async` and so it + assumes that neither *callable* nor iterating *iterable* will raise an + exception. + """ + queue: asyncio.Queue[_T | None] = asyncio.Queue(count * 2) + + async def worker() -> None: + while True: + item = await queue.get() + if item is None: + break + try: + await callable_(item, *args, **kwargs) + finally: + queue.task_done() + + async def fill_queue() -> None: + async for item in as_async_generator(iterable): + await queue.put(item) + for _ in range(count): + await queue.put(None) + + fill_task = asyncio.create_task(fill_queue()) + work_tasks = [asyncio.create_task(worker()) for _ in range(count)] + await asyncio.wait([fill_task, *work_tasks]) + + +class AsyncioLoopingCall: + """A simple implementation of a periodic call using asyncio, keeping + some API and behavior compatibility with + :class:`~twisted.internet.task.LoopingCall`. + + The function is called every *interval* seconds, independent of the finish + time of the previous call. If the function is still running when it's time + to call it again, calls are skipped until the function finishes. + + The function must not return a coroutine or a ``Deferred``. + + .. versionadded:: 2.14.0 + """ + + def __init__(self, func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs): + self._func: Callable[_P, _T] = func + self._args: tuple[Any, ...] = args + self._kwargs: dict[str, Any] = kwargs + self._task: asyncio.Task[None] | None = None + self.interval: float | None = None + self._start_time: float | None = None + + @property + def running(self) -> bool: + return self._start_time is not None + + def start(self, interval: float, now: bool = True) -> None: + """Start calling the function every *interval* seconds. + + :param interval: The interval in seconds between calls. + :type interval: float + + :param now: If ``True``, also call the function immediately. + :type now: bool + """ + if self.running: + raise RuntimeError("AsyncioLoopingCall already running") + + if interval <= 0: + raise ValueError("Interval must be greater than 0") + + self.interval = interval + self._start_time = time.monotonic() + if now: + self._call() + loop = asyncio.get_event_loop() + self._task = loop.create_task(self._loop()) + + def _to_sleep(self) -> float: + """Return the time to sleep until the next call.""" + assert self.interval is not None + assert self._start_time is not None + now = time.monotonic() + running_for = now - self._start_time + return self.interval - (running_for % self.interval) + + async def _loop(self) -> None: + """Run an infinite loop that calls the function periodically.""" + while self.running: + await asyncio.sleep(self._to_sleep()) + self._call() + + def stop(self) -> None: + """Stop the periodic calls.""" + self.interval = self._start_time = None + if self._task is not None: + self._task.cancel() + self._task = None + + def _call(self) -> None: + """Execute the function.""" + try: + result = self._func(*self._args, **self._kwargs) + except Exception: + logger.exception("Error calling the AsyncioLoopingCall function") + self.stop() + else: + if isinstance(result, (Coroutine, Deferred)): + self.stop() + raise TypeError( + "The AsyncioLoopingCall function must not return a coroutine or a Deferred" + ) + + +def create_looping_call( + func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs +) -> AsyncioLoopingCall | LoopingCall: + """Create an instance of a looping call class. + + This creates an instance of + :class:`~scrapy.utils.asyncio.AsyncioLoopingCall` or + :class:`~twisted.internet.task.LoopingCall`, depending on whether asyncio + support is available. + + .. versionadded:: 2.14.0 + """ + if is_asyncio_available(): + return AsyncioLoopingCall(func, *args, **kwargs) + return LoopingCall(func, *args, **kwargs) + + +def call_later( + delay: float, func: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts] +) -> CallLaterResult: + """Schedule a function to be called after a delay. + + This uses either :meth:`asyncio.loop.call_later` or + :meth:`reactor.callLater() `, + depending on whether asyncio support is available. + + .. versionadded:: 2.14.0 + """ + if is_asyncio_available(): + loop = asyncio.get_event_loop() + return CallLaterResult.from_asyncio(loop.call_later(delay, func, *args)) + + from twisted.internet import reactor + + return CallLaterResult.from_twisted(reactor.callLater(delay, func, *args)) + + +class CallLaterResult: + """An universal result for :func:`call_later`, wrapping either + :class:`asyncio.TimerHandle` or :class:`twisted.internet.base.DelayedCall`. + + The provided API is close to the :class:`asyncio.TimerHandle` one: there is + no ``active()`` (as there is no such public API in + :class:`asyncio.TimerHandle`) but ``cancel()`` can be called on already + called or cancelled instances. + + .. versionadded:: 2.14.0 + """ + + _timer_handle: asyncio.TimerHandle | None = None + _delayed_call: DelayedCall | None = None + + @classmethod + def from_asyncio(cls, timer_handle: asyncio.TimerHandle) -> Self: + """Create a CallLaterResult from an asyncio TimerHandle.""" + o = cls() + o._timer_handle = timer_handle + return o + + @classmethod + def from_twisted(cls, delayed_call: DelayedCall) -> Self: + """Create a CallLaterResult from a Twisted DelayedCall.""" + o = cls() + o._delayed_call = delayed_call + return o + + def cancel(self) -> None: + """Cancel the underlying delayed call. + + Does nothing if the delayed call was already called or cancelled. + """ + if self._timer_handle: + self._timer_handle.cancel() + self._timer_handle = None + elif self._delayed_call and self._delayed_call.active(): + self._delayed_call.cancel() + self._delayed_call = None + + +async def run_in_thread( + func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs +) -> _T: + """Call a function in a thread and return its result as a coroutine. + + This uses either :func:`asyncio.to_thread` or + :func:`twisted.internet.threads.deferToThread`, depending on whether + asyncio support is available. + + .. versionadded:: 2.15.0 + """ + if is_asyncio_available(): + return await asyncio.to_thread(func, *args, **kwargs) + + # circular import + from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415 + + return await maybe_deferred_to_future(deferToThread(func, *args, **kwargs)) diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 38884a9f0..403cd54a8 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -1,44 +1,46 @@ import random +from typing import Any from urllib.parse import urlencode from twisted.web.resource import Resource -from twisted.web.server import Site +from twisted.web.server import Request, Site class Root(Resource): isLeaf = True - def getChild(self, name, request): + def getChild(self, path: str, request: Request) -> Resource: return self - def render(self, request): + def render(self, request: Request) -> bytes: total = _getarg(request, b"total", 100, int) show = _getarg(request, b"show", 10, int) - nlist = [random.randint(1, total) for _ in range(show)] + nlist = [random.randint(1, total) for _ in range(show)] # noqa: S311 request.write(b"") + assert request.args is not None args = request.args.copy() for nl in nlist: args["n"] = nl argstr = urlencode(args, doseq=True) - request.write( - f"follow {nl}
".encode("utf8") - ) + request.write(f"follow {nl}
".encode()) request.write(b"") return b"" -def _getarg(request, name, default=None, type=str): - return type(request.args[name][0]) if name in request.args else default +def _getarg( + request: Request, name: bytes, default: Any = None, type_: type = str +) -> Any: + return type_(request.args[name][0]) if name in request.args else default if __name__ == "__main__": from twisted.internet import reactor - root = Root() + root = Root() # type: ignore[no-untyped-call] factory = Site(root) httpPort = reactor.listenTCP(8998, Site(root)) - def _print_listening(): + def _print_listening() -> None: httpHost = httpPort.getHost() print(f"Bench server at http://{httpHost.host}:{httpHost.port}") diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 53cfeddd0..73f86bc71 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -3,7 +3,7 @@ def is_botocore_available() -> bool: try: - import botocore # noqa: F401 + import botocore # noqa: F401,PLC0415 return True except ImportError: diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 641dfa4a2..4850b370b 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -1,35 +1,29 @@ +from __future__ import annotations + import numbers import os import sys -import warnings from configparser import ConfigParser from operator import itemgetter from pathlib import Path -from typing import ( - Any, - Callable, - Collection, - Dict, - Iterable, - List, - Mapping, - MutableMapping, - Optional, - Union, -) +from typing import TYPE_CHECKING, Any, cast -from scrapy.exceptions import ScrapyDeprecationWarning, UsageError +from scrapy.exceptions import UsageError from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values +if TYPE_CHECKING: + from collections.abc import Callable, Collection, Iterable, Mapping, MutableMapping + def build_component_list( compdict: MutableMapping[Any, Any], - custom: Any = None, + *, convert: Callable[[Any], Any] = update_classpath, -) -> List[Any]: - """Compose a component list from a { class: order } dictionary.""" +) -> list[Any]: + """Compose a component list from a :ref:`component priority dictionary + `.""" def _check_components(complist: Collection[Any]) -> None: if len({convert(c) for c in complist}) != len(complist): @@ -38,7 +32,7 @@ def build_component_list( "please update your settings" ) - def _map_keys(compdict: Mapping[Any, Any]) -> Union[BaseSettings, Dict[Any, Any]]: + def _map_keys(compdict: Mapping[Any, Any]) -> BaseSettings | dict[Any, Any]: if isinstance(compdict, BaseSettings): compbs = BaseSettings() for k, v in compdict.items(): @@ -50,8 +44,7 @@ def build_component_list( "convert to the same " "object, please update your settings" ) - else: - compbs.set(convert(k), v, priority=prio) + compbs.set(convert(k), v, priority=prio) return compbs _check_components(compdict) return {convert(k): v for k, v in compdict.items()} @@ -65,25 +58,12 @@ def build_component_list( "please provide a real number or None instead" ) - if custom is not None: - warnings.warn( - "The 'custom' attribute of build_component_list() is deprecated. " - "Please merge its value into 'compdict' manually or change your " - "code to use Settings.getwithbase().", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if isinstance(custom, (list, tuple)): - _check_components(custom) - return type(custom)(convert(c) for c in custom) # type: ignore[return-value] - compdict.update(custom) - _validate_values(compdict) compdict = without_none_values(_map_keys(compdict)) return [k for k, v in sorted(compdict.items(), key=itemgetter(1))] -def arglist_to_dict(arglist: List[str]) -> Dict[str, str]: +def arglist_to_dict(arglist: list[str]) -> dict[str, str]: """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a dict """ @@ -91,8 +71,8 @@ def arglist_to_dict(arglist: List[str]) -> Dict[str, str]: def closest_scrapy_cfg( - path: Union[str, os.PathLike] = ".", - prevpath: Optional[Union[str, os.PathLike]] = None, + path: str | os.PathLike[str] = ".", + prevpath: str | os.PathLike[str] | None = None, ) -> str: """Return the path to the closest scrapy.cfg file by traversing the current directory and its parents @@ -129,7 +109,7 @@ def get_config(use_closest: bool = True) -> ConfigParser: return cfg -def get_sources(use_closest: bool = True) -> List[str]: +def get_sources(use_closest: bool = True) -> list[str]: xdg_config_home = ( os.environ.get("XDG_CONFIG_HOME") or Path("~/.config").expanduser() ) @@ -145,8 +125,8 @@ def get_sources(use_closest: bool = True) -> List[str]: def feed_complete_default_values_from_settings( - feed: Dict[str, Any], settings: BaseSettings -) -> Dict[str, Any]: + feed: dict[str, Any], settings: BaseSettings +) -> dict[str, Any]: out = feed.copy() out.setdefault("batch_item_count", settings.getint("FEED_EXPORT_BATCH_ITEM_COUNT")) out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) @@ -163,17 +143,17 @@ def feed_complete_default_values_from_settings( def feed_process_params_from_cli( settings: BaseSettings, - output: List[str], - output_format: Optional[str] = None, - overwrite_output: Optional[List[str]] = None, -) -> Dict[str, Dict[str, Any]]: + output: list[str], + *, + overwrite_output: list[str] | None = None, +) -> dict[str, dict[str, Any]]: """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary suitable to be used as the FEEDS setting. """ valid_output_formats: Iterable[str] = without_none_values( - settings.getwithbase("FEED_EXPORTERS") + cast("dict[str, str]", settings.getwithbase("FEED_EXPORTERS")) ).keys() def check_valid_format(output_format: str) -> None: @@ -191,38 +171,10 @@ def feed_process_params_from_cli( raise UsageError( "Please use only one of -o/--output and -O/--overwrite-output" ) - if output_format: - raise UsageError( - "-t/--output-format is a deprecated command line option" - " and does not work in combination with -O/--overwrite-output." - " To specify a format please specify it after a colon at the end of the" - " output URI (i.e. -O :)." - " Example working in the tutorial: " - "scrapy crawl quotes -O quotes.json:json" - ) output = overwrite_output overwrite = True - if output_format: - if len(output) == 1: - check_valid_format(output_format) - message = ( - "The -t/--output-format command line option is deprecated in favor of " - "specifying the output format within the output URI using the -o/--output or the" - " -O/--overwrite-output option (i.e. -o/-O :). See the documentation" - " of the -o or -O option or the following examples for more information. " - "Examples working in the tutorial: " - "scrapy crawl quotes -o quotes.csv:csv or " - "scrapy crawl quotes -O quotes.json:json" - ) - warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) - return {output[0]: {"format": output_format}} - raise UsageError( - "The -t command-line option cannot be used if multiple output " - "URIs are specified" - ) - - result: Dict[str, Dict[str, Any]] = {} + result: dict[str, dict[str, Any]] = {} for element in output: try: feed_uri, feed_format = element.rsplit(":", 1) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 100f040bb..93fbb9033 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,18 +1,35 @@ +from __future__ import annotations + +import code +from collections.abc import Callable from functools import wraps +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Iterable + +EmbedFuncT = Callable[..., None] +KnownShellsT = dict[str, Callable[..., EmbedFuncT]] -def _embed_ipython_shell(namespace={}, banner=""): +def _embed_ipython_shell( + namespace: dict[str, Any] | None = None, banner: str = "" +) -> EmbedFuncT: """Start an IPython Shell""" try: - from IPython.terminal.embed import InteractiveShellEmbed - from IPython.terminal.ipapp import load_default_config + from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415 + from IPython.terminal.ipapp import load_default_config # noqa: PLC0415 except ImportError: - from IPython.frontend.terminal.embed import InteractiveShellEmbed - from IPython.frontend.terminal.ipapp import load_default_config + from IPython.frontend.terminal.embed import ( # type: ignore[import-not-found,no-redef] # noqa: T100,PLC0415 + InteractiveShellEmbed, + ) + from IPython.frontend.terminal.ipapp import ( # type: ignore[import-not-found,no-redef] # noqa: PLC0415 + load_default_config, + ) @wraps(_embed_ipython_shell) - def wrapper(namespace=namespace, banner=""): - config = load_default_config() + def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None: + config = load_default_config() # type: ignore[no-untyped-call] # Always use .instance() to ensure _instance propagation to all parents # this is needed for completion works well for new imports # and clear the instance to always have the fresh env @@ -26,50 +43,54 @@ def _embed_ipython_shell(namespace={}, banner=""): return wrapper -def _embed_bpython_shell(namespace={}, banner=""): +def _embed_bpython_shell( + namespace: dict[str, Any] | None = None, banner: str = "" +) -> EmbedFuncT: """Start a bpython shell""" - import bpython + import bpython # noqa: PLC0415 @wraps(_embed_bpython_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None: bpython.embed(locals_=namespace, banner=banner) return wrapper -def _embed_ptpython_shell(namespace={}, banner=""): +def _embed_ptpython_shell( + namespace: dict[str, Any] | None = None, banner: str = "" +) -> EmbedFuncT: """Start a ptpython shell""" - import ptpython.repl + import ptpython.repl # noqa: PLC0415 # pylint: disable=import-error @wraps(_embed_ptpython_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None: print(banner) ptpython.repl.embed(locals=namespace) return wrapper -def _embed_standard_shell(namespace={}, banner=""): +def _embed_standard_shell( + namespace: dict[str, Any] | None = None, banner: str = "" +) -> EmbedFuncT: """Start a standard python shell""" - import code - try: # readline module is only available on unix systems - import readline + import readline # noqa: PLC0415 except ImportError: pass else: - import rlcompleter # noqa: F401 + import rlcompleter # noqa: F401,PLC0415 - readline.parse_and_bind("tab:complete") + readline.parse_and_bind("tab:complete") # type: ignore[attr-defined,unused-ignore] @wraps(_embed_standard_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None: code.interact(banner=banner, local=namespace) return wrapper -DEFAULT_PYTHON_SHELLS = { +DEFAULT_PYTHON_SHELLS: KnownShellsT = { "ptpython": _embed_ptpython_shell, "ipython": _embed_ipython_shell, "bpython": _embed_bpython_shell, @@ -77,7 +98,9 @@ DEFAULT_PYTHON_SHELLS = { } -def get_shell_embed_func(shells=None, known_shells=None): +def get_shell_embed_func( + shells: Iterable[str] | None = None, known_shells: KnownShellsT | None = None +) -> EmbedFuncT | None: """Return the first acceptable shell-embed function from a given list of shell names. """ @@ -93,9 +116,14 @@ def get_shell_embed_func(shells=None, known_shells=None): return known_shells[shell]() except ImportError: continue + return None -def start_python_console(namespace=None, banner="", shells=None): +def start_python_console( + namespace: dict[str, Any] | None = None, + banner: str = "", + shells: Iterable[str] | None = None, +) -> None: """Start Python console bound to the given namespace. Readline support and tab completion will be used on Unix, if available. """ diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index f5dbbd64e..646335fb1 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -1,22 +1,33 @@ +from __future__ import annotations + import argparse import warnings from http.cookies import SimpleCookie from shlex import split +from typing import TYPE_CHECKING, Any, NoReturn from urllib.parse import urlparse from w3lib.http import basic_auth_header +if TYPE_CHECKING: + from collections.abc import Sequence + class DataAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + values: str | Sequence[Any] | None, + option_string: str | None = None, + ) -> None: value = str(values) - if value.startswith("$"): - value = value[1:] + value = value.removeprefix("$") setattr(namespace, self.dest, value) class CurlParser(argparse.ArgumentParser): - def error(self, message): + def error(self, message: str) -> NoReturn: error_msg = f"There was an error parsing the curl command: {message}" raise ValueError(error_msg) @@ -25,6 +36,7 @@ curl_parser = CurlParser() curl_parser.add_argument("url") curl_parser.add_argument("-H", "--header", dest="headers", action="append") curl_parser.add_argument("-X", "--request", dest="method") +curl_parser.add_argument("-b", "--cookie", dest="cookies", action="append") curl_parser.add_argument("-d", "--data", "--data-raw", dest="data", action=DataAction) curl_parser.add_argument("-u", "--user", dest="auth") @@ -42,9 +54,11 @@ for argument in safe_to_ignore_arguments: curl_parser.add_argument(*argument, action="store_true") -def _parse_headers_and_cookies(parsed_args): - headers = [] - cookies = {} +def _parse_headers_and_cookies( + parsed_args: argparse.Namespace, +) -> tuple[list[tuple[str, bytes]], dict[str, str]]: + headers: list[tuple[str, bytes]] = [] + cookies: dict[str, str] = {} for header in parsed_args.headers or (): name, val = header.split(":", 1) name = name.strip() @@ -55,6 +69,14 @@ def _parse_headers_and_cookies(parsed_args): else: headers.append((name, val)) + for cookie_param in parsed_args.cookies or (): + # curl can treat this parameter as either "key=value; key2=value2" pairs, or a filename. + # Scrapy will only support key-value pairs. + if "=" not in cookie_param: + continue + for name, morsel in SimpleCookie(cookie_param).items(): + cookies[name] = morsel.value + if parsed_args.auth: user, password = parsed_args.auth.split(":", 1) headers.append(("Authorization", basic_auth_header(user, password))) @@ -64,7 +86,7 @@ def _parse_headers_and_cookies(parsed_args): def curl_to_request_kwargs( curl_command: str, ignore_unknown_options: bool = True -) -> dict: +) -> dict[str, Any]: """Convert a cURL command syntax to Request kwargs. :param str curl_command: string containing the curl command @@ -82,9 +104,9 @@ def curl_to_request_kwargs( parsed_args, argv = curl_parser.parse_known_args(curl_args[1:]) if argv: - msg = f'Unrecognized options: {", ".join(argv)}' + msg = f"Unrecognized options: {', '.join(argv)}" if ignore_unknown_options: - warnings.warn(msg) + warnings.warn(msg, stacklevel=2) else: raise ValueError(msg) @@ -98,7 +120,7 @@ def curl_to_request_kwargs( method = parsed_args.method or "GET" - result = {"method": method.upper(), "url": url} + result: dict[str, Any] = {"method": method.upper(), "url": url} headers, cookies = _parse_headers_and_cookies(parsed_args) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 0ba2fe4e2..4e65c062e 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -8,25 +8,18 @@ This module must not depend on any module outside the Standard Library. from __future__ import annotations import collections +import contextlib import warnings import weakref +from collections import OrderedDict from collections.abc import Mapping -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Iterable, - Optional, - OrderedDict, - Sequence, - Tuple, - TypeVar, - Union, -) +from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, cast from scrapy.exceptions import ScrapyDeprecationWarning if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -35,11 +28,12 @@ _KT = TypeVar("_KT") _VT = TypeVar("_VT") -class CaselessDict(dict): +class CaselessDict(dict): # type: ignore[type-arg] __slots__ = () def __new__(cls, *args: Any, **kwargs: Any) -> Self: - from scrapy.http.headers import Headers + # circular import + from scrapy.http.headers import Headers # noqa: PLC0415 if issubclass(cls, CaselessDict) and not issubclass(cls, Headers): warnings.warn( @@ -52,7 +46,7 @@ class CaselessDict(dict): def __init__( self, - seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, ): super().__init__() if seq: @@ -89,10 +83,10 @@ class CaselessDict(dict): return dict.get(self, self.normkey(key), self.normvalue(def_val)) def setdefault(self, key: AnyStr, def_val: Any = None) -> Any: - return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) # type: ignore[arg-type] + return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) # doesn't fully implement MutableMapping.update() - def update(self, seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]]]) -> None: # type: ignore[override] + def update(self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]) -> None: # type: ignore[override] seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) super().update(iseq) @@ -105,20 +99,20 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) -class CaseInsensitiveDict(collections.UserDict): +class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]): """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups. """ - def __init__(self, *args, **kwargs) -> None: - self._keys: dict = {} + def __init__(self, *args: Any, **kwargs: Any) -> None: + self._keys: dict[str | bytes, Any] = {} super().__init__(*args, **kwargs) - def __getitem__(self, key: AnyStr) -> Any: + def __getitem__(self, key: str | bytes) -> Any: normalized_key = self._normkey(key) return super().__getitem__(self._keys[normalized_key.lower()]) - def __setitem__(self, key: AnyStr, value: Any) -> None: + def __setitem__(self, key: str | bytes, value: Any) -> None: normalized_key = self._normkey(key) try: lower_key = self._keys[normalized_key.lower()] @@ -128,19 +122,19 @@ class CaseInsensitiveDict(collections.UserDict): super().__setitem__(normalized_key, self._normvalue(value)) self._keys[normalized_key.lower()] = normalized_key - def __delitem__(self, key: AnyStr) -> None: + def __delitem__(self, key: str | bytes) -> None: normalized_key = self._normkey(key) stored_key = self._keys.pop(normalized_key.lower()) super().__delitem__(stored_key) - def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] + def __contains__(self, key: str | bytes) -> bool: # type: ignore[override] normalized_key = self._normkey(key) return normalized_key.lower() in self._keys def __repr__(self) -> str: return f"<{self.__class__.__name__}: {super().__repr__()}>" - def _normkey(self, key: AnyStr) -> AnyStr: + def _normkey(self, key: str | bytes) -> str | bytes: return key def _normvalue(self, value: Any) -> Any: @@ -153,9 +147,9 @@ class LocalCache(OrderedDict[_KT, _VT]): Older items expires first. """ - def __init__(self, limit: Optional[int] = None): + def __init__(self, limit: int | None = None): super().__init__() - self.limit: Optional[int] = limit + self.limit: int | None = limit def __setitem__(self, key: _KT, value: _VT) -> None: if self.limit: @@ -164,7 +158,7 @@ class LocalCache(OrderedDict[_KT, _VT]): super().__setitem__(key, value) -class LocalWeakReferencedCache(weakref.WeakKeyDictionary): +class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]): """ A weakref.WeakKeyDictionary implementation that uses LocalCache as its underlying data structure, making it ordered and capable of being size-limited. @@ -176,19 +170,18 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): it cannot be instantiated with an initial dictionary. """ - def __init__(self, limit: Optional[int] = None): + def __init__(self, limit: int | None = None): super().__init__() - self.data: LocalCache = LocalCache(limit=limit) + self.data: LocalCache[_KT, _VT] = LocalCache(limit=limit) - def __setitem__(self, key: _KT, value: _VT) -> None: - try: + def __setitem__(self, key: _KT, value: _VT | None) -> None: + # if raised, key is not weak-referenceable, skip caching + with contextlib.suppress(TypeError): super().__setitem__(key, value) - except TypeError: - pass # key is not weak-referenceable, skip caching - def __getitem__(self, key: _KT) -> Optional[_VT]: # type: ignore[override] + def __getitem__(self, key: _KT) -> _VT | None: try: - return super().__getitem__(key) + return cast("_VT", super().__getitem__(key)) except (TypeError, KeyError): return None # key is either not weak-referenceable or not cached @@ -196,8 +189,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): class SequenceExclude: """Object to test if an item is NOT within some sequence.""" - def __init__(self, seq: Sequence): - self.seq: Sequence = seq + def __init__(self, seq: Sequence[Any]): + self.seq: Sequence[Any] = seq def __contains__(self, item: Any) -> bool: return item not in self.seq diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 04186559f..a5bb6fa24 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -1,21 +1,34 @@ +from __future__ import annotations + +import inspect import warnings from functools import wraps -from typing import Any, Callable +from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, cast, overload -from twisted.internet import defer, threads -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, maybeDeferred from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.asyncio import run_in_thread +from scrapy.utils.defer import deferred_from_coro + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Callable, Coroutine -def deprecated(use_instead: Any = None) -> Callable: +_T = TypeVar("_T") +_P = ParamSpec("_P") + + +def deprecated( + use_instead: Any = None, +) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: """This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted when the function is used.""" - def deco(func: Callable) -> Callable: + def deco(func: Callable[_P, _T]) -> Callable[_P, _T]: @wraps(func) - def wrapped(*args: Any, **kwargs: Any) -> Any: + def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> _T: message = f"Call to deprecated function {func.__name__}." if use_instead: message += f" Use {use_instead} instead." @@ -30,23 +43,97 @@ def deprecated(use_instead: Any = None) -> Callable: return deco -def defers(func: Callable) -> Callable[..., Deferred]: +def defers(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: # pragma: no cover """Decorator to make sure a function always returns a deferred""" + warnings.warn( + "@defers is deprecated, you can use maybeDeferred() directly if needed.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) @wraps(func) - def wrapped(*a: Any, **kw: Any) -> Deferred: - return defer.maybeDeferred(func, *a, **kw) + def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]: + return maybeDeferred(func, *a, **kw) return wrapped -def inthread(func: Callable) -> Callable[..., Deferred]: +def inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: """Decorator to call a function in a thread and return a deferred with the - result + result. + + .. versionchanged:: 2.15.0 + Now uses :func:`asyncio.to_thread` if the asyncio support is available. """ @wraps(func) - def wrapped(*a: Any, **kw: Any) -> Deferred: - return threads.deferToThread(func, *a, **kw) + def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]: + return deferred_from_coro(run_in_thread(func, *a, **kw)) return wrapped + + +@overload +def _warn_spider_arg( + func: Callable[_P, Coroutine[Any, Any, _T]], +) -> Callable[_P, Coroutine[Any, Any, _T]]: ... + + +@overload +def _warn_spider_arg( + func: Callable[_P, AsyncGenerator[_T]], +) -> Callable[_P, AsyncGenerator[_T]]: ... + + +@overload +def _warn_spider_arg(func: Callable[_P, _T]) -> Callable[_P, _T]: ... + + +def _warn_spider_arg( + func: Callable[_P, _T], +) -> ( + Callable[_P, _T] + | Callable[_P, Coroutine[Any, Any, _T]] + | Callable[_P, AsyncGenerator[_T]] +): + """Decorator to warn if a ``spider`` argument is passed to a function.""" + + sig = inspect.signature(func) + + def check_args(*args: _P.args, **kwargs: _P.kwargs) -> None: + bound = sig.bind(*args, **kwargs) + if "spider" in bound.arguments: + warnings.warn( + f"Passing a 'spider' argument to {func.__qualname__}() is deprecated and " + "the argument will be removed in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=3, + ) + + if inspect.iscoroutinefunction(func): + + @wraps(func) + async def async_inner(*args: _P.args, **kwargs: _P.kwargs) -> _T: + check_args(*args, **kwargs) + return cast("_T", await func(*args, **kwargs)) + + return async_inner + + if inspect.isasyncgenfunction(func): + + @wraps(func) + async def asyncgen_inner( + *args: _P.args, **kwargs: _P.kwargs + ) -> AsyncGenerator[_T]: + check_args(*args, **kwargs) + async for item in func(*args, **kwargs): + yield item + + return asyncgen_inner + + @wraps(func) + def sync_inner(*args: _P.args, **kwargs: _P.kwargs) -> _T: + check_args(*args, **kwargs) + return func(*args, **kwargs) + + return sync_inner diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index bf3c5ef5b..29a34d4ef 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -1,109 +1,179 @@ """ Helper functions for dealing with Twisted deferreds """ + +from __future__ import annotations + import asyncio import inspect +import warnings from asyncio import Future +from collections import deque +from collections.abc import Awaitable, Coroutine, Iterable, Iterator from functools import wraps -from types import CoroutineType from typing import ( + TYPE_CHECKING, Any, - AsyncGenerator, - AsyncIterable, - AsyncIterator, - Awaitable, - Callable, - Coroutine, - Dict, - Generator, - Iterable, - Iterator, - List, - Optional, - Tuple, + Concatenate, + Generic, + ParamSpec, TypeVar, - Union, cast, overload, ) -from twisted.internet import defer -from twisted.internet.defer import Deferred, DeferredList, ensureDeferred +from twisted.internet.defer import Deferred, DeferredList, FirstError, fail, succeed from twisted.internet.task import Cooperator from twisted.python import failure -from twisted.python.failure import Failure -from scrapy.exceptions import IgnoreRequest -from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.python import global_object_name + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable + + from twisted.python.failure import Failure -def defer_fail(_failure: Failure) -> Deferred: +_T = TypeVar("_T") +_T2 = TypeVar("_T2") +_P = ParamSpec("_P") + + +_DEFER_DELAY = 0.1 + + +def defer_fail(_failure: Failure) -> Deferred[Any]: # pragma: no cover """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop It delays by 100ms so reactor has a chance to go through readers and writers before attending pending delayed calls, so do not set delay to zero. """ + warnings.warn( + "scrapy.utils.defer.defer_fail() is deprecated, use" + " twisted.internet.defer.fail(), plus an explicit sleep if needed.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + from twisted.internet import reactor - d: Deferred = Deferred() - reactor.callLater(0.1, d.errback, _failure) + d: Deferred[Any] = Deferred() + reactor.callLater(_DEFER_DELAY, d.errback, _failure) return d -def defer_succeed(result: Any) -> Deferred: +def defer_succeed(result: _T) -> Deferred[_T]: # pragma: no cover """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop It delays by 100ms so reactor has a chance to go through readers and writers before attending pending delayed calls, so do not set delay to zero. """ + warnings.warn( + "scrapy.utils.defer.defer_succeed() is deprecated, use" + " twisted.internet.defer.succeed(), plus an explicit sleep if needed.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + from twisted.internet import reactor - d: Deferred = Deferred() - reactor.callLater(0.1, d.callback, result) + d: Deferred[_T] = Deferred() + reactor.callLater(_DEFER_DELAY, d.callback, result) return d -def defer_result(result: Any) -> Deferred: +async def _defer_sleep_async() -> None: + """Delay by _DEFER_DELAY so reactor has a chance to go through readers and writers + before attending pending delayed calls, so do not set delay to zero. + """ + if is_asyncio_available(): + await asyncio.sleep(_DEFER_DELAY) + else: + from twisted.internet import reactor + + d: Deferred[None] = Deferred() + reactor.callLater(_DEFER_DELAY, d.callback, None) + await d + + +def defer_result(result: Any) -> Deferred[Any]: # pragma: no cover + warnings.warn( + "scrapy.utils.defer.defer_result() is deprecated, use" + " twisted.internet.defer.success() and twisted.internet.defer.fail()," + " plus an explicit sleep if needed, or explicit reactor.callLater().", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if isinstance(result, Deferred): return result + + from twisted.internet import reactor + + d: Deferred[Any] = Deferred() if isinstance(result, failure.Failure): - return defer_fail(result) - return defer_succeed(result) + reactor.callLater(_DEFER_DELAY, d.errback, result) + else: + reactor.callLater(_DEFER_DELAY, d.callback, result) + return d -def mustbe_deferred(f: Callable, *args: Any, **kw: Any) -> Deferred: +@overload +def mustbe_deferred( + f: Callable[_P, Deferred[_T]], *args: _P.args, **kw: _P.kwargs +) -> Deferred[_T]: ... + + +@overload +def mustbe_deferred( + f: Callable[_P, _T], *args: _P.args, **kw: _P.kwargs +) -> Deferred[_T]: ... + + +def mustbe_deferred( + f: Callable[_P, Deferred[_T] | _T], + *args: _P.args, + **kw: _P.kwargs, +) -> Deferred[_T]: # pragma: no cover """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ + warnings.warn( + "scrapy.utils.defer.mustbe_deferred() is deprecated, use" + " twisted.internet.defer.maybeDeferred(), with an explicit sleep if needed.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + result: _T | Deferred[_T] | Failure try: result = f(*args, **kw) - # FIXME: Hack to avoid introspecting tracebacks. This to speed up - # processing of IgnoreRequest errors which are, by far, the most common - # exception in Scrapy - see #125 - except IgnoreRequest as e: - return defer_fail(failure.Failure(e)) except Exception: - return defer_fail(failure.Failure()) - else: - return defer_result(result) + result = failure.Failure() + return defer_result(result) def parallel( - iterable: Iterable, count: int, callable: Callable, *args: Any, **named: Any -) -> Deferred: + iterable: Iterable[_T], + count: int, + callable: Callable[Concatenate[_T, _P], _T2], # noqa: A002 + *args: _P.args, + **named: _P.kwargs, +) -> Deferred[list[tuple[bool, Iterator[_T2]]]]: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. Taken from: https://jcalderone.livejournal.com/24285.html """ coop = Cooperator() - work = (callable(elem, *args, **named) for elem in iterable) + work: Iterator[_T2] = (callable(elem, *args, **named) for elem in iterable) return DeferredList([coop.coiterate(work) for _ in range(count)]) -class _AsyncCooperatorAdapter(Iterator): +class _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_T]): """A class that wraps an async iterable into a normal iterator suitable for using in Cooperator.coiterate(). As it's only needed for parallel_async(), it calls the callable directly in the callback, instead of providing a more @@ -151,28 +221,30 @@ class _AsyncCooperatorAdapter(Iterator): def __init__( self, - aiterable: AsyncIterable, - callable: Callable, - *callable_args: Any, - **callable_kwargs: Any, + aiterable: AsyncIterator[_T], + callable_: Callable[Concatenate[_T, _P], Deferred[Any] | None], + *callable_args: _P.args, + **callable_kwargs: _P.kwargs, ): - self.aiterator: AsyncIterator = aiterable.__aiter__() - self.callable: Callable = callable - self.callable_args: Tuple[Any, ...] = callable_args - self.callable_kwargs: Dict[str, Any] = callable_kwargs + self.aiterator: AsyncIterator[_T] = aiterable.__aiter__() + self.callable: Callable[Concatenate[_T, _P], Deferred[Any] | None] = callable_ + self.callable_args: tuple[Any, ...] = callable_args + self.callable_kwargs: dict[str, Any] = callable_kwargs self.finished: bool = False - self.waiting_deferreds: List[Deferred] = [] - self.anext_deferred: Optional[Deferred] = None + self.waiting_deferreds: deque[Deferred[Any]] = deque() + self.anext_deferred: Deferred[_T] | None = None - def _callback(self, result: Any) -> None: + def _callback(self, result: _T) -> None: # This gets called when the result from aiterator.__anext__() is available. # It calls the callable on it and sends the result to the oldest waiting Deferred # (by chaining if the result is a Deferred too or by firing if not). self.anext_deferred = None - result = self.callable(result, *self.callable_args, **self.callable_kwargs) - d = self.waiting_deferreds.pop(0) - if isinstance(result, Deferred): - result.chainDeferred(d) + callable_result = self.callable( + result, *self.callable_args, **self.callable_kwargs + ) + d = self.waiting_deferreds.popleft() + if isinstance(callable_result, Deferred): + callable_result.chainDeferred(d) else: d.callback(None) if self.waiting_deferreds: @@ -190,15 +262,15 @@ class _AsyncCooperatorAdapter(Iterator): def _call_anext(self) -> None: # This starts waiting for the next result from aiterator. # If aiterator is exhausted, _errback will be called. - self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) + self.anext_deferred = deferred_from_coro(anext(self.aiterator)) self.anext_deferred.addCallbacks(self._callback, self._errback) - def __next__(self) -> Deferred: + def __next__(self) -> Deferred[Any]: # This puts a new Deferred into self.waiting_deferreds and returns it. # It also calls __anext__() if needed. if self.finished: raise StopIteration - d: Deferred = Deferred() + d: Deferred[Any] = Deferred() self.waiting_deferreds.append(d) if not self.anext_deferred: self._call_anext() @@ -206,71 +278,77 @@ class _AsyncCooperatorAdapter(Iterator): def parallel_async( - async_iterable: AsyncIterable, + async_iterable: AsyncIterator[_T], count: int, - callable: Callable, - *args: Any, - **named: Any, -) -> Deferred: - """Like parallel but for async iterators""" + callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], # noqa: A002 + *args: _P.args, + **named: _P.kwargs, +) -> Deferred[list[tuple[bool, Iterator[Deferred[Any]]]]]: + """Like ``parallel`` but for async iterators""" coop = Cooperator() - work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named) - dl: Deferred = DeferredList([coop.coiterate(work) for _ in range(count)]) + work: Iterator[Deferred[Any]] = _AsyncCooperatorAdapter( + async_iterable, callable, *args, **named + ) + dl: Deferred[list[tuple[bool, Iterator[Deferred[Any]]]]] = DeferredList( + [coop.coiterate(work) for _ in range(count)] + ) return dl def process_chain( - callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any -) -> Deferred: + callbacks: Iterable[Callable[Concatenate[_T, _P], _T]], + input: _T, # noqa: A002 + *a: _P.args, + **kw: _P.kwargs, +) -> Deferred[_T]: # pragma: no cover """Return a Deferred built by chaining the given callbacks""" - d: Deferred = Deferred() + warnings.warn( + "process_chain() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + d: Deferred[_T] = Deferred() for x in callbacks: d.addCallback(x, *a, **kw) d.callback(input) return d -def process_chain_both( - callbacks: Iterable[Callable], - errbacks: Iterable[Callable], - input: Any, - *a: Any, - **kw: Any, -) -> Deferred: - """Return a Deferred built by chaining the given callbacks and errbacks""" - d: Deferred = Deferred() - for cb, eb in zip(callbacks, errbacks): - d.addCallbacks( - callback=cb, - errback=eb, - callbackArgs=a, - callbackKeywords=kw, - errbackArgs=a, - errbackKeywords=kw, - ) - if isinstance(input, failure.Failure): - d.errback(input) - else: - d.callback(input) - return d - - def process_parallel( - callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any -) -> Deferred: + callbacks: Iterable[Callable[Concatenate[_T, _P], _T2]], + input: _T, # noqa: A002 + *a: _P.args, + **kw: _P.kwargs, +) -> Deferred[list[_T2]]: # pragma: no cover """Return a Deferred with the output of all successful calls to the given callbacks """ - dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d: Deferred = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) - d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) - return d + warnings.warn( + "process_parallel() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + dfds = [succeed(input).addCallback(x, *a, **kw) for x in callbacks] + d: Deferred[list[tuple[bool, _T2]]] = DeferredList( + dfds, fireOnOneErrback=True, consumeErrors=True + ) + d2: Deferred[list[_T2]] = d.addCallback(lambda r: [x[1] for x in r]) + + def eb(failure: Failure) -> Failure: + assert isinstance(failure.value, FirstError) + return failure.value.subFailure + + d2.addErrback(eb) + return d2 def iter_errback( - iterable: Iterable, errback: Callable, *a: Any, **kw: Any -) -> Generator: - """Wraps an iterable calling an errback if an error is caught while + iterable: Iterable[_T], + errback: Callable[Concatenate[Failure, _P], Any], + *a: _P.args, + **kw: _P.kwargs, +) -> Iterable[_T]: + """Wrap an iterable calling an errback if an error is caught while iterating it. """ it = iter(iterable) @@ -284,85 +362,116 @@ def iter_errback( async def aiter_errback( - aiterable: AsyncIterable, errback: Callable, *a: Any, **kw: Any -) -> AsyncGenerator: - """Wraps an async iterable calling an errback if an error is caught while - iterating it. Similar to scrapy.utils.defer.iter_errback() + aiterable: AsyncIterator[_T], + errback: Callable[Concatenate[Failure, _P], Any], + *a: _P.args, + **kw: _P.kwargs, +) -> AsyncIterator[_T]: + """Wrap an async iterable calling an errback if an error is caught while + iterating it. Similar to :func:`scrapy.utils.defer.iter_errback`. """ - it = aiterable.__aiter__() + it = aiter(aiterable) while True: try: - yield await it.__anext__() + yield await anext(it) except StopAsyncIteration: break except Exception: errback(failure.Failure(), *a, **kw) -_CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future]) -_T = TypeVar("_T") +@overload +def deferred_from_coro(o: Awaitable[_T]) -> Deferred[_T]: ... @overload -def deferred_from_coro(o: _CT) -> Deferred: - ... +def deferred_from_coro(o: _T2) -> _T2: ... -@overload -def deferred_from_coro(o: _T) -> _T: - ... - - -def deferred_from_coro(o: _T) -> Union[Deferred, _T]: - """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" +def deferred_from_coro(o: Awaitable[_T] | _T2) -> Deferred[_T] | _T2: + """Convert a coroutine or other awaitable object into a Deferred, + or return the object as is if it isn't a coroutine.""" if isinstance(o, Deferred): return o - if asyncio.isfuture(o) or inspect.isawaitable(o): - if not is_asyncio_reactor_installed(): + if inspect.isawaitable(o): + if not is_asyncio_available(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" - return ensureDeferred(cast(Coroutine[Deferred, Any, Any], o)) + return Deferred.fromCoroutine(cast("Coroutine[Deferred[Any], Any, _T]", o)) # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor - event_loop = _get_asyncio_event_loop() - return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) + return Deferred.fromFuture(asyncio.ensure_future(o)) return o -def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: - """Converts a coroutine function into a function that returns a Deferred. +def deferred_f_from_coro_f( + coro_f: Callable[_P, Awaitable[_T]], +) -> Callable[_P, Deferred[_T]]: + """Convert a coroutine function into a function that returns a Deferred. The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it. This is useful for callback chains, as callback functions are called with the previous callback result. """ @wraps(coro_f) - def f(*coro_args: Any, **coro_kwargs: Any) -> Any: + def f(*coro_args: _P.args, **coro_kwargs: _P.kwargs) -> Deferred[_T]: return deferred_from_coro(coro_f(*coro_args, **coro_kwargs)) return f -def maybeDeferred_coro(f: Callable, *args: Any, **kw: Any) -> Deferred: +def maybeDeferred_coro( + f: Callable[_P, Any], *args: _P.args, **kw: _P.kwargs +) -> Deferred[Any]: # pragma: no cover + """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" + warnings.warn( + "maybeDeferred_coro() is deprecated and will be removed in a future Scrapy version.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return _maybeDeferred_coro(f, False, *args, **kw) + + +def _maybeDeferred_coro( + f: Callable[_P, Any], warn: bool, *args: _P.args, **kw: _P.kwargs +) -> Deferred[Any]: """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" try: result = f(*args, **kw) except: # noqa: E722 - return defer.fail(failure.Failure(captureVars=Deferred.debug)) + return fail(failure.Failure(captureVars=Deferred.debug)) + # when the deprecation period has ended we need to make sure the behavior + # of the public maybeDeferred_coro() function isn't changed, or drop it in + # the same release if isinstance(result, Deferred): + if warn: + warnings.warn( + f"{global_object_name(f)} returned a Deferred, this is deprecated." + f" Please refactor this function to return a coroutine.", + ScrapyDeprecationWarning, + stacklevel=2, + ) return result if asyncio.isfuture(result) or inspect.isawaitable(result): return deferred_from_coro(result) - if isinstance(result, failure.Failure): - return defer.fail(result) - return defer.succeed(result) + if isinstance(result, failure.Failure): # pragma: no cover + if warn: + warnings.warn( + f"{global_object_name(f)} returned a Failure, this is deprecated." + f" Please refactor this function to return a coroutine.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return fail(result) + return succeed(result) -def deferred_to_future(d: Deferred) -> Future: - """ - .. versionadded:: 2.6.0 +def deferred_to_future(d: Deferred[_T]) -> Future[_T]: + """Return an :class:`asyncio.Future` object that wraps *d*. - Return an :class:`asyncio.Future` object that wraps *d*. + This function requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be + installed. When :ref:`using the asyncio reactor `, you cannot await on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy @@ -376,26 +485,30 @@ def deferred_to_future(d: Deferred) -> Future: additional_request = scrapy.Request('https://example.org/price') deferred = self.crawler.engine.download(additional_request) additional_response = await deferred_to_future(deferred) + + .. versionchanged:: 2.14 + This function no longer installs an asyncio loop if called before the + Twisted asyncio reactor is installed. A :exc:`RuntimeError` is raised + in this case. """ - return d.asFuture(_get_asyncio_event_loop()) + if not is_asyncio_available(): + raise RuntimeError("deferred_to_future() requires AsyncioSelectorReactor.") + return d.asFuture(asyncio.get_event_loop()) -def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: - """ - .. versionadded:: 2.6.0 - - Return *d* as an object that can be awaited from a :ref:`Scrapy callable +def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: + """Return *d* as an object that can be awaited from a :ref:`Scrapy callable defined as a coroutine `. What you can await in Scrapy callables defined as coroutines depends on the value of :setting:`TWISTED_REACTOR`: - - When not using the asyncio reactor, you can only await on - :class:`~twisted.internet.defer.Deferred` objects. - - When :ref:`using the asyncio reactor `, you can only await on :class:`asyncio.Future` objects. + - When not using the asyncio reactor, you can only await on + :class:`~twisted.internet.defer.Deferred` objects. + If you want to write code that uses ``Deferred`` objects but works with any reactor, use this function on all ``Deferred`` objects:: @@ -406,6 +519,57 @@ def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: deferred = self.crawler.engine.download(additional_request) additional_response = await maybe_deferred_to_future(deferred) """ - if not is_asyncio_reactor_installed(): + if not is_asyncio_available(): return d return deferred_to_future(d) + + +def _schedule_coro(coro: Coroutine[Any, Any, Any]) -> None: + """Schedule the coroutine as a task or a Deferred. + + This doesn't store the reference to the task/Deferred, so a better + alternative is calling :func:`scrapy.utils.defer.deferred_from_coro`, + keeping the result, and adding proper exception handling (e.g. errbacks) to + it. + """ + if not is_asyncio_available(): + Deferred.fromCoroutine(coro) + return + loop = asyncio.get_event_loop() + loop.create_task(coro) # noqa: RUF006 + + +@overload +def ensure_awaitable(o: Awaitable[_T], _warn: str | None = None) -> Awaitable[_T]: ... + + +@overload +def ensure_awaitable(o: _T, _warn: str | None = None) -> Awaitable[_T]: ... + + +def ensure_awaitable(o: _T | Awaitable[_T], _warn: str | None = None) -> Awaitable[_T]: + """Convert any value to an awaitable object. + + For a :class:`~twisted.internet.defer.Deferred` object, use + :func:`maybe_deferred_to_future` to wrap it into a suitable object. For an + awaitable object of a different type, return it as is. For any other + value, return a coroutine that completes with that value. + + .. versionadded:: 2.14 + """ + if isinstance(o, Deferred): + if _warn: + warnings.warn( + f"{_warn} returned a Deferred, this is deprecated." + f" Please refactor this function to return a coroutine.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return maybe_deferred_to_future(o) + if inspect.isawaitable(o): + return o + + async def coro() -> _T: + return o + + return coro() diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index ea577c44a..359f819d7 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -1,10 +1,16 @@ """Some helpers for deprecation messages""" +from __future__ import annotations + import inspect import warnings -from typing import Any, Dict, List, Optional, Tuple, Type, overload +from typing import TYPE_CHECKING, Any, cast, overload from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.python import get_func_args_dict + +if TYPE_CHECKING: + from collections.abc import Callable def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> None: @@ -20,11 +26,11 @@ def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> No def create_deprecated_class( name: str, new_class: type, - clsdict: Optional[Dict[str, Any]] = None, - warn_category: Type[Warning] = ScrapyDeprecationWarning, + clsdict: dict[str, Any] | None = None, + warn_category: type[Warning] = ScrapyDeprecationWarning, warn_once: bool = True, - old_class_path: Optional[str] = None, - new_class_path: Optional[str] = None, + old_class_path: str | None = None, + new_class_path: str | None = None, subclass_warn_message: str = "{cls} inherits from deprecated class {old}, please inherit from {new}.", instance_warn_message: str = "{cls} is deprecated, instantiate {new} instead.", ) -> type: @@ -54,19 +60,20 @@ def create_deprecated_class( """ # https://github.com/python/mypy/issues/4177 - class DeprecatedClass(new_class.__class__): # type: ignore[misc, name-defined] - deprecated_class: Optional[type] = None + class DeprecatedClass(new_class.__class__): # type: ignore[misc,name-defined] + # pylint: disable=no-self-argument + deprecated_class: type | None = None warned_on_subclass: bool = False - def __new__( - metacls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any] + def __new__( # pylint: disable=bad-classmethod-argument + metacls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any] ) -> type: - cls = super().__new__(metacls, name, bases, clsdict_) + cls: type = super().__new__(metacls, name, bases, clsdict_) if metacls.deprecated_class is None: metacls.deprecated_class = cls return cls - def __init__(cls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any]): + def __init__(cls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any]): meta = cls.__class__ old = meta.deprecated_class if old in bases and not (warn_once and meta.warned_on_subclass): @@ -93,7 +100,7 @@ def create_deprecated_class( # is the deprecated class itself - subclasses of the # deprecated class should not use custom `__subclasscheck__` # method. - return super().__subclasscheck__(sub) + return cast("bool", super().__subclasscheck__(sub)) if not inspect.isclass(sub): raise TypeError("issubclass() arg 1 must be a class") @@ -123,28 +130,26 @@ def create_deprecated_class( # deprecated class is in jinja2 template). __module__ attribute is not # important enough to raise an exception as users may be unable # to fix inspect.stack() errors. - warnings.warn(f"Error detecting parent module: {e!r}") + warnings.warn(f"Error detecting parent module: {e!r}", stacklevel=2) return deprecated_cls -def _clspath(cls: type, forced: Optional[str] = None) -> str: +def _clspath(cls: type, forced: str | None = None) -> str: if forced is not None: return forced return f"{cls.__module__}.{cls.__name__}" -DEPRECATION_RULES: List[Tuple[str, str]] = [] +DEPRECATION_RULES: list[tuple[str, str]] = [] @overload -def update_classpath(path: str) -> str: - ... +def update_classpath(path: str) -> str: ... @overload -def update_classpath(path: Any) -> Any: - ... +def update_classpath(path: Any) -> Any: ... def update_classpath(path: Any) -> Any: @@ -155,6 +160,7 @@ def update_classpath(path: Any) -> Any: warnings.warn( f"`{path}` class is deprecated, use `{new_path}` instead", ScrapyDeprecationWarning, + stacklevel=2, ) return new_path return path @@ -178,6 +184,8 @@ def method_is_overridden(subclass: type, base_class: type, method_name: str) -> ... pass >>> class Sub4(Sub2): ... pass + >>> method_is_overridden(Base, Base, 'foo') + False >>> method_is_overridden(Sub1, Base, 'foo') False >>> method_is_overridden(Sub2, Base, 'foo') @@ -190,3 +198,35 @@ def method_is_overridden(subclass: type, base_class: type, method_name: str) -> base_method = getattr(base_class, method_name) sub_method = getattr(subclass, method_name) return base_method.__code__ is not sub_method.__code__ + + +def argument_is_required(func: Callable[..., Any], arg_name: str) -> bool: + """ + Check if a function argument is required (exists and doesn't have a default value). + + .. versionadded:: 2.14 + + >>> def func(a, b=1, c=None): + ... pass + >>> argument_is_required(func, 'a') + True + >>> argument_is_required(func, 'b') + False + >>> argument_is_required(func, 'c') + False + >>> argument_is_required(func, 'd') + False + """ + args = get_func_args_dict(func) + param = args.get(arg_name) + return param is not None and param.default is inspect.Parameter.empty + + +def warn_on_deprecated_spider_attribute(attribute_name: str, setting_name: str) -> None: + warnings.warn( + f"The '{attribute_name}' spider attribute is deprecated. " + "Use Spider.custom_settings or Spider.update_settings() instead. " + f"The corresponding setting name is '{setting_name}'.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index 596cf89e4..08e89dc0d 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -30,17 +30,17 @@ def _tty_supports_color() -> bool: def _colorize(text: str, colorize: bool = True) -> str: + # pylint: disable=no-name-in-module if not colorize or not sys.stdout.isatty() or not _tty_supports_color(): return text try: - from pygments import highlight + from pygments import highlight # noqa: PLC0415 except ImportError: return text - else: - from pygments.formatters import TerminalFormatter - from pygments.lexers import PythonLexer + from pygments.formatters import TerminalFormatter # noqa: PLC0415 + from pygments.lexers import PythonLexer # noqa: PLC0415 - return highlight(text, PythonLexer(), TerminalFormatter()) + return highlight(text, PythonLexer(), TerminalFormatter()) def pformat(obj: Any, *args: Any, **kwargs: Any) -> str: diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index a5f2a8c6e..085720f66 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -1,14 +1,16 @@ """Some debugging functions for working with the Scrapy engine""" +from __future__ import annotations + # used in global tests code from time import time # noqa: F401 -from typing import TYPE_CHECKING, Any, List, Tuple +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from scrapy.core.engine import ExecutionEngine -def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: +def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]: """Return a report of the current engine status""" tests = [ "time()-engine.start_time", @@ -16,10 +18,10 @@ def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: "engine.scraper.is_idle()", "engine.spider.name", "engine.spider_is_idle()", - "engine.slot.closing", - "len(engine.slot.inprogress)", - "len(engine.slot.scheduler.dqs or [])", - "len(engine.slot.scheduler.mqs)", + "engine._slot.closing", + "len(engine._slot.inprogress)", + "len(engine._slot.scheduler.dqs or [])", + "len(engine._slot.scheduler.mqs)", "len(engine.scraper.slot.queue)", "len(engine.scraper.slot.active)", "engine.scraper.slot.active_size", @@ -27,17 +29,17 @@ def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: "engine.scraper.slot.needs_backout()", ] - checks: List[Tuple[str, Any]] = [] + checks: list[tuple[str, Any]] = [] for test in tests: try: - checks += [(test, eval(test))] + checks += [(test, eval(test))] # noqa: S307 except Exception as e: checks += [(test, f"{type(e).__name__} (exception)")] return checks -def format_engine_status(engine: "ExecutionEngine") -> str: +def format_engine_status(engine: ExecutionEngine) -> str: checks = get_engine_status(engine) s = "Execution engine status\n\n" for test, result in checks: @@ -47,5 +49,5 @@ def format_engine_status(engine: "ExecutionEngine") -> str: return s -def print_engine_status(engine: "ExecutionEngine") -> None: +def print_engine_status(engine: ExecutionEngine) -> None: print(format_engine_status(engine)) diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index c77681a53..152f3374e 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -21,7 +21,7 @@ def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None: def ftp_store_file( *, path: str, - file: IO, + file: IO[bytes], host: str, port: int, username: str, diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index c7f74030e..5f95bccbd 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,31 +1,39 @@ +from __future__ import annotations + import struct from gzip import GzipFile from io import BytesIO -from typing import List +from typing import TYPE_CHECKING -from scrapy.http import Response +from ._compression import _CHUNK_SIZE, _check_max_size + +if TYPE_CHECKING: + from scrapy.http import Response -def gunzip(data: bytes) -> bytes: +def gunzip(data: bytes, *, max_size: int = 0) -> bytes: """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. """ f = GzipFile(fileobj=BytesIO(data)) - output_list: List[bytes] = [] + output_stream = BytesIO() chunk = b"." + decompressed_size = 0 while chunk: try: - chunk = f.read1(8196) - output_list.append(chunk) + chunk = f.read1(_CHUNK_SIZE) except (OSError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error - # some pages are quite small so output_list is empty - if output_list: + # some pages are quite small so output_stream is empty + if output_stream.getbuffer().nbytes > 0: break raise - return b"".join(output_list) + decompressed_size += len(chunk) + _check_max_size(decompressed_size, max_size) + output_stream.write(chunk) + return output_stream.getvalue() def gzip_magic_number(response: Response) -> bool: diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index d502e8910..58b4539bf 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -1,17 +1,21 @@ """Helper functions for scrapy.http objects (Request, Response)""" -from typing import Union +from __future__ import annotations + +from typing import TYPE_CHECKING from urllib.parse import ParseResult, urlparse from weakref import WeakKeyDictionary -from scrapy.http import Request, Response +if TYPE_CHECKING: + from scrapy.http import Request, Response -_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = ( + +_urlparse_cache: WeakKeyDictionary[Request | Response, ParseResult] = ( WeakKeyDictionary() ) -def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult: +def urlparse_cached(request_or_response: Request | Response) -> ParseResult: """Return urlparse.urlparse caching the result, where the argument can be a Request or Response object """ diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 55362efdf..9bb64c021 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,35 +1,26 @@ +from __future__ import annotations + import csv import logging import re from io import StringIO -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Generator, - Iterable, - List, - Literal, - Optional, - Union, - cast, - overload, -) +from typing import TYPE_CHECKING, Any, Literal, cast, overload +from warnings import warn +from lxml import etree + +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse from scrapy.selector import Selector -from scrapy.utils.python import re_rsearch, to_unicode +from scrapy.utils.python import re_rsearch if TYPE_CHECKING: - from lxml._types import SupportsReadClose + from collections.abc import Callable, Iterator logger = logging.getLogger(__name__) -def xmliter( - obj: Union[Response, str, bytes], nodename: str -) -> Generator[Selector, Any, None]: +def xmliter(obj: Response | str | bytes, nodename: str) -> Iterator[Selector]: """Return a iterator of Selector's over all nodes of a XML document, given the name of the node to iterate. Useful for parsing XML feeds. @@ -38,12 +29,22 @@ def xmliter( - a unicode string - a string encoded as utf-8 """ + warn( + ( + "xmliter is deprecated and its use strongly discouraged because " + "it is vulnerable to ReDoS attacks. Use xmliter_lxml instead. See " + "https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9" + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) + nodename_patt = re.escape(nodename) - DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S) - HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.S) - END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.S) - NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S) + DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.DOTALL) + HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.DOTALL) + END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.DOTALL) + NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.DOTALL) text = _body_or_str(obj) document_header_match = re.search(DOCUMENT_HEADER_RE, text) @@ -52,12 +53,14 @@ def xmliter( ) header_end_idx = re_rsearch(HEADER_END_RE, text) header_end = text[header_end_idx[1] :].strip() if header_end_idx else "" - namespaces: Dict[str, str] = {} + namespaces: dict[str, str] = {} if header_end: for tagname in reversed(re.findall(END_TAG_RE, header_end)): assert header_end_idx tag = re.search( - rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S + rf"<\s*{tagname}.*?xmlns[:=][^>]*>", + text[: header_end_idx[1]], + re.DOTALL, ) if tag: for x in re.findall(NAMESPACE_RE, tag.group()): @@ -68,7 +71,7 @@ def xmliter( nodetext = ( document_header + match.group().replace( - nodename, f'{nodename} {" ".join(namespaces.values())}', 1 + nodename, f"{nodename} {' '.join(namespaces.values())}", 1 ) + header_end ) @@ -76,20 +79,40 @@ def xmliter( def xmliter_lxml( - obj: Union[Response, str, bytes], + obj: Response | str | bytes, nodename: str, - namespace: Optional[str] = None, + namespace: str | None = None, prefix: str = "x", -) -> Generator[Selector, Any, None]: - from lxml import etree - +) -> Iterator[Selector]: reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( - cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding + reader, + encoding=reader.encoding, + events=("end", "start-ns"), + resolve_entities=False, + huge_tree=True, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) - for _, node in iterable: + needs_namespace_resolution = not namespace and ":" in nodename + if needs_namespace_resolution: + prefix, nodename = nodename.split(":", maxsplit=1) + for event, data in iterable: + if event == "start-ns": + assert isinstance(data, tuple) + if needs_namespace_resolution: + _prefix, _namespace = data + if _prefix != prefix: + continue + namespace = _namespace + needs_namespace_resolution = False + selxpath = f"//{prefix}:{nodename}" + tag = f"{{{namespace}}}{nodename}" + continue + assert isinstance(data, etree._Element) + node = data + if node.tag != tag: + continue nodetext = etree.tostring(node, encoding="unicode") node.clear() xs = Selector(text=nodetext, type="xml") @@ -99,9 +122,9 @@ def xmliter_lxml( class _StreamReader: - def __init__(self, obj: Union[Response, str, bytes]): + def __init__(self, obj: Response | str | bytes): self._ptr: int = 0 - self._text: Union[str, bytes] + self._text: str | bytes if isinstance(obj, TextResponse): self._text, self.encoding = obj.body, obj.encoding elif isinstance(obj, Response): @@ -124,21 +147,21 @@ class _StreamReader: def _read_string(self, n: int = 65535) -> bytes: s, e = self._ptr, self._ptr + n self._ptr = e - return cast(bytes, self._text)[s:e] + return cast("bytes", self._text)[s:e] def _read_unicode(self, n: int = 65535) -> bytes: s, e = self._ptr, self._ptr + n self._ptr = e - return cast(str, self._text)[s:e].encode("utf-8") + return cast("str", self._text)[s:e].encode("utf-8") def csviter( - obj: Union[Response, str, bytes], - delimiter: Optional[str] = None, - headers: Optional[List[str]] = None, - encoding: Optional[str] = None, - quotechar: Optional[str] = None, -) -> Generator[Dict[str, str], Any, None]: + obj: Response | str | bytes, + delimiter: str | None = None, + headers: list[str] | None = None, + encoding: str | None = None, + quotechar: str | None = None, +) -> Iterator[dict[str, str]]: """Returns an iterator of dictionaries from the given csv object obj can be: @@ -154,14 +177,17 @@ def csviter( quotechar is the character used to enclosure fields on the given obj. """ - encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8" - - def row_to_unicode(row_: Iterable) -> List[str]: - return [to_unicode(field, encoding) for field in row_] + if encoding is not None: # pragma: no cover + warn( + "The encoding argument of csviter() is ignored and will be removed" + " in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) lines = StringIO(_body_or_str(obj, unicode=True)) - kwargs: Dict[str, Any] = {} + kwargs: dict[str, Any] = {} if delimiter: kwargs["delimiter"] = delimiter if quotechar: @@ -170,13 +196,11 @@ def csviter( if not headers: try: - row = next(csv_r) + headers = next(csv_r) except StopIteration: return - headers = row_to_unicode(row) for row in csv_r: - row = row_to_unicode(row) if len(row) != len(headers): logger.warning( "ignoring row %(csvlnum)d (length: %(csvrow)d, " @@ -188,27 +212,22 @@ def csviter( }, ) continue - yield dict(zip(headers, row)) + yield dict(zip(headers, row, strict=False)) @overload -def _body_or_str(obj: Union[Response, str, bytes]) -> str: - ... +def _body_or_str(obj: Response | str | bytes) -> str: ... @overload -def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: - ... +def _body_or_str(obj: Response | str | bytes, unicode: Literal[True]) -> str: ... @overload -def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[False]) -> bytes: - ... +def _body_or_str(obj: Response | str | bytes, unicode: Literal[False]) -> bytes: ... -def _body_or_str( - obj: Union[Response, str, bytes], unicode: bool = True -) -> Union[str, bytes]: +def _body_or_str(obj: Response | str | bytes, unicode: bool = True) -> str | bytes: expected_types = (Response, str, bytes) if not isinstance(obj, expected_types): expected_types_str = " or ".join(t.__name__ for t in expected_types) diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index e230e4235..37e6aeb51 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -1,11 +1,14 @@ +from __future__ import annotations + from pathlib import Path -from typing import Optional +from typing import TYPE_CHECKING -from scrapy.settings import BaseSettings +if TYPE_CHECKING: + from scrapy.settings import BaseSettings -def job_dir(settings: BaseSettings) -> Optional[str]: - path: Optional[str] = settings["JOBDIR"] +def job_dir(settings: BaseSettings) -> str | None: + path: str | None = settings["JOBDIR"] if not path: return None if not Path(path).exists(): diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 2a38f151a..ee65d4155 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -1,27 +1,33 @@ from __future__ import annotations import logging +import pprint import sys +from collections.abc import MutableMapping from logging.config import dictConfig -from types import TracebackType -from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast +from typing import TYPE_CHECKING, Any, cast +from twisted.internet import asyncioreactor from twisted.python import log as twisted_log from twisted.python.failure import Failure import scrapy -from scrapy.settings import Settings -from scrapy.utils.versions import scrapy_components_versions +from scrapy.settings import Settings, _SettingsKey +from scrapy.utils.versions import get_versions if TYPE_CHECKING: + from types import TracebackType + from scrapy.crawler import Crawler + from scrapy.logformatter import LogFormatterResult + logger = logging.getLogger(__name__) def failure_to_exc_info( failure: Failure, -) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]: +) -> tuple[type[BaseException], BaseException, TracebackType | None] | None: """Extract exc_info from Failure instances""" if isinstance(failure, Failure): assert failure.type @@ -29,13 +35,13 @@ def failure_to_exc_info( return ( failure.type, failure.value, - cast(Optional[TracebackType], failure.getTracebackObject()), + cast("TracebackType | None", failure.getTracebackObject()), ) return None class TopLevelFormatter(logging.Filter): - """Keep only top level loggers's name (direct children from root) from + """Keep only top level loggers' name (direct children from root) from records. This filter will replace Scrapy loggers' names with 'scrapy'. This mimics @@ -46,8 +52,9 @@ class TopLevelFormatter(logging.Filter): ``loggers`` list where it should act. """ - def __init__(self, loggers: Optional[List[str]] = None): - self.loggers: List[str] = loggers or [] + def __init__(self, loggers: list[str] | None = None): + super().__init__() + self.loggers: list[str] = loggers or [] def filter(self, record: logging.LogRecord) -> bool: if any(record.name.startswith(logger + ".") for logger in self.loggers): @@ -65,6 +72,12 @@ DEFAULT_LOGGING = { "hpack": { "level": "ERROR", }, + "httpcore": { + "level": "ERROR", + }, + "httpx": { + "level": "WARNING", + }, "scrapy": { "level": "DEBUG", }, @@ -76,7 +89,8 @@ DEFAULT_LOGGING = { def configure_logging( - settings: Union[Settings, dict, None] = None, install_root_handler: bool = True + settings: Settings | dict[_SettingsKey, Any] | None = None, + install_root_handler: bool = True, ) -> None: """ Initialize logging defaults for Scrapy. @@ -114,29 +128,36 @@ def configure_logging( settings = Settings(settings) if settings.getbool("LOG_STDOUT"): - sys.stdout = StreamLogger(logging.getLogger("stdout")) # type: ignore[assignment] + sys.stdout = StreamLogger(logging.getLogger("stdout")) if install_root_handler: install_scrapy_root_handler(settings) -_scrapy_root_handler: Optional[logging.Handler] = None +_scrapy_root_handler: logging.Handler | None = None def install_scrapy_root_handler(settings: Settings) -> None: - global _scrapy_root_handler + global _scrapy_root_handler # noqa: PLW0603 + + _uninstall_scrapy_root_handler() + logging.root.setLevel(logging.NOTSET) + _scrapy_root_handler = _get_handler(settings) + logging.root.addHandler(_scrapy_root_handler) + + +def _uninstall_scrapy_root_handler() -> None: + global _scrapy_root_handler # noqa: PLW0603 if ( _scrapy_root_handler is not None and _scrapy_root_handler in logging.root.handlers ): logging.root.removeHandler(_scrapy_root_handler) - logging.root.setLevel(logging.NOTSET) - _scrapy_root_handler = _get_handler(settings) - logging.root.addHandler(_scrapy_root_handler) + _scrapy_root_handler = None -def get_scrapy_root_handler() -> Optional[logging.Handler]: +def get_scrapy_root_handler() -> logging.Handler | None: return _scrapy_root_handler @@ -168,20 +189,17 @@ def log_scrapy_info(settings: Settings) -> None: "Scrapy %(version)s started (bot: %(bot)s)", {"version": scrapy.__version__, "bot": settings["BOT_NAME"]}, ) - versions = [ - f"{name} {version}" - for name, version in scrapy_components_versions() - if name != "Scrapy" - ] - logger.info("Versions: %(versions)s", {"versions": ", ".join(versions)}) + software: list[str] = settings.getlist("LOG_VERSIONS") + if not software: + return + versions = pprint.pformat(dict(get_versions(software)), sort_dicts=False) + logger.info(f"Versions:\n{versions}") def log_reactor_info() -> None: from twisted.internet import reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) - from twisted.internet import asyncioreactor - if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor): logger.debug( "Using asyncio event loop: %s.%s", @@ -224,7 +242,9 @@ class LogCounterHandler(logging.Handler): self.crawler.stats.inc_value(sname) -def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]: +def logformatter_adapter( + logkws: LogFormatterResult, +) -> tuple[int, str, dict[str, Any] | tuple[Any, ...]]: """ Helper that takes the dictionary output from the methods in LogFormatter and adapts it into a tuple of positional arguments for logger.log calls, @@ -235,6 +255,20 @@ def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]: message = logkws.get("msg") or "" # NOTE: This also handles 'args' being an empty dict, that case doesn't # play well in logger.log calls - args = logkws if not logkws.get("args") else logkws["args"] + args = cast("dict[str, Any]", logkws) if not logkws.get("args") else logkws["args"] return (level, message, args) + + +# LoggerAdapter is only parameterized since Python 3.11 +class SpiderLoggerAdapter(logging.LoggerAdapter): # type: ignore[type-arg] + def process( + self, msg: str, kwargs: MutableMapping[str, Any] + ) -> tuple[str, MutableMapping[str, Any]]: + """Method that augments logging with additional 'extra' data""" + if isinstance(kwargs.get("extra"), MutableMapping): + kwargs["extra"].update(self.extra) + else: + kwargs["extra"] = self.extra + + return msg, kwargs diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index b38190cb3..0b67eaa34 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,4 +1,7 @@ """Helper functions which don't fit anywhere else""" + +from __future__ import annotations + import ast import hashlib import inspect @@ -10,32 +13,35 @@ from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules -from types import ModuleType -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Deque, - Generator, - Iterable, - List, - Optional, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Any, ParamSpec, Protocol, TypeVar, cast, overload from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item from scrapy.utils.datatypes import LocalWeakReferencedCache if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Iterator + from types import ModuleType + from scrapy import Spider + from scrapy.crawler import Crawler _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes +_ITER_T = TypeVar("_ITER_T", bound=dict[Any, Any] | Item | str | bytes) +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) +_P = ParamSpec("_P") +@overload +def arg_to_iter(arg: None) -> tuple[()]: ... +@overload +def arg_to_iter(arg: _ITER_T) -> Iterable[_ITER_T]: ... +@overload +def arg_to_iter(arg: Iterable[_T]) -> Iterable[_T]: ... +@overload +def arg_to_iter(arg: _T) -> Iterable[_T]: ... def arg_to_iter(arg: Any) -> Iterable[Any]: """Convert an argument to an iterable. The argument can be a None, single value, or an iterable. @@ -43,13 +49,13 @@ def arg_to_iter(arg: Any) -> Iterable[Any]: Exception: if arg is a dict, [arg] will be returned """ if arg is None: - return [] + return () if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"): - return cast(Iterable[Any], arg) + return cast("Iterable[Any]", arg) return [arg] -def load_object(path: Union[str, Callable]) -> Any: +def load_object(path: str | Callable[..., Any]) -> Any: """Load an object given its absolute object path, and return it. The object can be the import path of a class, function, variable or an @@ -69,7 +75,7 @@ def load_object(path: Union[str, Callable]) -> Any: try: dot = path.rindex(".") except ValueError: - raise ValueError(f"Error loading object '{path}': not a full path") + raise ValueError(f"Error loading object '{path}': not a full path") from None module, name = path[:dot], path[dot + 1 :] mod = import_module(module) @@ -77,34 +83,59 @@ def load_object(path: Union[str, Callable]) -> Any: try: obj = getattr(mod, name) except AttributeError: - raise NameError(f"Module '{module}' doesn't define any object named '{name}'") + raise NameError( + f"Module '{module}' doesn't define any object named '{name}'" + ) from None return obj -def walk_modules(path: str) -> List[ModuleType]: +def walk_modules_iter(path: str) -> Iterable[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that exception is thrown back. - For example: walk_modules('scrapy.utils') + For example: + >>> list(walk_modules_iter('scrapy.utils')) + [, ...] + >>> gen = walk_modules_iter('scrapy.utils.nonexistent') # error not raised until the generator is consumed + >>> list(gen) + Traceback (most recent call last): + ... + ModuleNotFoundError: No module named 'scrapy.utils.nonexistent' """ - mods: List[ModuleType] = [] mod = import_module(path) - mods.append(mod) + yield mod if hasattr(mod, "__path__"): for _, subpath, ispkg in iter_modules(mod.__path__): fullpath = path + "." + subpath if ispkg: - mods += walk_modules(fullpath) + yield from walk_modules_iter(fullpath) else: - submod = import_module(fullpath) - mods.append(submod) - return mods + yield import_module(fullpath) -def md5sum(file: IO) -> str: +def walk_modules(path: str) -> list[ModuleType]: # pragma: no cover + """ + Loads a module and all its submodules from the given module path and + returns them. If *any* module throws an exception while importing, that + exception is thrown back. + """ + warnings.warn( + ( + "The scrapy.utils.misc.walk_modules function is deprecated and will be " + "removed in a future version of Scrapy. " + "Use scrapy.utils.misc.walk_modules_iter instead." + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) + + return list(walk_modules_iter(path)) + + +def md5sum(file: IO[bytes]) -> str: """Calculate the md5 checksum of a file-like object without reading its whole content in memory. @@ -112,7 +143,15 @@ def md5sum(file: IO) -> str: >>> md5sum(BytesIO(b'file content to hash')) '784406af91dd5a54fbb9c84c2236595a' """ - m = hashlib.md5() + warnings.warn( + ( + "The scrapy.utils.misc.md5sum function is deprecated and will be " + "removed in a future version of Scrapy." + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) + m = hashlib.md5() # noqa: S324 while True: d = file.read(8096) if not d: @@ -121,55 +160,48 @@ def md5sum(file: IO) -> str: return m.hexdigest() -def rel_has_nofollow(rel: Optional[str]) -> bool: +def rel_has_nofollow(rel: str | None) -> bool: """Return True if link rel attribute has nofollow type""" return rel is not None and "nofollow" in rel.replace(",", " ").split() -def create_instance(objcls, settings, crawler, *args, **kwargs): - """Construct a class instance using its ``from_crawler`` or - ``from_settings`` constructors, if available. - - At least one of ``settings`` and ``crawler`` needs to be different from - ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used. - If ``crawler`` is ``None``, only the ``from_settings`` constructor will be - tried. - - ``*args`` and ``**kwargs`` are forwarded to the constructors. - - Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``. - - .. versionchanged:: 2.2 - Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an - extension has not been implemented correctly). - """ - warnings.warn( - "The create_instance() function is deprecated. " - "Please use build_from_crawler() or build_from_settings() instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - - if settings is None: - if crawler is None: - raise ValueError("Specify at least one of settings and crawler.") - settings = crawler.settings - if crawler and hasattr(objcls, "from_crawler"): - instance = objcls.from_crawler(crawler, *args, **kwargs) - method_name = "from_crawler" - elif hasattr(objcls, "from_settings"): - instance = objcls.from_settings(settings, *args, **kwargs) - method_name = "from_settings" - else: - instance = objcls(*args, **kwargs) - method_name = "__new__" - if instance is None: - raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") - return instance +class SupportsFromCrawler(Protocol[_T_co, _P]): + @classmethod + def from_crawler( + cls, crawler: Crawler, /, *args: _P.args, **kwargs: _P.kwargs + ) -> _T_co: ... -def build_from_crawler(objcls, crawler, /, *args, **kwargs): - """Construct a class instance using its ``from_crawler`` constructor. +@overload +def build_from_crawler( + objcls: SupportsFromCrawler[_T_co, _P], + crawler: Crawler, + /, + *args: _P.args, + **kwargs: _P.kwargs, +) -> _T_co: ... + + +@overload +def build_from_crawler( + objcls: Callable[_P, _T_co], + crawler: Crawler, + /, + *args: _P.args, + **kwargs: _P.kwargs, +) -> _T_co: ... + + +def build_from_crawler( + objcls: Any, + crawler: Crawler, + /, + *args: Any, + **kwargs: Any, +) -> Any: + """Construct a class instance using its ``from_crawler()`` or ``__init__()`` constructor. + + .. versionadded:: 2.12 ``*args`` and ``**kwargs`` are forwarded to the constructor. @@ -178,27 +210,6 @@ def build_from_crawler(objcls, crawler, /, *args, **kwargs): if hasattr(objcls, "from_crawler"): instance = objcls.from_crawler(crawler, *args, **kwargs) method_name = "from_crawler" - elif hasattr(objcls, "from_settings"): - instance = objcls.from_settings(crawler.settings, *args, **kwargs) - method_name = "from_settings" - else: - instance = objcls(*args, **kwargs) - method_name = "__new__" - if instance is None: - raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") - return instance - - -def build_from_settings(objcls, settings, /, *args, **kwargs): - """Construct a class instance using its ``from_settings`` constructor. - - ``*args`` and ``**kwargs`` are forwarded to the constructor. - - Raises ``TypeError`` if the resulting instance is ``None``. - """ - if hasattr(objcls, "from_settings"): - instance = objcls.from_settings(settings, *args, **kwargs) - method_name = "from_settings" else: instance = objcls(*args, **kwargs) method_name = "__new__" @@ -208,7 +219,7 @@ def build_from_settings(objcls, settings, /, *args, **kwargs): @contextmanager -def set_environ(**kwargs: str) -> Generator[None, Any, None]: +def set_environ(**kwargs: str) -> Iterator[None]: """Temporarily set environment variables inside the context manager and fully restore previous environment afterwards """ @@ -225,11 +236,11 @@ def set_environ(**kwargs: str) -> Generator[None, Any, None]: os.environ[k] = v -def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]: +def walk_callable(node: ast.AST) -> Iterable[ast.AST]: """Similar to ``ast.walk``, but walks only function body and skips nested functions defined within the node. """ - todo: Deque[ast.AST] = deque([node]) + todo: deque[ast.AST] = deque([node]) walked_func_def = False while todo: node = todo.popleft() @@ -241,10 +252,17 @@ def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]: yield node -_generator_callbacks_cache = LocalWeakReferencedCache(limit=128) +_generator_callbacks_cache: LocalWeakReferencedCache[Callable[..., Any], bool] = ( + LocalWeakReferencedCache(limit=128) +) -def is_generator_with_return_value(callable: Callable) -> bool: +def _returns_none(return_node: ast.Return) -> bool: + value = return_node.value + return value is None or (isinstance(value, ast.Constant) and value.value is None) + + +def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: # noqa: A002 """ Returns True if a callable is a generator function which includes a 'return' statement with a value different than None, False otherwise @@ -252,12 +270,6 @@ def is_generator_with_return_value(callable: Callable) -> bool: if callable in _generator_callbacks_cache: return bool(_generator_callbacks_cache[callable]) - def returns_none(return_node: ast.Return) -> bool: - value = return_node.value - return ( - value is None or isinstance(value, ast.NameConstant) and value.value is None - ) - if inspect.isgeneratorfunction(callable): func = callable while isinstance(func, partial): @@ -273,7 +285,7 @@ def is_generator_with_return_value(callable: Callable) -> bool: tree = ast.parse(code) for node in walk_callable(tree): - if isinstance(node, ast.Return) and not returns_none(node): + if isinstance(node, ast.Return) and not _returns_none(node): _generator_callbacks_cache[callable] = True return bool(_generator_callbacks_cache[callable]) @@ -281,11 +293,16 @@ def is_generator_with_return_value(callable: Callable) -> bool: return bool(_generator_callbacks_cache[callable]) -def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None: +def warn_on_generator_with_return_value( + spider: Spider, + callable: Callable[..., Any], # noqa: A002 +) -> None: """ Logs a warning if a callable is a generator function and includes a 'return' statement with a value different than None """ + if not spider.settings.getbool("WARN_ON_GENERATOR_RETURN_VALUE"): + return try: if is_generator_with_return_value(callable): warnings.warn( diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index db9a71273..4eda29e25 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,13 +1,16 @@ +from __future__ import annotations + import signal +from collections.abc import Callable from types import FrameType -from typing import Any, Callable, Dict, Optional, Union +from typing import Any, TypeAlias # copy of _HANDLER from typeshed/stdlib/signal.pyi -SignalHandlerT = Union[ - Callable[[int, Optional[FrameType]], Any], int, signal.Handlers, None -] +SignalHandlerT: TypeAlias = ( + Callable[[int, FrameType | None], Any] | int | signal.Handlers | None +) -signal_names: Dict[int, str] = {} +signal_names: dict[int, str] = {} for signame in dir(signal): if signame.startswith("SIG") and not signame.startswith("SIG_"): signum = getattr(signal, signame) @@ -24,7 +27,11 @@ def install_shutdown_handlers( (e.g. Pdb) """ signal.signal(signal.SIGTERM, function) - if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint: + if ( + signal.getsignal(signal.SIGINT) # pylint: disable=comparison-with-callable + == signal.default_int_handler + or override_sigint + ): signal.signal(signal.SIGINT, function) # Catch Ctrl-Break in windows if hasattr(signal, "SIGBREAK"): diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index de3c8eaf9..3e75c7729 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,9 +1,9 @@ +from __future__ import annotations + import os import warnings from importlib import import_module -from os import PathLike from pathlib import Path -from typing import Union from scrapy.exceptions import NotConfigured from scrapy.settings import Settings @@ -20,7 +20,8 @@ def inside_project() -> bool: import_module(scrapy_module) except ImportError as exc: warnings.warn( - f"Cannot import scrapy settings module {scrapy_module}: {exc}" + f"Cannot import scrapy settings module {scrapy_module}: {exc}", + stacklevel=2, ) else: return True @@ -46,7 +47,7 @@ def project_data_dir(project: str = "default") -> str: return str(d) -def data_path(path: Union[str, PathLike], createdir: bool = False) -> str: +def data_path(path: str | os.PathLike[str], createdir: bool = False) -> str: """ Return the given path joined with the .scrapy data directory. If given an absolute path, return it unmodified. diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7ca22fc80..ebd7dae9e 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,67 +1,35 @@ """ This module contains essential stuff that should've come with Python itself ;) """ -import collections.abc + +from __future__ import annotations + import gc import inspect import re import sys +import warnings import weakref +from collections.abc import AsyncIterator, Iterable, Mapping from functools import partial, wraps from itertools import chain -from typing import ( - Any, - AsyncGenerator, - AsyncIterable, - AsyncIterator, - Callable, - Dict, - Generator, - Iterable, - Iterator, - List, - Mapping, - Optional, - Pattern, - Tuple, - Union, - overload, -) -from warnings import warn +from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, overload from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + from re import Pattern -def flatten(x: Iterable) -> list: - """flatten(sequence) -> list - - Returns a single, flat list which contains all elements retrieved - from the sequence and all recursively contained sub-sequences - (iterables). - - Examples: - >>> [1, 2, [3,4], (5,6)] - [1, 2, [3, 4], (5, 6)] - >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)]) - [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10] - >>> flatten(["foo", "bar"]) - ['foo', 'bar'] - >>> flatten(["foo", ["baz", 42], "bar"]) - ['foo', 'baz', 42, 'bar'] - """ - return list(iflatten(x)) + # typing.Self requires Python 3.11 + from typing_extensions import Self -def iflatten(x: Iterable) -> Iterable: - """iflatten(sequence) -> iterator - - Similar to ``.flatten()``, but returns iterator instead""" - for el in x: - if is_listlike(el): - yield from iflatten(el) - else: - yield el +_T = TypeVar("_T") +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") +_P = ParamSpec("_P") def is_listlike(x: Any) -> bool: @@ -88,10 +56,10 @@ def is_listlike(x: Any) -> bool: return hasattr(x, "__iter__") and not isinstance(x, (str, bytes)) -def unique(list_: Iterable, key: Callable[[Any], Any] = lambda x: x) -> list: +def unique(list_: Iterable[_T], key: Callable[[_T], Any] = lambda x: x) -> list[_T]: """efficient function to uniquify a list preserving item order""" seen = set() - result = [] + result: list[_T] = [] for item in list_: seenkey = key(item) if seenkey in seen: @@ -102,7 +70,7 @@ def unique(list_: Iterable, key: Callable[[Any], Any] = lambda x: x) -> list: def to_unicode( - text: Union[str, bytes], encoding: Optional[str] = None, errors: str = "strict" + text: str | bytes, encoding: str | None = None, errors: str = "strict" ) -> str: """Return the unicode representation of a bytes object ``text``. If ``text`` is already an unicode object, return it as-is.""" @@ -110,8 +78,7 @@ def to_unicode( return text if not isinstance(text, (bytes, str)): raise TypeError( - "to_unicode must receive a bytes or str " - f"object, got {type(text).__name__}" + f"to_unicode must receive a bytes or str object, got {type(text).__name__}" ) if encoding is None: encoding = "utf-8" @@ -119,7 +86,7 @@ def to_unicode( def to_bytes( - text: Union[str, bytes], encoding: Optional[str] = None, errors: str = "strict" + text: str | bytes, encoding: str | None = None, errors: str = "strict" ) -> bytes: """Return the binary representation of ``text``. If ``text`` is already a bytes object, return it as-is.""" @@ -127,16 +94,26 @@ def to_bytes( return text if not isinstance(text, str): raise TypeError( - "to_bytes must receive a str or bytes " f"object, got {type(text).__name__}" + f"to_bytes must receive a str or bytes object, got {type(text).__name__}" ) if encoding is None: encoding = "utf-8" return text.encode(encoding, errors) +def _chunk_iter(text: str, chunk_size: int) -> Iterable[tuple[str, int]]: + offset = len(text) + while True: + offset -= chunk_size * 1024 + if offset <= 0: + break + yield (text[offset:], offset) + yield (text, 0) + + def re_rsearch( - pattern: Union[str, Pattern], text: str, chunk_size: int = 1024 -) -> Optional[Tuple[int, int]]: + pattern: str | Pattern[str], text: str, chunk_size: int = 1024 +) -> tuple[int, int] | None: """ This function does a reverse search in a text using a regular expression given in the attribute 'pattern'. @@ -150,34 +127,30 @@ def re_rsearch( the start position of the match, and the ending (regarding the entire text). """ - def _chunk_iter() -> Generator[Tuple[str, int], Any, None]: - offset = len(text) - while True: - offset -= chunk_size * 1024 - if offset <= 0: - break - yield (text[offset:], offset) - yield (text, 0) - if isinstance(pattern, str): pattern = re.compile(pattern) - for chunk, offset in _chunk_iter(): - matches = [match for match in pattern.finditer(chunk)] + for chunk, offset in _chunk_iter(text, chunk_size): + matches = list(pattern.finditer(chunk)) if matches: start, end = matches[-1].span() return offset + start, offset + end return None -def memoizemethod_noargs(method: Callable) -> Callable: +_SelfT = TypeVar("_SelfT") + + +def memoizemethod_noargs( + method: Callable[Concatenate[_SelfT, _P], _T], +) -> Callable[Concatenate[_SelfT, _P], _T]: """Decorator to cache the result of a method (without arguments) using a weak reference to its object """ - cache: weakref.WeakKeyDictionary[Any, Any] = weakref.WeakKeyDictionary() + cache: weakref.WeakKeyDictionary[_SelfT, _T] = weakref.WeakKeyDictionary() @wraps(method) - def new_method(self: Any, *args: Any, **kwargs: Any) -> Any: + def new_method(self: _SelfT, *args: _P.args, **kwargs: _P.kwargs) -> _T: if self not in cache: cache[self] = method(self, *args, **kwargs) return cache[self] @@ -194,7 +167,7 @@ def binary_is_text(data: bytes) -> bool: """Returns ``True`` if the given ``data`` argument (a ``bytes`` object) does not contain unprintable control characters. """ - warn( + warnings.warn( ( "scrapy.utils.python.binary_is_text is deprecated, use " "xtractmime.is_binary_data instead." @@ -207,37 +180,47 @@ def binary_is_text(data: bytes) -> bool: return all(c not in _BINARYCHARS for c in data) -def get_func_args(func: Callable, stripself: bool = False) -> List[str]: - """Return the argument name list of a callable object""" +def get_func_args_dict( + func: Callable[..., Any], stripself: bool = False +) -> Mapping[str, inspect.Parameter]: + """Return the argument dict of a callable object. + + .. versionadded:: 2.14 + """ if not callable(func): raise TypeError(f"func must be callable, got '{type(func).__name__}'") - args: List[str] = [] + args: Mapping[str, inspect.Parameter] try: sig = inspect.signature(func) except ValueError: - return args + return {} if isinstance(func, partial): partial_args = func.args partial_kw = func.keywords + args = {} for name, param in sig.parameters.items(): - if param.name in partial_args: + if name in partial_args: continue - if partial_kw and param.name in partial_kw: + if partial_kw and name in partial_kw: continue - args.append(name) + args[name] = param else: - for name in sig.parameters.keys(): - args.append(name) + args = sig.parameters - if stripself and args and args[0] == "self": - args = args[1:] + if stripself and args and "self" in args: + args = {k: v for k, v in args.items() if k != "self"} return args -def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]: +def get_func_args(func: Callable[..., Any], stripself: bool = False) -> list[str]: + """Return the argument name list of a callable object""" + return list(get_func_args_dict(func, stripself=stripself)) + + +def get_spec(func: Callable[..., Any]) -> tuple[list[str], dict[str, Any]]: """Returns (args, kwargs) tuple for a function >>> import re >>> get_spec(re.match) @@ -261,71 +244,51 @@ def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]: if inspect.isfunction(func) or inspect.ismethod(func): spec = inspect.getfullargspec(func) - elif hasattr(func, "__call__"): + elif hasattr(func, "__call__"): # noqa: B004 spec = inspect.getfullargspec(func.__call__) else: raise TypeError(f"{type(func)} is not callable") - defaults: Tuple[Any, ...] = spec.defaults or () + defaults: tuple[Any, ...] = spec.defaults or () firstdefault = len(spec.args) - len(defaults) args = spec.args[:firstdefault] - kwargs = dict(zip(spec.args[firstdefault:], defaults)) + kwargs = dict(zip(spec.args[firstdefault:], defaults, strict=False)) return args, kwargs -def equal_attributes( - obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable]]] -) -> bool: - """Compare two objects attributes""" - # not attributes given return False by default - if not attributes: - return False - - temp1, temp2 = object(), object() - for attr in attributes: - # support callables like itemgetter - if callable(attr): - if attr(obj1) != attr(obj2): - return False - elif getattr(obj1, attr, temp1) != getattr(obj2, attr, temp2): - return False - # all attributes equal - return True +@overload +def without_none_values(iterable: Mapping[_KT, _VT]) -> dict[_KT, _VT]: ... @overload -def without_none_values(iterable: Mapping) -> dict: - ... +def without_none_values(iterable: Iterable[_KT]) -> Iterable[_KT]: ... -@overload -def without_none_values(iterable: Iterable) -> Iterable: - ... - - -def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]: +def without_none_values( + iterable: Mapping[_KT, _VT] | Iterable[_KT], +) -> dict[_KT, _VT] | Iterable[_KT]: """Return a copy of ``iterable`` with all ``None`` entries removed. If ``iterable`` is a mapping, return a dictionary where all pairs that have value ``None`` have been removed. """ - if isinstance(iterable, collections.abc.Mapping): + if isinstance(iterable, Mapping): return {k: v for k, v in iterable.items() if v is not None} - else: - # the iterable __init__ must take another iterable - return type(iterable)(v for v in iterable if v is not None) # type: ignore[call-arg] + # the iterable __init__ must take another iterable + return type(iterable)(v for v in iterable if v is not None) # type: ignore[call-arg] def global_object_name(obj: Any) -> str: - """ - Return full name of a global object. + """Return the full import path of the given object. >>> from scrapy import Request >>> global_object_name(Request) 'scrapy.http.request.Request' + >>> global_object_name(Request.replace) + 'scrapy.http.request.Request.replace' """ - return f"{obj.__module__}.{obj.__name__}" + return f"{obj.__module__}.{obj.__qualname__}" if hasattr(sys, "pypy_version_info"): @@ -341,43 +304,76 @@ else: gc.collect() -class MutableChain(Iterable): - """ - Thin wrapper around itertools.chain, allowing to add iterables "in-place" - """ +class MutableChain(Iterable[_T]): # pragma: no cover + def __init__(self, *args: Iterable[_T]): + warnings.warn( + "MutableChain is deprecated and will be removed in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self.data: Iterator[_T] = chain.from_iterable(args) - def __init__(self, *args: Iterable): - self.data = chain.from_iterable(args) - - def extend(self, *iterables: Iterable) -> None: + def extend(self, *iterables: Iterable[_T]) -> None: self.data = chain(self.data, chain.from_iterable(iterables)) - def __iter__(self) -> Iterator: + def __iter__(self) -> Iterator[_T]: return self - def __next__(self) -> Any: + def __next__(self) -> _T: return next(self.data) -async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator: +async def _async_chain( + *iterables: Iterable[_T] | AsyncIterator[_T], +) -> AsyncIterator[_T]: for it in iterables: async for o in as_async_generator(it): yield o -class MutableAsyncChain(AsyncIterable): +class MutableAsyncChain(AsyncIterator[_T]): """ Similar to MutableChain but for async iterables """ - def __init__(self, *args: Union[Iterable, AsyncIterable]): - self.data = _async_chain(*args) + def __init__(self, *args: Iterable[_T] | AsyncIterator[_T]): + self.data: AsyncIterator[_T] = _async_chain(*args) - def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None: + def extend(self, *iterables: Iterable[_T] | AsyncIterator[_T]) -> None: self.data = _async_chain(self.data, _async_chain(*iterables)) - def __aiter__(self) -> AsyncIterator: + def __aiter__(self) -> Self: return self - async def __anext__(self) -> Any: + async def __anext__(self) -> _T: return await self.data.__anext__() + + +def _looks_like_import_path(value: str) -> bool: + """Return True if **value** looks like a valid Python import path or False + otherwise.""" + if not value: + return False + if any(c.isspace() for c in value): + return False + allowed_chars = set( + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_." + ) + if any(c not in allowed_chars for c in value): + return False + if value[0] == "." or value[-1] == ".": + return False + parts = value.split(".") + if any(part == "" for part in parts): + return False + return all(part.isidentifier() for part in parts) + + +def _iter_exc_causes(exc: BaseException) -> Iterable[BaseException]: + """Iterate over the exception causes/contexts.""" + seen: set[int] = set() + cur: BaseException | None = exc + while cur is not None and id(cur) not in seen: + seen.add(id(cur)) + yield cur + cur = cur.__cause__ or cur.__context__ diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index ad3d1d8bc..7ab58093a 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,100 +1,115 @@ +from __future__ import annotations + import asyncio import sys -from asyncio import AbstractEventLoop, AbstractEventLoopPolicy +import warnings from contextlib import suppress -from typing import Any, Callable, Dict, Optional, Sequence, Type -from warnings import catch_warnings, filterwarnings, warn +from typing import TYPE_CHECKING, Any, Generic, ParamSpec, TypeVar +from warnings import catch_warnings, filterwarnings from twisted.internet import asyncioreactor, error -from twisted.internet.base import DelayedCall +from twisted.internet.defer import Deferred -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import load_object +from scrapy.utils.python import global_object_name + +if TYPE_CHECKING: + from asyncio import AbstractEventLoop + from collections.abc import Callable + + from twisted.internet.protocol import ServerFactory + from twisted.internet.tcp import Port + + from scrapy.utils.asyncio import CallLaterResult -def listen_tcp(portrange, host, factory): +_T = TypeVar("_T") +_P = ParamSpec("_P") + + +def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] # noqa: RET503 """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor if len(portrange) > 2: raise ValueError(f"invalid portrange: {portrange}") if not portrange: - return reactor.listenTCP(0, factory, interface=host) - if not hasattr(portrange, "__iter__"): - return reactor.listenTCP(portrange, factory, interface=host) + return reactor.listenTCP(0, factory, interface=host) # type: ignore[no-any-return] if len(portrange) == 1: - return reactor.listenTCP(portrange[0], factory, interface=host) + return reactor.listenTCP(portrange[0], factory, interface=host) # type: ignore[no-any-return] for x in range(portrange[0], portrange[1] + 1): try: - return reactor.listenTCP(x, factory, interface=host) + return reactor.listenTCP(x, factory, interface=host) # type: ignore[no-any-return] except error.CannotListenError: if x == portrange[1]: raise -class CallLaterOnce: +class CallLaterOnce(Generic[_T]): """Schedule a function to be called in the next reactor loop, but only if it hasn't been already scheduled since the last time it ran. """ - def __init__(self, func: Callable, *a: Any, **kw: Any): - self._func: Callable = func - self._a: Sequence[Any] = a - self._kw: Dict[str, Any] = kw - self._call: Optional[DelayedCall] = None + def __init__(self, func: Callable[_P, _T], *a: _P.args, **kw: _P.kwargs): + self._func: Callable[_P, _T] = func + self._a: tuple[Any, ...] = a + self._kw: dict[str, Any] = kw + self._call: CallLaterResult | None = None + self._deferreds: list[Deferred[None]] = [] def schedule(self, delay: float = 0) -> None: - from twisted.internet import reactor + # circular import + from scrapy.utils.asyncio import call_later # noqa: PLC0415 if self._call is None: - self._call = reactor.callLater(delay, self) + self._call = call_later(delay, self) def cancel(self) -> None: if self._call: self._call.cancel() - def __call__(self) -> Any: + def __call__(self) -> _T: + # circular import + from scrapy.utils.asyncio import call_later # noqa: PLC0415 + self._call = None - return self._func(*self._a, **self._kw) + result = self._func(*self._a, **self._kw) + + for d in self._deferreds: + call_later(0, d.callback, None) + self._deferreds.clear() + + return result + + async def wait(self) -> None: + # circular import + from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415 + + d: Deferred[None] = Deferred() + self._deferreds.append(d) + await maybe_deferred_to_future(d) + + +_asyncio_reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" def set_asyncio_event_loop_policy() -> None: - """The policy functions from asyncio often behave unexpectedly, - so we restrict their use to the absolutely essential case. - This should only be used to install the reactor. - """ - _get_asyncio_event_loop_policy() + """Needed due to https://github.com/twisted/twisted/issues/12527.""" + if sys.platform != "win32": + return + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=r"'asyncio\.(get_event_loop_policy|WindowsSelectorEventLoopPolicy)' is deprecated", + category=DeprecationWarning, + ) + policy = asyncio.get_event_loop_policy() + if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy): + policy = asyncio.WindowsSelectorEventLoopPolicy() # pylint: disable=deprecated-class + asyncio.set_event_loop_policy(policy) -def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: - warn( - "Call to deprecated function " - "scrapy.utils.reactor.get_asyncio_event_loop_policy().\n" - "\n" - "Please use get_event_loop, new_event_loop and set_event_loop" - " from asyncio instead, as the corresponding policy methods may lead" - " to unexpected behaviour.\n" - "This function is replaced by set_asyncio_event_loop_policy and" - " is meant to be used only when the reactor is being installed.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return _get_asyncio_event_loop_policy() - - -def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: - policy = asyncio.get_event_loop_policy() - if ( - sys.version_info >= (3, 8) - and sys.platform == "win32" - and not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy) - ): - policy = asyncio.WindowsSelectorEventLoopPolicy() - asyncio.set_event_loop_policy(policy) - return policy - - -def install_reactor(reactor_path: str, event_loop_path: Optional[str] = None) -> None: +def install_reactor(reactor_path: str, event_loop_path: str | None = None) -> None: """Installs the :mod:`~twisted.internet.reactor` with the specified import path. Also installs the asyncio event loop with the specified import path if the asyncio reactor is enabled""" @@ -106,7 +121,7 @@ def install_reactor(reactor_path: str, event_loop_path: Optional[str] = None) -> asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") - installer_path = module + ["install"] + installer_path = [*module, "install"] installer = load_object(".".join(installer_path)) with suppress(error.ReactorAlreadyInstalledError): installer() @@ -116,12 +131,14 @@ def _get_asyncio_event_loop() -> AbstractEventLoop: return set_asyncio_event_loop(None) -def set_asyncio_event_loop(event_loop_path: Optional[str]) -> AbstractEventLoop: +def set_asyncio_event_loop(event_loop_path: str | None) -> AbstractEventLoop: """Sets and returns the event loop with specified import path.""" if event_loop_path is not None: - event_loop_class: Type[AbstractEventLoop] = load_object(event_loop_path) - event_loop = event_loop_class() - asyncio.set_event_loop(event_loop) + event_loop_class: type[AbstractEventLoop] = load_object(event_loop_path) + event_loop = _get_asyncio_event_loop() + if not isinstance(event_loop, event_loop_class): + event_loop = event_loop_class() + asyncio.set_event_loop(event_loop) else: try: with catch_warnings(): @@ -148,22 +165,34 @@ def set_asyncio_event_loop(event_loop_path: Optional[str]) -> AbstractEventLoop: def verify_installed_reactor(reactor_path: str) -> None: - """Raises :exc:`Exception` if the installed + """Raise :exc:`RuntimeError` if the installed :mod:`~twisted.internet.reactor` does not match the specified import - path.""" + path or if no reactor is installed.""" + if not is_reactor_installed(): + raise RuntimeError( + "verify_installed_reactor() called without an installed reactor." + ) + from twisted.internet import reactor - reactor_class = load_object(reactor_path) - if not reactor.__class__ == reactor_class: - msg = ( - "The installed reactor " - f"({reactor.__module__}.{reactor.__class__.__name__}) does not " - f"match the requested one ({reactor_path})" + expected_reactor_type = load_object(reactor_path) + reactor_type = type(reactor) + if not reactor_type == expected_reactor_type: + raise RuntimeError( + f"The installed reactor ({global_object_name(reactor_type)}) " + f"does not match the requested one ({reactor_path})" ) - raise Exception(msg) def verify_installed_asyncio_event_loop(loop_path: str) -> None: + """Raise :exc:`RuntimeError` if the even loop of the installed + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` + does not match the specified import path or if no reactor is installed.""" + if not is_reactor_installed(): + raise RuntimeError( + "verify_installed_asyncio_event_loop() called without an installed reactor." + ) + from twisted.internet import reactor loop_class = load_object(loop_path) @@ -173,16 +202,40 @@ def verify_installed_asyncio_event_loop(loop_path: str) -> None: f"{reactor._asyncioEventloop.__class__.__module__}" f".{reactor._asyncioEventloop.__class__.__qualname__}" ) - specified = f"{loop_class.__module__}.{loop_class.__qualname__}" - raise Exception( + raise RuntimeError( "Scrapy found an asyncio Twisted reactor already " f"installed, and its event loop class ({installed}) does " "not match the one specified in the ASYNCIO_EVENT_LOOP " - f"setting ({specified})" + f"setting ({global_object_name(loop_class)})" ) +def is_reactor_installed() -> bool: + """Check whether a :mod:`~twisted.internet.reactor` is installed.""" + return "twisted.internet.reactor" in sys.modules + + def is_asyncio_reactor_installed() -> bool: + """Check whether the installed reactor is :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. + + Raise a :exc:`RuntimeError` if no reactor is installed. + + In a future Scrapy version, when Scrapy supports running without a Twisted + reactor, this function won't be useful for checking if it's possible to use + asyncio features, so the code that that doesn't directly require a Twisted + reactor should use :func:`scrapy.utils.asyncio.is_asyncio_available` + instead of this function. + + .. versionchanged:: 2.13 + In earlier Scrapy versions this function silently installed the default + reactor if there was no reactor installed. Now it raises an exception to + prevent silent problems in this case. + """ + if not is_reactor_installed(): + raise RuntimeError( + "is_asyncio_reactor_installed() called without an installed reactor." + ) + from twisted.internet import reactor return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) diff --git a/scrapy/utils/reactorless.py b/scrapy/utils/reactorless.py new file mode 100644 index 000000000..aca76951b --- /dev/null +++ b/scrapy/utils/reactorless.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import sys +from importlib.abc import MetaPathFinder +from typing import TYPE_CHECKING + +from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.reactor import is_reactor_installed + +if TYPE_CHECKING: + from collections.abc import Sequence + from importlib.machinery import ModuleSpec + from types import ModuleType + + +def is_reactorless() -> bool: + """Check if we are running in the reactorless mode, i.e. with + :setting:`TWISTED_REACTOR_ENABLED` set to ``False``. + + As this checks the runtime state and not the setting itself, it can be + wrong when executed very early, before the reactor and/or the asyncio event + loop are initialized. + + .. note:: As this function uses + :func:`scrapy.utils.asyncio.is_asyncio_available()`, it has the same + limitations for detecting a running asyncio event loop as that one. + + .. versionadded:: 2.15.0 + """ + return is_asyncio_available() and not is_reactor_installed() + + +class ReactorImportHook(MetaPathFinder): + """Hook that prevents importing :mod:`twisted.internet.reactor`.""" + + def find_spec( + self, + fullname: str, + path: Sequence[str] | None, + target: ModuleType | None = None, + ) -> ModuleSpec | None: + if fullname == "twisted.internet.reactor": + raise ImportError( + f"Import of {fullname} is forbidden when running without a Twisted reactor," + f" as importing it installs the reactor, which can lead to unexpected behavior." + ) + return None + + +def install_reactor_import_hook() -> None: + """Prevent importing :mod:`twisted.internet.reactor`.""" + + sys.meta_path.insert(0, ReactorImportHook()) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index db0b44cf4..ffb7fae49 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -1,57 +1,41 @@ """ This module provides some useful functions for working with -scrapy.http.Request objects +scrapy.Request objects """ +from __future__ import annotations + import hashlib import json -import warnings -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Generator, - Iterable, - List, - Optional, - Protocol, - Tuple, - Type, - Union, -) +from typing import TYPE_CHECKING, Any, Protocol from urllib.parse import urlunparse from weakref import WeakKeyDictionary -from w3lib.http import basic_auth_header from w3lib.url import canonicalize_url from scrapy import Request, Spider -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: + from collections.abc import Iterable + + # typing.Self requires Python 3.11 + from typing_extensions import Self + from scrapy.crawler import Crawler -def _serialize_headers( - headers: Iterable[bytes], request: Request -) -> Generator[bytes, Any, None]: - for header in headers: - if header in request.headers: - yield header - yield from request.headers.getlist(header) - - -_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" -_fingerprint_cache = WeakKeyDictionary() +_fingerprint_cache: WeakKeyDictionary[ + Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes] +] = WeakKeyDictionary() def fingerprint( request: Request, *, - include_headers: Optional[Iterable[Union[bytes, str]]] = None, + include_headers: Iterable[bytes | str] | None = None, keep_fragments: bool = False, ) -> bytes: """ @@ -59,17 +43,15 @@ def fingerprint( The request fingerprint is a hash that uniquely identifies the resource the request points to. For example, take the following two urls: - - http://www.example.com/query?id=111&cat=222 - http://www.example.com/query?cat=222&id=111 + ``http://www.example.com/query?id=111&cat=222``, + ``http://www.example.com/query?cat=222&id=111``. Even though those are two different URLs both point to the same resource and are equivalent (i.e. they should return the same response). Another example are cookies used to store session ids. Suppose the following page is only accessible to authenticated users: - - http://www.example.com/members/offers.html + ``http://www.example.com/members/offers.html``. Lots of sites use a cookie to store the session id, which adds a random component to the HTTP Request and thus should be ignored when calculating @@ -84,17 +66,19 @@ def fingerprint( If you want to include them, set the keep_fragments argument to True (for instance when handling requests with a headless browser). """ - processed_include_headers: Optional[Tuple[bytes, ...]] = None + processed_include_headers: tuple[bytes, ...] | None = None if include_headers: processed_include_headers = tuple( to_bytes(h.lower()) for h in sorted(include_headers) ) + verbatim_url = bool(request.meta.get("verbatim_url")) + effective_keep_fragments = keep_fragments and not verbatim_url cache = _fingerprint_cache.setdefault(request, {}) - cache_key = (processed_include_headers, keep_fragments) + cache_key = (processed_include_headers, effective_keep_fragments, verbatim_url) if cache_key not in cache: # To decode bytes reliably (JSON does not support bytes), regardless of # character encoding, we use bytes.hex() - headers: Dict[str, List[str]] = {} + headers: dict[str, list[str]] = {} if processed_include_headers: for header in processed_include_headers: if header in request.headers: @@ -102,20 +86,25 @@ def fingerprint( header_value.hex() for header_value in request.headers.getlist(header) ] + if verbatim_url: + url = request.url + else: + url = canonicalize_url(request.url, keep_fragments=keep_fragments) fingerprint_data = { "method": to_unicode(request.method), - "url": canonicalize_url(request.url, keep_fragments=keep_fragments), + "url": url, "body": (request.body or b"").hex(), "headers": headers, } fingerprint_json = json.dumps(fingerprint_data, sort_keys=True) - cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest() + cache[cache_key] = hashlib.sha1( # noqa: S324 + fingerprint_json.encode() + ).digest() return cache[cache_key] class RequestFingerprinterProtocol(Protocol): - def fingerprint(self, request: Request) -> bytes: - ... + def fingerprint(self, request: Request) -> bytes: ... class RequestFingerprinter: @@ -123,49 +112,24 @@ class RequestFingerprinter: It takes into account a canonical version (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url - ` and the values of :attr:`request.method - ` and :attr:`request.body - `. It then generates an `SHA1 - `_ hash. - - .. seealso:: :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION`. + ` and the values of :attr:`request.method + ` and :attr:`request.body + `, unless :reqmeta:`verbatim_url` is true for that + request. It then generates an `SHA1 `_ + hash. """ @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def __init__(self, crawler: Optional["Crawler"] = None): - if crawler: - implementation = crawler.settings.get( - "REQUEST_FINGERPRINTER_IMPLEMENTATION" - ) - else: - implementation = "SENTINEL" - - if implementation != "SENTINEL": - message = ( - "'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n" - "And it will be removed in future version of Scrapy." - ) - warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) + def __init__(self, crawler: Crawler | None = None): self._fingerprint = fingerprint def fingerprint(self, request: Request) -> bytes: return self._fingerprint(request) -def request_authenticate( - request: Request, - username: str, - password: str, -) -> None: - """Authenticate the given request (in place) using the HTTP basic access - authentication mechanism (RFC 2617) and the given username and password - """ - request.headers["Authorization"] = basic_auth_header(username, password) - - def request_httprepr(request: Request) -> bytes: """Return the raw HTTP representation (as bytes) of the given request. This is provided only for reference since it's not the actual stream of @@ -183,7 +147,7 @@ def request_httprepr(request: Request) -> bytes: return s -def referer_str(request: Request) -> Optional[str]: +def referer_str(request: Request) -> str | None: """Return Referer HTTP header suitable for logging.""" referrer = request.headers.get("Referer") if referrer is None: @@ -191,13 +155,13 @@ def referer_str(request: Request) -> Optional[str]: return to_unicode(referrer, errors="replace") -def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request: +def request_from_dict(d: dict[str, Any], *, spider: Spider | None = None) -> Request: """Create a :class:`~scrapy.Request` object from a dict. If a spider is given, it will try to resolve the callbacks looking at the spider for methods with the same name. """ - request_cls: Type[Request] = load_object(d["_class"]) if "_class" in d else Request + request_cls: type[Request] = load_object(d["_class"]) if "_class" in d else Request kwargs = {key: value for key, value in d.items() if key in request_cls.attributes} if d.get("callback") and spider: kwargs["callback"] = _get_method(spider, d["callback"]) @@ -212,7 +176,7 @@ def _get_method(obj: Any, name: Any) -> Any: try: return getattr(obj, name) except AttributeError: - raise ValueError(f"Method {name!r} not found in: {obj}") + raise ValueError(f"Method {name!r} not found in: {obj}") from None def request_to_curl(request: Request) -> str: @@ -238,7 +202,8 @@ def request_to_curl(request: Request) -> str: cookies = f"--cookie '{cookie}'" elif isinstance(request.cookies, list): cookie = "; ".join( - f"{list(c.keys())[0]}={list(c.values())[0]}" for c in request.cookies + f"{next(iter(c.keys()))}={next(iter(c.values()))}" + for c in request.cookies ) cookies = f"--cookie '{cookie}'" diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 2ab167478..48ef9b1ff 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -2,6 +2,9 @@ This module provides some useful functions for working with scrapy.http.Response objects """ + +from __future__ import annotations + import os import re import tempfile @@ -9,7 +12,7 @@ import webbrowser from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data -from typing import Any, Callable, Iterable, Optional, Sequence, Tuple, Type, Union +from typing import TYPE_CHECKING, Any from urllib.parse import urlparse from warnings import warn from weakref import WeakKeyDictionary @@ -25,7 +28,6 @@ from xtractmime.mimegroups import ( is_xml_mime_type, ) -import scrapy from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import ( Headers, @@ -37,6 +39,9 @@ from scrapy.http import ( ) from scrapy.utils.python import to_bytes, to_unicode +if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Sequence + _ENCODING_MIME_TYPE_MAP = { b"br": b"application/brotli", b"compress": b"application/x-compress", @@ -49,12 +54,16 @@ _MIME_TYPES = MimeTypes() _mime_overrides = get_data("scrapy", "mime.types") or b"" _MIME_TYPES.readfp(StringIO(_mime_overrides.decode())) +_metaref_cache: WeakKeyDictionary[Response, tuple[None, None] | tuple[float, str]] = ( + WeakKeyDictionary() +) -def _is_compressed_mime_type(mime_type): + +def _is_compressed_mime_type(mime_type: bytes) -> bool: return mime_type in _ENCODING_MIME_TYPES -def _is_other_text_mime_type(mime_type): +def _is_other_text_mime_type(mime_type: bytes) -> bool: return ( mime_type.startswith(b"text/") or mime_type == b"application/x-javascript" @@ -64,7 +73,7 @@ def _is_other_text_mime_type(mime_type): def _get_encoding_or_mime_type_from_headers( headers: Headers, -) -> Tuple[Optional[bytes], Optional[bytes]]: +) -> tuple[bytes | None, bytes | None]: if b"Content-Encoding" in headers: encodings = [ item.strip() @@ -84,7 +93,7 @@ def _get_encoding_or_mime_type_from_headers( ) ): return None, headers[b"Content-Type"] - if b"Content-Disposition" in headers and headers[b"Content-Disposition"]: + if headers.get(b"Content-Disposition"): path = ( headers[b"Content-Disposition"] .split(b";")[-1] @@ -99,11 +108,13 @@ def _get_encoding_or_mime_type_from_headers( return None, None -def _get_mime_type_from_encoding(encoding): - return _ENCODING_MIME_TYPE_MAP.get(encoding, None) or b"application/" + encoding +def _get_mime_type_from_encoding(encoding: bytes) -> bytes: + return _ENCODING_MIME_TYPE_MAP.get(encoding) or b"application/" + encoding -def _get_encoding_or_mime_type_from_path(path): +def _get_encoding_or_mime_type_from_path( + path: str, +) -> tuple[bytes | None, bytes | None]: mimetype, encoding = _MIME_TYPES.guess_type(path, strict=False) if encoding: return encoding.encode(), None @@ -112,7 +123,7 @@ def _get_encoding_or_mime_type_from_path(path): return None, None -def _get_response_class_from_mime_type(mime_type): +def _get_response_class_from_mime_type(mime_type: bytes | None) -> type[Response]: if not mime_type: return Response if is_html_mime_type(mime_type): @@ -145,31 +156,26 @@ def get_base_url(response: TextResponse) -> str: return response.base_url -_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = ( - WeakKeyDictionary() -) - - def get_meta_refresh( - response: "scrapy.http.response.text.TextResponse", + response: TextResponse, ignore_tags: Iterable[str] = ("script", "noscript"), -) -> Union[Tuple[None, None], Tuple[float, str]]: +) -> tuple[None, None] | tuple[float, str]: """Parse the http-equiv refresh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] _metaref_cache[response] = html.get_meta_refresh( - text, response.url, response.encoding, ignore_tags=ignore_tags + text, response.base_url, response.encoding, ignore_tags=ignore_tags ) return _metaref_cache[response] def get_response_class( *, - url: Optional[str] = None, - body: Optional[bytes] = None, - declared_mime_types: Optional[Sequence[bytes]] = None, - http_headers: Optional[Headers] = None, -) -> Type[Response]: + url: str | None = None, + body: bytes | None = None, + declared_mime_types: Sequence[bytes] | None = None, + http_headers: Headers | None = None, +) -> type[Response]: """Guess the most appropriate Response class based on the given arguments.""" mime_type = next(iter(declared_mime_types or []), None) @@ -217,37 +223,58 @@ def get_response_class( return _get_response_class_from_mime_type(mime_type) -def response_status_message(status: Union[bytes, float, int, str]) -> str: +def response_status_message(status: bytes | float | str) -> str: """Return status code plus status text descriptive message""" status_int = int(status) message = http.RESPONSES.get(status_int, "Unknown Status") return f"{status_int} {to_unicode(message)}" +def _remove_html_comments(body: bytes) -> bytes: + start = body.find(b"", start + 1) + if end == -1: + return body[:start] + body = body[:start] + body[end + 3 :] + start = body.find(b"", b"", body, flags=re.DOTALL) - body = re.sub(rb"(|\s.*?>))", to_bytes(repl), body) + if b"' + body = re.sub(rb"]*?>)", to_bytes(repl), body, count=1) ext = ".html" elif isinstance(response, TextResponse): ext = ".txt" else: - raise TypeError("Unsupported response type: " f"{response.__class__.__name__}") + raise TypeError(f"Unsupported response type: {response.__class__.__name__}") fd, fname = tempfile.mkstemp(ext) os.write(fd, body) os.close(fd) diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 3b4f67f00..5d06bbe30 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -26,14 +26,10 @@ class ScrapyJSONEncoder(json.JSONEncoder): return str(o) if isinstance(o, defer.Deferred): return str(o) - if is_item(o): - return ItemAdapter(o).asdict() if isinstance(o, Request): return f"<{type(o).__name__} {o.method} {o.url}>" if isinstance(o, Response): return f"<{type(o).__name__} {o.status} {o.url}>" + if is_item(o): + return ItemAdapter(o).asdict() return super().default(o) - - -class ScrapyJSONDecoder(json.JSONDecoder): - pass diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 21a12a19e..ee391a49e 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,8 +1,13 @@ """Helper functions for working with signals""" -import collections.abc + +from __future__ import annotations + +import asyncio import logging +import warnings +from collections.abc import Awaitable, Callable, Generator, Sequence from typing import Any as TypingAny -from typing import List, Tuple +from typing import cast from pydispatch.dispatcher import ( Anonymous, @@ -12,12 +17,18 @@ from pydispatch.dispatcher import ( liveReceivers, ) from pydispatch.robustapply import robustApply -from twisted.internet.defer import Deferred, DeferredList +from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks from twisted.python.failure import Failure -from scrapy.exceptions import StopDownload -from scrapy.utils.defer import maybeDeferred_coro +from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload +from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.defer import ( + _maybeDeferred_coro, + ensure_awaitable, + maybe_deferred_to_future, +) from scrapy.utils.log import failure_to_exc_info +from scrapy.utils.python import global_object_name logger = logging.getLogger(__name__) @@ -26,25 +37,21 @@ def send_catch_log( signal: TypingAny = Any, sender: TypingAny = Anonymous, *arguments: TypingAny, - **named: TypingAny -) -> List[Tuple[TypingAny, TypingAny]]: - """Like pydispatcher.robust.sendRobust but it also logs errors and returns + **named: TypingAny, +) -> list[tuple[TypingAny, TypingAny]]: + """Like ``pydispatcher.robust.sendRobust()`` but it also logs errors and returns Failures instead of exceptions. """ dont_log = named.pop("dont_log", ()) - dont_log = ( - tuple(dont_log) - if isinstance(dont_log, collections.abc.Sequence) - else (dont_log,) - ) + dont_log = tuple(dont_log) if isinstance(dont_log, Sequence) else (dont_log,) dont_log += (StopDownload,) - spider = named.get("spider", None) - responses: List[Tuple[TypingAny, TypingAny]] = [] + spider = named.get("spider") + responses: list[tuple[TypingAny, TypingAny]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): result: TypingAny try: response = robustApply( - receiver, signal=signal, sender=sender, *arguments, **named + receiver, *arguments, signal=signal, sender=sender, **named ) if isinstance(response, Deferred): logger.error( @@ -72,14 +79,29 @@ def send_catch_log_deferred( signal: TypingAny = Any, sender: TypingAny = Anonymous, *arguments: TypingAny, - **named: TypingAny -) -> Deferred: - """Like send_catch_log but supports returning deferreds on signal handlers. - Returns a deferred that gets fired once all signal handlers deferreds were - fired. - """ + **named: TypingAny, +) -> Deferred[list[tuple[TypingAny, TypingAny]]]: + """Like :func:`send_catch_log` but supports :ref:`asynchronous signal handlers + `. - def logerror(failure: Failure, recv: Any) -> Failure: + Returns a deferred that gets fired once all signal handlers have finished. + """ + warnings.warn( + "send_catch_log_deferred() is deprecated, use send_catch_log_async() instead", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return _send_catch_log_deferred(signal, sender, *arguments, **named) + + +@inlineCallbacks +def _send_catch_log_deferred( + signal: TypingAny, + sender: TypingAny, + *arguments: TypingAny, + **named: TypingAny, +) -> Generator[Deferred[TypingAny], TypingAny, list[tuple[TypingAny, TypingAny]]]: + def logerror(failure: Failure, recv: TypingAny) -> Failure: if dont_log is None or not isinstance(failure.value, dont_log): logger.error( "Error caught on signal handler: %(receiver)s", @@ -90,23 +112,114 @@ def send_catch_log_deferred( return failure dont_log = named.pop("dont_log", None) - spider = named.get("spider", None) - dfds = [] + spider = named.get("spider") + dfds: list[Deferred[tuple[TypingAny, TypingAny]]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): - d = maybeDeferred_coro( - robustApply, receiver, signal=signal, sender=sender, *arguments, **named + d: Deferred[TypingAny] = _maybeDeferred_coro( + robustApply, + True, + receiver, + *arguments, + signal=signal, + sender=sender, + **named, ) d.addErrback(logerror, receiver) - d.addBoth(lambda result: (receiver, result)) - dfds.append(d) - d = DeferredList(dfds) - d.addCallback(lambda out: [x[1] for x in out]) - return d + # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html + d2: Deferred[tuple[TypingAny, TypingAny]] = d.addBoth( + lambda result: ( + receiver, # pylint: disable=cell-var-from-loop # noqa: B023 + result, + ) + ) + dfds.append(d2) + + results = yield DeferredList(dfds) + return [result[1] for result in results] + + +async def send_catch_log_async( + signal: TypingAny = Any, + sender: TypingAny = Anonymous, + *arguments: TypingAny, + **named: TypingAny, +) -> list[tuple[TypingAny, TypingAny]]: + """Like :func:`send_catch_log` but supports :ref:`asynchronous signal handlers + `. + + Returns a coroutine that completes once all signal handlers have finished. + + .. versionadded:: 2.14 + """ + # note that this returns exceptions instead of Failures in the second tuple member + if is_asyncio_available(): + return await _send_catch_log_asyncio(signal, sender, *arguments, **named) + results = await maybe_deferred_to_future( + _send_catch_log_deferred(signal, sender, *arguments, **named) + ) + return [ + (receiver, result.value if isinstance(result, Failure) else result) + for receiver, result in results + ] + + +async def _send_catch_log_asyncio( + signal: TypingAny = Any, + sender: TypingAny = Anonymous, + *arguments: TypingAny, + **named: TypingAny, +) -> list[tuple[TypingAny, TypingAny]]: + """Like :func:`send_catch_log` but supports :ref:`asynchronous signal handlers + `. + + Returns a coroutine that completes once all signal handlers have finished. + + This function requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be + installed. + + .. versionadded:: 2.14 + """ + dont_log = named.pop("dont_log", ()) + dont_log = tuple(dont_log) if isinstance(dont_log, Sequence) else (dont_log,) + spider = named.get("spider") + handlers: list[Awaitable[TypingAny]] = [] + for receiver in liveReceivers(getAllReceivers(sender, signal)): + + async def handler( + receiver: Callable[..., Any], + ) -> tuple[Callable[..., Any], TypingAny]: + result: TypingAny + try: + result = await ensure_awaitable( + robustApply( + receiver, *arguments, signal=signal, sender=sender, **named + ), + _warn=global_object_name(receiver), + ) + except dont_log as ex: # pylint: disable=catching-non-exception + result = ex + except Exception as ex: + logger.error( + "Error caught on signal handler: %(receiver)s", + {"receiver": receiver}, + exc_info=True, + extra={"spider": spider}, + ) + result = ex + return (receiver, result) + + handlers.append(handler(receiver)) + + return cast( + "list[tuple[TypingAny, TypingAny]]", + await asyncio.gather(*handlers, return_exceptions=True), + ) def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None: """Disconnect all signal handlers. Useful for cleaning up after running - tests + tests. """ for receiver in liveReceivers(getAllReceivers(sender, signal)): disconnect(receiver, signal=signal, sender=sender) diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 3d2ecc9a7..1520a4ff0 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -4,48 +4,132 @@ Module for processing Sitemaps. Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ -from typing import Any, Dict, Generator, Iterator, Optional + +from __future__ import annotations + +import warnings +from io import BytesIO, StringIO +from typing import TYPE_CHECKING, Any from urllib.parse import urljoin import lxml.etree +from scrapy.exceptions import ScrapyDeprecationWarning + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + class Sitemap: """Class to parse Sitemap (type=urlset) and Sitemap Index (type=sitemapindex) files""" - def __init__(self, xmltext: str): - xmlp = lxml.etree.XMLParser( - recover=True, remove_comments=True, resolve_entities=False + __slots__ = ("type", "xmliter") + + def __init__(self, xmltext: str | bytes): + if isinstance(xmltext, str): + warnings.warn( + "Passing `str` type as `xmltext` is deprecated, use `bytes`", + ScrapyDeprecationWarning, + stacklevel=2, + ) + xmltext = xmltext.encode() + + self.xmliter = lxml.etree.iterparse( + BytesIO(xmltext), + recover=True, + remove_comments=True, + resolve_entities=False, + remove_blank_text=True, + collect_ids=False, + remove_pis=True, + events=("start", "end"), ) - self._root = lxml.etree.fromstring(xmltext, parser=xmlp) - rt = self._root.tag - self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt + _, root = next(self.xmliter) + self.type = self._get_tag_name(root) - def __iter__(self) -> Iterator[Dict[str, Any]]: - for elem in self._root.getchildren(): - d: Dict[str, Any] = {} - for el in elem.getchildren(): - tag = el.tag - name = tag.split("}", 1)[1] if "}" in tag else tag + def __iter__(self) -> Iterator[dict[str, Any]]: + for event, elem in self.xmliter: + if event == "start": + continue - if name == "link": - if "href" in el.attrib: - d.setdefault("alternate", []).append(el.get("href")) - else: - d[name] = el.text.strip() if el.text else "" + if self._get_tag_name(elem) not in {"url", "sitemap"}: + continue - if "loc" in d: + if d := self._process_sitemap_element(elem): yield d + def _process_sitemap_element( + self, elem: lxml.etree._Element + ) -> dict[str, Any] | None: + d: dict[str, Any] = {} + alternate: list[str] = [] + has_loc = False + + for el in elem: + try: + tag_name = self._get_tag_name(el) + if not tag_name: + continue + + if tag_name == "link": + if href := el.get("href"): + alternate.append(href) + else: + d[tag_name] = el.text.strip() if el.text else "" + if not has_loc and tag_name == "loc": + has_loc = True + finally: + el.clear() + elem.clear() + parent = elem.getparent() + if parent is not None: + while elem.getprevious() is not None: + del parent[0] + + if not has_loc: + return None + + if alternate: + d["alternate"] = alternate + + return d + + @staticmethod + def _get_tag_name(elem: lxml.etree._Element) -> str: + if TYPE_CHECKING: + assert isinstance(elem.tag, str) + _, _, localname = elem.tag.partition("}") + return localname or elem.tag + def sitemap_urls_from_robots( - robots_text: str, base_url: Optional[str] = None -) -> Generator[str, Any, None]: - """Return an iterator over all sitemap urls contained in the given - robots.txt file - """ - for line in robots_text.splitlines(): - if line.lstrip().lower().startswith("sitemap:"): - url = line.split(":", 1)[1].strip() + robots_text: str | bytes, + base_url: str | None = None, +) -> Iterable[str]: + if isinstance(robots_text, bytes): + for line in BytesIO(robots_text): + if line.lstrip()[:8].lower() == b"sitemap:": + try: + url = line.partition(b":")[2].strip().decode() + except UnicodeDecodeError: + continue + yield urljoin(base_url or "", url) + + else: + warnings.warn( + "Passing `str` type as `robots_text` is deprecated, use `bytes`", + ScrapyDeprecationWarning, + stacklevel=2, + ) + yield from _sitemap_urls_from_robots_str(robots_text, base_url) + + +def _sitemap_urls_from_robots_str( + robots_text: str, + base_url: str | None = None, +) -> Iterable[str]: + for line in StringIO(robots_text): + if line.lstrip()[:8].lower() == "sitemap:": + url = line.partition(":")[2].strip() yield urljoin(base_url or "", url) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 704df8657..9f43df1ce 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -2,30 +2,21 @@ from __future__ import annotations import inspect import logging -from types import CoroutineType, ModuleType -from typing import ( - TYPE_CHECKING, - Any, - AsyncGenerator, - Generator, - Iterable, - Literal, - Optional, - Type, - TypeVar, - Union, - overload, -) +from typing import TYPE_CHECKING, Any, TypeVar, overload -from twisted.internet.defer import Deferred - -from scrapy import Request from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter if TYPE_CHECKING: - from scrapy.spiderloader import SpiderLoader + from collections.abc import AsyncGenerator, Iterable + from types import CoroutineType, ModuleType + + from twisted.internet.defer import Deferred + + from scrapy import Request + from scrapy.spiderloader import SpiderLoaderProtocol + logger = logging.getLogger(__name__) @@ -34,38 +25,32 @@ _T = TypeVar("_T") # https://stackoverflow.com/questions/60222982 @overload -def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: # type: ignore[misc] - ... +def iterate_spider_output(result: AsyncGenerator[_T]) -> AsyncGenerator[_T]: ... # type: ignore[overload-overlap] @overload -def iterate_spider_output(result: CoroutineType) -> Deferred: - ... +def iterate_spider_output(result: CoroutineType[Any, Any, _T]) -> Deferred[_T]: ... @overload -def iterate_spider_output(result: _T) -> Iterable: - ... +def iterate_spider_output(result: _T) -> Iterable[Any]: ... -def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]: +def iterate_spider_output( + result: Any, +) -> Iterable[Any] | AsyncGenerator[_T] | Deferred[_T]: if inspect.isasyncgen(result): return result + d: Deferred[_T] = deferred_from_coro(result) if inspect.iscoroutine(result): - d = deferred_from_coro(result) - d.addCallback(iterate_spider_output) - return d - return arg_to_iter(deferred_from_coro(result)) + return d.addCallback(iterate_spider_output) + return arg_to_iter(d) -def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]: +def iter_spider_classes(module: ModuleType) -> Iterable[type[Spider]]: """Return an iterator over all spider classes defined in the given module that can be instantiated (i.e. which have name) """ - # this needs to be imported here until get rid of the spider manager - # singleton in scrapy.spider.spiders - from scrapy.spiders import Spider - for obj in vars(module).values(): if ( inspect.isclass(obj) @@ -78,44 +63,41 @@ def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None @overload def spidercls_for_request( - spider_loader: SpiderLoader, + spider_loader: SpiderLoaderProtocol, request: Request, - default_spidercls: Type[Spider], + default_spidercls: type[Spider], log_none: bool = ..., log_multiple: bool = ..., -) -> Type[Spider]: - ... +) -> type[Spider]: ... @overload def spidercls_for_request( - spider_loader: SpiderLoader, + spider_loader: SpiderLoaderProtocol, request: Request, - default_spidercls: Literal[None], + default_spidercls: None, log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: - ... +) -> type[Spider] | None: ... @overload def spidercls_for_request( - spider_loader: SpiderLoader, + spider_loader: SpiderLoaderProtocol, request: Request, *, log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: - ... +) -> type[Spider] | None: ... def spidercls_for_request( - spider_loader: SpiderLoader, + spider_loader: SpiderLoaderProtocol, request: Request, - default_spidercls: Optional[Type[Spider]] = None, + default_spidercls: type[Spider] | None = None, log_none: bool = False, log_multiple: bool = False, -) -> Optional[Type[Spider]]: +) -> type[Spider] | None: """Return a spider class that handles the given Request. This will look for the spiders that can handle the given request (using diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index e74769c65..22e9414b8 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,12 +1,120 @@ -from typing import Any, Optional +from __future__ import annotations -import OpenSSL._util as pyOpenSSLutil # type: ignore[import-untyped] +import logging +import ssl +from typing import TYPE_CHECKING, Any, TypedDict, TypeVar + +import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL import OpenSSL.version -from OpenSSL.crypto import X509Name +from twisted.internet.ssl import CertificateOptions, TLSVersion +from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1 from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + from collections.abc import Callable + + from OpenSSL.crypto import X509Name + + from scrapy.settings import BaseSettings + +logger = logging.getLogger(__name__) + +_T = TypeVar("_T") + + +# common + + +def _get_tls_version_limit( + settings: BaseSettings, setting_name: str, converter: Callable[[str], _T] +) -> _T | None: + setting: str | None = settings[setting_name] + if setting is None: + return None + try: + return converter(setting) + except Exception as ex: + raise ValueError(f"Unknown {setting_name} value: {setting}") from ex + + +def _get_tls_version_limits( + settings: BaseSettings, converter: Callable[[str], _T] +) -> tuple[_T | None, _T | None]: + return ( + _get_tls_version_limit(settings, "DOWNLOAD_TLS_MIN_VERSION", converter), + _get_tls_version_limit(settings, "DOWNLOAD_TLS_MAX_VERSION", converter), + ) + + +# stdlib ssl module utils + +_STDLIB_VERSION_MAP: dict[str, ssl.TLSVersion] = { + "TLSv1.0": ssl.TLSVersion.TLSv1, + "TLSv1.1": ssl.TLSVersion.TLSv1_1, + "TLSv1.2": ssl.TLSVersion.TLSv1_2, + "TLSv1.3": ssl.TLSVersion.TLSv1_3, +} + + +def _make_ssl_context(settings: BaseSettings) -> ssl.SSLContext: + """Create an :class:`ssl.SSLContext` instance according to the settings. + + It's intended to be used in an HTTPS download handler. + """ + + tls_min_ver, tls_max_ver = _get_tls_version_limits( + settings, _STDLIB_VERSION_MAP.__getitem__ + ) + ciphers_setting: str | None = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] + verify_setting = settings.getbool("DOWNLOAD_VERIFY_CERTIFICATES") + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + if verify_setting: + ctx.check_hostname = True + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_default_certs() + else: + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + if tls_min_ver is not None: + ctx.minimum_version = tls_min_ver + if tls_max_ver is not None: + ctx.maximum_version = tls_max_ver + if ciphers_setting: + ctx.set_ciphers(ciphers_setting) + return ctx + + +def _make_insecure_ssl_ctx() -> ssl.SSLContext: + """Create an SSL context that doesn't verify certificates. + + Compared to :func:`~scrapy.utils.ssl._make_ssl_context` this is much more + simple. + """ + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return ctx + + +def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None: + cipher = sslobj.cipher() + logger.debug( + f"SSL connection to {sslobj.server_hostname}" + f" using protocol {sslobj.version()}," + f" cipher {cipher[0] if cipher else None}" + ) + if cert := sslobj.getpeercert(): + # Not available without certificate verification + logger.debug( + f'SSL connection certificate: issuer "{cert["issuer"]}", subject "{cert["subject"]}"' + ) + + +# pyOpenSSL utils + def ffi_buf_to_string(buf: Any) -> str: return to_unicode(pyOpenSSLutil.ffi.string(buf)) @@ -16,13 +124,12 @@ def x509name_to_string(x509name: X509Name) -> str: # from OpenSSL.crypto.X509Name.__repr__ result_buffer: Any = pyOpenSSLutil.ffi.new("char[]", 512) pyOpenSSLutil.lib.X509_NAME_oneline( - x509name._name, result_buffer, len(result_buffer) # type: ignore[attr-defined] + x509name._name, result_buffer, len(result_buffer) ) - return ffi_buf_to_string(result_buffer) -def get_temp_key_info(ssl_object: Any) -> Optional[str]: +def get_temp_key_info(ssl_object: Any) -> str | None: # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key() if not hasattr(pyOpenSSLutil.lib, "SSL_get_server_tmp_key"): # removed in cryptography 40.0.0 @@ -61,3 +168,61 @@ def get_openssl_version() -> str: system_openssl_bytes = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) system_openssl = system_openssl_bytes.decode("ascii", errors="replace") return f"{OpenSSL.version.__version__} ({system_openssl})" + + +def _log_ssl_conn_debug_info(hostname: str, connection: OpenSSL.SSL.Connection) -> None: + logger.debug( + "SSL connection to %s using protocol %s, cipher %s", + hostname, + connection.get_protocol_version_name(), + connection.get_cipher_name(), + ) + server_cert = connection.get_peer_certificate() + if server_cert: + logger.debug( + 'SSL connection certificate: issuer "%s", subject "%s"', + x509name_to_string(server_cert.get_issuer()), + x509name_to_string(server_cert.get_subject()), + ) + key_info = get_temp_key_info(connection._ssl) + if key_info: + logger.debug("SSL temp key: %s", key_info) + + +# Twisted-specific + + +class _CertificateOptionsVersionKwargs(TypedDict, total=False): + lowerMaximumSecurityTo: TLSVersion + insecurelyLowerMinimumTo: TLSVersion + raiseMinimumTo: TLSVersion + + +def _get_cert_options_version_kwargs( + min_version: TLSVersion | None, max_version: TLSVersion | None +) -> _CertificateOptionsVersionKwargs: + """Get TLS version kwargs for + :class:`~twisted.internet.ssl.CertificateOptions` for the given limits.""" + result: _CertificateOptionsVersionKwargs = {} + if max_version: + if TWISTED_TLS_LIMITS_OFFBY1: + # lowerMaximumSecurityTo is treated as 1 version lower than the passed one + versions = list(TLSVersion.iterconstants()) + max_index = versions.index(max_version) + if max_index + 1 >= len(versions): + raise ValueError( + f"Due to an error in Twisted < 26.4.0 cannot set the maximum TLS version to {max_version.name}" + ) + max_version = versions[max_index + 1] + result["lowerMaximumSecurityTo"] = max_version + if min_version: + # We cannot pass both insecurelyLowerMinimumTo and raiseMinimumTo, + # so we need to know the direction. + + # 1.0 in Twisted 22.8.0 and older, 1.2 in Twisted 22.10.0 and newer + default_min = CertificateOptions._defaultMinimumTLSVersion + if min_version < default_min: + result["insecurelyLowerMinimumTo"] = min_version + elif min_version > default_min: + result["raiseMinimumTo"] = min_version + return result diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 6b22f3bfa..977d5a42a 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -1,13 +1,17 @@ """Helper functions for working with templates""" +from __future__ import annotations + import re import string -from os import PathLike from pathlib import Path -from typing import Any, Union +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from os import PathLike -def render_templatefile(path: Union[str, PathLike], **kwargs: Any) -> None: +def render_templatefile(path: str | PathLike[str], **kwargs: Any) -> None: path_obj = Path(path) raw = path_obj.read_text("utf8") diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 7a8c5c859..b4e20c3c6 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -2,93 +2,82 @@ This module contains some assorted functions used in tests """ +from __future__ import annotations + import asyncio import os +import warnings from importlib import import_module from pathlib import Path -from posixpath import split -from typing import Any, Coroutine, Dict, List, Optional, Tuple, Type -from unittest import TestCase, mock +from typing import TYPE_CHECKING, Any, TypeVar, cast -from twisted.internet.defer import Deferred -from twisted.trial.unittest import SkipTest +from twisted.web.client import Agent -from scrapy import Spider -from scrapy.crawler import Crawler -from scrapy.utils.boto import is_botocore_available +from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner, CrawlerRunnerBase +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed +from scrapy.utils.spider import DefaultSpider + +if TYPE_CHECKING: + from collections.abc import Awaitable + + from twisted.internet.defer import Deferred + from twisted.web.client import Response as TxResponse + + from scrapy import Spider + from scrapy.crawler import Crawler -def assert_gcs_environ() -> None: - if "GCS_PROJECT_ID" not in os.environ: - raise SkipTest("GCS_PROJECT_ID not found") +_T = TypeVar("_T") -def skip_if_no_boto() -> None: - if not is_botocore_available(): - raise SkipTest("missing botocore library") +def get_reactor_settings() -> dict[str, Any]: + """Return a settings dict that works with the installed reactor. - -def get_gcs_content_and_delete( - bucket: Any, path: str -) -> Tuple[bytes, List[Dict[str, str]], Any]: - from google.cloud import storage - - client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) - bucket = client.get_bucket(bucket) - blob = bucket.get_blob(path) - content = blob.download_as_string() - acl = list(blob.acl) # loads acl before it will be deleted - bucket.delete_blob(path) - return content, acl, blob - - -def get_ftp_content_and_delete( - path: str, - host: str, - port: int, - username: str, - password: str, - use_active_mode: bool = False, -) -> bytes: - from ftplib import FTP - - ftp = FTP() - ftp.connect(host, port) - ftp.login(username, password) - if use_active_mode: - ftp.set_pasv(False) - ftp_data: List[bytes] = [] - - def buffer_data(data: bytes) -> None: - ftp_data.append(data) - - ftp.retrbinary(f"RETR {path}", buffer_data) - dirname, filename = split(path) - ftp.cwd(dirname) - ftp.delete(filename) - return b"".join(ftp_data) - - -class TestSpider(Spider): - name = "test" + ``Crawler._apply_settings()`` checks that the installed reactor matches the + settings, so tests that run the crawler in the current process may need to + pass a correct :setting:`TWISTED_REACTOR` setting value when creating it. + """ + settings: dict[str, Any] = {} + if is_reactor_installed(): + if not is_asyncio_reactor_installed(): + settings["TWISTED_REACTOR"] = None + else: + # We are either running Scrapy tests for the reactorless mode, or + # running some 3rd-party library tests for the reactorless mode, or + # running some 3rd-party library tests without initializing a reactor + # properly. The first two cases are fine, but we cannot distinguish the + # last one from them. + settings["TWISTED_REACTOR_ENABLED"] = False + settings["DOWNLOAD_HANDLERS"] = { + "ftp": None, + "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + } + return settings def get_crawler( - spidercls: Optional[Type[Spider]] = None, - settings_dict: Optional[Dict[str, Any]] = None, + spidercls: type[Spider] | None = None, + settings_dict: dict[str, Any] | None = None, prevent_warnings: bool = True, ) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level priority. """ - from scrapy.crawler import CrawlerRunner - - # Set by default settings that prevent deprecation warnings. - settings: Dict[str, Any] = {} - settings.update(settings_dict or {}) - runner = CrawlerRunner(settings) - crawler = runner.create_crawler(spidercls or TestSpider) + # When needed, useful settings can be added here, e.g. ones that prevent + # deprecation warnings. + settings: dict[str, Any] = { + **get_reactor_settings(), + **(settings_dict or {}), + } + runner: CrawlerRunnerBase + if is_reactor_installed(): + runner = CrawlerRunner(settings) + else: + runner = AsyncCrawlerRunner(settings) + crawler = runner.create_crawler(spidercls or DefaultSpider) crawler._apply_settings() return crawler @@ -100,7 +89,7 @@ def get_pythonpath() -> str: return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get("PYTHONPATH", "") -def get_testenv() -> Dict[str, str]: +def get_testenv() -> dict[str, str]: """Return a OS environment dict suitable to fork processes that need to import this installation of Scrapy, instead of a system installed one. """ @@ -109,42 +98,22 @@ def get_testenv() -> Dict[str, str]: return env -def assert_samelines( - testcase: TestCase, text1: str, text2: str, msg: Optional[str] = None -) -> None: - """Asserts text1 and text2 have the same lines, ignoring differences in - line endings between platforms - """ - testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) - - -def get_from_asyncio_queue(value: Any) -> Coroutine: - q: asyncio.Queue = asyncio.Queue() +def get_from_asyncio_queue(value: _T) -> Awaitable[_T]: + q: asyncio.Queue[_T] = asyncio.Queue() getter = q.get() q.put_nowait(value) return getter -def mock_google_cloud_storage() -> Tuple[Any, Any, Any]: - """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob - classes and set their proper return values. - """ - from google.cloud.storage import Blob, Bucket, Client +def get_web_client_agent_req(url: str) -> Deferred[TxResponse]: # pragma: no cover + warnings.warn( + "The get_web_client_agent_req() function is deprecated" + " and will be removed in a future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) - client_mock = mock.create_autospec(Client) - - bucket_mock = mock.create_autospec(Bucket) - client_mock.get_bucket.return_value = bucket_mock - - blob_mock = mock.create_autospec(Blob) - bucket_mock.blob.return_value = blob_mock - - return (client_mock, bucket_mock, blob_mock) - - -def get_web_client_agent_req(url: str) -> Deferred: from twisted.internet import reactor - from twisted.web.client import Agent # imports twisted.internet.reactor agent = Agent(reactor) - return agent.request(b"GET", url.encode("utf-8")) + return cast("Deferred[TxResponse]", agent.request(b"GET", url.encode("utf-8"))) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py deleted file mode 100644 index 3bdffcaa7..000000000 --- a/scrapy/utils/testproc.py +++ /dev/null @@ -1,63 +0,0 @@ -from __future__ import annotations - -import os -import sys -from typing import Iterable, List, Optional, Tuple, cast - -from twisted.internet.defer import Deferred -from twisted.internet.error import ProcessTerminated -from twisted.internet.protocol import ProcessProtocol -from twisted.python.failure import Failure - - -class ProcessTest: - command: Optional[str] = None - prefix = [sys.executable, "-m", "scrapy.cmdline"] - cwd = os.getcwd() # trial chdirs to temp dir - - def execute( - self, - args: Iterable[str], - check_code: bool = True, - settings: Optional[str] = None, - ) -> Deferred: - from twisted.internet import reactor - - env = os.environ.copy() - if settings is not None: - env["SCRAPY_SETTINGS_MODULE"] = settings - assert self.command - cmd = self.prefix + [self.command] + list(args) - pp = TestProcessProtocol() - pp.deferred.addCallback(self._process_finished, cmd, check_code) - reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) - return pp.deferred - - def _process_finished( - self, pp: TestProcessProtocol, cmd: List[str], check_code: bool - ) -> Tuple[int, bytes, bytes]: - if pp.exitcode and check_code: - msg = f"process {cmd} exit with code {pp.exitcode}" - msg += f"\n>>> stdout <<<\n{pp.out.decode()}" - msg += "\n" - msg += f"\n>>> stderr <<<\n{pp.err.decode()}" - raise RuntimeError(msg) - return cast(int, pp.exitcode), pp.out, pp.err - - -class TestProcessProtocol(ProcessProtocol): - def __init__(self) -> None: - self.deferred: Deferred = Deferred() - self.out: bytes = b"" - self.err: bytes = b"" - self.exitcode: Optional[int] = None - - def outReceived(self, data: bytes) -> None: - self.out += data - - def errReceived(self, data: bytes) -> None: - self.err += data - - def processEnded(self, status: Failure) -> None: - self.exitcode = cast(ProcessTerminated, status.value).exitCode - self.deferred.callback(self) diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py deleted file mode 100644 index de9ce992a..000000000 --- a/scrapy/utils/testsite.py +++ /dev/null @@ -1,55 +0,0 @@ -from urllib.parse import urljoin - -from twisted.web import resource, server, static, util - - -class SiteTest: - def setUp(self): - from twisted.internet import reactor - - super().setUp() - self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") - self.baseurl = f"http://localhost:{self.site.getHost().port}/" - - def tearDown(self): - super().tearDown() - self.site.stopListening() - - def url(self, path): - return urljoin(self.baseurl, path) - - -class NoMetaRefreshRedirect(util.Redirect): - def render(self, request): - content = util.Redirect.render(self, request) - return content.replace( - b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"' - ) - - -def test_site(): - r = resource.Resource() - r.putChild(b"text", static.Data(b"Works", "text/plain")) - r.putChild( - b"html", - static.Data( - b"

Works

World

", - "text/html", - ), - ) - r.putChild( - b"enc-gb18030", - static.Data(b"

gb18030 encoding

", "text/html; charset=gb18030"), - ) - r.putChild(b"redirect", util.Redirect(b"/redirected")) - r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")) - r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) - return server.Site(r) - - -if __name__ == "__main__": - from twisted.internet import reactor - - port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") - print(f"http://localhost:{port.getHost().port}/") - reactor.run() diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 9ff9a273f..87df10a02 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -7,21 +7,32 @@ subclass from object_ref (instead of object). About performance: This library has a minimal performance impact when enabled, and no performance penalty at all when disabled (as object_ref becomes just an alias to object in that case). + +.. note:: PyPy uses a tracing garbage collector, so objects may + remain in the ``live_refs`` longer than expected, even after they + go out of scope. If deterministic behavior is required, you may need + to explicitly trigger garbage collection or call ``trackref.live_refs.clear()``. """ +from __future__ import annotations + from collections import defaultdict from operator import itemgetter -from time import time -from typing import TYPE_CHECKING, Any, DefaultDict, Iterable +from time import monotonic_ns +from types import NoneType +from typing import TYPE_CHECKING, Any from weakref import WeakKeyDictionary if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self -NoneType = type(None) -live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) +live_refs: defaultdict[type, WeakKeyDictionary[object, float]] = defaultdict( + WeakKeyDictionary +) class object_ref: @@ -29,9 +40,9 @@ class object_ref: __slots__ = () - def __new__(cls, *args: Any, **kwargs: Any) -> "Self": + def __new__(cls, *args: Any, **kwargs: Any) -> Self: obj = object.__new__(cls) - live_refs[cls][obj] = time() + live_refs[cls][obj] = monotonic_ns() return obj @@ -39,14 +50,14 @@ class object_ref: def format_live_refs(ignore: Any = NoneType) -> str: """Return a tabular representation of tracked objects""" s = "Live References\n\n" - now = time() + now_ns = monotonic_ns() for cls, wdict in sorted(live_refs.items(), key=lambda x: x[0].__name__): if not wdict: continue if issubclass(cls, ignore): continue - oldest = min(wdict.values()) - s += f"{cls.__name__:<30} {len(wdict):6} oldest: {int(now - oldest)}s ago\n" + oldest_ns = min(wdict.values()) + s += f"{cls.__name__:<30} {len(wdict):6} oldest: {int((now_ns - oldest_ns) // 1e9)}s ago\n" return s @@ -62,6 +73,7 @@ def get_oldest(class_name: str) -> Any: if not wdict: break return min(wdict.items(), key=itemgetter(1))[0] + return None def iter_all(class_name: str) -> Iterable[Any]: @@ -69,4 +81,4 @@ def iter_all(class_name: str) -> Iterable[Any]: for cls, wdict in live_refs.items(): if cls.__name__ == class_name: return wdict.keys() - return [] + return () diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 22b4197f9..8f75e2618 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -1,26 +1,22 @@ """ This module contains general purpose URL functions not found in the standard library. - -Some of the functions that used to be imported from this module have been moved -to the w3lib.url module. Always import those from there instead. """ + +from __future__ import annotations + import re -from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast -from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse +from typing import TYPE_CHECKING, TypeAlias +from urllib.parse import ParseResult, urlparse, urlunparse -# scrapy.utils.url was moved to w3lib.url and import * ensures this -# move doesn't break old code -from w3lib.url import * -from w3lib.url import _safe_chars, _unquotepath # noqa: F401 - -from scrapy.utils.python import to_unicode +from w3lib.url import any_to_uri, parse_url if TYPE_CHECKING: + from collections.abc import Iterable + from scrapy import Spider - -UrlT = Union[str, bytes, ParseResult] +UrlT: TypeAlias = str | bytes | ParseResult def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: @@ -28,15 +24,18 @@ def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: host = parse_url(url).netloc.lower() if not host: return False - domains = [d.lower() for d in domains] - return any((host == d) or (host.endswith(f".{d}")) for d in domains) + return any((host == d) or (host.endswith(f".{d}")) for d in map(str.lower, domains)) -def url_is_from_spider(url: UrlT, spider: Type["Spider"]) -> bool: +def _spider_domains(spider: type[Spider]) -> Iterable[str]: + yield spider.name + if allowed_domains := getattr(spider, "allowed_domains", None): + yield from allowed_domains + + +def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool: """Return True if the url belongs to the given spider""" - return url_is_from_any_domain( - url, [spider.name] + list(getattr(spider, "allowed_domains", [])) - ) + return url_is_from_any_domain(url, _spider_domains(spider)) def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool: @@ -45,47 +44,9 @@ def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool: return any(lowercase_path.endswith(ext) for ext in extensions) -def parse_url(url: UrlT, encoding: Optional[str] = None) -> ParseResult: - """Return urlparsed url from the given argument (which could be an already - parsed url) - """ - if isinstance(url, ParseResult): - return url - return cast(ParseResult, urlparse(to_unicode(url, encoding))) - - -def escape_ajax(url: str) -> str: - """ - Return the crawlable url according to: - https://developers.google.com/webmasters/ajax-crawling/docs/getting-started - - >>> escape_ajax("www.example.com/ajax.html#!key=value") - 'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue' - >>> escape_ajax("www.example.com/ajax.html?k1=v1&k2=v2#!key=value") - 'www.example.com/ajax.html?k1=v1&k2=v2&_escaped_fragment_=key%3Dvalue' - >>> escape_ajax("www.example.com/ajax.html?#!key=value") - 'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue' - >>> escape_ajax("www.example.com/ajax.html#!") - 'www.example.com/ajax.html?_escaped_fragment_=' - - URLs that are not "AJAX crawlable" (according to Google) returned as-is: - - >>> escape_ajax("www.example.com/ajax.html#key=value") - 'www.example.com/ajax.html#key=value' - >>> escape_ajax("www.example.com/ajax.html#") - 'www.example.com/ajax.html#' - >>> escape_ajax("www.example.com/ajax.html") - 'www.example.com/ajax.html' - """ - defrag, frag = urldefrag(url) - if not frag.startswith("!"): - return url - return add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:]) - - def add_http_if_no_scheme(url: str) -> str: """Add http as the default scheme if it is missing from the url.""" - match = re.match(r"^\w+://", url, flags=re.I) + match = re.match(r"^\w+://", url, flags=re.IGNORECASE) if not match: parts = urlparse(url) scheme = "http:" if parts.netloc else "http://" @@ -167,13 +128,19 @@ def strip_url( parsed_url.username or parsed_url.password ): netloc = netloc.split("@")[-1] - if strip_default_port and parsed_url.port: - if (parsed_url.scheme, parsed_url.port) in ( + + if ( + strip_default_port + and parsed_url.port + and (parsed_url.scheme, parsed_url.port) + in { ("http", 80), ("https", 443), ("ftp", 21), - ): - netloc = netloc.replace(f":{parsed_url.port}", "") + } + ): + netloc = netloc.replace(f":{parsed_url.port}", "") + return urlunparse( ( parsed_url.scheme, diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index 9b637bdb0..6bd96a215 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -1,32 +1,32 @@ +from __future__ import annotations + import platform import sys -from typing import List, Tuple +from importlib.metadata import version -import cryptography -import cssselect import lxml.etree -import parsel -import twisted -import w3lib -import scrapy +from scrapy.settings.default_settings import LOG_VERSIONS from scrapy.utils.ssl import get_openssl_version +_DEFAULT_SOFTWARE: list[str] = ["Scrapy", *LOG_VERSIONS] -def scrapy_components_versions() -> List[Tuple[str, str]]: - lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) - libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) - return [ - ("Scrapy", scrapy.__version__), - ("lxml", lxml_version), - ("libxml2", libxml2_version), - ("cssselect", cssselect.__version__), - ("parsel", parsel.__version__), - ("w3lib", w3lib.__version__), - ("Twisted", twisted.version.short()), - ("Python", sys.version.replace("\n", "- ")), - ("pyOpenSSL", get_openssl_version()), - ("cryptography", cryptography.__version__), - ("Platform", platform.platform()), - ] +def _version(item: str) -> str: + lowercase_item = item.lower() + if lowercase_item == "libxml2": + return ".".join(map(str, lxml.etree.LIBXML_VERSION)) + if lowercase_item == "platform": + return platform.platform() + if lowercase_item == "pyopenssl": + return get_openssl_version() + if lowercase_item == "python": + return sys.version.replace("\n", "- ") + return version(item) + + +def get_versions( + software: list[str] | None = None, +) -> list[tuple[str, str]]: + software = software or _DEFAULT_SOFTWARE + return [(item, _version(item)) for item in software] diff --git a/sep/sep-004.rst b/sep/sep-004.rst index b1cef2600..7a4ebe886 100644 --- a/sep/sep-004.rst +++ b/sep/sep-004.rst @@ -11,7 +11,7 @@ SEP-004: Library API ==================== .. note:: the library API has been implemented, but slightly different from proposed in this SEP. You can run a Scrapy crawler inside a Twisted - reactor, but not outside it. + reactor, but not outside it. Introduction ============ diff --git a/sep/sep-007.rst b/sep/sep-007.rst index 0ca2036ce..73ce0d338 100644 --- a/sep/sep-007.rst +++ b/sep/sep-007.rst @@ -96,7 +96,7 @@ specified, else utf-8 is used) and returns a new unicode object. E.g: ``clean_spaces`` ---------------- - + Converts multispaces into single spaces for the given string. E.g: :: diff --git a/sep/sep-008.rst b/sep/sep-008.rst index be5987e39..baf1fde23 100644 --- a/sep/sep-008.rst +++ b/sep/sep-008.rst @@ -37,8 +37,8 @@ Modules and classes - ``scrapy.contrib.itemparser.ItemParser`` - ``scrapy.contrib.itemparser.XPathItemParser`` -- ``scrapy.contrib.itemparser.parsers.``MapConcat`` *(ex. ``TreeExpander``)* -- ``scrapy.contrib.itemparser.parsers.``TakeFirst`` +- ``scrapy.contrib.itemparser.parsers.MapConcat`` *(ex. ``TreeExpander``)* +- ``scrapy.contrib.itemparser.parsers.TakeFirst`` - ``scrapy.contrib.itemparser.parsers.Join`` - ``scrapy.contrib.itemparser.parsers.Identity`` @@ -70,11 +70,11 @@ Alternative Public API Proposal - ``ItemLoader.replace_value()`` - ``ItemLoader.load_item()`` *(returns loaded item)* -- ``ItemLoader.get_stored_values()`` or ``ItemLoader.get_values()`` *(returns the ``ItemLoader values)* +- ``ItemLoader.get_stored_values()`` or ``ItemLoader.get_values()`` *(returns the ``ItemLoader`` values)* - ``ItemLoader.get_output_value()`` -- ``ItemLoader.get_input_processor()`` or ``ItemLoader.get_in_processor()`` *(short version)* -- ``ItemLoader.get_output_processor()`` or ``ItemLoader.get_out_processor()`` *(short version)* +- ``ItemLoader.get_input_processor()`` or ``ItemLoader.get_in_processor()`` *(short version)* +- ``ItemLoader.get_output_processor()`` or ``ItemLoader.get_out_processor()`` *(short version)* - ``ItemLoader.context`` diff --git a/sep/sep-014.rst b/sep/sep-014.rst index e03a2b0f6..0a2e6b51e 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -21,7 +21,7 @@ Current flaws and inconsistencies 2. Link extractors are inflexible and hard to maintain, link processing/filtering is tightly coupled. (e.g. canonicalize) 3. Isn't possible to crawl an url directly from command line because the Spider - does not know which callback use. + does not know which callback use. These flaws will be corrected by the changes proposed in this SEP. @@ -55,7 +55,7 @@ Request Extractors Request Extractors takes response object and determines which requests follow. This is an enhancement to ``LinkExtractors`` which returns urls (links), -Request Extractors return Request objects. +Request Extractors return Request objects. Request Processors ------------------ diff --git a/sep/sep-018.rst b/sep/sep-018.rst index 9ac62c090..29b1f860e 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -140,7 +140,7 @@ Example: The data flow with Spider Middleware v2 is as follows: -1. When a response arrives from the engine, it it passed through all the spider +1. When a response arrives from the engine, it is passed through all the spider middlewares (in descending order). The result of each middleware ``process_response`` is kept and then returned along with the spider callback result @@ -200,7 +200,7 @@ the same spider: # extract item from response return item -The Spider Middleware that implements spider code +The Spider Middleware that implements spider code ================================================= There's gonna be one middleware that will take care of calling the proper @@ -619,13 +619,13 @@ Resolved: ``manager.scraper.process_request()`` instead of ``manager.engine.crawl()`` - should we support adding additional start requests from a spider middleware? - - Yes - there is a spider middleware method (``start_requests``) for that + - Yes - there is a spider middleware method (``start_requests()``) for that - should ``process_response()`` receive a ``request`` argument with the ``request`` that originated it?. ``response.request`` is the latest request, not the original one (think of redirections), but it does carry the ``meta`` of the original one. The original one may not be available anymore (in memory) if we're using a persistent scheduler., but in that case it would be - the deserialized request from the persistent scheduler queue. + the deserialized request from the persistent scheduler queue. - No - this would make implementation more complex and we're not sure it's really needed diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 151e784c6..000000000 --- a/setup.cfg +++ /dev/null @@ -1,24 +0,0 @@ -[bdist_rpm] -doc_files = docs AUTHORS INSTALL LICENSE README.rst - -[bdist_wheel] -universal=1 - -[mypy] -ignore_missing_imports = true - -# Interface classes are hard to support - -[mypy-twisted.internet.interfaces] -follow_imports = skip - -[mypy-scrapy.interfaces] -ignore_errors = True - -[mypy-twisted.internet.reactor] -follow_imports = skip - -# FIXME: remove the following sections once the issues are solved - -[mypy-scrapy.settings.default_settings] -ignore_errors = True diff --git a/setup.py b/setup.py deleted file mode 100644 index 98501d447..000000000 --- a/setup.py +++ /dev/null @@ -1,76 +0,0 @@ -from pathlib import Path - -from setuptools import find_packages, setup - -version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() - - -install_requires = [ - "Twisted>=18.9.0", - "cryptography>=36.0.0", - "cssselect>=0.9.1", - "itemloaders>=1.0.1", - "parsel>=1.5.0", - "pyOpenSSL>=21.0.0", - "queuelib>=1.4.2", - "service_identity>=18.1.0", - "w3lib>=1.17.0", - "zope.interface>=5.1.0", - "protego>=0.1.15", - "itemadapter>=0.1.0", - "setuptools", - "packaging", - "tldextract", - "lxml>=4.4.1", - "xtractmime>=0.2.0", -] -extras_require = { - ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"], - ':platform_python_implementation == "PyPy"': ["PyPyDispatcher>=2.1.0"], -} - - -setup( - name="Scrapy", - version=version, - url="https://scrapy.org", - project_urls={ - "Documentation": "https://docs.scrapy.org/", - "Source": "https://github.com/scrapy/scrapy", - "Tracker": "https://github.com/scrapy/scrapy/issues", - }, - description="A high-level Web Crawling and Web Scraping framework", - long_description=open("README.rst", encoding="utf-8").read(), - author="Scrapy developers", - author_email="pablo@pablohoffman.com", - maintainer="Pablo Hoffman", - maintainer_email="pablo@pablohoffman.com", - license="BSD", - packages=find_packages(exclude=("tests", "tests.*")), - include_package_data=True, - zip_safe=False, - entry_points={"console_scripts": ["scrapy = scrapy.cmdline:execute"]}, - classifiers=[ - "Framework :: Scrapy", - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Topic :: Internet :: WWW/HTTP", - "Topic :: Software Development :: Libraries :: Application Frameworks", - "Topic :: Software Development :: Libraries :: Python Modules", - ], - python_requires=">=3.8", - install_requires=install_requires, - extras_require=extras_require, -) diff --git a/tests/AsyncCrawlerProcess/args_settings.py b/tests/AsyncCrawlerProcess/args_settings.py new file mode 100644 index 000000000..5e162e782 --- /dev/null +++ b/tests/AsyncCrawlerProcess/args_settings.py @@ -0,0 +1,25 @@ +from typing import Any + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess, Crawler + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + @classmethod + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("FOO", kwargs.get("foo")) + return spider + + async def start(self): + self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") + return + yield + + +process = AsyncCrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider, foo=42) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_custom_loop.py b/tests/AsyncCrawlerProcess/asyncio_custom_loop.py new file mode 100644 index 000000000..172e36b7b --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_custom_loop.py @@ -0,0 +1,20 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_different.py b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_different.py new file mode 100644 index 000000000..d76da51a1 --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_different.py @@ -0,0 +1,23 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + custom_settings = { + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": None, + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_same.py b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_same.py new file mode 100644 index 000000000..bd4a99e14 --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_same.py @@ -0,0 +1,23 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + custom_settings = { + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py new file mode 100644 index 000000000..e000a23db --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import asyncio +import sys + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.defer import deferred_from_coro + + +class UppercasePipeline: + async def _open_spider(self, spider): + spider.logger.info("async pipeline opened!") + await asyncio.sleep(0.1) + + def open_spider(self, spider): + return deferred_from_coro(self._open_spider(spider)) + + def process_item(self, item): + return {"url": item["url"].upper()} + + +class UrlSpider(Spider): + name = "url_spider" + start_urls = ["data:,"] + custom_settings = { + "ITEM_PIPELINES": {UppercasePipeline: 100}, + } + + def parse(self, response): + yield {"url": response.url} + + +if __name__ == "__main__": + ASYNCIO_EVENT_LOOP: str | None + try: + ASYNCIO_EVENT_LOOP = sys.argv[1] + except IndexError: + ASYNCIO_EVENT_LOOP = None + + process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP, + } + ) + process.crawl(UrlSpider) + process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py b/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py new file mode 100644 index 000000000..3c47eb826 --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py @@ -0,0 +1,27 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.reactor import is_asyncio_reactor_installed + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py new file mode 100644 index 000000000..e025e17d1 --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py @@ -0,0 +1,53 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + is_reactor_installed, +) + +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed before is_asyncio_reactor_installed()." + ) + +try: + is_asyncio_reactor_installed() +except RuntimeError: + pass +else: + raise RuntimeError("is_asyncio_reactor_installed() did not raise RuntimeError.") + +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed after is_asyncio_reactor_installed()." + ) + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + +if not is_asyncio_reactor_installed(): + raise RuntimeError("Wrong reactor installed after install_reactor().") + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py new file mode 100644 index 000000000..4257bc0ac --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -0,0 +1,29 @@ +import asyncio +import sys + +from twisted.internet import asyncioreactor + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + +if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +asyncioreactor.install(asyncio.get_event_loop()) + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py new file mode 100644 index 000000000..9c6fd090b --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -0,0 +1,31 @@ +import asyncio +import sys + +from twisted.internet import asyncioreactor +from uvloop import Loop + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + +if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +asyncio.set_event_loop(Loop()) +asyncioreactor.install(asyncio.get_event_loop()) + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py new file mode 100644 index 000000000..a6520ca22 --- /dev/null +++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py @@ -0,0 +1,35 @@ +import sys + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class CachingHostnameResolverSpider(scrapy.Spider): + """ + Finishes in a finite amount of time (does not hang indefinitely in the DNS resolution) + """ + + name = "caching_hostname_resolver_spider" + + async def start(self): + yield scrapy.Request(self.url) + + def parse(self, response): + for _ in range(10): + yield scrapy.Request( + response.url, dont_filter=True, callback=self.ignore_response + ) + + def ignore_response(self, response): + self.logger.info(repr(response.ip_address)) + + +if __name__ == "__main__": + process = AsyncCrawlerProcess( + settings={ + "RETRY_ENABLED": False, + "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + } + ) + process.crawl(CachingHostnameResolverSpider, url=sys.argv[1]) + process.start() diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py new file mode 100644 index 000000000..55d2ef711 --- /dev/null +++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class CachingHostnameResolverSpider(scrapy.Spider): + """ + Finishes without a scrapy.exceptions.CannotResolveHostError exception + """ + + name = "caching_hostname_resolver_spider" + start_urls = ["http://[::1]"] + + +if __name__ == "__main__": + process = AsyncCrawlerProcess( + settings={ + "RETRY_ENABLED": False, + "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + } + ) + process.crawl(CachingHostnameResolverSpider) + process.start() diff --git a/tests/AsyncCrawlerProcess/default_name_resolver.py b/tests/AsyncCrawlerProcess/default_name_resolver.py new file mode 100644 index 000000000..4c8897f8f --- /dev/null +++ b/tests/AsyncCrawlerProcess/default_name_resolver.py @@ -0,0 +1,18 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class IPv6Spider(scrapy.Spider): + """ + Raises a scrapy.exceptions.CannotResolveHostError: + the default name resolver does not handle IPv6 addresses. + """ + + name = "ipv6_spider" + start_urls = ["http://[::1]"] + + +if __name__ == "__main__": + process = AsyncCrawlerProcess(settings={"RETRY_ENABLED": False}) + process.crawl(IPv6Spider) + process.start() diff --git a/tests/AsyncCrawlerProcess/multi.py b/tests/AsyncCrawlerProcess/multi.py new file mode 100644 index 000000000..2eede5471 --- /dev/null +++ b/tests/AsyncCrawlerProcess/multi.py @@ -0,0 +1,17 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactor_default.py b/tests/AsyncCrawlerProcess/reactor_default.py new file mode 100644 index 000000000..9638652bd --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactor_default.py @@ -0,0 +1,18 @@ +from twisted.internet import reactor # noqa: F401,TID253 + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess(settings={}) + +d = process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_custom_settings.py b/tests/AsyncCrawlerProcess/reactorless_custom_settings.py new file mode 100644 index 000000000..f49741441 --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_custom_settings.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.reactorless import is_reactorless + +if TYPE_CHECKING: + from asyncio import Task + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + custom_settings = { + "TWISTED_REACTOR_ENABLED": False, + } + + async def start(self): + self.logger.info(f"is_reactorless(): {is_reactorless()}") + return + yield + + +def log_task_exception(task: Task[None]) -> None: + try: + task.result() + except Exception: + logging.exception("Crawl task failed") # noqa: LOG015 + + +process = AsyncCrawlerProcess() +task = process.crawl(NoRequestsSpider) +task.add_done_callback(log_task_exception) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_datauri.py b/tests/AsyncCrawlerProcess/reactorless_datauri.py new file mode 100644 index 000000000..5270b7b83 --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_datauri.py @@ -0,0 +1,18 @@ +from scrapy import Request, Spider +from scrapy.crawler import AsyncCrawlerProcess + + +class DataSpider(Spider): + name = "data" + + async def start(self): + yield Request("data:,foo") + + def parse(self, response): + return {"data": response.text} + + +process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) + +process.crawl(DataSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_import_hook.py b/tests/AsyncCrawlerProcess/reactorless_import_hook.py new file mode 100644 index 000000000..57e2a5c26 --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_import_hook.py @@ -0,0 +1,18 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + import twisted.internet.reactor # noqa: F401 + + return + yield + + +process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) + +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_reactor.py b/tests/AsyncCrawlerProcess/reactorless_reactor.py new file mode 100644 index 000000000..481112ca7 --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_reactor.py @@ -0,0 +1,6 @@ +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.reactor import install_reactor + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + +AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) diff --git a/tests/AsyncCrawlerProcess/reactorless_simple.py b/tests/AsyncCrawlerProcess/reactorless_simple.py new file mode 100644 index 000000000..33b4e8cb2 --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_simple.py @@ -0,0 +1,18 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.reactorless import is_reactorless + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + self.logger.info(f"is_reactorless(): {is_reactorless()}") + return + yield + + +process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) + +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_sleeping.py b/tests/AsyncCrawlerProcess/reactorless_sleeping.py new file mode 100644 index 000000000..12101d221 --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_sleeping.py @@ -0,0 +1,20 @@ +import asyncio +import sys + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class SleepingSpider(scrapy.Spider): + name = "sleeping" + + start_urls = ["data:,;"] + + async def parse(self, response): + await asyncio.sleep(int(sys.argv[1])) + + +process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) + +process.crawl(SleepingSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_telnetconsole_disabled.py b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_disabled.py new file mode 100644 index 000000000..e59a365bc --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_disabled.py @@ -0,0 +1,21 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR_ENABLED": False, + "TELNETCONSOLE_ENABLED": False, + } +) + +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_telnetconsole_enabled.py b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_enabled.py new file mode 100644 index 000000000..34daf8779 --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_enabled.py @@ -0,0 +1,21 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR_ENABLED": False, + "TELNETCONSOLE_ENABLED": True, + } +) + +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/simple.py b/tests/AsyncCrawlerProcess/simple.py new file mode 100644 index 000000000..368e05608 --- /dev/null +++ b/tests/AsyncCrawlerProcess/simple.py @@ -0,0 +1,18 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.reactorless import is_reactorless + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + self.logger.info(f"is_reactorless(): {is_reactorless()}") + return + yield + + +process = AsyncCrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/sleeping.py b/tests/AsyncCrawlerProcess/sleeping.py new file mode 100644 index 000000000..88caf5032 --- /dev/null +++ b/tests/AsyncCrawlerProcess/sleeping.py @@ -0,0 +1,20 @@ +import asyncio +import sys + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class SleepingSpider(scrapy.Spider): + name = "sleeping" + + start_urls = ["data:,;"] + + async def parse(self, response): + await asyncio.sleep(int(sys.argv[1])) + + +process = AsyncCrawlerProcess(settings={}) + +process.crawl(SleepingSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_asyncio.py b/tests/AsyncCrawlerProcess/twisted_reactor_asyncio.py new file mode 100644 index 000000000..dc820ea3a --- /dev/null +++ b/tests/AsyncCrawlerProcess/twisted_reactor_asyncio.py @@ -0,0 +1,15 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class AsyncioReactorSpider(scrapy.Spider): + name = "asyncio_reactor" + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } +) +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings.py new file mode 100644 index 000000000..5fd48274a --- /dev/null +++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings.py @@ -0,0 +1,14 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class AsyncioReactorSpider(scrapy.Spider): + name = "asyncio_reactor" + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = AsyncCrawlerProcess() +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_same.py new file mode 100644 index 000000000..c205c3cd2 --- /dev/null +++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_same.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class AsyncioReactorSpider1(scrapy.Spider): + name = "asyncio_reactor1" + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +class AsyncioReactorSpider2(scrapy.Spider): + name = "asyncio_reactor2" + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = AsyncCrawlerProcess() +process.crawl(AsyncioReactorSpider1) +process.crawl(AsyncioReactorSpider2) +process.start() diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py new file mode 100644 index 000000000..b85f60f87 --- /dev/null +++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + +if TYPE_CHECKING: + from asyncio import Task + + +class AsyncioReactorSpider(scrapy.Spider): + name = "asyncio_reactor" + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", + } + + +def log_task_exception(task: Task[None]) -> None: + try: + task.result() + except Exception: + logging.exception("Crawl task failed") # noqa: LOG015 + + +process = AsyncCrawlerProcess() +task = process.crawl(AsyncioReactorSpider) +task.add_done_callback(log_task_exception) +process.start() diff --git a/tests/AsyncCrawlerRunner/custom_loop_different.py b/tests/AsyncCrawlerRunner/custom_loop_different.py new file mode 100644 index 000000000..89cf0e536 --- /dev/null +++ b/tests/AsyncCrawlerRunner/custom_loop_different.py @@ -0,0 +1,31 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/AsyncCrawlerRunner/custom_loop_same.py b/tests/AsyncCrawlerRunner/custom_loop_same.py new file mode 100644 index 000000000..43d0dc053 --- /dev/null +++ b/tests/AsyncCrawlerRunner/custom_loop_same.py @@ -0,0 +1,31 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor", "uvloop.Loop") +react(main) diff --git a/tests/AsyncCrawlerRunner/multi_parallel.py b/tests/AsyncCrawlerRunner/multi_parallel.py new file mode 100644 index 000000000..f1af9f794 --- /dev/null +++ b/tests/AsyncCrawlerRunner/multi_parallel.py @@ -0,0 +1,28 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + runner.crawl(NoRequestsSpider) + runner.crawl(NoRequestsSpider) + await runner.join() + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/AsyncCrawlerRunner/multi_seq.py b/tests/AsyncCrawlerRunner/multi_seq.py new file mode 100644 index 000000000..987f7a514 --- /dev/null +++ b/tests/AsyncCrawlerRunner/multi_seq.py @@ -0,0 +1,27 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/AsyncCrawlerRunner/no_reactor.py b/tests/AsyncCrawlerRunner/no_reactor.py new file mode 100644 index 000000000..6a473e5e8 --- /dev/null +++ b/tests/AsyncCrawlerRunner/no_reactor.py @@ -0,0 +1,22 @@ +import asyncio + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +async def main() -> None: + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +asyncio.run(main()) diff --git a/tests/AsyncCrawlerRunner/reactorless_custom_settings.py b/tests/AsyncCrawlerRunner/reactorless_custom_settings.py new file mode 100644 index 000000000..9b5efc3f9 --- /dev/null +++ b/tests/AsyncCrawlerRunner/reactorless_custom_settings.py @@ -0,0 +1,27 @@ +import asyncio + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactorless import is_reactorless + + +class NoRequestsSpider(Spider): + name = "no_request" + custom_settings = { + "TWISTED_REACTOR_ENABLED": False, + } + + async def start(self): + self.logger.info(f"is_reactorless(): {is_reactorless()}") + return + yield + + +async def main() -> None: + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +asyncio.run(main()) diff --git a/tests/AsyncCrawlerRunner/reactorless_datauri.py b/tests/AsyncCrawlerRunner/reactorless_datauri.py new file mode 100644 index 000000000..d964f9c1c --- /dev/null +++ b/tests/AsyncCrawlerRunner/reactorless_datauri.py @@ -0,0 +1,24 @@ +import asyncio + +from scrapy import Request, Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.log import configure_logging + + +class DataSpider(Spider): + name = "data" + + async def start(self): + yield Request("data:,foo") + + def parse(self, response): + return {"data": response.text} + + +async def main() -> None: + configure_logging() + runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) + await runner.crawl(DataSpider) + + +asyncio.run(main()) diff --git a/tests/AsyncCrawlerRunner/reactorless_reactor.py b/tests/AsyncCrawlerRunner/reactorless_reactor.py new file mode 100644 index 000000000..4889b2ddc --- /dev/null +++ b/tests/AsyncCrawlerRunner/reactorless_reactor.py @@ -0,0 +1,24 @@ +import asyncio + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +async def main() -> None: + configure_logging() + runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +asyncio.run(main()) diff --git a/tests/AsyncCrawlerRunner/reactorless_simple.py b/tests/AsyncCrawlerRunner/reactorless_simple.py new file mode 100644 index 000000000..e91c7c89b --- /dev/null +++ b/tests/AsyncCrawlerRunner/reactorless_simple.py @@ -0,0 +1,24 @@ +import asyncio + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactorless import is_reactorless + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + self.logger.info(f"is_reactorless(): {is_reactorless()}") + return + yield + + +async def main() -> None: + configure_logging() + runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) + await runner.crawl(NoRequestsSpider) + + +asyncio.run(main()) diff --git a/tests/AsyncCrawlerRunner/simple.py b/tests/AsyncCrawlerRunner/simple.py new file mode 100644 index 000000000..29b132d75 --- /dev/null +++ b/tests/AsyncCrawlerRunner/simple.py @@ -0,0 +1,28 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor +from scrapy.utils.reactorless import is_reactorless + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + self.logger.info(f"is_reactorless(): {is_reactorless()}") + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/AsyncCrawlerRunner/simple_default_reactor.py b/tests/AsyncCrawlerRunner/simple_default_reactor.py new file mode 100644 index 000000000..ae052f188 --- /dev/null +++ b/tests/AsyncCrawlerRunner/simple_default_reactor.py @@ -0,0 +1,24 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +react(main) diff --git a/tests/CrawlerProcess/args_settings.py b/tests/CrawlerProcess/args_settings.py index a46a8806b..c8a3d0a5b 100644 --- a/tests/CrawlerProcess/args_settings.py +++ b/tests/CrawlerProcess/args_settings.py @@ -13,9 +13,10 @@ class NoRequestsSpider(scrapy.Spider): spider.settings.set("FOO", kwargs.get("foo")) return spider - def start_requests(self): + async def start(self): self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") - return [] + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/asyncio_custom_loop.py b/tests/CrawlerProcess/asyncio_custom_loop.py index 5e72aa6d4..bd78a0de7 100644 --- a/tests/CrawlerProcess/asyncio_custom_loop.py +++ b/tests/CrawlerProcess/asyncio_custom_loop.py @@ -5,8 +5,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index 1afef4d24..eb2e1105c 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -1,6 +1,7 @@ +from __future__ import annotations + import asyncio import sys -from typing import Optional from scrapy import Spider from scrapy.crawler import CrawlerProcess @@ -15,7 +16,7 @@ class UppercasePipeline: def open_spider(self, spider): return deferred_from_coro(self._open_spider(spider)) - def process_item(self, item, spider): + def process_item(self, item): return {"url": item["url"].upper()} @@ -31,7 +32,7 @@ class UrlSpider(Spider): if __name__ == "__main__": - ASYNCIO_EVENT_LOOP: Optional[str] + ASYNCIO_EVENT_LOOP: str | None try: ASYNCIO_EVENT_LOOP = sys.argv[1] except IndexError: diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py index 6df6d76fa..6bb6fb3c6 100644 --- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py @@ -1,17 +1,26 @@ import scrapy from scrapy.crawler import CrawlerProcess +from scrapy.utils.reactor import is_asyncio_reactor_installed + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, } ) process.crawl(NoRequestsSpider) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index 01d23c963..4e8d3db12 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,26 +1,62 @@ -import asyncio -import sys +import scrapy +from scrapy.crawler import CrawlerProcess +from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + is_reactor_installed, +) -from twisted.internet import asyncioreactor +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed before is_asyncio_reactor_installed()." + ) -if sys.version_info >= (3, 8) and sys.platform == "win32": - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -asyncioreactor.install(asyncio.get_event_loop()) +try: + is_asyncio_reactor_installed() +except RuntimeError: + pass +else: + raise RuntimeError("is_asyncio_reactor_installed() did not raise RuntimeError.") -import scrapy # noqa: E402 -from scrapy.crawler import CrawlerProcess # noqa: E402 +try: + is_asyncio_available() +except RuntimeError: + pass +else: + raise RuntimeError("is_asyncio_available() did not raise RuntimeError.") + +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed after is_asyncio_reactor_installed()." + ) + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + +if not is_asyncio_reactor_installed(): + raise RuntimeError("Wrong reactor installed after install_reactor().") + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") + if not is_asyncio_available(): + raise RuntimeError("ReactorCheckExtension requires asyncio support.") class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, } ) process.crawl(NoRequestsSpider) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index 9dc8ce46b..7c50277b9 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -4,19 +4,20 @@ import sys from twisted.internet import asyncioreactor from twisted.python import log -if sys.version_info >= (3, 8) and sys.platform == "win32": - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -asyncioreactor.install(asyncio.get_event_loop()) +import scrapy +from scrapy.crawler import CrawlerProcess -import scrapy # noqa: E402 -from scrapy.crawler import CrawlerProcess # noqa: E402 +if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +asyncioreactor.install() class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index be9c83b95..a2e63a0d0 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -4,20 +4,22 @@ import sys from twisted.internet import asyncioreactor from uvloop import Loop -if sys.version_info >= (3, 8) and sys.platform == "win32": - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -asyncio.set_event_loop(Loop()) -asyncioreactor.install(asyncio.get_event_loop()) +import scrapy +from scrapy.crawler import CrawlerProcess -import scrapy # noqa: E402 -from scrapy.crawler import CrawlerProcess # noqa: E402 +if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +loop = Loop() +asyncio.set_event_loop(loop) +asyncioreactor.install(loop) class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py index 7b0497bde..7f687e7d9 100644 --- a/tests/CrawlerProcess/caching_hostname_resolver.py +++ b/tests/CrawlerProcess/caching_hostname_resolver.py @@ -11,7 +11,7 @@ class CachingHostnameResolverSpider(scrapy.Spider): name = "caching_hostname_resolver_spider" - def start_requests(self): + async def start(self): yield scrapy.Request(self.url) def parse(self, response): @@ -28,7 +28,7 @@ if __name__ == "__main__": process = CrawlerProcess( settings={ "RETRY_ENABLED": False, - "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", } ) process.crawl(CachingHostnameResolverSpider, url=sys.argv[1]) diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py index d5d19e27e..da9c16cb8 100644 --- a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py +++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py @@ -4,7 +4,7 @@ from scrapy.crawler import CrawlerProcess class CachingHostnameResolverSpider(scrapy.Spider): """ - Finishes without a twisted.internet.error.DNSLookupError exception + Finishes without a scrapy.exceptions.CannotResolveHostError exception """ name = "caching_hostname_resolver_spider" @@ -15,7 +15,7 @@ if __name__ == "__main__": process = CrawlerProcess( settings={ "RETRY_ENABLED": False, - "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", } ) process.crawl(CachingHostnameResolverSpider) diff --git a/tests/CrawlerProcess/default_name_resolver.py b/tests/CrawlerProcess/default_name_resolver.py index cfc4b38b7..f4c129fdf 100644 --- a/tests/CrawlerProcess/default_name_resolver.py +++ b/tests/CrawlerProcess/default_name_resolver.py @@ -4,7 +4,7 @@ from scrapy.crawler import CrawlerProcess class IPv6Spider(scrapy.Spider): """ - Raises a twisted.internet.error.DNSLookupError: + Raises a scrapy.exceptions.CannotResolveHostError: the default name resolver does not handle IPv6 addresses. """ diff --git a/tests/CrawlerProcess/multi.py b/tests/CrawlerProcess/multi.py index 9f7eaf2ae..0058896b5 100644 --- a/tests/CrawlerProcess/multi.py +++ b/tests/CrawlerProcess/multi.py @@ -5,8 +5,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index 078cb72cb..cbe6427ea 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -1,4 +1,5 @@ -from twisted.internet import reactor # noqa: F401 +from twisted.internet import reactor # noqa: F401,TID253 +from twisted.python import log import scrapy from scrapy.crawler import CrawlerProcess @@ -7,11 +8,13 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess(settings={}) -process.crawl(NoRequestsSpider) +d = process.crawl(NoRequestsSpider) +d.addErrback(log.err) process.start() diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index eee808c32..f7802fbc6 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -1,4 +1,4 @@ -from twisted.internet import reactor # noqa: F401 +from twisted.internet import reactor # noqa: F401,TID253 from twisted.python import log import scrapy @@ -8,8 +8,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index 814a2a46d..53941568a 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -1,4 +1,5 @@ from twisted.internet import selectreactor +from twisted.python import log import scrapy from scrapy.crawler import CrawlerProcess @@ -9,11 +10,13 @@ selectreactor.install() class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess(settings={}) -process.crawl(NoRequestsSpider) +d = process.crawl(NoRequestsSpider) +d.addErrback(log.err) process.start() diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index 38ca4c4f1..5739d77ae 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -17,8 +17,9 @@ installReactor(reactor) class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py index b397608ec..c488f7526 100644 --- a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py @@ -9,8 +9,9 @@ selectreactor.install() class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/reactorless.py b/tests/CrawlerProcess/reactorless.py new file mode 100644 index 000000000..c9384010e --- /dev/null +++ b/tests/CrawlerProcess/reactorless.py @@ -0,0 +1,3 @@ +from scrapy.crawler import CrawlerProcess + +CrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py index 2d8769501..d5a7ec53a 100644 --- a/tests/CrawlerProcess/simple.py +++ b/tests/CrawlerProcess/simple.py @@ -1,12 +1,15 @@ import scrapy from scrapy.crawler import CrawlerProcess +from scrapy.utils.reactorless import is_reactorless class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + self.logger.info(f"is_reactorless(): {is_reactorless()}") + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py index 45479ea4f..cb8f869e1 100644 --- a/tests/CrawlerProcess/sleeping.py +++ b/tests/CrawlerProcess/sleeping.py @@ -1,3 +1,5 @@ +import sys + from twisted.internet.defer import Deferred import scrapy @@ -14,7 +16,7 @@ class SleepingSpider(scrapy.Spider): from twisted.internet import reactor d = Deferred() - reactor.callLater(int(self.sleep), d.callback, None) + reactor.callLater(int(sys.argv[1]), d.callback, None) await maybe_deferred_to_future(d) diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py new file mode 100644 index 000000000..c275e0583 --- /dev/null +++ b/tests/CrawlerRunner/change_reactor.py @@ -0,0 +1,32 @@ +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + async def start(self): + return + yield + + +configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"}) + + +from scrapy.utils.reactor import install_reactor # noqa: E402 + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + +runner = CrawlerRunner() + +d = runner.crawl(NoRequestsSpider) + +from twisted.internet import reactor # noqa: E402,TID253 + +d.addBoth(callback=lambda _: reactor.stop()) +reactor.run() diff --git a/tests/CrawlerRunner/custom_loop_different.py b/tests/CrawlerRunner/custom_loop_different.py new file mode 100644 index 000000000..86ba1ed47 --- /dev/null +++ b/tests/CrawlerRunner/custom_loop_different.py @@ -0,0 +1,29 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +def main(reactor): + configure_logging() + runner = CrawlerRunner() + return runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/CrawlerRunner/custom_loop_same.py b/tests/CrawlerRunner/custom_loop_same.py new file mode 100644 index 000000000..98b8dde87 --- /dev/null +++ b/tests/CrawlerRunner/custom_loop_same.py @@ -0,0 +1,29 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +def main(reactor): + configure_logging() + runner = CrawlerRunner() + return runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor", "uvloop.Loop") +react(main) diff --git a/tests/CrawlerRunner/explicit_default_reactor.py b/tests/CrawlerRunner/explicit_default_reactor.py new file mode 100644 index 000000000..9eb8a39bb --- /dev/null +++ b/tests/CrawlerRunner/explicit_default_reactor.py @@ -0,0 +1,28 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": None, + } + + async def start(self): + return + yield + + +def main(reactor): + configure_logging( + {"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"} + ) + runner = CrawlerRunner() + return runner.crawl(NoRequestsSpider) + + +react(main) diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 23260ab0d..fae206610 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -1,23 +1,34 @@ -from urllib.parse import urlparse +# ruff: noqa: E402 -from twisted.internet import reactor -from twisted.names import cache +from __future__ import annotations + +from typing import TYPE_CHECKING + +from scrapy.utils.reactor import install_reactor +from tests.mockserver.dns import MockDNSServer +from tests.mockserver.http import MockServer + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + + +from twisted.names import cache, resolve from twisted.names import hosts as hostsModule -from twisted.names import resolve from twisted.names.client import Resolver from twisted.python.runtime import platform from scrapy import Request, Spider from scrapy.crawler import CrawlerRunner +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import configure_logging -from tests.mockserver import MockDNSServer, MockServer + +if TYPE_CHECKING: + from twisted.names.common import ResolverBase # https://stackoverflow.com/a/32784190 -def createResolver(servers=None, resolvconf=None, hosts=None): - if hosts is None: - hosts = b"/etc/hosts" if platform.getType() == "posix" else r"c:\windows\hosts" - theResolver = Resolver(resolvconf, servers) +def createResolver(servers: list[tuple[str, int]]) -> ResolverBase: + hosts = b"/etc/hosts" if platform.getType() == "posix" else r"c:\windows\hosts" + theResolver = Resolver(None, servers) hostResolver = hostsModule.Resolver(hosts) chain = [hostResolver, cache.CacheResolver(), theResolver] return resolve.ResolverChain(chain) @@ -26,11 +37,11 @@ def createResolver(servers=None, resolvconf=None, hosts=None): class LocalhostSpider(Spider): name = "localhost_spider" - def start_requests(self): + async def start(self): yield Request(self.url) def parse(self, response): - netloc = urlparse(response.url).netloc + netloc = urlparse_cached(response).netloc host = netloc.split(":")[0] self.logger.info(f"Host: {host}") self.logger.info(f"Type: {type(response.ip_address)}") @@ -38,8 +49,10 @@ class LocalhostSpider(Spider): if __name__ == "__main__": + from twisted.internet import reactor + with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: - port = urlparse(mock_http_server.http_address).port + port = mock_http_server.http_port url = f"http://not.a.real.domain:{port}/echo" servers = [(mock_dns_server.host, mock_dns_server.port)] diff --git a/tests/CrawlerRunner/multi_parallel.py b/tests/CrawlerRunner/multi_parallel.py new file mode 100644 index 000000000..51feccd0a --- /dev/null +++ b/tests/CrawlerRunner/multi_parallel.py @@ -0,0 +1,26 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +def main(reactor): + configure_logging() + runner = CrawlerRunner() + runner.crawl(NoRequestsSpider) + runner.crawl(NoRequestsSpider) + return runner.join() + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/CrawlerRunner/multi_seq.py b/tests/CrawlerRunner/multi_seq.py new file mode 100644 index 000000000..f6549be9b --- /dev/null +++ b/tests/CrawlerRunner/multi_seq.py @@ -0,0 +1,27 @@ +from twisted.internet.defer import inlineCallbacks +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@inlineCallbacks +def main(reactor): + configure_logging() + runner = CrawlerRunner() + yield runner.crawl(NoRequestsSpider) + yield runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/CrawlerRunner/no_reactor.py b/tests/CrawlerRunner/no_reactor.py new file mode 100644 index 000000000..1405de00f --- /dev/null +++ b/tests/CrawlerRunner/no_reactor.py @@ -0,0 +1,19 @@ +from twisted.python import log + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +configure_logging() +runner = CrawlerRunner() +d = runner.crawl(NoRequestsSpider) +d.addErrback(log.err) diff --git a/tests/CrawlerRunner/reactorless.py b/tests/CrawlerRunner/reactorless.py new file mode 100644 index 000000000..bad2ae8d9 --- /dev/null +++ b/tests/CrawlerRunner/reactorless.py @@ -0,0 +1,3 @@ +from scrapy.crawler import CrawlerRunner + +CrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) diff --git a/tests/CrawlerRunner/simple.py b/tests/CrawlerRunner/simple.py new file mode 100644 index 000000000..47c0fe04f --- /dev/null +++ b/tests/CrawlerRunner/simple.py @@ -0,0 +1,26 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor +from scrapy.utils.reactorless import is_reactorless + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + self.logger.info(f"is_reactorless(): {is_reactorless()}") + return + yield + + +def main(reactor): + configure_logging() + runner = CrawlerRunner() + return runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/__init__.py b/tests/__init__.py index 5f0c0f7ad..cd52ade58 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -15,13 +15,6 @@ os.environ["http_proxy"] = "" os.environ["https_proxy"] = "" os.environ["ftp_proxy"] = "" -# Absolutize paths to coverage config and output file because tests that -# spawn subprocesses also changes current working directory. -_sourceroot = Path(__file__).resolve().parent.parent -if "COV_CORE_CONFIG" in os.environ: - os.environ["COVERAGE_FILE"] = str(_sourceroot / ".coverage") - os.environ["COV_CORE_CONFIG"] = str(_sourceroot / os.environ["COV_CORE_CONFIG"]) - tests_datadir = str(Path(__file__).parent.resolve() / "sample_data") diff --git a/tests/ftpserver.py b/tests/ftpserver.py deleted file mode 100644 index 0c6ca3621..000000000 --- a/tests/ftpserver.py +++ /dev/null @@ -1,24 +0,0 @@ -from argparse import ArgumentParser - -from pyftpdlib.authorizers import DummyAuthorizer -from pyftpdlib.handlers import FTPHandler -from pyftpdlib.servers import FTPServer - - -def main(): - parser = ArgumentParser() - parser.add_argument("-d", "--directory") - args = parser.parse_args() - - authorizer = DummyAuthorizer() - full_permissions = "elradfmwMT" - authorizer.add_anonymous(args.directory, perm=full_permissions) - handler = FTPHandler - handler.authorizer = authorizer - address = ("127.0.0.1", 2121) - server = FTPServer(address, handler) - server.serve_forever() - - -if __name__ == "__main__": - main() diff --git a/tests/keys/example-com.gen.README b/tests/keys/example-com.gen.README index 955e7b057..026226b2c 100644 --- a/tests/keys/example-com.gen.README +++ b/tests/keys/example-com.gen.README @@ -21,4 +21,3 @@ Locality Name (eg, city) [New York]:The Internet Organization Name (eg, company) [Example, LLC]:Scrapy Common Name (e.g. server FQDN or YOUR name) [Example Company]:www.example.com Email Address [test@example.com]: - diff --git a/tests/keys/localhost-ip.gen.README b/tests/keys/localhost-ip.gen.README index 8e94e1217..6424ad83e 100644 --- a/tests/keys/localhost-ip.gen.README +++ b/tests/keys/localhost-ip.gen.README @@ -18,4 +18,3 @@ Organization Name (eg, company) [Internet Widgits Pty Ltd]:Scrapy Organizational Unit Name (eg, section) []:. Common Name (e.g. server FQDN or YOUR name) []:127.0.0.1 Email Address []:. - diff --git a/tests/keys/localhost.gen.README b/tests/keys/localhost.gen.README index 19c29a725..65f3a2d5c 100644 --- a/tests/keys/localhost.gen.README +++ b/tests/keys/localhost.gen.README @@ -18,4 +18,3 @@ Organization Name (eg, company) [Internet Widgits Pty Ltd]:Scrapy Organizational Unit Name (eg, section) []:. Common Name (e.g. server FQDN or YOUR name) []:localhost Email Address []:. - diff --git a/tests/keys/mitmproxy-ca.pem b/tests/keys/mitmproxy-ca.pem index cdef75f99..61a690cc8 100644 --- a/tests/keys/mitmproxy-ca.pem +++ b/tests/keys/mitmproxy-ca.pem @@ -1,50 +1,47 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCYp6U4G9YWITYB -/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7ZDiT -NUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMgz1BJ -u6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniIggUH -JrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE6HFB -eIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/0zls -90iyQ3E/AgMBAAECggEBAJA1dyAdM85uC04vKVNUJM1GDp0xS+0syBReJaKRI3nJ -epoCj+RqxGag1pdaYLI0G84NTPqECz9LOyLdqpPgEfKRIxWlf9oWmSnfnXskArd8 -VfVcWYl6tEPv1TToTZIBmCbYLBFVbLxG/GrbK6uokdhUsqbdXwEKok2IEaSTRlDn -v8BVXte00d9VEKKpmI6EY3f45uPQPHuJNcitP2HGW1mT/C6XoZR6wj+VvoRgUGQT -I7PuktbYpQlLV+oX0uZz9frPGhjydUq0Jti5v3QAJEb+7D0cKrkZW+7fYDx4YkRU -oDiuWEyO2kfpff52Qxs+xUXMiAyw6/8+TamKoAi1TIECgYEAyAzoztW6W4CjL2au -/hN5VmbAvuBxq1m1G5KgXM1myX9V2CgH6OKwzJQNSCEfKMNOjqxB99T7C3tMCjgG -gmbUzylTeciQFF+crrl2Rn/6qZS9dCo1hagb3K5eXMhLXoP425Y4sypNPPqULhPn -YrUDFNAf89rRLqP1KMPLZ+uO7EECgYEAw1lWPxGV+X85iQxYN9xoX85htfJSBXTf -dLirQ4bkykOxSA6ZzFuhDO/G373Q1rze4tmEO790uOCeaiXGgeWC1A+2PMO957i5 -9FqhDIkmerfdIttdEUMM9rQwuTcLnixGZkT5GHDzjtNinaIVB+pv7twRAESqN9dC -QXh7IF7g/X8CgYBMhQOX+hCqZ24D95cAAJrs/ajEWj2geVPZFCDa3oZulJJVeBpu -bieKWScra9/rS6mE0Ub6cTEFl0fisMNspcDI7NnNP3Y9FMVt3+rp1JIgw5AkGvEW -CtN9egUGIGcT5A8Qj0lo3slkhcSgS2S6UNq431MZh51z5askyJ/JREULAQKBgFrR -OatwfYzUfOcd+hVePpfr1rlDwqYOw6P8BoMKP2tZNR4Oy6maH7Fn98kk8eYjQGuu -PC+avqUEqCEpFrRlAwGbnFl7ltoXozvatmyhhmYe/Iur+ASCa5B2DQDOenQ6mTAK -eNPIDzMjSwGFzMk1UHx3it/ZDFmRlZfibzuJYIf5AoGBAIaPHk4qadK/XpcD4Wwx -BOsDEIz27DGWdwWfd5r3EcV4zX/wNzH0G1Z8eydNjUqKzufMZgFwpcTu0Evesl1/ -B8kC8sLHxQoG5SvBu4dBxMwKIU9O9uFnX5SUYZUDpCtUYyZ+GtGom41Jwg5ENrwy -HzPh2taMnCA0h1fNLFFBkw88 ------END PRIVATE KEY----- +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAuq7ujTpHoSwQn9/hFJT837jU/T7xLuyXjkAEfL7uVDuPWSdF +AJy+QJsuL6INMKMlxLUb1RRzxQgAmtYN1dIEbTPplffCNbfYm3FXg0mZlxg1UBg9 +rE1bPwuz/B+M76S35EIiKQlpaCFErLQi5oyhw9FIBvYLZxxgfeDfDPiXQlBvtHix +n9TFqNoLNZkAX+auh2Wj2SSjM74pBQWsuVZLkF5CAwYuMQkpEplCV/QHNX3ZeNdQ +YNFvpA7CxENa3sTZyHpeTmWoOdJXgJFveAWL4ZhSvkSd0HDuPPJp4JckETNESa9B +qOwFfj36SM+5dRiCwiuzwAQ+oaFjisMXuRbVFQIDAQABAoIBABqFaJmCupNgnboA +xcq1QdmMuiGCNCRs8zj/ykNoopYv4fUR+aEVI4gtI5obxRDwVJjF+/7BCZNnyCI3 +H78NN5jGA7zM9nfINwsaRor9xUasZ0KKNxTH5pslz/uVBeIzvfY9GPpIfoOPGmEI +tF6Zgw+9JyTqBoOvCdxIOpfupxqB8TQ0z4UbFUuBiEkGuJ+o8C1rX8Wze0JUl0qG +BOwhQtaCn/yrm+dTXZ5XaelJY5mcwgFy22Jiynmm6TbLhyZlACd2Q/MGak7o1TJL +QgvvGMlcVrK7MZ3TJN+wzwWfwAAjXnT3Xvd5pD5yunZJoNe8YyFOCMlh0swNG5Zt +0tGeX78CgYEA+m6gYGKTNWFnqlqMZRfGTqiqVZeVQKftcLdA1dkscnffRP9bvKOW +9TbgzoGHiyZnjZBDFTUuy67El8RXIMsxYy3GYuGRdUSLS63Fr2af4pBQIYvW6OmG +UZlcAP6ZAhUzn409XGlXaac3F30hFeKdC62+V5ZMnfPlVhHRCoKaaqsCgYEAvtV4 +FuU5sFKyhKPPV3rzaNZtL0swvtBIuODH1oAWhPNySQvCu+45W0EOOAPPpsYP4wGX +G+otOSp4RLdlVXNhkh1rpJzeK77KZ5ZY+ShkuHD+uL/iRARwl/gh2Ve1aqUrm1LE +9ldchmQGvLalN9HalzeW//xHA3X9SF4Vo16Dvz8CgYABeZlUOABp9hLoO/RLvCIc +4H1wV543bUXGvi2RlN/gJLiZ7W8a41PGSfZ1AOpNdYJyoQDkJRYLeRILWsqwlMHL +tb9PYci7ihXP8kwRxmb2rKbsK6iuYoG6BU83akh4bKuLKwfLfYtYQfXfG4uQV29Z +XEKcvXPiEkethBlZGH/UVQKBgCa9Pvum3OcmYob6mgSwOOl3XgLTyLlzns+pEehB +aFDk+rZJZOaxnYMg2boVS/oXCvKSSBKqnzOTo4aPlEqceZonzspD7fYDbSNKKhWq +VYf7qDno+g3EuPagsH5mh7V2gjutub4oTegaNiPpD/Ec8Lrx1f1xQRk6wogGUW4w +qZ4RAoGBAPg1LezV8mlesF5mhj+KubYP4l1Zf9geAeQprjDbFsA0BEAS2KsWgmwR +Ye1fmek7jDjCPLQ4Amq030mLJuQGEM3cZPqjKX2sBZ8fQcgw7pWJWMvKMTBA4Aah +zQx1KXwHJANMWq/0QSFDq/LGJ2OYMlV2F0tH3P5Kp7ZASTyc78ux +-----END RSA PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIDoTCCAomgAwIBAgIGDodLQx9+MA0GCSqGSIb3DQEBCwUAMCgxEjAQBgNVBAMM -CW1pdG1wcm94eTESMBAGA1UECgwJbWl0bXByb3h5MB4XDTIwMDgxMjE3MDMyNloX -DTIzMDgxNDE3MDMyNlowKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAlt -aXRtcHJveHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCYp6U4G9YW -ITYB/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7 -ZDiTNUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMg -z1BJu6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniI -ggUHJrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE -6HFBeIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/ -0zls90iyQ3E/AgMBAAGjgdAwgc0wDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhC -AQEEBAMCAgQweAYDVR0lBHEwbwYIKwYBBQUHAwEGCCsGAQUFBwMCBggrBgEFBQcD -BAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQBgjcCARYGCisGAQQBgjcKAwEG -CisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG+EIEATAOBgNVHQ8BAf8EBAMC -AQYwHQYDVR0OBBYEFBCsLPpFz3l9rOOfGmfs+VRc3jhJMA0GCSqGSIb3DQEBCwUA -A4IBAQADTpA15na6U5qqDCe0rr39fkS1/dY804Xnz7g/L3AsxPE1KOMijuJa8sKd -kKwba1173FwMupfK39zY8jUxL8Qprdi92RO6CpoFUsL/icpA///lYhzUSqt32qwe -gRNW3mtYBimOk6KH1NOfQnJolWpJh+g1OEsitQKEeKwIn5Hz+8/yS5tbwLgdnMlY -1/it1H70JSdE7nfJueqN4cFfBsm6XaHZzacJJmN7WP88fd+zztnSQsBFbLlnjnqj -envCDIwCrMywKNMqEBMwmBEGSAF47fVNYj6KzDAtMvBdDkYaHWpBf4tnFfk6v0wj -wiKjdLjCmJgjGAQjRw5VYJ8JI0XO +MIIDNTCCAh2gAwIBAgIUcGDiCmOuhfxMGFS/otcGGFkOSAEwDQYJKoZIhvcNAQEL +BQAwKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAltaXRtcHJveHkwHhcN +MjQwMjI3MTMwNTQ4WhcNMzQwMjI2MTMwNTQ4WjAoMRIwEAYDVQQDDAltaXRtcHJv +eHkxEjAQBgNVBAoMCW1pdG1wcm94eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBALqu7o06R6EsEJ/f4RSU/N+41P0+8S7sl45ABHy+7lQ7j1knRQCcvkCb +Li+iDTCjJcS1G9UUc8UIAJrWDdXSBG0z6ZX3wjW32JtxV4NJmZcYNVAYPaxNWz8L +s/wfjO+kt+RCIikJaWghRKy0IuaMocPRSAb2C2ccYH3g3wz4l0JQb7R4sZ/Uxaja +CzWZAF/mrodlo9kkozO+KQUFrLlWS5BeQgMGLjEJKRKZQlf0BzV92XjXUGDRb6QO +wsRDWt7E2ch6Xk5lqDnSV4CRb3gFi+GYUr5EndBw7jzyaeCXJBEzREmvQajsBX49 ++kjPuXUYgsIrs8AEPqGhY4rDF7kW1RUCAwEAAaNXMFUwDwYDVR0TAQH/BAUwAwEB +/zATBgNVHSUEDDAKBggrBgEFBQcDATAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FOjFT0G7itqsrCij2InhRSfB0sEkMA0GCSqGSIb3DQEBCwUAA4IBAQCVMa5/xlH4 +GUbrWNMdxr9LL7Dh+vK0wYCfAsc/kO2zCq8iVt/MaqVLel/bKcQhvE5RZHvyep13 +x7378OfCqqHkDDDNroWIvij84ZtMUaM53tF13G/ZGOlNsoLNynWs9IVVvqGKsH7o +/buJ1RNArI/0irF0UD7qrMmo1p6SYanZhqdh2PphNy9NS3FsfrfnuWvf+/TRp9Ts +L8058B0p/LIL0OB5trYFircC3iKSOuRl0ERD2ufgSqsSVEYm1mc6UIxv+d1iFD+Q +8CRUF88icQXrec1TCbhh0CfdDxz+FYSTnW0DR0L75coa/CBmRxAjnrkLoXRr3Y1d +sTjU4zDdBcBw -----END CERTIFICATE----- diff --git a/tests/keys/mitmproxy-dhparam.pem b/tests/keys/mitmproxy-dhparam.pem new file mode 100644 index 000000000..c10121fbf --- /dev/null +++ b/tests/keys/mitmproxy-dhparam.pem @@ -0,0 +1,14 @@ + +-----BEGIN DH PARAMETERS----- +MIICCAKCAgEAyT6LzpwVFS3gryIo29J5icvgxCnCebcdSe/NHMkD8dKJf8suFCg3 +O2+dguLakSVif/t6dhImxInJk230HmfC8q93hdcg/j8rLGJYDKu3ik6H//BAHKIv +j5O9yjU3rXCfmVJQic2Nne39sg3CreAepEts2TvYHhVv3TEAzEqCtOuTjgDv0ntJ +Gwpj+BJBRQGG9NvprX1YGJ7WOFBP/hWU7d6tgvE6Xa7T/u9QIKpYHMIkcN/l3ZFB +chZEqVlyrcngtSXCROTPcDOQ6Q8QzhaBJS+Z6rcsd7X+haiQqvoFcmaJ08Ks6LQC +ZIL2EtYJw8V8z7C0igVEBIADZBI6OTbuuhDwRw//zU1uq52Oc48CIZlGxTYG/Evq +o9EWAXUYVzWkDSTeBH1r4z/qLPE2cnhtMxbFxuvK53jGB0emy2y1Ei6IhKshJ5qX +IB/aE7SSHyQ3MDHHkCmQJCsOd4Mo26YX61NZ+n501XjqpCBQ2+DfZCBh8Va2wDyv +A2Ryg9SUz8j0AXViRNMJgJrr446yro/FuJZwnQcO3WQnXeqSBnURqKjmqkeFP+d8 +6mk2tqJaY507lRNqtGlLnj7f5RNoBFJDCLBNurVgfvq9TCVWKDIFD4vZRjCrnl6I +rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI= +-----END DH PARAMETERS----- diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index 2869ff8f7..98fe58860 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -1,9 +1,10 @@ """DBM-like dummy module""" -import collections -from typing import Any, DefaultDict + +from collections import defaultdict +from typing import Any -class DummyDB(dict): +class DummyDB(dict): # type: ignore[type-arg] """Provide dummy DBM-like interface.""" def close(self): @@ -13,10 +14,10 @@ class DummyDB(dict): error = KeyError -_DATABASES: DefaultDict[Any, DummyDB] = collections.defaultdict(DummyDB) +_DATABASES: defaultdict[Any, DummyDB] = defaultdict(DummyDB) -def open(file, flag="r", mode=0o666): +def open(file, flag="r", mode=0o666): # noqa: A001 """Open or create a dummy database compatible. Arguments ``flag`` and ``mode`` are ignored. diff --git a/tests/mockserver.py b/tests/mockserver.py deleted file mode 100644 index 647b0682e..000000000 --- a/tests/mockserver.py +++ /dev/null @@ -1,411 +0,0 @@ -import argparse -import json -import os -import random -import sys -from pathlib import Path -from shutil import rmtree -from subprocess import PIPE, Popen -from tempfile import mkdtemp -from typing import Dict -from urllib.parse import urlencode - -from OpenSSL import SSL -from twisted.internet import defer, reactor, ssl -from twisted.internet.protocol import ServerFactory -from twisted.internet.task import deferLater -from twisted.names import dns, error -from twisted.names.server import DNSServerFactory -from twisted.web import resource, server -from twisted.web.server import NOT_DONE_YET, GzipEncoderFactory, Site -from twisted.web.static import File -from twisted.web.util import redirectTo - -from scrapy.utils.python import to_bytes, to_unicode - - -def getarg(request, name, default=None, type=None): - if name in request.args: - value = request.args[name][0] - if type is not None: - value = type(value) - return value - return default - - -def get_mockserver_env() -> Dict[str, str]: - """Return a OS environment dict suitable to run mockserver processes.""" - - tests_path = Path(__file__).parent.parent - pythonpath = str(tests_path) + os.pathsep + os.environ.get("PYTHONPATH", "") - env = os.environ.copy() - env["PYTHONPATH"] = pythonpath - return env - - -# most of the following resources are copied from twisted.web.test.test_webclient -class ForeverTakingResource(resource.Resource): - """ - L{ForeverTakingResource} is a resource which never finishes responding - to requests. - """ - - def __init__(self, write=False): - resource.Resource.__init__(self) - self._write = write - - def render(self, request): - if self._write: - request.write(b"some bytes") - return server.NOT_DONE_YET - - -class ErrorResource(resource.Resource): - def render(self, request): - request.setResponseCode(401) - if request.args.get(b"showlength"): - request.setHeader(b"content-length", b"0") - return b"" - - -class NoLengthResource(resource.Resource): - def render(self, request): - return b"nolength" - - -class HostHeaderResource(resource.Resource): - """ - A testing resource which renders itself as the value of the host header - from the request. - """ - - def render(self, request): - return request.requestHeaders.getRawHeaders(b"host")[0] - - -class PayloadResource(resource.Resource): - """ - A testing resource which renders itself as the contents of the request body - as long as the request body is 100 bytes long, otherwise which renders - itself as C{"ERROR"}. - """ - - def render(self, request): - data = request.content.read() - contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0] - if len(data) != 100 or int(contentLength) != 100: - return b"ERROR" - return data - - -class BrokenDownloadResource(resource.Resource): - def render(self, request): - # only sends 3 bytes even though it claims to send 5 - request.setHeader(b"content-length", b"5") - request.write(b"abc") - return b"" - - -class LeafResource(resource.Resource): - isLeaf = True - - def deferRequest(self, request, delay, f, *a, **kw): - def _cancelrequest(_): - # silence CancelledError - d.addErrback(lambda _: None) - d.cancel() - - d = deferLater(reactor, delay, f, *a, **kw) - request.notifyFinish().addErrback(_cancelrequest) - return d - - -class Follow(LeafResource): - def render(self, request): - total = getarg(request, b"total", 100, type=int) - show = getarg(request, b"show", 1, type=int) - order = getarg(request, b"order", b"desc") - maxlatency = getarg(request, b"maxlatency", 0, type=float) - n = getarg(request, b"n", total, type=int) - if order == b"rand": - nlist = [random.randint(1, total) for _ in range(show)] - else: # order == "desc" - nlist = range(n, max(n - show, 0), -1) - - lag = random.random() * maxlatency - self.deferRequest(request, lag, self.renderRequest, request, nlist) - return NOT_DONE_YET - - def renderRequest(self, request, nlist): - s = """ """ - args = request.args.copy() - for nl in nlist: - args[b"n"] = [to_bytes(str(nl))] - argstr = urlencode(args, doseq=True) - s += f"follow {nl}
" - s += """""" - request.write(to_bytes(s)) - request.finish() - - -class Delay(LeafResource): - def render_GET(self, request): - n = getarg(request, b"n", 1, type=float) - b = getarg(request, b"b", 1, type=int) - if b: - # send headers now and delay body - request.write("") - self.deferRequest(request, n, self._delayedRender, request, n) - return NOT_DONE_YET - - def _delayedRender(self, request, n): - request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n")) - request.finish() - - -class Status(LeafResource): - def render_GET(self, request): - n = getarg(request, b"n", 200, type=int) - request.setResponseCode(n) - return b"" - - -class Raw(LeafResource): - def render_GET(self, request): - request.startedWriting = 1 - self.deferRequest(request, 0, self._delayedRender, request) - return NOT_DONE_YET - - render_POST = render_GET - - def _delayedRender(self, request): - raw = getarg(request, b"raw", b"HTTP 1.1 200 OK\n") - request.startedWriting = 1 - request.write(raw) - request.channel.transport.loseConnection() - request.finish() - - -class Echo(LeafResource): - def render_GET(self, request): - output = { - "headers": dict( - (to_unicode(k), [to_unicode(v) for v in vs]) - for k, vs in request.requestHeaders.getAllRawHeaders() - ), - "body": to_unicode(request.content.read()), - } - return to_bytes(json.dumps(output)) - - render_POST = render_GET - - -class RedirectTo(LeafResource): - def render(self, request): - goto = getarg(request, b"goto", b"/") - # we force the body content, otherwise Twisted redirectTo() - # returns HTML with None: + from twisted.internet import reactor + + clients = [MockDNSResolver()] + factory = DNSServerFactory(clients=clients) + protocol = dns.DNSDatagramProtocol(controller=factory) + listener = reactor.listenUDP(0, protocol) + + def print_listening(): + host = listener.getHost() + print(f"{host.host}:{host.port}") + + reactor.callWhenRunning(print_listening) + reactor.run() + + +if __name__ == "__main__": + main() diff --git a/tests/mockserver/ftp.py b/tests/mockserver/ftp.py new file mode 100644 index 000000000..22efc966b --- /dev/null +++ b/tests/mockserver/ftp.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import re +import sys +from argparse import ArgumentParser +from pathlib import Path +from shutil import rmtree +from subprocess import PIPE, Popen +from tempfile import mkdtemp + +from pyftpdlib.authorizers import DummyAuthorizer +from pyftpdlib.handlers import FTPHandler +from pyftpdlib.servers import FTPServer + +from tests.utils import get_script_run_env + + +class MockFTPServer: + """Creates an FTP server on a random port with a default passwordless user + (anonymous) and a temporary root path that you can read from the + :attr:`path` attribute.""" + + def __init__(self) -> None: + self.proc: Popen[str] | None = None + self.host: str = "127.0.0.1" + self.port: int | None = None + self.path: Path | None = None + + def __enter__(self): + self.path = Path(mkdtemp()) + self.proc = Popen( + [sys.executable, "-u", "-m", "tests.mockserver.ftp", "-d", str(self.path)], + stderr=PIPE, + env=get_script_run_env(), + text=True, + ) + for line in self.proc.stderr: + if "starting FTP server" in line and ( + m := re.search(r"starting FTP server on ([^ :]+):(\d+),", line) + ): + self.port = int(m.group(2)) + break + else: + self.proc.kill() + self.proc.communicate() + raise RuntimeError( + "The FTP server failed to start or the output is unrecognized" + ) + return self + + def __exit__(self, exc_type, exc_value, traceback): + rmtree(str(self.path)) + self.proc.kill() + self.proc.communicate() + + def url(self, path): + return f"ftp://{self.host}:{self.port}/{path}" + + +def main() -> None: + parser = ArgumentParser() + parser.add_argument("-d", "--directory", required=True) + args = parser.parse_args() + + authorizer = DummyAuthorizer() + full_permissions = "elradfmwMT" + authorizer.add_anonymous(args.directory, perm=full_permissions) + handler = FTPHandler + handler.authorizer = authorizer + address = ("127.0.0.1", 0) + server = FTPServer(address, handler) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py new file mode 100644 index 000000000..7ad873c02 --- /dev/null +++ b/tests/mockserver/http.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from pathlib import Path + +from twisted.web import resource +from twisted.web.static import Data, File +from twisted.web.util import Redirect + +from tests import tests_datadir + +from .http_base import BaseMockServer, main_factory +from .http_resources import ( + ArbitraryLengthPayloadResource, + BrokenChunkedResource, + BrokenDownloadResource, + ChunkedResource, + ClientIPResource, + Compress, + ContentLengthHeaderResource, + Delay, + Drop, + DuplicateHeaderResource, + Echo, + EmptyContentTypeHeaderResource, + Follow, + ForeverTakingResource, + HostHeaderResource, + LargeChunkedFileResource, + NoMetaRefreshRedirect, + Partial, + PayloadResource, + Raw, + RedirectTo, + ResponseHeadersResource, + SetCookie, + Status, + UriResource, +) + + +class Root(resource.Resource): + def __init__(self): + super().__init__() + self.putChild(b"status", Status()) + self.putChild(b"follow", Follow()) + self.putChild(b"delay", Delay()) + self.putChild(b"partial", Partial()) + self.putChild(b"drop", Drop()) + self.putChild(b"raw", Raw()) + self.putChild(b"echo", Echo()) + self.putChild(b"payload", PayloadResource()) + self.putChild(b"alpayload", ArbitraryLengthPayloadResource()) + self.putChild(b"static", File(str(Path(tests_datadir, "test_site/")))) + self.putChild(b"redirect-to", RedirectTo()) + self.putChild(b"text", Data(b"Works", "text/plain")) + self.putChild( + b"html", + Data( + b"

Works

World

", + "text/html", + ), + ) + self.putChild( + b"enc-gb18030", + Data(b"

gb18030 encoding

", "text/html; charset=gb18030"), + ) + self.putChild(b"redirect", Redirect(b"/redirected")) + self.putChild( + b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected") + ) + self.putChild(b"redirected", Data(b"Redirected here", "text/plain")) + numbers = [str(x).encode("utf8") for x in range(2**18)] + self.putChild(b"numbers", Data(b"".join(numbers), "text/plain")) + self.putChild(b"wait", ForeverTakingResource()) + self.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) + self.putChild(b"host", HostHeaderResource()) + self.putChild(b"client-ip", ClientIPResource()) + self.putChild(b"broken", BrokenDownloadResource()) + self.putChild(b"chunked", ChunkedResource()) + self.putChild(b"broken-chunked", BrokenChunkedResource()) + self.putChild(b"contentlength", ContentLengthHeaderResource()) + self.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) + self.putChild(b"largechunkedfile", LargeChunkedFileResource()) + self.putChild(b"compress", Compress()) + self.putChild(b"duplicate-header", DuplicateHeaderResource()) + self.putChild(b"response-headers", ResponseHeadersResource()) + self.putChild(b"set-cookie", SetCookie()) + self.putChild(b"uri", UriResource()) + + def getChild(self, path, request): + return self + + def render(self, request): + return b"Scrapy mock HTTP server\n" + + +class MockServer(BaseMockServer): + module_name = "tests.mockserver.http" + + +main = main_factory(Root) + + +if __name__ == "__main__": + main() diff --git a/tests/mockserver/http_base.py b/tests/mockserver/http_base.py new file mode 100644 index 000000000..7b38409ff --- /dev/null +++ b/tests/mockserver/http_base.py @@ -0,0 +1,147 @@ +"""Base classes and functions for HTTP mockservers.""" + +from __future__ import annotations + +import argparse +import sys +from abc import ABC, abstractmethod +from subprocess import PIPE, Popen +from typing import TYPE_CHECKING +from urllib.parse import urlparse + +from twisted.web.server import Site + +from tests.utils import get_script_run_env + +from .utils import ssl_context_factory + +if TYPE_CHECKING: + from collections.abc import Callable + + from twisted.web import resource + + +class BaseMockServer(ABC): + listen_http: bool = True + listen_https: bool = True + + @property + @abstractmethod + def module_name(self) -> str: + raise NotImplementedError + + def __init__(self) -> None: + if not self.listen_http and not self.listen_https: + raise ValueError("At least one of listen_http and listen_https must be set") + + self.proc: Popen[str] | None = None + self.host: str = "127.0.0.1" + self.http_port: int | None = None + self.https_port: int | None = None + + def __enter__(self): + self.proc = Popen( + [sys.executable, "-u", "-m", self.module_name, *self.get_additional_args()], + stdout=PIPE, + env=get_script_run_env(), + text=True, + ) + if self.listen_http: + http_address = self.proc.stdout.readline().strip() + http_parsed = urlparse(http_address) + self.http_port = http_parsed.port + if self.listen_https: + https_address = self.proc.stdout.readline().strip() + https_parsed = urlparse(https_address) + self.https_port = https_parsed.port + return self + + def __exit__(self, exc_type, exc_value, traceback): + if self.proc: + self.proc.kill() + self.proc.communicate() + + def get_additional_args(self) -> list[str]: + return [] + + def port(self, is_secure: bool = False) -> int: + if not is_secure and not self.listen_http: + raise ValueError("This server doesn't provide HTTP") + if is_secure and not self.listen_https: + raise ValueError("This server doesn't provide HTTPS") + port = self.https_port if is_secure else self.http_port + assert port is not None + return port + + def url(self, path: str, is_secure: bool = False) -> str: + port = self.port(is_secure) + scheme = "https" if is_secure else "http" + return f"{scheme}://{self.host}:{port}{path}" + + +def main_factory( + resource_class: type[resource.Resource], + *, + listen_http: bool = True, + listen_https: bool = True, +) -> Callable[[], None]: + if not listen_http and not listen_https: + raise ValueError("At least one of listen_http and listen_https must be set") + + def main() -> None: + from twisted.internet import reactor + + root = resource_class() + factory = Site(root) + + if listen_http: + http_port = reactor.listenTCP(0, factory) + + if listen_https: + parser = argparse.ArgumentParser() + parser.add_argument("--keyfile", help="SSL key file") + parser.add_argument("--certfile", help="SSL certificate file") + parser.add_argument( + "--cipher-string", + default=None, + help="SSL cipher string (optional)", + ) + parser.add_argument( + "--tls-min-version", + default=None, + help="Minimum accepted TLS version (optional)", + ) + parser.add_argument( + "--tls-max-version", + default=None, + help="Maximum accepted TLS version (optional)", + ) + args = parser.parse_args() + context_factory_kw = {} + if args.keyfile: + context_factory_kw["keyfile"] = args.keyfile + if args.certfile: + context_factory_kw["certfile"] = args.certfile + if args.cipher_string: + context_factory_kw["cipher_string"] = args.cipher_string + if args.tls_min_version: + context_factory_kw["tls_min_version"] = args.tls_min_version + if args.tls_max_version: + context_factory_kw["tls_max_version"] = args.tls_max_version + context_factory = ssl_context_factory(**context_factory_kw) + https_port = reactor.listenSSL(0, factory, context_factory) + + def print_listening(): + if listen_http: + http_host = http_port.getHost() + http_address = f"http://{http_host.host}:{http_host.port}" + print(http_address) + if listen_https: + https_host = https_port.getHost() + https_address = f"https://{https_host.host}:{https_host.port}" + print(https_address) + + reactor.callWhenRunning(print_listening) + reactor.run() + + return main diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.py new file mode 100644 index 000000000..98ac6cf6a --- /dev/null +++ b/tests/mockserver/http_resources.py @@ -0,0 +1,381 @@ +from __future__ import annotations + +import gzip +import json +import random +from typing import TYPE_CHECKING, ParamSpec, TypeVar +from urllib.parse import urlencode + +from twisted.internet.task import deferLater +from twisted.web import resource, server +from twisted.web.server import NOT_DONE_YET +from twisted.web.util import Redirect, redirectTo + +from scrapy.utils.python import to_bytes, to_unicode + +if TYPE_CHECKING: + from collections.abc import Callable + + from twisted.internet.defer import Deferred + from twisted.web.http import Request + + +_T = TypeVar("_T") +_P = ParamSpec("_P") + + +def getarg(request, name, default=None, type_=None): + if name in request.args: + value = request.args[name][0] + if type_ is not None: + value = type_(value) + return value + return default + + +def close_connection(request): + # We have to force a disconnection for HTTP/1.1 clients. Otherwise + # client keeps the connection open waiting for more data. + request.channel.loseConnection() + request.finish() + + +# most of the following resources are copied from twisted.web.test.test_webclient +class ForeverTakingResource(resource.Resource): + """ + L{ForeverTakingResource} is a resource which never finishes responding + to requests. + """ + + def __init__(self, write=False): + resource.Resource.__init__(self) + self._write = write + + def render(self, request): + if self._write: + request.write(b"some bytes") + return server.NOT_DONE_YET + + +class HostHeaderResource(resource.Resource): + """ + A testing resource which renders itself as the value of the host header + from the request. + """ + + def render(self, request): + return request.requestHeaders.getRawHeaders(b"host")[0] + + +class ClientIPResource(resource.Resource): + """ + A testing resource which renders itself as the request client IP address. + """ + + def render(self, request): + client_address = request.getClientAddress() + if client_address is None or client_address.host is None: + return b"" + return to_bytes(client_address.host) + + +class PayloadResource(resource.Resource): + """ + A testing resource which renders itself as the contents of the request body + as long as the request body is 100 bytes long, otherwise which renders + itself as C{"ERROR"}. + """ + + def render(self, request): + data = request.content.read() + contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0] + if len(data) != 100 or int(contentLength) != 100: + return b"ERROR" + return data + + +class LeafResource(resource.Resource): + isLeaf = True + + def deferRequest( + self, + request: Request, + delay: float, + f: Callable[_P, _T], + *a: _P.args, + **kw: _P.kwargs, + ) -> Deferred[_T]: + from twisted.internet import reactor + + def _cancelrequest(_): + # silence CancelledError + d.addErrback(lambda _: None) + d.cancel() + + d = deferLater(reactor, delay, f, *a, **kw) + request.notifyFinish().addErrback(_cancelrequest) + return d + + +class Follow(LeafResource): + def render(self, request): + total = getarg(request, b"total", 100, type_=int) + show = getarg(request, b"show", 1, type_=int) + order = getarg(request, b"order", b"desc") + maxlatency = getarg(request, b"maxlatency", 0, type_=float) + n = getarg(request, b"n", total, type_=int) + if order == b"rand": + nlist = [random.randint(1, total) for _ in range(show)] + else: # order == "desc" + nlist = range(n, max(n - show, 0), -1) + + lag = random.random() * maxlatency + self.deferRequest(request, lag, self.renderRequest, request, nlist) + return NOT_DONE_YET + + def renderRequest(self, request, nlist): + s = """ """ + args = request.args.copy() + for nl in nlist: + args[b"n"] = [to_bytes(str(nl))] + argstr = urlencode(args, doseq=True) + s += f"follow {nl}
" + s += """""" + request.write(to_bytes(s)) + request.finish() + + +class Delay(LeafResource): + def render_GET(self, request): + n = getarg(request, b"n", 1, type_=float) + b = getarg(request, b"b", 1, type_=int) + if b: + # send headers now and delay body + request.write("") + self.deferRequest(request, n, self._delayedRender, request, n) + return NOT_DONE_YET + + def _delayedRender(self, request, n): + request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n")) + request.finish() + + +class Status(LeafResource): + def render_GET(self, request): + n = getarg(request, b"n", 200, type_=int) + request.setResponseCode(n) + return b"" + + +class Raw(LeafResource): + def render_GET(self, request): + request.startedWriting = 1 + self.deferRequest(request, 0, self._delayedRender, request) + return NOT_DONE_YET + + render_POST = render_GET + + def _delayedRender(self, request): + raw = getarg(request, b"raw", b"HTTP 1.1 200 OK\n") + request.startedWriting = 1 + request.write(raw) + request.channel.transport.loseConnection() + request.finish() + + +class Echo(LeafResource): + def render_GET(self, request): + output = { + "headers": { + to_unicode(k): [to_unicode(v) for v in vs] + for k, vs in request.requestHeaders.getAllRawHeaders() + }, + "body": to_unicode(request.content.read()), + } + return to_bytes(json.dumps(output)) + + render_POST = render_GET + + +class RedirectTo(LeafResource): + def render(self, request): + goto = getarg(request, b"goto", b"/") + # we force the body content, otherwise Twisted redirectTo() + # returns HTML with bytes: + content: bytes = Redirect.render(self, request) + return content.replace( + b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"' + ) + + +class ContentLengthHeaderResource(resource.Resource): + """ + A testing resource which renders itself as the value of the Content-Length + header from the request. + """ + + def render(self, request): + return request.requestHeaders.getRawHeaders(b"content-length")[0] + + +class ChunkedResource(resource.Resource): + def render(self, request): + from twisted.internet import reactor + + def response(): + request.write(b"chunked ") + request.write(b"content\n") + request.finish() + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class BrokenChunkedResource(resource.Resource): + def render(self, request): + from twisted.internet import reactor + + def response(): + request.write(b"chunked ") + request.write(b"content\n") + # Disable terminating chunk on finish. + request.chunked = False + close_connection(request) + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class BrokenDownloadResource(resource.Resource): + def render(self, request): + from twisted.internet import reactor + + def response(): + request.setHeader(b"Content-Length", b"20") + request.write(b"partial") + close_connection(request) + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class EmptyContentTypeHeaderResource(resource.Resource): + """ + A testing resource which renders itself as the value of request body + without content-type header in response. + """ + + def render(self, request): + request.setHeader("content-type", "") + return request.content.read() + + +class LargeChunkedFileResource(resource.Resource): + def render(self, request): + from twisted.internet import reactor + + def response(): + for _ in range(1024): + request.write(b"x" * 1024) + request.finish() + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class DuplicateHeaderResource(resource.Resource): + def render(self, request): + request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"]) + return b"" + + +class UriResource(resource.Resource): + """Return the full uri that was requested""" + + def getChild(self, path, request): + return self + + def render(self, request): + # Note: this is an ugly hack for CONNECT request timeout test. + # Returning some data here fail SSL/TLS handshake + # ToDo: implement proper HTTPS proxy tests, not faking them. + if request.method != b"CONNECT": + return request.uri + request.transport.write(b"HTTP/1.1 200 Connection established\r\n\r\n") + return NOT_DONE_YET + + +class ResponseHeadersResource(resource.Resource): + """Return a response with headers set from the JSON request body""" + + def render(self, request): + body = json.loads(request.content.read().decode()) + for header_name, header_value in body.items(): + request.responseHeaders.setRawHeaders(header_name, [header_value]) + return json.dumps(body).encode("utf-8") + + +class Compress(resource.Resource): + """Compress the data sent in the request url params and set Content-Encoding header""" + + def render(self, request): + data = request.args.get(b"data")[0] + + accept_encoding_header = request.getHeader(b"accept-encoding") + + # include common encoding schemes here + if accept_encoding_header == b"gzip": + request.setHeader(b"Content-Encoding", b"gzip") + return gzip.compress(data) + + # just set this to trigger a test failure if no valid accept-encoding header was set + request.setResponseCode(500) + return b"Did not receive a valid accept-encoding header" + + +class SetCookie(resource.Resource): + """Return a response with a Set-Cookie header for each request url parameter""" + + def render(self, request): + for cookie_name, cookie_values in request.args.items(): + for cookie_value in cookie_values: + cookie = (cookie_name.decode() + "=" + cookie_value.decode()).encode() + request.setHeader(b"Set-Cookie", cookie) + return b"" diff --git a/tests/mockserver/mitm_proxy.py b/tests/mockserver/mitm_proxy.py new file mode 100644 index 000000000..56620f84e --- /dev/null +++ b/tests/mockserver/mitm_proxy.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import re +import sys +from pathlib import Path +from subprocess import PIPE, Popen +from urllib.parse import urlsplit, urlunsplit + + +class MitmProxy: + auth_user = "scrapy" + auth_pass = "scrapy" + + def __init__(self, mode: str | None = None) -> None: + self.mode = mode + + def start(self) -> str: + script = """ +import sys +from mitmproxy.tools.main import mitmdump +sys.argv[0] = "mitmdump" +sys.exit(mitmdump()) + """ + cert_path = Path(__file__).parent.parent.resolve() / "keys" + args = [ + "--listen-host", + "127.0.0.1", + "--listen-port", + "0", + "--proxyauth", + f"{self.auth_user}:{self.auth_pass}", + "--set", + f"confdir={cert_path}", + "--ssl-insecure", + "-s", + str(Path(__file__).with_name("mitm_proxy_addon.py")), + ] + if self.mode: + args += ["--mode", self.mode] + self.proc: Popen[str] = Popen( + [ + sys.executable, + "-u", + "-c", + script, + *args, + ], + stdout=PIPE, + text=True, + ) + assert self.proc.stdout is not None + scheme = "socks5" if self.mode == "socks5" else "http" + line = "" + for line in self.proc.stdout: + m = re.search(r"listening at (?:\w+://)?([^:]+:\d+)", line) + if m: + host_port = m.group(1) + return f"{scheme}://{self.auth_user}:{self.auth_pass}@{host_port}" + self.stop() + raise RuntimeError(f"Failed to parse mitmdump output: {line}") + + def stop(self) -> None: + self.proc.kill() + self.proc.communicate() + + +def wrong_credentials(proxy_url: str) -> str: + bad_auth_proxy = list(urlsplit(proxy_url)) + bad_auth_proxy[1] = bad_auth_proxy[1].replace("scrapy:scrapy@", "wrong:wronger@") + return urlunsplit(bad_auth_proxy) diff --git a/tests/mockserver/mitm_proxy_addon.py b/tests/mockserver/mitm_proxy_addon.py new file mode 100644 index 000000000..04ba2e4a1 --- /dev/null +++ b/tests/mockserver/mitm_proxy_addon.py @@ -0,0 +1,5 @@ +def response(flow) -> None: + # add custom headers to be able to check that the request went through the proxy + flow.response.headers["X-Via-Mitmproxy"] = "1" + if flow.client_conn.tls_established: + flow.response.headers["X-Via-Mitmproxy-TLS"] = "1" diff --git a/tests/mockserver/proxy_echo.py b/tests/mockserver/proxy_echo.py new file mode 100644 index 000000000..6345726b6 --- /dev/null +++ b/tests/mockserver/proxy_echo.py @@ -0,0 +1,17 @@ +# This is only used by tests.test_downloader_handlers_http_base.TestHttpProxyBase + +from __future__ import annotations + +from .http_base import BaseMockServer, main_factory +from .http_resources import UriResource + + +class ProxyEchoMockServer(BaseMockServer): + module_name = "tests.mockserver.proxy_echo" + + +main = main_factory(UriResource) + + +if __name__ == "__main__": + main() diff --git a/tests/mockserver/simple_https.py b/tests/mockserver/simple_https.py new file mode 100644 index 000000000..943775fa5 --- /dev/null +++ b/tests/mockserver/simple_https.py @@ -0,0 +1,60 @@ +# This is only used by tests.test_downloader_handlers_http_base.TestSimpleHttpsBase + +from __future__ import annotations + +from twisted.web import resource +from twisted.web.static import Data + +from .http_base import BaseMockServer, main_factory + + +class Root(resource.Resource): + def __init__(self): + resource.Resource.__init__(self) + self.putChild(b"file", Data(b"0123456789", "text/plain")) + + def getChild(self, path, request): + return self + + +class SimpleMockServer(BaseMockServer): + listen_http = False + module_name = "tests.mockserver.simple_https" + + def __init__( + self, + keyfile: str, + certfile: str, + *, + cipher_string: str | None = None, + tls_min_version: str | None = None, + tls_max_version: str | None = None, + ): + super().__init__() + self.keyfile = keyfile + self.certfile = certfile + self.cipher_string = cipher_string or "" + self.tls_min_version = tls_min_version + self.tls_max_version = tls_max_version + + def get_additional_args(self) -> list[str]: + args = [ + "--keyfile", + self.keyfile, + "--certfile", + self.certfile, + ] + if self.cipher_string is not None: + args.extend(["--cipher-string", self.cipher_string]) + if self.tls_min_version is not None: + args.extend(["--tls-min-version", self.tls_min_version]) + if self.tls_max_version is not None: + args.extend(["--tls-max-version", self.tls_max_version]) + return args + + +main = main_factory(Root, listen_http=False) + + +if __name__ == "__main__": + main() diff --git a/tests/mockserver/utils.py b/tests/mockserver/utils.py new file mode 100644 index 000000000..7aa656780 --- /dev/null +++ b/tests/mockserver/utils.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, cast + +from cryptography.hazmat.primitives.serialization import load_pem_private_key +from cryptography.x509 import load_pem_x509_certificate +from OpenSSL import SSL +from OpenSSL.crypto import FILETYPE_PEM, load_certificate, load_privatekey +from twisted.internet.ssl import CertificateOptions, ContextFactory + +from scrapy.core.downloader.tls import _TWISTED_VERSION_MAP +from scrapy.utils._deps_compat import PYOPENSSL_WANTS_X509_PKEY +from scrapy.utils.python import to_bytes +from scrapy.utils.ssl import _get_cert_options_version_kwargs + +if TYPE_CHECKING: + from twisted.internet.interfaces import IOpenSSLContextFactory + + +def ssl_context_factory( + keyfile: str = "keys/localhost.key", + certfile: str = "keys/localhost.crt", + *, + cipher_string: str | None = None, + tls_min_version: str | None = None, + tls_max_version: str | None = None, +) -> IOpenSSLContextFactory: + keyfile_path = Path(__file__).parent.parent / keyfile + certfile_path = Path(__file__).parent.parent / certfile + + if not PYOPENSSL_WANTS_X509_PKEY: + cert = load_pem_x509_certificate(certfile_path.read_bytes()) + key = load_pem_private_key(keyfile_path.read_bytes(), password=None) + else: + cert = load_certificate(FILETYPE_PEM, certfile_path.read_bytes()) # type: ignore[assignment] + key = load_privatekey(FILETYPE_PEM, keyfile_path.read_bytes()) # type: ignore[assignment] + + tls_min = _TWISTED_VERSION_MAP.get(tls_min_version) if tls_min_version else None + tls_max = _TWISTED_VERSION_MAP.get(tls_max_version) if tls_max_version else None + tls_version_kwargs = _get_cert_options_version_kwargs(tls_min, tls_max) + # https://github.com/twisted/twisted/issues/12638 + factory: CertificateOptions = CertificateOptions( + privateKey=key, # type: ignore[arg-type] + certificate=cert, # type: ignore[arg-type] + **tls_version_kwargs, + ) + if cipher_string: + ctx = factory.getContext() + # disabling TLS1.3 because it unconditionally enables some strong ciphers + ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_3) + ctx.set_cipher_list(to_bytes(cipher_string)) + return cast("ContextFactory", factory) diff --git a/tests/pipelines.py b/tests/pipelines.py index d9fc12676..4f00d6919 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -4,13 +4,13 @@ Some pipelines used for testing class ZeroDivisionErrorPipeline: - def open_spider(self, spider): + def open_spider(self): 1 / 0 - def process_item(self, item, spider): + def process_item(self, item): return item class ProcessWithZeroDivisionErrorPipeline: - def process_item(self, item, spider): + def process_item(self, item): 1 / 0 diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index 5b75674f5..000000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -# Tests requirements -attrs -pexpect >= 4.8.0 -pyftpdlib >= 1.5.8 -pytest -pytest-cov==4.0.0 -pytest-xdist -sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 -testfixtures -uvloop; platform_system != "Windows" - -bpython # optional for shell wrapper tests -brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests -# 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072 -brotli==1.0.9; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests -zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests -ipython -pywin32; sys_platform == "win32" diff --git a/tests/sample_data/compressed/bomb-br.bin b/tests/sample_data/compressed/bomb-br.bin new file mode 100644 index 000000000..50059866f --- /dev/null +++ b/tests/sample_data/compressed/bomb-br.bin @@ -0,0 +1,2 @@ +Ÿ;§¯ø”nÞªVp SmoY2·å +ï(ä)-д=_o \ No newline at end of file diff --git a/tests/sample_data/compressed/bomb-deflate.bin b/tests/sample_data/compressed/bomb-deflate.bin new file mode 100644 index 000000000..3598aca07 Binary files /dev/null and b/tests/sample_data/compressed/bomb-deflate.bin differ diff --git a/tests/sample_data/compressed/bomb-gzip.bin b/tests/sample_data/compressed/bomb-gzip.bin new file mode 100644 index 000000000..64aa0c369 Binary files /dev/null and b/tests/sample_data/compressed/bomb-gzip.bin differ diff --git a/tests/sample_data/compressed/bomb-zstd.bin b/tests/sample_data/compressed/bomb-zstd.bin new file mode 100644 index 000000000..4b0efa8a4 Binary files /dev/null and b/tests/sample_data/compressed/bomb-zstd.bin differ diff --git a/tests/sample_data/compressed/html-gzip-deflate-gzip.bin b/tests/sample_data/compressed/html-gzip-deflate-gzip.bin new file mode 100644 index 000000000..d66f4c5a0 Binary files /dev/null and b/tests/sample_data/compressed/html-gzip-deflate-gzip.bin differ diff --git a/tests/sample_data/compressed/html-gzip-deflate.bin b/tests/sample_data/compressed/html-gzip-deflate.bin new file mode 100644 index 000000000..5066842ef Binary files /dev/null and b/tests/sample_data/compressed/html-gzip-deflate.bin differ diff --git a/tests/spiders.py b/tests/spiders.py index 3df153a12..065a0e6d7 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -1,8 +1,12 @@ """ Some spiders used for testing and benchmarking """ + +from __future__ import annotations + import asyncio import time +from typing import TYPE_CHECKING, Any from urllib.parse import urlencode from twisted.internet import defer @@ -15,21 +19,32 @@ from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.spiders.crawl import CrawlSpider, Rule from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future -from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req +from scrapy.utils.test import get_from_asyncio_queue + +if TYPE_CHECKING: + from scrapy.settings import _SettingsKey + from tests.mockserver.http import MockServer class MockServerSpider(Spider): - def __init__(self, mockserver=None, *args, **kwargs): + def __init__( + self, + *args, + mockserver: MockServer | None = None, + is_secure: bool = False, + **kwargs: Any, + ): super().__init__(*args, **kwargs) self.mockserver = mockserver + self.is_secure = is_secure class MetaSpider(MockServerSpider): name = "meta" - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) - self.meta = {} + self.meta: dict[str, Any] = {} def closed(self, reason): self.meta["close_reason"] = reason @@ -65,7 +80,7 @@ class DelaySpider(MetaSpider): self.b = b self.t1 = self.t2 = self.t2_err = 0 - def start_requests(self): + async def start(self): self.t1 = time.time() url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") yield Request(url, callback=self.parse, errback=self.errback) @@ -77,10 +92,32 @@ class DelaySpider(MetaSpider): self.t2_err = time.time() +class LogSpider(MetaSpider): + name = "log_spider" + + def log_debug(self, message: str, extra: dict[str, Any] | None = None): + self.logger.debug(message, extra=extra) + + def log_info(self, message: str, extra: dict[str, Any] | None = None): + self.logger.info(message, extra=extra) + + def log_warning(self, message: str, extra: dict[str, Any] | None = None): + self.logger.warning(message, extra=extra) + + def log_error(self, message: str, extra: dict[str, Any] | None = None): + self.logger.error(message, extra=extra) + + def log_critical(self, message: str, extra: dict[str, Any] | None = None): + self.logger.critical(message, extra=extra) + + def parse(self, response): + pass + + class SlowSpider(DelaySpider): name = "slow" - def start_requests(self): + async def start(self): # 1st response is fast url = self.mockserver.url("/delay?n=0&b=0") yield Request(url, callback=self.parse, errback=self.errback) @@ -150,7 +187,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): status = await get_from_asyncio_queue(response.status) self.logger.info(f"Got response {status}, req_id {req_id}") if req_id > 0: - return + return None reqs = [] for i in range(1, 3): req = Request(self.start_urls[0], dont_filter=True, meta={"req_id": i}) @@ -173,28 +210,24 @@ class AsyncDefDeferredDirectSpider(SimpleSpider): name = "asyncdef_deferred_direct" async def parse(self, response): - resp = await get_web_client_agent_req(self.mockserver.url("/status?n=200")) - yield {"code": resp.code} + await defer.succeed(None) + yield {"code": 200} class AsyncDefDeferredWrappedSpider(SimpleSpider): name = "asyncdef_deferred_wrapped" async def parse(self, response): - resp = await deferred_to_future( - get_web_client_agent_req(self.mockserver.url("/status?n=200")) - ) - yield {"code": resp.code} + await deferred_to_future(defer.succeed(None)) + yield {"code": 200} class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider): name = "asyncdef_deferred_wrapped" async def parse(self, response): - resp = await maybe_deferred_to_future( - get_web_client_agent_req(self.mockserver.url("/status?n=200")) - ) - yield {"code": resp.code} + await maybe_deferred_to_future(defer.succeed(None)) + yield {"code": 200} class AsyncDefAsyncioGenSpider(SimpleSpider): @@ -230,7 +263,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): callback=cb, ) - def start_requests(self): + async def start(self): for i in range(1, self.initial_reqs + 1): yield self._get_req(i) @@ -259,6 +292,24 @@ class ItemSpider(FollowAllSpider): yield {} +class MaxItemsAndRequestsSpider(FollowAllSpider): + def __init__(self, max_items=10, max_requests=10, *args, **kwargs): + super().__init__(*args, **kwargs) + self.max_items = max_items + self.max_requests = max_requests + + def parse(self, response): + self.items_scraped = 0 + self.pages_crawled = 1 # account for the start url + for request in super().parse(response): + if self.pages_crawled < self.max_requests: + yield request + self.pages_crawled += 1 + if self.items_scraped < self.max_items: + yield Item() + self.items_scraped += 1 + + class DefaultError(Exception): pass @@ -276,7 +327,7 @@ class ErrorSpider(FollowAllSpider): self.raise_exception() -class BrokenStartRequestsSpider(FollowAllSpider): +class BrokenStartSpider(FollowAllSpider): fail_before_yield = False fail_yielding = False @@ -284,7 +335,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): super().__init__(*a, **kw) self.seedsseen = [] - def start_requests(self): + async def start(self): if self.fail_before_yield: 1 / 0 @@ -295,21 +346,32 @@ class BrokenStartRequestsSpider(FollowAllSpider): if self.fail_yielding: 2 / 0 - assert ( - self.seedsseen - ), "All start requests consumed before any download happened" + assert self.seedsseen, "All seeds consumed before any download happened" def parse(self, response): self.seedsseen.append(response.meta.get("seed")) yield from super().parse(response) +class StartItemSpider(FollowAllSpider): + async def start(self): + yield {"name": "test item"} + + +class StartGoodAndBadOutput(FollowAllSpider): + async def start(self): + yield {"a": "a"} + yield Request("data:,a") + yield "data:,b" + yield object() + + class SingleRequestSpider(MetaSpider): seed = None callback_func = None errback_func = None - def start_requests(self): + async def start(self): if isinstance(self.seed, Request): yield self.seed.replace(callback=self.parse, errback=self.on_error) else: @@ -321,22 +383,24 @@ class SingleRequestSpider(MetaSpider): return self.callback_func(response) if "next" in response.meta: return response.meta["next"] + return None def on_error(self, failure): self.meta["failure"] = failure if callable(self.errback_func): return self.errback_func(failure) + return None -class DuplicateStartRequestsSpider(MockServerSpider): +class DuplicateStartSpider(MockServerSpider): dont_filter = True name = "duplicatestartrequests" distinct_urls = 2 dupe_factor = 3 - def start_requests(self): - for i in range(0, self.distinct_urls): - for j in range(0, self.dupe_factor): + async def start(self): + for i in range(self.distinct_urls): + for _ in range(self.dupe_factor): url = self.mockserver.url(f"/echo?headers=1&body=test{i}") yield Request(url, dont_filter=self.dont_filter) @@ -354,15 +418,15 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): """ name = "crawl_spider_with_parse_method" - custom_settings: dict = { + custom_settings: dict[_SettingsKey, Any] = { "RETRY_HTTP_CODES": [], # no need to retry } rules = (Rule(LinkExtractor(), callback="parse", follow=True),) - def start_requests(self): + async def start(self): test_body = b""" - Page title<title></head> + <head><title>Page title

Item 200

Item 201

@@ -413,10 +477,10 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = "crawl_spider_with_errback" rules = (Rule(LinkExtractor(), callback="parse", errback="errback", follow=True),) - def start_requests(self): + async def start(self): test_body = b""" - Page title<title></head> + <head><title>Page title

Item 200

Item 201

@@ -458,9 +522,9 @@ class BytesReceivedCallbackSpider(MetaSpider): crawler.signals.connect(spider.bytes_received, signals.bytes_received) return spider - def start_requests(self): + async def start(self): body = b"a" * self.full_response_length - url = self.mockserver.url("/alpayload") + url = self.mockserver.url("/alpayload", is_secure=self.is_secure) yield Request(url, method="POST", body=body, errback=self.errback) def parse(self, response): @@ -487,8 +551,11 @@ class HeadersReceivedCallbackSpider(MetaSpider): crawler.signals.connect(spider.headers_received, signals.headers_received) return spider - def start_requests(self): - yield Request(self.mockserver.url("/status"), errback=self.errback) + async def start(self): + yield Request( + self.mockserver.url("/status", is_secure=self.is_secure), + errback=self.errback, + ) def parse(self, response): self.meta["response"] = response diff --git a/tests/test_addons.py b/tests/test_addons.py index f1b01bc5c..3ad8cf7ff 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,15 +1,13 @@ import itertools -from typing import Any, Dict +from typing import Any from unittest.mock import patch -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest - from scrapy import Spider -from scrapy.crawler import Crawler, CrawlerRunner +from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunner from scrapy.exceptions import NotConfigured from scrapy.settings import BaseSettings, Settings -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings +from tests.utils.decorators import inline_callbacks_test class SimpleAddon: @@ -17,7 +15,7 @@ class SimpleAddon: pass -def get_addon_cls(config: Dict[str, Any]) -> type: +def get_addon_cls(config: dict[str, Any]) -> type: class AddonWithConfig: def update_settings(self, settings: BaseSettings): settings.update(config, priority="addon") @@ -39,7 +37,7 @@ class CreateInstanceAddon: settings.update(self.config, "addon") -class AddonTest(unittest.TestCase): +class TestAddon: def test_update_settings(self): settings = BaseSettings() settings.set("KEY1", "default", priority="default") @@ -47,31 +45,31 @@ class AddonTest(unittest.TestCase): addon_config = {"KEY1": "addon", "KEY2": "addon", "KEY3": "addon"} testaddon = get_addon_cls(addon_config)() testaddon.update_settings(settings) - self.assertEqual(settings["KEY1"], "addon") - self.assertEqual(settings["KEY2"], "project") - self.assertEqual(settings["KEY3"], "addon") + assert settings["KEY1"] == "addon" + assert settings["KEY2"] == "project" + assert settings["KEY3"] == "addon" -class AddonManagerTest(unittest.TestCase): +class TestAddonManager: def test_load_settings(self): settings_dict = { "ADDONS": {"tests.test_addons.SimpleAddon": 0}, } crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons - self.assertIsInstance(manager.addons[0], SimpleAddon) + assert isinstance(manager.addons[0], SimpleAddon) def test_notconfigured(self): class NotConfiguredAddon: def update_settings(self, settings): - raise NotConfigured() + raise NotConfigured settings_dict = { "ADDONS": {NotConfiguredAddon: 0}, } crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons - self.assertFalse(manager.addons) + assert not manager.addons def test_load_settings_order(self): # Get three addons with different settings @@ -86,8 +84,8 @@ class AddonManagerTest(unittest.TestCase): settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} crawler = get_crawler(settings_dict=settings) manager = crawler.addons - self.assertEqual([a.number for a in manager.addons], expected_order) - self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1]) + assert [a.number for a in manager.addons] == expected_order + assert crawler.settings.getint("KEY1") == expected_order[-1] def test_build_from_crawler(self): settings_dict = { @@ -96,8 +94,8 @@ class AddonManagerTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons - self.assertIsInstance(manager.addons[0], CreateInstanceAddon) - self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val") + assert isinstance(manager.addons[0], CreateInstanceAddon) + assert crawler.settings.get("MYADDON_KEY") == "val" def test_settings_priority(self): config = { @@ -105,82 +103,87 @@ class AddonManagerTest(unittest.TestCase): } settings_dict = { "ADDONS": {get_addon_cls(config): 1}, + **get_reactor_settings(), } crawler = get_crawler(settings_dict=settings_dict) - self.assertEqual(crawler.settings.getint("KEY"), 15) + assert crawler.settings.getint("KEY") == 15 + + runner_cls = ( + CrawlerRunner + if settings_dict.get("TWISTED_REACTOR_ENABLED", True) + else AsyncCrawlerRunner + ) settings = Settings(settings_dict) settings.set("KEY", 0, priority="default") - runner = CrawlerRunner(settings) + runner = runner_cls(settings) crawler = runner.create_crawler(Spider) crawler._apply_settings() - self.assertEqual(crawler.settings.getint("KEY"), 15) + assert crawler.settings.getint("KEY") == 15 settings_dict = { "KEY": 20, # priority=project "ADDONS": {get_addon_cls(config): 1}, + **get_reactor_settings(), } settings = Settings(settings_dict) settings.set("KEY", 0, priority="default") - runner = CrawlerRunner(settings) + runner = runner_cls(settings) crawler = runner.create_crawler(Spider) - self.assertEqual(crawler.settings.getint("KEY"), 20) + assert crawler.settings.getint("KEY") == 20 def test_fallback_workflow(self): - FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" + FALLBACK_SETTING = "MY_FALLBACK_SCHEDULER" class AddonWithFallback: def update_settings(self, settings): if not settings.get(FALLBACK_SETTING): settings.set( FALLBACK_SETTING, - settings.getwithbase("DOWNLOAD_HANDLERS")["https"], + settings.get("SCHEDULER"), "addon", ) - settings["DOWNLOAD_HANDLERS"]["https"] = "AddonHandler" + settings["SCHEDULER"] = "AddonScheduler" settings_dict = { "ADDONS": {AddonWithFallback: 1}, } crawler = get_crawler(settings_dict=settings_dict) - self.assertEqual( - crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" - ) - self.assertEqual( - crawler.settings.get(FALLBACK_SETTING), - "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", + assert crawler.settings.get("SCHEDULER") == "AddonScheduler" + assert ( + crawler.settings.get(FALLBACK_SETTING) == "scrapy.core.scheduler.Scheduler" ) settings_dict = { "ADDONS": {AddonWithFallback: 1}, - "DOWNLOAD_HANDLERS": {"https": "UserHandler"}, + "SCHEDULER": "UserScheduler", } crawler = get_crawler(settings_dict=settings_dict) - self.assertEqual( - crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" - ) - self.assertEqual(crawler.settings.get(FALLBACK_SETTING), "UserHandler") + assert crawler.settings.get("SCHEDULER") == "AddonScheduler" + assert crawler.settings.get(FALLBACK_SETTING) == "UserScheduler" def test_logging_message(self): class LoggedAddon: def update_settings(self, settings): pass - with patch("scrapy.addons.logger") as logger_mock: - with patch("scrapy.addons.build_from_crawler") as build_from_crawler_mock: - settings_dict = { - "ADDONS": {LoggedAddon: 1}, - } - addon = LoggedAddon() - build_from_crawler_mock.return_value = addon - crawler = get_crawler(settings_dict=settings_dict) - logger_mock.info.assert_called_once_with( - "Enabled addons:\n%(addons)s", - {"addons": [addon]}, - extra={"crawler": crawler}, - ) + with ( + patch("scrapy.addons.logger") as logger_mock, + patch("scrapy.addons.build_from_crawler") as build_from_crawler_mock, + ): + settings_dict = { + "ADDONS": {LoggedAddon: 1}, + } + addon = LoggedAddon() + build_from_crawler_mock.return_value = addon + crawler = get_crawler(settings_dict=settings_dict) + logger_mock.info.assert_called_once_with( + "Enabled addons:\n%(addons)s", + {"addons": [addon]}, + extra={"crawler": crawler}, + ) - @inlineCallbacks + @inline_callbacks_test def test_enable_addon_in_spider(self): class MySpider(Spider): name = "myspider" @@ -194,9 +197,15 @@ class AddonManagerTest(unittest.TestCase): return spider settings = Settings() + settings.setdict(get_reactor_settings()) settings.set("KEY", "default", priority="default") - runner = CrawlerRunner(settings) + runner_cls = ( + CrawlerRunner + if settings.getbool("TWISTED_REACTOR_ENABLED", True) + else AsyncCrawlerRunner + ) + runner = runner_cls(settings) crawler = runner.create_crawler(MySpider) - self.assertEqual(crawler.settings.get("KEY"), "default") + assert crawler.settings.get("KEY") == "default" yield crawler.crawl() - self.assertEqual(crawler.settings.get("KEY"), "addon") + assert crawler.settings.get("KEY") == "addon" diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 38ede70e4..9a7447e16 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -1,66 +1,110 @@ -from twisted.internet import defer -from twisted.trial.unittest import TestCase - from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer -from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider, SlowSpider +from tests.mockserver.http import MockServer +from tests.spiders import ( + ErrorSpider, + FollowAllSpider, + ItemSpider, + MaxItemsAndRequestsSpider, + SlowSpider, +) +from tests.utils.decorators import inline_callbacks_test -class TestCloseSpider(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() +class TestCloseSpider: + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) - @defer.inlineCallbacks + @inline_callbacks_test def test_closespider_itemcount(self): close_on = 5 crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on}) yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_itemcount") + assert reason == "closespider_itemcount" itemcount = crawler.stats.get_value("item_scraped_count") - self.assertTrue(itemcount >= close_on) + assert itemcount >= close_on - @defer.inlineCallbacks + @inline_callbacks_test def test_closespider_pagecount(self): close_on = 5 crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on}) yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_pagecount") + assert reason == "closespider_pagecount" pagecount = crawler.stats.get_value("response_received_count") - self.assertTrue(pagecount >= close_on) + assert pagecount >= close_on - @defer.inlineCallbacks + @inline_callbacks_test + def test_closespider_pagecount_no_item(self): + close_on = 5 + max_items = 5 + max_requests = close_on + max_items + crawler = get_crawler( + MaxItemsAndRequestsSpider, + { + "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on, + }, + ) + yield crawler.crawl( + max_items=max_items, max_requests=max_requests, mockserver=self.mockserver + ) + reason = crawler.spider.meta["close_reason"] + assert reason == "closespider_pagecount_no_item" + pagecount = crawler.stats.get_value("response_received_count") + itemcount = crawler.stats.get_value("item_scraped_count") + assert pagecount <= close_on + itemcount + + @inline_callbacks_test + def test_closespider_pagecount_no_item_with_pagecount(self): + close_on_pagecount_no_item = 5 + close_on_pagecount = 20 + crawler = get_crawler( + FollowAllSpider, + { + "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on_pagecount_no_item, + "CLOSESPIDER_PAGECOUNT": close_on_pagecount, + }, + ) + yield crawler.crawl(mockserver=self.mockserver) + reason = crawler.spider.meta["close_reason"] + assert reason == "closespider_pagecount_no_item" + pagecount = crawler.stats.get_value("response_received_count") + assert pagecount < close_on_pagecount + + @inline_callbacks_test def test_closespider_errorcount(self): close_on = 5 crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on}) yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_errorcount") + assert reason == "closespider_errorcount" key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}" errorcount = crawler.stats.get_value(key) - self.assertTrue(errorcount >= close_on) + assert crawler.stats.get_value("spider_exceptions/count") >= close_on + assert errorcount >= close_on - @defer.inlineCallbacks + @inline_callbacks_test def test_closespider_timeout(self): close_on = 0.1 crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on}) yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_timeout") + assert reason == "closespider_timeout" total_seconds = crawler.stats.get_value("elapsed_time_seconds") - self.assertTrue(total_seconds >= close_on) + assert total_seconds >= close_on - @defer.inlineCallbacks + @inline_callbacks_test def test_closespider_timeout_no_item(self): timeout = 1 crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) yield crawler.crawl(n=3, mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_timeout_no_item") + assert reason == "closespider_timeout_no_item" total_seconds = crawler.stats.get_value("elapsed_time_seconds") - self.assertTrue(total_seconds >= timeout) + assert total_seconds >= timeout diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 25ded143c..98a85bc17 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -4,7 +4,6 @@ import pstats import shutil import sys import tempfile -import unittest from io import StringIO from pathlib import Path from subprocess import PIPE, Popen @@ -12,27 +11,27 @@ from subprocess import PIPE, Popen from scrapy.utils.test import get_testenv -class CmdlineTest(unittest.TestCase): - def setUp(self): +class TestCmdline: + def setup_method(self): self.env = get_testenv() tests_path = Path(__file__).parent.parent self.env["PYTHONPATH"] += os.pathsep + str(tests_path.parent) self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings" def _execute(self, *new_args, **kwargs): - encoding = getattr(sys.stdout, "encoding") or "utf-8" - args = (sys.executable, "-m", "scrapy.cmdline") + new_args + encoding = sys.stdout.encoding or "utf-8" + args = (sys.executable, "-m", "scrapy.cmdline", *new_args) proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs) comm = proc.communicate()[0].strip() return comm.decode(encoding) def test_default_settings(self): - self.assertEqual(self._execute("settings", "--get", "TEST1"), "default") + assert self._execute("settings", "--get", "TEST1") == "default" def test_override_settings_using_set_arg(self): - self.assertEqual( - self._execute("settings", "--get", "TEST1", "-s", "TEST1=override"), - "override", + assert ( + self._execute("settings", "--get", "TEST1", "-s", "TEST1=override") + == "override" ) def test_profiling(self): @@ -40,14 +39,14 @@ class CmdlineTest(unittest.TestCase): filename = path / "res.prof" try: self._execute("version", "--profile", str(filename)) - self.assertTrue(filename.exists()) + assert filename.exists() out = StringIO() stats = pstats.Stats(str(filename), stream=out) stats.print_stats() out.seek(0) stats = out.read() - self.assertIn(str(Path("scrapy", "commands", "version.py")), stats) - self.assertIn("tottime", stats) + assert str(Path("scrapy", "commands", "version.py")) in stats + assert "tottime" in stats finally: shutil.rmtree(path) @@ -62,15 +61,14 @@ class CmdlineTest(unittest.TestCase): "EXTENSIONS=" + json.dumps(EXTENSIONS), ) # XXX: There's gotta be a smarter way to do this... - self.assertNotIn("...", settingsstr) + assert "..." not in settingsstr for char in ("'", "<", ">"): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) - self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys()) - self.assertEqual(200, settingsdict[EXT_PATH]) + assert set(settingsdict.keys()) == set(EXTENSIONS.keys()) + assert settingsdict[EXT_PATH] == 200 def test_pathlib_path_as_feeds_key(self): - self.assertEqual( - self._execute("settings", "--get", "FEEDS"), - json.dumps({"items.csv": {"format": "csv", "fields": ["price", "name"]}}), + assert self._execute("settings", "--get", "FEEDS") == json.dumps( + {"items.csv": {"format": "csv", "fields": ["price", "name"]}} ) diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py index 5cb09b5c0..f17543575 100644 --- a/tests/test_cmdline_crawl_with_pipeline/__init__.py +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -1,19 +1,21 @@ import sys -import unittest from pathlib import Path from subprocess import PIPE, Popen -class CmdlineCrawlPipelineTest(unittest.TestCase): +class TestCmdlineCrawlPipeline: def _execute(self, spname): args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname) cwd = Path(__file__).resolve().parent proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd) - proc.communicate() - return proc.returncode + _, stderr = proc.communicate() + return proc.returncode, stderr def test_open_spider_normally_in_pipeline(self): - self.assertEqual(self._execute("normal"), 0) + returncode, _ = self._execute("normal") + assert returncode == 0 def test_exception_at_open_spider_in_pipeline(self): - self.assertEqual(self._execute("exception"), 1) + returncode, _ = self._execute("exception") + # An exception in pipeline's open_spider should result in a non-zero exit code + assert returncode == 1 diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py index af15cac68..577ecd686 100644 --- a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py +++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py @@ -2,13 +2,13 @@ class TestSpiderPipeline: def open_spider(self, spider): pass - def process_item(self, item, spider): + def process_item(self, item): return item class TestSpiderExceptionPipeline: def open_spider(self, spider): - raise Exception("exception") + raise RuntimeError("exception") - def process_item(self, item, spider): + def process_item(self, item): return item diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 592494aba..815f87026 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -1,16 +1,29 @@ -from tests.test_commands import CommandTest +from __future__ import annotations + +import sys +from io import StringIO +from typing import TYPE_CHECKING +from unittest import TestCase +from unittest.mock import MagicMock, Mock, PropertyMock, call, patch + +from scrapy.commands.check import Command, TextTestResult +from tests.test_commands import TestProjectBase +from tests.utils.cmdline import proc + +if TYPE_CHECKING: + from pathlib import Path -class CheckCommandTest(CommandTest): - command = "check" +class DummyTestCase(TestCase): + pass - def setUp(self): - super().setUp() - self.spider_name = "check_spider" - self.spider = (self.proj_mod_path / "spiders" / "checkspider.py").resolve() - def _write_contract(self, contracts, parse_def): - self.spider.write_text( +class TestCheckCommand(TestProjectBase): + spider_name = "check_spider" + + def _write_contract(self, proj_path: Path, contracts: str, parse_def: str) -> None: + spider = proj_path / self.project_name / "spiders" / "checkspider.py" + spider.write_text( f""" import scrapy @@ -18,6 +31,10 @@ class CheckSpider(scrapy.Spider): name = '{self.spider_name}' start_urls = ['data:,'] + custom_settings = {{ + "DOWNLOAD_DELAY": 0, + }} + def parse(self, response, **cb_kwargs): \"\"\" @url data:, @@ -28,32 +45,50 @@ class CheckSpider(scrapy.Spider): encoding="utf-8", ) - def _test_contract(self, contracts="", parse_def="pass"): - self._write_contract(contracts, parse_def) - p, out, err = self.proc("check") - self.assertNotIn("F", out) - self.assertIn("OK", err) - self.assertEqual(p.returncode, 0) + def _test_contract( + self, + proj_path: Path, + contracts: str = "", + parse_def: str = "pass", + use_reactor: bool = True, + ) -> None: + self._write_contract(proj_path, contracts, parse_def) + args = ["check"] + if not use_reactor: + args += ["-s", "TWISTED_REACTOR_ENABLED=False"] + ret, out, err = proc(*args, cwd=proj_path) + assert "F" not in out + assert "OK" in err + assert ret == 0 - def test_check_returns_requests_contract(self): + def test_check_returns_requests_contract(self, proj_path: Path) -> None: contracts = """ @returns requests 1 """ parse_def = """ yield scrapy.Request(url='http://next-url.com') """ - self._test_contract(contracts, parse_def) + self._test_contract(proj_path, contracts, parse_def) - def test_check_returns_items_contract(self): + def test_check_no_reactor(self, proj_path: Path) -> None: + contracts = """ + @returns requests 1 + """ + parse_def = """ + yield scrapy.Request(url='http://next-url.com') + """ + self._test_contract(proj_path, contracts, parse_def, use_reactor=False) + + def test_check_returns_items_contract(self, proj_path: Path) -> None: contracts = """ @returns items 1 """ parse_def = """ yield {'key1': 'val1', 'key2': 'val2'} """ - self._test_contract(contracts, parse_def) + self._test_contract(proj_path, contracts, parse_def) - def test_check_cb_kwargs_contract(self): + def test_check_cb_kwargs_contract(self, proj_path: Path) -> None: contracts = """ @cb_kwargs {"arg1": "val1", "arg2": "val2"} """ @@ -61,18 +96,18 @@ class CheckSpider(scrapy.Spider): if len(cb_kwargs.items()) == 0: raise Exception("Callback args not set") """ - self._test_contract(contracts, parse_def) + self._test_contract(proj_path, contracts, parse_def) - def test_check_scrapes_contract(self): + def test_check_scrapes_contract(self, proj_path: Path) -> None: contracts = """ @scrapes key1 key2 """ parse_def = """ yield {'key1': 'val1', 'key2': 'val2'} """ - self._test_contract(contracts, parse_def) + self._test_contract(proj_path, contracts, parse_def) - def test_check_all_default_contracts(self): + def test_check_all_default_contracts(self, proj_path: Path) -> None: contracts = """ @returns items 1 @returns requests 1 @@ -85,12 +120,115 @@ class CheckSpider(scrapy.Spider): if len(cb_kwargs.items()) == 0: raise Exception("Callback args not set") """ - self._test_contract(contracts, parse_def) + self._test_contract(proj_path, contracts, parse_def) - def test_SCRAPY_CHECK_set(self): + def test_SCRAPY_CHECK_set(self, proj_path: Path) -> None: parse_def = """ import os if not os.environ.get('SCRAPY_CHECK'): raise Exception('SCRAPY_CHECK not set') """ - self._test_contract(parse_def=parse_def) + self._test_contract(proj_path, parse_def=parse_def) + + def test_printSummary_with_unsuccessful_test_result_without_errors_and_without_failures( + self, + ) -> None: + result = TextTestResult(MagicMock(), descriptions=False, verbosity=1) + start_time = 1.0 + stop_time = 2.0 + result.testsRun = 5 + result.failures = [] + result.errors = [] + result.unexpectedSuccesses = [DummyTestCase(), DummyTestCase()] + with patch.object(result.stream, "write") as mock_write: + result.printSummary(start_time, stop_time) + mock_write.assert_has_calls([call("FAILED"), call("\n")]) + + def test_printSummary_with_unsuccessful_test_result_with_only_failures( + self, + ) -> None: + result = TextTestResult(MagicMock(), descriptions=False, verbosity=1) + start_time = 1.0 + stop_time = 2.0 + result.testsRun = 5 + result.failures = [(DummyTestCase(), "failure")] + result.errors = [] + with patch.object(result.stream, "writeln") as mock_write: + result.printSummary(start_time, stop_time) + mock_write.assert_called_with(" (failures=1)") + + def test_printSummary_with_unsuccessful_test_result_with_only_errors(self) -> None: + result = TextTestResult(MagicMock(), descriptions=False, verbosity=1) + start_time = 1.0 + stop_time = 2.0 + result.testsRun = 5 + result.failures = [] + result.errors = [(DummyTestCase(), "error")] + with patch.object(result.stream, "writeln") as mock_write: + result.printSummary(start_time, stop_time) + mock_write.assert_called_with(" (errors=1)") + + def test_printSummary_with_unsuccessful_test_result_with_both_failures_and_errors( + self, + ) -> None: + result = TextTestResult(MagicMock(), descriptions=False, verbosity=1) + start_time = 1.0 + stop_time = 2.0 + result.testsRun = 5 + result.failures = [(DummyTestCase(), "failure")] + result.errors = [(DummyTestCase(), "error")] + with patch.object(result.stream, "writeln") as mock_write: + result.printSummary(start_time, stop_time) + mock_write.assert_called_with(" (failures=1, errors=1)") + + @patch("scrapy.commands.check.ContractsManager") + def test_run_with_opts_list_prints_spider(self, cm_cls_mock) -> None: + output = StringIO() + sys.stdout = output + cmd = Command() + cmd.settings = Mock( + get_component_priority_dict_with_base=Mock(return_value={}), + ) + cm_cls_mock.return_value = cm_mock = Mock() + spider_loader_mock = Mock() + cmd.crawler_process = Mock(spider_loader=spider_loader_mock) + spider_name = "FakeSpider" + spider_cls_mock = Mock() + type(spider_cls_mock).name = PropertyMock(return_value=spider_name) + spider_loader_mock.load.side_effect = lambda x: {spider_name: spider_cls_mock}[ + x + ] + tested_methods = ["fakeMethod1", "fakeMethod2"] + cm_mock.tested_methods_from_spidercls.side_effect = lambda x: { + spider_cls_mock: tested_methods + }[x] + + cmd.run([spider_name], Mock(list=True)) + + assert output.getvalue() == "FakeSpider\n * fakeMethod1\n * fakeMethod2\n" + sys.stdout = sys.__stdout__ + + @patch("scrapy.commands.check.ContractsManager") + def test_run_without_opts_list_does_not_crawl_spider_with_no_tested_methods( + self, cm_cls_mock + ) -> None: + cmd = Command() + cmd.settings = Mock( + get_component_priority_dict_with_base=Mock(return_value={}), + ) + cm_cls_mock.return_value = cm_mock = Mock() + spider_loader_mock = Mock() + cmd.crawler_process = Mock(spider_loader=spider_loader_mock) + spider_name = "FakeSpider" + spider_cls_mock = Mock() + spider_loader_mock.load.side_effect = lambda x: {spider_name: spider_cls_mock}[ + x + ] + tested_methods: list[str] = [] + cm_mock.tested_methods_from_spidercls.side_effect = lambda x: { + spider_cls_mock: tested_methods + }[x] + + cmd.run([spider_name], Mock(list=False)) + + cmd.crawler_process.crawl.assert_not_called() diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py new file mode 100644 index 000000000..dd69d36ba --- /dev/null +++ b/tests/test_command_crawl.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from tests.test_commands import TestProjectBase +from tests.utils.cmdline import proc + +if TYPE_CHECKING: + from collections.abc import Iterable + from pathlib import Path + + +class TestCrawlCommand(TestProjectBase): + def crawl( + self, code: str, proj_path: Path, args: Iterable[str] = () + ) -> tuple[int, str, str]: + (proj_path / self.project_name / "spiders" / "myspider.py").write_text( + code, encoding="utf-8" + ) + return proc("crawl", "myspider", *args, cwd=proj_path) + + def get_log(self, code: str, proj_path: Path, args: Iterable[str] = ()) -> str: + _, _, stderr = self.crawl(code, proj_path, args=args) + return stderr + + def test_no_output(self, proj_path: Path) -> None: + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('It works!') + return + yield +""" + log = self.get_log(spider_code, proj_path) + assert "[myspider] DEBUG: It works!" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Spider closed (finished)" in log + + def test_output(self, proj_path: Path) -> None: + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return + yield +""" + args = ["-o", "example.json"] + log = self.get_log(spider_code, proj_path, args=args) + assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log + + def test_overwrite_output(self, proj_path: Path) -> None: + spider_code = """ +import json +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug( + 'FEEDS: {}'.format( + json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) + ) + ) + return + yield +""" + j = proj_path / "example.json" + j.write_text("not empty", encoding="utf-8") + args = ["-O", "example.json"] + log = self.get_log(spider_code, proj_path, args=args) + assert ( + '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' + in log + ) + with j.open(encoding="utf-8") as f2: + first_line = f2.readline() + assert first_line != "not empty" + + def test_output_and_overwrite_output(self, proj_path: Path) -> None: + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + return + yield +""" + args = ["-o", "example1.json", "-O", "example2.json"] + log = self.get_log(spider_code, proj_path, args=args) + assert ( + "error: Please use only one of -o/--output and -O/--overwrite-output" in log + ) + + def test_default_reactor(self, proj_path: Path) -> None: + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('It works!') + return + yield +""" + log = self.get_log(spider_code, proj_path, args=("-s", "TWISTED_REACTOR=")) + assert "[myspider] DEBUG: It works!" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + assert "Spider closed (finished)" in log + + def test_no_reactor(self, proj_path: Path) -> None: + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('It works!') + return + yield +""" + log = self.get_log( + spider_code, proj_path, args=("-s", "TWISTED_REACTOR_ENABLED=False") + ) + assert "[myspider] DEBUG: It works!" in log + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index d2027d1c2..d98dac968 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -1,35 +1,38 @@ -from twisted.internet import defer -from twisted.trial import unittest +from __future__ import annotations -from scrapy.utils.testproc import ProcessTest -from scrapy.utils.testsite import SiteTest +from typing import TYPE_CHECKING + +from tests.utils.cmdline import proc + +if TYPE_CHECKING: + from tests.mockserver.http import MockServer -class FetchTest(ProcessTest, SiteTest, unittest.TestCase): - command = "fetch" +class TestFetchCommand: + def test_output(self, mockserver: MockServer) -> None: + _, out, _ = proc("fetch", mockserver.url("/text")) + assert out.strip() == "Works" - @defer.inlineCallbacks - def test_output(self): - _, out, _ = yield self.execute([self.url("/text")]) - self.assertEqual(out.strip(), b"Works") + def test_redirect_default(self, mockserver: MockServer) -> None: + _, out, _ = proc("fetch", mockserver.url("/redirect")) + assert out.strip() == "Redirected here" - @defer.inlineCallbacks - def test_redirect_default(self): - _, out, _ = yield self.execute([self.url("/redirect")]) - self.assertEqual(out.strip(), b"Redirected here") - - @defer.inlineCallbacks - def test_redirect_disabled(self): - _, out, err = yield self.execute( - ["--no-redirect", self.url("/redirect-no-meta-refresh")] + def test_redirect_disabled(self, mockserver: MockServer) -> None: + _, _, err = proc( + "fetch", "--no-redirect", mockserver.url("/redirect-no-meta-refresh") ) err = err.strip() - self.assertIn(b"downloader/response_status_count/302", err, err) - self.assertNotIn(b"downloader/response_status_count/200", err, err) + assert "downloader/response_status_count/302" in err + assert "downloader/response_status_count/200" not in err - @defer.inlineCallbacks - def test_headers(self): - _, out, _ = yield self.execute([self.url("/text"), "--headers"]) - out = out.replace(b"\r", b"") # required on win32 - assert b"Server: TwistedWeb" in out, out - assert b"Content-Type: text/plain" in out + def test_headers(self, mockserver: MockServer) -> None: + _, out, _ = proc("fetch", mockserver.url("/text"), "--headers") + out = out.replace("\r", "") # required on win32 + assert "Server: TwistedWeb" in out + assert "Content-Type: text/plain" in out + + def test_no_reactor(self, mockserver: MockServer) -> None: + _, out, _ = proc( + "fetch", "-s", "TWISTED_REACTOR_ENABLED=False", mockserver.url("/text") + ) + assert out.strip() == "Works" diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py new file mode 100644 index 000000000..67e3eb50a --- /dev/null +++ b/tests/test_command_genspider.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from tests.test_commands import TestProjectBase +from tests.utils.cmdline import call, proc + + +def find_in_file(filename: Path, regex: str) -> re.Match[str] | None: + """Find first pattern occurrence in file""" + pattern = re.compile(regex) + with filename.open("r", encoding="utf-8") as f: + for line in f: + match = pattern.search(line) + if match is not None: + return match + return None + + +class TestGenspiderCommand(TestProjectBase): + def test_arguments(self, proj_path: Path) -> None: + spider = proj_path / self.project_name / "spiders" / "test_name.py" + # only pass one argument. spider script shouldn't be created + assert call("genspider", "test_name", cwd=proj_path) == 2 + assert not spider.exists() + # pass two arguments . spider script should be created + assert call("genspider", "test_name", "test.com", cwd=proj_path) == 0 + assert spider.exists() + + @pytest.mark.parametrize( + "tplname", + [ + "basic", + "crawl", + "xmlfeed", + "csvfeed", + ], + ) + def test_template(self, tplname: str, proj_path: Path) -> None: + args = [f"--template={tplname}"] if tplname else [] + spname = "test_spider" + spmodule = f"{self.project_name}.spiders.{spname}" + spfile = proj_path / self.project_name / "spiders" / f"{spname}.py" + _, out, _ = proc("genspider", spname, "test.com", *args, cwd=proj_path) + assert ( + f"Created spider {spname!r} using template {tplname!r} in module:\n {spmodule}" + in out + ) + assert spfile.exists() + modify_time_before = spfile.stat().st_mtime + _, out, _ = proc("genspider", spname, "test.com", *args, cwd=proj_path) + assert f"Spider {spname!r} already exists in module" in out + modify_time_after = spfile.stat().st_mtime + assert modify_time_after == modify_time_before + + def test_list(self, proj_path: Path) -> None: + assert call("genspider", "--list", cwd=proj_path) == 0 + + def test_dump(self, proj_path: Path) -> None: + assert call("genspider", "--dump=basic", cwd=proj_path) == 0 + assert call("genspider", "-d", "basic", cwd=proj_path) == 0 + + def test_same_name_as_project(self, proj_path: Path) -> None: + assert call("genspider", self.project_name, cwd=proj_path) == 2 + assert not ( + proj_path / self.project_name / "spiders" / f"{self.project_name}.py" + ).exists() + + @pytest.mark.parametrize("force", [True, False]) + def test_same_filename_as_existing_spider( + self, force: bool, proj_path: Path + ) -> None: + file_name = "example" + file_path = proj_path / self.project_name / "spiders" / f"{file_name}.py" + assert call("genspider", file_name, "example.com", cwd=proj_path) == 0 + assert file_path.exists() + + # change name of spider but not its file name + with file_path.open("r+", encoding="utf-8") as spider_file: + file_data = spider_file.read() + file_data = file_data.replace('name = "example"', 'name = "renamed"') + spider_file.seek(0) + spider_file.write(file_data) + spider_file.truncate() + modify_time_before = file_path.stat().st_mtime + file_contents_before = file_data + + if force: + _, out, _ = proc( + "genspider", "--force", file_name, "example.com", cwd=proj_path + ) + assert ( + f"Created spider {file_name!r} using template 'basic' in module" in out + ) + modify_time_after = file_path.stat().st_mtime + assert modify_time_after != modify_time_before + file_contents_after = file_path.read_text(encoding="utf-8") + assert file_contents_after != file_contents_before + else: + _, out, _ = proc("genspider", file_name, "example.com", cwd=proj_path) + assert f"{file_path.resolve()} already exists" in out + modify_time_after = file_path.stat().st_mtime + assert modify_time_after == modify_time_before + file_contents_after = file_path.read_text(encoding="utf-8") + assert file_contents_after == file_contents_before + + @pytest.mark.parametrize( + ("url", "domain"), + [ + ("test.com", "test.com"), + ("https://test.com", "test.com"), + ], + ) + def test_url(self, url: str, domain: str, proj_path: Path) -> None: + assert call("genspider", "--force", "test_name", url, cwd=proj_path) == 0 + spider = proj_path / self.project_name / "spiders" / "test_name.py" + m = find_in_file(spider, r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]") + assert m is not None + assert m.group(1) == domain + m = find_in_file(spider, r"start_urls\s*=\s*\[['\"](.+)['\"]\]") + assert m is not None + assert m.group(1) == f"https://{domain}" + + @pytest.mark.parametrize( + ("url", "expected", "template"), + [ + # basic + ("https://test.com", "https://test.com", "basic"), + ("http://test.com", "http://test.com", "basic"), + ("http://test.com/other/path", "http://test.com/other/path", "basic"), + ("test.com/other/path", "https://test.com/other/path", "basic"), + # crawl + ("https://test.com", "https://test.com", "crawl"), + ("http://test.com", "http://test.com", "crawl"), + ("http://test.com/other/path", "http://test.com/other/path", "crawl"), + ("test.com/other/path", "https://test.com/other/path", "crawl"), + ("test.com", "https://test.com", "crawl"), + # xmlfeed + ("https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"), + ("http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed"), + ("test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"), + # csvfeed + ("https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"), + ("http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed"), + ("test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"), + ], + ) + def test_template_start_urls( + self, url: str, expected: str, template: str, proj_path: Path + ) -> None: + assert ( + call( + "genspider", "-t", template, "--force", "test_name", url, cwd=proj_path + ) + == 0 + ) + spider = proj_path / self.project_name / "spiders" / "test_name.py" + m = find_in_file(spider, r"start_urls\s*=\s*\[['\"](.+)['\"]\]") + assert m is not None + assert m.group(1) == expected + + +class TestGenspiderStandaloneCommand: + def test_generate_standalone_spider(self, tmp_path: Path) -> None: + call("genspider", "example", "example.com", cwd=tmp_path) + assert Path(tmp_path, "example.py").exists() + + @pytest.mark.parametrize("force", [True, False]) + def test_same_name_as_existing_file(self, force: bool, tmp_path: Path) -> None: + file_name = "example" + file_path = Path(tmp_path, file_name + ".py") + _, out, _ = proc("genspider", file_name, "example.com", cwd=tmp_path) + assert f"Created spider {file_name!r} using template 'basic' " in out + assert file_path.exists() + modify_time_before = file_path.stat().st_mtime + file_contents_before = file_path.read_text(encoding="utf-8") + + if force: + # use different template to ensure contents were changed + _, out, _ = proc( + "genspider", + "--force", + "-t", + "crawl", + file_name, + "example.com", + cwd=tmp_path, + ) + assert f"Created spider {file_name!r} using template 'crawl' " in out + modify_time_after = file_path.stat().st_mtime + assert modify_time_after != modify_time_before + file_contents_after = file_path.read_text(encoding="utf-8") + assert file_contents_after != file_contents_before + else: + _, out, _ = proc("genspider", file_name, "example.com", cwd=tmp_path) + assert ( + f"{Path(tmp_path, file_name + '.py').resolve()} already exists" in out + ) + modify_time_after = file_path.stat().st_mtime + assert modify_time_after == modify_time_before + file_contents_after = file_path.read_text(encoding="utf-8") + assert file_contents_after == file_contents_before diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 9356d6b79..c210a06e8 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,30 +1,29 @@ -import argparse -import os -from pathlib import Path +from __future__ import annotations -from twisted.internet import defer +import argparse +import re +from typing import TYPE_CHECKING + +import pytest from scrapy.commands import parse from scrapy.settings import Settings -from scrapy.utils.python import to_unicode -from scrapy.utils.testproc import ProcessTest -from scrapy.utils.testsite import SiteTest -from tests.test_commands import CommandTest +from tests.test_commands import TestProjectBase +from tests.utils.cmdline import call, proc + +if TYPE_CHECKING: + from pathlib import Path + + from tests.mockserver.http import MockServer -def _textmode(bstr): - """Normalize input the same as writing to a file - and reading from it in text mode""" - return to_unicode(bstr).replace(os.linesep, "\n") +class TestParseCommand(TestProjectBase): + spider_name = "parse_spider" - -class ParseCommandTest(ProcessTest, SiteTest, CommandTest): - command = "parse" - - def setUp(self): - super().setUp() - self.spider_name = "parse_spider" - (self.proj_mod_path / "spiders" / "myspider.py").write_text( + @pytest.fixture(autouse=True) + def create_files(self, proj_path: Path) -> None: + proj_mod_path = proj_path / self.project_name + (proj_mod_path / "spiders" / "myspider.py").write_text( f""" import scrapy from scrapy.linkextractors import LinkExtractor @@ -33,7 +32,13 @@ from scrapy.utils.test import get_from_asyncio_queue import asyncio -class AsyncDefAsyncioReturnSpider(scrapy.Spider): +class BaseSpider(scrapy.Spider): + custom_settings = {{ + "DOWNLOAD_DELAY": 0, + }} + + +class AsyncDefAsyncioReturnSpider(BaseSpider): name = "asyncdef_asyncio_return" async def parse(self, response): @@ -42,7 +47,7 @@ class AsyncDefAsyncioReturnSpider(scrapy.Spider): self.logger.info(f"Got response {{status}}") return [{{'id': 1}}, {{'id': 2}}] -class AsyncDefAsyncioReturnSingleElementSpider(scrapy.Spider): +class AsyncDefAsyncioReturnSingleElementSpider(BaseSpider): name = "asyncdef_asyncio_return_single_element" async def parse(self, response): @@ -51,7 +56,7 @@ class AsyncDefAsyncioReturnSingleElementSpider(scrapy.Spider): self.logger.info(f"Got response {{status}}") return {{'foo': 42}} -class AsyncDefAsyncioGenLoopSpider(scrapy.Spider): +class AsyncDefAsyncioGenLoopSpider(BaseSpider): name = "asyncdef_asyncio_gen_loop" async def parse(self, response): @@ -60,7 +65,7 @@ class AsyncDefAsyncioGenLoopSpider(scrapy.Spider): yield {{'foo': i}} self.logger.info(f"Got response {{response.status}}") -class AsyncDefAsyncioSpider(scrapy.Spider): +class AsyncDefAsyncioSpider(BaseSpider): name = "asyncdef_asyncio" async def parse(self, response): @@ -68,7 +73,7 @@ class AsyncDefAsyncioSpider(scrapy.Spider): status = await get_from_asyncio_queue(response.status) self.logger.debug(f"Got response {{status}}") -class AsyncDefAsyncioGenExcSpider(scrapy.Spider): +class AsyncDefAsyncioGenExcSpider(BaseSpider): name = "asyncdef_asyncio_gen_exc" async def parse(self, response): @@ -90,7 +95,8 @@ class MySpider(scrapy.Spider): custom_settings = {{ "DOWNLOADER_MIDDLEWARES": {{ CallbackSignatureDownloaderMiddleware: 0, - }} + }}, + "DOWNLOAD_DELAY": 0, }} def parse(self, response): @@ -123,6 +129,10 @@ class MySpider(scrapy.Spider): class MyGoodCrawlSpider(CrawlSpider): name = 'goodcrawl{self.spider_name}' + custom_settings = {{ + "DOWNLOAD_DELAY": 0, + }} + rules = ( Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), Rule(LinkExtractor(allow=r'/text'), follow=True), @@ -139,6 +149,10 @@ class MyBadCrawlSpider(CrawlSpider): '''Spider which doesn't define a parse_item callback while using it in a rule.''' name = 'badcrawl{self.spider_name}' + custom_settings = {{ + "DOWNLOAD_DELAY": 0, + }} + rules = ( Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), ) @@ -149,288 +163,338 @@ class MyBadCrawlSpider(CrawlSpider): encoding="utf-8", ) - (self.proj_mod_path / "pipelines.py").write_text( + (proj_mod_path / "pipelines.py").write_text( """ import logging class MyPipeline: component_name = 'my_pipeline' - def process_item(self, item, spider): + def process_item(self, item): logging.info('It Works!') return item """, encoding="utf-8", ) - with (self.proj_mod_path / "settings.py").open("a", encoding="utf-8") as f: + with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f: f.write( f""" ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} """ ) - @defer.inlineCallbacks - def test_spider_arguments(self): - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "-a", - "test_arg=1", - "-c", - "parse", - "--verbose", - self.url("/html"), - ] + def test_spider_arguments(self, proj_path: Path, mockserver: MockServer) -> None: + _, _, stderr = proc( + "parse", + "--spider", + self.spider_name, + "-a", + "test_arg=1", + "-c", + "parse", + "--verbose", + mockserver.url("/html"), + cwd=proj_path, ) - self.assertIn("DEBUG: It Works!", _textmode(stderr)) + assert "DEBUG: It Works!" in stderr - @defer.inlineCallbacks - def test_request_with_meta(self): + def test_request_with_meta(self, proj_path: Path, mockserver: MockServer) -> None: raw_json_string = '{"foo" : "baz"}' - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "--meta", - raw_json_string, - "-c", - "parse_request_with_meta", - "--verbose", - self.url("/html"), - ] + _, _, stderr = proc( + "parse", + "--spider", + self.spider_name, + "--meta", + raw_json_string, + "-c", + "parse_request_with_meta", + "--verbose", + mockserver.url("/html"), + cwd=proj_path, ) - self.assertIn("DEBUG: It Works!", _textmode(stderr)) + assert "DEBUG: It Works!" in stderr - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "-m", - raw_json_string, - "-c", - "parse_request_with_meta", - "--verbose", - self.url("/html"), - ] + _, _, stderr = proc( + "parse", + "--spider", + self.spider_name, + "-m", + raw_json_string, + "-c", + "parse_request_with_meta", + "--verbose", + mockserver.url("/html"), + cwd=proj_path, ) - self.assertIn("DEBUG: It Works!", _textmode(stderr)) + assert "DEBUG: It Works!" in stderr - @defer.inlineCallbacks - def test_request_with_cb_kwargs(self): + def test_request_with_cb_kwargs( + self, proj_path: Path, mockserver: MockServer + ) -> None: raw_json_string = '{"foo" : "bar", "key": "value"}' - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "--cbkwargs", - raw_json_string, - "-c", - "parse_request_with_cb_kwargs", - "--verbose", - self.url("/html"), - ] + _, _, stderr = proc( + "parse", + "--spider", + self.spider_name, + "--cbkwargs", + raw_json_string, + "-c", + "parse_request_with_cb_kwargs", + "--verbose", + mockserver.url("/html"), + cwd=proj_path, ) - log = _textmode(stderr) - self.assertIn("DEBUG: It Works!", log) - self.assertIn( - "DEBUG: request.callback signature: (response, foo=None, key=None)", log + assert "DEBUG: It Works!" in stderr + assert ( + "DEBUG: request.callback signature: (response, foo=None, key=None)" + in stderr ) - @defer.inlineCallbacks - def test_request_without_meta(self): - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "-c", - "parse_request_without_meta", - "--nolinks", - self.url("/html"), - ] + def test_request_without_meta( + self, proj_path: Path, mockserver: MockServer + ) -> None: + _, _, stderr = proc( + "parse", + "--spider", + self.spider_name, + "-c", + "parse_request_without_meta", + "--nolinks", + mockserver.url("/html"), + cwd=proj_path, ) - self.assertIn("DEBUG: It Works!", _textmode(stderr)) + assert "DEBUG: It Works!" in stderr - @defer.inlineCallbacks - def test_pipelines(self): - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "--pipelines", - "-c", - "parse", - "--verbose", - self.url("/html"), - ] + def test_pipelines(self, proj_path: Path, mockserver: MockServer) -> None: + _, _, stderr = proc( + "parse", + "--spider", + self.spider_name, + "--pipelines", + "-c", + "parse", + "--verbose", + mockserver.url("/html"), + cwd=proj_path, ) - self.assertIn("INFO: It Works!", _textmode(stderr)) + assert "INFO: It Works!" in stderr - @defer.inlineCallbacks - def test_async_def_asyncio_parse_items_list(self): - status, out, stderr = yield self.execute( - [ - "--spider", - "asyncdef_asyncio_return", - "-c", - "parse", - self.url("/html"), - ] + def test_async_def_asyncio_parse_items_list( + self, proj_path: Path, mockserver: MockServer + ) -> None: + _, out, stderr = proc( + "parse", + "--spider", + "asyncdef_asyncio_return", + "-c", + "parse", + mockserver.url("/html"), + cwd=proj_path, ) - self.assertIn("INFO: Got response 200", _textmode(stderr)) - self.assertIn("{'id': 1}", _textmode(out)) - self.assertIn("{'id': 2}", _textmode(out)) + assert "INFO: Got response 200" in stderr + assert "{'id': 1}" in out + assert "{'id': 2}" in out - @defer.inlineCallbacks - def test_async_def_asyncio_parse_items_single_element(self): - status, out, stderr = yield self.execute( - [ - "--spider", - "asyncdef_asyncio_return_single_element", - "-c", - "parse", - self.url("/html"), - ] + def test_async_def_asyncio_parse_items_single_element( + self, proj_path: Path, mockserver: MockServer + ) -> None: + _, out, stderr = proc( + "parse", + "--spider", + "asyncdef_asyncio_return_single_element", + "-c", + "parse", + mockserver.url("/html"), + cwd=proj_path, ) - self.assertIn("INFO: Got response 200", _textmode(stderr)) - self.assertIn("{'foo': 42}", _textmode(out)) + assert "INFO: Got response 200" in stderr + assert "{'foo': 42}" in out - @defer.inlineCallbacks - def test_async_def_asyncgen_parse_loop(self): - status, out, stderr = yield self.execute( - [ - "--spider", - "asyncdef_asyncio_gen_loop", - "-c", - "parse", - self.url("/html"), - ] + def test_async_def_asyncgen_parse_loop( + self, proj_path: Path, mockserver: MockServer + ) -> None: + _, out, stderr = proc( + "parse", + "--spider", + "asyncdef_asyncio_gen_loop", + "-c", + "parse", + mockserver.url("/html"), + cwd=proj_path, ) - self.assertIn("INFO: Got response 200", _textmode(stderr)) + assert "INFO: Got response 200" in stderr for i in range(10): - self.assertIn(f"{{'foo': {i}}}", _textmode(out)) + assert f"{{'foo': {i}}}" in out - @defer.inlineCallbacks - def test_async_def_asyncgen_parse_exc(self): - status, out, stderr = yield self.execute( - [ - "--spider", - "asyncdef_asyncio_gen_exc", - "-c", - "parse", - self.url("/html"), - ] + def test_async_def_asyncgen_parse_exc( + self, proj_path: Path, mockserver: MockServer + ) -> None: + _, out, stderr = proc( + "parse", + "--spider", + "asyncdef_asyncio_gen_exc", + "-c", + "parse", + mockserver.url("/html"), + cwd=proj_path, ) - self.assertIn("ValueError", _textmode(stderr)) + assert "ValueError" in stderr for i in range(7): - self.assertIn(f"{{'foo': {i}}}", _textmode(out)) + assert f"{{'foo': {i}}}" in out - @defer.inlineCallbacks - def test_async_def_asyncio_parse(self): - _, _, stderr = yield self.execute( - [ - "--spider", - "asyncdef_asyncio", - "-c", - "parse", - self.url("/html"), - ] + def test_async_def_asyncio_parse( + self, proj_path: Path, mockserver: MockServer + ) -> None: + _, _, stderr = proc( + "parse", + "--spider", + "asyncdef_asyncio", + "-c", + "parse", + mockserver.url("/html"), + cwd=proj_path, ) - self.assertIn("DEBUG: Got response 200", _textmode(stderr)) + assert "DEBUG: Got response 200" in stderr - @defer.inlineCallbacks - def test_parse_items(self): - status, out, stderr = yield self.execute( - ["--spider", self.spider_name, "-c", "parse", self.url("/html")] + def test_parse_items(self, proj_path: Path, mockserver: MockServer) -> None: + _, out, _ = proc( + "parse", + "--spider", + self.spider_name, + "-c", + "parse", + mockserver.url("/html"), + cwd=proj_path, ) - self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) + assert "[{}, {'foo': 'bar'}]" in out - @defer.inlineCallbacks - def test_parse_items_no_callback_passed(self): - status, out, stderr = yield self.execute( - ["--spider", self.spider_name, self.url("/html")] + def test_parse_items_no_callback_passed( + self, proj_path: Path, mockserver: MockServer + ) -> None: + _, out, _ = proc( + "parse", + "--spider", + self.spider_name, + mockserver.url("/html"), + cwd=proj_path, ) - self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) + assert "[{}, {'foo': 'bar'}]" in out - @defer.inlineCallbacks - def test_wrong_callback_passed(self): - status, out, stderr = yield self.execute( - ["--spider", self.spider_name, "-c", "dummy", self.url("/html")] + def test_wrong_callback_passed( + self, proj_path: Path, mockserver: MockServer + ) -> None: + _, out, stderr = proc( + "parse", + "--spider", + self.spider_name, + "-c", + "dummy", + mockserver.url("/html"), + cwd=proj_path, ) - self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") - self.assertIn("""Cannot find callback""", _textmode(stderr)) + assert re.search(r"# Scraped Items -+\r?\n\[\]", out) + assert "Cannot find callback" in stderr - @defer.inlineCallbacks - def test_crawlspider_matching_rule_callback_set(self): + def test_crawlspider_matching_rule_callback_set( + self, proj_path: Path, mockserver: MockServer + ) -> None: """If a rule matches the URL, use it's defined callback.""" - status, out, stderr = yield self.execute( - ["--spider", "goodcrawl" + self.spider_name, "-r", self.url("/html")] + _, out, _ = proc( + "parse", + "--spider", + "goodcrawl" + self.spider_name, + "-r", + mockserver.url("/html"), + cwd=proj_path, ) - self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) + assert "[{}, {'foo': 'bar'}]" in out - @defer.inlineCallbacks - def test_crawlspider_matching_rule_default_callback(self): + def test_crawlspider_matching_rule_default_callback( + self, proj_path: Path, mockserver: MockServer + ) -> None: """If a rule match but it has no callback set, use the 'parse' callback.""" - status, out, stderr = yield self.execute( - ["--spider", "goodcrawl" + self.spider_name, "-r", self.url("/text")] + _, out, _ = proc( + "parse", + "--spider", + "goodcrawl" + self.spider_name, + "-r", + mockserver.url("/text"), + cwd=proj_path, ) - self.assertIn("""[{}, {'nomatch': 'default'}]""", _textmode(out)) + assert "[{}, {'nomatch': 'default'}]" in out - @defer.inlineCallbacks - def test_spider_with_no_rules_attribute(self): + def test_spider_with_no_rules_attribute( + self, proj_path: Path, mockserver: MockServer + ) -> None: """Using -r with a spider with no rule should not produce items.""" - status, out, stderr = yield self.execute( - ["--spider", self.spider_name, "-r", self.url("/html")] + _, out, stderr = proc( + "parse", + "--spider", + self.spider_name, + "-r", + mockserver.url("/html"), + cwd=proj_path, ) - self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") - self.assertIn("""No CrawlSpider rules found""", _textmode(stderr)) + assert re.search(r"# Scraped Items -+\r?\n\[\]", out) + assert "No CrawlSpider rules found" in stderr - @defer.inlineCallbacks - def test_crawlspider_missing_callback(self): - status, out, stderr = yield self.execute( - ["--spider", "badcrawl" + self.spider_name, "-r", self.url("/html")] + def test_crawlspider_missing_callback( + self, proj_path: Path, mockserver: MockServer + ) -> None: + _, out, _ = proc( + "parse", + "--spider", + "badcrawl" + self.spider_name, + "-r", + mockserver.url("/html"), + cwd=proj_path, ) - self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") + assert re.search(r"# Scraped Items -+\r?\n\[\]", out) - @defer.inlineCallbacks - def test_crawlspider_no_matching_rule(self): + def test_crawlspider_no_matching_rule( + self, proj_path: Path, mockserver: MockServer + ) -> None: """The requested URL has no matching rule, so no items should be scraped""" - status, out, stderr = yield self.execute( - ["--spider", "badcrawl" + self.spider_name, "-r", self.url("/enc-gb18030")] + _, out, stderr = proc( + "parse", + "--spider", + "badcrawl" + self.spider_name, + "-r", + mockserver.url("/enc-gb18030"), + cwd=proj_path, ) - self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") - self.assertIn("""Cannot find a rule that matches""", _textmode(stderr)) + assert re.search(r"# Scraped Items -+\r?\n\[\]", out) + assert "Cannot find a rule that matches" in stderr - @defer.inlineCallbacks - def test_crawlspider_not_exists_with_not_matched_url(self): - status, out, stderr = yield self.execute([self.url("/invalid_url")]) - self.assertEqual(status, 0) + def test_crawlspider_not_exists_with_not_matched_url( + self, proj_path: Path, mockserver: MockServer + ) -> None: + assert call("parse", mockserver.url("/invalid_url"), cwd=proj_path) == 0 - @defer.inlineCallbacks - def test_output_flag(self): + def test_output_flag(self, proj_path: Path, mockserver: MockServer) -> None: """Checks if a file was created successfully having correct format containing correct data in it. """ file_name = "data.json" - file_path = Path(self.proj_path, file_name) - yield self.execute( - [ - "--spider", - self.spider_name, - "-c", - "parse", - "-o", - file_name, - self.url("/html"), - ] + file_path = proj_path / file_name + proc( + "parse", + "--spider", + self.spider_name, + "-c", + "parse", + "-o", + file_name, + mockserver.url("/html"), + cwd=proj_path, ) - self.assertTrue(file_path.exists()) - self.assertTrue(file_path.is_file()) + assert file_path.exists() + assert file_path.is_file() content = '[\n{},\n{"foo": "bar"}\n]' - self.assertEqual(file_path.read_text(encoding="utf-8"), content) + assert file_path.read_text(encoding="utf-8") == content def test_parse_add_options(self): command = parse.Command() @@ -445,7 +509,23 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} namespace = parser.parse_args( ["--verbose", "--nolinks", "-d", "2", "--spider", self.spider_name] ) - self.assertTrue(namespace.nolinks) - self.assertEqual(namespace.depth, 2) - self.assertEqual(namespace.spider, self.spider_name) - self.assertTrue(namespace.verbose) + assert namespace.nolinks + assert namespace.depth == 2 + assert namespace.spider == self.spider_name + assert namespace.verbose + + def test_no_reactor(self, proj_path: Path, mockserver: MockServer) -> None: + _, out, stderr = proc( + "parse", + "--spider", + "asyncdef_asyncio_return", + "-c", + "parse", + mockserver.url("/html"), + "-s", + "TWISTED_REACTOR_ENABLED=False", + cwd=proj_path, + ) + assert "INFO: Got response 200" in stderr + assert "{'id': 1}" in out + assert "{'id': 2}" in out diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py new file mode 100644 index 000000000..b1455611e --- /dev/null +++ b/tests/test_command_runspider.py @@ -0,0 +1,361 @@ +from __future__ import annotations + +import asyncio +import inspect +import platform +import sys +from typing import TYPE_CHECKING + +import pytest + +from tests.test_crawler import ExceptionSpider, NoRequestsSpider +from tests.utils.cmdline import proc + +if TYPE_CHECKING: + from collections.abc import Iterable + from pathlib import Path + + +class TestRunSpiderCommand: + spider_filename = "myspider.py" + + debug_log_spider = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug("It Works!") + return + yield +""" + + badspider = """ +import scrapy + +class BadSpider(scrapy.Spider): + name = "bad" + async def start(self): + raise Exception("oops!") + yield + """ + + def runspider( + self, cwd: Path, code: str, name: str | None = None, args: Iterable[str] = () + ) -> tuple[int, str, str]: + fname = cwd / (name or self.spider_filename) + fname.write_text(code, encoding="utf-8") + return proc("runspider", str(fname), *args, cwd=cwd) + + def get_log( + self, cwd: Path, code: str, name: str | None = None, args: Iterable[str] = () + ) -> str: + _, _, stderr = self.runspider(cwd, code, name, args=args) + return stderr + + def test_runspider(self, tmp_path: Path) -> None: + log = self.get_log(tmp_path, self.debug_log_spider) + assert "DEBUG: It Works!" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "INFO: Spider closed (finished)" in log + + def test_run_fail_spider(self, tmp_path: Path) -> None: + ret, _, _ = self.runspider( + tmp_path, "import scrapy\n" + inspect.getsource(ExceptionSpider) + ) + assert ret != 0 + + def test_run_good_spider(self, tmp_path: Path) -> None: + ret, _, _ = self.runspider( + tmp_path, "import scrapy\n" + inspect.getsource(NoRequestsSpider) + ) + assert ret == 0 + + def test_runspider_log_level(self, tmp_path: Path) -> None: + log = self.get_log( + tmp_path, self.debug_log_spider, args=("-s", "LOG_LEVEL=INFO") + ) + assert "DEBUG: It Works!" not in log + assert "INFO: Spider opened" in log + + def test_runspider_default_reactor(self, tmp_path: Path) -> None: + log = self.get_log( + tmp_path, self.debug_log_spider, args=("-s", "TWISTED_REACTOR=") + ) + assert "DEBUG: It Works!" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + assert "INFO: Spider opened" in log + assert "INFO: Closing spider (finished)" in log + assert "INFO: Spider closed (finished)" in log + + def test_runspider_dnscache_disabled(self, tmp_path: Path) -> None: + # see https://github.com/scrapy/scrapy/issues/2811 + # The spider below should not be able to connect to localhost:12345, + # which is intended, + # but this should not be because of DNS lookup error + # assumption: localhost will resolve in all cases (true?) + dnscache_spider = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + start_urls = ['http://localhost:12345'] + + custom_settings = { + "ROBOTSTXT_OBEY": False, + "RETRY_ENABLED": False, + } + + def parse(self, response): + return {'test': 'value'} +""" + log = self.get_log( + tmp_path, dnscache_spider, args=("-s", "DNSCACHE_ENABLED=False") + ) + assert "CannotResolveHostError" not in log + assert "INFO: Spider opened" in log + + @pytest.mark.parametrize("value", [False, True]) + def test_runspider_log_short_names(self, tmp_path: Path, value: bool) -> None: + log1 = self.get_log( + tmp_path, self.debug_log_spider, args=("-s", f"LOG_SHORT_NAMES={value}") + ) + assert "[myspider] DEBUG: It Works!" in log1 + assert ("[scrapy]" in log1) is value + assert ("[scrapy.core.engine]" in log1) is not value + + def test_runspider_no_spider_found(self, tmp_path: Path) -> None: + log = self.get_log(tmp_path, "from scrapy.spiders import Spider\n") + assert "No spider found in file" in log + + def test_runspider_file_not_found(self) -> None: + _, _, log = proc("runspider", "some_non_existent_file") + assert "File not found: some_non_existent_file" in log + + def test_runspider_unable_to_load(self, tmp_path: Path) -> None: + log = self.get_log(tmp_path, "", name="myspider.txt") + assert "Unable to load" in log + + def test_start_errors(self, tmp_path: Path) -> None: + log = self.get_log(tmp_path, self.badspider, name="badspider.py") + assert "start" in log + assert "badspider.py" in log, log + + def test_asyncio_enabled_true(self, tmp_path: Path) -> None: + log = self.get_log( + tmp_path, + self.debug_log_spider, + args=[ + "-s", + "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", + ], + ) + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_asyncio_enabled_default(self, tmp_path: Path) -> None: + log = self.get_log(tmp_path, self.debug_log_spider) + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_asyncio_enabled_false(self, tmp_path: Path) -> None: + log = self.get_log( + tmp_path, + self.debug_log_spider, + args=["-s", "TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor"], + ) + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + + @pytest.mark.requires_uvloop + def test_custom_asyncio_loop_enabled_true(self, tmp_path: Path) -> None: + log = self.get_log( + tmp_path, + self.debug_log_spider, + args=[ + "-s", + "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "-s", + "ASYNCIO_EVENT_LOOP=uvloop.Loop", + ], + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + def test_custom_asyncio_loop_enabled_false(self, tmp_path: Path) -> None: + log = self.get_log( + tmp_path, + self.debug_log_spider, + args=[ + "-s", + "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", + ], + ) + if sys.platform != "win32": + loop = asyncio.new_event_loop() + else: + loop = asyncio.SelectorEventLoop() + assert ( + f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}" + in log + ) + + def test_no_reactor(self, tmp_path: Path) -> None: + log = self.get_log( + tmp_path, + self.debug_log_spider, + args=[ + "-s", + "TWISTED_REACTOR_ENABLED=False", + ], + ) + assert "Not using a Twisted reactor" in log + + def test_output(self, tmp_path: Path) -> None: + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return + yield +""" + args = ["-o", "example.json"] + log = self.get_log(tmp_path, spider_code, args=args) + assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log + + def test_overwrite_output(self, tmp_path: Path) -> None: + spider_code = """ +import json +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug( + 'FEEDS: {}'.format( + json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) + ) + ) + return + yield +""" + (tmp_path / "example.json").write_text("not empty", encoding="utf-8") + args = ["-O", "example.json"] + log = self.get_log(tmp_path, spider_code, args=args) + assert ( + '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' + in log + ) + with (tmp_path / "example.json").open(encoding="utf-8") as f2: + first_line = f2.readline() + assert first_line != "not empty" + + def test_output_and_overwrite_output(self, tmp_path: Path) -> None: + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + return + yield +""" + args = ["-o", "example1.json", "-O", "example2.json"] + log = self.get_log(tmp_path, spider_code, args=args) + assert ( + "error: Please use only one of -o/--output and -O/--overwrite-output" in log + ) + + def test_output_stdout(self, tmp_path: Path) -> None: + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return + yield +""" + args = ["-o", "-:json"] + log = self.get_log(tmp_path, spider_code, args=args) + assert "[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}" in log + + @pytest.mark.parametrize("arg", ["output.json:json", "output.json"]) + def test_absolute_path(self, tmp_path: Path, arg: str) -> None: + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} + """ + + args = ["-o", str(tmp_path / arg)] + log = self.get_log(tmp_path, spider_code, args=args) + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {tmp_path / 'output.json'}" + in log + ) + + def test_args_change_settings(self, tmp_path: Path) -> None: + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("FOO", kwargs.get("foo")) + return spider + + async def start(self): + self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") + return + yield +""" + args = ["-a", "foo=42"] + log = self.get_log(tmp_path, spider_code, args=args) + assert "Spider closed (finished)" in log + assert "The value of FOO is 42" in log + + +@pytest.mark.skipif( + platform.system() != "Windows", reason="Windows required for .pyw files" +) +class TestWindowsRunSpiderCommand(TestRunSpiderCommand): + spider_filename = "myspider.pyw" + + def test_start_errors(self, tmp_path: Path) -> None: + log = self.get_log(tmp_path, self.badspider, name="badspider.pyw") + assert "start" in log + assert "badspider.pyw" in log + + def test_runspider_unable_to_load(self, tmp_path: Path) -> None: + pytest.skip("Already Tested in 'RunSpiderCommandTest'") diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 7918d94b2..1585835cc 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -1,147 +1,147 @@ +from __future__ import annotations + import os import sys from io import BytesIO from pathlib import Path +from typing import TYPE_CHECKING +import pytest from pexpect.popen_spawn import PopenSpawn -from twisted.internet import defer -from twisted.trial import unittest -from scrapy.utils.testproc import ProcessTest -from scrapy.utils.testsite import SiteTest +from scrapy.utils.reactor import _asyncio_reactor_path from tests import NON_EXISTING_RESOLVABLE, tests_datadir -from tests.mockserver import MockServer +from tests.utils.cmdline import proc + +if TYPE_CHECKING: + from tests.mockserver.http import MockServer -class ShellTest(ProcessTest, SiteTest, unittest.TestCase): - command = "shell" +class TestShellCommand: + def test_empty(self) -> None: + _, out, _ = proc("shell", "-c", "item") + assert "{}" in out - @defer.inlineCallbacks - def test_empty(self): - _, out, _ = yield self.execute(["-c", "item"]) - assert b"{}" in out + def test_empty_no_reactor(self) -> None: + _, out, _ = proc( + "shell", "-c", "item", "--set", "TWISTED_REACTOR_ENABLED=False" + ) + assert "{}" in out - @defer.inlineCallbacks - def test_response_body(self): - _, out, _ = yield self.execute([self.url("/text"), "-c", "response.body"]) - assert b"Works" in out + def test_response_body(self, mockserver: MockServer) -> None: + _, out, _ = proc("shell", mockserver.url("/text"), "-c", "response.body") + assert "Works" in out - @defer.inlineCallbacks - def test_response_type_text(self): - _, out, _ = yield self.execute([self.url("/text"), "-c", "type(response)"]) - assert b"TextResponse" in out + def test_response_type_text(self, mockserver: MockServer) -> None: + _, out, _ = proc("shell", mockserver.url("/text"), "-c", "type(response)") + assert "TextResponse" in out - @defer.inlineCallbacks - def test_response_type_html(self): - _, out, _ = yield self.execute([self.url("/html"), "-c", "type(response)"]) - assert b"HtmlResponse" in out + def test_response_type_html(self, mockserver: MockServer) -> None: + _, out, _ = proc("shell", mockserver.url("/html"), "-c", "type(response)") + assert "HtmlResponse" in out - @defer.inlineCallbacks - def test_response_selector_html(self): + def test_response_selector_html(self, mockserver: MockServer) -> None: xpath = "response.xpath(\"//p[@class='one']/text()\").get()" - _, out, _ = yield self.execute([self.url("/html"), "-c", xpath]) - self.assertEqual(out.strip(), b"Works") + _, out, _ = proc("shell", mockserver.url("/html"), "-c", xpath) + assert out.strip() == "Works" - @defer.inlineCallbacks - def test_response_encoding_gb18030(self): - _, out, _ = yield self.execute( - [self.url("/enc-gb18030"), "-c", "response.encoding"] + def test_response_encoding_gb18030(self, mockserver: MockServer) -> None: + _, out, _ = proc( + "shell", mockserver.url("/enc-gb18030"), "-c", "response.encoding" ) - self.assertEqual(out.strip(), b"gb18030") + assert out.strip() == "gb18030" - @defer.inlineCallbacks - def test_redirect(self): - _, out, _ = yield self.execute([self.url("/redirect"), "-c", "response.url"]) - assert out.strip().endswith(b"/redirected") + def test_redirect(self, mockserver: MockServer) -> None: + _, out, _ = proc("shell", mockserver.url("/redirect"), "-c", "response.url") + assert out.strip().endswith("/redirected") - @defer.inlineCallbacks - def test_redirect_follow_302(self): - _, out, _ = yield self.execute( - [self.url("/redirect-no-meta-refresh"), "-c", "response.status"] + def test_redirect_follow_302(self, mockserver: MockServer) -> None: + _, out, _ = proc( + "shell", + mockserver.url("/redirect-no-meta-refresh"), + "-c", + "response.status", ) - assert out.strip().endswith(b"200") + assert out.strip().endswith("200") - @defer.inlineCallbacks - def test_redirect_not_follow_302(self): - _, out, _ = yield self.execute( - [ - "--no-redirect", - self.url("/redirect-no-meta-refresh"), - "-c", - "response.status", - ] + def test_redirect_not_follow_302(self, mockserver: MockServer) -> None: + _, out, _ = proc( + "shell", + "--no-redirect", + mockserver.url("/redirect-no-meta-refresh"), + "-c", + "response.status", ) - assert out.strip().endswith(b"302") + assert out.strip().endswith("302") - @defer.inlineCallbacks - def test_fetch_redirect_follow_302(self): + def test_fetch_redirect_follow_302(self, mockserver: MockServer) -> None: """Test that calling ``fetch(url)`` follows HTTP redirects by default.""" - url = self.url("/redirect-no-meta-refresh") + url = mockserver.url("/redirect-no-meta-refresh") code = f"fetch('{url}')" - errcode, out, errout = yield self.execute(["-c", code]) - self.assertEqual(errcode, 0, out) - assert b"Redirecting (302)" in errout - assert b"Crawled (200)" in errout + ret, out, err = proc("shell", "-c", code) + assert ret == 0, out + assert "Redirecting (302)" in err + assert "Crawled (200)" in err - @defer.inlineCallbacks - def test_fetch_redirect_not_follow_302(self): + def test_fetch_redirect_not_follow_302(self, mockserver: MockServer) -> None: """Test that calling ``fetch(url, redirect=False)`` disables automatic redirects.""" - url = self.url("/redirect-no-meta-refresh") + url = mockserver.url("/redirect-no-meta-refresh") code = f"fetch('{url}', redirect=False)" - errcode, out, errout = yield self.execute(["-c", code]) - self.assertEqual(errcode, 0, out) - assert b"Crawled (302)" in errout + ret, out, err = proc("shell", "-c", code) + assert ret == 0, out + assert "Crawled (302)" in err - @defer.inlineCallbacks - def test_request_replace(self): - url = self.url("/text") + def test_request_replace(self, mockserver: MockServer) -> None: + url = mockserver.url("/text") code = f"fetch('{url}') or fetch(response.request.replace(method='POST'))" - errcode, out, _ = yield self.execute(["-c", code]) - self.assertEqual(errcode, 0, out) + ret, out, _ = proc("shell", "-c", code) + assert ret == 0, out - @defer.inlineCallbacks - def test_scrapy_import(self): - url = self.url("/text") + def test_scrapy_import(self, mockserver: MockServer) -> None: + url = mockserver.url("/text") code = f"fetch(scrapy.Request('{url}'))" - errcode, out, _ = yield self.execute(["-c", code]) - self.assertEqual(errcode, 0, out) + ret, out, _ = proc("shell", "-c", code) + assert ret == 0, out - @defer.inlineCallbacks - def test_local_file(self): + def test_local_file(self) -> None: filepath = Path(tests_datadir, "test_site", "index.html") - _, out, _ = yield self.execute([str(filepath), "-c", "item"]) - assert b"{}" in out + _, out, _ = proc("shell", str(filepath), "-c", "item") + assert "{}" in out - @defer.inlineCallbacks - def test_local_nofile(self): + def test_local_nofile(self) -> None: filepath = "file:///tests/sample_data/test_site/nothinghere.html" - errcode, out, err = yield self.execute( - [filepath, "-c", "item"], check_code=False - ) - self.assertEqual(errcode, 1, out or err) - self.assertIn(b"No such file or directory", err) + ret, out, err = proc("shell", filepath, "-c", "item") + assert ret == 1, out or err + assert "No such file or directory" in err - @defer.inlineCallbacks - def test_dns_failures(self): + def test_dns_failures(self, mockserver: MockServer) -> None: if NON_EXISTING_RESOLVABLE: - raise unittest.SkipTest("Non-existing hosts are resolvable") + pytest.skip("Non-existing hosts are resolvable") url = "www.somedomainthatdoesntexi.st" - errcode, out, err = yield self.execute([url, "-c", "item"], check_code=False) - self.assertEqual(errcode, 1, out or err) - self.assertIn(b"DNS lookup failed", err) + ret, out, err = proc("shell", url, "-c", "item") + assert ret == 1, out or err + assert "CannotResolveHostError" in err - @defer.inlineCallbacks - def test_shell_fetch_async(self): - reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" - url = self.url("/html") + def test_shell_fetch_async(self, mockserver: MockServer) -> None: + url = mockserver.url("/html") code = f"fetch('{url}')" - args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"] - _, _, err = yield self.execute(args, check_code=True) - self.assertNotIn(b"RuntimeError: There is no current event loop in thread", err) + ret, _, err = proc( + "shell", "-c", code, "--set", f"TWISTED_REACTOR={_asyncio_reactor_path}" + ) + assert ret == 0, err + assert "RuntimeError: There is no current event loop in thread" not in err + + def test_shell_fetch_no_reactor(self, mockserver: MockServer) -> None: + url = mockserver.url("/html") + code = f"fetch('{url}')" + ret, _, err = proc( + "shell", "-c", code, "--set", "TWISTED_REACTOR_ENABLED=False" + ) + assert ret == 0, err -class InteractiveShellTest(unittest.TestCase): - def test_fetch(self): +class TestInteractiveShell: + def test_fetch(self, mockserver: MockServer) -> None: args = ( sys.executable, "-m", @@ -154,11 +154,10 @@ class InteractiveShellTest(unittest.TestCase): p = PopenSpawn(args, env=env, timeout=5) p.logfile_read = logfile p.expect_exact("Available Scrapy objects") - with MockServer() as mockserver: - p.sendline(f"fetch('{mockserver.url('/')}')") - p.sendline("type(response)") - p.expect_exact("HtmlResponse") + p.sendline(f"fetch('{mockserver.url('/')}')") + p.sendline("type(response)") + p.expect_exact("HtmlResponse") p.sendeof() - p.wait() + p.wait() # type: ignore[no-untyped-call] logfile.seek(0) - self.assertNotIn("Traceback", logfile.read().decode()) + assert "Traceback" not in logfile.read().decode() diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py new file mode 100644 index 000000000..2a9d0ed57 --- /dev/null +++ b/tests/test_command_startproject.py @@ -0,0 +1,292 @@ +from __future__ import annotations + +import os +import subprocess +import sys +from contextlib import contextmanager +from itertools import chain +from pathlib import Path +from shutil import copytree +from stat import S_IWRITE as ANYONE_WRITE_PERMISSION + +import scrapy +from scrapy.commands.startproject import IGNORE +from scrapy.utils.test import get_testenv +from tests.utils.cmdline import call, proc + + +class TestStartprojectCommand: + project_name = "testproject" + + @staticmethod + def _assert_files_exist(project_dir: Path, project_name: str) -> None: + assert (project_dir / "scrapy.cfg").exists() + assert (project_dir / project_name).exists() + assert (project_dir / project_name / "__init__.py").exists() + assert (project_dir / project_name / "items.py").exists() + assert (project_dir / project_name / "pipelines.py").exists() + assert (project_dir / project_name / "settings.py").exists() + assert (project_dir / project_name / "spiders" / "__init__.py").exists() + + def test_startproject(self, tmp_path: Path) -> None: + # with no dir argument creates the project in the "self.project_name" subdir of cwd + assert call("startproject", self.project_name, cwd=tmp_path) == 0 + self._assert_files_exist(tmp_path / self.project_name, self.project_name) + + assert call("startproject", self.project_name, cwd=tmp_path) == 1 + assert call("startproject", "wrong---project---name") == 1 + assert call("startproject", "sys") == 1 + + def test_startproject_with_project_dir(self, tmp_path: Path) -> None: + # with a dir arg creates the project in the specified dir + project_dir = tmp_path / "project" + assert ( + call("startproject", self.project_name, str(project_dir), cwd=tmp_path) == 0 + ) + self._assert_files_exist(project_dir, self.project_name) + + assert ( + call( + "startproject", self.project_name, str(project_dir) + "2", cwd=tmp_path + ) + == 0 + ) + + assert ( + call("startproject", self.project_name, str(project_dir), cwd=tmp_path) == 1 + ) + assert ( + call( + "startproject", self.project_name + "2", str(project_dir), cwd=tmp_path + ) + == 1 + ) + assert call("startproject", "wrong---project---name") == 1 + assert call("startproject", "sys") == 1 + assert call("startproject") == 2 + assert ( + call("startproject", self.project_name, str(project_dir), "another_params") + == 2 + ) + + def test_existing_project_dir(self, tmp_path: Path) -> None: + project_name = self.project_name + "_existing" + project_path = tmp_path / project_name + project_path.mkdir() + + assert call("startproject", project_name, cwd=tmp_path) == 0 + self._assert_files_exist(project_path, project_name) + + +def get_permissions_dict( + path: str | os.PathLike[str], renamings=None, ignore=None +) -> dict[str, str]: + def get_permissions(path: Path) -> str: + return oct(path.stat().st_mode) + + path_obj = Path(path) + + renamings = renamings or () + permissions_dict = { + ".": get_permissions(path_obj), + } + for root, dirs, files in os.walk(path_obj): + nodes = list(chain(dirs, files)) + if ignore: + ignored_names = ignore(root, nodes) + nodes = [node for node in nodes if node not in ignored_names] + for node in nodes: + absolute_path = Path(root, node) + relative_path = str(absolute_path.relative_to(path)) + for search_string, replacement in renamings: + relative_path = relative_path.replace(search_string, replacement) + permissions = get_permissions(absolute_path) + permissions_dict[relative_path] = permissions + return permissions_dict + + +class TestStartprojectTemplates: + def test_startproject_template_override(self, tmp_path: Path) -> None: + tmpl = tmp_path / "templates" + tmpl_proj = tmpl / "project" + project_name = "testproject" + + copytree(Path(scrapy.__path__[0], "templates"), tmpl) + (tmpl_proj / "root_template").write_bytes(b"") + + args = ["--set", f"TEMPLATES_DIR={tmpl}"] + _, out, _ = proc("startproject", project_name, *args, cwd=tmp_path) + assert f"New Scrapy project '{project_name}', using template directory" in out + assert str(tmpl_proj) in out + assert (tmp_path / project_name / "root_template").exists() + + def test_startproject_permissions_from_writable(self, tmp_path: Path) -> None: + """Check that generated files have the right permissions when the + template folder has the same permissions as in the project, i.e. + everything is writable.""" + scrapy_path = scrapy.__path__[0] + project_template = Path(scrapy_path, "templates", "project") + project_name = "startproject1" + renamings = ( + ("module", project_name), + (".tmpl", ""), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + destination = tmp_path / "proj" + destination.mkdir() + process = subprocess.Popen( + ( + sys.executable, + "-m", + "scrapy.cmdline", + "startproject", + project_name, + ), + cwd=destination, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + env=get_testenv(), + ) + process.wait() + + project_dir = destination / project_name + actual_permissions = get_permissions_dict(project_dir) + + assert actual_permissions == expected_permissions + + def test_startproject_permissions_from_read_only(self, tmp_path: Path) -> None: + """Check that generated files have the right permissions when the + template folder has been made read-only, which is something that some + systems do. + + See https://github.com/scrapy/scrapy/pull/4604 + """ + scrapy_path = scrapy.__path__[0] + templates_dir = Path(scrapy_path, "templates") + project_template = Path(templates_dir, "project") + project_name = "startproject2" + renamings = ( + ("module", project_name), + (".tmpl", ""), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + def _make_read_only(path: Path): + current_permissions = path.stat().st_mode + path.chmod(current_permissions & ~ANYONE_WRITE_PERMISSION) + + read_only_templates_dir = tmp_path / "templates" + copytree(templates_dir, read_only_templates_dir) + + for root, dirs, files in os.walk(read_only_templates_dir): + for node in chain(dirs, files): + _make_read_only(Path(root, node)) + + destination = tmp_path / "proj" + destination.mkdir() + assert ( + call( + "startproject", + project_name, + "--set", + f"TEMPLATES_DIR={read_only_templates_dir}", + cwd=destination, + ) + == 0 + ) + + project_dir = destination / project_name + actual_permissions = get_permissions_dict(project_dir) + + assert actual_permissions == expected_permissions + + def test_startproject_permissions_unchanged_in_destination( + self, tmp_path: Path + ) -> None: + """Check that preexisting folders and files in the destination folder + do not see their permissions modified.""" + scrapy_path = scrapy.__path__[0] + project_template = Path(scrapy_path, "templates", "project") + project_name = "startproject3" + renamings = ( + ("module", project_name), + (".tmpl", ""), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + destination = tmp_path / "proj" + project_dir = destination / project_name + project_dir.mkdir(parents=True) + + existing_nodes = { + f"{permissions:o}{extension}": permissions + for extension in ("", ".d") + for permissions in ( + 0o444, + 0o555, + 0o644, + 0o666, + 0o755, + 0o777, + ) + } + for node, permissions in existing_nodes.items(): + path = project_dir / node + if node.endswith(".d"): + path.mkdir(mode=permissions) + else: + path.touch(mode=permissions) + expected_permissions[node] = oct(path.stat().st_mode) + + assert call("startproject", project_name, ".", cwd=project_dir) == 0 + + actual_permissions = get_permissions_dict(project_dir) + + assert actual_permissions == expected_permissions + + def test_startproject_permissions_umask_022(self, tmp_path: Path) -> None: + """Check that generated files have the right permissions when the + system uses a umask value that causes new files to have different + permissions than those from the template folder.""" + + @contextmanager + def umask(new_mask): + cur_mask = os.umask(new_mask) + yield + os.umask(cur_mask) + + scrapy_path = scrapy.__path__[0] + project_template = Path(scrapy_path, "templates", "project") + project_name = "umaskproject" + renamings = ( + ("module", project_name), + (".tmpl", ""), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + with umask(0o002): + destination = tmp_path / "proj" + destination.mkdir() + assert call("startproject", project_name, cwd=destination) == 0 + + project_dir = destination / project_name + actual_permissions = get_permissions_dict(project_dir) + + assert actual_permissions == expected_permissions diff --git a/tests/test_command_version.py b/tests/test_command_version.py index a52d0d13c..c4ca9d07b 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -1,45 +1,25 @@ -import sys - -from twisted.internet import defer -from twisted.trial import unittest - import scrapy -from scrapy.utils.testproc import ProcessTest +from tests.utils.cmdline import proc -class VersionTest(ProcessTest, unittest.TestCase): - command = "version" +class TestVersionCommand: + def test_output(self) -> None: + _, out, _ = proc("version") + assert out.strip() == f"Scrapy {scrapy.__version__}" - @defer.inlineCallbacks - def test_output(self): - encoding = getattr(sys.stdout, "encoding") or "utf-8" - _, out, _ = yield self.execute([]) - self.assertEqual( - out.strip().decode(encoding), - f"Scrapy {scrapy.__version__}", - ) - - @defer.inlineCallbacks - def test_verbose_output(self): - encoding = getattr(sys.stdout, "encoding") or "utf-8" - _, out, _ = yield self.execute(["-v"]) - headers = [ - line.partition(":")[0].strip() - for line in out.strip().decode(encoding).splitlines() + def test_verbose_output(self) -> None: + _, out, _ = proc("version", "-v") + headers = [line.partition(":")[0].strip() for line in out.strip().splitlines()] + assert headers == [ + "Scrapy", + "lxml", + "libxml2", + "cssselect", + "parsel", + "w3lib", + "Twisted", + "Python", + "pyOpenSSL", + "cryptography", + "Platform", ] - self.assertEqual( - headers, - [ - "Scrapy", - "lxml", - "libxml2", - "cssselect", - "parsel", - "w3lib", - "Twisted", - "Python", - "pyOpenSSL", - "cryptography", - "Platform", - ], - ) diff --git a/tests/test_commands.py b/tests/test_commands.py index 36f800850..edb03da1b 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,37 +1,36 @@ -import argparse -import inspect -import json -import os -import platform -import re -import subprocess -import sys -import tempfile -from contextlib import contextmanager -from itertools import chain -from pathlib import Path -from shutil import copytree, rmtree -from stat import S_IWRITE as ANYONE_WRITE_PERMISSION -from tempfile import mkdtemp -from threading import Timer -from typing import Dict, Generator, Optional, Union -from unittest import skipIf +from __future__ import annotations -from pytest import mark -from twisted.trial import unittest +import argparse +import json +from io import StringIO +from shutil import copytree +from typing import TYPE_CHECKING +from unittest import mock + +import pytest import scrapy +from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view -from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings -from scrapy.utils.python import to_unicode -from scrapy.utils.test import get_testenv -from tests.test_crawler import ExceptionSpider, NoRequestsSpider +from scrapy.utils.reactor import _asyncio_reactor_path +from tests.utils.cmdline import call, proc + +if TYPE_CHECKING: + from pathlib import Path -class CommandSettings(unittest.TestCase): - def setUp(self): - self.command = ScrapyCommand() +class EmptyCommand(ScrapyCommand): + def short_desc(self) -> str: + return "" + + def run(self, args: list[str], opts: argparse.Namespace) -> None: + pass + + +class TestCommandSettings: + def setup_method(self): + self.command = EmptyCommand() self.command.settings = Settings() self.parser = argparse.ArgumentParser( formatter_class=ScrapyHelpFormatter, conflict_handler="resolve" @@ -44,10 +43,8 @@ class CommandSettings(unittest.TestCase): args=["-s", f"FEEDS={feeds_json}", "spider.py"] ) self.command.process_options(args, opts) - self.assertIsInstance( - self.command.settings["FEEDS"], scrapy.settings.BaseSettings - ) - self.assertEqual(dict(self.command.settings["FEEDS"]), json.loads(feeds_json)) + assert isinstance(self.command.settings["FEEDS"], scrapy.settings.BaseSettings) + assert dict(self.command.settings["FEEDS"]) == json.loads(feeds_json) def test_help_formatter(self): formatter = ScrapyHelpFormatter(prog="scrapy") @@ -58,987 +55,315 @@ class CommandSettings(unittest.TestCase): "\n", "Global Options:\n", ] - self.assertEqual( - formatter._join_parts(part_strings), - ( - "Usage\n=====\n scrapy genspider [options] \n\n\n" - "Optional Arguments\n==================\n\n" - "Global Options\n--------------\n" - ), + assert formatter._join_parts(part_strings) == ( + "Usage\n=====\n scrapy genspider [options] \n\n\n" + "Optional Arguments\n==================\n\n" + "Global Options\n--------------\n" ) -class ProjectTest(unittest.TestCase): +class TestProjectBase: + """A base class for tests that may need a Scrapy project.""" + project_name = "testproject" - def setUp(self): - self.temp_path = mkdtemp() - self.cwd = self.temp_path - self.proj_path = Path(self.temp_path, self.project_name) - self.proj_mod_path = self.proj_path / self.project_name - self.env = get_testenv() + @pytest.fixture(scope="session") + def _proj_path_cached(self, tmp_path_factory: pytest.TempPathFactory) -> Path: + """Create a Scrapy project in a temporary directory and return its path. - def tearDown(self): - rmtree(self.temp_path) - - def call(self, *new_args, **kwargs): - with tempfile.TemporaryFile() as out: - args = (sys.executable, "-m", "scrapy.cmdline") + new_args - return subprocess.call( - args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **kwargs - ) - - def proc(self, *new_args, **popen_kwargs): - args = (sys.executable, "-m", "scrapy.cmdline") + new_args - p = subprocess.Popen( - args, - cwd=popen_kwargs.pop("cwd", self.cwd), - env=self.env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - **popen_kwargs, - ) - - def kill_proc(): - p.kill() - p.communicate() - assert False, "Command took too much time to complete" - - timer = Timer(15, kill_proc) - try: - timer.start() - stdout, stderr = p.communicate() - finally: - timer.cancel() - - return p, to_unicode(stdout), to_unicode(stderr) - - def find_in_file( - self, filename: Union[str, os.PathLike], regex - ) -> Optional[re.Match]: - """Find first pattern occurrence in file""" - pattern = re.compile(regex) - with Path(filename).open("r", encoding="utf-8") as f: - for line in f: - match = pattern.search(line) - if match is not None: - return match - return None - - -class StartprojectTest(ProjectTest): - def test_startproject(self): - p, out, err = self.proc("startproject", self.project_name) - print(out) - print(err, file=sys.stderr) - self.assertEqual(p.returncode, 0) - - assert Path(self.proj_path, "scrapy.cfg").exists() - assert Path(self.proj_path, "testproject").exists() - assert Path(self.proj_mod_path, "__init__.py").exists() - assert Path(self.proj_mod_path, "items.py").exists() - assert Path(self.proj_mod_path, "pipelines.py").exists() - assert Path(self.proj_mod_path, "settings.py").exists() - assert Path(self.proj_mod_path, "spiders", "__init__.py").exists() - - self.assertEqual(1, self.call("startproject", self.project_name)) - self.assertEqual(1, self.call("startproject", "wrong---project---name")) - self.assertEqual(1, self.call("startproject", "sys")) - - def test_startproject_with_project_dir(self): - project_dir = mkdtemp() - self.assertEqual(0, self.call("startproject", self.project_name, project_dir)) - - assert Path(project_dir, "scrapy.cfg").exists() - assert Path(project_dir, "testproject").exists() - assert Path(project_dir, self.project_name, "__init__.py").exists() - assert Path(project_dir, self.project_name, "items.py").exists() - assert Path(project_dir, self.project_name, "pipelines.py").exists() - assert Path(project_dir, self.project_name, "settings.py").exists() - assert Path(project_dir, self.project_name, "spiders", "__init__.py").exists() - - self.assertEqual( - 0, self.call("startproject", self.project_name, project_dir + "2") - ) - - self.assertEqual(1, self.call("startproject", self.project_name, project_dir)) - self.assertEqual( - 1, self.call("startproject", self.project_name + "2", project_dir) - ) - self.assertEqual(1, self.call("startproject", "wrong---project---name")) - self.assertEqual(1, self.call("startproject", "sys")) - self.assertEqual(2, self.call("startproject")) - self.assertEqual( - 2, - self.call("startproject", self.project_name, project_dir, "another_params"), - ) - - def test_existing_project_dir(self): - project_dir = mkdtemp() - project_name = self.project_name + "_existing" - project_path = Path(project_dir, project_name) - project_path.mkdir() - - p, out, err = self.proc("startproject", project_name, cwd=project_dir) - print(out) - print(err, file=sys.stderr) - self.assertEqual(p.returncode, 0) - - assert Path(project_path, "scrapy.cfg").exists() - assert Path(project_path, project_name).exists() - assert Path(project_path, project_name, "__init__.py").exists() - assert Path(project_path, project_name, "items.py").exists() - assert Path(project_path, project_name, "pipelines.py").exists() - assert Path(project_path, project_name, "settings.py").exists() - assert Path(project_path, project_name, "spiders", "__init__.py").exists() - - -def get_permissions_dict( - path: Union[str, os.PathLike], renamings=None, ignore=None -) -> Dict[str, str]: - def get_permissions(path: Path) -> str: - return oct(path.stat().st_mode) - - path_obj = Path(path) - - renamings = renamings or tuple() - permissions_dict = { - ".": get_permissions(path_obj), - } - for root, dirs, files in os.walk(path_obj): - nodes = list(chain(dirs, files)) - if ignore: - ignored_names = ignore(root, nodes) - nodes = [node for node in nodes if node not in ignored_names] - for node in nodes: - absolute_path = Path(root, node) - relative_path = str(absolute_path.relative_to(path)) - for search_string, replacement in renamings: - relative_path = relative_path.replace(search_string, replacement) - permissions = get_permissions(absolute_path) - permissions_dict[relative_path] = permissions - return permissions_dict - - -class StartprojectTemplatesTest(ProjectTest): - maxDiff = None - - def setUp(self): - super().setUp() - self.tmpl = str(Path(self.temp_path, "templates")) - self.tmpl_proj = str(Path(self.tmpl, "project")) - - def test_startproject_template_override(self): - copytree(Path(scrapy.__path__[0], "templates"), self.tmpl) - Path(self.tmpl_proj, "root_template").write_bytes(b"") - assert Path(self.tmpl_proj, "root_template").exists() - - args = ["--set", f"TEMPLATES_DIR={self.tmpl}"] - p, out, err = self.proc("startproject", self.project_name, *args) - self.assertIn( - f"New Scrapy project '{self.project_name}', " "using template directory", - out, - ) - self.assertIn(self.tmpl_proj, out) - assert Path(self.proj_path, "root_template").exists() - - def test_startproject_permissions_from_writable(self): - """Check that generated files have the right permissions when the - template folder has the same permissions as in the project, i.e. - everything is writable.""" - scrapy_path = scrapy.__path__[0] - project_template = Path(scrapy_path, "templates", "project") - project_name = "startproject1" - renamings = ( - ("module", project_name), - (".tmpl", ""), - ) - expected_permissions = get_permissions_dict( - project_template, - renamings, - IGNORE, - ) - - destination = mkdtemp() - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scrapy.cmdline", - "startproject", - project_name, - ), - cwd=destination, - env=self.env, - ) - process.wait() - - project_dir = Path(destination, project_name) - actual_permissions = get_permissions_dict(project_dir) - - self.assertEqual(actual_permissions, expected_permissions) - - def test_startproject_permissions_from_read_only(self): - """Check that generated files have the right permissions when the - template folder has been made read-only, which is something that some - systems do. - - See https://github.com/scrapy/scrapy/pull/4604 + Used as a cache for ``proj_path``. """ - scrapy_path = scrapy.__path__[0] - templates_dir = Path(scrapy_path, "templates") - project_template = Path(templates_dir, "project") - project_name = "startproject2" - renamings = ( - ("module", project_name), - (".tmpl", ""), - ) - expected_permissions = get_permissions_dict( - project_template, - renamings, - IGNORE, - ) + tmp_path = tmp_path_factory.mktemp("proj") + call("startproject", self.project_name, cwd=tmp_path) + return tmp_path / self.project_name - def _make_read_only(path: Path): - current_permissions = path.stat().st_mode - path.chmod(current_permissions & ~ANYONE_WRITE_PERMISSION) - - read_only_templates_dir = str(Path(mkdtemp()) / "templates") - copytree(templates_dir, read_only_templates_dir) - - for root, dirs, files in os.walk(read_only_templates_dir): - for node in chain(dirs, files): - _make_read_only(Path(root, node)) - - destination = mkdtemp() - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scrapy.cmdline", - "startproject", - project_name, - "--set", - f"TEMPLATES_DIR={read_only_templates_dir}", - ), - cwd=destination, - env=self.env, - ) - process.wait() - - project_dir = Path(destination, project_name) - actual_permissions = get_permissions_dict(project_dir) - - self.assertEqual(actual_permissions, expected_permissions) - - def test_startproject_permissions_unchanged_in_destination(self): - """Check that preexisting folders and files in the destination folder - do not see their permissions modified.""" - scrapy_path = scrapy.__path__[0] - project_template = Path(scrapy_path, "templates", "project") - project_name = "startproject3" - renamings = ( - ("module", project_name), - (".tmpl", ""), - ) - expected_permissions = get_permissions_dict( - project_template, - renamings, - IGNORE, - ) - - destination = mkdtemp() - project_dir = Path(destination, project_name) - - existing_nodes = { - oct(permissions)[2:] + extension: permissions - for extension in ("", ".d") - for permissions in ( - 0o444, - 0o555, - 0o644, - 0o666, - 0o755, - 0o777, - ) - } - project_dir.mkdir() - for node, permissions in existing_nodes.items(): - path = project_dir / node - if node.endswith(".d"): - path.mkdir(mode=permissions) - else: - path.touch(mode=permissions) - expected_permissions[node] = oct(path.stat().st_mode) - - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scrapy.cmdline", - "startproject", - project_name, - ".", - ), - cwd=project_dir, - env=self.env, - ) - process.wait() - - actual_permissions = get_permissions_dict(project_dir) - - self.assertEqual(actual_permissions, expected_permissions) - - def test_startproject_permissions_umask_022(self): - """Check that generated files have the right permissions when the - system uses a umask value that causes new files to have different - permissions than those from the template folder.""" - - @contextmanager - def umask(new_mask): - cur_mask = os.umask(new_mask) - yield - os.umask(cur_mask) - - scrapy_path = scrapy.__path__[0] - project_template = Path(scrapy_path, "templates", "project") - project_name = "umaskproject" - renamings = ( - ("module", project_name), - (".tmpl", ""), - ) - expected_permissions = get_permissions_dict( - project_template, - renamings, - IGNORE, - ) - - with umask(0o002): - destination = mkdtemp() - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scrapy.cmdline", - "startproject", - project_name, - ), - cwd=destination, - env=self.env, - ) - process.wait() - - project_dir = Path(destination, project_name) - actual_permissions = get_permissions_dict(project_dir) - - self.assertEqual(actual_permissions, expected_permissions) + @pytest.fixture + def proj_path(self, tmp_path: Path, _proj_path_cached: Path) -> Path: + """Copy a pre-generated Scrapy project into a temporary directory and return its path.""" + proj_path = tmp_path / self.project_name + copytree(_proj_path_cached, proj_path) + return proj_path -class CommandTest(ProjectTest): - def setUp(self): - super().setUp() - self.call("startproject", self.project_name) - self.cwd = Path(self.temp_path, self.project_name) - self.env["SCRAPY_SETTINGS_MODULE"] = f"{self.project_name}.settings" +class TestCommandCrawlerProcess(TestProjectBase): + """Test that the command uses the expected kind of *CrawlerProcess + and produces expected errors when needed.""" + name = "crawl" + NORMAL_MSG = "Using CrawlerProcess" + ASYNC_MSG = "Using AsyncCrawlerProcess" -class GenspiderCommandTest(CommandTest): - def test_arguments(self): - # only pass one argument. spider script shouldn't be created - self.assertEqual(2, self.call("genspider", "test_name")) - assert not Path(self.proj_mod_path, "spiders", "test_name.py").exists() - # pass two arguments . spider script should be created - self.assertEqual(0, self.call("genspider", "test_name", "test.com")) - assert Path(self.proj_mod_path, "spiders", "test_name.py").exists() - - def test_template(self, tplname="crawl"): - args = [f"--template={tplname}"] if tplname else [] - spname = "test_spider" - spmodule = f"{self.project_name}.spiders.{spname}" - p, out, err = self.proc("genspider", spname, "test.com", *args) - self.assertIn( - f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}", - out, - ) - self.assertTrue(Path(self.proj_mod_path, "spiders", "test_spider.py").exists()) - modify_time_before = ( - Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime - ) - p, out, err = self.proc("genspider", spname, "test.com", *args) - self.assertIn(f"Spider {spname!r} already exists in module", out) - modify_time_after = ( - Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime - ) - self.assertEqual(modify_time_after, modify_time_before) - - def test_template_basic(self): - self.test_template("basic") - - def test_template_csvfeed(self): - self.test_template("csvfeed") - - def test_template_xmlfeed(self): - self.test_template("xmlfeed") - - def test_list(self): - self.assertEqual(0, self.call("genspider", "--list")) - - def test_dump(self): - self.assertEqual(0, self.call("genspider", "--dump=basic")) - self.assertEqual(0, self.call("genspider", "-d", "basic")) - - def test_same_name_as_project(self): - self.assertEqual(2, self.call("genspider", self.project_name)) - assert not Path( - self.proj_mod_path, "spiders", f"{self.project_name}.py" - ).exists() - - def test_same_filename_as_existing_spider(self, force=False): - file_name = "example" - file_path = Path(self.proj_mod_path, "spiders", f"{file_name}.py") - self.assertEqual(0, self.call("genspider", file_name, "example.com")) - assert file_path.exists() - - # change name of spider but not its file name - with file_path.open("r+", encoding="utf-8") as spider_file: - file_data = spider_file.read() - file_data = file_data.replace('name = "example"', 'name = "renamed"') - spider_file.seek(0) - spider_file.write(file_data) - spider_file.truncate() - modify_time_before = file_path.stat().st_mtime - file_contents_before = file_data - - if force: - p, out, err = self.proc("genspider", "--force", file_name, "example.com") - self.assertIn( - f"Created spider {file_name!r} using template 'basic' in module", out - ) - modify_time_after = file_path.stat().st_mtime - self.assertNotEqual(modify_time_after, modify_time_before) - file_contents_after = file_path.read_text(encoding="utf-8") - self.assertNotEqual(file_contents_after, file_contents_before) - else: - p, out, err = self.proc("genspider", file_name, "example.com") - self.assertIn(f"{file_path.resolve()} already exists", out) - modify_time_after = file_path.stat().st_mtime - self.assertEqual(modify_time_after, modify_time_before) - file_contents_after = file_path.read_text(encoding="utf-8") - self.assertEqual(file_contents_after, file_contents_before) - - def test_same_filename_as_existing_spider_force(self): - self.test_same_filename_as_existing_spider(force=True) - - def test_url(self, url="test.com", domain="test.com"): - self.assertEqual(0, self.call("genspider", "--force", "test_name", url)) - self.assertEqual( - domain, - self.find_in_file( - Path(self.proj_mod_path, "spiders", "test_name.py"), - r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]", - ).group(1), - ) - self.assertEqual( - f"https://{domain}", - self.find_in_file( - Path(self.proj_mod_path, "spiders", "test_name.py"), - r"start_urls\s*=\s*\[['\"](.+)['\"]\]", - ).group(1), - ) - - def test_url_schema(self): - self.test_url("https://test.com", "test.com") - - def test_template_start_urls( - self, url="test.com", expected="https://test.com", template="basic" - ): - self.assertEqual( - 0, self.call("genspider", "-t", template, "--force", "test_name", url) - ) - self.assertEqual( - expected, - self.find_in_file( - Path(self.proj_mod_path, "spiders", "test_name.py"), - r"start_urls\s*=\s*\[['\"](.+)['\"]\]", - ).group(1), - ) - - def test_genspider_basic_start_urls(self): - self.test_template_start_urls("https://test.com", "https://test.com", "basic") - self.test_template_start_urls("http://test.com", "http://test.com", "basic") - self.test_template_start_urls( - "http://test.com/other/path", "http://test.com/other/path", "basic" - ) - self.test_template_start_urls( - "test.com/other/path", "https://test.com/other/path", "basic" - ) - - def test_genspider_crawl_start_urls(self): - self.test_template_start_urls("https://test.com", "https://test.com", "crawl") - self.test_template_start_urls("http://test.com", "http://test.com", "crawl") - self.test_template_start_urls( - "http://test.com/other/path", "http://test.com/other/path", "crawl" - ) - self.test_template_start_urls( - "test.com/other/path", "https://test.com/other/path", "crawl" - ) - self.test_template_start_urls("test.com", "https://test.com", "crawl") - - def test_genspider_xmlfeed_start_urls(self): - self.test_template_start_urls( - "https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" - ) - self.test_template_start_urls( - "http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed" - ) - self.test_template_start_urls( - "test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" - ) - - def test_genspider_csvfeed_start_urls(self): - self.test_template_start_urls( - "https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" - ) - self.test_template_start_urls( - "http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed" - ) - self.test_template_start_urls( - "test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" - ) - - -class GenspiderStandaloneCommandTest(ProjectTest): - def test_generate_standalone_spider(self): - self.call("genspider", "example", "example.com") - assert Path(self.temp_path, "example.py").exists() - - def test_same_name_as_existing_file(self, force=False): - file_name = "example" - file_path = Path(self.temp_path, file_name + ".py") - p, out, err = self.proc("genspider", file_name, "example.com") - self.assertIn(f"Created spider {file_name!r} using template 'basic' ", out) - assert file_path.exists() - modify_time_before = file_path.stat().st_mtime - file_contents_before = file_path.read_text(encoding="utf-8") - - if force: - # use different template to ensure contents were changed - p, out, err = self.proc( - "genspider", "--force", "-t", "crawl", file_name, "example.com" - ) - self.assertIn(f"Created spider {file_name!r} using template 'crawl' ", out) - modify_time_after = file_path.stat().st_mtime - self.assertNotEqual(modify_time_after, modify_time_before) - file_contents_after = file_path.read_text(encoding="utf-8") - self.assertNotEqual(file_contents_after, file_contents_before) - else: - p, out, err = self.proc("genspider", file_name, "example.com") - self.assertIn( - f"{Path(self.temp_path, file_name + '.py').resolve()} already exists", - out, - ) - modify_time_after = file_path.stat().st_mtime - self.assertEqual(modify_time_after, modify_time_before) - file_contents_after = file_path.read_text(encoding="utf-8") - self.assertEqual(file_contents_after, file_contents_before) - - def test_same_name_as_existing_file_force(self): - self.test_same_name_as_existing_file(force=True) - - -class MiscCommandsTest(CommandTest): - def test_list(self): - self.assertEqual(0, self.call("list")) - - -class RunSpiderCommandTest(CommandTest): - spider_filename = "myspider.py" - - debug_log_spider = """ + @pytest.fixture(autouse=True) + def create_files(self, proj_path: Path) -> None: + proj_mod_path = proj_path / self.project_name + (proj_mod_path / "spiders" / "sp.py").write_text(""" import scrapy class MySpider(scrapy.Spider): - name = 'myspider' + name = 'sp' - def start_requests(self): - self.logger.debug("It Works!") - return [] -""" + custom_settings = {} - badspider = """ -import scrapy + async def start(self): + self.logger.debug('It works!') + return + yield +""") -class BadSpider(scrapy.Spider): - name = "bad" - def start_requests(self): - raise Exception("oops!") - """ + (proj_mod_path / "spiders" / "aiosp.py").write_text(""" +import asyncio - @contextmanager - def _create_file(self, content, name=None) -> Generator[str, None, None]: - tmpdir = Path(self.mktemp()) - tmpdir.mkdir() - if name: - fname = (tmpdir / name).resolve() - else: - fname = (tmpdir / self.spider_filename).resolve() - fname.write_text(content, encoding="utf-8") - try: - yield str(fname) - finally: - rmtree(tmpdir) - - def runspider(self, code, name=None, args=()): - with self._create_file(code, name) as fname: - return self.proc("runspider", fname, *args) - - def get_log(self, code, name=None, args=()): - p, stdout, stderr = self.runspider(code, name, args=args) - return stderr - - def test_runspider(self): - log = self.get_log(self.debug_log_spider) - self.assertIn("DEBUG: It Works!", log) - self.assertIn("INFO: Spider opened", log) - self.assertIn("INFO: Closing spider (finished)", log) - self.assertIn("INFO: Spider closed (finished)", log) - - def test_run_fail_spider(self): - proc, _, _ = self.runspider( - "import scrapy\n" + inspect.getsource(ExceptionSpider) - ) - ret = proc.returncode - self.assertNotEqual(ret, 0) - - def test_run_good_spider(self): - proc, _, _ = self.runspider( - "import scrapy\n" + inspect.getsource(NoRequestsSpider) - ) - ret = proc.returncode - self.assertEqual(ret, 0) - - def test_runspider_log_level(self): - log = self.get_log(self.debug_log_spider, args=("-s", "LOG_LEVEL=INFO")) - self.assertNotIn("DEBUG: It Works!", log) - self.assertIn("INFO: Spider opened", log) - - def test_runspider_dnscache_disabled(self): - # see https://github.com/scrapy/scrapy/issues/2811 - # The spider below should not be able to connect to localhost:12345, - # which is intended, - # but this should not be because of DNS lookup error - # assumption: localhost will resolve in all cases (true?) - dnscache_spider = """ import scrapy class MySpider(scrapy.Spider): - name = 'myspider' - start_urls = ['http://localhost:12345'] + name = 'aiosp' - def parse(self, response): - return {'test': 'value'} -""" - log = self.get_log(dnscache_spider, args=("-s", "DNSCACHE_ENABLED=False")) - self.assertNotIn("DNSLookupError", log) - self.assertIn("INFO: Spider opened", log) + custom_settings = {} - def test_runspider_log_short_names(self): - log1 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=1")) - self.assertIn("[myspider] DEBUG: It Works!", log1) - self.assertIn("[scrapy]", log1) - self.assertNotIn("[scrapy.core.engine]", log1) + async def start(self): + await asyncio.sleep(0.01) + self.logger.debug('It works!') + return + yield +""") - log2 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=0")) - self.assertIn("[myspider] DEBUG: It Works!", log2) - self.assertNotIn("[scrapy]", log2) - self.assertIn("[scrapy.core.engine]", log2) + self._append_settings(proj_mod_path, "LOG_LEVEL = 'DEBUG'\n") - def test_runspider_no_spider_found(self): - log = self.get_log("from scrapy.spiders import Spider\n") - self.assertIn("No spider found in file", log) + @staticmethod + def _append_settings(proj_mod_path: Path, text: str) -> None: + """Add text to the end of the project settings.py.""" + with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f: + f.write(text) - def test_runspider_file_not_found(self): - _, _, log = self.proc("runspider", "some_non_existent_file") - self.assertIn("File not found: some_non_existent_file", log) + @staticmethod + def _replace_custom_settings( + proj_mod_path: Path, spider_name: str, text: str + ) -> None: + """Replace custom_settings in the given spider file with the given text.""" + spider_path = proj_mod_path / "spiders" / f"{spider_name}.py" + with spider_path.open("r+", encoding="utf-8") as f: + content = f.read() + content = content.replace( + "custom_settings = {}", f"custom_settings = {text}" + ) + f.seek(0) + f.write(content) + f.truncate() - def test_runspider_unable_to_load(self): - log = self.get_log("", name="myspider.txt") - self.assertIn("Unable to load", log) + def _assert_spider_works(self, msg: str, proj_path: Path, *args: str) -> None: + """The command uses the expected *CrawlerProcess, the spider works.""" + _, _, err = proc(self.name, *args, cwd=proj_path) + assert msg in err + assert "It works!" in err + assert "Spider closed (finished)" in err - def test_start_requests_errors(self): - log = self.get_log(self.badspider, name="badspider.py") - self.assertIn("start_requests", log) - self.assertIn("badspider.py", log) + def _assert_spider_asyncio_fail( + self, msg: str, proj_path: Path, *args: str + ) -> None: + """The command uses the expected *CrawlerProcess, the spider fails to use asyncio.""" + _, _, err = proc(self.name, *args, cwd=proj_path) + assert msg in err + assert "no running event loop" in err - def test_asyncio_enabled_true(self): - log = self.get_log( - self.debug_log_spider, - args=[ + def test_project_settings(self, proj_path: Path) -> None: + """The reactor is set via the project default settings (to the asyncio value). + + AsyncCrawlerProcess, the asyncio reactor, both spiders work.""" + for spider in ["sp", "aiosp"]: + self._assert_spider_works(self.ASYNC_MSG, proj_path, spider) + + def test_cmdline_asyncio(self, proj_path: Path) -> None: + """The reactor is set via the command line to the asyncio value. + AsyncCrawlerProcess, the asyncio reactor, both spiders work.""" + for spider in ["sp", "aiosp"]: + self._assert_spider_works( + self.ASYNC_MSG, + proj_path, + spider, "-s", - "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", - ], - ) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log - ) - - def test_asyncio_enabled_default(self): - log = self.get_log(self.debug_log_spider, args=[]) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log - ) - - def test_asyncio_enabled_false(self): - log = self.get_log( - self.debug_log_spider, - args=["-s", "TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor"], - ) - self.assertIn( - "Using reactor: twisted.internet.selectreactor.SelectReactor", log - ) - self.assertNotIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log - ) - - @mark.requires_uvloop - def test_custom_asyncio_loop_enabled_true(self): - log = self.get_log( - self.debug_log_spider, - args=[ - "-s", - "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "-s", - "ASYNCIO_EVENT_LOOP=uvloop.Loop", - ], - ) - self.assertIn("Using asyncio event loop: uvloop.Loop", log) - - def test_custom_asyncio_loop_enabled_false(self): - log = self.get_log( - self.debug_log_spider, - args=[ - "-s", - "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", - ], - ) - import asyncio - - if sys.platform != "win32": - loop = asyncio.new_event_loop() - else: - loop = asyncio.SelectorEventLoop() - self.assertIn( - f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", - log, - ) - - def test_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - def start_requests(self): - self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return [] -""" - args = ["-o", "example.json"] - log = self.get_log(spider_code, args=args) - self.assertIn( - "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log - ) - - def test_overwrite_output(self): - spider_code = """ -import json -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - def start_requests(self): - self.logger.debug( - 'FEEDS: {}'.format( - json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) + f"TWISTED_REACTOR={_asyncio_reactor_path}", ) + + def test_project_settings_explicit_asyncio(self, proj_path: Path) -> None: + """The reactor explicitly is set via the project settings to the asyncio value. + + AsyncCrawlerProcess, the asyncio reactor, both spiders work.""" + self._append_settings( + proj_path / self.project_name, + f"TWISTED_REACTOR = '{_asyncio_reactor_path}'\n", ) - return [] + + for spider in ["sp", "aiosp"]: + self._assert_spider_works(self.ASYNC_MSG, proj_path, spider) + + def test_cmdline_empty(self, proj_path: Path) -> None: + """The reactor is set via the command line to the empty value. + + CrawlerProcess, the default reactor, only the normal spider works.""" + self._assert_spider_works( + self.NORMAL_MSG, proj_path, "sp", "-s", "TWISTED_REACTOR=" + ) + self._assert_spider_asyncio_fail( + self.NORMAL_MSG, proj_path, "aiosp", "-s", "TWISTED_REACTOR=" + ) + + def test_project_settings_empty(self, proj_path: Path) -> None: + """The reactor is set via the project settings to the empty value. + + CrawlerProcess, the default reactor, only the normal spider works.""" + self._append_settings(proj_path / self.project_name, "TWISTED_REACTOR = None\n") + + self._assert_spider_works(self.NORMAL_MSG, proj_path, "sp") + self._assert_spider_asyncio_fail( + self.NORMAL_MSG, proj_path, "aiosp", "-s", "TWISTED_REACTOR=" + ) + + def test_spider_settings_asyncio(self, proj_path: Path) -> None: + """The reactor is set via the spider settings to the asyncio value. + + AsyncCrawlerProcess, the asyncio reactor, both spiders work.""" + for spider in ["sp", "aiosp"]: + self._replace_custom_settings( + proj_path / self.project_name, + spider, + f"{{'TWISTED_REACTOR': '{_asyncio_reactor_path}'}}", + ) + self._assert_spider_works(self.ASYNC_MSG, proj_path, spider) + + def test_spider_settings_asyncio_cmdline_empty(self, proj_path: Path) -> None: + """The reactor is set via the spider settings to the asyncio value + and via command line to the empty value. The command line value takes + precedence so the spider settings don't matter. + + CrawlerProcess, the default reactor, only the normal spider works.""" + for spider in ["sp", "aiosp"]: + self._replace_custom_settings( + proj_path / self.project_name, + spider, + f"{{'TWISTED_REACTOR': '{_asyncio_reactor_path}'}}", + ) + + self._assert_spider_works( + self.NORMAL_MSG, proj_path, "sp", "-s", "TWISTED_REACTOR=" + ) + self._assert_spider_asyncio_fail( + self.NORMAL_MSG, proj_path, "aiosp", "-s", "TWISTED_REACTOR=" + ) + + def test_project_empty_spider_settings_asyncio(self, proj_path: Path) -> None: + """The reactor is set via the project settings to the empty value + and via the spider settings to the asyncio value. CrawlerProcess is + chosen based on the project settings, but the asyncio reactor is chosen + based on the spider settings. + + CrawlerProcess, the asyncio reactor, both spiders work.""" + self._append_settings(proj_path / self.project_name, "TWISTED_REACTOR = None\n") + for spider in ["sp", "aiosp"]: + self._replace_custom_settings( + proj_path / self.project_name, + spider, + f"{{'TWISTED_REACTOR': '{_asyncio_reactor_path}'}}", + ) + self._assert_spider_works(self.NORMAL_MSG, proj_path, spider) + + def test_project_asyncio_spider_settings_select(self, proj_path: Path) -> None: + """The reactor is set via the project settings to the asyncio value + and via the spider settings to the select value. AsyncCrawlerProcess + is chosen based on the project settings, and the conflicting reactor + setting in the spider settings causes an exception. + + AsyncCrawlerProcess, the asyncio reactor, both spiders produce a + mismatched reactor exception.""" + self._append_settings( + proj_path / self.project_name, + f"TWISTED_REACTOR = '{_asyncio_reactor_path}'\n", + ) + for spider in ["sp", "aiosp"]: + self._replace_custom_settings( + proj_path / self.project_name, + spider, + "{'TWISTED_REACTOR': 'twisted.internet.selectreactor.SelectReactor'}", + ) + _, _, err = proc(self.name, spider, cwd=proj_path) + assert self.ASYNC_MSG in err + assert ( + "The installed reactor (twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + " does not match the requested one" + " (twisted.internet.selectreactor.SelectReactor)" + ) in err + + def test_project_asyncio_spider_settings_select_forced( + self, proj_path: Path + ) -> None: + """The reactor is set via the project settings to the asyncio value + and via the spider settings to the select value, CrawlerProcess is + forced via the project settings. The reactor is chosen based on the + spider settings. + + CrawlerProcess, the select reactor, only the normal spider works.""" + self._append_settings( + proj_path / self.project_name, "FORCE_CRAWLER_PROCESS = True\n" + ) + for spider in ["sp", "aiosp"]: + self._replace_custom_settings( + proj_path / self.project_name, + spider, + "{'TWISTED_REACTOR': 'twisted.internet.selectreactor.SelectReactor'}", + ) + + self._assert_spider_works(self.NORMAL_MSG, proj_path, "sp") + self._assert_spider_asyncio_fail(self.NORMAL_MSG, proj_path, "aiosp") + + +class TestMiscCommands(TestProjectBase): + def test_list(self, proj_path: Path) -> None: + assert call("list", cwd=proj_path) == 0 + + def test_list_subdir(self, proj_path: Path) -> None: + """Test that commands work in a subdirectory of the project.""" + subdir = proj_path / "subdir" + subdir.mkdir(exist_ok=True) + assert call("list", cwd=subdir) == 0 + + def test_command_not_found(self) -> None: + na_msg = """ +The list command is not available from this location. +These commands are only available from within a project: check, crawl, edit, list, parse. """ - Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") - args = ["-O", "example.json"] - log = self.get_log(spider_code, args=args) - self.assertIn( - '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', - log, - ) - with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: - first_line = f2.readline() - self.assertNotEqual(first_line, "not empty") - - def test_output_and_overwrite_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - def start_requests(self): - return [] + not_found_msg = """ +Unknown command: abc """ - args = ["-o", "example1.json", "-O", "example2.json"] - log = self.get_log(spider_code, args=args) - self.assertIn( - "error: Please use only one of -o/--output and -O/--overwrite-output", log - ) - - def test_output_stdout(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - def start_requests(self): - self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return [] -""" - args = ["-o", "-:json"] - log = self.get_log(spider_code, args=args) - self.assertIn("[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}", log) - - @skipIf(platform.system() == "Windows", reason="Linux only") - def test_absolute_path_linux(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - start_urls = ["data:,"] - - def parse(self, response): - yield {"hello": "world"} - """ - temp_dir = mkdtemp() - - args = ["-o", f"{temp_dir}/output1.json:json"] - log = self.get_log(spider_code, args=args) - self.assertIn( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json", - log, - ) - - args = ["-o", f"{temp_dir}/output2.json"] - log = self.get_log(spider_code, args=args) - self.assertIn( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json", - log, - ) - - @skipIf(platform.system() != "Windows", reason="Windows only") - def test_absolute_path_windows(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - start_urls = ["data:,"] - - def parse(self, response): - yield {"hello": "world"} - """ - temp_dir = mkdtemp() - - args = ["-o", f"{temp_dir}\\output1.json:json"] - log = self.get_log(spider_code, args=args) - self.assertIn( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json", - log, - ) - - args = ["-o", f"{temp_dir}\\output2.json"] - log = self.get_log(spider_code, args=args) - self.assertIn( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json", - log, - ) - - def test_args_change_settings(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - @classmethod - def from_crawler(cls, crawler, *args, **kwargs): - spider = super().from_crawler(crawler, *args, **kwargs) - spider.settings.set("FOO", kwargs.get("foo")) - return spider - - def start_requests(self): - self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") - return [] -""" - args = ["-a", "foo=42"] - log = self.get_log(spider_code, args=args) - self.assertIn("Spider closed (finished)", log) - self.assertIn("The value of FOO is 42", log) + params = [ + ("list", False, na_msg), + ("abc", False, not_found_msg), + ("abc", True, not_found_msg), + ] + for cmdname, inproject, message in params: + with mock.patch("sys.stdout", new=StringIO()) as out: + _print_unknown_command_msg(Settings(), cmdname, inproject) + assert out.getvalue().strip() == message.strip() -@skipIf(platform.system() != "Windows", "Windows required for .pyw files") -class WindowsRunSpiderCommandTest(RunSpiderCommandTest): - spider_filename = "myspider.pyw" - - def setUp(self): - super().setUp() - - def test_start_requests_errors(self): - log = self.get_log(self.badspider, name="badspider.pyw") - self.assertIn("start_requests", log) - self.assertIn("badspider.pyw", log) - - def test_run_good_spider(self): - super().test_run_good_spider() - - def test_runspider(self): - super().test_runspider() - - def test_runspider_dnscache_disabled(self): - super().test_runspider_dnscache_disabled() - - def test_runspider_log_level(self): - super().test_runspider_log_level() - - def test_runspider_log_short_names(self): - super().test_runspider_log_short_names() - - def test_runspider_no_spider_found(self): - super().test_runspider_no_spider_found() - - def test_output(self): - super().test_output() - - def test_overwrite_output(self): - super().test_overwrite_output() - - def test_runspider_unable_to_load(self): - raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") +class TestBenchCommand: + @pytest.mark.parametrize("use_reactor", [True, False]) + def test_run(self, use_reactor: bool) -> None: + args: list[str] = [ + "bench", + "-s", + "LOGSTATS_INTERVAL=0.001", + "-s", + "CLOSESPIDER_TIMEOUT=0.01", + ] + if not use_reactor: + args += ["-s", "TWISTED_REACTOR_ENABLED=False"] + _, _, err = proc(*args) + assert "INFO: Crawled" in err + assert "Unhandled Error" not in err + assert "log_count/ERROR" not in err -class BenchCommandTest(CommandTest): - def test_run(self): - _, _, log = self.proc( - "bench", "-s", "LOGSTATS_INTERVAL=0.001", "-s", "CLOSESPIDER_TIMEOUT=0.01" - ) - self.assertIn("INFO: Crawled", log) - self.assertNotIn("Unhandled Error", log) - - -class ViewCommandTest(CommandTest): - def test_methods(self): +class TestViewCommand: + def test_methods(self) -> None: command = view.Command() command.settings = Settings() parser = argparse.ArgumentParser( @@ -1048,102 +373,14 @@ class ViewCommandTest(CommandTest): conflict_handler="resolve", ) command.add_options(parser) - self.assertEqual(command.short_desc(), "Open URL in browser, as seen by Scrapy") - self.assertIn( - "URL using the Scrapy downloader and show its", command.long_desc() - ) + assert command.short_desc() == "Open URL in browser, as seen by Scrapy" + assert "URL using the Scrapy downloader and show its" in command.long_desc() -class CrawlCommandTest(CommandTest): - def crawl(self, code, args=()): - Path(self.proj_mod_path, "spiders", "myspider.py").write_text( - code, encoding="utf-8" - ) - return self.proc("crawl", "myspider", *args) - - def get_log(self, code, args=()): - _, _, stderr = self.crawl(code, args=args) - return stderr - - def test_no_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - def start_requests(self): - self.logger.debug('It works!') - return [] -""" - log = self.get_log(spider_code) - self.assertIn("[myspider] DEBUG: It works!", log) - - def test_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - def start_requests(self): - self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return [] -""" - args = ["-o", "example.json"] - log = self.get_log(spider_code, args=args) - self.assertIn( - "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log - ) - - def test_overwrite_output(self): - spider_code = """ -import json -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - def start_requests(self): - self.logger.debug( - 'FEEDS: {}'.format( - json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) - ) - ) - return [] -""" - Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") - args = ["-O", "example.json"] - log = self.get_log(spider_code, args=args) - self.assertIn( - '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', - log, - ) - with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: - first_line = f2.readline() - self.assertNotEqual(first_line, "not empty") - - def test_output_and_overwrite_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - def start_requests(self): - return [] -""" - args = ["-o", "example1.json", "-O", "example2.json"] - log = self.get_log(spider_code, args=args) - self.assertIn( - "error: Please use only one of -o/--output and -O/--overwrite-output", log - ) - - -class HelpMessageTest(CommandTest): - def setUp(self): - super().setUp() - self.commands = [ +class TestHelpMessage(TestProjectBase): + @pytest.mark.parametrize( + "command", + [ "parse", "startproject", "view", @@ -1158,9 +395,34 @@ class HelpMessageTest(CommandTest): "genspider", "check", "bench", - ] + ], + ) + def test_help_messages(self, proj_path: Path, command: str) -> None: + _, out, _ = proc(command, "-h", cwd=proj_path) + assert "Usage" in out - def test_help_messages(self): - for command in self.commands: - _, out, _ = self.proc(command, "-h") - self.assertIn("Usage", out) + +class TestPopCommandName: + def test_valid_command(self) -> None: + argv = ["scrapy", "crawl", "my_spider"] + command = _pop_command_name(argv) + assert command == "crawl" + assert argv == ["scrapy", "my_spider"] + + def test_no_command(self) -> None: + argv = ["scrapy"] + command = _pop_command_name(argv) + assert command is None + assert argv == ["scrapy"] + + def test_option_before_command(self) -> None: + argv = ["scrapy", "-h", "crawl"] + command = _pop_command_name(argv) + assert command == "crawl" + assert argv == ["scrapy", "-h"] + + def test_option_after_command(self) -> None: + argv = ["scrapy", "crawl", "-h"] + command = _pop_command_name(argv) + assert command == "crawl" + assert argv == ["scrapy", "-h"] diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 1459e0b5f..008e326ec 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,13 +1,12 @@ from unittest import TextTestResult -from twisted.internet import defer +import pytest from twisted.python import failure -from twisted.trial import unittest -from scrapy import FormRequest from scrapy.contracts import Contract, ContractsManager from scrapy.contracts.default import ( CallbackKeywordArgumentsContract, + MetadataContract, ReturnsContract, ScrapesContract, UrlContract, @@ -17,10 +16,11 @@ from scrapy.item import Field, Item from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer +from tests.mockserver.http import MockServer +from tests.utils.decorators import inline_callbacks_test -class TestItem(Item): +class DemoItem(Item): name = Field() url = Field() @@ -29,6 +29,16 @@ class ResponseMock: url = "http://scrapy.org" +class ResponseMetaMock(ResponseMock): + meta = None + + +class TaggedRequest(Request): + def __init__(self, url, contract_tag=None, **kwargs): + super().__init__(url, **kwargs) + self.contract_tag = contract_tag + + class CustomSuccessContract(Contract): name = "custom_success_contract" @@ -44,16 +54,17 @@ class CustomFailContract(Contract): raise TypeError("Error in adjust_request_args") -class CustomFormContract(Contract): - name = "custom_form" - request_cls = FormRequest +class CustomTaggedRequestContract(Contract): + name = "custom_tagged_request" + request_cls = TaggedRequest def adjust_request_args(self, args): - args["formdata"] = {"name": "scrapy"} + args["contract_tag"] = "custom" + args["method"] = "POST" return args -class TestSpider(Spider): +class DemoSpider(Spider): name = "demo_spider" def returns_request(self, response): @@ -75,7 +86,7 @@ class TestSpider(Spider): @url http://scrapy.org @returns items 1 1 """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def returns_request_cb_kwargs(self, response, url): """method which returns request @@ -91,7 +102,7 @@ class TestSpider(Spider): @cb_kwargs {"name": "Scrapy"} @returns items 1 1 """ - return TestItem(name=name, url=response.url) + return DemoItem(name=name, url=response.url) def returns_item_cb_kwargs_error_unexpected_keyword(self, response): """method which returns item @@ -99,14 +110,14 @@ class TestSpider(Spider): @cb_kwargs {"arg": "value"} @returns items 1 1 """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def returns_item_cb_kwargs_error_missing_argument(self, response, arg): """method which returns item @url http://scrapy.org @returns items 1 1 """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def returns_dict_item(self, response): """method which returns item @@ -120,7 +131,7 @@ class TestSpider(Spider): @url http://scrapy.org @returns items 0 0 """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def returns_dict_fail(self, response): """method which returns item @@ -135,7 +146,7 @@ class TestSpider(Spider): @returns items 1 1 @scrapes name url """ - return TestItem(name="test", url=response.url) + return DemoItem(name="test", url=response.url) def scrapes_dict_item_ok(self, response): """returns item with name and url @@ -151,7 +162,7 @@ class TestSpider(Spider): @returns items 1 1 @scrapes name url """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def scrapes_dict_item_fail(self, response): """returns item with no name @@ -173,14 +184,50 @@ class TestSpider(Spider): """method with no url @returns items 1 1 """ - pass - def custom_form(self, response): + def custom_tagged_request(self, response): """ @url http://scrapy.org - @custom_form + @custom_tagged_request """ - pass + + def invalid_regex(self, response): + """method with invalid regex + @ Scrapy is awsome + """ + + def invalid_regex_with_valid_contract(self, response): + """method with invalid regex + @ scrapy is awsome + @url http://scrapy.org + """ + + def returns_request_meta(self, response): + """method which returns request + @url https://example.org + @meta {"cookiejar": "session1"} + @returns requests 1 + """ + return Request( + "https://example.org", meta=response.meta, callback=self.returns_item_meta + ) + + def returns_item_meta(self, response): + """method which returns item + @url http://scrapy.org + @meta {"key": "example"} + @returns items 1 1 + """ + return DemoItem(name="example", url=response.url) + + def returns_error_missing_meta(self, response): + """method which depends of metadata be defined + + @url http://scrapy.org + @returns items 1 + """ + key = response.meta["key"] + yield {key: "value"} class CustomContractSuccessSpider(Spider): @@ -190,7 +237,6 @@ class CustomContractSuccessSpider(Spider): """ @custom_success_contract """ - pass class CustomContractFailSpider(Spider): @@ -200,93 +246,88 @@ class CustomContractFailSpider(Spider): """ @custom_fail_contract """ - pass -class InheritsTestSpider(TestSpider): +class InheritsDemoSpider(DemoSpider): name = "inherits_demo_spider" -class ContractsManagerTest(unittest.TestCase): +class TestContractsManager: contracts = [ UrlContract, CallbackKeywordArgumentsContract, + MetadataContract, ReturnsContract, ScrapesContract, - CustomFormContract, + CustomTaggedRequestContract, CustomSuccessContract, CustomFailContract, ] - def setUp(self): + def setup_method(self): self.conman = ContractsManager(self.contracts) self.results = TextTestResult(stream=None, descriptions=False, verbosity=0) def should_succeed(self): - self.assertFalse(self.results.failures) - self.assertFalse(self.results.errors) + assert not self.results.failures + assert not self.results.errors def should_fail(self): - self.assertTrue(self.results.failures) - self.assertFalse(self.results.errors) + assert self.results.failures + assert not self.results.errors def should_error(self): - self.assertTrue(self.results.errors) + assert self.results.errors def test_contracts(self): - spider = TestSpider() + spider = DemoSpider() # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request) - self.assertEqual(len(contracts), 2) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, ReturnsContract]), + assert len(contracts) == 2 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, ReturnsContract] ) # returns request for valid method request = self.conman.from_method(spider.returns_request, self.results) - self.assertNotEqual(request, None) + assert request is not None # no request for missing url request = self.conman.from_method(spider.parse_no_url, self.results) - self.assertEqual(request, None) + assert request is None def test_cb_kwargs(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request_cb_kwargs) - self.assertEqual(len(contracts), 3) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]), + assert len(contracts) == 3 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, CallbackKeywordArgumentsContract, ReturnsContract] ) contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs) - self.assertEqual(len(contracts), 3) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]), + assert len(contracts) == 3 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, CallbackKeywordArgumentsContract, ReturnsContract] ) contracts = self.conman.extract_contracts( spider.returns_item_cb_kwargs_error_unexpected_keyword ) - self.assertEqual(len(contracts), 3) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]), + assert len(contracts) == 3 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, CallbackKeywordArgumentsContract, ReturnsContract] ) contracts = self.conman.extract_contracts( spider.returns_item_cb_kwargs_error_missing_argument ) - self.assertEqual(len(contracts), 2) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, ReturnsContract]), + assert len(contracts) == 2 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, ReturnsContract] ) # returns_request @@ -315,8 +356,52 @@ class ContractsManagerTest(unittest.TestCase): request.callback(response, **request.cb_kwargs) self.should_error() + def test_meta(self): + spider = DemoSpider() + + # extract contracts correctly + contracts = self.conman.extract_contracts(spider.returns_request_meta) + assert len(contracts) == 3 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, MetadataContract, ReturnsContract] + ) + + contracts = self.conman.extract_contracts(spider.returns_item_meta) + assert len(contracts) == 3 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, MetadataContract, ReturnsContract] + ) + + response = ResponseMetaMock() + + # returns_request + request = self.conman.from_method(spider.returns_request_meta, self.results) + assert request.meta["cookiejar"] == "session1" + response.meta = request.meta + request.callback(response) + assert response.meta["cookiejar"] == "session1" + self.should_succeed() + + response = ResponseMetaMock() + + # returns_item + request = self.conman.from_method(spider.returns_item_meta, self.results) + assert request.meta["key"] == "example" + response.meta = request.meta + request.callback(ResponseMetaMock) + assert response.meta["key"] == "example" + self.should_succeed() + + response = ResponseMetaMock() + + request = self.conman.from_method( + spider.returns_error_missing_meta, self.results + ) + request.callback(response) + self.should_error() + def test_returns(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() # returns_item @@ -345,7 +430,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_fail() def test_returns_async(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() request = self.conman.from_method(spider.returns_request_async, self.results) @@ -353,7 +438,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_error() def test_scrapes(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() # scrapes_item_ok @@ -385,6 +470,21 @@ class ContractsManagerTest(unittest.TestCase): message = "ContractFail: Missing fields: name, url" assert message in self.results.failures[-1][-1] + def test_regex(self): + spider = DemoSpider() + response = ResponseMock() + + # invalid regex + request = self.conman.from_method(spider.invalid_regex, self.results) + self.should_succeed() + + # invalid regex with valid contract + request = self.conman.from_method( + spider.invalid_regex_with_valid_contract, self.results + ) + self.should_succeed() + request.callback(response) + def test_custom_contracts(self): self.conman.from_spider(CustomContractSuccessSpider(), self.results) self.should_succeed() @@ -393,7 +493,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_error() def test_errback(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() try: @@ -404,10 +504,10 @@ class ContractsManagerTest(unittest.TestCase): request = self.conman.from_method(spider.returns_request, self.results) request.errback(failure_mock) - self.assertFalse(self.results.failures) - self.assertTrue(self.results.errors) + assert not self.results.failures + assert self.results.errors - @defer.inlineCallbacks + @inline_callbacks_test def test_same_url(self): class TestSameUrlSpider(Spider): name = "test_same_url" @@ -416,19 +516,20 @@ class ContractsManagerTest(unittest.TestCase): super().__init__(*args, **kwargs) self.visited = 0 - def start_requests(s): - return self.conman.from_spider(s, self.results) + async def start(self_): # pylint: disable=no-self-argument + for item_or_request in self.conman.from_spider(self_, self.results): + yield item_or_request def parse_first(self, response): self.visited += 1 - return TestItem() + return DemoItem() def parse_second(self, response): self.visited += 1 - return TestItem() + return DemoItem() with MockServer() as mockserver: - contract_doc = f'@url {mockserver.url("/status?n=200")}' + contract_doc = f"@url {mockserver.url('/status?n=200')}" TestSameUrlSpider.parse_first.__doc__ = contract_doc TestSameUrlSpider.parse_second.__doc__ = contract_doc @@ -436,16 +537,67 @@ class ContractsManagerTest(unittest.TestCase): crawler = get_crawler(TestSameUrlSpider) yield crawler.crawl() - self.assertEqual(crawler.spider.visited, 2) + assert crawler.spider.visited == 2 - def test_form_contract(self): - spider = TestSpider() - request = self.conman.from_method(spider.custom_form, self.results) - self.assertEqual(request.method, "POST") - self.assertIsInstance(request, FormRequest) + def test_custom_tagged_request_contract(self): + spider = DemoSpider() + request = self.conman.from_method(spider.custom_tagged_request, self.results) + assert request.method == "POST" + assert isinstance(request, TaggedRequest) + assert request.contract_tag == "custom" def test_inherited_contracts(self): - spider = InheritsTestSpider() + spider = InheritsDemoSpider() requests = self.conman.from_spider(spider, self.results) - self.assertTrue(requests) + assert requests + assert any( + isinstance(request, TaggedRequest) for request in requests if request + ) + + +class CustomFailContractPreProcess(Contract): + name = "test_contract" + + def pre_process(self, response): + raise KeyboardInterrupt("Pre-process exception") + + +class CustomFailContractPostProcess(Contract): + name = "test_contract" + + def post_process(self, response): + raise KeyboardInterrupt("Post-process exception") + + +class TestCustomContractPrePostProcess: + def setup_method(self): + self.results = TextTestResult(stream=None, descriptions=False, verbosity=0) + + def test_pre_hook_keyboard_interrupt(self): + spider = DemoSpider() + response = ResponseMock() + contract = CustomFailContractPreProcess(spider.returns_request) + conman = ContractsManager([contract]) + + request = conman.from_method(spider.returns_request, self.results) + contract.add_pre_hook(request, self.results) + with pytest.raises(KeyboardInterrupt, match="Pre-process exception"): + request.callback(response, **request.cb_kwargs) + + assert not self.results.failures + assert not self.results.errors + + def test_post_hook_keyboard_interrupt(self): + spider = DemoSpider() + response = ResponseMock() + contract = CustomFailContractPostProcess(spider.returns_request) + conman = ContractsManager([contract]) + + request = conman.from_method(spider.returns_request, self.results) + contract.add_post_hook(request, self.results) + with pytest.raises(KeyboardInterrupt, match="Post-process exception"): + request.callback(response, **request.cb_kwargs) + + assert not self.results.failures + assert not self.results.errors diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 9a6e9e4ff..abeaa2f65 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -1,11 +1,283 @@ -from twisted.trial import unittest +from __future__ import annotations -from scrapy.core.downloader import Slot +import warnings +from typing import TYPE_CHECKING, cast + +import OpenSSL.SSL +import pytest +from pytest_twisted import async_yield_fixture +from twisted.internet.protocol import Factory +from twisted.internet.protocol import Protocol as TxProtocol +from twisted.internet.ssl import optionsForClientTLS +from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol +from twisted.web import server, static +from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody +from twisted.web.client import Response as TxResponse + +from scrapy.core.downloader import Downloader, Slot, tls +from scrapy.core.downloader.contextfactory import ( + _load_context_factory_from_settings, + _ScrapyClientContextFactory, +) +from scrapy.core.downloader.handlers.http11 import _RequestBodyProducer +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils._deps_compat import ( + PYOPENSSL_SET_CIPHER_LIST_TMP_CONN, + TWISTED_TLS_NEW_IMPL, +) +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.python import to_bytes +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.test import get_crawler +from tests.mockserver.http_resources import PayloadResource +from tests.mockserver.utils import ssl_context_factory +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from twisted.internet.defer import Deferred + from twisted.internet.ssl import ContextFactory + from twisted.web.iweb import IBodyProducer -class SlotTest(unittest.TestCase): +class TestSlot: def test_repr(self): slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) - self.assertEqual( - repr(slot), "Slot(concurrency=8, delay=0.10, randomize_delay=True)" + assert repr(slot) == "Slot(concurrency=8, delay=0.1, randomize_delay=True)" + + +@pytest.mark.requires_reactor # this test is related to the Twisted HTTP code +class TestContextFactoryBase: + context_factory: ContextFactory | None = None + + @async_yield_fixture + async def server_url(self, tmp_path): + (tmp_path / "file").write_bytes(b"0123456789") + r = static.File(str(tmp_path)) + r.putChild(b"payload", PayloadResource()) + site = server.Site(r, timeout=None) + port = self._listen(site) + portno = port.getHost().port + + yield f"https://127.0.0.1:{portno}/" + + await port.stopListening() + + def _listen(self, site): + from twisted.internet import reactor + + return reactor.listenSSL( + 0, + site, + contextFactory=self.context_factory or ssl_context_factory(), + interface="127.0.0.1", ) + + @staticmethod + async def get_page( + url: str, + client_context_factory: BrowserLikePolicyForHTTPS, + body: str | None = None, + ) -> bytes: + from twisted.internet import reactor + + agent = Agent(reactor, contextFactory=client_context_factory) + body_producer = _RequestBodyProducer(body.encode()) if body else None + response: TxResponse = cast( + "TxResponse", + await maybe_deferred_to_future( + agent.request( + b"GET", + url.encode(), + bodyProducer=cast("IBodyProducer", body_producer), + ) + ), + ) + with warnings.catch_warnings(): + # https://github.com/twisted/twisted/issues/8227 + warnings.filterwarnings( + "ignore", + category=DeprecationWarning, + message=r".*does not have an abortConnection method", + ) + d: Deferred[bytes] = readBody(response) # type: ignore[arg-type] + return await maybe_deferred_to_future(d) + + +class TestContextFactory(TestContextFactoryBase): + @pytest.fixture + def factory(self) -> _ScrapyClientContextFactory: + crawler = get_crawler() + return _load_context_factory_from_settings(crawler) + + @staticmethod + def _get_dummy_protocol() -> TLSMemoryBIOProtocol: + # from Twisted src/twisted/web/test/test_agent.py::dummyTLSProtocol() + factory = TLSMemoryBIOFactory( + optionsForClientTLS("example.com"), True, Factory.forProtocol(TxProtocol) + ) + return factory.buildProtocol(None) + + @coroutine_test + async def test_payload( + self, factory: _ScrapyClientContextFactory, server_url: str + ) -> None: + s = "0123456789" * 10 + body = await self.get_page(server_url + "payload", factory, body=s) + assert body == to_bytes(s) + + @pytest.mark.skipif( + TWISTED_TLS_NEW_IMPL, + reason="The context is not stored on this Twisted version", + ) + def test_no_context_sharing(self, factory: _ScrapyClientContextFactory) -> None: + """Every call to creatorForNetloc() should give a fresh context.""" + creator1 = factory.creatorForNetloc(b"website1.tld", 443) + assert creator1._hostnameBytes == b"website1.tld" + creator2 = factory.creatorForNetloc(b"website2.tld", 443) + assert creator2._hostnameBytes == b"website2.tld" + assert creator1._ctx is not creator2._ctx # type: ignore[attr-defined] + + def test_no_context_sharing_with_conn( + self, factory: _ScrapyClientContextFactory + ) -> None: + """Like test_no_context_sharing() but get the context from a connection.""" + creator1 = factory.creatorForNetloc(b"website1.tld", 443) + assert creator1._hostnameBytes == b"website1.tld" + conn1 = creator1.clientConnectionForTLS(self._get_dummy_protocol()) + + creator2 = factory.creatorForNetloc(b"website2.tld", 443) + assert creator2._hostnameBytes == b"website2.tld" + conn2 = creator2.clientConnectionForTLS(self._get_dummy_protocol()) + + assert conn1.get_context() is not conn2.get_context() + + @pytest.mark.skipif( + PYOPENSSL_SET_CIPHER_LIST_TMP_CONN, + reason="Fails or doesn't make sense on this pyOpenSSL version", + ) + def test_no_immutable_ctx_warning( + self, factory: _ScrapyClientContextFactory + ) -> None: + """There should be no pyOpenSSL context modification warning. + + pyOpenSSL < 25.1.0 doesn't produce this warning, and on 25.1.0 it's + always produced due to + https://github.com/scrapy/scrapy/issues/6859#issuecomment-4294917851. + """ + with warnings.catch_warnings(): + warnings.filterwarnings( + "error", + category=DeprecationWarning, + message="Attempting to mutate a Context after a Connection was created", + ) + factory.creatorForNetloc(b"website.tld", 443) + + def test_ctx_flags(self, factory: _ScrapyClientContextFactory) -> None: + """The context should have the expected flags set.""" + creator = factory.creatorForNetloc(b"website.tld", 443) + conn = creator.clientConnectionForTLS(self._get_dummy_protocol()) + ctx = conn.get_context() + # fragile but pyOpenSSL doesn't have Context.get_options() + options = OpenSSL.SSL._lib.SSL_CTX_get_options(ctx._context) # type: ignore[attr-defined] + assert options & 0x4 # OP_LEGACY_SERVER_CONNECT + + +class TestContextFactoryTLSMethod(TestContextFactoryBase): + async def _assert_factory_works( + self, server_url: str, client_context_factory: _ScrapyClientContextFactory + ) -> None: + s = "0123456789" * 10 + body = await self.get_page( + server_url + "payload", client_context_factory, body=s + ) + assert body == to_bytes(s) + + @coroutine_test + async def test_setting_default(self, server_url: str) -> None: + crawler = get_crawler() + client_context_factory = _load_context_factory_from_settings(crawler) + assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD + await self._assert_factory_works(server_url, client_context_factory) + + def test_setting_none(self): + crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": None}) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match="Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is deprecated", + ), + pytest.raises(KeyError), + ): + _load_context_factory_from_settings(crawler) + + def test_setting_bad(self): + crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match="Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is deprecated", + ), + pytest.raises(KeyError), + ): + _load_context_factory_from_settings(crawler) + + @pytest.mark.filterwarnings( + r"ignore:Passing method to twisted\.internet\.ssl\.CertificateOptions:DeprecationWarning" + ) + @coroutine_test + async def test_setting_explicit(self, server_url: str) -> None: + crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "TLSv1.2"}) + with pytest.warns( + ScrapyDeprecationWarning, + match="Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is deprecated", + ): + client_context_factory = _load_context_factory_from_settings(crawler) + assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD + await self._assert_factory_works(server_url, client_context_factory) + + @coroutine_test + async def test_direct_from_crawler(self, server_url: str) -> None: + # the setting is ignored + crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) + client_context_factory = build_from_crawler( + _ScrapyClientContextFactory, crawler + ) + assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD + await self._assert_factory_works(server_url, client_context_factory) + + @pytest.mark.filterwarnings( + r"ignore:Passing method to twisted\.internet\.ssl\.CertificateOptions:DeprecationWarning" + ) + @coroutine_test + async def test_direct_init(self, server_url: str) -> None: + client_context_factory = _ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD) + assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD + await self._assert_factory_works(server_url, client_context_factory) + + +@coroutine_test +async def test_fetch_deprecated_spider_arg(): + class CustomDownloader(Downloader): + def fetch(self, request, spider): # pylint: disable=signature-differs + return super().fetch(request, spider) + + crawler = get_crawler(DefaultSpider, {"DOWNLOADER": CustomDownloader}) + with pytest.warns( + ScrapyDeprecationWarning, + match=r"The fetch\(\) method of .+\.CustomDownloader requires a spider argument", + ): + await crawler.crawl_async() + + +def test_deprecated_tls_module_names() -> None: + with pytest.warns( + ScrapyDeprecationWarning, + match="scrapy.core.downloader.tls.METHOD_TLS is deprecated", + ): + assert tls.METHOD_TLS == "TLS" + with pytest.warns( + ScrapyDeprecationWarning, + match="scrapy.core.downloader.tls.openssl_methods is deprecated", + ): + assert isinstance(tls.openssl_methods, dict) diff --git a/tests/test_core_scraper.py b/tests/test_core_scraper.py new file mode 100644 index 000000000..1b8ba08ab --- /dev/null +++ b/tests/test_core_scraper.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from scrapy.utils.test import get_crawler +from tests.spiders import SimpleSpider +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + import pytest + + from tests.mockserver.http import MockServer + + +@coroutine_test +async def test_scraper_exception( + mockserver: MockServer, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + crawler = get_crawler(SimpleSpider) + monkeypatch.setattr( + "scrapy.core.engine.Scraper.handle_spider_output_async", + lambda *args, **kwargs: 1 / 0, + ) + await crawler.crawl_async(url=mockserver.url("/")) + assert "Scraper bug processing" in caplog.text diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 96d43b2b9..ada4c31ce 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,26 +1,28 @@ +from __future__ import annotations + import json import logging -import unittest from ipaddress import IPv4Address from socket import gethostbyname -from urllib.parse import urlparse +from typing import TYPE_CHECKING, Any +from urllib.parse import urlencode, urlparse -from pytest import mark -from testfixtures import LogCapture -from twisted.internet import defer +import pytest +from cryptography.x509 import load_der_x509_certificate +from twisted.internet.defer import succeed from twisted.internet.ssl import Certificate from twisted.python.failure import Failure -from twisted.trial.unittest import TestCase -from scrapy import signals -from scrapy.crawler import CrawlerRunner -from scrapy.exceptions import StopDownload +from scrapy import Spider, signals +from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner +from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload from scrapy.http import Request from scrapy.http.response import Response +from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future +from scrapy.utils.engine import format_engine_status, get_engine_status from scrapy.utils.python import to_unicode -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings from tests import NON_EXISTING_RESOLVABLE -from tests.mockserver import MockServer from tests.spiders import ( AsyncDefAsyncioGenComplexSpider, AsyncDefAsyncioGenExcSpider, @@ -34,7 +36,7 @@ from tests.spiders import ( AsyncDefDeferredMaybeWrappedSpider, AsyncDefDeferredWrappedSpider, AsyncDefSpider, - BrokenStartRequestsSpider, + BrokenStartSpider, BytesReceivedCallbackSpider, BytesReceivedErrbackSpider, CrawlSpiderWithAsyncCallback, @@ -43,55 +45,58 @@ from tests.spiders import ( CrawlSpiderWithParseMethod, CrawlSpiderWithProcessRequestCallbackKeywordArguments, DelaySpider, - DuplicateStartRequestsSpider, + DuplicateStartSpider, FollowAllSpider, HeadersReceivedCallbackSpider, HeadersReceivedErrbackSpider, SimpleSpider, SingleRequestSpider, + StartGoodAndBadOutput, + StartItemSpider, ) +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from scrapy.statscollectors import StatsCollector + from tests.mockserver.http import MockServer -class CrawlTestCase(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() - - def tearDown(self): - self.mockserver.__exit__(None, None, None) - - @defer.inlineCallbacks - def test_follow_all(self): +class TestCrawl: + @coroutine_test + async def test_follow_all(self, mockserver: MockServer) -> None: crawler = get_crawler(FollowAllSpider) - yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, FollowAllSpider) + assert len(crawler.spider.urls_visited) == 11 # 10 + start_url - @defer.inlineCallbacks - def test_fixed_delay(self): - yield self._test_delay(total=3, delay=0.2) + @coroutine_test + async def test_fixed_delay(self, mockserver: MockServer) -> None: + await self._test_delay(mockserver, total=3, delay=0.2) - @defer.inlineCallbacks - def test_randomized_delay(self): - yield self._test_delay(total=3, delay=0.1, randomize=True) + @coroutine_test + async def test_randomized_delay(self, mockserver: MockServer) -> None: + await self._test_delay(mockserver, total=3, delay=0.1, randomize=True) - @defer.inlineCallbacks - def _test_delay(self, total, delay, randomize=False): - crawl_kwargs = dict( - maxlatency=delay * 2, - mockserver=self.mockserver, - total=total, - ) + @staticmethod + async def _test_delay( + mockserver: MockServer, total: int, delay: float, randomize: bool = False + ) -> None: + crawl_kwargs = { + "maxlatency": delay * 2, + "mockserver": mockserver, + "total": total, + } tolerance = 1 - (0.6 if randomize else 0.2) settings = {"DOWNLOAD_DELAY": delay, "RANDOMIZE_DOWNLOAD_DELAY": randomize} crawler = get_crawler(FollowAllSpider, settings) - yield crawler.crawl(**crawl_kwargs) + await crawler.crawl_async(**crawl_kwargs) + assert crawler.spider + assert isinstance(crawler.spider, FollowAllSpider) times = crawler.spider.times total_time = times[-1] - times[0] average = total_time / (len(times) - 1) - self.assertTrue( - average > delay * tolerance, f"download delay too small: {average}" - ) + assert average > delay * tolerance, f"download delay too small: {average}" # Ensure that the same test parameters would cause a failure if no # download delay is set. Otherwise, it means we are using a combination @@ -99,125 +104,170 @@ class CrawlTestCase(TestCase): # code above to have any meaning. settings["DOWNLOAD_DELAY"] = 0 crawler = get_crawler(FollowAllSpider, settings) - yield crawler.crawl(**crawl_kwargs) + await crawler.crawl_async(**crawl_kwargs) + assert crawler.spider + assert isinstance(crawler.spider, FollowAllSpider) times = crawler.spider.times total_time = times[-1] - times[0] average = total_time / (len(times) - 1) - self.assertFalse( - average > delay / tolerance, "test total or delay values are too small" - ) + assert average <= delay / tolerance, "test total or delay values are too small" - @defer.inlineCallbacks - def test_timeout_success(self): + @coroutine_test + async def test_timeout_success(self, mockserver: MockServer) -> None: crawler = get_crawler(DelaySpider) - yield crawler.crawl(n=0.5, mockserver=self.mockserver) - self.assertTrue(crawler.spider.t1 > 0) - self.assertTrue(crawler.spider.t2 > 0) - self.assertTrue(crawler.spider.t2 > crawler.spider.t1) + await crawler.crawl_async(n=0.5, mockserver=mockserver) + assert isinstance(crawler.spider, DelaySpider) + assert crawler.spider.t1 > 0 + assert crawler.spider.t2 > 0 + assert crawler.spider.t2 > crawler.spider.t1 - @defer.inlineCallbacks - def test_timeout_failure(self): + @coroutine_test + async def test_timeout_failure(self, mockserver: MockServer) -> None: crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) - yield crawler.crawl(n=0.5, mockserver=self.mockserver) - self.assertTrue(crawler.spider.t1 > 0) - self.assertTrue(crawler.spider.t2 == 0) - self.assertTrue(crawler.spider.t2_err > 0) - self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) + await crawler.crawl_async(n=0.5, mockserver=mockserver) + assert isinstance(crawler.spider, DelaySpider) + assert crawler.spider.t1 > 0 + assert crawler.spider.t2 == 0 + assert crawler.spider.t2_err > 0 + assert crawler.spider.t2_err > crawler.spider.t1 # server hangs after receiving response headers crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) - yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver) - self.assertTrue(crawler.spider.t1 > 0) - self.assertTrue(crawler.spider.t2 == 0) - self.assertTrue(crawler.spider.t2_err > 0) - self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) + await crawler.crawl_async(n=0.5, b=1, mockserver=mockserver) + assert isinstance(crawler.spider, DelaySpider) + assert crawler.spider.t1 > 0 + assert crawler.spider.t2 == 0 + assert crawler.spider.t2_err > 0 + assert crawler.spider.t2_err > crawler.spider.t1 - @defer.inlineCallbacks - def test_retry_503(self): + @coroutine_test + async def test_retry_503( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=503"), mockserver=self.mockserver + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + mockserver.url("/status?n=503"), mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - @defer.inlineCallbacks - def test_retry_conn_failed(self): + @coroutine_test + async def test_retry_conn_failed( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - "http://localhost:65432/status?n=503", mockserver=self.mockserver + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + "http://localhost:65432/status?n=503", mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - @defer.inlineCallbacks - def test_retry_dns_error(self): + @coroutine_test + async def test_retry_dns_error( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: if NON_EXISTING_RESOLVABLE: - raise unittest.SkipTest("Non-existing hosts are resolvable") + pytest.skip("Non-existing hosts are resolvable") crawler = get_crawler(SimpleSpider) - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): # try to fetch the homepage of a nonexistent domain - yield crawler.crawl( - "http://dns.resolution.invalid./", mockserver=self.mockserver + await crawler.crawl_async( + "http://dns.resolution.invalid./", mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - @defer.inlineCallbacks - def test_start_requests_bug_before_yield(self): - with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(BrokenStartRequestsSpider) - yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) + @coroutine_test + async def test_start_bug_before_yield( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.ERROR): + crawler = get_crawler(BrokenStartSpider) + await crawler.crawl_async(fail_before_yield=1, mockserver=mockserver) - self.assertEqual(len(log.records), 1) - record = log.records[0] - self.assertIsNotNone(record.exc_info) - self.assertIs(record.exc_info[0], ZeroDivisionError) + assert len(caplog.records) == 1 + record = caplog.records[0] + assert record.exc_info is not None + assert record.exc_info[0] is ZeroDivisionError - @defer.inlineCallbacks - def test_start_requests_bug_yielding(self): - with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(BrokenStartRequestsSpider) - yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) + @coroutine_test + async def test_start_bug_yielding( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.ERROR): + crawler = get_crawler(BrokenStartSpider) + await crawler.crawl_async(fail_yielding=1, mockserver=mockserver) - self.assertEqual(len(log.records), 1) - record = log.records[0] - self.assertIsNotNone(record.exc_info) - self.assertIs(record.exc_info[0], ZeroDivisionError) + assert len(caplog.records) == 1 + record = caplog.records[0] + assert record.exc_info is not None + assert record.exc_info[0] is ZeroDivisionError - @defer.inlineCallbacks - def test_start_requests_laziness(self): + @coroutine_test + async def test_start_items( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + items = [] + + def _on_item_scraped(item): + items.append(item) + + with caplog.at_level(logging.ERROR): + crawler = get_crawler(StartItemSpider) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) + await crawler.crawl_async(mockserver=mockserver) + + assert len(caplog.records) == 0 + assert items == [{"name": "test item"}] + + @coroutine_test + async def test_start_unsupported_output( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + """Anything that is not a request is assumed to be an item, avoiding a + potentially expensive call to itemadapter.is_item(), and letting + instead things fail when ItemAdapter is actually used on the + corresponding non-item object.""" + + items = [] + + def _on_item_scraped(item): + items.append(item) + + with caplog.at_level(logging.ERROR): + crawler = get_crawler(StartGoodAndBadOutput) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) + await crawler.crawl_async(mockserver=mockserver) + + assert len(caplog.records) == 0 + assert len(items) == 3 + assert not any(isinstance(item, Request) for item in items) + + @coroutine_test + async def test_start_dupes(self, mockserver: MockServer) -> None: settings = {"CONCURRENT_REQUESTS": 1} - crawler = get_crawler(BrokenStartRequestsSpider, settings) - yield crawler.crawl(mockserver=self.mockserver) - self.assertTrue( - crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99), - crawler.spider.seedsseen, + crawler = get_crawler(DuplicateStartSpider, settings) + await crawler.crawl_async( + dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=mockserver ) + assert isinstance(crawler.spider, DuplicateStartSpider) + assert crawler.spider.visited == 6 - @defer.inlineCallbacks - def test_start_requests_dupes(self): - settings = {"CONCURRENT_REQUESTS": 1} - crawler = get_crawler(DuplicateStartRequestsSpider, settings) - yield crawler.crawl( - dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver - ) - self.assertEqual(crawler.spider.visited, 6) - - crawler = get_crawler(DuplicateStartRequestsSpider, settings) - yield crawler.crawl( + crawler = get_crawler(DuplicateStartSpider, settings) + await crawler.crawl_async( dont_filter=False, distinct_urls=3, dupe_factor=4, - mockserver=self.mockserver, + mockserver=mockserver, ) - self.assertEqual(crawler.spider.visited, 3) + assert isinstance(crawler.spider, DuplicateStartSpider) + assert crawler.spider.visited == 3 - @defer.inlineCallbacks - def test_unbounded_response(self): + @coroutine_test + async def test_unbounded_response( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: # Completeness of responses without Content-Length or Transfer-Encoding # can not be determined, we treat them as valid but flagged as "partial" - from urllib.parse import urlencode - query = urlencode( { "raw": """\ @@ -240,40 +290,45 @@ with multiples lines } ) crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + mockserver.url(f"/raw?{query}"), mockserver=mockserver ) - self.assertEqual(str(log).count("Got response 200"), 1) + assert caplog.text.count("Got response 200") == 1 - @defer.inlineCallbacks - def test_retry_conn_lost(self): + @coroutine_test + async def test_retry_conn_lost( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: # connection lost after receiving data crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + mockserver.url("/drop?abort=0"), mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - @defer.inlineCallbacks - def test_retry_conn_aborted(self): + @coroutine_test + async def test_retry_conn_aborted( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: # connection lost before receiving data crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + mockserver.url("/drop?abort=1"), mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - def _assert_retried(self, log): - self.assertEqual(str(log).count("Retrying"), 2) - self.assertEqual(str(log).count("Gave up retrying"), 1) + @staticmethod + def _assert_retried(log: str) -> None: + assert str(log).count("Retrying") == 2 + assert str(log).count("Gave up retrying") == 1 - @defer.inlineCallbacks - def test_referer_header(self): + @coroutine_test + async def test_referer_header(self, mockserver: MockServer) -> None: """Referer header is set by RefererMiddleware unless it is already set""" - req0 = Request(self.mockserver.url("/echo?headers=1&body=0"), dont_filter=1) + req0 = Request(mockserver.url("/echo?headers=1&body=0"), dont_filter=True) req1 = req0.replace() req2 = req0.replace(headers={"Referer": None}) req3 = req0.replace(headers={"Referer": "http://example.com"}) @@ -281,146 +336,154 @@ with multiples lines req1.meta["next"] = req2 req2.meta["next"] = req3 crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl(seed=req0, mockserver=self.mockserver) + await crawler.crawl_async(seed=req0, mockserver=mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) # basic asserts in case of weird communication errors - self.assertIn("responses", crawler.spider.meta) - self.assertNotIn("failures", crawler.spider.meta) - # start requests doesn't set Referer header + assert "responses" in crawler.spider.meta + assert "failures" not in crawler.spider.meta + # start() doesn't set Referer header echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) - self.assertNotIn("Referer", echo0["headers"]) - # following request sets Referer to start request url + assert "Referer" not in echo0["headers"] + # following request sets Referer to the source request url echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body)) - self.assertEqual(echo1["headers"].get("Referer"), [req0.url]) + assert echo1["headers"].get("Referer") == [req0.url] # next request avoids Referer header echo2 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) - self.assertNotIn("Referer", echo2["headers"]) + assert "Referer" not in echo2["headers"] # last request explicitly sets a Referer header echo3 = json.loads(to_unicode(crawler.spider.meta["responses"][3].body)) - self.assertEqual(echo3["headers"].get("Referer"), ["http://example.com"]) - - @defer.inlineCallbacks - def test_engine_status(self): - from scrapy.utils.engine import get_engine_status + assert echo3["headers"].get("Referer") == ["http://example.com"] + @coroutine_test + async def test_engine_status(self, mockserver: MockServer) -> None: est = [] def cb(response): est.append(get_engine_status(crawler.engine)) crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl( - seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver + await crawler.crawl_async( + seed=mockserver.url("/"), callback_func=cb, mockserver=mockserver ) - self.assertEqual(len(est), 1, est) + assert isinstance(crawler.spider, SingleRequestSpider) + assert len(est) == 1, est s = dict(est[0]) - self.assertEqual(s["engine.spider.name"], crawler.spider.name) - self.assertEqual(s["len(engine.scraper.slot.active)"], 1) - - @defer.inlineCallbacks - def test_format_engine_status(self): - from scrapy.utils.engine import format_engine_status + assert s["engine.spider.name"] == crawler.spider.name + assert s["len(engine.scraper.slot.active)"] == 1 + @coroutine_test + async def test_format_engine_status(self, mockserver: MockServer) -> None: est = [] def cb(response): est.append(format_engine_status(crawler.engine)) crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl( - seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver + await crawler.crawl_async( + seed=mockserver.url("/"), callback_func=cb, mockserver=mockserver ) - self.assertEqual(len(est), 1, est) + assert isinstance(crawler.spider, SingleRequestSpider) + assert len(est) == 1, est est = est[0].split("\n")[2:-2] # remove header & footer # convert to dict est = [x.split(":") for x in est] est = [x for sublist in est for x in sublist] # flatten est = [x.lstrip().rstrip() for x in est] it = iter(est) - s = dict(zip(it, it)) + s = dict(zip(it, it, strict=False)) - self.assertEqual(s["engine.spider.name"], crawler.spider.name) - self.assertEqual(s["len(engine.scraper.slot.active)"], "1") + assert s["engine.spider.name"] == crawler.spider.name + assert s["len(engine.scraper.slot.active)"] == "1" - @defer.inlineCallbacks - def test_graceful_crawl_error_handling(self): - """ - Test whether errors happening anywhere in Crawler.crawl() are properly - reported (and not somehow swallowed) after a graceful engine shutdown. - The errors should not come from within Scrapy's core but from within - spiders/middlewares/etc., e.g. raised in Spider.start_requests(), - SpiderMiddleware.process_start_requests(), etc. - """ - - class TestError(Exception): - pass - - class FaultySpider(SimpleSpider): - def start_requests(self): - raise TestError - - crawler = get_crawler(FaultySpider) - yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError) - self.assertFalse(crawler.crawling) - - @defer.inlineCallbacks - def test_open_spider_error_on_faulty_pipeline(self): + @coroutine_test + async def test_open_spider_error_on_faulty_pipeline( + self, mockserver: MockServer + ) -> None: settings = { "ITEM_PIPELINES": { "tests.pipelines.ZeroDivisionErrorPipeline": 300, } } crawler = get_crawler(SimpleSpider, settings) - yield self.assertFailure( - crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver - ), - ZeroDivisionError, - ) - self.assertFalse(crawler.crawling) + with pytest.raises(ZeroDivisionError): + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver + ) + assert not crawler.crawling - @defer.inlineCallbacks - def test_crawlerrunner_accepts_crawler(self): + @coroutine_test + async def test_open_spider_error_on_faulty_pipeline_crawl( + self, mockserver: MockServer + ) -> None: + # cover the except block in Crawler.crawl() + settings = { + "ITEM_PIPELINES": { + "tests.pipelines.ZeroDivisionErrorPipeline": 300, + } + } + crawler = get_crawler(SimpleSpider, settings) + with pytest.raises(ZeroDivisionError): + await maybe_deferred_to_future( + crawler.crawl(mockserver.url("/status?n=200"), mockserver=mockserver) + ) + assert not crawler.crawling + + @coroutine_test + async def test_crawlerrunner_accepts_crawler( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(SimpleSpider) runner = CrawlerRunner() - with LogCapture() as log: - yield runner.crawl( - crawler, - self.mockserver.url("/status?n=200"), - mockserver=self.mockserver, + with caplog.at_level(logging.DEBUG): + await maybe_deferred_to_future( + runner.crawl( + crawler, + mockserver.url("/status?n=200"), + mockserver=mockserver, + ) ) - self.assertIn("Got response 200", str(log)) + assert "Got response 200" in caplog.text - @defer.inlineCallbacks - def test_crawl_multiple(self): - runner = CrawlerRunner({"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7"}) + @coroutine_test + async def test_crawl_multiple( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + settings_dict = get_reactor_settings() + runner_cls = ( + CrawlerRunner + if settings_dict.get("TWISTED_REACTOR_ENABLED", True) + else AsyncCrawlerRunner + ) + runner = runner_cls(settings_dict) runner.crawl( SimpleSpider, - self.mockserver.url("/status?n=200"), - mockserver=self.mockserver, + mockserver.url("/status?n=200"), + mockserver=mockserver, ) runner.crawl( SimpleSpider, - self.mockserver.url("/status?n=503"), - mockserver=self.mockserver, + mockserver.url("/status?n=503"), + mockserver=mockserver, ) - with LogCapture() as log: - yield runner.join() + with caplog.at_level(logging.DEBUG): + await ensure_awaitable(runner.join()) - self._assert_retried(log) - self.assertIn("Got response 200", str(log)) + self._assert_retried(caplog.text) + assert "Got response 200" in caplog.text + + @coroutine_test + async def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None: + crawler = get_crawler(SimpleSpider) + await crawler.crawl_async("foo://bar") + assert "NotSupported: Unsupported URL scheme 'foo'" in caplog.text -class CrawlSpiderTestCase(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() - - def tearDown(self): - self.mockserver.__exit__(None, None, None) - - @defer.inlineCallbacks - def _run_spider(self, spider_cls): +class TestCrawlSpider: + @staticmethod + async def _run_spider( + spider_cls: type[Spider], mockserver: MockServer + ) -> tuple[list[Any], StatsCollector]: items = [] def _on_item_scraped(item): @@ -428,100 +491,119 @@ class CrawlSpiderTestCase(TestCase): crawler = get_crawler(spider_cls) crawler.signals.connect(_on_item_scraped, signals.item_scraped) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver - ) - return log, items, crawler.stats + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver + ) + assert crawler.stats + return items, crawler.stats - @defer.inlineCallbacks - def test_crawlspider_with_parse(self): + @coroutine_test + async def test_crawlspider_with_parse( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithParseMethod) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - self.assertIn("[parse] status 200 (foo: None)", str(log)) - self.assertIn("[parse] status 201 (foo: None)", str(log)) - self.assertIn("[parse] status 202 (foo: bar)", str(log)) + assert "[parse] status 200 (foo: None)" in caplog.text + assert "[parse] status 201 (foo: None)" in caplog.text + assert "[parse] status 202 (foo: bar)" in caplog.text - @defer.inlineCallbacks - def test_crawlspider_with_async_callback(self): + @coroutine_test + async def test_crawlspider_with_async_callback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithAsyncCallback) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - self.assertIn("[parse_async] status 200 (foo: None)", str(log)) - self.assertIn("[parse_async] status 201 (foo: None)", str(log)) - self.assertIn("[parse_async] status 202 (foo: bar)", str(log)) + assert "[parse_async] status 200 (foo: None)" in caplog.text + assert "[parse_async] status 201 (foo: None)" in caplog.text + assert "[parse_async] status 202 (foo: bar)" in caplog.text - @defer.inlineCallbacks - def test_crawlspider_with_async_generator_callback(self): + @coroutine_test + async def test_crawlspider_with_async_generator_callback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - self.assertIn("[parse_async_gen] status 200 (foo: None)", str(log)) - self.assertIn("[parse_async_gen] status 201 (foo: None)", str(log)) - self.assertIn("[parse_async_gen] status 202 (foo: bar)", str(log)) + assert "[parse_async_gen] status 200 (foo: None)" in caplog.text + assert "[parse_async_gen] status 201 (foo: None)" in caplog.text + assert "[parse_async_gen] status 202 (foo: bar)" in caplog.text - @defer.inlineCallbacks - def test_crawlspider_with_errback(self): + @coroutine_test + async def test_crawlspider_with_errback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithErrback) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - self.assertIn("[parse] status 200 (foo: None)", str(log)) - self.assertIn("[parse] status 201 (foo: None)", str(log)) - self.assertIn("[parse] status 202 (foo: bar)", str(log)) - self.assertIn("[errback] status 404", str(log)) - self.assertIn("[errback] status 500", str(log)) - self.assertIn("[errback] status 501", str(log)) + assert "[parse] status 200 (foo: None)" in caplog.text + assert "[parse] status 201 (foo: None)" in caplog.text + assert "[parse] status 202 (foo: bar)" in caplog.text + assert "[errback] status 404" in caplog.text + assert "[errback] status 500" in caplog.text + assert "[errback] status 501" in caplog.text - @defer.inlineCallbacks - def test_crawlspider_process_request_cb_kwargs(self): + @coroutine_test + async def test_crawlspider_process_request_cb_kwargs( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - self.assertIn("[parse] status 200 (foo: process_request)", str(log)) - self.assertIn("[parse] status 201 (foo: process_request)", str(log)) - self.assertIn("[parse] status 202 (foo: bar)", str(log)) + assert "[parse] status 200 (foo: process_request)" in caplog.text + assert "[parse] status 201 (foo: process_request)" in caplog.text + assert "[parse] status 202 (foo: bar)" in caplog.text - @defer.inlineCallbacks - def test_async_def_parse(self): + @coroutine_test + async def test_async_def_parse( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(AsyncDefSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver ) - self.assertIn("Got response 200", str(log)) + assert "Got response 200" in caplog.text - @mark.only_asyncio() - @defer.inlineCallbacks - def test_async_def_asyncio_parse(self): + @pytest.mark.only_asyncio + @coroutine_test + async def test_async_def_asyncio_parse( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler( AsyncDefAsyncioSpider, { "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor" }, ) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver ) - self.assertIn("Got response 200", str(log)) + assert "Got response 200" in caplog.text - @mark.only_asyncio() - @defer.inlineCallbacks - def test_async_def_asyncio_parse_items_list(self): - log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider) - self.assertIn("Got response 200", str(log)) - self.assertIn({"id": 1}, items) - self.assertIn({"id": 2}, items) + @pytest.mark.only_asyncio + @coroutine_test + async def test_async_def_asyncio_parse_items_list( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + items, _ = await self._run_spider(AsyncDefAsyncioReturnSpider, mockserver) + assert "Got response 200" in caplog.text + assert {"id": 1} in items + assert {"id": 2} in items - @mark.only_asyncio() - @defer.inlineCallbacks - def test_async_def_asyncio_parse_items_single_element(self): + @pytest.mark.only_asyncio + @coroutine_test + async def test_async_def_asyncio_parse_items_single_element( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: items = [] def _on_item_scraped(item): @@ -529,177 +611,432 @@ class CrawlSpiderTestCase(TestCase): crawler = get_crawler(AsyncDefAsyncioReturnSingleElementSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver ) - self.assertIn("Got response 200", str(log)) - self.assertIn({"foo": 42}, items) + assert "Got response 200" in caplog.text + assert {"foo": 42} in items - @mark.only_asyncio() - @defer.inlineCallbacks - def test_async_def_asyncgen_parse(self): - log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider) - self.assertIn("Got response 200", str(log)) + @pytest.mark.only_asyncio + @coroutine_test + async def test_async_def_asyncgen_parse( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + _, stats = await self._run_spider(AsyncDefAsyncioGenSpider, mockserver) + assert "Got response 200" in caplog.text itemcount = stats.get_value("item_scraped_count") - self.assertEqual(itemcount, 1) + assert itemcount == 1 - @mark.only_asyncio() - @defer.inlineCallbacks - def test_async_def_asyncgen_parse_loop(self): - log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider) - self.assertIn("Got response 200", str(log)) + @pytest.mark.only_asyncio + @coroutine_test + async def test_async_def_asyncgen_parse_loop( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + items, stats = await self._run_spider( + AsyncDefAsyncioGenLoopSpider, mockserver + ) + assert "Got response 200" in caplog.text itemcount = stats.get_value("item_scraped_count") - self.assertEqual(itemcount, 10) + assert itemcount == 10 for i in range(10): - self.assertIn({"foo": i}, items) + assert {"foo": i} in items - @mark.only_asyncio() - @defer.inlineCallbacks - def test_async_def_asyncgen_parse_exc(self): - log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider) - log = str(log) - self.assertIn("Spider error processing", log) - self.assertIn("ValueError", log) + @pytest.mark.only_asyncio + @coroutine_test + async def test_async_def_asyncgen_parse_exc( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + items, stats = await self._run_spider( + AsyncDefAsyncioGenExcSpider, mockserver + ) + assert "Spider error processing" in caplog.text + assert "ValueError" in caplog.text itemcount = stats.get_value("item_scraped_count") - self.assertEqual(itemcount, 7) + assert itemcount == 7 for i in range(7): - self.assertIn({"foo": i}, items) + assert {"foo": i} in items - @mark.only_asyncio() - @defer.inlineCallbacks - def test_async_def_asyncgen_parse_complex(self): - _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider) + @pytest.mark.only_asyncio + @coroutine_test + async def test_async_def_asyncgen_parse_complex( + self, mockserver: MockServer + ) -> None: + items, stats = await self._run_spider( + AsyncDefAsyncioGenComplexSpider, mockserver + ) itemcount = stats.get_value("item_scraped_count") - self.assertEqual(itemcount, 156) + assert itemcount == 156 # some random items for i in [1, 4, 21, 22, 207, 311]: - self.assertIn({"index": i}, items) + assert {"index": i} in items for i in [10, 30, 122]: - self.assertIn({"index2": i}, items) + assert {"index2": i} in items - @mark.only_asyncio() - @defer.inlineCallbacks - def test_async_def_asyncio_parse_reqs_list(self): - log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider) + @pytest.mark.only_asyncio + @coroutine_test + async def test_async_def_asyncio_parse_reqs_list( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + await self._run_spider(AsyncDefAsyncioReqsReturnSpider, mockserver) for req_id in range(3): - self.assertIn(f"Got response 200, req_id {req_id}", str(log)) + assert f"Got response 200, req_id {req_id}" in caplog.text - @mark.only_not_asyncio() - @defer.inlineCallbacks - def test_async_def_deferred_direct(self): - _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider) - self.assertEqual(items, [{"code": 200}]) + @pytest.mark.only_not_asyncio + @coroutine_test + async def test_async_def_deferred_direct(self, mockserver: MockServer) -> None: + items, _ = await self._run_spider(AsyncDefDeferredDirectSpider, mockserver) + assert items == [{"code": 200}] - @mark.only_asyncio() - @defer.inlineCallbacks - def test_async_def_deferred_wrapped(self): - log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider) - self.assertEqual(items, [{"code": 200}]) + @pytest.mark.only_asyncio + @coroutine_test + async def test_async_def_deferred_wrapped(self, mockserver: MockServer) -> None: + items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider, mockserver) + assert items == [{"code": 200}] - @defer.inlineCallbacks - def test_async_def_deferred_maybe_wrapped(self): - _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider) - self.assertEqual(items, [{"code": 200}]) + @coroutine_test + async def test_async_def_deferred_maybe_wrapped( + self, mockserver: MockServer + ) -> None: + items, _ = await self._run_spider( + AsyncDefDeferredMaybeWrappedSpider, mockserver + ) + assert items == [{"code": 200}] - @defer.inlineCallbacks - def test_response_ssl_certificate_none(self): + @coroutine_test + async def test_response_ssl_certificate_none(self, mockserver: MockServer) -> None: crawler = get_crawler(SingleRequestSpider) - url = self.mockserver.url("/echo?body=test", is_secure=False) - yield crawler.crawl(seed=url, mockserver=self.mockserver) - self.assertIsNone(crawler.spider.meta["responses"][0].certificate) + url = mockserver.url("/echo?body=test", is_secure=False) + await crawler.crawl_async(seed=url, mockserver=mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) + assert crawler.spider.meta["responses"][0].certificate is None - @defer.inlineCallbacks - def test_response_ssl_certificate(self): + @pytest.mark.parametrize( + "url", + [ + "/echo?body=test", + pytest.param( + "/status?n=200", + marks=pytest.mark.xfail( + 'config.getoption("--reactor") != "none"', + reason="With HTTP11DownloadHandler, responses with no body are returned early and contain no certificate", + strict=True, + ), + ), + ], + ) + @coroutine_test + async def test_response_ssl_certificate( + self, mockserver: MockServer, url: str + ) -> None: crawler = get_crawler(SingleRequestSpider) - url = self.mockserver.url("/echo?body=test", is_secure=True) - yield crawler.crawl(seed=url, mockserver=self.mockserver) + url = mockserver.url(url, is_secure=True) + await crawler.crawl_async(seed=url, mockserver=mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) cert = crawler.spider.meta["responses"][0].certificate - self.assertIsInstance(cert, Certificate) - self.assertEqual(cert.getSubject().commonName, b"localhost") - self.assertEqual(cert.getIssuer().commonName, b"localhost") + assert cert is not None + if isinstance(cert, Certificate): # Twisted + assert cert.getSubject().commonName == b"localhost" + assert cert.getIssuer().commonName == b"localhost" + elif isinstance(cert, bytes): # DER bytes + cert_x509 = load_der_x509_certificate(cert) + assert cert_x509.subject.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE" + assert cert_x509.issuer.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE" - @mark.xfail(reason="Responses with no body return early and contain no certificate") - @defer.inlineCallbacks - def test_response_ssl_certificate_empty_response(self): + @pytest.mark.parametrize( + "url", + [ + "/echo?body=test", + pytest.param( + "/status?n=200", + marks=pytest.mark.xfail( + 'config.getoption("--reactor") != "none"', + reason="With HTTP11DownloadHandler, responses with no body are returned early and contain no ip_address", + strict=True, + ), + ), + ], + ) + @coroutine_test + async def test_response_ip_address(self, mockserver: MockServer, url: str) -> None: crawler = get_crawler(SingleRequestSpider) - url = self.mockserver.url("/status?n=200", is_secure=True) - yield crawler.crawl(seed=url, mockserver=self.mockserver) - cert = crawler.spider.meta["responses"][0].certificate - self.assertIsInstance(cert, Certificate) - self.assertEqual(cert.getSubject().commonName, b"localhost") - self.assertEqual(cert.getIssuer().commonName, b"localhost") - - @defer.inlineCallbacks - def test_dns_server_ip_address_none(self): - crawler = get_crawler(SingleRequestSpider) - url = self.mockserver.url("/status?n=200") - yield crawler.crawl(seed=url, mockserver=self.mockserver) - ip_address = crawler.spider.meta["responses"][0].ip_address - self.assertIsNone(ip_address) - - @defer.inlineCallbacks - def test_dns_server_ip_address(self): - crawler = get_crawler(SingleRequestSpider) - url = self.mockserver.url("/echo?body=test") + url = mockserver.url(url) expected_netloc, _ = urlparse(url).netloc.split(":") - yield crawler.crawl(seed=url, mockserver=self.mockserver) + await crawler.crawl_async(seed=url, mockserver=mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) ip_address = crawler.spider.meta["responses"][0].ip_address - self.assertIsInstance(ip_address, IPv4Address) - self.assertEqual(str(ip_address), gethostbyname(expected_netloc)) + assert isinstance(ip_address, IPv4Address) + assert str(ip_address) == gethostbyname(expected_netloc) - @defer.inlineCallbacks - def test_bytes_received_stop_download_callback(self): + @coroutine_test + async def test_bytes_received_stop_download_callback( + self, mockserver: MockServer + ) -> None: crawler = get_crawler(BytesReceivedCallbackSpider) - yield crawler.crawl(mockserver=self.mockserver) - self.assertIsNone(crawler.spider.meta.get("failure")) - self.assertIsInstance(crawler.spider.meta["response"], Response) - self.assertEqual( - crawler.spider.meta["response"].body, - crawler.spider.meta.get("bytes_received"), + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, BytesReceivedCallbackSpider) + assert crawler.spider.meta.get("failure") is None + assert isinstance(crawler.spider.meta["response"], Response) + assert crawler.spider.meta["response"].body == crawler.spider.meta.get( + "bytes_received" ) - self.assertLess( - len(crawler.spider.meta["response"].body), - crawler.spider.full_response_length, + assert ( + len(crawler.spider.meta["response"].body) + < crawler.spider.full_response_length ) - @defer.inlineCallbacks - def test_bytes_received_stop_download_errback(self): + @coroutine_test + async def test_bytes_received_stop_download_errback( + self, mockserver: MockServer + ) -> None: crawler = get_crawler(BytesReceivedErrbackSpider) - yield crawler.crawl(mockserver=self.mockserver) - self.assertIsNone(crawler.spider.meta.get("response")) - self.assertIsInstance(crawler.spider.meta["failure"], Failure) - self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload) - self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response) - self.assertEqual( - crawler.spider.meta["failure"].value.response.body, - crawler.spider.meta.get("bytes_received"), - ) - self.assertLess( - len(crawler.spider.meta["failure"].value.response.body), - crawler.spider.full_response_length, + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, BytesReceivedErrbackSpider) + assert crawler.spider.meta.get("response") is None + assert isinstance(crawler.spider.meta["failure"], Failure) + assert isinstance(crawler.spider.meta["failure"].value, StopDownload) + assert isinstance(crawler.spider.meta["failure"].value.response, Response) + assert crawler.spider.meta[ + "failure" + ].value.response.body == crawler.spider.meta.get("bytes_received") + assert ( + len(crawler.spider.meta["failure"].value.response.body) + < crawler.spider.full_response_length ) - @defer.inlineCallbacks - def test_headers_received_stop_download_callback(self): + @coroutine_test + async def test_headers_received_stop_download_callback( + self, mockserver: MockServer + ) -> None: crawler = get_crawler(HeadersReceivedCallbackSpider) - yield crawler.crawl(mockserver=self.mockserver) - self.assertIsNone(crawler.spider.meta.get("failure")) - self.assertIsInstance(crawler.spider.meta["response"], Response) - self.assertEqual( - crawler.spider.meta["response"].headers, - crawler.spider.meta.get("headers_received"), + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, HeadersReceivedCallbackSpider) + assert crawler.spider.meta.get("failure") is None + assert isinstance(crawler.spider.meta["response"], Response) + assert crawler.spider.meta["response"].headers == crawler.spider.meta.get( + "headers_received" ) - @defer.inlineCallbacks - def test_headers_received_stop_download_errback(self): + @coroutine_test + async def test_headers_received_stop_download_errback( + self, mockserver: MockServer + ) -> None: crawler = get_crawler(HeadersReceivedErrbackSpider) - yield crawler.crawl(mockserver=self.mockserver) - self.assertIsNone(crawler.spider.meta.get("response")) - self.assertIsInstance(crawler.spider.meta["failure"], Failure) - self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload) - self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response) - self.assertEqual( - crawler.spider.meta["failure"].value.response.headers, - crawler.spider.meta.get("headers_received"), - ) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, HeadersReceivedErrbackSpider) + assert crawler.spider.meta.get("response") is None + assert isinstance(crawler.spider.meta["failure"], Failure) + assert isinstance(crawler.spider.meta["failure"].value, StopDownload) + assert isinstance(crawler.spider.meta["failure"].value.response, Response) + assert crawler.spider.meta[ + "failure" + ].value.response.headers == crawler.spider.meta.get("headers_received") + + @coroutine_test + async def test_spider_callback_deferred_deprecated( + self, mockserver: MockServer + ) -> None: + def cb(response: Response) -> Any: + return succeed(None) + + crawler = get_crawler(SingleRequestSpider) + with pytest.warns( + ScrapyDeprecationWarning, + match="Returning Deferreds from spider callbacks is deprecated", + ): + await crawler.crawl_async(seed=mockserver.url("/"), callback_func=cb) + + @coroutine_test + async def test_spider_errback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + failures = [] + + def eb(failure: Failure) -> Failure: + failures.append(failure) + return failure + + crawler = get_crawler(SingleRequestSpider) + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb + ) + assert len(failures) == 1 + assert "HTTP status code is not handled or not allowed" in caplog.text + assert "Spider error processing" not in caplog.text + + @coroutine_test + async def test_spider_errback_silence( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + failures = [] + + def eb(failure: Failure) -> None: + failures.append(failure) + + crawler = get_crawler(SingleRequestSpider) + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb + ) + assert len(failures) == 1 + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text + + @coroutine_test + async def test_spider_errback_exception( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + def eb(failure: Failure) -> None: + raise ValueError("foo") + + crawler = get_crawler(SingleRequestSpider) + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb + ) + assert "Spider error processing" in caplog.text + + @coroutine_test + async def test_spider_errback_item( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + def eb(failure: Failure) -> Any: + return {"foo": "bar"} + + crawler = get_crawler(SingleRequestSpider) + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text + assert "'item_scraped_count': 1" in caplog.text + + @coroutine_test + async def test_spider_errback_request( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + def eb(failure: Failure) -> Request: + return Request(mockserver.url("/")) + + crawler = get_crawler(SingleRequestSpider) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text + assert "Crawled (200)" in caplog.text + + @coroutine_test + async def test_spider_errback_downloader_error( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + failures = [] + + def eb(failure: Failure) -> Failure: + failures.append(failure) + return failure + + crawler = get_crawler(SingleRequestSpider) + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert len(failures) == 1 + assert "Error downloading" in caplog.text + assert "Spider error processing" not in caplog.text + + @coroutine_test + async def test_spider_errback_downloader_error_exception( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + def eb(failure: Failure) -> None: + raise ValueError("foo") + + crawler = get_crawler(SingleRequestSpider) + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "Error downloading" in caplog.text + assert "Spider error processing" in caplog.text + + @coroutine_test + async def test_spider_errback_downloader_error_item( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + def eb(failure: Failure) -> Any: + return {"foo": "bar"} + + crawler = get_crawler(SingleRequestSpider) + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text + assert "'item_scraped_count': 1" in caplog.text + + @coroutine_test + async def test_spider_errback_downloader_error_request( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + def eb(failure: Failure) -> Request: + return Request(mockserver.url("/")) + + crawler = get_crawler(SingleRequestSpider) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + seed=mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text + assert "Crawled (200)" in caplog.text + + @coroutine_test + async def test_spider_errback_deferred_deprecated( + self, mockserver: MockServer + ) -> None: + def eb(failure: Failure) -> Any: + return succeed(None) + + crawler = get_crawler(SingleRequestSpider) + with pytest.warns( + ScrapyDeprecationWarning, + match="Returning Deferreds from spider errbacks is deprecated", + ): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb + ) + + @coroutine_test + async def test_raise_closespider( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + def cb(response): + raise CloseSpider + + crawler = get_crawler(SingleRequestSpider) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(seed=mockserver.url("/"), callback_func=cb) + assert "Closing spider (cancelled)" in caplog.text + assert "Spider error processing" not in caplog.text + + @coroutine_test + async def test_raise_closespider_reason( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + def cb(response): + raise CloseSpider("my_reason") + + crawler = get_crawler(SingleRequestSpider) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(seed=mockserver.url("/"), callback_func=cb) + assert "Closing spider (my_reason)" in caplog.text + assert "Spider error processing" not in caplog.text diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 989208694..853d6cfaa 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,59 +1,71 @@ +from __future__ import annotations + +import asyncio import logging -import os -import platform -import signal -import subprocess -import sys +import re import warnings from pathlib import Path -from typing import List +from typing import Any, ClassVar import pytest -from packaging.version import parse as parse_version -from pexpect.popen_spawn import PopenSpawn -from pytest import mark, raises -from twisted.internet.defer import Deferred, inlineCallbacks -from twisted.trial import unittest -from w3lib import __version__ as w3lib_version from zope.interface.exceptions import MultipleInvalid import scrapy from scrapy import Spider -from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner +from scrapy.crawler import ( + AsyncCrawlerProcess, + AsyncCrawlerRunner, + Crawler, + CrawlerProcess, + CrawlerRunner, + CrawlerRunnerBase, +) from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.extensions import telnet from scrapy.extensions.throttle import AutoThrottle -from scrapy.settings import Settings, default_settings -from scrapy.spiderloader import SpiderLoader -from scrapy.utils.log import configure_logging, get_scrapy_root_handler +from scrapy.settings import Settings, _SettingsKey, default_settings +from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future +from scrapy.utils.log import ( + _uninstall_scrapy_root_handler, + configure_logging, + get_scrapy_root_handler, +) from scrapy.utils.spider import DefaultSpider -from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer, get_mockserver_env +from scrapy.utils.test import get_crawler, get_reactor_settings +from tests.utils.decorators import coroutine_test -# To prevent warnings. -BASE_SETTINGS = { - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", -} +BASE_SETTINGS: dict[str, Any] = {} -def get_raw_crawler(spidercls=None, settings_dict=None): +def get_raw_crawler( + spidercls: type[Spider] | None = None, settings_dict: dict[str, Any] | None = None +) -> Crawler: """get_crawler alternative that only calls the __init__ method of the crawler.""" settings = Settings() + settings.setdict(get_reactor_settings()) settings.setdict(settings_dict or {}) return Crawler(spidercls or DefaultSpider, settings) -class BaseCrawlerTest(unittest.TestCase): - def assertOptionIsDefault(self, settings, key): - self.assertIsInstance(settings, Settings) - self.assertEqual(settings[key], getattr(default_settings, key)) +class TestBaseCrawler: + @staticmethod + def assertOptionIsDefault(settings: Settings, key: str) -> None: + assert isinstance(settings, Settings) + assert settings[key] == getattr(default_settings, key) -class CrawlerTestCase(BaseCrawlerTest): - def test_populate_spidercls_settings(self): - spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = {**BASE_SETTINGS, "TEST1": "project", "TEST3": "project"} +class TestCrawler(TestBaseCrawler): + def test_populate_spidercls_settings(self) -> None: + spider_settings: dict[_SettingsKey, Any] = { + "TEST1": "spider", + "TEST2": "spider", + } + project_settings = { + **BASE_SETTINGS, + "TEST1": "project", + "TEST3": "project", + **get_reactor_settings(), + } class CustomSettingsSpider(DefaultSpider): custom_settings = spider_settings @@ -63,44 +75,67 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = Crawler(CustomSettingsSpider, settings) crawler._apply_settings() - self.assertEqual(crawler.settings.get("TEST1"), "spider") - self.assertEqual(crawler.settings.get("TEST2"), "spider") - self.assertEqual(crawler.settings.get("TEST3"), "project") + assert crawler.settings.get("TEST1") == "spider" + assert crawler.settings.get("TEST2") == "spider" + assert crawler.settings.get("TEST3") == "project" - self.assertFalse(settings.frozen) - self.assertTrue(crawler.settings.frozen) + assert not settings.frozen + assert crawler.settings.frozen - def test_crawler_accepts_dict(self): + def test_crawler_accepts_dict(self) -> None: crawler = get_crawler(DefaultSpider, {"foo": "bar"}) - self.assertEqual(crawler.settings["foo"], "bar") + assert crawler.settings["foo"] == "bar" self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") - def test_crawler_accepts_None(self): + def test_crawler_accepts_None(self) -> None: with warnings.catch_warnings(): warnings.simplefilter("ignore", ScrapyDeprecationWarning) crawler = Crawler(DefaultSpider) self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") - def test_crawler_rejects_spider_objects(self): - with raises(ValueError): - Crawler(DefaultSpider()) + def test_crawler_rejects_spider_objects(self) -> None: + with pytest.raises(ValueError, match="spidercls argument must be a class"): + Crawler(DefaultSpider()) # type: ignore[arg-type] - @inlineCallbacks - def test_crawler_crawl_twice_deprecated(self): + @coroutine_test + async def test_crawler_crawl_twice_seq_unsupported(self) -> None: crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) - yield crawler.crawl() - with pytest.warns( - ScrapyDeprecationWarning, - match=r"Running Crawler.crawl\(\) more than once is deprecated", - ): - yield crawler.crawl() + await maybe_deferred_to_future(crawler.crawl()) + with pytest.raises(RuntimeError, match="more than once on the same instance"): + await maybe_deferred_to_future(crawler.crawl()) - def test_get_addon(self): + @coroutine_test + async def test_crawler_crawl_async_twice_seq_unsupported(self) -> None: + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) + await crawler.crawl_async() + with pytest.raises(RuntimeError, match="more than once on the same instance"): + await crawler.crawl_async() + + @coroutine_test + async def test_crawler_crawl_twice_parallel_unsupported(self) -> None: + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) + d1 = crawler.crawl() + d2 = crawler.crawl() + await maybe_deferred_to_future(d1) + with pytest.raises(RuntimeError, match="Crawling already taking place"): + await maybe_deferred_to_future(d2) + + @pytest.mark.only_asyncio + @coroutine_test + async def test_crawler_crawl_async_twice_parallel_unsupported(self) -> None: + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) + t1 = asyncio.create_task(crawler.crawl_async()) + t2 = asyncio.create_task(crawler.crawl_async()) + await t1 + with pytest.raises(RuntimeError, match="Crawling already taking place"): + await t2 + + def test_get_addon(self) -> None: class ParentAddon: pass class TrackingAddon(ParentAddon): - instances = [] + instances: ClassVar[list[TrackingAddon]] = [] def __init__(self): TrackingAddon.instances.append(self) @@ -115,46 +150,49 @@ class CrawlerTestCase(BaseCrawlerTest): }, } crawler = get_crawler(settings_dict=settings) - self.assertEqual(len(TrackingAddon.instances), 1) + assert len(TrackingAddon.instances) == 1 expected = TrackingAddon.instances[-1] addon = crawler.get_addon(TrackingAddon) - self.assertEqual(addon, expected) + assert addon == expected - addon = crawler.get_addon(DefaultSpider) - self.assertIsNone(addon) + addon = crawler.get_addon(DefaultSpider) # type: ignore[assignment] + assert addon is None addon = crawler.get_addon(ParentAddon) - self.assertEqual(addon, expected) + assert addon == expected class ChildAddon(TrackingAddon): pass addon = crawler.get_addon(ChildAddon) - self.assertIsNone(addon) + assert addon is None - @inlineCallbacks - def test_get_downloader_middleware(self): + @coroutine_test + async def test_get_downloader_middleware(self) -> None: class ParentDownloaderMiddleware: pass class TrackingDownloaderMiddleware(ParentDownloaderMiddleware): - instances = [] + instances: ClassVar[list[TrackingDownloaderMiddleware]] = [] def __init__(self): TrackingDownloaderMiddleware.instances.append(self) class MySpider(Spider): name = "myspider" + cls: ClassVar[type[Any]] + result: ClassVar[Any] @classmethod def from_crawler(cls, crawler): return cls(crawler=crawler) - def __init__(self, crawler): + def __init__(self, crawler, **kwargs: Any): + super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def start(self): MySpider.result = crawler.get_downloader_middleware(MySpider.cls) return yield @@ -168,36 +206,35 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingDownloaderMiddleware - yield crawler.crawl() - self.assertEqual(len(TrackingDownloaderMiddleware.instances), 1) - self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1]) + await crawler.crawl_async() + assert len(TrackingDownloaderMiddleware.instances) == 1 + assert MySpider.result == TrackingDownloaderMiddleware.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider - yield crawler.crawl() - self.assertIsNone(MySpider.result) + await crawler.crawl_async() + assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentDownloaderMiddleware - yield crawler.crawl() - self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1]) + await crawler.crawl_async() + assert MySpider.result == TrackingDownloaderMiddleware.instances[-1] class ChildDownloaderMiddleware(TrackingDownloaderMiddleware): pass crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildDownloaderMiddleware - yield crawler.crawl() - self.assertIsNone(MySpider.result) + await crawler.crawl_async() + assert MySpider.result is None - def test_get_downloader_middleware_not_crawling(self): + def test_get_downloader_middleware_not_crawling(self) -> None: crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) - self.assertRaises( - RuntimeError, crawler.get_downloader_middleware, DefaultSpider - ) + with pytest.raises(RuntimeError): + crawler.get_downloader_middleware(DefaultSpider) - @inlineCallbacks - def test_get_downloader_middleware_no_engine(self): + @coroutine_test + async def test_get_downloader_middleware_no_engine(self) -> None: class MySpider(Spider): name = "myspider" @@ -210,31 +247,34 @@ class CrawlerTestCase(BaseCrawlerTest): raise crawler = get_raw_crawler(MySpider, BASE_SETTINGS) - with raises(RuntimeError): - yield crawler.crawl() + with pytest.raises(RuntimeError): + await crawler.crawl_async() - @inlineCallbacks - def test_get_extension(self): + @coroutine_test + async def test_get_extension(self) -> None: class ParentExtension: pass class TrackingExtension(ParentExtension): - instances = [] + instances: ClassVar[list[TrackingExtension]] = [] def __init__(self): TrackingExtension.instances.append(self) class MySpider(Spider): name = "myspider" + cls: ClassVar[type[Any]] + result: ClassVar[Any] @classmethod def from_crawler(cls, crawler): return cls(crawler=crawler) - def __init__(self, crawler): + def __init__(self, crawler, **kwargs: Any): + super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def start(self): MySpider.result = crawler.get_extension(MySpider.cls) return yield @@ -248,34 +288,35 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingExtension - yield crawler.crawl() - self.assertEqual(len(TrackingExtension.instances), 1) - self.assertEqual(MySpider.result, TrackingExtension.instances[-1]) + await crawler.crawl_async() + assert len(TrackingExtension.instances) == 1 + assert MySpider.result == TrackingExtension.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider - yield crawler.crawl() - self.assertIsNone(MySpider.result) + await crawler.crawl_async() + assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentExtension - yield crawler.crawl() - self.assertEqual(MySpider.result, TrackingExtension.instances[-1]) + await crawler.crawl_async() + assert MySpider.result == TrackingExtension.instances[-1] class ChildExtension(TrackingExtension): pass crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildExtension - yield crawler.crawl() - self.assertIsNone(MySpider.result) + await crawler.crawl_async() + assert MySpider.result is None - def test_get_extension_not_crawling(self): + def test_get_extension_not_crawling(self) -> None: crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) - self.assertRaises(RuntimeError, crawler.get_extension, DefaultSpider) + with pytest.raises(RuntimeError): + crawler.get_extension(DefaultSpider) - @inlineCallbacks - def test_get_extension_no_engine(self): + @coroutine_test + async def test_get_extension_no_engine(self) -> None: class MySpider(Spider): name = "myspider" @@ -288,31 +329,34 @@ class CrawlerTestCase(BaseCrawlerTest): raise crawler = get_raw_crawler(MySpider, BASE_SETTINGS) - with raises(RuntimeError): - yield crawler.crawl() + with pytest.raises(RuntimeError): + await crawler.crawl_async() - @inlineCallbacks - def test_get_item_pipeline(self): + @coroutine_test + async def test_get_item_pipeline(self) -> None: class ParentItemPipeline: pass class TrackingItemPipeline(ParentItemPipeline): - instances = [] + instances: ClassVar[list[TrackingItemPipeline]] = [] def __init__(self): TrackingItemPipeline.instances.append(self) class MySpider(Spider): name = "myspider" + cls: ClassVar[type[Any]] + result: ClassVar[Any] @classmethod def from_crawler(cls, crawler): return cls(crawler=crawler) - def __init__(self, crawler): + def __init__(self, crawler, **kwargs: Any): + super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def start(self): MySpider.result = crawler.get_item_pipeline(MySpider.cls) return yield @@ -326,34 +370,35 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingItemPipeline - yield crawler.crawl() - self.assertEqual(len(TrackingItemPipeline.instances), 1) - self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1]) + await crawler.crawl_async() + assert len(TrackingItemPipeline.instances) == 1 + assert MySpider.result == TrackingItemPipeline.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider - yield crawler.crawl() - self.assertIsNone(MySpider.result) + await crawler.crawl_async() + assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentItemPipeline - yield crawler.crawl() - self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1]) + await crawler.crawl_async() + assert MySpider.result == TrackingItemPipeline.instances[-1] class ChildItemPipeline(TrackingItemPipeline): pass crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildItemPipeline - yield crawler.crawl() - self.assertIsNone(MySpider.result) + await crawler.crawl_async() + assert MySpider.result is None - def test_get_item_pipeline_not_crawling(self): + def test_get_item_pipeline_not_crawling(self) -> None: crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) - self.assertRaises(RuntimeError, crawler.get_item_pipeline, DefaultSpider) + with pytest.raises(RuntimeError): + crawler.get_item_pipeline(DefaultSpider) - @inlineCallbacks - def test_get_item_pipeline_no_engine(self): + @coroutine_test + async def test_get_item_pipeline_no_engine(self) -> None: class MySpider(Spider): name = "myspider" @@ -366,31 +411,34 @@ class CrawlerTestCase(BaseCrawlerTest): raise crawler = get_raw_crawler(MySpider, BASE_SETTINGS) - with raises(RuntimeError): - yield crawler.crawl() + with pytest.raises(RuntimeError): + await crawler.crawl_async() - @inlineCallbacks - def test_get_spider_middleware(self): + @coroutine_test + async def test_get_spider_middleware(self) -> None: class ParentSpiderMiddleware: pass class TrackingSpiderMiddleware(ParentSpiderMiddleware): - instances = [] + instances: ClassVar[list[TrackingSpiderMiddleware]] = [] def __init__(self): TrackingSpiderMiddleware.instances.append(self) class MySpider(Spider): name = "myspider" + cls: ClassVar[type[Any]] + result: ClassVar[Any] @classmethod def from_crawler(cls, crawler): return cls(crawler=crawler) - def __init__(self, crawler): + def __init__(self, crawler, **kwargs: Any): + super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def start(self): MySpider.result = crawler.get_spider_middleware(MySpider.cls) return yield @@ -404,34 +452,35 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingSpiderMiddleware - yield crawler.crawl() - self.assertEqual(len(TrackingSpiderMiddleware.instances), 1) - self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1]) + await crawler.crawl_async() + assert len(TrackingSpiderMiddleware.instances) == 1 + assert MySpider.result == TrackingSpiderMiddleware.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider - yield crawler.crawl() - self.assertIsNone(MySpider.result) + await crawler.crawl_async() + assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentSpiderMiddleware - yield crawler.crawl() - self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1]) + await crawler.crawl_async() + assert MySpider.result == TrackingSpiderMiddleware.instances[-1] class ChildSpiderMiddleware(TrackingSpiderMiddleware): pass crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildSpiderMiddleware - yield crawler.crawl() - self.assertIsNone(MySpider.result) + await crawler.crawl_async() + assert MySpider.result is None - def test_get_spider_middleware_not_crawling(self): + def test_get_spider_middleware_not_crawling(self) -> None: crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) - self.assertRaises(RuntimeError, crawler.get_spider_middleware, DefaultSpider) + with pytest.raises(RuntimeError): + crawler.get_spider_middleware(DefaultSpider) - @inlineCallbacks - def test_get_spider_middleware_no_engine(self): + @coroutine_test + async def test_get_spider_middleware_no_engine(self) -> None: class MySpider(Spider): name = "myspider" @@ -444,23 +493,24 @@ class CrawlerTestCase(BaseCrawlerTest): raise crawler = get_raw_crawler(MySpider, BASE_SETTINGS) - with raises(RuntimeError): - yield crawler.crawl() + with pytest.raises(RuntimeError): + await crawler.crawl_async() -class SpiderSettingsTestCase(unittest.TestCase): - def test_spider_custom_settings(self): +class TestSpiderSettings: + def test_spider_custom_settings(self) -> None: class MySpider(scrapy.Spider): name = "spider" custom_settings = {"AUTOTHROTTLE_ENABLED": True} crawler = get_crawler(MySpider) + assert crawler.extensions enabled_exts = [e.__class__ for e in crawler.extensions.middlewares] - self.assertIn(AutoThrottle, enabled_exts) + assert AutoThrottle in enabled_exts -class CrawlerLoggingTestCase(unittest.TestCase): - def test_no_root_handler_installed(self): +class TestCrawlerLogging: + def test_no_root_handler_installed(self) -> None: handler = get_scrapy_root_handler() if handler is not None: logging.root.removeHandler(handler) @@ -471,44 +521,61 @@ class CrawlerLoggingTestCase(unittest.TestCase): get_crawler(MySpider) assert get_scrapy_root_handler() is None - def test_spider_custom_settings_log_level(self): - log_file = Path(self.mktemp()) + @coroutine_test + async def test_spider_custom_settings_log_level(self, tmp_path: Path) -> None: + log_file = Path(tmp_path, "log.txt") log_file.write_text("previous message\n", encoding="utf-8") + info_count = None + class MySpider(scrapy.Spider): name = "spider" custom_settings = { "LOG_LEVEL": "INFO", "LOG_FILE": str(log_file), - # settings to avoid extra warnings - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", - "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE, } - configure_logging() - self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) - crawler = get_crawler(MySpider) - self.assertEqual(get_scrapy_root_handler().level, logging.INFO) - info_count = crawler.stats.get_value("log_count/INFO") - logging.debug("debug message") - logging.info("info message") - logging.warning("warning message") - logging.error("error message") + async def start(self): + info_count_start = crawler.stats.get_value("log_count/INFO") + logging.debug("debug message") # noqa: LOG015 + logging.info("info message") # noqa: LOG015 + logging.warning("warning message") # noqa: LOG015 + logging.error("error message") # noqa: LOG015 + nonlocal info_count + info_count = ( + crawler.stats.get_value("log_count/INFO") - info_count_start + ) + return + yield + + try: + configure_logging() + handler = get_scrapy_root_handler() + assert handler is not None + assert handler.level == logging.DEBUG + crawler = get_crawler(MySpider) + handler = get_scrapy_root_handler() + assert handler is not None + assert handler.level == logging.INFO + await crawler.crawl_async() + finally: + _uninstall_scrapy_root_handler() logged = log_file.read_text(encoding="utf-8") - self.assertIn("previous message", logged) - self.assertNotIn("debug message", logged) - self.assertIn("info message", logged) - self.assertIn("warning message", logged) - self.assertIn("error message", logged) - self.assertEqual(crawler.stats.get_value("log_count/ERROR"), 1) - self.assertEqual(crawler.stats.get_value("log_count/WARNING"), 1) - self.assertEqual(crawler.stats.get_value("log_count/INFO") - info_count, 1) - self.assertEqual(crawler.stats.get_value("log_count/DEBUG", 0), 0) + assert "previous message" in logged + assert "debug message" not in logged + assert "info message" in logged + assert "warning message" in logged + assert "error message" in logged + assert crawler.stats + assert crawler.stats.get_value("log_count/ERROR") == 1 + assert crawler.stats.get_value("log_count/WARNING") == 1 + assert info_count == 1 + assert crawler.stats.get_value("log_count/DEBUG", 0) == 0 - def test_spider_custom_settings_log_append(self): - log_file = Path(self.mktemp()) + def test_spider_custom_settings_log_append(self, tmp_path: Path) -> None: + log_file = Path(tmp_path, "log.txt") log_file.write_text("previous message\n", encoding="utf-8") class MySpider(scrapy.Spider): @@ -516,56 +583,90 @@ class CrawlerLoggingTestCase(unittest.TestCase): custom_settings = { "LOG_FILE": str(log_file), "LOG_FILE_APPEND": False, - # disable telnet if not available to avoid an extra warning - "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE, } - configure_logging() - get_crawler(MySpider) - logging.debug("debug message") + try: + configure_logging() + get_crawler(MySpider) + logging.debug("debug message") # noqa: LOG015 + finally: + _uninstall_scrapy_root_handler() logged = log_file.read_text(encoding="utf-8") - self.assertNotIn("previous message", logged) - self.assertIn("debug message", logged) + assert "previous message" not in logged + assert "debug message" in logged class SpiderLoaderWithWrongInterface: - def unneeded_method(self): + def unneeded_method(self) -> None: pass -class CustomSpiderLoader(SpiderLoader): - pass - - -class CrawlerRunnerTestCase(BaseCrawlerTest): - def test_spider_manager_verify_interface(self): +class TestCrawlerRunner(TestBaseCrawler): + def test_spider_manager_verify_interface(self) -> None: settings = Settings( { "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, } ) - self.assertRaises(MultipleInvalid, CrawlerRunner, settings) + with pytest.raises(MultipleInvalid): + CrawlerRunner(settings) - def test_crawler_runner_accepts_dict(self): + def test_crawler_runner_accepts_dict(self) -> None: runner = CrawlerRunner({"foo": "bar"}) - self.assertEqual(runner.settings["foo"], "bar") + assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - def test_crawler_runner_accepts_None(self): + def test_crawler_runner_accepts_None(self) -> None: runner = CrawlerRunner() self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") -class CrawlerProcessTest(BaseCrawlerTest): - def test_crawler_process_accepts_dict(self): - runner = CrawlerProcess({"foo": "bar"}) - self.assertEqual(runner.settings["foo"], "bar") +class TestAsyncCrawlerRunner(TestBaseCrawler): + def test_spider_manager_verify_interface(self) -> None: + settings = Settings( + { + "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, + } + ) + with pytest.raises(MultipleInvalid): + AsyncCrawlerRunner(settings) + + def test_crawler_runner_accepts_dict(self) -> None: + runner = AsyncCrawlerRunner({"foo": "bar"}) + assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - def test_crawler_process_accepts_None(self): - runner = CrawlerProcess() + def test_crawler_runner_accepts_None(self) -> None: + runner = AsyncCrawlerRunner() + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") + + +class TestCrawlerProcess(TestBaseCrawler): + def test_crawler_process_accepts_dict(self) -> None: + runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False) + assert runner.settings["foo"] == "bar" + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") + + def test_crawler_process_accepts_None(self) -> None: + runner = CrawlerProcess(install_root_handler=False) + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") + + +@pytest.mark.only_asyncio +class TestAsyncCrawlerProcess(TestBaseCrawler): + def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None: + runner = AsyncCrawlerProcess( + {"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"}, + install_root_handler=False, + ) + assert runner.settings["foo"] == "bar" + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") + + @pytest.mark.requires_reactor # can't pass TWISTED_REACTOR_ENABLED=False + def test_crawler_process_accepts_None(self) -> None: + runner = AsyncCrawlerProcess(install_root_handler=False) self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") @@ -580,349 +681,130 @@ class ExceptionSpider(scrapy.Spider): class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield -@mark.usefixtures("reactor_pytest") -class CrawlerRunnerHasSpider(unittest.TestCase): - def _runner(self): - return CrawlerRunner({"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7"}) +@pytest.mark.requires_reactor # CrawlerRunner requires a reactor +class TestCrawlerRunnerHasSpider: + @pytest.fixture + def runner(self) -> CrawlerRunnerBase: + return CrawlerRunner(get_reactor_settings()) - @inlineCallbacks - def test_crawler_runner_bootstrap_successful(self): - runner = self._runner() - yield runner.crawl(NoRequestsSpider) - self.assertFalse(runner.bootstrap_failed) + @staticmethod + async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None: + await ensure_awaitable(runner.crawl(spider)) - @inlineCallbacks - def test_crawler_runner_bootstrap_successful_for_several(self): - runner = self._runner() - yield runner.crawl(NoRequestsSpider) - yield runner.crawl(NoRequestsSpider) - self.assertFalse(runner.bootstrap_failed) + @coroutine_test + async def test_crawler_runner_bootstrap_successful( + self, runner: CrawlerRunnerBase + ) -> None: + await self._crawl(runner, NoRequestsSpider) + assert not runner.bootstrap_failed - @inlineCallbacks - def test_crawler_runner_bootstrap_failed(self): - runner = self._runner() + @coroutine_test + async def test_crawler_runner_bootstrap_successful_for_several( + self, runner: CrawlerRunnerBase + ) -> None: + await self._crawl(runner, NoRequestsSpider) + await self._crawl(runner, NoRequestsSpider) + assert not runner.bootstrap_failed + @coroutine_test + async def test_crawler_runner_bootstrap_failed( + self, runner: CrawlerRunnerBase + ) -> None: try: - yield runner.crawl(ExceptionSpider) + await self._crawl(runner, ExceptionSpider) except ValueError: pass else: - self.fail("Exception should be raised from spider") + pytest.fail("Exception should be raised from spider") - self.assertTrue(runner.bootstrap_failed) - - @inlineCallbacks - def test_crawler_runner_bootstrap_failed_for_several(self): - runner = self._runner() + assert runner.bootstrap_failed + @coroutine_test + async def test_crawler_runner_bootstrap_failed_for_several( + self, runner: CrawlerRunnerBase + ) -> None: try: - yield runner.crawl(ExceptionSpider) + await self._crawl(runner, ExceptionSpider) except ValueError: pass else: - self.fail("Exception should be raised from spider") + pytest.fail("Exception should be raised from spider") - yield runner.crawl(NoRequestsSpider) + await self._crawl(runner, NoRequestsSpider) - self.assertTrue(runner.bootstrap_failed) + assert runner.bootstrap_failed - @inlineCallbacks - def test_crawler_runner_asyncio_enabled_true(self): - if self.reactor_pytest == "asyncio": + @coroutine_test + async def test_crawler_runner_asyncio_enabled_true( + self, reactor_pytest: str + ) -> None: + if reactor_pytest != "asyncio": + runner = CrawlerRunner( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + ) + with pytest.raises( + Exception, + match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)", + ): + await self._crawl(runner, NoRequestsSpider) + else: CrawlerRunner( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } ) - else: - msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" - with self.assertRaisesRegex(Exception, msg): - runner = CrawlerRunner( - settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", - } - ) - yield runner.crawl(NoRequestsSpider) -class ScriptRunnerMixin: - script_dir: Path - cwd = os.getcwd() +@pytest.mark.only_asyncio +class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider): + @staticmethod + def _runner() -> CrawlerRunnerBase: + return AsyncCrawlerRunner(get_reactor_settings()) - def get_script_args(self, script_name: str, *script_args: str) -> List[str]: - script_path = self.script_dir / script_name - return [sys.executable, str(script_path)] + list(script_args) - - def run_script(self, script_name: str, *script_args: str) -> str: - args = self.get_script_args(script_name, *script_args) - p = subprocess.Popen( - args, - env=get_mockserver_env(), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - stdout, stderr = p.communicate() - return stderr.decode("utf-8") + def test_crawler_runner_asyncio_enabled_true(self) -> None: # type: ignore[override] + pytest.skip("This test is only for CrawlerRunner") -class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): - script_dir = Path(__file__).parent.resolve() / "CrawlerProcess" - - def test_simple(self): - log = self.run_script("simple.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log - ) - - def test_multi(self): - log = self.run_script("multi.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log - ) - self.assertNotIn("ReactorAlreadyInstalledError", log) - - def test_reactor_default(self): - log = self.run_script("reactor_default.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log - ) - self.assertNotIn("ReactorAlreadyInstalledError", log) - - def test_reactor_default_twisted_reactor_select(self): - log = self.run_script("reactor_default_twisted_reactor_select.py") - if platform.system() in ["Windows", "Darwin"]: - # The goal of this test function is to test that, when a reactor is - # installed (the default one here) and a different reactor is - # configured (select here), an error raises. - # - # In Windows the default reactor is the select reactor, so that - # error does not raise. - # - # If that ever becomes the case on more platforms (i.e. if Linux - # also starts using the select reactor by default in a future - # version of Twisted), then we will need to rethink this test. - self.assertIn("Spider closed (finished)", log) - else: - self.assertNotIn("Spider closed (finished)", log) - self.assertIn( - ( - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ), - log, - ) - - def test_reactor_select(self): - log = self.run_script("reactor_select.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn("ReactorAlreadyInstalledError", log) - - def test_reactor_select_twisted_reactor_select(self): - log = self.run_script("reactor_select_twisted_reactor_select.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn("ReactorAlreadyInstalledError", log) - - def test_reactor_select_subclass_twisted_reactor_select(self): - log = self.run_script("reactor_select_subclass_twisted_reactor_select.py") - self.assertNotIn("Spider closed (finished)", log) - self.assertIn( - ( - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ), - log, - ) - - def test_asyncio_enabled_no_reactor(self): - log = self.run_script("asyncio_enabled_no_reactor.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log - ) - - def test_asyncio_enabled_reactor(self): - log = self.run_script("asyncio_enabled_reactor.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log - ) - - @mark.skipif( - parse_version(w3lib_version) >= parse_version("2.0.0"), - reason="w3lib 2.0.0 and later do not allow invalid domains.", +@pytest.mark.parametrize( + ("settings", "items"), + [ + ({}, default_settings.LOG_VERSIONS), + ({"LOG_VERSIONS": ["itemadapter"]}, ["itemadapter"]), + ({"LOG_VERSIONS": []}, None), + ], +) +def test_log_scrapy_info( + settings: dict[str, Any], items: list[str] | None, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level("INFO"): + CrawlerProcess(settings, install_root_handler=False) + assert ( + caplog.records[0].getMessage() + == f"Scrapy {scrapy.__version__} started (bot: scrapybot)" + ), repr(caplog.records[0].msg) + if not items: + assert len(caplog.records) == 1 + return + version_string = caplog.records[1].getMessage() + expected_items_pattern = "',\n '".join( + f"{item}': '[^']+('\n +'[^']+)*" for item in items ) - def test_ipv6_default_name_resolver(self): - log = self.run_script("default_name_resolver.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", - log, - ) - self.assertIn( - "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", - log, - ) - - def test_caching_hostname_resolver_ipv6(self): - log = self.run_script("caching_hostname_resolver_ipv6.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn("twisted.internet.error.DNSLookupError", log) - - def test_caching_hostname_resolver_finite_execution(self): - with MockServer() as mock_server: - http_address = mock_server.http_address.replace("0.0.0.0", "127.0.0.1") - log = self.run_script("caching_hostname_resolver.py", http_address) - self.assertIn("Spider closed (finished)", log) - self.assertNotIn("ERROR: Error downloading", log) - self.assertNotIn("TimeoutError", log) - self.assertNotIn("twisted.internet.error.DNSLookupError", log) - - def test_twisted_reactor_select(self): - log = self.run_script("twisted_reactor_select.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.selectreactor.SelectReactor", log - ) - - @mark.skipif( - platform.system() == "Windows", reason="PollReactor is not supported on Windows" - ) - def test_twisted_reactor_poll(self): - log = self.run_script("twisted_reactor_poll.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) - - def test_twisted_reactor_asyncio(self): - log = self.run_script("twisted_reactor_asyncio.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log - ) - - def test_twisted_reactor_asyncio_custom_settings(self): - log = self.run_script("twisted_reactor_custom_settings.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log - ) - - def test_twisted_reactor_asyncio_custom_settings_same(self): - log = self.run_script("twisted_reactor_custom_settings_same.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log - ) - - def test_twisted_reactor_asyncio_custom_settings_conflict(self): - log = self.run_script("twisted_reactor_custom_settings_conflict.py") - self.assertIn( - "Using reactor: twisted.internet.selectreactor.SelectReactor", log - ) - self.assertIn( - "(twisted.internet.selectreactor.SelectReactor) does not match the requested one", - log, - ) - - @mark.requires_uvloop - def test_custom_loop_asyncio(self): - log = self.run_script("asyncio_custom_loop.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log - ) - self.assertIn("Using asyncio event loop: uvloop.Loop", log) - - @mark.requires_uvloop - def test_custom_loop_asyncio_deferred_signal(self): - log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log - ) - self.assertIn("Using asyncio event loop: uvloop.Loop", log) - self.assertIn("async pipeline opened!", log) - - @mark.requires_uvloop - def test_asyncio_enabled_reactor_same_loop(self): - log = self.run_script("asyncio_enabled_reactor_same_loop.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log - ) - self.assertIn("Using asyncio event loop: uvloop.Loop", log) - - @mark.requires_uvloop - def test_asyncio_enabled_reactor_different_loop(self): - log = self.run_script("asyncio_enabled_reactor_different_loop.py") - self.assertNotIn("Spider closed (finished)", log) - self.assertIn( - ( - "does not match the one specified in the ASYNCIO_EVENT_LOOP " - "setting (uvloop.Loop)" - ), - log, - ) - - def test_default_loop_asyncio_deferred_signal(self): - log = self.run_script("asyncio_deferred_signal.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log - ) - self.assertNotIn("Using asyncio event loop: uvloop.Loop", log) - self.assertIn("async pipeline opened!", log) - - def test_args_change_settings(self): - log = self.run_script("args_settings.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn("The value of FOO is 42", log) - - def test_shutdown_graceful(self): - sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py", "-a", "sleep=3") - p = PopenSpawn(args, timeout=5) - p.expect_exact("Spider opened") - p.expect_exact("Crawled (200)") - p.kill(sig) - p.expect_exact("shutting down gracefully") - p.expect_exact("Spider closed (shutdown)") - p.wait() - - @inlineCallbacks - def test_shutdown_forced(self): - from twisted.internet import reactor - - sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py", "-a", "sleep=10") - p = PopenSpawn(args, timeout=5) - p.expect_exact("Spider opened") - p.expect_exact("Crawled (200)") - p.kill(sig) - p.expect_exact("shutting down gracefully") - # sending the second signal too fast often causes problems - d = Deferred() - reactor.callLater(0.1, d.callback, None) - yield d - p.kill(sig) - p.expect_exact("forcing unclean shutdown") - p.wait() + assert re.search(r"^Versions:\n{'" + expected_items_pattern + "'}$", version_string) -class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): - script_dir = Path(__file__).parent.resolve() / "CrawlerRunner" - - def test_response_ip_address(self): - log = self.run_script("ip_address.py") - self.assertIn("INFO: Spider closed (finished)", log) - self.assertIn("INFO: Host: not.a.real.domain", log) - self.assertIn("INFO: Type: ", log) - self.assertIn("INFO: IP address: 127.0.0.1", log) +@coroutine_test +async def test_deprecated_crawler_stop() -> None: + crawler = get_crawler(DefaultSpider) + d = crawler.crawl() + await maybe_deferred_to_future(d) + with pytest.warns( + ScrapyDeprecationWarning, match=r"Crawler.stop\(\) is deprecated" + ): + await maybe_deferred_to_future(crawler.stop()) diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py new file mode 100644 index 000000000..146d94ecd --- /dev/null +++ b/tests/test_crawler_subprocess.py @@ -0,0 +1,575 @@ +from __future__ import annotations + +import platform +import re +import signal +import subprocess +import sys +from abc import ABC, abstractmethod +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest +from packaging.version import parse as parse_version +from pexpect.popen_spawn import PopenSpawn +from w3lib import __version__ as w3lib_version + +from tests.utils import async_sleep, get_script_run_env +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from tests.mockserver.http import MockServer + + +class ScriptRunnerMixin(ABC): + @property + @abstractmethod + def script_dir(self) -> Path: + raise NotImplementedError + + @staticmethod + def get_script_dir(name: str) -> Path: + return Path(__file__).parent.resolve() / name + + def get_script_args(self, script_name: str, *script_args: str) -> list[str]: + script_path = self.script_dir / script_name + return [sys.executable, str(script_path), *script_args] + + def run_script(self, script_name: str, *script_args: str) -> str: + args = self.get_script_args(script_name, *script_args) + p = subprocess.Popen( + args, + env=get_script_run_env(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + _, stderr = p.communicate() + return stderr.decode("utf-8") + + +class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): + """Common tests between CrawlerProcess and AsyncCrawlerProcess, + with the same file names and expectations. + """ + + def test_simple(self) -> None: + log = self.run_script("simple.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "is_reactorless(): False" in log + + def test_multi(self) -> None: + log = self.run_script("multi.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "ReactorAlreadyInstalledError" not in log + + def test_reactor_default(self) -> None: + log = self.run_script("reactor_default.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + ) in log + + def test_asyncio_enabled_no_reactor(self) -> None: + log = self.run_script("asyncio_enabled_no_reactor.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "RuntimeError" not in log + + def test_asyncio_enabled_reactor(self) -> None: + log = self.run_script("asyncio_enabled_reactor.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "RuntimeError" not in log + + @pytest.mark.skipif( + parse_version(w3lib_version) >= parse_version("2.0.0"), + reason="w3lib 2.0.0 and later do not allow invalid domains.", + ) + def test_ipv6_default_name_resolver(self) -> None: + log = self.run_script("default_name_resolver.py") + assert "Spider closed (finished)" in log + assert ( + "'downloader/exception_type_count/scrapy.exceptions.CannotResolveHostError': 1," + in log + ) + assert ( + "scrapy.exceptions.CannotResolveHostError: DNS lookup failed: no results for hostname lookup: ::1." + in log + ) + + def test_caching_hostname_resolver_ipv6(self) -> None: + log = self.run_script("caching_hostname_resolver_ipv6.py") + assert "Spider closed (finished)" in log + assert "scrapy.exceptions.CannotResolveHostError" not in log + + def test_caching_hostname_resolver_finite_execution( + self, mockserver: MockServer + ) -> None: + log = self.run_script("caching_hostname_resolver.py", mockserver.url("/")) + assert "Spider closed (finished)" in log + assert "ERROR: Error downloading" not in log + assert "TimeoutError" not in log + assert "scrapy.exceptions.CannotResolveHostError" not in log + + def test_twisted_reactor_asyncio(self) -> None: + log = self.run_script("twisted_reactor_asyncio.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_twisted_reactor_asyncio_custom_settings(self) -> None: + log = self.run_script("twisted_reactor_custom_settings.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_twisted_reactor_asyncio_custom_settings_same(self) -> None: + log = self.run_script("twisted_reactor_custom_settings_same.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + @pytest.mark.requires_uvloop + def test_custom_loop_asyncio(self) -> None: + log = self.run_script("asyncio_custom_loop.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_custom_loop_asyncio_deferred_signal(self) -> None: + log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + assert "async pipeline opened!" in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_same_loop(self) -> None: + log = self.run_script("asyncio_enabled_reactor_same_loop.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_different_loop(self) -> None: + log = self.run_script("asyncio_enabled_reactor_different_loop.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ) in log + + def test_default_loop_asyncio_deferred_signal(self) -> None: + log = self.run_script("asyncio_deferred_signal.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" not in log + assert "async pipeline opened!" in log + + def test_args_change_settings(self) -> None: + log = self.run_script("args_settings.py") + assert "Spider closed (finished)" in log + assert "The value of FOO is 42" in log + + def _test_shutdown_graceful(self, script: str = "sleeping.py") -> None: + sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK # type: ignore[attr-defined] + args = self.get_script_args(script, "3") + p = PopenSpawn(args, timeout=5, env=get_script_run_env()) + p.expect_exact("Spider opened") + p.expect_exact("Crawled (200)") + p.kill(sig) + p.expect_exact("shutting down gracefully") + p.expect_exact("Spider closed (shutdown)") + p.wait() # type: ignore[no-untyped-call] + + def test_shutdown_graceful(self) -> None: + self._test_shutdown_graceful() + + async def _test_shutdown_forced(self, script: str = "sleeping.py") -> None: + sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK # type: ignore[attr-defined] + args = self.get_script_args(script, "10") + p = PopenSpawn(args, timeout=5, env=get_script_run_env()) + p.expect_exact("Spider opened") + p.expect_exact("Crawled (200)") + p.kill(sig) + p.expect_exact("shutting down gracefully") + # sending the second signal too fast often causes problems + await async_sleep(0.01) + p.kill(sig) + p.expect_exact("forcing unclean shutdown") + p.wait() # type: ignore[no-untyped-call] + + @coroutine_test + async def test_shutdown_forced(self) -> None: + await self._test_shutdown_forced() + + +class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("CrawlerProcess") + + def test_reactor_default_twisted_reactor_select(self) -> None: + log = self.run_script("reactor_default_twisted_reactor_select.py") + if platform.system() in ["Windows", "Darwin"]: + # The goal of this test function is to test that, when a reactor is + # installed (the default one here) and a different reactor is + # configured (select here), an error raises. + # + # In Windows the default reactor is the select reactor, so that + # error does not raise. + # + # If that ever becomes the case on more platforms (i.e. if Linux + # also starts using the select reactor by default in a future + # version of Twisted), then we will need to rethink this test. + assert "Spider closed (finished)" in log + else: + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + def test_reactor_select(self) -> None: + log = self.run_script("reactor_select.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + ) in log + + def test_reactor_select_twisted_reactor_select(self) -> None: + log = self.run_script("reactor_select_twisted_reactor_select.py") + assert "Spider closed (finished)" in log + assert "ReactorAlreadyInstalledError" not in log + + def test_reactor_select_subclass_twisted_reactor_select(self) -> None: + log = self.run_script("reactor_select_subclass_twisted_reactor_select.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + def test_twisted_reactor_select(self) -> None: + log = self.run_script("twisted_reactor_select.py") + assert "Spider closed (finished)" in log + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + + @pytest.mark.skipif( + platform.system() == "Windows", reason="PollReactor is not supported on Windows" + ) + def test_twisted_reactor_poll(self) -> None: + log = self.run_script("twisted_reactor_poll.py") + assert "Spider closed (finished)" in log + assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log + + def test_twisted_reactor_asyncio_custom_settings_conflict(self) -> None: + log = self.run_script("twisted_reactor_custom_settings_conflict.py") + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + assert ( + "(twisted.internet.selectreactor.SelectReactor) does not match the requested one" + in log + ) + + def test_reactorless(self) -> None: + log = self.run_script("reactorless.py") + assert ( + "RuntimeError: CrawlerProcess doesn't support TWISTED_REACTOR_ENABLED=False" + in log + ) + + +class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("AsyncCrawlerProcess") + + def test_twisted_reactor_custom_settings_select(self) -> None: + log = self.run_script("twisted_reactor_custom_settings_select.py") + assert "Spider closed (finished)" not in log + assert ( + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor) " + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_same_loop(self) -> None: + log = self.run_script("asyncio_custom_loop_custom_settings_same.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_different_loop(self) -> None: + log = self.run_script("asyncio_custom_loop_custom_settings_different.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ) in log + + def test_reactorless_simple(self) -> None: + log = self.run_script("reactorless_simple.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "is_reactorless(): True" in log + assert "ERROR: " not in log + assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2 + assert log.count("WARNING: ") == 2 + + def test_reactorless_custom_settings(self) -> None: + """Setting TWISTED_REACTOR_ENABLED=False in spider settings is not + currently supported, AsyncCrawlerProcess will install a reactor in this + case. + """ + log = self.run_script("reactorless_custom_settings.py") + assert "Spider closed (finished)" not in log + assert ( + "TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed." + in log + ) + + def test_reactorless_datauri(self) -> None: + log = self.run_script("reactorless_datauri.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "{'data': 'foo'}" in log + assert "'item_scraped_count': 1" in log + assert "ERROR: " not in log + assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2 + assert log.count("WARNING: ") == 2 + + def test_reactorless_import_hook(self) -> None: + log = self.run_script("reactorless_import_hook.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "ImportError: Import of twisted.internet.reactor is forbidden" in log + + def test_reactorless_telnetconsole_default(self) -> None: + """By default TWISTED_REACTOR_ENABLED=False silently sets TELNETCONSOLE_ENABLED=False.""" + log = self.run_script("reactorless_simple.py") # no need for a separate script + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "The TelnetConsole extension requires a Twisted reactor" not in log + assert "scrapy.extensions.telnet.TelnetConsole" not in log + + def test_reactorless_telnetconsole_disabled(self) -> None: + """Explicit TELNETCONSOLE_ENABLED=False, there are no warnings.""" + log = self.run_script("reactorless_telnetconsole_disabled.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "The TelnetConsole extension requires a Twisted reactor" not in log + assert "scrapy.extensions.telnet.TelnetConsole" not in log + + def test_reactorless_telnetconsole_enabled(self) -> None: + """Explicit TELNETCONSOLE_ENABLED=True, the user gets a warning.""" + log = self.run_script("reactorless_telnetconsole_enabled.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "The TelnetConsole extension requires a Twisted reactor" in log + + def test_reactorless_reactor(self) -> None: + log = self.run_script("reactorless_reactor.py") + assert ( + "RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed" + in log + ) + + def test_shutdown_graceful(self) -> None: + self._test_shutdown_graceful("reactorless_sleeping.py") + + @coroutine_test + async def test_shutdown_forced(self) -> None: + await self._test_shutdown_forced("reactorless_sleeping.py") + + +class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): + """Common tests between CrawlerRunner and AsyncCrawlerRunner, + with the same file names and expectations. + """ + + def test_simple(self) -> None: + log = self.run_script("simple.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "is_reactorless(): False" in log + + def test_multi_parallel(self) -> None: + log = self.run_script("multi_parallel.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Spider opened.+Closing spider.+Closing spider", + log, + re.DOTALL, + ) + + def test_multi_seq(self) -> None: + log = self.run_script("multi_seq.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Closing spider.+Spider opened.+Closing spider", + log, + re.DOTALL, + ) + + @pytest.mark.requires_uvloop + def test_custom_loop_same(self) -> None: + log = self.run_script("custom_loop_same.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_custom_loop_different(self) -> None: + log = self.run_script("custom_loop_different.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ) in log + + def test_no_reactor(self) -> None: + log = self.run_script("no_reactor.py") + assert "Spider closed (finished)" not in log + assert ( + "RuntimeError: We expected a Twisted reactor to be installed but it isn't." + in log + ) + + +class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("CrawlerRunner") + + def test_explicit_default_reactor(self) -> None: + log = self.run_script("explicit_default_reactor.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + + def test_response_ip_address(self) -> None: + log = self.run_script("ip_address.py") + assert "INFO: Spider closed (finished)" in log + assert "INFO: Host: not.a.real.domain" in log + assert "INFO: Type: " in log + assert "INFO: IP address: 127.0.0.1" in log + + def test_change_default_reactor(self) -> None: + log = self.run_script("change_reactor.py") + assert ( + "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "DEBUG: Using asyncio event loop" in log + + def test_reactorless(self) -> None: + log = self.run_script("reactorless.py") + assert ( + "RuntimeError: CrawlerRunner doesn't support TWISTED_REACTOR_ENABLED=False" + in log + ) + + +class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("AsyncCrawlerRunner") + + def test_simple_default_reactor(self) -> None: + log = self.run_script("simple_default_reactor.py") + assert "Spider closed (finished)" not in log + assert ( + "RuntimeError: When TWISTED_REACTOR_ENABLED is True, " + "AsyncCrawlerRunner requires that the installed Twisted reactor" + ) in log + + def test_reactorless_simple(self) -> None: + log = self.run_script("reactorless_simple.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "is_reactorless(): True" in log + assert "ERROR: " not in log + assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2 + assert log.count("WARNING: ") == 2 + + def test_reactorless_custom_settings(self) -> None: + """Setting TWISTED_REACTOR_ENABLED=False in spider settings is not + currently supported, AsyncCrawlerRunner will expect a reactor installed + by the user. + """ + log = self.run_script("reactorless_custom_settings.py") + assert "Spider closed (finished)" not in log + assert "We expected a Twisted reactor to be installed but it isn't." in log + + def test_reactorless_datauri(self) -> None: + log = self.run_script("reactorless_datauri.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "{'data': 'foo'}" in log + assert "'item_scraped_count': 1" in log + assert "ERROR: " not in log + assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2 + assert log.count("WARNING: ") == 2 + + def test_reactorless_reactor(self) -> None: + log = self.run_script("reactorless_reactor.py") + assert ( + "RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed" + in log + ) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index a39ed0694..4436efd9b 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,24 +1,13 @@ import os import re from configparser import ConfigParser -from importlib import import_module from pathlib import Path +import pytest from twisted import version as twisted_version -from twisted.trial import unittest -class ScrapyUtilsTest(unittest.TestCase): - def test_required_openssl_version(self): - try: - module = import_module("OpenSSL") - except ImportError: - raise unittest.SkipTest("OpenSSL is not available") - - if hasattr(module, "__version__"): - installed_version = [int(x) for x in module.__version__.split(".")[:2]] - assert installed_version >= [0, 6], "OpenSSL >= 0.6 required" - +class TestScrapyUtils: def test_pinned_twisted_version(self): """When running tests within a Tox environment with pinned dependencies, make sure that the version of Twisted is the pinned @@ -27,13 +16,13 @@ class ScrapyUtilsTest(unittest.TestCase): See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011 """ if not os.environ.get("_SCRAPY_PINNED", None): - self.skipTest("Not in a pinned environment") + pytest.skip("Not in a pinned environment") tox_config_file_path = Path(__file__).parent / ".." / "tox.ini" config_parser = ConfigParser() config_parser.read(tox_config_file_path) - pattern = r"Twisted\[http2\]==([\d.]+)" + pattern = r"Twisted==([\d.]+)" match = re.search(pattern, config_parser["pinned"]["deps"]) pinned_twisted_version_string = match[1] - self.assertEqual(twisted_version.short(), pinned_twisted_version_string) + assert twisted_version.short() == pinned_twisted_version_string diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py new file mode 100644 index 000000000..fdacad963 --- /dev/null +++ b/tests/test_downloader_handler_httpx.py @@ -0,0 +1,161 @@ +"""Tests for scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler.""" + +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING, Any, ClassVar + +import pytest + +from scrapy import Request +from scrapy.core.downloader.handlers._httpx import ( + HAS_HTTP2, + HAS_SOCKS, + HttpxDownloadHandler, +) +from scrapy.exceptions import DownloadFailedError +from tests.test_downloader_handlers_http_base import ( + TestHttpBase, + TestHttpProxyBase, + TestHttpsBase, + TestHttpsCustomCiphersBase, + TestHttpsInvalidDNSIdBase, + TestHttpsInvalidDNSPatternBase, + TestHttpsTLSVersionBase, + TestHttpsWrongHostnameBase, + TestHttpWithCrawlerBase, + TestMitmProxyBase, + TestRealWebsiteBase, + TestSimpleHttpsBase, +) +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from scrapy.core.downloader.handlers import DownloadHandlerProtocol + from tests.mockserver.http import MockServer + + +pytestmark = pytest.mark.only_asyncio + +pytest.importorskip("httpx") + + +class HttpxDownloadHandlerMixin: + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HttpxDownloadHandler + + @property + def settings_dict(self) -> dict[str, Any] | None: + return { + "DOWNLOAD_HANDLERS": { + "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + } + } + + +class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase): + handler_supports_bindaddress_meta = False + + @pytest.mark.skipif( + sys.platform == "darwin", + reason="127.0.0.2 is not available on macOS by default", + ) + @coroutine_test + async def test_bind_address_port_warning( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + request = Request(mockserver.url("/client-ip")) + async with self.get_dh( + {"DOWNLOAD_BIND_ADDRESS": ("127.0.0.2", 12345)} + ) as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"127.0.0.2" + assert "DOWNLOAD_BIND_ADDRESS specifies a port (12345)" in caplog.text + assert "Ignoring the port" in caplog.text + + +class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase): + handler_supports_bindaddress_meta = False + tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher" + + @pytest.mark.skip(reason="The check is Twisted-specific") + def test_verify_certs_deprecated(self) -> None: # type: ignore[override] + pass + + +@pytest.mark.skipif(not HAS_HTTP2, reason="No HTTP/2 support in HttpxDownloadHandler") +class TestHttp2(TestHttps): + http2 = True + handler_supports_http2_dataloss = False + + default_handler_settings: ClassVar[dict[str, Any]] = { + "HTTPX_HTTP2_ENABLED": True, + } + + @coroutine_test + async def test_protocol(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/host", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.protocol == "HTTP/2" + + @coroutine_test + async def test_data_loss_handling(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/broken", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + with pytest.raises(DownloadFailedError): + await download_handler.download_request(request) + + +class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase): + pass + + +class TestHttpsWrongHostname(HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase): + pass + + +class TestHttpsInvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase): + pass + + +class TestHttpsInvalidDNSPattern( + HttpxDownloadHandlerMixin, TestHttpsInvalidDNSPatternBase +): + pass + + +class TestHttpsCustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase): + pass + + +class TestHttpsTLSVersion(HttpxDownloadHandlerMixin, TestHttpsTLSVersionBase): + pass + + +class TestHttpWithCrawler(HttpxDownloadHandlerMixin, TestHttpWithCrawlerBase): + pass + + +class TestHttpsWithCrawler(TestHttpWithCrawler): + is_secure = True + + +class TestHttpProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase): + expected_http_proxy_request_body = b"http://example.com/" + + +class TestHttpsProxy(TestHttpProxy): + is_secure = True + + +@pytest.mark.requires_mitmproxy +class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase): + handler_supports_socks = HAS_SOCKS + + +@pytest.mark.requires_internet +class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase): + pass diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py new file mode 100644 index 000000000..361e91382 --- /dev/null +++ b/tests/test_downloader_handler_twisted_ftp.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +import os +import sys +from abc import ABC, abstractmethod +from pathlib import Path +from tempfile import mkstemp +from typing import TYPE_CHECKING, Any + +import pytest +from pytest_twisted import async_yield_fixture +from twisted.cred import checkers, credentials, portal + +from scrapy import Spider +from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler +from scrapy.crawler import Crawler +from scrapy.exceptions import NotConfigured +from scrapy.http import HtmlResponse, Request, Response +from scrapy.http.response.text import TextResponse +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.python import to_bytes +from scrapy.utils.test import get_crawler + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Generator + + from twisted.protocols.ftp import FTPFactory + + +pytestmark = pytest.mark.requires_reactor # FTPDownloadHandler requires a reactor + + +class TestFTPBase(ABC): + username = "scrapy" + password = "passwd" + req_meta: dict[str, Any] = {"ftp_user": username, "ftp_password": password} + + test_files = ( + ("file.txt", b"I have the power!"), + ("file with spaces.txt", b"Moooooooooo power!"), + ("html-file-without-extension", b"\n."), + ) + + @abstractmethod + def _create_files(self, root: Path) -> None: + raise NotImplementedError + + @abstractmethod + def _get_factory(self, tmp_path: Path) -> FTPFactory: + raise NotImplementedError + + @async_yield_fixture # type: ignore[untyped-decorator] + async def server_url(self, tmp_path: Path) -> AsyncGenerator[str]: + from twisted.internet import reactor + + self._create_files(tmp_path) + factory = self._get_factory(tmp_path) + port = reactor.listenTCP(0, factory, interface="127.0.0.1") + portno = port.getHost().port + + yield f"https://127.0.0.1:{portno}/" + + await port.stopListening() + + @staticmethod + @pytest.fixture + def dh() -> Generator[FTPDownloadHandler]: + crawler = get_crawler() + dh = build_from_crawler(FTPDownloadHandler, crawler) + + yield dh + + # if the test was skipped, there will be no client attribute + if hasattr(dh, "client"): + assert dh.client.transport + dh.client.transport.loseConnection() + + @deferred_f_from_coro_f + async def test_ftp_download_success( + self, server_url: str, dh: FTPDownloadHandler + ) -> None: + request = Request(url=server_url + "file.txt", meta=self.req_meta) + r = await dh.download_request(request) + assert r.status == 200 + assert r.body == b"I have the power!" + assert r.headers == {b"Local Filename": [b""], b"Size": [b"17"]} + assert r.protocol is None + + @deferred_f_from_coro_f + async def test_ftp_download_path_with_spaces( + self, server_url: str, dh: FTPDownloadHandler + ) -> None: + request = Request( + url=server_url + "file with spaces.txt", + meta=self.req_meta, + ) + r = await dh.download_request(request) + assert r.status == 200 + assert r.body == b"Moooooooooo power!" + assert r.headers == {b"Local Filename": [b""], b"Size": [b"18"]} + + @deferred_f_from_coro_f + async def test_ftp_download_nonexistent( + self, server_url: str, dh: FTPDownloadHandler + ) -> None: + request = Request(url=server_url + "nonexistent.txt", meta=self.req_meta) + r = await dh.download_request(request) + assert r.status == 404 + assert r.body == b"['550 nonexistent.txt: No such file or directory.']" + + @deferred_f_from_coro_f + async def test_ftp_local_filename( + self, server_url: str, dh: FTPDownloadHandler + ) -> None: + f, local_fname = mkstemp() + fname_bytes = to_bytes(local_fname) + local_path = Path(local_fname) + os.close(f) + meta = {"ftp_local_filename": fname_bytes} + meta.update(self.req_meta) + request = Request(url=server_url + "file.txt", meta=meta) + r = await dh.download_request(request) + assert r.body == fname_bytes + assert r.headers == {b"Local Filename": [fname_bytes], b"Size": [b"17"]} + assert local_path.exists() + assert local_path.read_bytes() == b"I have the power!" + local_path.unlink() + + @pytest.mark.parametrize( + ("filename", "response_class"), + [ + ("file.txt", TextResponse), + ("html-file-without-extension", HtmlResponse), + ], + ) + @deferred_f_from_coro_f + async def test_response_class( + self, + filename: str, + response_class: type[Response], + server_url: str, + dh: FTPDownloadHandler, + ) -> None: + f, local_fname = mkstemp() + local_fname_path = Path(local_fname) + os.close(f) + meta = {} + meta.update(self.req_meta) + request = Request(url=server_url + filename, meta=meta) + r = await dh.download_request(request) + assert type(r) is response_class # pylint: disable=unidiomatic-typecheck + local_fname_path.unlink() + + +class TestFTP(TestFTPBase): + def _create_files(self, root: Path) -> None: + userdir = root / self.username + userdir.mkdir() + for filename, content in self.test_files: + (userdir / filename).write_bytes(content) + + def _get_factory(self, root): + from twisted.protocols.ftp import FTPFactory, FTPRealm + + realm = FTPRealm(anonymousRoot=str(root), userHome=str(root)) + p = portal.Portal(realm) + users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() + users_checker.addUser(self.username, self.password) + p.registerChecker(users_checker, credentials.IUsernamePassword) + return FTPFactory(portal=p) + + @deferred_f_from_coro_f + async def test_invalid_credentials( + self, server_url: str, dh: FTPDownloadHandler, reactor_pytest: str + ) -> None: + if reactor_pytest == "asyncio" and sys.platform == "win32": + pytest.skip( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) + + from twisted.protocols.ftp import ConnectionLost + + meta = dict(self.req_meta) + meta.update({"ftp_password": "invalid"}) + request = Request(url=server_url + "file.txt", meta=meta) + with pytest.raises(ConnectionLost): + await dh.download_request(request) + + +class TestAnonymousFTP(TestFTPBase): + username = "anonymous" + req_meta = {} + + def _create_files(self, root: Path) -> None: + for filename, content in self.test_files: + (root / filename).write_bytes(content) + + def _get_factory(self, tmp_path): + from twisted.protocols.ftp import FTPFactory, FTPRealm + + realm = FTPRealm(anonymousRoot=str(tmp_path)) + p = portal.Portal(realm) + p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous) + return FTPFactory(portal=p, userAnonymous=self.username) + + +def test_not_configured_without_reactor() -> None: + crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False}) + with pytest.raises(NotConfigured): + FTPDownloadHandler.from_crawler(crawler) diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py new file mode 100644 index 000000000..79750a136 --- /dev/null +++ b/tests/test_downloader_handler_twisted_http11.py @@ -0,0 +1,119 @@ +"""Tests for scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler.""" + +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING, Any + +import pytest + +from scrapy import Spider +from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler +from scrapy.crawler import Crawler +from scrapy.exceptions import NotConfigured +from tests.test_downloader_handlers_http_base import ( + TestHttpBase, + TestHttpProxyBase, + TestHttpsBase, + TestHttpsCustomCiphersBase, + TestHttpsInvalidDNSIdBase, + TestHttpsInvalidDNSPatternBase, + TestHttpsTLSVersionBase, + TestHttpsWrongHostnameBase, + TestHttpWithCrawlerBase, + TestMitmProxyBase, + TestRealWebsiteBase, + TestSimpleHttpsBase, +) + +if TYPE_CHECKING: + from scrapy.core.downloader.handlers import DownloadHandlerProtocol + + +pytestmark = pytest.mark.requires_reactor # HTTP11DownloadHandler requires a reactor + + +class HTTP11DownloadHandlerMixin: + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP11DownloadHandler + + @property + def settings_dict(self) -> dict[str, Any] | None: + return { + "DOWNLOAD_HANDLERS": { + "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", + "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", + } + } + + +def test_not_configured_without_reactor() -> None: + crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False}) + with pytest.raises(NotConfigured): + HTTP11DownloadHandler.from_crawler(crawler) + + +class TestHttp(HTTP11DownloadHandlerMixin, TestHttpBase): + pass + + +class TestHttps(HTTP11DownloadHandlerMixin, TestHttpsBase): + pass + + +class TestSimpleHttps(HTTP11DownloadHandlerMixin, TestSimpleHttpsBase): + pass + + +class TestHttpsWrongHostname(HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase): + pass + + +class TestHttpsInvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): + pass + + +class TestHttpsInvalidDNSPattern( + HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase +): + pass + + +class TestHttpsCustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase): + pass + + +class TestHttpsTLSVersion(HTTP11DownloadHandlerMixin, TestHttpsTLSVersionBase): + pass + + +class TestHttpWithCrawler(HTTP11DownloadHandlerMixin, TestHttpWithCrawlerBase): + pass + + +class TestHttpsWithCrawler(TestHttpWithCrawler): + is_secure = True + + +class TestHttpProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase): + pass + + +class TestHttpsProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase): + is_secure = True + # not implemented + handler_supports_tls_in_tls = False + + +@pytest.mark.requires_mitmproxy +class TestMitmProxy(HTTP11DownloadHandlerMixin, TestMitmProxyBase): + # not implemented + handler_supports_tls_in_tls = False + + +@pytest.mark.requires_internet +class TestRealWebsite(HTTP11DownloadHandlerMixin, TestRealWebsiteBase): + @property + def platform_cert_store_works(self) -> bool: + return sys.platform != "win32" diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py new file mode 100644 index 000000000..5f79a5453 --- /dev/null +++ b/tests/test_downloader_handler_twisted_http2.py @@ -0,0 +1,212 @@ +"""Tests for scrapy.core.downloader.handlers.http2.H2DownloadHandler.""" + +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING, Any + +import pytest +from testfixtures import LogCapture +from twisted.web.http import H2_ENABLED + +from scrapy import Spider +from scrapy.crawler import Crawler +from scrapy.exceptions import DownloadFailedError, NotConfigured +from scrapy.http import Request +from tests.test_downloader_handlers_http_base import ( + TestHttpProxyBase, + TestHttpsBase, + TestHttpsCustomCiphersBase, + TestHttpsInvalidDNSIdBase, + TestHttpsInvalidDNSPatternBase, + TestHttpsTLSVersionBase, + TestHttpsWrongHostnameBase, + TestHttpWithCrawlerBase, + TestMitmProxyBase, + TestRealWebsiteBase, +) +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from scrapy.core.downloader.handlers import DownloadHandlerProtocol + from tests.mockserver.http import MockServer + + +pytestmark = [ + pytest.mark.requires_reactor, # H2DownloadHandler requires a reactor + pytest.mark.skipif( + not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled" + ), +] + + +class H2DownloadHandlerMixin: + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + # the import can fail when H2_ENABLED is False + from scrapy.core.downloader.handlers.http2 import ( # noqa: PLC0415 + H2DownloadHandler, + ) + + return H2DownloadHandler + + @property + def settings_dict(self) -> dict[str, Any] | None: + return { + "DOWNLOAD_HANDLERS": { + "http": None, + "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler", + } + } + + +def test_not_configured_without_reactor() -> None: + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler # noqa: PLC0415 + + crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False}) + with pytest.raises(NotConfigured): + H2DownloadHandler.from_crawler(crawler) + + +class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase): + http2 = True + handler_supports_http2_dataloss = False + + @coroutine_test + async def test_protocol(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/host", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.protocol == "h2" + + def test_download_conn_failed(self) -> None: # type: ignore[override] + # Unlike HTTP11DownloadHandler which raises it from download_request() + # (without any special handling), here ConnectionRefusedError (raised in + # twisted.internet.endpoints.startConnectionAttempts()) bubbles up as + # an unhandled exception in a Deferred and the handler waits until + # DOWNLOAD_TIMEOUT. + pytest.skip("The handler doesn't properly reraise ConnectionRefusedError") + + def test_download_dns_error(self) -> None: # type: ignore[override] + # Unlike HTTP11DownloadHandler which raises it from download_request() + # (without any special handling), here DNSLookupError (raised in + # twisted.internet.endpoints.startConnectionAttempts()) bubbles up as + # an unhandled exception in a Deferred and the handler waits until + # DOWNLOAD_TIMEOUT. + pytest.skip("The handler doesn't properly reraise DNSLookupError") + + @coroutine_test + async def test_concurrent_requests_same_domain( + self, mockserver: MockServer + ) -> None: + request1 = Request(mockserver.url("/text", is_secure=self.is_secure)) + request2 = Request( + mockserver.url("/echo", is_secure=self.is_secure), method="POST" + ) + async with self.get_dh() as download_handler: + response1 = await download_handler.download_request(request1) + assert response1.body == b"Works" + response2 = await download_handler.download_request(request2) + assert response2.headers["Content-Length"] == b"79" + + @pytest.mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247") + @coroutine_test + async def test_connect_request(self, mockserver: MockServer) -> None: + request = Request( + mockserver.url("/file", is_secure=self.is_secure), method="CONNECT" + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"" + + @coroutine_test + async def test_custom_content_length_good(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/contentlength", is_secure=self.is_secure)) + custom_content_length = str(len(request.body)) + request.headers["Content-Length"] = custom_content_length + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.text == custom_content_length + + @coroutine_test + async def test_custom_content_length_bad(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/contentlength", is_secure=self.is_secure)) + actual_content_length = str(len(request.body)) + bad_content_length = str(len(request.body) + 1) + request.headers["Content-Length"] = bad_content_length + async with self.get_dh() as download_handler: + with LogCapture() as log: + response = await download_handler.download_request(request) + assert response.text == actual_content_length + log.check_present( + ( + "scrapy.core.http2.stream", + "WARNING", + f"Ignoring bad Content-Length header " + f"{bad_content_length!r} of request {request}, sending " + f"{actual_content_length!r} instead", + ) + ) + + @coroutine_test + async def test_data_loss_handling(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/broken", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + with pytest.raises(DownloadFailedError): + await download_handler.download_request(request) + + +class TestHttp2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): + pass + + +class TestHttp2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): + pass + + +class TestHttp2InvalidDNSPattern( + H2DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase +): + pass + + +class TestHttp2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase): + pass + + +class TestHttp2TLSVersion(H2DownloadHandlerMixin, TestHttpsTLSVersionBase): + pass + + +class TestHttp2WithCrawler(H2DownloadHandlerMixin, TestHttpWithCrawlerBase): + is_secure = True + + def test_bytes_received_stop_download_callback(self) -> None: # type: ignore[override] + pytest.skip("bytes_received support is not implemented") + + def test_bytes_received_stop_download_errback(self) -> None: # type: ignore[override] + pytest.skip("bytes_received support is not implemented") + + def test_headers_received_stop_download_callback(self) -> None: # type: ignore[override] + pytest.skip("headers_received support is not implemented") + + def test_headers_received_stop_download_errback(self) -> None: # type: ignore[override] + pytest.skip("headers_received support is not implemented") + + +@pytest.mark.skip(reason="Proxy support is not implemented yet") +class TestHttp2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): + is_secure = True + + +@pytest.mark.skip(reason="Proxy support is not implemented yet") +@pytest.mark.requires_mitmproxy +class TestMitmProxy(H2DownloadHandlerMixin, TestMitmProxyBase): + pass + + +@pytest.mark.requires_internet +class TestRealWebsite(H2DownloadHandlerMixin, TestRealWebsiteBase): + @property + def platform_cert_store_works(self) -> bool: + return sys.platform != "win32" diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index a0c46dd09..eadb7740e 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,57 +1,40 @@ +"""Tests for DownloadHandlers and for specific non-HTTP download handlers.""" + +from __future__ import annotations + import contextlib import os -import shutil -import sys -import tempfile -from base64 import b64encode from pathlib import Path -from typing import Optional, Type -from unittest import SkipTest, mock +from tempfile import mkdtemp, mkstemp +from unittest import mock -from testfixtures import LogCapture -from twisted.cred import checkers, credentials, portal -from twisted.internet import defer, error, reactor -from twisted.protocols.policies import WrappingFactory -from twisted.trial import unittest -from twisted.web import resource, server, static, util -from twisted.web._newclient import ResponseFailed -from twisted.web.http import _DataLoss +import pytest from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler -from scrapy.core.downloader.handlers.http import HTTPDownloadHandler -from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler -from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler -from scrapy.exceptions import NotConfigured -from scrapy.http import Headers, HtmlResponse, Request, XmlResponse -from scrapy.http.response.text import TextResponse -from scrapy.spiders import Spider +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.http import Request +from scrapy.responsetypes import responsetypes +from scrapy.utils.boto import is_botocore_available from scrapy.utils.misc import build_from_crawler -from scrapy.utils.python import to_bytes -from scrapy.utils.test import get_crawler, skip_if_no_boto -from tests import NON_EXISTING_RESOLVABLE -from tests.mockserver import ( - Echo, - ForeverTakingResource, - HostHeaderResource, - MockServer, - NoLengthResource, - PayloadResource, - ssl_context_factory, -) -from tests.spiders import SingleRequestSpider +from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test class DummyDH: lazy = False + async def download_request(self, request): + pass + class DummyLazyDH: - # Default is lazy for backward compatibility - pass + # Default (but deprecated) is lazy for backward compatibility + async def download_request(self, request): + pass class OffDH: @@ -65,813 +48,151 @@ class OffDH: return cls(crawler) -class LoadTestCase(unittest.TestCase): +class BuggyDH: + lazy = False + + def __init__(self, crawler): + raise ValueError + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + +class TestLoad: def test_enabled_handler(self): handlers = {"scheme": DummyDH} crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) dh = DownloadHandlers(crawler) - self.assertIn("scheme", dh._schemes) - self.assertIn("scheme", dh._handlers) - self.assertNotIn("scheme", dh._notconfigured) + assert "scheme" in dh._schemes + assert "scheme" in dh._handlers + assert "scheme" not in dh._notconfigured def test_not_configured_handler(self): handlers = {"scheme": OffDH} crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) dh = DownloadHandlers(crawler) - self.assertIn("scheme", dh._schemes) - self.assertNotIn("scheme", dh._handlers) - self.assertIn("scheme", dh._notconfigured) + assert "scheme" in dh._schemes + assert "scheme" not in dh._handlers + assert "scheme" in dh._notconfigured + + def test_buggy_handler(self, caplog: pytest.LogCaptureFixture) -> None: + handlers = {"scheme": BuggyDH} + crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) + dh = DownloadHandlers(crawler) + assert "scheme" in dh._schemes + assert "scheme" not in dh._handlers + assert "scheme" in dh._notconfigured + assert ( + 'Loading "" for scheme "scheme"' + in caplog.text + ) def test_disabled_handler(self): handlers = {"scheme": None} crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) dh = DownloadHandlers(crawler) - self.assertNotIn("scheme", dh._schemes) - for scheme in handlers: # force load handlers - dh._get_handler(scheme) - self.assertNotIn("scheme", dh._handlers) - self.assertIn("scheme", dh._notconfigured) + assert "scheme" not in dh._schemes + assert dh._get_handler("scheme") is None + assert "scheme" not in dh._handlers + assert "scheme" in dh._notconfigured + # get the handler again to cover the code that gets it from dh._notconfigured + assert dh._get_handler("scheme") is None def test_lazy_handlers(self): handlers = {"scheme": DummyLazyDH} crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) - dh = DownloadHandlers(crawler) - self.assertIn("scheme", dh._schemes) - self.assertNotIn("scheme", dh._handlers) - for scheme in handlers: # force load lazy handler - dh._get_handler(scheme) - self.assertIn("scheme", dh._handlers) - self.assertNotIn("scheme", dh._notconfigured) + with pytest.warns( + ScrapyDeprecationWarning, + match="DummyLazyDH doesn't define a 'lazy' attribute", + ): + dh = DownloadHandlers(crawler) + assert "scheme" in dh._schemes + assert "scheme" not in dh._handlers + handler = dh._get_handler("scheme") # force load lazy handler + assert handler + assert "scheme" in dh._handlers + assert "scheme" not in dh._notconfigured -class FileTestCase(unittest.TestCase): - def setUp(self): +class TestFile: + def setup_method(self): # add a special char to check that they are handled correctly - self.tmpname = Path(self.mktemp() + "^") + self.fd, self.tmpname = mkstemp(suffix="^") Path(self.tmpname).write_text("0123456789", encoding="utf-8") - handler = build_from_crawler(FileDownloadHandler, get_crawler()) - self.download_request = handler.download_request + download_handler = build_from_crawler(FileDownloadHandler, get_crawler()) + self.download_request = download_handler.download_request - def tearDown(self): - self.tmpname.unlink() + def teardown_method(self): + os.close(self.fd) + Path(self.tmpname).unlink() - def test_download(self): - def _test(response): - self.assertEqual(response.url, request.url) - self.assertEqual(response.status, 200) - self.assertEqual(response.body, b"0123456789") - self.assertEqual(response.protocol, None) - - request = Request(path_to_file_uri(str(self.tmpname))) + @coroutine_test + async def test_download(self): + request = Request(path_to_file_uri(self.tmpname)) assert request.url.upper().endswith("%5E") - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_non_existent(self): - request = Request(path_to_file_uri(self.mktemp())) - d = self.download_request(request, Spider("foo")) - return self.assertFailure(d, OSError) - - -class ContentLengthHeaderResource(resource.Resource): - """ - A testing resource which renders itself as the value of the Content-Length - header from the request. - """ - - def render(self, request): - return request.requestHeaders.getRawHeaders(b"content-length")[0] - - -class ChunkedResource(resource.Resource): - def render(self, request): - def response(): - request.write(b"chunked ") - request.write(b"content\n") - request.finish() - - reactor.callLater(0, response) - return server.NOT_DONE_YET - - -class BrokenChunkedResource(resource.Resource): - def render(self, request): - def response(): - request.write(b"chunked ") - request.write(b"content\n") - # Disable terminating chunk on finish. - request.chunked = False - closeConnection(request) - - reactor.callLater(0, response) - return server.NOT_DONE_YET - - -class BrokenDownloadResource(resource.Resource): - def render(self, request): - def response(): - request.setHeader(b"Content-Length", b"20") - request.write(b"partial") - closeConnection(request) - - reactor.callLater(0, response) - return server.NOT_DONE_YET - - -def closeConnection(request): - # We have to force a disconnection for HTTP/1.1 clients. Otherwise - # client keeps the connection open waiting for more data. - if hasattr(request.channel, "loseConnection"): # twisted >=16.3.0 - request.channel.loseConnection() - else: - request.channel.transport.loseConnection() - request.finish() - - -class EmptyContentTypeHeaderResource(resource.Resource): - """ - A testing resource which renders itself as the value of request body - without content-type header in response. - """ - - def render(self, request): - request.setHeader("content-type", "") - return request.content.read() - - -class LargeChunkedFileResource(resource.Resource): - def render(self, request): - def response(): - for i in range(1024): - request.write(b"x" * 1024) - request.finish() - - reactor.callLater(0, response) - return server.NOT_DONE_YET - - -class DuplicateHeaderResource(resource.Resource): - def render(self, request): - request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"]) - return b"" - - -class HttpTestCase(unittest.TestCase): - scheme = "http" - download_handler_cls: Type = HTTPDownloadHandler - - # only used for HTTPS tests - keyfile = "keys/localhost.key" - certfile = "keys/localhost.crt" - - def setUp(self): - self.tmpname = Path(self.mktemp()) - self.tmpname.mkdir() - (self.tmpname / "file").write_bytes(b"0123456789") - r = static.File(str(self.tmpname)) - r.putChild(b"redirect", util.Redirect(b"/file")) - r.putChild(b"wait", ForeverTakingResource()) - r.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) - r.putChild(b"nolength", NoLengthResource()) - r.putChild(b"host", HostHeaderResource()) - r.putChild(b"payload", PayloadResource()) - r.putChild(b"broken", BrokenDownloadResource()) - r.putChild(b"chunked", ChunkedResource()) - r.putChild(b"broken-chunked", BrokenChunkedResource()) - r.putChild(b"contentlength", ContentLengthHeaderResource()) - r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) - r.putChild(b"largechunkedfile", LargeChunkedFileResource()) - r.putChild(b"duplicate-header", DuplicateHeaderResource()) - r.putChild(b"echo", Echo()) - self.site = server.Site(r, timeout=None) - self.wrapper = WrappingFactory(self.site) - self.host = "localhost" - if self.scheme == "https": - # Using WrappingFactory do not enable HTTP/2 failing all the - # tests with H2DownloadHandler - self.port = reactor.listenSSL( - 0, - self.site, - ssl_context_factory(self.keyfile, self.certfile), - interface=self.host, - ) - else: - self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) - self.portno = self.port.getHost().port - self.download_handler = build_from_crawler( - self.download_handler_cls, get_crawler() - ) - self.download_request = self.download_handler.download_request - - @defer.inlineCallbacks - def tearDown(self): - yield self.port.stopListening() - if hasattr(self.download_handler, "close"): - yield self.download_handler.close() - shutil.rmtree(self.tmpname) - - def getURL(self, path): - return f"{self.scheme}://{self.host}:{self.portno}/{path}" - - def test_download(self): - request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d - - def test_download_head(self): - request = Request(self.getURL("file"), method="HEAD") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"") - return d - - def test_redirect_status(self): - request = Request(self.getURL("redirect")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.status) - d.addCallback(self.assertEqual, 302) - return d - - def test_redirect_status_head(self): - request = Request(self.getURL("redirect"), method="HEAD") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.status) - d.addCallback(self.assertEqual, 302) - return d - - @defer.inlineCallbacks - def test_timeout_download_from_spider_nodata_rcvd(self): - if self.reactor_pytest == "asyncio" and sys.platform == "win32": - # https://twistedmatrix.com/trac/ticket/10279 - raise unittest.SkipTest( - "This test produces DirtyReactorAggregateError on Windows with asyncio" - ) - - # client connects but no data is received - spider = Spider("foo") - meta = {"download_timeout": 0.5} - request = Request(self.getURL("wait"), meta=meta) - d = self.download_request(request, spider) - yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) - - @defer.inlineCallbacks - def test_timeout_download_from_spider_server_hangs(self): - if self.reactor_pytest == "asyncio" and sys.platform == "win32": - # https://twistedmatrix.com/trac/ticket/10279 - raise unittest.SkipTest( - "This test produces DirtyReactorAggregateError on Windows with asyncio" - ) - # client connects, server send headers and some body bytes but hangs - spider = Spider("foo") - meta = {"download_timeout": 0.5} - request = Request(self.getURL("hang-after-headers"), meta=meta) - d = self.download_request(request, spider) - yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) - - def test_host_header_not_in_request_headers(self): - def _test(response): - self.assertEqual(response.body, to_bytes(f"{self.host}:{self.portno}")) - self.assertEqual(request.headers, {}) - - request = Request(self.getURL("host")) - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_host_header_seted_in_request_headers(self): - host = self.host + ":" + str(self.portno) - - def _test(response): - self.assertEqual(response.body, host.encode()) - self.assertEqual(request.headers.get("Host"), host.encode()) - - request = Request(self.getURL("host"), headers={"Host": host}) - return self.download_request(request, Spider("foo")).addCallback(_test) - - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"localhost") - return d - - def test_content_length_zero_bodyless_post_request_headers(self): - """Tests if "Content-Length: 0" is sent for bodyless POST requests. - - This is not strictly required by HTTP RFCs but can cause trouble - for some web servers. - See: - https://github.com/scrapy/scrapy/issues/823 - https://issues.apache.org/jira/browse/TS-2902 - https://github.com/kennethreitz/requests/issues/405 - https://bugs.python.org/issue14721 - """ - - def _test(response): - self.assertEqual(response.body, b"0") - - request = Request(self.getURL("contentlength"), method="POST") - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_content_length_zero_bodyless_post_only_one(self): - def _test(response): - import json - - headers = Headers(json.loads(response.text)["headers"]) - contentlengths = headers.getlist("Content-Length") - self.assertEqual(len(contentlengths), 1) - self.assertEqual(contentlengths, [b"0"]) - - request = Request(self.getURL("echo"), method="POST") - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_payload(self): - body = b"1" * 100 # PayloadResource requires body length to be 100 - request = Request(self.getURL("payload"), method="POST", body=body) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, body) - return d - - def test_response_header_content_length(self): - request = Request(self.getURL("file"), method=b"GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.headers[b"content-length"]) - d.addCallback(self.assertEqual, b"159") - return d - - def _test_response_class(self, filename, body, response_class): - def _test(response): - self.assertEqual(type(response), response_class) - - request = Request(self.getURL(filename), body=body) - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_response_class_from_url(self): - return self._test_response_class("foo.html", b"", HtmlResponse) - - def test_response_class_from_body(self): - return self._test_response_class( - "foo", - b"\n.", - HtmlResponse, - ) - - def test_get_duplicate_header(self): - def _test(response): - self.assertEqual( - response.headers.getlist(b"Set-Cookie"), - [b"a=b", b"c=d"], - ) - - request = Request(self.getURL("duplicate-header")) - return self.download_request(request, Spider("foo")).addCallback(_test) - - -class Http10TestCase(HttpTestCase): - """HTTP 1.0 test case""" - - download_handler_cls: Type = HTTP10DownloadHandler - - def test_protocol(self): - request = Request(self.getURL("host"), method="GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, "HTTP/1.0") - return d - - -class Https10TestCase(Http10TestCase): - scheme = "https" - - -class Http11TestCase(HttpTestCase): - """HTTP 1.1 test case""" - - download_handler_cls: Type = HTTP11DownloadHandler - - def test_download_without_maxsize_limit(self): - request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d - - def test_response_class_choosing_request(self): - """Tests choosing of correct response type - in case of Content-Type is empty but body contains text. - """ - xml_body = b' None: + crawler = get_crawler() + with pytest.raises(NotConfigured, match="missing botocore library"): + build_from_crawler(S3DownloadHandler, crawler) - test_files = ( - ("file.txt", b"I have the power!"), - ("file with spaces.txt", b"Moooooooooo power!"), - ("html-file-without-extension", b"\n."), - ) - def setUp(self): - from twisted.protocols.ftp import FTPFactory, FTPRealm - - from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler - - # setup dirs and test file - self.directory = Path(self.mktemp()) - self.directory.mkdir() - userdir = self.directory / self.username - userdir.mkdir() - for filename, content in self.test_files: - (userdir / filename).write_bytes(content) - - # setup server - realm = FTPRealm( - anonymousRoot=str(self.directory), userHome=str(self.directory) - ) - p = portal.Portal(realm) - users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() - users_checker.addUser(self.username, self.password) - p.registerChecker(users_checker, credentials.IUsernamePassword) - self.factory = FTPFactory(portal=p) - self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") - self.portNum = self.port.getHost().port +class TestDataURI: + def setup_method(self): crawler = get_crawler() - self.download_handler = build_from_crawler(FTPDownloadHandler, crawler) - self.addCleanup(self.port.stopListening) + download_handler = build_from_crawler(DataURIDownloadHandler, crawler) + self.download_request = download_handler.download_request - def tearDown(self): - shutil.rmtree(self.directory) - - def _add_test_callbacks(self, deferred, callback=None, errback=None): - def _clean(data): - self.download_handler.client.transport.loseConnection() - return data - - deferred.addCallback(_clean) - if callback: - deferred.addCallback(callback) - if errback: - deferred.addErrback(errback) - return deferred - - def test_ftp_download_success(self): - request = Request( - url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=self.req_meta - ) - d = self.download_handler.download_request(request, None) - - def _test(r): - self.assertEqual(r.status, 200) - self.assertEqual(r.body, b"I have the power!") - self.assertEqual(r.headers, {b"Local Filename": [b""], b"Size": [b"17"]}) - self.assertIsNone(r.protocol) - - return self._add_test_callbacks(d, _test) - - def test_ftp_download_path_with_spaces(self): - request = Request( - url=f"ftp://127.0.0.1:{self.portNum}/file with spaces.txt", - meta=self.req_meta, - ) - d = self.download_handler.download_request(request, None) - - def _test(r): - self.assertEqual(r.status, 200) - self.assertEqual(r.body, b"Moooooooooo power!") - self.assertEqual(r.headers, {b"Local Filename": [b""], b"Size": [b"18"]}) - - return self._add_test_callbacks(d, _test) - - def test_ftp_download_nonexistent(self): - request = Request( - url=f"ftp://127.0.0.1:{self.portNum}/nonexistent.txt", meta=self.req_meta - ) - d = self.download_handler.download_request(request, None) - - def _test(r): - self.assertEqual(r.status, 404) - - return self._add_test_callbacks(d, _test) - - def test_ftp_local_filename(self): - f, local_fname = tempfile.mkstemp() - fname_bytes = to_bytes(local_fname) - local_fname = Path(local_fname) - os.close(f) - meta = {"ftp_local_filename": fname_bytes} - meta.update(self.req_meta) - request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=meta) - d = self.download_handler.download_request(request, None) - - def _test(r): - self.assertEqual(r.body, fname_bytes) - self.assertEqual( - r.headers, {b"Local Filename": [fname_bytes], b"Size": [b"17"]} - ) - self.assertTrue(local_fname.exists()) - self.assertEqual(local_fname.read_bytes(), b"I have the power!") - local_fname.unlink() - - return self._add_test_callbacks(d, _test) - - def _test_response_class(self, filename, response_class): - f, local_fname = tempfile.mkstemp() - local_fname = Path(local_fname) - os.close(f) - meta = {} - meta.update(self.req_meta) - request = Request(url=f"ftp://127.0.0.1:{self.portNum}/{filename}", meta=meta) - d = self.download_handler.download_request(request, None) - - def _test(r): - self.assertEqual(type(r), response_class) - local_fname.unlink() - - return self._add_test_callbacks(d, _test) - - def test_response_class_from_url(self): - return self._test_response_class("file.txt", TextResponse) - - def test_response_class_from_body(self): - return self._test_response_class("html-file-without-extension", HtmlResponse) - - -class FTPTestCase(BaseFTPTestCase): - def test_invalid_credentials(self): - if self.reactor_pytest == "asyncio" and sys.platform == "win32": - raise unittest.SkipTest( - "This test produces DirtyReactorAggregateError on Windows with asyncio" - ) - from twisted.protocols.ftp import ConnectionLost - - meta = dict(self.req_meta) - meta.update({"ftp_password": "invalid"}) - request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=meta) - d = self.download_handler.download_request(request, None) - - def _test(r): - self.assertEqual(r.type, ConnectionLost) - - return self._add_test_callbacks(d, errback=_test) - - -class AnonymousFTPTestCase(BaseFTPTestCase): - username = "anonymous" - req_meta = {} - - def setUp(self): - from twisted.protocols.ftp import FTPFactory, FTPRealm - - from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler - - # setup dir and test file - self.directory = Path(self.mktemp()) - self.directory.mkdir() - - for filename, content in self.test_files: - (self.directory / filename).write_bytes(content) - - # setup server for anonymous access - realm = FTPRealm(anonymousRoot=str(self.directory)) - p = portal.Portal(realm) - p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous) - - self.factory = FTPFactory(portal=p, userAnonymous=self.username) - self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") - self.portNum = self.port.getHost().port - crawler = get_crawler() - self.download_handler = build_from_crawler(FTPDownloadHandler, crawler) - self.addCleanup(self.port.stopListening) - - def tearDown(self): - shutil.rmtree(self.directory) - - -class DataURITestCase(unittest.TestCase): - def setUp(self): - crawler = get_crawler() - self.download_handler = build_from_crawler(DataURIDownloadHandler, crawler) - self.download_request = self.download_handler.download_request - self.spider = Spider("foo") - - def test_response_attrs(self): + @coroutine_test + async def test_response_attrs(self): uri = "data:,A%20brief%20note" - - def _test(response): - self.assertEqual(response.url, uri) - self.assertFalse(response.headers) - request = Request(uri) - return self.download_request(request, self.spider).addCallback(_test) - - def test_default_mediatype_encoding(self): - def _test(response): - self.assertEqual(response.text, "A brief note") - self.assertIsInstance(response, TextResponse) - self.assertEqual(response.encoding, "US-ASCII") + response = await self.download_request(request) + assert response.url == uri + assert not response.headers + @coroutine_test + async def test_default_mediatype_encoding(self): request = Request("data:,A%20brief%20note") - return self.download_request(request, self.spider).addCallback(_test) - - def test_default_mediatype(self): - def _test(response): - self.assertEqual(response.text, "\u038e\u03a3\u038e") - self.assertIsInstance(response, TextResponse) - self.assertEqual(response.encoding, "iso-8859-7") + response = await self.download_request(request) + assert response.text == "A brief note" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "US-ASCII" + @coroutine_test + async def test_default_mediatype(self): request = Request("data:;charset=iso-8859-7,%be%d3%be") - return self.download_request(request, self.spider).addCallback(_test) - - def test_text_charset(self): - def _test(response): - self.assertEqual(response.text, "\u038e\u03a3\u038e") - self.assertEqual(response.body, b"\xbe\xd3\xbe") - self.assertEqual(response.encoding, "iso-8859-7") + response = await self.download_request(request) + assert response.text == "\u038e\u03a3\u038e" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "iso-8859-7" + @coroutine_test + async def test_text_charset(self): request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") - return self.download_request(request, self.spider).addCallback(_test) - - def test_mediatype_parameters(self): - def _test(response): - self.assertEqual(response.text, "\u038e\u03a3\u038e") - self.assertIsInstance(response, TextResponse) - self.assertEqual(response.encoding, "utf-8") + response = await self.download_request(request) + assert response.text == "\u038e\u03a3\u038e" + assert response.body == b"\xbe\xd3\xbe" + assert response.encoding == "iso-8859-7" + @coroutine_test + async def test_mediatype_parameters(self): request = Request( "data:text/plain;foo=%22foo;bar%5C%22%22;" "charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22" ",%CE%8E%CE%A3%CE%8E" ) - return self.download_request(request, self.spider).addCallback(_test) - - def test_base64(self): - def _test(response): - self.assertEqual(response.text, "Hello, world.") + response = await self.download_request(request) + assert response.text == "\u038e\u03a3\u038e" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "utf-8" + @coroutine_test + async def test_base64(self): request = Request("data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D") - return self.download_request(request, self.spider).addCallback(_test) - - def test_protocol(self): - def _test(response): - self.assertIsNone(response.protocol) + response = await self.download_request(request) + assert response.text == "Hello, world." + @coroutine_test + async def test_protocol(self): request = Request("data:,") - return self.download_request(request, self.spider).addCallback(_test) - - def test_body_mime_type(self): - """Test that the body, and not only the declared MIME type, is taken - into account when choosing a response class.""" - - def _test(response): - self.assertIsInstance(response, HtmlResponse) - - html = "\n." - base64_html = b64encode(html.encode()).decode() - data_uri = f"data:application/unknown;base64,{base64_html}" - request = Request(data_uri) - return self.download_request(request, self.spider).addCallback(_test) + response = await self.download_request(request) + assert response.protocol is None diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py deleted file mode 100644 index 322075043..000000000 --- a/tests/test_downloader_handlers_http2.py +++ /dev/null @@ -1,254 +0,0 @@ -import json -from unittest import mock, skipIf - -from pytest import mark -from testfixtures import LogCapture -from twisted.internet import defer, error, reactor -from twisted.trial import unittest -from twisted.web import server -from twisted.web.error import SchemeNotSupported -from twisted.web.http import H2_ENABLED - -from scrapy.http import Request -from scrapy.spiders import Spider -from scrapy.utils.misc import build_from_crawler -from scrapy.utils.test import get_crawler -from tests.mockserver import ssl_context_factory -from tests.test_downloader_handlers import ( - Http11MockServerTestCase, - Http11ProxyTestCase, - Https11CustomCiphers, - Https11TestCase, - UriResource, -) - - -@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") -class Https2TestCase(Https11TestCase): - scheme = "https" - HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" - - @classmethod - def setUpClass(cls): - from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - - cls.download_handler_cls = H2DownloadHandler - - def test_protocol(self): - request = Request(self.getURL("host"), method="GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, "h2") - return d - - @defer.inlineCallbacks - def test_download_with_maxsize_very_large_file(self): - with mock.patch("scrapy.core.http2.stream.logger") as logger: - request = Request(self.getURL("largechunkedfile")) - - def check(logger): - logger.error.assert_called_once_with(mock.ANY) - - d = self.download_request(request, Spider("foo", download_maxsize=1500)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) - - # As the error message is logged in the dataReceived callback, we - # have to give a bit of time to the reactor to process the queue - # after closing the connection. - d = defer.Deferred() - d.addCallback(check) - reactor.callLater(0.1, d.callback, logger) - yield d - - @defer.inlineCallbacks - def test_unsupported_scheme(self): - request = Request("ftp://unsupported.scheme") - d = self.download_request(request, Spider("foo")) - yield self.assertFailure(d, SchemeNotSupported) - - def test_download_broken_content_cause_data_loss(self, url="broken"): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_chunked_content_cause_data_loss(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_content_allow_data_loss(self, url="broken"): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_chunked_content_allow_data_loss(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_chunked_content_allow_data_loss_via_setting(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_concurrent_requests_same_domain(self): - spider = Spider("foo") - - request1 = Request(self.getURL("file")) - d1 = self.download_request(request1, spider) - d1.addCallback(lambda r: r.body) - d1.addCallback(self.assertEqual, b"0123456789") - - request2 = Request(self.getURL("echo"), method="POST") - d2 = self.download_request(request2, spider) - d2.addCallback(lambda r: r.headers["Content-Length"]) - d2.addCallback(self.assertEqual, b"79") - - return defer.DeferredList([d1, d2]) - - @mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247") - def test_connect_request(self): - request = Request(self.getURL("file"), method="CONNECT") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"") - return d - - def test_custom_content_length_good(self): - request = Request(self.getURL("contentlength")) - custom_content_length = str(len(request.body)) - request.headers["Content-Length"] = custom_content_length - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.text) - d.addCallback(self.assertEqual, custom_content_length) - return d - - def test_custom_content_length_bad(self): - request = Request(self.getURL("contentlength")) - actual_content_length = str(len(request.body)) - bad_content_length = str(len(request.body) + 1) - request.headers["Content-Length"] = bad_content_length - log = LogCapture() - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.text) - d.addCallback(self.assertEqual, actual_content_length) - d.addCallback( - lambda _: log.check_present( - ( - "scrapy.core.http2.stream", - "WARNING", - f"Ignoring bad Content-Length header " - f"{bad_content_length!r} of request {request}, sending " - f"{actual_content_length!r} instead", - ) - ) - ) - d.addCallback(lambda _: log.uninstall()) - return d - - def test_duplicate_header(self): - request = Request(self.getURL("echo")) - header, value1, value2 = "Custom-Header", "foo", "bar" - request.headers.appendlist(header, value1) - request.headers.appendlist(header, value2) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: json.loads(r.text)["headers"][header]) - d.addCallback(self.assertEqual, [value1, value2]) - return d - - -class Https2WrongHostnameTestCase(Https2TestCase): - tls_log_message = ( - 'SSL connection certificate: issuer "/C=XW/ST=XW/L=The ' - 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", ' - 'subject "/C=XW/ST=XW/L=The ' - 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"' - ) - - # above tests use a server certificate for "localhost", - # client connection to "localhost" too. - # here we test that even if the server certificate is for another domain, - # "www.example.com" in this case, - # the tests still pass - keyfile = "keys/example-com.key.pem" - certfile = "keys/example-com.cert.pem" - - -class Https2InvalidDNSId(Https2TestCase): - """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" - - def setUp(self): - super().setUp() - self.host = "127.0.0.1" - - -class Https2InvalidDNSPattern(Https2TestCase): - """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" - - keyfile = "keys/localhost.ip.key" - certfile = "keys/localhost.ip.crt" - - def setUp(self): - try: - from service_identity.exceptions import CertificateError # noqa: F401 - except ImportError: - raise unittest.SkipTest("cryptography lib is too old") - self.tls_log_message = ( - 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' - 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' - ) - super().setUp() - - -@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") -class Https2CustomCiphers(Https11CustomCiphers): - scheme = "https" - - @classmethod - def setUpClass(cls): - from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - - cls.download_handler_cls = H2DownloadHandler - - -class Http2MockServerTestCase(Http11MockServerTestCase): - """HTTP 2.0 test case with MockServer""" - - settings_dict = { - "DOWNLOAD_HANDLERS": { - "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler" - } - } - - -@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") -class Https2ProxyTestCase(Http11ProxyTestCase): - # only used for HTTPS tests - keyfile = "keys/localhost.key" - certfile = "keys/localhost.crt" - - scheme = "https" - host = "127.0.0.1" - - expected_http_proxy_request_body = b"/" - - @classmethod - def setUpClass(cls): - from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - - cls.download_handler_cls = H2DownloadHandler - - def setUp(self): - site = server.Site(UriResource(), timeout=None) - self.port = reactor.listenSSL( - 0, - site, - ssl_context_factory(self.keyfile, self.certfile), - interface=self.host, - ) - self.portno = self.port.getHost().port - self.download_handler = build_from_crawler( - self.download_handler_cls, get_crawler() - ) - self.download_request = self.download_handler.download_request - - def getURL(self, path): - return f"{self.scheme}://{self.host}:{self.portno}/{path}" - - @defer.inlineCallbacks - def test_download_with_proxy_https_timeout(self): - with self.assertRaises(NotImplementedError): - yield super().test_download_with_proxy_https_timeout() diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py new file mode 100644 index 000000000..0e1ff07c9 --- /dev/null +++ b/tests/test_downloader_handlers_http_base.py @@ -0,0 +1,1551 @@ +"""Base classes for HTTP download handler tests.""" + +from __future__ import annotations + +import gzip +import json +import logging +import os +import re +import sys +from abc import ABC, abstractmethod +from contextlib import asynccontextmanager +from http import HTTPStatus +from ipaddress import IPv4Address +from socket import gethostbyname +from typing import TYPE_CHECKING, Any, ClassVar +from urllib.parse import urlparse + +import pytest +from cryptography.x509 import load_der_x509_certificate +from twisted.internet.ssl import Certificate +from twisted.python.failure import Failure + +from scrapy.exceptions import ( + CannotResolveHostError, + DownloadCancelledError, + DownloadConnectionRefusedError, + DownloadFailedError, + DownloadTimeoutError, + ResponseDataLossError, + ScrapyDeprecationWarning, + StopDownload, + UnsupportedURLSchemeError, +) +from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse +from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1 +from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.test import get_crawler +from tests import NON_EXISTING_RESOLVABLE +from tests.mockserver.mitm_proxy import wrong_credentials +from tests.mockserver.proxy_echo import ProxyEchoMockServer +from tests.mockserver.simple_https import SimpleMockServer +from tests.spiders import ( + BytesReceivedCallbackSpider, + BytesReceivedErrbackSpider, + HeadersReceivedCallbackSpider, + HeadersReceivedErrbackSpider, + SimpleSpider, + SingleRequestSpider, +) +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Generator + + from scrapy.core.downloader.handlers import DownloadHandlerProtocol + from tests.mockserver.http import MockServer + + +class TestHttpBase(ABC): + is_secure: bool = False + http2: bool = False + # whether the handler supports per-request bindaddress + handler_supports_bindaddress_meta: bool = True + # RFC 9113 §8.1.1 explicitly says that a Content-Length mismatch is a + # stream error (of type PROTOCOL_ERROR) so the client will send + # RST_STREAM. Some libraries do only this while e.g. h2 also closes the + # connection (see handling of ProtocolError in + # h2.connection.H2Connection.receive_data()), thus closing all streams that + # were using it, and we handle this as a normal exception. + handler_supports_http2_dataloss: bool = True + # default headers added by the underlying library that cannot be suppressed + always_present_req_headers: ClassVar[frozenset[str]] = frozenset() + default_handler_settings: ClassVar[dict[str, Any]] = {} + + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + @asynccontextmanager + async def get_dh( + self, settings_dict: dict[str, Any] | None = None + ) -> AsyncGenerator[DownloadHandlerProtocol]: + settings_dict = { + **self.default_handler_settings, + **(settings_dict or {}), + } + crawler = get_crawler(DefaultSpider, settings_dict) + crawler.spider = crawler._create_spider() + dh = build_from_crawler(self.download_handler_cls, crawler) + try: + yield dh + finally: + await dh.close() + + @coroutine_test + async def test_unsupported_scheme(self) -> None: + request = Request("unsupp://unsupported.scheme") + async with self.get_dh() as download_handler: + with pytest.raises(UnsupportedURLSchemeError): + await download_handler.download_request(request) + + @coroutine_test + async def test_download(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/text", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"Works" + + @coroutine_test + async def test_download_head(self, mockserver: MockServer) -> None: + request = Request( + mockserver.url("/text", is_secure=self.is_secure), method="HEAD" + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"" + + @pytest.mark.parametrize( + "http_status", + [ + pytest.param(http_status, id=f"status={http_status.value}") + for http_status in HTTPStatus + if http_status.value == 200 or http_status.value // 100 in (4, 5) + ], + ) + @coroutine_test + async def test_download_has_correct_http_status_code( + self, mockserver: MockServer, http_status: HTTPStatus + ) -> None: + request = Request( + mockserver.url(f"/status?n={http_status.value}", is_secure=self.is_secure) + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == http_status.value + + @coroutine_test + async def test_server_receives_correct_request_headers( + self, mockserver: MockServer + ) -> None: + request_headers = { + # common request headers + "Accept": "text/html", + "Accept-Charset": "utf-8", + "Accept-Datetime": "Thu, 31 May 2007 20:35:00 GMT", + "Accept-Encoding": "gzip, deflate", + # custom headers + "X-Custom-Header": "Custom Value", + } + + request = Request( + mockserver.url("/echo", is_secure=self.is_secure), + headers=request_headers, + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == HTTPStatus.OK + body = json.loads(response.body.decode("utf-8")) + assert "headers" in body + for header_name, header_value in request_headers.items(): + assert header_name in body["headers"] + assert body["headers"][header_name] == [header_value] + + @coroutine_test + async def test_request_header_none(self, mockserver: MockServer) -> None: + """Adding a header with None as the value should not send that header.""" + request_headers = { + "Cookie": None, + "X-Custom-Header": None, + } + request = Request( + mockserver.url("/echo", is_secure=self.is_secure), + headers=request_headers, + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == HTTPStatus.OK + body = json.loads(response.body.decode("utf-8")) + assert "headers" in body + for header_name in request_headers: + assert header_name not in body["headers"] + + @pytest.mark.parametrize( + "request_headers", + [ + {"X-Custom-Header": ["foo", "bar"]}, + [("X-Custom-Header", "foo"), ("X-Custom-Header", "bar")], + ], + ) + @coroutine_test + async def test_request_header_duplicate( + self, mockserver: MockServer, request_headers: Any + ) -> None: + """All values for a header should be sent.""" + request = Request( + mockserver.url("/echo", is_secure=self.is_secure), + headers=request_headers, + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == HTTPStatus.OK + body = json.loads(response.body.decode("utf-8")) + assert "headers" in body + assert body["headers"]["X-Custom-Header"] == ["foo", "bar"] + + @coroutine_test + async def test_server_receives_no_extra_headers( + self, mockserver: MockServer + ) -> None: + """Test that the handler doesn't add headers to the request.""" + request = Request(mockserver.url("/echo", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == HTTPStatus.OK + body = json.loads(response.body.decode("utf-8")) + assert "headers" in body + received_headers = set(body["headers"].keys()) + allowed_headers = { + "Connection", + "Content-Length", + "Host", + } | self.always_present_req_headers + extra_headers = received_headers - allowed_headers + assert not extra_headers, body["headers"] + + @coroutine_test + async def test_server_receives_correct_request_body( + self, mockserver: MockServer + ) -> None: + request_body = { + "message": "It works!", + } + request = Request( + mockserver.url("/echo", is_secure=self.is_secure), + body=json.dumps(request_body), + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == HTTPStatus.OK + body = json.loads(response.body.decode("utf-8")) + assert json.loads(body["body"]) == request_body + + @coroutine_test + async def test_download_has_correct_response_headers( + self, mockserver: MockServer + ) -> None: + # these headers will be set on the response in the resource and returned + response_headers = { + # common response headers + "Access-Control-Allow-Origin": "*", + "Allow": "Get, Head", + "Age": "12", + "Cache-Control": "max-age=3600", + "Content-Encoding": "gzip", + "Content-MD5": "Q2hlY2sgSW50ZWdyaXR5IQ==", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 15 Nov 1994 08:12:31 GMT", + "Pragma": "no-cache", + "Retry-After": "120", + "Set-Cookie": "CookieName=CookieValue; Max-Age=3600; Version=1", + "WWW-Authenticate": "Basic", + # custom headers + "X-Custom-Header": "Custom Header Value", + } + + request = Request( + mockserver.url("/response-headers", is_secure=self.is_secure), + headers={"content-type": "application/json"}, + body=json.dumps(response_headers), + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == 200 + for header_name, header_value in response_headers.items(): + assert header_name in response.headers, ( + f"Response was missing expected header {header_name}" + ) + assert response.headers.getlist(header_name) == [ + header_value.encode(encoding="utf-8") + ] + + @coroutine_test + async def test_download_no_extra_response_headers( + self, mockserver: MockServer + ) -> None: + """Test that the handler doesn't add headers to the response.""" + request = Request( + mockserver.url("/response-headers", is_secure=self.is_secure), + headers={"content-type": "application/json"}, + body=json.dumps({}), + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == 200 + received_headers = set(response.headers.keys()) + allowed_headers = { + b"Content-Length", + b"Content-Type", + b"Date", + b"Server", + } + extra_headers = received_headers - allowed_headers + assert not extra_headers, response.headers + + @coroutine_test + async def test_redirect_status(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/redirect", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == 302 + assert response.headers["Location"] == b"/redirected" + + @coroutine_test + async def test_redirect_status_head(self, mockserver: MockServer) -> None: + request = Request( + mockserver.url("/redirect", is_secure=self.is_secure), method="HEAD" + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == 302 + assert response.headers["Location"] == b"/redirected" + + @coroutine_test + async def test_timeout_download_from_spider_nodata_rcvd( + self, mockserver: MockServer, reactor_pytest: str + ) -> None: + if reactor_pytest == "asyncio" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + pytest.skip( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) + + # client connects but no data is received + meta = {"download_timeout": 0.5} + request = Request(mockserver.url("/wait", is_secure=self.is_secure), meta=meta) + async with self.get_dh() as download_handler: + d = deferred_from_coro(download_handler.download_request(request)) + with pytest.raises(DownloadTimeoutError): + await maybe_deferred_to_future(d) + + @coroutine_test + async def test_timeout_download_from_spider_server_hangs( + self, mockserver: MockServer, reactor_pytest: str + ) -> None: + if reactor_pytest == "asyncio" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + pytest.skip( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) + # client connects, server send headers and some body bytes but hangs + meta = {"download_timeout": 0.5} + request = Request( + mockserver.url("/hang-after-headers", is_secure=self.is_secure), meta=meta + ) + async with self.get_dh() as download_handler: + d = deferred_from_coro(download_handler.download_request(request)) + with pytest.raises(DownloadTimeoutError): + await maybe_deferred_to_future(d) + + @pytest.mark.parametrize("send_header", [True, False]) + @coroutine_test + async def test_host_header(self, send_header: bool, mockserver: MockServer) -> None: + host_port = f"{mockserver.host}:{mockserver.port(is_secure=self.is_secure)}" + request = Request( + mockserver.url("/host", is_secure=self.is_secure), + headers={"Host": host_port} if send_header else {}, + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == host_port.encode() + if send_header: + assert request.headers.get("Host") == host_port.encode() + else: + assert not request.headers + + @coroutine_test + async def test_content_length_zero_bodyless_post_request_headers( + self, mockserver: MockServer + ) -> None: + """Tests if "Content-Length: 0" is sent for bodyless POST requests. + + This is not strictly required by HTTP RFCs but can cause trouble + for some web servers. + See: + https://github.com/scrapy/scrapy/issues/823 + https://issues.apache.org/jira/browse/TS-2902 + https://github.com/kennethreitz/requests/issues/405 + https://bugs.python.org/issue14721 + """ + request = Request( + mockserver.url("/contentlength", is_secure=self.is_secure), method="POST" + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"0" + + @coroutine_test + async def test_content_length_zero_bodyless_post_only_one( + self, mockserver: MockServer + ) -> None: + request = Request( + mockserver.url("/echo", is_secure=self.is_secure), method="POST" + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + headers = Headers(json.loads(response.text)["headers"]) + contentlengths = headers.getlist("Content-Length") + assert len(contentlengths) == 1 + assert contentlengths == [b"0"] + + @coroutine_test + async def test_payload(self, mockserver: MockServer) -> None: + body = b"1" * 100 # PayloadResource requires body length to be 100 + request = Request( + mockserver.url("/payload", is_secure=self.is_secure), + method="POST", + body=body, + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == body + + @coroutine_test + async def test_response_header_content_length(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/text", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.headers[b"content-length"] == b"5" + + @pytest.mark.parametrize( + ("filename", "body", "response_class"), + [ + ("foo.html", b"", HtmlResponse), + ("foo", b"\n.", HtmlResponse), + ], + ) + @coroutine_test + async def test_response_class( + self, + filename: str, + body: bytes, + response_class: type[Response], + mockserver: MockServer, + ) -> None: + request = Request( + mockserver.url(f"/{filename}", is_secure=self.is_secure), body=body + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert type(response) is response_class # pylint: disable=unidiomatic-typecheck + + @coroutine_test + async def test_get_duplicate_header(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/duplicate-header", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] + + @coroutine_test + async def test_download_is_not_automatically_gzip_decoded( + self, mockserver: MockServer + ) -> None: + """Test download handler does not automatically decode content using the scheme provided in Content-Encoding header""" + + data = "compress-me" + + # send a request to mock resource that gzip encodes the "data" url parameter + request = Request( + mockserver.url(f"/compress?data={data}", is_secure=self.is_secure), + headers={ + "accept-encoding": "gzip", + }, + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + + assert response.status == 200 + + # check that the Content-Encoding header is gzip + content_encoding = response.headers[b"Content-Encoding"] + assert content_encoding == b"gzip" + + # check that the response is still encoded + # by checking for the magic number that is always included at the start of a gzip encoding + # see https://datatracker.ietf.org/doc/html/rfc1952#page-5 section 2.3.1 + GZIP_MAGIC = b"\x1f\x8b" + assert response.body[:2] == GZIP_MAGIC, "Response body was not in gzip format" + + # check that a gzip decoding matches the data sent in the request + expected_decoding = bytes(data, encoding="utf-8") + assert gzip.decompress(response.body) == expected_decoding + + @coroutine_test + async def test_no_cookie_processing_or_persistence( + self, mockserver: MockServer + ) -> None: + cookie_name = "foo" + cookie_value = "bar" + + # check that cookies are not modified + request = Request( + mockserver.url( + f"/set-cookie?{cookie_name}={cookie_value}", is_secure=self.is_secure + ) + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == 200 + set_cookie = response.headers.get(b"Set-Cookie") + assert set_cookie == f"{cookie_name}={cookie_value}".encode() + + # check that cookies are not sent in the next request + request = Request(mockserver.url("/echo", is_secure=self.is_secure)) + response = await download_handler.download_request(request) + assert response.status == 200 + headers = Headers(json.loads(response.text)["headers"]) + assert "Cookie" not in headers + assert "cookie" not in headers + + @coroutine_test + async def test_download_latency(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/text", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + await download_handler.download_request(request) + assert "download_latency" in request.meta + latency = request.meta["download_latency"] + if sys.version_info < (3, 13) and sys.platform == "win32": + # time.monotonic() resolution is too low here: + # https://docs.python.org/3/whatsnew/3.13.html#time + assert latency >= 0 + else: + assert latency > 0 + + @coroutine_test + async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/text", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"Works" + + @coroutine_test + async def test_response_class_choosing_request( + self, mockserver: MockServer + ) -> None: + """Tests choosing of correct response type + in case of Content-Type is empty but body contains text. + """ + body = b"Some plain text\ndata with tabs\t and null bytes\0" + request = Request( + mockserver.url("/nocontenttype", is_secure=self.is_secure), body=body + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck + + @coroutine_test + async def test_download_with_maxsize( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + request = Request(mockserver.url("/text", is_secure=self.is_secure)) + + # 10 is minimal size for this request and the limit is only counted on + # response body. (regardless of headers) + async with self.get_dh({"DOWNLOAD_MAXSIZE": 5}) as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"Works" + + caplog.clear() + msg = "Expected to receive 5 bytes which is larger than download max size (4)" + async with self.get_dh({"DOWNLOAD_MAXSIZE": 4}) as download_handler: + with pytest.raises(DownloadCancelledError, match=re.escape(msg)): + await download_handler.download_request(request) + assert msg in caplog.text + + @coroutine_test + async def test_download_with_maxsize_very_large_file( + self, mockserver: MockServer, caplog: pytest.LogCaptureFixture + ) -> None: + request = Request(mockserver.url("/largechunkedfile", is_secure=self.is_secure)) + async with self.get_dh({"DOWNLOAD_MAXSIZE": 1_500}) as download_handler: + with pytest.raises(DownloadCancelledError): + await download_handler.download_request(request) + assert re.search( + r"Received \d+ bytes which is larger than download max size \(1500\)", + caplog.text, + ) + + @coroutine_test + async def test_download_with_maxsize_per_req(self, mockserver: MockServer) -> None: + meta = {"download_maxsize": 2} + request = Request(mockserver.url("/text", is_secure=self.is_secure), meta=meta) + async with self.get_dh() as download_handler: + with pytest.raises(DownloadCancelledError): + await download_handler.download_request(request) + + @coroutine_test + async def test_download_with_small_maxsize_via_setting( + self, mockserver: MockServer + ) -> None: + request = Request(mockserver.url("/text", is_secure=self.is_secure)) + async with self.get_dh({"DOWNLOAD_MAXSIZE": 2}) as download_handler: + with pytest.raises(DownloadCancelledError): + await download_handler.download_request(request) + + @coroutine_test + async def test_download_with_large_maxsize_via_setting( + self, mockserver: MockServer + ) -> None: + request = Request(mockserver.url("/text", is_secure=self.is_secure)) + async with self.get_dh({"DOWNLOAD_MAXSIZE": 100}) as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"Works" + + @coroutine_test + async def test_download_with_warnsize( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + request = Request(mockserver.url("/text", is_secure=self.is_secure)) + async with self.get_dh({"DOWNLOAD_WARNSIZE": 4}) as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"Works" + assert ( + "Expected to receive 5 bytes which is larger than download warn size (4)" + in caplog.text + ) + + @coroutine_test + async def test_download_with_warnsize_no_content_length( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + request = Request( + mockserver.url("/delay?n=0.1", is_secure=self.is_secure), + ) + async with self.get_dh({"DOWNLOAD_WARNSIZE": 10}) as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"Response delayed for 0.100 seconds\n" + assert ( + "Received 35 bytes which is larger than download warn size (10)" + in caplog.text + ) + + @coroutine_test + async def test_download_chunked_content(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/chunked", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"chunked content\n" + + @coroutine_test + async def test_download_cause_data_loss(self, mockserver: MockServer) -> None: + if self.http2 and not self.handler_supports_http2_dataloss: + pytest.skip("This handler doesn't support dataloss on HTTP/2") + request = Request(mockserver.url("/broken", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + with pytest.raises(ResponseDataLossError): + await download_handler.download_request(request) + + @coroutine_test + async def test_download_cause_data_loss_double_warning( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + if self.http2 and not self.handler_supports_http2_dataloss: + pytest.skip("This handler doesn't support dataloss on HTTP/2") + request = Request(mockserver.url("/broken", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + with pytest.raises(ResponseDataLossError): + await download_handler.download_request(request) + assert "Got data loss" in caplog.text + caplog.clear() + with pytest.raises(ResponseDataLossError): + await download_handler.download_request(request) + # no repeated warning + assert "Got data loss" not in caplog.text + + @coroutine_test + async def test_download_allow_data_loss_broken( + self, mockserver: MockServer + ) -> None: + if self.http2 and not self.handler_supports_http2_dataloss: + pytest.skip("This handler doesn't support dataloss on HTTP/2") + request = Request( + mockserver.url("/broken", is_secure=self.is_secure), + meta={"download_fail_on_dataloss": False}, + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.flags == ["dataloss"] + assert response.text == "partial" + + @coroutine_test + async def test_download_allow_data_loss_broken_chunked( + self, mockserver: MockServer + ) -> None: + if self.http2: + pytest.skip("Chunked encoding is specific to HTTP/1.1") + request = Request( + mockserver.url("/broken-chunked", is_secure=self.is_secure), + meta={"download_fail_on_dataloss": False}, + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.flags == ["dataloss"] + assert response.text == "chunked content\n" + + @coroutine_test + async def test_download_allow_data_loss_via_setting( + self, mockserver: MockServer + ) -> None: + if self.http2 and not self.handler_supports_http2_dataloss: + pytest.skip("This handler doesn't support dataloss on HTTP/2") + request = Request(mockserver.url("/broken", is_secure=self.is_secure)) + async with self.get_dh( + {"DOWNLOAD_FAIL_ON_DATALOSS": False} + ) as download_handler: + response = await download_handler.download_request(request) + assert response.flags == ["dataloss"] + + @coroutine_test + async def test_download_conn_failed(self) -> None: + # copy of TestCrawl.test_retry_conn_failed() + scheme = "https" if self.is_secure else "http" + request = Request(f"{scheme}://localhost:65432/") + async with self.get_dh() as download_handler: + with pytest.raises(DownloadConnectionRefusedError): + await download_handler.download_request(request) + + @coroutine_test + async def test_download_conn_lost(self, mockserver: MockServer) -> None: + # copy of TestCrawl.test_retry_conn_lost() + request = Request(mockserver.url("/drop?abort=0", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + with pytest.raises(ResponseDataLossError): + await download_handler.download_request(request) + + @coroutine_test + async def test_download_conn_aborted(self, mockserver: MockServer) -> None: + # copy of TestCrawl.test_retry_conn_aborted() + if self.http2: + # it may be possible to write a separate resource that does something + # suitable on HTTP/2 without sending Content-Length + pytest.skip( + "On HTTP/2 this triggers a Content-Length mismatch error instead." + ) + request = Request(mockserver.url("/drop?abort=1", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + with pytest.raises(DownloadFailedError): + await download_handler.download_request(request) + + @pytest.mark.skipif( + NON_EXISTING_RESOLVABLE, reason="Non-existing hosts are resolvable" + ) + @coroutine_test + async def test_download_dns_error(self) -> None: + # copy of TestCrawl.test_retry_dns_error() + scheme = "https" if self.is_secure else "http" + request = Request(f"{scheme}://dns.resolution.invalid./") + async with self.get_dh() as download_handler: + with pytest.raises(CannotResolveHostError): + await download_handler.download_request(request) + + @coroutine_test + async def test_protocol(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/host", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.protocol == "HTTP/1.1" + + @pytest.mark.skipif( + sys.platform == "darwin", + reason="127.0.0.2 is not available on macOS by default", + ) + @pytest.mark.parametrize("setting_value", [("127.0.0.2", 0), "127.0.0.2"]) + @coroutine_test + async def test_download_bind_address_setting( + self, mockserver: MockServer, setting_value: Any + ) -> None: + request = Request(mockserver.url("/client-ip", is_secure=self.is_secure)) + async with self.get_dh( + {"DOWNLOAD_BIND_ADDRESS": setting_value} + ) as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"127.0.0.2" + + @pytest.mark.skipif( + sys.platform == "darwin", + reason="127.0.0.2 is not available on macOS by default", + ) + @pytest.mark.parametrize("meta_value", [("127.0.0.2", 0), "127.0.0.2"]) + @coroutine_test + async def test_download_bind_address_meta( + self, mockserver: MockServer, caplog: pytest.LogCaptureFixture, meta_value: Any + ) -> None: + request = Request( + mockserver.url("/client-ip", is_secure=self.is_secure), + meta={"bindaddress": meta_value}, + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + if self.handler_supports_bindaddress_meta: + assert response.body == b"127.0.0.2" + else: + assert ( + "The 'bindaddress' request meta key is not supported by" in caplog.text + ) + + @coroutine_test + async def test_verbatim_url(self, mockserver: MockServer) -> None: + # Square brackets are encoded by safe_url_string (w3lib). + path = "/uri/items?data[0]=a" + url = mockserver.url(path, is_secure=self.is_secure) + + # Without verbatim_url, the brackets are percent-encoded before the + # request reaches the server. + request = Request(url) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"/uri/items?data%5B0%5D=a" + + # With verbatim_url=True the URL is sent to the server as-is. + request = Request(url, meta={"verbatim_url": True}) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == path.encode() + + +class TestHttpsBase(TestHttpBase): + is_secure = True + + tls_log_message = ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' + 'subject "/C=IE/O=Scrapy/CN=localhost"' + ) + + def test_download_conn_lost(self) -> None: # type: ignore[override] + # For some reason (maybe related to TLS shutdown flow, and maybe the + # mockserver resource can be fixed so that this works) HTTPS clients + # (not just Scrapy) hang on /drop?abort=0. + pytest.skip("Unable to test on HTTPS") + + @coroutine_test + async def test_tls_logging( + self, mockserver: MockServer, caplog: pytest.LogCaptureFixture + ) -> None: + request = Request(mockserver.url("/text", is_secure=self.is_secure)) + async with self.get_dh( + {"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True} + ) as download_handler: + with caplog.at_level("DEBUG"): + response = await download_handler.download_request(request) + assert response.body == b"Works" + assert self.tls_log_message in caplog.text + + @coroutine_test + async def test_verify_certs_deprecated(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/text", is_secure=self.is_secure)) + with ( # noqa: PT031 + pytest.warns( + ScrapyDeprecationWarning, + match="'DOWNLOADER_CLIENTCONTEXTFACTORY' setting is deprecated", + ), + pytest.warns( + ScrapyDeprecationWarning, + match="BrowserLikeContextFactory is deprecated", + ), + ): + async with self.get_dh( + { + "DOWNLOADER_CLIENTCONTEXTFACTORY": "scrapy.core.downloader.contextfactory.BrowserLikeContextFactory" + } + ) as download_handler: + with pytest.raises( + (DownloadConnectionRefusedError, DownloadFailedError) + ): + await download_handler.download_request(request) + + @coroutine_test + async def test_verify_certs(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/text", is_secure=self.is_secure)) + async with self.get_dh( + {"DOWNLOAD_VERIFY_CERTIFICATES": True} + ) as download_handler: + with pytest.raises((DownloadConnectionRefusedError, DownloadFailedError)): + await download_handler.download_request(request) + + +class TestSimpleHttpsBase(ABC): + """Base class for special cases tested with just one simple request""" + + keyfile = "keys/localhost.key" + certfile = "keys/localhost.crt" + host = "localhost" + cipher_string: str | None = None + + @pytest.fixture(scope="class") + def simple_mockserver(self) -> Generator[SimpleMockServer]: + with SimpleMockServer( + self.keyfile, self.certfile, cipher_string=self.cipher_string + ) as simple_mockserver: + yield simple_mockserver + + @pytest.fixture(scope="class") + def url(self, simple_mockserver: SimpleMockServer) -> str: + # need to use self.host instead of what mockserver returns + return f"https://{self.host}:{simple_mockserver.port(is_secure=True)}/file" + + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + @asynccontextmanager + async def get_dh(self) -> AsyncGenerator[DownloadHandlerProtocol]: + if self.cipher_string is not None: + settings_dict = {"DOWNLOADER_CLIENT_TLS_CIPHERS": self.cipher_string} + else: + settings_dict = None + crawler = get_crawler(DefaultSpider, settings_dict=settings_dict) + crawler.spider = crawler._create_spider() + dh = build_from_crawler(self.download_handler_cls, crawler) + try: + yield dh + finally: + await dh.close() + + @coroutine_test + async def test_download(self, url: str) -> None: + request = Request(url) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"0123456789" + + +class TestHttpsWrongHostnameBase(TestSimpleHttpsBase): + # above tests use a server certificate for "localhost", + # client connection to "localhost" too. + # here we test that even if the server certificate is for another domain, + # "www.example.com" in this case, + # the tests still pass + keyfile = "keys/example-com.key.pem" + certfile = "keys/example-com.cert.pem" + + +class TestHttpsInvalidDNSIdBase(TestSimpleHttpsBase): + """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" + + host = "127.0.0.1" + + +class TestHttpsInvalidDNSPatternBase(TestSimpleHttpsBase): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = "keys/localhost.ip.key" + certfile = "keys/localhost.ip.crt" + + +class TestHttpsCustomCiphersBase(TestSimpleHttpsBase): + cipher_string = "CAMELLIA256-SHA" + + +class TestHttpsTLSVersionBase(ABC): + keyfile = "keys/localhost.key" + certfile = "keys/localhost.crt" + + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + @asynccontextmanager + async def get_dh( + self, client_tls_min: str | None, client_tls_max: str | None + ) -> AsyncGenerator[DownloadHandlerProtocol]: + settings = {} + if client_tls_min is not None: + settings["DOWNLOAD_TLS_MIN_VERSION"] = client_tls_min + if client_tls_max is not None: + settings["DOWNLOAD_TLS_MAX_VERSION"] = client_tls_max + crawler = get_crawler(DefaultSpider, settings_dict=settings) + crawler.spider = crawler._create_spider() + dh = build_from_crawler(self.download_handler_cls, crawler) + try: + yield dh + finally: + await dh.close() + + @pytest.mark.parametrize( + ( + "server_tls_min", + "server_tls_max", + "client_tls_min", + "client_tls_max", + "expect_success", + ), + [ + pytest.param(None, None, None, None, True, id="no-limits"), + pytest.param(None, None, None, "TLSv1.2", True, id="client-max-tls1.2"), + pytest.param(None, None, "TLSv1.3", None, True, id="client-min-tls1.3"), + pytest.param( + "TLSv1.3", + None, + None, + "TLSv1.2", + False, + id="client-max-below-server-min", + ), + pytest.param( + None, + "TLSv1.2", + "TLSv1.3", + None, + False, + id="client-min-above-server-max", + ), + pytest.param(None, "TLSv1.2", None, "TLSv1.2", True, id="both-tls1.2"), + pytest.param("TLSv1.3", None, "TLSv1.3", None, True, id="both-tls1.3"), + pytest.param( + None, + None, + "TLSv1.0", + None, + True, + id="client-min-tls1.0", + marks=pytest.mark.filterwarnings( + r"ignore:ssl\.TLSVersion\.TLSv1 is deprecated:DeprecationWarning" + ), + ), + pytest.param( + "TLSv1.0", + None, + "TLSv1.0", + None, + True, + id="both-min-tls1.0", + marks=pytest.mark.filterwarnings( + r"ignore:ssl\.TLSVersion\.TLSv1 is deprecated:DeprecationWarning" + ), + ), + pytest.param( + "TLSv1.2", + "TLSv1.3", + "TLSv1.2", + "TLSv1.3", + True, + id="both-tls1.2-1.3", + marks=pytest.mark.xfail( + TWISTED_TLS_LIMITS_OFFBY1, + reason="Can't set max to 1.3 on this Twisted version", + strict=True, + ), + ), + ], + ) + @coroutine_test + async def test_download( + self, + server_tls_min: str | None, + server_tls_max: str | None, + client_tls_min: str | None, + client_tls_max: str | None, + expect_success: bool, + ) -> None: + with SimpleMockServer( + self.keyfile, + self.certfile, + tls_min_version=server_tls_min, + tls_max_version=server_tls_max, + ) as simple_mockserver: + url = f"https://localhost:{simple_mockserver.port(is_secure=True)}/file" + request = Request(url) + async with self.get_dh(client_tls_min, client_tls_max) as dh: + if expect_success: + response = await dh.download_request(request) + assert response.body == b"0123456789" + else: + with pytest.raises( + (DownloadConnectionRefusedError, DownloadFailedError) + ): + await dh.download_request(request) + + @coroutine_test + async def test_invalid_min_version_setting(self) -> None: + with pytest.raises( + ValueError, match="Unknown DOWNLOAD_TLS_MIN_VERSION value: invalid" + ): + async with self.get_dh(client_tls_min="invalid", client_tls_max=None): + pass + + @coroutine_test + async def test_invalid_max_version_setting(self) -> None: + with pytest.raises( + ValueError, match="Unknown DOWNLOAD_TLS_MAX_VERSION value: invalid" + ): + async with self.get_dh(client_tls_min=None, client_tls_max="invalid"): + pass + + +class TestHttpWithCrawlerBase(ABC): + @property + @abstractmethod + def settings_dict(self) -> dict[str, Any] | None: + raise NotImplementedError + + is_secure = False + + @coroutine_test + async def test_download_with_content_length(self, mockserver: MockServer) -> None: + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid + # download it + await maybe_deferred_to_future( + crawler.crawl( + seed=Request( + url=mockserver.url("/partial", is_secure=self.is_secure), + meta={"download_maxsize": 1000}, + ) + ) + ) + assert crawler.spider + failure = crawler.spider.meta["failure"] # type: ignore[attr-defined] + assert isinstance(failure.value, DownloadCancelledError) + + @coroutine_test + async def test_download(self, mockserver: MockServer) -> None: + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + await maybe_deferred_to_future( + crawler.crawl( + seed=Request(url=mockserver.url("", is_secure=self.is_secure)) + ) + ) + assert crawler.spider + failure = crawler.spider.meta.get("failure") # type: ignore[attr-defined] + assert failure is None + reason = crawler.spider.meta["close_reason"] # type: ignore[attr-defined] + assert reason == "finished" + + @coroutine_test + async def test_response_ssl_certificate(self, mockserver: MockServer) -> None: + if not self.is_secure: + pytest.skip("Only applies to HTTPS") + # copy of TestCrawl.test_response_ssl_certificate() + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + url = mockserver.url("/echo?body=test", is_secure=self.is_secure) + await crawler.crawl_async(seed=url, mockserver=mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) + cert = crawler.spider.meta["responses"][0].certificate + assert cert is not None + if isinstance(cert, Certificate): # Twisted + assert cert.getSubject().commonName == b"localhost" + assert cert.getIssuer().commonName == b"localhost" + elif isinstance(cert, bytes): # DER bytes + cert_x509 = load_der_x509_certificate(cert) + assert cert_x509.subject.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE" + assert cert_x509.issuer.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE" + + @coroutine_test + async def test_response_ip_address(self, mockserver: MockServer) -> None: + # copy of TestCrawl.test_response_ip_address() + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + url = mockserver.url("/echo?body=test", is_secure=self.is_secure) + expected_netloc, _ = urlparse(url).netloc.split(":") + await crawler.crawl_async(seed=url, mockserver=mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) + ip_address = crawler.spider.meta["responses"][0].ip_address + assert isinstance(ip_address, IPv4Address) + assert str(ip_address) == gethostbyname(expected_netloc) + + @coroutine_test + async def test_bytes_received_stop_download_callback( + self, mockserver: MockServer + ) -> None: + # copy of TestCrawl.test_bytes_received_stop_download_callback() + crawler = get_crawler(BytesReceivedCallbackSpider, self.settings_dict) + await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure) + assert isinstance(crawler.spider, BytesReceivedCallbackSpider) + assert crawler.spider.meta.get("failure") is None + assert isinstance(crawler.spider.meta["response"], Response) + assert crawler.spider.meta["response"].body == crawler.spider.meta.get( + "bytes_received" + ) + assert ( + len(crawler.spider.meta["response"].body) + < crawler.spider.full_response_length + ) + + @coroutine_test + async def test_bytes_received_stop_download_errback( + self, mockserver: MockServer + ) -> None: + # copy of TestCrawl.test_bytes_received_stop_download_errback() + crawler = get_crawler(BytesReceivedErrbackSpider, self.settings_dict) + await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure) + assert isinstance(crawler.spider, BytesReceivedErrbackSpider) + assert crawler.spider.meta.get("response") is None + assert isinstance(crawler.spider.meta["failure"], Failure) + assert isinstance(crawler.spider.meta["failure"].value, StopDownload) + assert isinstance(crawler.spider.meta["failure"].value.response, Response) + assert crawler.spider.meta[ + "failure" + ].value.response.body == crawler.spider.meta.get("bytes_received") + assert ( + len(crawler.spider.meta["failure"].value.response.body) + < crawler.spider.full_response_length + ) + + @coroutine_test + async def test_headers_received_stop_download_callback( + self, mockserver: MockServer + ) -> None: + # copy of TestCrawl.test_headers_received_stop_download_callback() + crawler = get_crawler(HeadersReceivedCallbackSpider, self.settings_dict) + await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure) + assert isinstance(crawler.spider, HeadersReceivedCallbackSpider) + assert crawler.spider.meta.get("failure") is None + assert isinstance(crawler.spider.meta["response"], Response) + assert crawler.spider.meta["response"].headers == crawler.spider.meta.get( + "headers_received" + ) + + @coroutine_test + async def test_headers_received_stop_download_errback( + self, mockserver: MockServer + ) -> None: + # copy of TestCrawl.test_headers_received_stop_download_errback() + crawler = get_crawler(HeadersReceivedErrbackSpider, self.settings_dict) + await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure) + assert isinstance(crawler.spider, HeadersReceivedErrbackSpider) + assert crawler.spider.meta.get("response") is None + assert isinstance(crawler.spider.meta["failure"], Failure) + assert isinstance(crawler.spider.meta["failure"].value, StopDownload) + assert isinstance(crawler.spider.meta["failure"].value.response, Response) + assert crawler.spider.meta[ + "failure" + ].value.response.headers == crawler.spider.meta.get("headers_received") + + +class TestHttpProxyBase(ABC): + is_secure = False + expected_http_proxy_request_body = b"http://example.com" + # whether the handler supports HTTPS proxies with HTTPS destinations + handler_supports_tls_in_tls: bool = True + + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + @pytest.fixture(scope="session") + def proxy_mockserver(self) -> Generator[ProxyEchoMockServer]: + with ProxyEchoMockServer() as proxy: + yield proxy + + @asynccontextmanager + async def get_dh(self) -> AsyncGenerator[DownloadHandlerProtocol]: + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() + dh = build_from_crawler(self.download_handler_cls, crawler) + try: + yield dh + finally: + await dh.close() + + @coroutine_test + async def test_download_with_proxy( + self, proxy_mockserver: ProxyEchoMockServer + ) -> None: + http_proxy = proxy_mockserver.url("", is_secure=self.is_secure) + request = Request("http://example.com", meta={"proxy": http_proxy}) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body + + @coroutine_test + async def test_download_without_proxy( + self, proxy_mockserver: ProxyEchoMockServer + ) -> None: + request = Request( + proxy_mockserver.url("/path/to/resource", is_secure=self.is_secure) + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == 200 + assert response.url == request.url + assert response.body == b"/path/to/resource" + + @coroutine_test + async def test_download_with_proxy_https_timeout( + self, proxy_mockserver: ProxyEchoMockServer + ) -> None: + if NON_EXISTING_RESOLVABLE: + pytest.skip("Non-existing hosts are resolvable") + if self.is_secure and not self.handler_supports_tls_in_tls: + pytest.skip("HTTPS proxies for HTTPS destinations are not supported") + http_proxy = proxy_mockserver.url("", is_secure=self.is_secure) + domain = "https://no-such-domain.nosuch" + request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2}) + async with self.get_dh() as download_handler: + with pytest.raises(DownloadTimeoutError) as exc_info: + await download_handler.download_request(request) + assert domain in str(exc_info.value) + + @coroutine_test + async def test_download_with_proxy_without_http_scheme( + self, proxy_mockserver: ProxyEchoMockServer + ) -> None: + http_proxy = f"{proxy_mockserver.host}:{proxy_mockserver.port()}" + request = Request("http://example.com", meta={"proxy": http_proxy}) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body + + +class TestMitmProxyBase(ABC): + # whether the handler supports HTTPS proxies with HTTPS destinations + handler_supports_tls_in_tls: bool = True + handler_supports_socks: bool = False + + @property + @abstractmethod + def settings_dict(self) -> dict[str, Any] | None: + raise NotImplementedError + + @pytest.mark.parametrize( + "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] + ) + @pytest.mark.usefixtures("mitm_proxy_server") + @coroutine_test + async def test_http_proxy( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool + ) -> None: + """HTTP proxy, HTTP or HTTPS destination.""" + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + seed=mockserver.url("/status?n=200", is_secure=https_dest) + ) + assert isinstance(crawler.spider, SingleRequestSpider) + self._assert_got_response_code(200, caplog.text) + self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest) + + @pytest.mark.parametrize( + "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] + ) + @pytest.mark.usefixtures("mitm_proxy_server_https") + @coroutine_test + async def test_https_proxy( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool + ) -> None: + """HTTPS proxy, HTTP or HTTPS destination.""" + if https_dest and not self.handler_supports_tls_in_tls: + pytest.skip("HTTPS proxies for HTTPS destinations are not supported") + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + seed=mockserver.url("/status?n=200", is_secure=https_dest) + ) + assert isinstance(crawler.spider, SingleRequestSpider) + self._assert_got_response_code(200, caplog.text) + self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest) + + @pytest.mark.parametrize( + "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] + ) + @pytest.mark.usefixtures("mitm_proxy_server") + @coroutine_test + async def test_http_proxy_auth_error( + self, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, + mockserver: MockServer, + https_dest: bool, + ) -> None: + """HTTP proxy, HTTP or HTTPS destination, wrong proxy creds.""" + envvar = "https_proxy" if https_dest else "http_proxy" + monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar])) + crawler = get_crawler(SimpleSpider, self.settings_dict) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + mockserver.url("/status?n=200", is_secure=https_dest) + ) + # The proxy returns a 407 error code but it does not reach the client; + # it just sees an exception. + self._assert_got_auth_exception(caplog.text) + + @pytest.mark.parametrize( + "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] + ) + @pytest.mark.usefixtures("mitm_proxy_server") + @coroutine_test + async def test_dont_leak_proxy_authorization_header( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool + ) -> None: + """HTTP proxy, HTTP or HTTPS destination. Check that the auth header + is not sent to the destination.""" + request = Request(mockserver.url("/echo", is_secure=https_dest)) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async(seed=request) + assert isinstance(crawler.spider, SingleRequestSpider) + self._assert_got_response_code(200, caplog.text) + self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest) + echo = json.loads(crawler.spider.meta["responses"][0].text) + assert "Proxy-Authorization" not in echo["headers"] + + @pytest.mark.parametrize( + "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] + ) + @pytest.mark.usefixtures("socks5_proxy_server") + @coroutine_test + async def test_download_with_socks_proxy( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool + ) -> None: + """SOCKS5 proxy, HTTP or HTTPS destination.""" + if not self.handler_supports_socks: + pytest.skip("SOCKS proxies are not supported") + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + seed=mockserver.url("/status?n=200", is_secure=https_dest) + ) + assert isinstance(crawler.spider, SingleRequestSpider) + self._assert_got_response_code(200, caplog.text) + self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest) + + @pytest.mark.parametrize( + "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] + ) + @pytest.mark.usefixtures("socks5_proxy_server") + @coroutine_test + async def test_socks_proxy_auth_error( + self, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, + mockserver: MockServer, + https_dest: bool, + ) -> None: + if not self.handler_supports_socks: + pytest.skip("SOCKS proxies are not supported") + envvar = "https_proxy" if https_dest else "http_proxy" + monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar])) + crawler = get_crawler(SimpleSpider, self.settings_dict) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + mockserver.url("/status?n=200", is_secure=https_dest) + ) + assert "DownloadConnectionRefusedError" in caplog.text + + @staticmethod + def _assert_headers(headers: Headers, https_dest: bool) -> None: + assert b"X-Via-Mitmproxy" in headers + if https_dest: + assert b"X-Via-Mitmproxy-TLS" in headers + + @staticmethod + def _assert_got_response_code(code: int, log: str) -> None: + assert str(log).count(f"Crawled ({code})") == 1 + + @staticmethod + def _assert_got_auth_exception(log: str) -> None: + assert "Proxy Authentication Required" in log or "407" in log + + +class TestRealWebsiteBase(ABC): + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + @property + @abstractmethod + def settings_dict(self) -> dict[str, Any] | None: + raise NotImplementedError + + @property + def platform_cert_store_works(self) -> bool: + """Whether valid certificates can be verified. + + Twisted on Windows cannot do that out of the box, see e.g. + https://github.com/twisted/twisted/issues/6371. + """ + return True + + @asynccontextmanager + async def get_dh( + self, settings_dict: dict[str, Any] | None = None + ) -> AsyncGenerator[DownloadHandlerProtocol]: + crawler = get_crawler(DefaultSpider, settings_dict) + crawler.spider = crawler._create_spider() + dh = build_from_crawler(self.download_handler_cls, crawler) + try: + yield dh + finally: + await dh.close() + + @coroutine_test + async def test_download(self) -> None: + request = Request("https://books.toscrape.com/") + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == 200 + assert "All products | Books to Scrape - Sandbox" in response.text + + @coroutine_test + async def test_download_with_spider(self) -> None: + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + await maybe_deferred_to_future( + crawler.crawl(seed=Request("https://books.toscrape.com/")) + ) + assert isinstance(crawler.spider, SingleRequestSpider) + failure = crawler.spider.meta.get("failure") + assert failure is None + reason = crawler.spider.meta["close_reason"] + assert reason == "finished" + + @coroutine_test + async def test_verify_certs(self) -> None: + if not self.platform_cert_store_works: + pytest.skip("Cannot verify certificates") + request = Request("https://books.toscrape.com/") + async with self.get_dh( + {"DOWNLOAD_VERIFY_CERTIFICATES": True} + ) as download_handler: + response = await download_handler.download_request(request) + assert response.status == 200 + assert "All products | Books to Scrape - Sandbox" in response.text + + @pytest.mark.parametrize("verify_certs", [True, False]) + @coroutine_test + async def test_tls_logging( + self, caplog: pytest.LogCaptureFixture, verify_certs: bool + ) -> None: + if verify_certs and not self.platform_cert_store_works: + pytest.skip("Cannot verify certificates") + request = Request("https://books.toscrape.com/") + async with self.get_dh( + { + "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True, + "DOWNLOAD_VERIFY_CERTIFICATES": verify_certs, + } + ) as download_handler: + with caplog.at_level("DEBUG"): + response = await download_handler.download_request(request) + assert response.status == 200 + assert "SSL connection to books.toscrape.com using protocol" in caplog.text diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 062e8a8b4..d321c9047 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,67 +1,75 @@ +from __future__ import annotations + import asyncio +from contextlib import asynccontextmanager +from gzip import BadGzipFile +from typing import TYPE_CHECKING from unittest import mock -from pytest import mark -from twisted.internet import defer -from twisted.internet.defer import Deferred -from twisted.python.failure import Failure -from twisted.trial.unittest import TestCase +import pytest +from twisted.internet.defer import Deferred, succeed from scrapy.core.downloader.middleware import DownloaderMiddlewareManager -from scrapy.exceptions import _InvalidOutput +from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider +from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, get_from_asyncio_queue +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator -class ManagerTestCase(TestCase): +class TestManagerBase: settings_dict = None - def setUp(self): - self.crawler = get_crawler(Spider, self.settings_dict) - self.spider = self.crawler._create_spider("foo") - self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler) - # some mw depends on stats collector - self.crawler.stats.open_spider(self.spider) - return self.mwman.open_spider(self.spider) + # should be a fixture but async fixtures that use Futures are problematic with pytest-twisted + @asynccontextmanager + async def get_mwman(self) -> AsyncGenerator[DownloaderMiddlewareManager]: + crawler = get_crawler(Spider, self.settings_dict) + crawler.spider = crawler._create_spider("foo") + mwman = DownloaderMiddlewareManager.from_crawler(crawler) + crawler.engine = crawler._create_engine() + await crawler.engine.open_spider_async() + try: + yield mwman + finally: + await crawler.engine.close_spider_async() - def tearDown(self): - self.crawler.stats.close_spider(self.spider, "") - return self.mwman.close_spider(self.spider) - - def _download(self, request, response=None): + @staticmethod + async def _download( + mwman: DownloaderMiddlewareManager, + request: Request, + response: Response | None = None, + ) -> Response | Request: """Executes downloader mw manager's download method and returns - the result (Request or Response) or raise exception in case of + the result (Request or Response) or raises exception in case of failure. """ if not response: response = Response(request.url) - def download_func(**kwargs): + async def download_func(request: Request) -> Response: return response - dfd = self.mwman.download(download_func, request, self.spider) - # catch deferred result and return the value - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - ret = results[0] - if isinstance(ret, Failure): - ret.raiseException() - return ret + return await mwman.download_async(download_func, request) -class DefaultsTest(ManagerTestCase): +class TestDefaults(TestManagerBase): """Tests default behavior with default settings""" - def test_request_response(self): + @coroutine_test + async def test_request_response(self): req = Request("http://example.com/index.html") resp = Response(req.url, status=200) - ret = self._download(req, resp) - self.assertTrue(isinstance(ret, Response), "Non-response returned") + async with self.get_mwman() as mwman: + ret = await self._download(mwman, req, resp) + assert isinstance(ret, Response), "Non-response returned" - def test_3xx_and_invalid_gzipped_body_must_redirect(self): + @coroutine_test + async def test_3xx_and_invalid_gzipped_body_must_redirect(self): """Regression test for a failure when redirecting a compressed request. @@ -86,15 +94,15 @@ class DefaultsTest(ManagerTestCase): "Location": "http://example.com/login", }, ) - ret = self._download(request=req, response=resp) - self.assertTrue(isinstance(ret, Request), f"Not redirected: {ret!r}") - self.assertEqual( - to_bytes(ret.url), - resp.headers["Location"], - "Not redirected to location header", + async with self.get_mwman() as mwman: + ret = await self._download(mwman, req, resp) + assert isinstance(ret, Request), f"Not redirected: {ret!r}" + assert to_bytes(ret.url) == resp.headers["Location"], ( + "Not redirected to location header" ) - def test_200_and_invalid_gzipped_body_must_fail(self): + @coroutine_test + async def test_200_and_invalid_gzipped_body_must_fail(self): req = Request("http://example.com") body = b"

You are being redirected

" resp = Response( @@ -108,160 +116,231 @@ class DefaultsTest(ManagerTestCase): "Location": "http://example.com/login", }, ) - self.assertRaises(OSError, self._download, request=req, response=resp) + with pytest.raises(BadGzipFile): + async with self.get_mwman() as mwman: + await self._download(mwman, req, resp) -class ResponseFromProcessRequestTest(ManagerTestCase): +class TestResponseFromProcessRequest(TestManagerBase): """Tests middleware returning a response from process_request.""" - def test_download_func_not_called(self): + @coroutine_test + async def test_download_func_not_called(self): + req = Request("http://example.com/index.html") resp = Response("http://example.com/index.html") + download_func = mock.MagicMock() class ResponseMiddleware: - def process_request(self, request, spider): + def process_request(self, request): return resp - self.mwman._add_middleware(ResponseMiddleware()) + async with self.get_mwman() as mwman: + mwman._add_middleware(ResponseMiddleware()) + result = await mwman.download_async(download_func, req) + assert result is resp + assert not download_func.called + +class TestResponseFromProcessException(TestManagerBase): + """Tests middleware returning a response from process_exception.""" + + @coroutine_test + async def test_process_response_called(self): req = Request("http://example.com/index.html") - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) + resp = Response("http://example.com/index.html") + calls = [] - self.assertIs(results[0], resp) - self.assertFalse(download_func.called) + def download_func(request): + raise ValueError("test") + + class ResponseMiddleware: + def process_response(self, request, response): + calls.append("process_response") + return resp + + def process_exception(self, request, exception): + calls.append("process_exception") + return resp + + async with self.get_mwman() as mwman: + mwman._add_middleware(ResponseMiddleware()) + result = await mwman.download_async(download_func, req) + assert result is resp + assert calls == [ + "process_exception", + "process_response", + ] -class ProcessRequestInvalidOutput(ManagerTestCase): - """Invalid return value for process_request method should raise an exception""" - - def test_invalid_process_request(self): +class TestInvalidOutput(TestManagerBase): + @coroutine_test + async def test_invalid_process_request(self): + """Invalid return value for process_request method should raise an exception""" req = Request("http://example.com/index.html") class InvalidProcessRequestMiddleware: - def process_request(self, request, spider): + def process_request(self, request): return 1 - self.mwman._add_middleware(InvalidProcessRequestMiddleware()) - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self.assertIsInstance(results[0], Failure) - self.assertIsInstance(results[0].value, _InvalidOutput) + async with self.get_mwman() as mwman: + mwman._add_middleware(InvalidProcessRequestMiddleware()) + with pytest.raises(_InvalidOutput): + await self._download(mwman, req) - -class ProcessResponseInvalidOutput(ManagerTestCase): - """Invalid return value for process_response method should raise an exception""" - - def test_invalid_process_response(self): + @coroutine_test + async def test_invalid_process_response(self): + """Invalid return value for process_response method should raise an exception""" req = Request("http://example.com/index.html") class InvalidProcessResponseMiddleware: - def process_response(self, request, response, spider): + def process_response(self, request, response): return 1 - self.mwman._add_middleware(InvalidProcessResponseMiddleware()) - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self.assertIsInstance(results[0], Failure) - self.assertIsInstance(results[0].value, _InvalidOutput) + async with self.get_mwman() as mwman: + mwman._add_middleware(InvalidProcessResponseMiddleware()) + with pytest.raises(_InvalidOutput): + await self._download(mwman, req) - -class ProcessExceptionInvalidOutput(ManagerTestCase): - """Invalid return value for process_exception method should raise an exception""" - - def test_invalid_process_exception(self): + @coroutine_test + async def test_invalid_process_exception(self): + """Invalid return value for process_exception method should raise an exception""" req = Request("http://example.com/index.html") class InvalidProcessExceptionMiddleware: - def process_request(self, request, spider): - raise Exception() + def process_request(self, request): + raise RuntimeError - def process_exception(self, request, exception, spider): + def process_exception(self, request, exception): return 1 - self.mwman._add_middleware(InvalidProcessExceptionMiddleware()) - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self.assertIsInstance(results[0], Failure) - self.assertIsInstance(results[0].value, _InvalidOutput) + async with self.get_mwman() as mwman: + mwman._add_middleware(InvalidProcessExceptionMiddleware()) + with pytest.raises(_InvalidOutput): + await self._download(mwman, req) -class MiddlewareUsingDeferreds(ManagerTestCase): - """Middlewares using Deferreds should work""" +class TestMiddlewareUsingDeferreds(TestManagerBase): + """Middlewares using Deferreds (deprecated) should work""" - def test_deferred(self): + @coroutine_test + async def test_deferred(self): + req = Request("http://example.com/index.html") resp = Response("http://example.com/index.html") + download_func = mock.MagicMock() class DeferredMiddleware: def cb(self, result): return result - def process_request(self, request, spider): + def process_request(self, request): d = Deferred() d.addCallback(self.cb) d.callback(resp) return d - self.mwman._add_middleware(DeferredMiddleware()) - req = Request("http://example.com/index.html") - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - self.assertIs(results[0], resp) - self.assertFalse(download_func.called) + async with self.get_mwman() as mwman: + mwman._add_middleware(DeferredMiddleware()) + with pytest.warns( + ScrapyDeprecationWarning, + match="returned a Deferred, this is deprecated", + ): + result = await mwman.download_async(download_func, req) + assert result is resp + assert not download_func.called -@mark.usefixtures("reactor_pytest") -class MiddlewareUsingCoro(ManagerTestCase): +class TestMiddlewareUsingCoro(TestManagerBase): """Middlewares using asyncio coroutines should work""" - def test_asyncdef(self): + @coroutine_test + async def test_asyncdef(self): + req = Request("http://example.com/index.html") resp = Response("http://example.com/index.html") + download_func = mock.MagicMock() class CoroMiddleware: - async def process_request(self, request, spider): - await defer.succeed(42) + async def process_request(self, request): + await succeed(42) return resp - self.mwman._add_middleware(CoroMiddleware()) + async with self.get_mwman() as mwman: + mwman._add_middleware(CoroMiddleware()) + result = await mwman.download_async(download_func, req) + assert result is resp + assert not download_func.called + + @pytest.mark.only_asyncio + @coroutine_test + async def test_asyncdef_asyncio(self): req = Request("http://example.com/index.html") - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - self.assertIs(results[0], resp) - self.assertFalse(download_func.called) - - @mark.only_asyncio() - def test_asyncdef_asyncio(self): resp = Response("http://example.com/index.html") + download_func = mock.MagicMock() class CoroMiddleware: - async def process_request(self, request, spider): + async def process_request(self, request): await asyncio.sleep(0.1) - result = await get_from_asyncio_queue(resp) - return result + return await get_from_asyncio_queue(resp) - self.mwman._add_middleware(CoroMiddleware()) + async with self.get_mwman() as mwman: + mwman._add_middleware(CoroMiddleware()) + result = await mwman.download_async(download_func, req) + assert result is resp + assert not download_func.called + + +class TestDownloadDeprecated(TestManagerBase): + @coroutine_test + async def test_mwman_download(self): req = Request("http://example.com/index.html") - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) + resp = Response(req.url, status=200) - self.assertIs(results[0], resp) - self.assertFalse(download_func.called) + def download_func(request: Request, spider: Spider) -> Deferred[Response]: + return succeed(resp) + + async with self.get_mwman() as mwman: + with pytest.warns( + ScrapyDeprecationWarning, + match=r"DownloaderMiddlewareManager.download\(\) is deprecated, use download_async\(\) instead", + ): + ret = await maybe_deferred_to_future( + mwman.download(download_func, req, mwman.crawler.spider) + ) + assert isinstance(ret, Response) + + +class TestDeprecatedSpiderArg(TestManagerBase): + @coroutine_test + async def test_deprecated_spider_arg(self): + req = Request("http://example.com/index.html") + resp = Response("http://example.com/index.html") + download_func = mock.MagicMock() + + class DeprecatedSpiderArgMiddleware: + def process_request(self, request, spider): + 1 / 0 + + def process_response(self, request, response, spider): + return response + + def process_exception(self, request, exception, spider): + return resp + + async with self.get_mwman() as mwman: + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"process_request\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"process_response\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"process_exception\(\) requires a spider argument", + ), + ): + mwman._add_middleware(DeprecatedSpiderArgMiddleware()) + result = await mwman.download_async(download_func, req) + assert result is resp + assert not download_func.called diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py deleted file mode 100644 index 043dc0a12..000000000 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ /dev/null @@ -1,63 +0,0 @@ -import unittest - -from scrapy.downloadermiddlewares.ajaxcrawl import AjaxCrawlMiddleware -from scrapy.http import HtmlResponse, Request, Response -from scrapy.spiders import Spider -from scrapy.utils.test import get_crawler - -__doctests__ = ["scrapy.downloadermiddlewares.ajaxcrawl"] - - -class AjaxCrawlMiddlewareTest(unittest.TestCase): - def setUp(self): - crawler = get_crawler(Spider, {"AJAXCRAWL_ENABLED": True}) - self.spider = crawler._create_spider("foo") - self.mw = AjaxCrawlMiddleware.from_crawler(crawler) - - def _ajaxcrawlable_body(self): - return b'' - - def _req_resp(self, url, req_kwargs=None, resp_kwargs=None): - req = Request(url, **(req_kwargs or {})) - resp = HtmlResponse(url, request=req, **(resp_kwargs or {})) - return req, resp - - def test_non_get(self): - req, resp = self._req_resp("http://example.com/", {"method": "HEAD"}) - resp2 = self.mw.process_response(req, resp, self.spider) - self.assertEqual(resp, resp2) - - def test_binary_response(self): - req = Request("http://example.com/") - resp = Response("http://example.com/", body=b"foobar\x00\x01\x02", request=req) - resp2 = self.mw.process_response(req, resp, self.spider) - self.assertIs(resp, resp2) - - def test_ajaxcrawl(self): - req, resp = self._req_resp( - "http://example.com/", - {"meta": {"foo": "bar"}}, - {"body": self._ajaxcrawlable_body()}, - ) - req2 = self.mw.process_response(req, resp, self.spider) - self.assertEqual(req2.url, "http://example.com/?_escaped_fragment_=") - self.assertEqual(req2.meta["foo"], "bar") - - def test_ajaxcrawl_loop(self): - req, resp = self._req_resp( - "http://example.com/", {}, {"body": self._ajaxcrawlable_body()} - ) - req2 = self.mw.process_response(req, resp, self.spider) - resp2 = HtmlResponse(req2.url, body=resp.body, request=req2) - resp3 = self.mw.process_response(req2, resp2, self.spider) - - assert isinstance(resp3, HtmlResponse), (resp3.__class__, resp3) - self.assertEqual(resp3.request.url, "http://example.com/?_escaped_fragment_=") - assert resp3 is resp2 - - def test_noncrawlable_body(self): - req, resp = self._req_resp( - "http://example.com/", {}, {"body": b""} - ) - resp2 = self.mw.process_response(req, resp, self.spider) - self.assertIs(resp, resp2) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 4a81a638e..225562644 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -1,5 +1,4 @@ import logging -from unittest import TestCase import pytest from testfixtures import LogCapture @@ -9,11 +8,12 @@ from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response -from scrapy.settings import Settings -from scrapy.spiders import Spider from scrapy.utils.python import to_bytes +from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler +UNSET = object() + def _cookie_to_set_cookie_value(cookie): """Given a cookie defined as a dictionary with name and value keys, and @@ -23,7 +23,7 @@ def _cookie_to_set_cookie_value(cookie): for key in ("name", "value", "path", "domain"): if cookie.get(key) is None: if key in ("name", "value"): - return + return None continue if isinstance(cookie[key], (bool, float, int, str)): decoded[key] = str(cookie[key]) @@ -42,7 +42,7 @@ def _cookie_to_set_cookie_value(cookie): def _cookies_to_set_cookie_list(cookies): """Given a group of cookie defined either as a dictionary or as a list of dictionaries (i.e. in a format supported by the cookies parameter of - Request), return the equivalen list of strings that can be associated to a + Request), return the equivalent list of strings that can be associated to a ``Set-Cookie`` header.""" if not cookies: return [] @@ -51,49 +51,49 @@ def _cookies_to_set_cookie_list(cookies): return filter(None, (_cookie_to_set_cookie_value(cookie) for cookie in cookies)) -class CookiesMiddlewareTest(TestCase): +class TestCookiesMiddleware: def assertCookieValEqual(self, first, second, msg=None): def split_cookies(cookies): return sorted([s.strip() for s in to_bytes(cookies).split(b";")]) - return self.assertEqual(split_cookies(first), split_cookies(second), msg=msg) + assert split_cookies(first) == split_cookies(second), msg - def setUp(self): - self.spider = Spider("foo") - self.mw = CookiesMiddleware() - self.redirect_middleware = RedirectMiddleware(settings=Settings()) + def setup_method(self): + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() + self.mw = CookiesMiddleware.from_crawler(crawler) + self.redirect_middleware = RedirectMiddleware.from_crawler(crawler) - def tearDown(self): + def teardown_method(self): del self.mw del self.redirect_middleware def test_basic(self): req = Request("http://scrapytest.org/") - assert self.mw.process_request(req, self.spider) is None + assert self.mw.process_request(req) is None assert "Cookie" not in req.headers headers = {"Set-Cookie": "C1=value1; path=/"} res = Response("http://scrapytest.org/", headers=headers) - assert self.mw.process_response(req, res, self.spider) is res + assert self.mw.process_response(req, res) is res req2 = Request("http://scrapytest.org/sub1/") - assert self.mw.process_request(req2, self.spider) is None - self.assertEqual(req2.headers.get("Cookie"), b"C1=value1") + assert self.mw.process_request(req2) is None + assert req2.headers.get("Cookie") == b"C1=value1" def test_setting_false_cookies_enabled(self): - self.assertRaises( - NotConfigured, - CookiesMiddleware.from_crawler, - get_crawler(settings_dict={"COOKIES_ENABLED": False}), - ) + with pytest.raises(NotConfigured): + CookiesMiddleware.from_crawler( + get_crawler(settings_dict={"COOKIES_ENABLED": False}) + ) def test_setting_default_cookies_enabled(self): - self.assertIsInstance( + assert isinstance( CookiesMiddleware.from_crawler(get_crawler()), CookiesMiddleware ) def test_setting_true_cookies_enabled(self): - self.assertIsInstance( + assert isinstance( CookiesMiddleware.from_crawler( get_crawler(settings_dict={"COOKIES_ENABLED": True}) ), @@ -112,9 +112,9 @@ class CookiesMiddlewareTest(TestCase): res = Response( "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"} ) - mw.process_response(req, res, crawler.spider) + mw.process_response(req, res) req2 = Request("http://scrapytest.org/sub1/") - mw.process_request(req2, crawler.spider) + mw.process_request(req2) log.check( ( @@ -143,35 +143,35 @@ class CookiesMiddlewareTest(TestCase): res = Response( "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"} ) - mw.process_response(req, res, crawler.spider) + mw.process_response(req, res) req2 = Request("http://scrapytest.org/sub1/") - mw.process_request(req2, crawler.spider) + mw.process_request(req2) log.check() def test_do_not_break_on_non_utf8_header(self): req = Request("http://scrapytest.org/") - assert self.mw.process_request(req, self.spider) is None + assert self.mw.process_request(req) is None assert "Cookie" not in req.headers headers = {"Set-Cookie": b"C1=in\xa3valid; path=/", "Other": b"ignore\xa3me"} res = Response("http://scrapytest.org/", headers=headers) - assert self.mw.process_response(req, res, self.spider) is res + assert self.mw.process_response(req, res) is res req2 = Request("http://scrapytest.org/sub1/") - assert self.mw.process_request(req2, self.spider) is None - self.assertIn("Cookie", req2.headers) + assert self.mw.process_request(req2) is None + assert "Cookie" in req2.headers def test_dont_merge_cookies(self): # merge some cookies into jar headers = {"Set-Cookie": "C1=value1; path=/"} req = Request("http://scrapytest.org/") res = Response("http://scrapytest.org/", headers=headers) - assert self.mw.process_response(req, res, self.spider) is res + assert self.mw.process_response(req, res) is res # test Cookie header is not seted to request req = Request("http://scrapytest.org/dontmerge", meta={"dont_merge_cookies": 1}) - assert self.mw.process_request(req, self.spider) is None + assert self.mw.process_request(req) is None assert "Cookie" not in req.headers # check that returned cookies are not merged back to jar @@ -179,17 +179,17 @@ class CookiesMiddlewareTest(TestCase): "http://scrapytest.org/dontmerge", headers={"Set-Cookie": "dont=mergeme; path=/"}, ) - assert self.mw.process_response(req, res, self.spider) is res + assert self.mw.process_response(req, res) is res # check that cookies are merged back req = Request("http://scrapytest.org/mergeme") - assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers.get("Cookie"), b"C1=value1") + assert self.mw.process_request(req) is None + assert req.headers.get("Cookie") == b"C1=value1" # check that cookies are merged when dont_merge_cookies is passed as 0 req = Request("http://scrapytest.org/mergeme", meta={"dont_merge_cookies": 0}) - assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers.get("Cookie"), b"C1=value1") + assert self.mw.process_request(req) is None + assert req.headers.get("Cookie") == b"C1=value1" def test_complex_cookies(self): # merge some cookies into jar @@ -216,11 +216,11 @@ class CookiesMiddlewareTest(TestCase): ] req = Request("http://scrapytest.org/", cookies=cookies) - self.mw.process_request(req, self.spider) + self.mw.process_request(req) # embed C1 and C3 for scrapytest.org/foo req = Request("http://scrapytest.org/foo") - self.mw.process_request(req, self.spider) + self.mw.process_request(req) assert req.headers.get("Cookie") in ( b"C1=value1; C3=value3", b"C3=value3; C1=value1", @@ -228,25 +228,25 @@ class CookiesMiddlewareTest(TestCase): # embed C2 for scrapytest.org/bar req = Request("http://scrapytest.org/bar") - self.mw.process_request(req, self.spider) - self.assertEqual(req.headers.get("Cookie"), b"C2=value2") + self.mw.process_request(req) + assert req.headers.get("Cookie") == b"C2=value2" # embed nothing for scrapytest.org/baz req = Request("http://scrapytest.org/baz") - self.mw.process_request(req, self.spider) + self.mw.process_request(req) assert "Cookie" not in req.headers def test_merge_request_cookies(self): req = Request("http://scrapytest.org/", cookies={"galleta": "salada"}) - assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers.get("Cookie"), b"galleta=salada") + assert self.mw.process_request(req) is None + assert req.headers.get("Cookie") == b"galleta=salada" headers = {"Set-Cookie": "C1=value1; path=/"} res = Response("http://scrapytest.org/", headers=headers) - assert self.mw.process_response(req, res, self.spider) is res + assert self.mw.process_response(req, res) is res req2 = Request("http://scrapytest.org/sub1/") - assert self.mw.process_request(req2, self.spider) is None + assert self.mw.process_request(req2) is None self.assertCookieValEqual( req2.headers.get("Cookie"), b"C1=value1; galleta=salada" @@ -258,15 +258,15 @@ class CookiesMiddlewareTest(TestCase): cookies={"galleta": "salada"}, meta={"cookiejar": "store1"}, ) - assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers.get("Cookie"), b"galleta=salada") + assert self.mw.process_request(req) is None + assert req.headers.get("Cookie") == b"galleta=salada" headers = {"Set-Cookie": "C1=value1; path=/"} res = Response("http://scrapytest.org/", headers=headers, request=req) - assert self.mw.process_response(req, res, self.spider) is res + assert self.mw.process_response(req, res) is res req2 = Request("http://scrapytest.org/", meta=res.meta) - assert self.mw.process_request(req2, self.spider) is None + assert self.mw.process_request(req2) is None self.assertCookieValEqual( req2.headers.get("Cookie"), b"C1=value1; galleta=salada" ) @@ -276,63 +276,63 @@ class CookiesMiddlewareTest(TestCase): cookies={"galleta": "dulce"}, meta={"cookiejar": "store2"}, ) - assert self.mw.process_request(req3, self.spider) is None - self.assertEqual(req3.headers.get("Cookie"), b"galleta=dulce") + assert self.mw.process_request(req3) is None + assert req3.headers.get("Cookie") == b"galleta=dulce" headers = {"Set-Cookie": "C2=value2; path=/"} res2 = Response("http://scrapytest.org/", headers=headers, request=req3) - assert self.mw.process_response(req3, res2, self.spider) is res2 + assert self.mw.process_response(req3, res2) is res2 req4 = Request("http://scrapytest.org/", meta=res2.meta) - assert self.mw.process_request(req4, self.spider) is None + assert self.mw.process_request(req4) is None self.assertCookieValEqual( req4.headers.get("Cookie"), b"C2=value2; galleta=dulce" ) # cookies from hosts with port req5_1 = Request("http://scrapytest.org:1104/") - assert self.mw.process_request(req5_1, self.spider) is None + assert self.mw.process_request(req5_1) is None headers = {"Set-Cookie": "C1=value1; path=/"} res5_1 = Response( "http://scrapytest.org:1104/", headers=headers, request=req5_1 ) - assert self.mw.process_response(req5_1, res5_1, self.spider) is res5_1 + assert self.mw.process_response(req5_1, res5_1) is res5_1 req5_2 = Request("http://scrapytest.org:1104/some-redirected-path") - assert self.mw.process_request(req5_2, self.spider) is None - self.assertEqual(req5_2.headers.get("Cookie"), b"C1=value1") + assert self.mw.process_request(req5_2) is None + assert req5_2.headers.get("Cookie") == b"C1=value1" req5_3 = Request("http://scrapytest.org/some-redirected-path") - assert self.mw.process_request(req5_3, self.spider) is None - self.assertEqual(req5_3.headers.get("Cookie"), b"C1=value1") + assert self.mw.process_request(req5_3) is None + assert req5_3.headers.get("Cookie") == b"C1=value1" # skip cookie retrieval for not http request req6 = Request("file:///scrapy/sometempfile") - assert self.mw.process_request(req6, self.spider) is None - self.assertEqual(req6.headers.get("Cookie"), None) + assert self.mw.process_request(req6) is None + assert req6.headers.get("Cookie") is None def test_local_domain(self): request = Request("http://example-host/", cookies={"currencyCookie": "USD"}) - assert self.mw.process_request(request, self.spider) is None - self.assertIn("Cookie", request.headers) - self.assertEqual(b"currencyCookie=USD", request.headers["Cookie"]) + assert self.mw.process_request(request) is None + assert "Cookie" in request.headers + assert request.headers["Cookie"] == b"currencyCookie=USD" @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_from_default_request_headers_middleware(self): - DEFAULT_REQUEST_HEADERS = dict(Cookie="default=value; asdf=qwerty") + DEFAULT_REQUEST_HEADERS = {"Cookie": "default=value; asdf=qwerty"} mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items()) # overwrite with values from 'cookies' request argument req1 = Request("http://example.org", cookies={"default": "something"}) - assert mw_default_headers.process_request(req1, self.spider) is None - assert self.mw.process_request(req1, self.spider) is None + assert mw_default_headers.process_request(req1) is None + assert self.mw.process_request(req1) is None self.assertCookieValEqual( req1.headers["Cookie"], b"default=something; asdf=qwerty" ) # keep both req2 = Request("http://example.com", cookies={"a": "b"}) - assert mw_default_headers.process_request(req2, self.spider) is None - assert self.mw.process_request(req2, self.spider) is None + assert mw_default_headers.process_request(req2) is None + assert self.mw.process_request(req2) is None self.assertCookieValEqual( req2.headers["Cookie"], b"default=value; a=b; asdf=qwerty" ) @@ -341,13 +341,13 @@ class CookiesMiddlewareTest(TestCase): def test_keep_cookie_header(self): # keep only cookies from 'Cookie' request header req1 = Request("http://scrapytest.org", headers={"Cookie": "a=b; c=d"}) - assert self.mw.process_request(req1, self.spider) is None + assert self.mw.process_request(req1) is None self.assertCookieValEqual(req1.headers["Cookie"], "a=b; c=d") # keep cookies from both 'Cookie' request header and 'cookies' keyword req2 = Request( "http://scrapytest.org", headers={"Cookie": "a=b; c=d"}, cookies={"e": "f"} ) - assert self.mw.process_request(req2, self.spider) is None + assert self.mw.process_request(req2) is None self.assertCookieValEqual(req2.headers["Cookie"], "a=b; c=d; e=f") # overwrite values from 'Cookie' request header with 'cookies' keyword req3 = Request( @@ -355,40 +355,40 @@ class CookiesMiddlewareTest(TestCase): headers={"Cookie": "a=b; c=d"}, cookies={"a": "new", "e": "f"}, ) - assert self.mw.process_request(req3, self.spider) is None + assert self.mw.process_request(req3) is None self.assertCookieValEqual(req3.headers["Cookie"], "a=new; c=d; e=f") def test_request_cookies_encoding(self): # 1) UTF8-encoded bytes - req1 = Request("http://example.org", cookies={"a": "á".encode("utf8")}) - assert self.mw.process_request(req1, self.spider) is None + req1 = Request("http://example.org", cookies={"a": "á".encode()}) + assert self.mw.process_request(req1) is None self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1") # 2) Non UTF8-encoded bytes req2 = Request("http://example.org", cookies={"a": "á".encode("latin1")}) - assert self.mw.process_request(req2, self.spider) is None + assert self.mw.process_request(req2) is None self.assertCookieValEqual(req2.headers["Cookie"], b"a=\xc3\xa1") # 3) String req3 = Request("http://example.org", cookies={"a": "á"}) - assert self.mw.process_request(req3, self.spider) is None + assert self.mw.process_request(req3) is None self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1") @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_request_headers_cookie_encoding(self): # 1) UTF8-encoded bytes - req1 = Request("http://example.org", headers={"Cookie": "a=á".encode("utf8")}) - assert self.mw.process_request(req1, self.spider) is None + req1 = Request("http://example.org", headers={"Cookie": "a=á".encode()}) + assert self.mw.process_request(req1) is None self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1") # 2) Non UTF8-encoded bytes req2 = Request("http://example.org", headers={"Cookie": "a=á".encode("latin1")}) - assert self.mw.process_request(req2, self.spider) is None + assert self.mw.process_request(req2) is None self.assertCookieValEqual(req2.headers["Cookie"], b"a=\xc3\xa1") # 3) String req3 = Request("http://example.org", headers={"Cookie": "a=á"}) - assert self.mw.process_request(req3, self.spider) is None + assert self.mw.process_request(req3) is None self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1") def test_invalid_cookies(self): @@ -402,31 +402,31 @@ class CookiesMiddlewareTest(TestCase): ) as lc: cookies1 = [{"value": "bar"}, {"name": "key", "value": "value1"}] req1 = Request("http://example.org/1", cookies=cookies1) - assert self.mw.process_request(req1, self.spider) is None + assert self.mw.process_request(req1) is None cookies2 = [{"name": "foo"}, {"name": "key", "value": "value2"}] req2 = Request("http://example.org/2", cookies=cookies2) - assert self.mw.process_request(req2, self.spider) is None + assert self.mw.process_request(req2) is None cookies3 = [{"name": "foo", "value": None}, {"name": "key", "value": ""}] req3 = Request("http://example.org/3", cookies=cookies3) - assert self.mw.process_request(req3, self.spider) is None + assert self.mw.process_request(req3) is None lc.check( ( "scrapy.downloadermiddlewares.cookies", "WARNING", "Invalid cookie found in request :" - " {'value': 'bar'} ('name' is missing)", + " {'value': 'bar', 'secure': False} ('name' is missing)", ), ( "scrapy.downloadermiddlewares.cookies", "WARNING", "Invalid cookie found in request :" - " {'name': 'foo'} ('value' is missing)", + " {'name': 'foo', 'secure': False} ('value' is missing)", ), ( "scrapy.downloadermiddlewares.cookies", "WARNING", "Invalid cookie found in request :" - " {'name': 'foo', 'value': None} ('value' is missing)", + " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)", ), ) self.assertCookieValEqual(req1.headers["Cookie"], "key=value1") @@ -436,22 +436,22 @@ class CookiesMiddlewareTest(TestCase): def test_primitive_type_cookies(self): # Boolean req1 = Request("http://example.org", cookies={"a": True}) - assert self.mw.process_request(req1, self.spider) is None + assert self.mw.process_request(req1) is None self.assertCookieValEqual(req1.headers["Cookie"], b"a=True") # Float req2 = Request("http://example.org", cookies={"a": 9.5}) - assert self.mw.process_request(req2, self.spider) is None + assert self.mw.process_request(req2) is None self.assertCookieValEqual(req2.headers["Cookie"], b"a=9.5") # Integer req3 = Request("http://example.org", cookies={"a": 10}) - assert self.mw.process_request(req3, self.spider) is None + assert self.mw.process_request(req3) is None self.assertCookieValEqual(req3.headers["Cookie"], b"a=10") # String req4 = Request("http://example.org", cookies={"a": "b"}) - assert self.mw.process_request(req4, self.spider) is None + assert self.mw.process_request(req4) is None self.assertCookieValEqual(req4.headers["Cookie"], b"a=b") def _test_cookie_redirect( @@ -471,9 +471,9 @@ class CookiesMiddlewareTest(TestCase): target.setdefault("status", 301) request1 = Request(cookies=input_cookies, **source) - self.mw.process_request(request1, self.spider) + self.mw.process_request(request1) cookies = request1.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies1 else None) + assert cookies == (b"a=b" if cookies1 else None) response = Response( headers={ @@ -481,21 +481,14 @@ class CookiesMiddlewareTest(TestCase): }, **target, ) - self.assertEqual( - self.mw.process_response(request1, response, self.spider), - response, - ) + assert self.mw.process_response(request1, response) == response - request2 = self.redirect_middleware.process_response( - request1, - response, - self.spider, - ) - self.assertIsInstance(request2, Request) + request2 = self.redirect_middleware.process_response(request1, response) + assert isinstance(request2, Request) - self.mw.process_request(request2, self.spider) + self.mw.process_request(request2) cookies = request2.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies2 else None) + assert cookies == (b"a=b" if cookies2 else None) def test_cookie_redirect_same_domain(self): self._test_cookie_redirect( @@ -567,15 +560,11 @@ class CookiesMiddlewareTest(TestCase): **target, ) - request2 = self.redirect_middleware.process_response( - request1, - response, - self.spider, - ) - self.assertIsInstance(request2, Request) + request2 = self.redirect_middleware.process_response(request1, response) + assert isinstance(request2, Request) cookies = request2.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies2 else None) + assert cookies == (b"a=b" if cookies2 else None) def test_cookie_header_redirect_same_domain(self): self._test_cookie_header_redirect( @@ -623,14 +612,14 @@ class CookiesMiddlewareTest(TestCase): ] request1 = Request(url1, cookies=input_cookies) - self.mw.process_request(request1, self.spider) + self.mw.process_request(request1) cookies = request1.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies1 else None) + assert cookies == (b"a=b" if cookies1 else None) request2 = Request(url2) - self.mw.process_request(request2, self.spider) + self.mw.process_request(request2) cookies = request2.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies2 else None) + assert cookies == (b"a=b" if cookies2 else None) def test_user_set_cookie_domain_suffix_private(self): self._test_user_set_cookie_domain_followup( @@ -677,7 +666,7 @@ class CookiesMiddlewareTest(TestCase): cookies, ): request1 = Request(url1) - self.mw.process_request(request1, self.spider) + self.mw.process_request(request1) input_cookies = [ { @@ -691,15 +680,12 @@ class CookiesMiddlewareTest(TestCase): "Set-Cookie": _cookies_to_set_cookie_list(input_cookies), } response = Response(url1, status=200, headers=headers) - self.assertEqual( - self.mw.process_response(request1, response, self.spider), - response, - ) + assert self.mw.process_response(request1, response) == response request2 = Request(url2) - self.mw.process_request(request2, self.spider) + self.mw.process_request(request2) actual_cookies = request2.headers.get("Cookie") - self.assertEqual(actual_cookies, b"a=b" if cookies else None) + assert actual_cookies == (b"a=b" if cookies else None) def test_server_set_cookie_domain_suffix_private(self): self._test_server_set_cookie_domain_followup( @@ -732,3 +718,99 @@ class CookiesMiddlewareTest(TestCase): "co.uk", cookies=True, ) + + def _test_cookie_redirect_scheme_change( + self, secure, from_scheme, to_scheme, cookies1, cookies2, cookies3 + ): + """When a redirect causes the URL scheme to change from *from_scheme* + to *to_scheme*, while domain and port remain the same, and given a + cookie on the initial request with its secure attribute set to + *secure*, check if the cookie should be set on the Cookie header of the + initial request (*cookies1*), if it should be kept by the redirect + middleware (*cookies2*), and if it should be present on the Cookie + header in the redirected request (*cookie3*).""" + cookie_kwargs = {} + if secure is not UNSET: + cookie_kwargs["secure"] = secure + input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}] + + request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies) + self.mw.process_request(request1) + cookies = request1.headers.get("Cookie") + assert cookies == (b"a=b" if cookies1 else None) + + response = Response( + f"{from_scheme}://a.example", + headers={"Location": f"{to_scheme}://a.example"}, + status=301, + ) + assert self.mw.process_response(request1, response) == response + + request2 = self.redirect_middleware.process_response(request1, response) + assert isinstance(request2, Request) + cookies = request2.headers.get("Cookie") + assert cookies == (b"a=b" if cookies2 else None) + + self.mw.process_request(request2) + cookies = request2.headers.get("Cookie") + assert cookies == (b"a=b" if cookies3 else None) + + def test_cookie_redirect_secure_undefined_downgrade(self): + self._test_cookie_redirect_scheme_change( + secure=UNSET, + from_scheme="https", + to_scheme="http", + cookies1=True, + cookies2=False, + cookies3=False, + ) + + def test_cookie_redirect_secure_undefined_upgrade(self): + self._test_cookie_redirect_scheme_change( + secure=UNSET, + from_scheme="http", + to_scheme="https", + cookies1=True, + cookies2=True, + cookies3=True, + ) + + def test_cookie_redirect_secure_false_downgrade(self): + self._test_cookie_redirect_scheme_change( + secure=False, + from_scheme="https", + to_scheme="http", + cookies1=True, + cookies2=False, + cookies3=True, + ) + + def test_cookie_redirect_secure_false_upgrade(self): + self._test_cookie_redirect_scheme_change( + secure=False, + from_scheme="http", + to_scheme="https", + cookies1=True, + cookies2=True, + cookies3=True, + ) + + def test_cookie_redirect_secure_true_downgrade(self): + self._test_cookie_redirect_scheme_change( + secure=True, + from_scheme="https", + to_scheme="http", + cookies1=True, + cookies2=False, + cookies3=False, + ) + + def test_cookie_redirect_secure_true_upgrade(self): + self._test_cookie_redirect_scheme_change( + secure=True, + from_scheme="http", + to_scheme="https", + cookies1=False, + cookies2=False, + cookies3=True, + ) diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index 27d6224b4..e697de9b7 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -1,5 +1,3 @@ -from unittest import TestCase - from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.http import Request from scrapy.spiders import Spider @@ -7,29 +5,28 @@ from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler -class TestDefaultHeadersMiddleware(TestCase): - def get_defaults_spider_mw(self): +class TestDefaultHeadersMiddleware: + def get_defaults_mw(self): crawler = get_crawler(Spider) - spider = crawler._create_spider("foo") defaults = { to_bytes(k): [to_bytes(v)] for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items() } - return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler) + return defaults, DefaultHeadersMiddleware.from_crawler(crawler) def test_process_request(self): - defaults, spider, mw = self.get_defaults_spider_mw() + defaults, mw = self.get_defaults_mw() req = Request("http://www.scrapytest.org") - mw.process_request(req, spider) - self.assertEqual(req.headers, defaults) + mw.process_request(req) + assert req.headers == defaults def test_update_headers(self): - defaults, spider, mw = self.get_defaults_spider_mw() + defaults, mw = self.get_defaults_mw() headers = {"Accept-Language": ["es"], "Test-Header": ["test"]} bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]} req = Request("http://www.scrapytest.org", headers=headers) - self.assertEqual(req.headers, bytes_headers) + assert req.headers == bytes_headers - mw.process_request(req, spider) + mw.process_request(req) defaults.update(bytes_headers) - self.assertEqual(req.headers, defaults) + assert req.headers == defaults diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index 44458ade8..c744d259c 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -1,12 +1,10 @@ -import unittest - from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class DownloadTimeoutMiddlewareTest(unittest.TestCase): +class TestDownloadTimeoutMiddleware: def get_request_spider_mw(self, settings=None): crawler = get_crawler(Spider, settings) spider = crawler._create_spider("foo") @@ -16,26 +14,24 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase): def test_default_download_timeout(self): req, spider, mw = self.get_request_spider_mw() mw.spider_opened(spider) - assert mw.process_request(req, spider) is None - self.assertEqual(req.meta.get("download_timeout"), 180) + assert mw.process_request(req) is None + assert req.meta.get("download_timeout") == 180 def test_string_download_timeout(self): req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"}) mw.spider_opened(spider) - assert mw.process_request(req, spider) is None - self.assertEqual(req.meta.get("download_timeout"), 20.1) + assert mw.process_request(req) is None + assert req.meta.get("download_timeout") == 20.1 - def test_spider_has_download_timeout(self): - req, spider, mw = self.get_request_spider_mw() - spider.download_timeout = 2 + def test_setting_has_download_timeout(self): + req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2}) mw.spider_opened(spider) - assert mw.process_request(req, spider) is None - self.assertEqual(req.meta.get("download_timeout"), 2) + assert mw.process_request(req) is None + assert req.meta.get("download_timeout") == 2 def test_request_has_download_timeout(self): - req, spider, mw = self.get_request_spider_mw() - spider.download_timeout = 2 + req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2}) mw.spider_opened(spider) req.meta["download_timeout"] = 1 - assert mw.process_request(req, spider) is None - self.assertEqual(req.meta.get("download_timeout"), 1) + assert mw.process_request(req) is None + assert req.meta.get("download_timeout") == 1 diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index fc110e6cc..522a3002f 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,5 +1,4 @@ -import unittest - +import pytest from w3lib.http import basic_auth_header from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware @@ -7,78 +6,78 @@ from scrapy.http import Request from scrapy.spiders import Spider -class TestSpiderLegacy(Spider): +class LegacySpider(Spider): http_user = "foo" http_pass = "bar" -class TestSpider(Spider): +class DomainSpider(Spider): http_user = "foo" http_pass = "bar" http_auth_domain = "example.com" -class TestSpiderAny(Spider): +class AnyDomainSpider(Spider): http_user = "foo" http_pass = "bar" http_auth_domain = None -class HttpAuthMiddlewareLegacyTest(unittest.TestCase): - def setUp(self): - self.spider = TestSpiderLegacy("foo") +class TestHttpAuthMiddlewareLegacy: + def setup_method(self): + self.spider = LegacySpider("foo") def test_auth(self): - with self.assertRaises(AttributeError): - mw = HttpAuthMiddleware() + mw = HttpAuthMiddleware() + with pytest.raises(AttributeError): mw.spider_opened(self.spider) -class HttpAuthMiddlewareTest(unittest.TestCase): - def setUp(self): +class TestHttpAuthMiddleware: + def setup_method(self): self.mw = HttpAuthMiddleware() - self.spider = TestSpider("foo") - self.mw.spider_opened(self.spider) + spider = DomainSpider("foo") + self.mw.spider_opened(spider) - def tearDown(self): + def teardown_method(self): del self.mw def test_no_auth(self): req = Request("http://example-noauth.com/") - assert self.mw.process_request(req, self.spider) is None - self.assertNotIn("Authorization", req.headers) + assert self.mw.process_request(req) is None + assert "Authorization" not in req.headers def test_auth_domain(self): req = Request("http://example.com/") - assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) + assert self.mw.process_request(req) is None + assert req.headers["Authorization"] == basic_auth_header("foo", "bar") def test_auth_subdomain(self): req = Request("http://foo.example.com/") - assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) + assert self.mw.process_request(req) is None + assert req.headers["Authorization"] == basic_auth_header("foo", "bar") def test_auth_already_set(self): - req = Request("http://example.com/", headers=dict(Authorization="Digest 123")) - assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], b"Digest 123") + req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) + assert self.mw.process_request(req) is None + assert req.headers["Authorization"] == b"Digest 123" -class HttpAuthAnyMiddlewareTest(unittest.TestCase): - def setUp(self): +class TestHttpAuthAnyMiddleware: + def setup_method(self): self.mw = HttpAuthMiddleware() - self.spider = TestSpiderAny("foo") - self.mw.spider_opened(self.spider) + spider = AnyDomainSpider("foo") + self.mw.spider_opened(spider) - def tearDown(self): + def teardown_method(self): del self.mw def test_auth(self): req = Request("http://example.com/") - assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) + assert self.mw.process_request(req) is None + assert req.headers["Authorization"] == basic_auth_header("foo", "bar") def test_auth_already_set(self): - req = Request("http://example.com/", headers=dict(Authorization="Digest 123")) - assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], b"Digest 123") + req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) + assert self.mw.process_request(req) is None + assert req.headers["Authorization"] == b"Digest 123" diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index f80eff3e6..548c0d8ee 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -1,28 +1,36 @@ +from __future__ import annotations + import email.utils import shutil import tempfile import time -import unittest from contextlib import contextmanager +from typing import TYPE_CHECKING, Any + +import pytest from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware from scrapy.exceptions import IgnoreRequest from scrapy.http import HtmlResponse, Request, Response -from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +if TYPE_CHECKING: + from collections.abc import Generator -class _BaseTest(unittest.TestCase): - storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" - policy_class = "scrapy.extensions.httpcache.RFC2616Policy" + from scrapy.crawler import Crawler - def setUp(self): + +class TestBase: + """Base class with common setup and helper methods.""" + + policy_class: str + storage_class: str + + def setup_method(self): self.yesterday = email.utils.formatdate(time.time() - 86400) self.today = email.utils.formatdate() self.tomorrow = email.utils.formatdate(time.time() + 86400) - self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider("example.com") self.tmpdir = tempfile.mkdtemp() self.request = Request("http://www.example.com", headers={"User-Agent": "test"}) self.response = Response( @@ -31,13 +39,11 @@ class _BaseTest(unittest.TestCase): body=b"test body", status=202, ) - self.crawler.stats.open_spider(self.spider) - def tearDown(self): - self.crawler.stats.close_spider(self.spider, "") + def teardown_method(self): shutil.rmtree(self.tmpdir) - def _get_settings(self, **new_settings): + def _get_settings(self, **new_settings: Any) -> dict[str, Any]: settings = { "HTTPCACHE_ENABLED": True, "HTTPCACHE_DIR": self.tmpdir, @@ -47,138 +53,109 @@ class _BaseTest(unittest.TestCase): "HTTPCACHE_STORAGE": self.storage_class, } settings.update(new_settings) - return Settings(settings) + return settings @contextmanager - def _storage(self, **new_settings): - with self._middleware(**new_settings) as mw: - yield mw.storage - - @contextmanager - def _policy(self, **new_settings): - with self._middleware(**new_settings) as mw: - yield mw.policy - - @contextmanager - def _middleware(self, **new_settings): + def _get_crawler(self, **new_settings: Any) -> Generator[Crawler]: settings = self._get_settings(**new_settings) - mw = HttpCacheMiddleware(settings, self.crawler.stats) - mw.spider_opened(self.spider) + crawler = get_crawler(Spider, settings) + crawler.spider = crawler._create_spider("example.com") + assert crawler.stats + crawler.stats.open_spider() try: - yield mw + yield crawler finally: - mw.spider_closed(self.spider) + crawler.stats.close_spider() + + @contextmanager + def _storage(self, **new_settings: Any): + with self._middleware(**new_settings) as mw: + yield mw.storage, mw.crawler + + @contextmanager + def _middleware(self, **new_settings: Any) -> Generator[HttpCacheMiddleware]: + with self._get_crawler(**new_settings) as crawler: + assert crawler.spider + mw = HttpCacheMiddleware.from_crawler(crawler) + mw.spider_opened(crawler.spider) + try: + yield mw + finally: + mw.spider_closed(crawler.spider) def assertEqualResponse(self, response1, response2): - self.assertEqual(response1.url, response2.url) - self.assertEqual(response1.status, response2.status) - self.assertEqual(response1.headers, response2.headers) - self.assertEqual(response1.body, response2.body) - - def assertEqualRequest(self, request1, request2): - self.assertEqual(request1.url, request2.url) - self.assertEqual(request1.headers, request2.headers) - self.assertEqual(request1.body, request2.body) - - def assertEqualRequestButWithCacheValidators(self, request1, request2): - self.assertEqual(request1.url, request2.url) - assert b"If-None-Match" not in request1.headers - assert b"If-Modified-Since" not in request1.headers - assert any( - h in request2.headers for h in (b"If-None-Match", b"If-Modified-Since") - ) - self.assertEqual(request1.body, request2.body) - - def test_dont_cache(self): - with self._middleware() as mw: - self.request.meta["dont_cache"] = True - mw.process_response(self.request, self.response, self.spider) - self.assertEqual( - mw.storage.retrieve_response(self.spider, self.request), None - ) - - with self._middleware() as mw: - self.request.meta["dont_cache"] = False - mw.process_response(self.request, self.response, self.spider) - if mw.policy.should_cache_response(self.response, self.request): - self.assertIsInstance( - mw.storage.retrieve_response(self.spider, self.request), - self.response.__class__, - ) + assert response1.url == response2.url + assert response1.status == response2.status + assert response1.headers == response2.headers + assert response1.body == response2.body -class DefaultStorageTest(_BaseTest): +class StorageTestMixin: + """Mixin containing storage-specific test methods.""" + def test_storage(self): - with self._storage() as storage: + with self._storage() as (storage, crawler): request2 = self.request.copy() - assert storage.retrieve_response(self.spider, request2) is None + assert storage.retrieve_response(crawler.spider, request2) is None - storage.store_response(self.spider, self.request, self.response) - response2 = storage.retrieve_response(self.spider, request2) + storage.store_response(crawler.spider, self.request, self.response) + response2 = storage.retrieve_response(crawler.spider, request2) assert isinstance(response2, HtmlResponse) # content-type header self.assertEqualResponse(self.response, response2) time.sleep(2) # wait for cache to expire - assert storage.retrieve_response(self.spider, request2) is None + assert storage.retrieve_response(crawler.spider, request2) is None def test_storage_never_expire(self): - with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as storage: - assert storage.retrieve_response(self.spider, self.request) is None - storage.store_response(self.spider, self.request, self.response) + with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as (storage, crawler): + assert storage.retrieve_response(crawler.spider, self.request) is None + storage.store_response(crawler.spider, self.request, self.response) time.sleep(0.5) # give the chance to expire - assert storage.retrieve_response(self.spider, self.request) + assert storage.retrieve_response(crawler.spider, self.request) def test_storage_no_content_type_header(self): """Test that the response body is used to get the right response class even if there is no Content-Type header""" - with self._storage() as storage: - assert storage.retrieve_response(self.spider, self.request) is None + with self._storage() as (storage, crawler): + assert storage.retrieve_response(crawler.spider, self.request) is None response = Response( "http://www.example.com", body=b"\n.", status=202, ) - storage.store_response(self.spider, self.request, response) - cached_response = storage.retrieve_response(self.spider, self.request) - self.assertIsInstance(cached_response, HtmlResponse) + storage.store_response(crawler.spider, self.request, response) + cached_response = storage.retrieve_response(crawler.spider, self.request) + assert isinstance(cached_response, HtmlResponse) self.assertEqualResponse(response, cached_response) -class DbmStorageTest(DefaultStorageTest): - storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" +class PolicyTestMixin: + """Mixin containing policy-specific test methods.""" + + def test_dont_cache(self): + with self._middleware() as mw: + self.request.meta["dont_cache"] = True + mw.process_response(self.request, self.response) + assert mw.storage.retrieve_response(mw.crawler.spider, self.request) is None + + with self._middleware() as mw: + self.request.meta["dont_cache"] = False + mw.process_response(self.request, self.response) + if mw.policy.should_cache_response(self.response, self.request): + assert isinstance( + mw.storage.retrieve_response(mw.crawler.spider, self.request), + self.response.__class__, + ) -class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): - dbm_module = "tests.mocks.dummydbm" - - def _get_settings(self, **new_settings): - new_settings.setdefault("HTTPCACHE_DBM_MODULE", self.dbm_module) - return super()._get_settings(**new_settings) - - def test_custom_dbm_module_loaded(self): - # make sure our dbm module has been loaded - with self._storage() as storage: - self.assertEqual(storage.dbmodule.__name__, self.dbm_module) - - -class FilesystemStorageTest(DefaultStorageTest): - storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" - - -class FilesystemStorageGzipTest(FilesystemStorageTest): - def _get_settings(self, **new_settings): - new_settings.setdefault("HTTPCACHE_GZIP", True) - return super()._get_settings(**new_settings) - - -class DummyPolicyTest(_BaseTest): - policy_class = "scrapy.extensions.httpcache.DummyPolicy" +class DummyPolicyTestMixin(PolicyTestMixin): + """Mixin containing dummy policy specific test methods.""" def test_middleware(self): with self._middleware() as mw: - assert mw.process_request(self.request, self.spider) is None - mw.process_response(self.request, self.response, self.spider) - response = mw.process_request(self.request, self.spider) + assert mw.process_request(self.request) is None + mw.process_response(self.request, self.response) + response = mw.process_request(self.request) assert isinstance(response, HtmlResponse) self.assertEqualResponse(self.response, response) assert "cached" in response.flags @@ -187,20 +164,19 @@ class DummyPolicyTest(_BaseTest): with self._middleware() as mw: req = Request("http://host.com/path") res = Response("http://host2.net/test.html") - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) - cached = mw.process_request(req, self.spider) + assert mw.process_request(req) is None + mw.process_response(req, res) + cached = mw.process_request(req) assert isinstance(cached, Response) self.assertEqualResponse(res, cached) assert "cached" in cached.flags def test_middleware_ignore_missing(self): with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw: - self.assertRaises( - IgnoreRequest, mw.process_request, self.request, self.spider - ) - mw.process_response(self.request, self.response, self.spider) - response = mw.process_request(self.request, self.spider) + with pytest.raises(IgnoreRequest): + mw.process_request(self.request) + mw.process_response(self.request, self.response) + response = mw.process_request(self.request) assert isinstance(response, HtmlResponse) self.assertEqualResponse(self.response, response) assert "cached" in response.flags @@ -209,10 +185,10 @@ class DummyPolicyTest(_BaseTest): # http responses are cached by default req, res = Request("http://test.com/"), Response("http://test.com/") with self._middleware() as mw: - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) + assert mw.process_request(req) is None + mw.process_response(req, res) - cached = mw.process_request(req, self.spider) + cached = mw.process_request(req) assert isinstance(cached, Response), type(cached) self.assertEqualResponse(res, cached) assert "cached" in cached.flags @@ -220,19 +196,19 @@ class DummyPolicyTest(_BaseTest): # file response is not cached by default req, res = Request("file:///tmp/t.txt"), Response("file:///tmp/t.txt") with self._middleware() as mw: - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) + assert mw.process_request(req) is None + mw.process_response(req, res) - assert mw.storage.retrieve_response(self.spider, req) is None - assert mw.process_request(req, self.spider) is None + assert mw.storage.retrieve_response(mw.crawler.spider, req) is None + assert mw.process_request(req) is None # s3 scheme response is cached by default req, res = Request("s3://bucket/key"), Response("http://bucket/key") with self._middleware() as mw: - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) + assert mw.process_request(req) is None + mw.process_response(req, res) - cached = mw.process_request(req, self.spider) + cached = mw.process_request(req) assert isinstance(cached, Response), type(cached) self.assertEqualResponse(res, cached) assert "cached" in cached.flags @@ -240,41 +216,45 @@ class DummyPolicyTest(_BaseTest): # ignore s3 scheme req, res = Request("s3://bucket/key2"), Response("http://bucket/key2") with self._middleware(HTTPCACHE_IGNORE_SCHEMES=["s3"]) as mw: - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) + assert mw.process_request(req) is None + mw.process_response(req, res) - assert mw.storage.retrieve_response(self.spider, req) is None - assert mw.process_request(req, self.spider) is None + assert mw.storage.retrieve_response(mw.crawler.spider, req) is None + assert mw.process_request(req) is None def test_middleware_ignore_http_codes(self): # test response is not cached with self._middleware(HTTPCACHE_IGNORE_HTTP_CODES=[202]) as mw: - assert mw.process_request(self.request, self.spider) is None - mw.process_response(self.request, self.response, self.spider) + assert mw.process_request(self.request) is None + mw.process_response(self.request, self.response) - assert mw.storage.retrieve_response(self.spider, self.request) is None - assert mw.process_request(self.request, self.spider) is None + assert mw.storage.retrieve_response(mw.crawler.spider, self.request) is None + assert mw.process_request(self.request) is None # test response is cached with self._middleware(HTTPCACHE_IGNORE_HTTP_CODES=[203]) as mw: - mw.process_response(self.request, self.response, self.spider) - response = mw.process_request(self.request, self.spider) + mw.process_response(self.request, self.response) + response = mw.process_request(self.request) assert isinstance(response, HtmlResponse) self.assertEqualResponse(self.response, response) assert "cached" in response.flags -class RFC2616PolicyTest(DefaultStorageTest): - policy_class = "scrapy.extensions.httpcache.RFC2616Policy" +class RFC2616PolicyTestMixin(PolicyTestMixin): + """Mixin containing RFC2616 policy specific test methods.""" - def _process_requestresponse(self, mw, request, response): + @staticmethod + def _process_requestresponse( + mw: HttpCacheMiddleware, request: Request, response: Response | None + ) -> Response | Request: result = None try: - result = mw.process_request(request, self.spider) + result = mw.process_request(request) if result: assert isinstance(result, (Request, Response)) return result - result = mw.process_response(request, response, self.spider) + assert response is not None + result = mw.process_response(request, response) assert isinstance(result, Response) return result except Exception: @@ -294,11 +274,11 @@ class RFC2616PolicyTest(DefaultStorageTest): # response for a request with no-store must not be cached res1 = self._process_requestresponse(mw, req1, res0) self.assertEqualResponse(res1, res0) - assert mw.storage.retrieve_response(self.spider, req1) is None + assert mw.storage.retrieve_response(mw.crawler.spider, req1) is None # Re-do request without no-store and expect it to be cached res2 = self._process_requestresponse(mw, req0, res0) assert "cached" not in res2.flags - res3 = mw.process_request(req0, self.spider) + res3 = mw.process_request(req0) assert "cached" in res3.flags self.assertEqualResponse(res2, res3) # request with no-cache directive must not return cached response @@ -354,12 +334,13 @@ class RFC2616PolicyTest(DefaultStorageTest): ) self.assertEqualResponse(res1, res0) self.assertEqualResponse(res2, res0) - resc = mw.storage.retrieve_response(self.spider, req0) + resc = mw.storage.retrieve_response(mw.crawler.spider, req0) if shouldcache: self.assertEqualResponse(resc, res1) - assert "cached" in res2.flags and res2.status != 304 + assert "cached" in res2.flags + assert res2.status != 304 else: - self.assertFalse(resc) + assert not resc assert "cached" not in res2.flags # cache unconditionally unless response contains no-store or is a 304 @@ -377,12 +358,13 @@ class RFC2616PolicyTest(DefaultStorageTest): ) self.assertEqualResponse(res1, res0) self.assertEqualResponse(res2, res0) - resc = mw.storage.retrieve_response(self.spider, req0) + resc = mw.storage.retrieve_response(mw.crawler.spider, req0) if shouldcache: self.assertEqualResponse(resc, res1) - assert "cached" in res2.flags and res2.status != 304 + assert "cached" in res2.flags + assert res2.status != 304 else: - self.assertFalse(resc) + assert not resc assert "cached" not in res2.flags def test_cached_and_fresh(self): @@ -443,7 +425,7 @@ class RFC2616PolicyTest(DefaultStorageTest): # validate cached response if request max-age set as 0 req1 = req0.replace(headers={"Cache-Control": "max-age=0"}) res304 = res0.replace(status=304) - assert mw.process_request(req1, self.spider) is None + assert mw.process_request(req1) is None res3 = self._process_requestresponse(mw, req1, res304) self.assertEqualResponse(res1, res3) assert "cached" in res3.flags @@ -535,14 +517,14 @@ class RFC2616PolicyTest(DefaultStorageTest): self._process_requestresponse(mw, req0, res0) for e in mw.DOWNLOAD_EXCEPTIONS: # Simulate encountering an error on download attempts - assert mw.process_request(req0, self.spider) is None - res1 = mw.process_exception(req0, e("foo"), self.spider) + assert mw.process_request(req0) is None + res1 = mw.process_exception(req0, e("foo")) # Use cached response as recovery assert "cached" in res1.flags self.assertEqualResponse(res0, res1) # Do not use cached response for unhandled exceptions - mw.process_request(req0, self.spider) - assert mw.process_exception(req0, Exception("foo"), self.spider) is None + mw.process_request(req0) + assert mw.process_exception(req0, Exception("foo")) is None def test_ignore_response_cache_controls(self): sampledata = [ @@ -568,5 +550,49 @@ class RFC2616PolicyTest(DefaultStorageTest): assert "cached" in res2.flags -if __name__ == "__main__": - unittest.main() +# Concrete test classes that combine storage and policy mixins + + +class TestFilesystemStorageWithDummyPolicy( + TestBase, StorageTestMixin, DummyPolicyTestMixin +): + storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" + policy_class = "scrapy.extensions.httpcache.DummyPolicy" + + +class TestFilesystemStorageWithRFC2616Policy( + TestBase, StorageTestMixin, RFC2616PolicyTestMixin +): + storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" + policy_class = "scrapy.extensions.httpcache.RFC2616Policy" + + +class TestDbmStorageWithDummyPolicy(TestBase, StorageTestMixin, DummyPolicyTestMixin): + storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" + policy_class = "scrapy.extensions.httpcache.DummyPolicy" + + +class TestDbmStorageWithRFC2616Policy( + TestBase, StorageTestMixin, RFC2616PolicyTestMixin +): + storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" + policy_class = "scrapy.extensions.httpcache.RFC2616Policy" + + +class TestDbmStorageWithCustomDbmModule(TestDbmStorageWithDummyPolicy): + dbm_module = "tests.mocks.dummydbm" + + def _get_settings(self, **new_settings) -> dict[str, Any]: + new_settings.setdefault("HTTPCACHE_DBM_MODULE", self.dbm_module) + return super()._get_settings(**new_settings) + + def test_custom_dbm_module_loaded(self): + # make sure our dbm module has been loaded + with self._storage() as (storage, _): + assert storage.dbmodule.__name__ == self.dbm_module + + +class TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPolicy): + def _get_settings(self, **new_settings) -> dict[str, Any]: + new_settings.setdefault("HTTPCACHE_GZIP", True) + return super()._get_settings(**new_settings) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 003132488..c1acddd45 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,15 +1,17 @@ from gzip import GzipFile from io import BytesIO +from logging import WARNING from pathlib import Path -from unittest import SkipTest, TestCase +import pytest +from testfixtures import LogCapture from w3lib.encoding import resolve_encoding from scrapy.downloadermiddlewares.httpcompression import ( ACCEPTED_ENCODINGS, HttpCompressionMiddleware, ) -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider from scrapy.utils.gz import gunzip @@ -21,9 +23,11 @@ SAMPLEDIR = Path(tests_datadir, "compressed") FORMAT = { "gzip": ("html-gzip.bin", "gzip"), - "x-gzip": ("html-gzip.bin", "gzip"), + "x-gzip": ("html-gzip.bin", "x-gzip"), "rawdeflate": ("html-rawdeflate.bin", "deflate"), "zlibdeflate": ("html-zlibdeflate.bin", "deflate"), + "gzip-deflate": ("html-gzip-deflate.bin", "gzip, deflate"), + "gzip-deflate-gzip": ("html-gzip-deflate-gzip.bin", "gzip, deflate, gzip"), "br": ("html-br.bin", "br"), "br,gzip": ("html-br-gzip.bin", "br,gzip"), # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin @@ -35,19 +39,48 @@ FORMAT = { "html-zstd-streaming-no-content-size.bin", "zstd", ), + **{ + f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id) + for format_id in ( + "br", # 34 → 11 511 612 + "deflate", # 27 968 → 11 511 612 + "gzip", # 27 988 → 11 511 612 + "zstd", # 1 096 → 11 511 612 + ) + }, } -class HttpCompressionTest(TestCase): - def setUp(self): +def _skip_if_no_br() -> None: + try: + try: + import brotli # noqa: PLC0415 + + brotli.Decompressor.can_accept_more_data + except (ImportError, AttributeError): + import brotlicffi # noqa: PLC0415 + + brotlicffi.Decompressor.can_accept_more_data + except (ImportError, AttributeError): + pytest.skip("no brotli support") + + +def _skip_if_no_zstd() -> None: + try: + import zstandard # noqa: F401,PLC0415 + except ImportError: + pytest.skip("no zstd support (zstandard)") + + +class TestHttpCompression: + def setup_method(self): self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider("scrapytest.org") self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) - self.crawler.stats.open_spider(self.spider) + self.crawler.stats.open_spider() def _getresponse(self, coding): if coding not in FORMAT: - raise ValueError() + raise ValueError samplefile, contentencoding = FORMAT[coding] @@ -68,27 +101,24 @@ class HttpCompressionTest(TestCase): return response def assertStatsEqual(self, key, value): - self.assertEqual( - self.crawler.stats.get_value(key, spider=self.spider), - value, - str(self.crawler.stats.get_stats(self.spider)), + assert self.crawler.stats.get_value(key) == value, str( + self.crawler.stats.get_stats() ) def test_setting_false_compression_enabled(self): - self.assertRaises( - NotConfigured, - HttpCompressionMiddleware.from_crawler, - get_crawler(settings_dict={"COMPRESSION_ENABLED": False}), - ) + with pytest.raises(NotConfigured): + HttpCompressionMiddleware.from_crawler( + get_crawler(settings_dict={"COMPRESSION_ENABLED": False}) + ) def test_setting_default_compression_enabled(self): - self.assertIsInstance( + assert isinstance( HttpCompressionMiddleware.from_crawler(get_crawler()), HttpCompressionMiddleware, ) def test_setting_true_compression_enabled(self): - self.assertIsInstance( + assert isinstance( HttpCompressionMiddleware.from_crawler( get_crawler(settings_dict={"COMPRESSION_ENABLED": True}) ), @@ -98,44 +128,28 @@ class HttpCompressionTest(TestCase): def test_process_request(self): request = Request("http://scrapytest.org") assert "Accept-Encoding" not in request.headers - self.mw.process_request(request, self.spider) - self.assertEqual( - request.headers.get("Accept-Encoding"), b", ".join(ACCEPTED_ENCODINGS) - ) + self.mw.process_request(request) + assert request.headers.get("Accept-Encoding") == b", ".join(ACCEPTED_ENCODINGS) def test_process_response_gzip(self): response = self._getresponse("gzip") request = response.request - self.assertEqual(response.headers["Content-Encoding"], b"gzip") - newresponse = self.mw.process_response(request, response, self.spider) + assert response.headers["Content-Encoding"] == b"gzip" + newresponse = self.mw.process_response(request, response) assert newresponse is not response assert newresponse.body.startswith(b"= 1.2.0 to decode 'br'." + ), + ), + ) + assert newresponse is not response + assert newresponse.headers.getlist("Content-Encoding") == [b"br"] + + def test_process_response_zstd(self): + _skip_if_no_zstd() + raw_content = None for check_key in FORMAT: if not check_key.startswith("zstd-"): continue response = self._getresponse(check_key) request = response.request - self.assertEqual(response.headers["Content-Encoding"], b"zstd") - newresponse = self.mw.process_response(request, response, self.spider) + assert response.headers["Content-Encoding"] == b"zstd" + newresponse = self.mw.process_response(request, response) if raw_content is None: raw_content = newresponse.body else: @@ -194,12 +235,42 @@ class HttpCompressionTest(TestCase): assert newresponse.body.startswith(b" body size after " + "decompression (11511612 B) is larger than the download " + "warning size (10000000 B)." + ), + ), + ) + + def test_download_warnsize_setting_br(self): + _skip_if_no_br() + + self._test_download_warnsize_setting("br") + + def test_download_warnsize_setting_deflate(self): + self._test_download_warnsize_setting("deflate") + + def test_download_warnsize_setting_gzip(self): + self._test_download_warnsize_setting("gzip") + + def test_download_warnsize_setting_zstd(self): + _skip_if_no_zstd() + + self._test_download_warnsize_setting("zstd") + + def _test_download_warnsize_spider_attr(self, compression_id): + class DownloadWarnSizeSpider(Spider): + download_warnsize = 10_000_000 + + crawler = get_crawler(DownloadWarnSizeSpider) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + response = self._getresponse(f"bomb-{compression_id}") + + with LogCapture( + "scrapy.downloadermiddlewares.httpcompression", + propagate=False, + level=WARNING, + ) as log: + mw.process_response(response.request, response) + log.check( + ( + "scrapy.downloadermiddlewares.httpcompression", + "WARNING", + ( + "<200 http://scrapytest.org/> body size after " + "decompression (11511612 B) is larger than the download " + "warning size (10000000 B)." + ), + ), + ) + + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") + def test_download_warnsize_spider_attr_br(self): + _skip_if_no_br() + + self._test_download_warnsize_spider_attr("br") + + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") + def test_download_warnsize_spider_attr_deflate(self): + self._test_download_warnsize_spider_attr("deflate") + + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") + def test_download_warnsize_spider_attr_gzip(self): + self._test_download_warnsize_spider_attr("gzip") + + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") + def test_download_warnsize_spider_attr_zstd(self): + _skip_if_no_zstd() + + self._test_download_warnsize_spider_attr("zstd") + + def _test_download_warnsize_request_meta(self, compression_id): + crawler = get_crawler(Spider) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + response = self._getresponse(f"bomb-{compression_id}") + response.meta["download_warnsize"] = 10_000_000 + + with LogCapture( + "scrapy.downloadermiddlewares.httpcompression", + propagate=False, + level=WARNING, + ) as log: + mw.process_response(response.request, response) + log.check( + ( + "scrapy.downloadermiddlewares.httpcompression", + "WARNING", + ( + "<200 http://scrapytest.org/> body size after " + "decompression (11511612 B) is larger than the download " + "warning size (10000000 B)." + ), + ), + ) + + def test_download_warnsize_request_meta_br(self): + _skip_if_no_br() + + self._test_download_warnsize_request_meta("br") + + def test_download_warnsize_request_meta_deflate(self): + self._test_download_warnsize_request_meta("deflate") + + def test_download_warnsize_request_meta_gzip(self): + self._test_download_warnsize_request_meta("gzip") + + def test_download_warnsize_request_meta_zstd(self): + _skip_if_no_zstd() + + self._test_download_warnsize_request_meta("zstd") + + def _get_truncated_response(self, compression_id): + crawler = get_crawler(Spider) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + response = self._getresponse(compression_id) + truncated_body = response.body[: len(response.body) // 2] + response = response.replace(body=truncated_body) + return mw.process_response(response.request, response) + + def test_process_truncated_response_br(self): + _skip_if_no_br() + resp = self._get_truncated_response("br") + assert resp.body.startswith(b" req.priority + def setup_method(self): + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() + self.mw = self.mwcls.from_crawler(crawler) + + def get_response(self, request, location, status=302): + headers = {"Location": location} + return Response(request.url, status=status, headers=headers) def test_redirect_3xx_permanent(self): - def _test(method, status=301): + def _test(method, status: int): url = f"http://www.example.com/{status}" url2 = "http://www.example.com/redirected" req = Request(url, method=method) rsp = Response(url, headers={"Location": url2}, status=status) - req2 = self.mw.process_response(req, rsp, self.spider) + req2 = self.mw.process_response(req, rsp) assert isinstance(req2, Request) - self.assertEqual(req2.url, url2) - self.assertEqual(req2.method, method) + assert req2.url == url2 + assert req2.method == method # response without Location header but with status code is 3XX should be ignored del rsp.headers["Location"] - assert self.mw.process_response(req, rsp, self.spider) is rsp - - _test("GET") - _test("POST") - _test("HEAD") + assert self.mw.process_response(req, rsp) is rsp _test("GET", status=307) _test("POST", status=307) @@ -52,50 +59,211 @@ class RedirectMiddlewareTest(unittest.TestCase): _test("POST", status=308) _test("HEAD", status=308) - def test_dont_redirect(self): - url = "http://www.example.com/301" - url2 = "http://www.example.com/redirected" - req = Request(url, meta={"dont_redirect": True}) - rsp = Response(url, headers={"Location": url2}, status=301) - - r = self.mw.process_response(req, rsp, self.spider) - assert isinstance(r, Response) - assert r is rsp - - # Test that it redirects when dont_redirect is False - req = Request(url, meta={"dont_redirect": False}) - rsp = Response(url2, status=200) - - r = self.mw.process_response(req, rsp, self.spider) - assert isinstance(r, Response) - assert r is rsp - - def test_redirect_302(self): - url = "http://www.example.com/302" - url2 = "http://www.example.com/redirected2" - req = Request( - url, + @pytest.mark.parametrize("status", [301, 302, 303]) + def test_method_becomes_get(self, status): + source_url = f"http://www.example.com/{status}" + target_url = "http://www.example.com/redirected2" + request = Request( + source_url, method="POST", body="test", headers={"Content-Type": "text/plain", "Content-length": "4"}, ) + response = Response(source_url, headers={"Location": target_url}, status=status) + redirect_request = self.mw.process_response(request, response) + assert isinstance(redirect_request, Request) + assert redirect_request.url == target_url + assert redirect_request.method == "GET" + assert "Content-Type" not in redirect_request.headers + assert "Content-Length" not in redirect_request.headers + assert not redirect_request.body + + @pytest.mark.parametrize("status", [301, 302]) + @pytest.mark.parametrize("method", ["PUT", "DELETE"]) + def test_method_not_converted_on_301_302(self, status, method): + url = f"http://www.example.com/{status}" + url2 = "http://www.example.com/redirected" + body = b"test-body" + req = Request( + url, + method=method, + body=body, + headers={"Content-Type": "text/plain", "Content-Length": str(len(body))}, + ) + rsp = Response(url, headers={"Location": url2}, status=status) + + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == url2 + assert req2.method == method + assert req2.body == body + assert req2.headers[b"Content-Type"] == b"text/plain" + assert req2.headers[b"Content-Length"] == str(len(body)).encode() + + @pytest.mark.parametrize("method", ["PUT", "DELETE"]) + def test_method_converted_on_303(self, method): + status = 303 + url = f"http://www.example.com/{status}" + url2 = "http://www.example.com/redirected" + req = Request(url, method=method) + rsp = Response(url, headers={"Location": url2}, status=status) + + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == url2 + assert req2.method == "GET" + + def test_get_method_body_preserved_on_303(self): + status = 303 + url = f"http://www.example.com/{status}" + url2 = "http://www.example.com/redirected" + body = b"test-body" + req = Request( + url, + method="GET", + body=body, + headers={"Content-Type": "text/plain", "Content-Length": str(len(body))}, + ) + rsp = Response(url, headers={"Location": url2}, status=status) + + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == url2 + assert req2.method == "GET" + assert req2.body == body + assert req2.headers[b"Content-Type"] == b"text/plain" + assert req2.headers[b"Content-Length"] == str(len(body)).encode() + + def test_redirect_strips_content_headers(self): + url = "http://www.example.com/303" + url2 = "http://www.example.com/redirected" + headers = { + "Content-Type": "application/json", + "Content-Length": "100", + "Content-Encoding": "gzip", + "Content-Language": "en", + "Content-Location": "http://www.example.com/original", + "X-Custom": "foo", + } + req = Request(url, method="POST", headers=headers, body=b"foo") + rsp = Response(url, headers={"Location": url2}, status=303) + + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == url2 + assert req2.method == "GET" + assert req2.body == b"" + assert "Content-Type" not in req2.headers + assert "Content-Length" not in req2.headers + assert "Content-Encoding" not in req2.headers + assert "Content-Language" not in req2.headers + assert "Content-Location" not in req2.headers + assert req2.headers["X-Custom"] == b"foo" + + @pytest.mark.parametrize( + ( + "referer", + "source_url", + "target_url", + "policy_header", + "expected_referer", + ), + [ + ( + None, + "http://www.example.com/302", + "http://www.example.com/redirected", + None, + b"http://www.example.com/302", + ), + ( + "http://example.com/old", + "http://www.example.com/302", + "http://www.example.com/redirected", + None, + b"http://www.example.com/302", + ), + ( + "https://example.com/old", + "https://www.example.com/302", + "http://www.example.com/redirected", + None, + None, + ), + ( + "http://example.com/old", + "http://www.example.com/foo/bar", + "http://www.example.com/redirected", + "origin", + b"http://www.example.com/", + ), + ], + ) + def test_redirect_referer( + self, referer, source_url, target_url, policy_header, expected_referer + ): + headers = {"Referer": referer} if referer else {} + source_request = Request(source_url, headers=headers) + resp_headers = {"Location": target_url} + if policy_header: + resp_headers["Referrer-Policy"] = policy_header + response = Response(source_url, headers=resp_headers, status=302) + crawler = get_crawler() + referer_mw = build_from_crawler(RefererMiddleware, crawler) + redirect_mw = self.mwcls.from_crawler(crawler) + redirect_mw._referer_spider_middleware = referer_mw + redirect_request = redirect_mw.process_response(source_request, response) + if expected_referer: + assert redirect_request.headers.get("Referer") == expected_referer + else: + assert "Referer" not in redirect_request.headers + + def test_redirect_strips_referer_no_middleware(self): + source_url = "http://www.example.com/302" + redirect_url = "http://www.example.com/redirected" + source_request = Request( + source_url, headers={"Referer": "http://example.com/old"} + ) + response = Response(source_url, headers={"Location": redirect_url}, status=302) + redirect_mw = self.mwcls.from_crawler(get_crawler()) + redirect_mw._referer_spider_middleware = None + redirect_request = redirect_mw.process_response(source_request, response) + assert "Referer" not in redirect_request.headers + + @pytest.mark.parametrize("status", [307, 308]) + def test_cross_origin_maintain_body(self, status): + source_url = "https://example.com" + target_url = "https://attacker.example" + body = b"secret" + request = Request( + source_url, + method="POST", + body=body, + headers={ + "Content-Type": "application/json", + "Content-Length": str(len(body)), + }, + ) + response1 = Response( + source_url, headers={"Location": target_url}, status=status + ) + redirect_request = self.mw.process_response(request, response1) + assert isinstance(redirect_request, Request) + assert redirect_request.url == target_url + assert redirect_request.method == "POST" + assert redirect_request.body == body + assert redirect_request.headers[b"Content-Type"] == b"application/json" + assert redirect_request.headers[b"Content-Length"] == str(len(body)).encode() + + def test_redirect_keeps_fragment(self): + url = "http://www.example.com/301#frag" + url2 = "http://www.example.com/redirected" + req = Request(url) rsp = Response(url, headers={"Location": url2}, status=302) - req2 = self.mw.process_response(req, rsp, self.spider) + req2 = self.mw.process_response(req, rsp) assert isinstance(req2, Request) - self.assertEqual(req2.url, url2) - self.assertEqual(req2.method, "GET") - assert ( - "Content-Type" not in req2.headers - ), "Content-Type header must not be present in redirected request" - assert ( - "Content-Length" not in req2.headers - ), "Content-Length header must not be present in redirected request" - assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" - - # response without Location header but with status code is 3XX should be ignored - del rsp.headers["Location"] - assert self.mw.process_response(req, rsp, self.spider) is rsp + assert req2.url == "http://www.example.com/redirected#frag" def test_redirect_302_head(self): url = "http://www.example.com/302" @@ -103,14 +271,10 @@ class RedirectMiddlewareTest(unittest.TestCase): req = Request(url, method="HEAD") rsp = Response(url, headers={"Location": url2}, status=302) - req2 = self.mw.process_response(req, rsp, self.spider) + req2 = self.mw.process_response(req, rsp) assert isinstance(req2, Request) - self.assertEqual(req2.url, url2) - self.assertEqual(req2.method, "HEAD") - - # response without Location header but with status code is 3XX should be ignored - del rsp.headers["Location"] - assert self.mw.process_response(req, rsp, self.spider) is rsp + assert req2.url == url2 + assert req2.method == "HEAD" def test_redirect_302_relative(self): url = "http://www.example.com/302" @@ -119,95 +283,19 @@ class RedirectMiddlewareTest(unittest.TestCase): req = Request(url, method="HEAD") rsp = Response(url, headers={"Location": url2}, status=302) - req2 = self.mw.process_response(req, rsp, self.spider) + req2 = self.mw.process_response(req, rsp) assert isinstance(req2, Request) - self.assertEqual(req2.url, url3) - self.assertEqual(req2.method, "HEAD") - - # response without Location header but with status code is 3XX should be ignored - del rsp.headers["Location"] - assert self.mw.process_response(req, rsp, self.spider) is rsp - - def test_max_redirect_times(self): - self.mw.max_redirect_times = 1 - req = Request("http://scrapytest.org/302") - rsp = Response( - "http://scrapytest.org/302", headers={"Location": "/redirected"}, status=302 - ) - - req = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req, Request) - assert "redirect_times" in req.meta - self.assertEqual(req.meta["redirect_times"], 1) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) - - def test_ttl(self): - self.mw.max_redirect_times = 100 - req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1}) - rsp = Response( - "http://www.scrapytest.org/302", - headers={"Location": "/redirected"}, - status=302, - ) - - req = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req, Request) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) - - def test_redirect_urls(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = Response( - "http://scrapytest.org/first", - headers={"Location": "/redirected"}, - status=302, - ) - req2 = self.mw.process_response(req1, rsp1, self.spider) - rsp2 = Response( - "http://scrapytest.org/redirected", - headers={"Location": "/redirected2"}, - status=302, - ) - req3 = self.mw.process_response(req2, rsp2, self.spider) - - self.assertEqual(req2.url, "http://scrapytest.org/redirected") - self.assertEqual(req2.meta["redirect_urls"], ["http://scrapytest.org/first"]) - self.assertEqual(req3.url, "http://scrapytest.org/redirected2") - self.assertEqual( - req3.meta["redirect_urls"], - ["http://scrapytest.org/first", "http://scrapytest.org/redirected"], - ) - - def test_redirect_reasons(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = Response( - "http://scrapytest.org/first", - headers={"Location": "/redirected1"}, - status=301, - ) - req2 = self.mw.process_response(req1, rsp1, self.spider) - rsp2 = Response( - "http://scrapytest.org/redirected1", - headers={"Location": "/redirected2"}, - status=301, - ) - req3 = self.mw.process_response(req2, rsp2, self.spider) - - self.assertEqual(req2.meta["redirect_reasons"], [301]) - self.assertEqual(req3.meta["redirect_reasons"], [301, 301]) + assert req2.url == url3 + assert req2.method == "HEAD" def test_spider_handling(self): - smartspider = self.crawler._create_spider("smarty") - smartspider.handle_httpstatus_list = [404, 301, 302] + self.mw.crawler.spider.handle_httpstatus_list = [404, 301, 302] url = "http://www.example.com/301" url2 = "http://www.example.com/redirected" req = Request(url) rsp = Response(url, headers={"Location": url2}, status=301) - r = self.mw.process_response(req, rsp, smartspider) - self.assertIs(r, rsp) + r = self.mw.process_response(req, rsp) + assert r is rsp def test_request_meta_handling(self): url = "http://www.example.com/301" @@ -215,8 +303,8 @@ class RedirectMiddlewareTest(unittest.TestCase): def _test_passthrough(req): rsp = Response(url, headers={"Location": url2}, status=301, request=req) - r = self.mw.process_response(req, rsp, self.spider) - self.assertIs(r, rsp) + r = self.mw.process_response(req, rsp) + assert r is rsp _test_passthrough( Request(url, meta={"handle_httpstatus_list": [404, 301, 302]}) @@ -231,157 +319,142 @@ class RedirectMiddlewareTest(unittest.TestCase): headers={"Location": latin1_location}, status=302, ) - req_result = self.mw.process_response(req, resp, self.spider) + req_result = self.mw.process_response(req, resp) perc_encoded_utf8_url = "http://scrapytest.org/a%E7%E3o" - self.assertEqual(perc_encoded_utf8_url, req_result.url) + assert perc_encoded_utf8_url == req_result.url def test_utf8_location(self): req = Request("http://scrapytest.org/first") - utf8_location = "/ação".encode("utf-8") # header using UTF-8 encoding + utf8_location = "/ação".encode() # header using UTF-8 encoding resp = Response( "http://scrapytest.org/first", headers={"Location": utf8_location}, status=302, ) - req_result = self.mw.process_response(req, resp, self.spider) + req_result = self.mw.process_response(req, resp) perc_encoded_utf8_url = "http://scrapytest.org/a%C3%A7%C3%A3o" - self.assertEqual(perc_encoded_utf8_url, req_result.url) + assert perc_encoded_utf8_url == req_result.url + + def test_no_location(self): + request = Request("https://example.com") + response = Response(request.url, status=302) + assert self.mw.process_response(request, response) is response -class MetaRefreshMiddlewareTest(unittest.TestCase): - def setUp(self): - crawler = get_crawler(Spider) - self.spider = crawler._create_spider("foo") - self.mw = MetaRefreshMiddleware.from_crawler(crawler) - - def _body(self, interval=5, url="http://example.org/newpage"): - html = f"""""" - return html.encode("utf-8") - - def test_priority_adjust(self): - req = Request("http://a.com") - rsp = HtmlResponse(req.url, body=self._body()) - req2 = self.mw.process_response(req, rsp, self.spider) - assert req2.priority > req.priority - - def test_meta_refresh(self): - req = Request(url="http://example.org") - rsp = HtmlResponse(req.url, body=self._body()) - req2 = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req2, Request) - self.assertEqual(req2.url, "http://example.org/newpage") - - def test_meta_refresh_with_high_interval(self): - # meta-refresh with high intervals don't trigger redirects - req = Request(url="http://example.org") - rsp = HtmlResponse( - url="http://example.org", body=self._body(interval=1000), encoding="utf-8" - ) - rsp2 = self.mw.process_response(req, rsp, self.spider) - assert rsp is rsp2 - - def test_meta_refresh_trough_posted_request(self): - req = Request( - url="http://example.org", - method="POST", - body="test", - headers={"Content-Type": "text/plain", "Content-length": "4"}, - ) - rsp = HtmlResponse(req.url, body=self._body()) - req2 = self.mw.process_response(req, rsp, self.spider) - - assert isinstance(req2, Request) - self.assertEqual(req2.url, "http://example.org/newpage") - self.assertEqual(req2.method, "GET") - assert ( - "Content-Type" not in req2.headers - ), "Content-Type header must not be present in redirected request" - assert ( - "Content-Length" not in req2.headers - ), "Content-Length header must not be present in redirected request" - assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" - - def test_max_redirect_times(self): - self.mw.max_redirect_times = 1 - req = Request("http://scrapytest.org/max") - rsp = HtmlResponse(req.url, body=self._body()) - - req = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req, Request) - assert "redirect_times" in req.meta - self.assertEqual(req.meta["redirect_times"], 1) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) - - def test_ttl(self): - self.mw.max_redirect_times = 100 - req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1}) - rsp = HtmlResponse(req.url, body=self._body()) - - req = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req, Request) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) - - def test_redirect_urls(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = HtmlResponse(req1.url, body=self._body(url="/redirected")) - req2 = self.mw.process_response(req1, rsp1, self.spider) - assert isinstance(req2, Request), req2 - rsp2 = HtmlResponse(req2.url, body=self._body(url="/redirected2")) - req3 = self.mw.process_response(req2, rsp2, self.spider) - assert isinstance(req3, Request), req3 - self.assertEqual(req2.url, "http://scrapytest.org/redirected") - self.assertEqual(req2.meta["redirect_urls"], ["http://scrapytest.org/first"]) - self.assertEqual(req3.url, "http://scrapytest.org/redirected2") - self.assertEqual( - req3.meta["redirect_urls"], - ["http://scrapytest.org/first", "http://scrapytest.org/redirected"], - ) - - def test_redirect_reasons(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = HtmlResponse( - "http://scrapytest.org/first", body=self._body(url="/redirected") - ) - req2 = self.mw.process_response(req1, rsp1, self.spider) - rsp2 = HtmlResponse( - "http://scrapytest.org/redirected", body=self._body(url="/redirected1") - ) - req3 = self.mw.process_response(req2, rsp2, self.spider) - - self.assertEqual(req2.meta["redirect_reasons"], ["meta refresh"]) - self.assertEqual( - req3.meta["redirect_reasons"], ["meta refresh", "meta refresh"] - ) - - def test_ignore_tags_default(self): - req = Request(url="http://example.org") - body = ( - """""" - ) - rsp = HtmlResponse(req.url, body=body.encode()) - req2 = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req2, Request) - self.assertEqual(req2.url, "http://example.org/newpage") - - def test_ignore_tags_1_x_list(self): - """Test that Scrapy 1.x behavior remains possible""" - settings = {"METAREFRESH_IGNORE_TAGS": ["script", "noscript"]} - crawler = get_crawler(Spider, settings) - mw = MetaRefreshMiddleware.from_crawler(crawler) - req = Request(url="http://example.org") - body = ( - """""" - ) - rsp = HtmlResponse(req.url, body=body.encode()) - response = mw.process_response(req, rsp, self.spider) - assert isinstance(response, Response) +@pytest.mark.parametrize(SCHEME_PARAMS, REDIRECT_SCHEME_CASES) +def test_redirect_schemes(url, location, target): + crawler = get_crawler(Spider) + mw = RedirectMiddleware.from_crawler(crawler) + request = Request(url) + response = Response(url, headers={"Location": location}, status=301) + redirect = mw.process_response(request, response) + if target is None: + assert redirect == response + else: + assert isinstance(redirect, Request) + assert redirect.url == target -if __name__ == "__main__": - unittest.main() +@pytest.mark.parametrize( + ("policy", "source_url", "target_url", "expected_referrer"), + [ + # The policy header affects the outcome. + # (without it, the https → http switch would drop the referer) + ( + POLICY_UNSAFE_URL, + "https://a.example/1", + "http://a.example/2", + b"https://a.example/1", + ), + # The policy header can get the Referer header removed. + ( + POLICY_NO_REFERRER, + "http://a.example/1", + "http://a.example/2", + None, + ), + # The policy header can get the Referer header edited (path stripped). + ( + POLICY_ORIGIN, + "http://a.example/1", + "http://a.example/2", + b"http://a.example/", + ), + ], +) +def test_response_referrer_policy(policy, source_url, target_url, expected_referrer): + crawler = get_crawler() + referrer_mw = build_from_crawler(RefererMiddleware, crawler) + redirect_mw = build_from_crawler(RedirectMiddleware, crawler) + redirect_mw._referer_spider_middleware = referrer_mw + source_request = Request(source_url) + extra_headers = {} + if policy: + extra_headers["Referrer-Policy"] = policy + response_redirect = Response( + source_request.url, + status=301, + headers={"Location": target_url, **extra_headers}, + ) + source_request = redirect_mw.process_response(source_request, response_redirect) + assert isinstance(source_request, Request) + + assert source_request.headers.get("Referer") == expected_referrer + + +def test_no_warning_when_referer_middleware_present(caplog): + crawler = get_crawler() + crawler.get_spider_middleware = MagicMock(return_value=MagicMock()) + mw = build_from_crawler(RedirectMiddleware, crawler) + caplog.clear() + with caplog.at_level(logging.WARNING): + mw._engine_started() + assert not [ + record + for record in caplog.records + if record.name == "scrapy.downloadermiddlewares.redirect" + ] + + +def test_warning_redirect_middleware(caplog): + crawler = get_crawler() + crawler.get_spider_middleware = MagicMock(return_value=None) + mw = build_from_crawler(RedirectMiddleware, crawler) + with caplog.at_level(logging.WARNING): + mw._engine_started() + assert ( + "scrapy.downloadermiddlewares.redirect.RedirectMiddleware found no " + "scrapy.spidermiddlewares.referer.RefererMiddleware" + ) in caplog.text + assert ( + "enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)" + in caplog.text + ) + assert ( + "replace scrapy.downloadermiddlewares.redirect.RedirectMiddleware " + "with a subclass that overrides the handle_referer() method" + ) in caplog.text + + +def test_warning_subclass(caplog): + class MyRedirectMiddleware(RedirectMiddleware): + pass + + crawler = get_crawler() + crawler.get_spider_middleware = MagicMock(return_value=None) + mw = build_from_crawler(MyRedirectMiddleware, crawler) + with caplog.at_level(logging.WARNING): + mw._engine_started() + assert ( + "test_warning_subclass..MyRedirectMiddleware found no " + "scrapy.spidermiddlewares.referer.RefererMiddleware" + ) in caplog.text + assert ( + "enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)" + in caplog.text + ) + assert "edit " in caplog.text + assert "test_warning_subclass..MyRedirectMiddleware" in caplog.text + assert ( + "(if defined in your code base) to override the handle_referer() method" + ) in caplog.text diff --git a/tests/test_downloadermiddleware_redirect_base.py b/tests/test_downloadermiddleware_redirect_base.py new file mode 100644 index 000000000..44ade93b7 --- /dev/null +++ b/tests/test_downloadermiddleware_redirect_base.py @@ -0,0 +1,1023 @@ +from __future__ import annotations + +from itertools import product + +import pytest + +from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware +from scrapy.exceptions import IgnoreRequest +from scrapy.http import Request, Response +from scrapy.utils.misc import set_environ +from scrapy.utils.test import get_crawler + +SCHEME_PARAMS = ("url", "location", "target") +HTTP_SCHEMES = ("http", "https") +NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo") +REDIRECT_SCHEME_CASES = ( + # http/https → http/https redirects + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + f"{output_scheme}://example.com/b", + ) + for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2) + ), + # http/https → data/file/ftp/s3/foo does not redirect + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + None, + ) + for input_scheme in HTTP_SCHEMES + for output_scheme in NON_HTTP_SCHEMES + ), + # http/https → relative redirects + *( + ( + f"{scheme}://example.com/a", + location, + f"{scheme}://example.com/b", + ) + for scheme in HTTP_SCHEMES + for location in ("//example.com/b", "/b") + ), + # Note: We do not test data/file/ftp/s3 schemes for the initial URL + # because their download handlers cannot return a status code of 3xx. +) + + +class Base: + class Test: + def test_priority_adjust(self): + req = Request("http://a.example") + rsp = self.get_response(req, "http://a.example/redirected") + req2 = self.mw.process_response(req, rsp) + assert req2.priority > req.priority + + def test_dont_redirect(self): + url = "http://www.example.com/301" + url2 = "http://www.example.com/redirected" + req = Request(url, meta={"dont_redirect": True}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp) + assert isinstance(r, Response) + assert r is rsp + + # Test that it redirects when dont_redirect is False + req = Request(url, meta={"dont_redirect": False}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp) + assert isinstance(r, Request) + + def test_post(self): + url = "http://www.example.com/302" + url2 = "http://www.example.com/redirected2" + req = Request( + url, + method="POST", + body="test", + headers={"Content-Type": "text/plain", "Content-length": "4"}, + ) + rsp = self.get_response(req, url2) + + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == url2 + assert req2.method == "GET" + assert "Content-Type" not in req2.headers, ( + "Content-Type header must not be present in redirected request" + ) + assert "Content-Length" not in req2.headers, ( + "Content-Length header must not be present in redirected request" + ) + assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" + + def test_max_redirect_times(self): + self.mw.max_redirect_times = 1 + req = Request("http://a.example/302") + rsp = self.get_response(req, "/redirected") + + req = self.mw.process_response(req, rsp) + assert isinstance(req, Request) + assert "redirect_times" in req.meta + assert req.meta["redirect_times"] == 1 + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp) + + def test_ttl(self): + self.mw.max_redirect_times = 100 + req = Request("http://a.example/302", meta={"redirect_ttl": 1}) + rsp = self.get_response(req, "/a") + + req = self.mw.process_response(req, rsp) + assert isinstance(req, Request) + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp) + + def test_redirect_urls(self): + req1 = Request("http://a.example/first") + rsp1 = self.get_response(req1, "/redirected") + req2 = self.mw.process_response(req1, rsp1) + rsp2 = self.get_response(req1, "/redirected2") + req3 = self.mw.process_response(req2, rsp2) + + assert req2.url == "http://a.example/redirected" + assert req2.meta["redirect_urls"] == ["http://a.example/first"] + assert req3.url == "http://a.example/redirected2" + assert req3.meta["redirect_urls"] == [ + "http://a.example/first", + "http://a.example/redirected", + ] + + def test_redirect_reasons(self): + req1 = Request("http://a.example/first") + rsp1 = self.get_response(req1, "/redirected1") + req2 = self.mw.process_response(req1, rsp1) + rsp2 = self.get_response(req2, "/redirected2") + req3 = self.mw.process_response(req2, rsp2) + assert req2.meta["redirect_reasons"] == [self.reason] + assert req3.meta["redirect_reasons"] == [self.reason, self.reason] + + def test_cross_origin_header_dropping(self): + safe_headers = {"A": "B"} + cookie_header = {"Cookie": "a=b"} + authorization_header = {"Authorization": "Bearer 123456"} + + original_request = Request( + "https://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers. + internal_response = self.get_response( + original_request, "https://example.com/a" + ) + internal_redirect_request = self.mw.process_response( + original_request, internal_response + ) + assert isinstance(internal_redirect_request, Request) + assert original_request.headers == internal_redirect_request.headers + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers also when the scheme is http. + http_request = Request( + "http://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + http_response = self.get_response(http_request, "http://example.com/a") + http_redirect_request = self.mw.process_response( + http_request, http_response + ) + assert isinstance(http_redirect_request, Request) + assert http_request.headers == http_redirect_request.headers + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_explicit_port_response = self.get_response( + original_request, "https://example.com:443/a" + ) + to_explicit_port_redirect_request = self.mw.process_response( + original_request, to_explicit_port_response + ) + assert isinstance(to_explicit_port_redirect_request, Request) + assert original_request.headers == to_explicit_port_redirect_request.headers + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_implicit_port_response = self.get_response( + original_request, "https://example.com/a" + ) + to_implicit_port_redirect_request = self.mw.process_response( + original_request, to_implicit_port_response + ) + assert isinstance(to_implicit_port_redirect_request, Request) + assert original_request.headers == to_implicit_port_redirect_request.headers + + # A port change drops the Authorization header because the origin + # changes, but keeps the Cookie header because the domain remains the + # same. + different_port_response = self.get_response( + original_request, "https://example.com:8080/a" + ) + different_port_redirect_request = self.mw.process_response( + original_request, different_port_response + ) + assert isinstance(different_port_redirect_request, Request) + assert { + **safe_headers, + **cookie_header, + } == different_port_redirect_request.headers.to_unicode_dict() + + # A domain change drops both the Authorization and the Cookie header. + external_response = self.get_response( + original_request, "https://example.org/a" + ) + external_redirect_request = self.mw.process_response( + original_request, external_response + ) + assert isinstance(external_redirect_request, Request) + assert safe_headers == external_redirect_request.headers.to_unicode_dict() + + # A scheme upgrade (http → https) drops the Authorization header + # because the origin changes, but keeps the Cookie header because the + # domain remains the same. + upgrade_response = self.get_response(http_request, "https://example.com/a") + upgrade_redirect_request = self.mw.process_response( + http_request, upgrade_response + ) + assert isinstance(upgrade_redirect_request, Request) + assert { + **safe_headers, + **cookie_header, + } == upgrade_redirect_request.headers.to_unicode_dict() + + # A scheme downgrade (https → http) drops the Authorization header + # because the origin changes, and the Cookie header because its value + # cannot indicate whether the cookies were secure (HTTPS-only) or not. + # + # Note: If the Cookie header is set by the cookie management + # middleware, as recommended in the docs, the dropping of Cookie on + # scheme downgrade is not an issue, because the cookie management + # middleware will add again the Cookie header to the new request if + # appropriate. + downgrade_response = self.get_response( + original_request, "http://example.com/a" + ) + downgrade_redirect_request = self.mw.process_response( + original_request, downgrade_response + ) + assert isinstance(downgrade_redirect_request, Request) + assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict() + + def test_meta_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_http_to_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_to_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_proxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request2.meta["_auth_proxy"] == "https://b.example" + assert request2.meta["proxy"] == "https://b.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_proxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_unproxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request2.meta["_auth_proxy"] == "https://b.example" + assert request2.meta["proxy"] == "https://b.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_unproxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_proxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request1.meta["_auth_proxy"] == "https://b.example" + assert request1.meta["proxy"] == "https://b.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request3.meta["_auth_proxy"] == "https://b.example" + assert request3.meta["proxy"] == "https://b.example" + + def test_system_proxy_proxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request1.meta["_auth_proxy"] == "https://b.example" + assert request1.meta["proxy"] == "https://b.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request3.meta["_auth_proxy"] == "https://b.example" + assert request3.meta["proxy"] == "https://b.example" + + def test_system_proxy_unproxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_unproxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py new file mode 100644 index 000000000..416fbc2aa --- /dev/null +++ b/tests/test_downloadermiddleware_redirect_metarefresh.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import logging +from itertools import chain +from unittest.mock import MagicMock + +import pytest + +from scrapy.downloadermiddlewares.redirect import MetaRefreshMiddleware +from scrapy.http import HtmlResponse, Request, Response +from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler +from tests.test_downloadermiddleware_redirect_base import ( + HTTP_SCHEMES, + NON_HTTP_SCHEMES, + REDIRECT_SCHEME_CASES, + SCHEME_PARAMS, + Base, +) + + +def meta_refresh_body(url, interval=5): + html = f"""""" + return html.encode("utf-8") + + +class TestMetaRefreshMiddleware(Base.Test): + mwcls = MetaRefreshMiddleware + reason = "meta refresh" + + def setup_method(self): + crawler = get_crawler(Spider) + self.mw = self.mwcls.from_crawler(crawler) + + def _body(self, interval=5, url="http://example.org/newpage"): + return meta_refresh_body(url, interval) + + def get_response(self, request, location): + return HtmlResponse(request.url, body=self._body(url=location)) + + def test_meta_refresh(self): + req = Request(url="http://example.org") + rsp = HtmlResponse(req.url, body=self._body()) + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == "http://example.org/newpage" + + def test_meta_refresh_with_high_interval(self): + # meta-refresh with high intervals don't trigger redirects + req = Request(url="http://example.org") + rsp = HtmlResponse( + url="http://example.org", body=self._body(interval=1000), encoding="utf-8" + ) + rsp2 = self.mw.process_response(req, rsp) + assert rsp is rsp2 + + def test_meta_refresh_trough_posted_request(self): + req = Request( + url="http://example.org", + method="POST", + body="test", + headers={"Content-Type": "text/plain", "Content-length": "4"}, + ) + rsp = HtmlResponse(req.url, body=self._body()) + req2 = self.mw.process_response(req, rsp) + + assert isinstance(req2, Request) + assert req2.url == "http://example.org/newpage" + assert req2.method == "GET" + assert "Content-Type" not in req2.headers, ( + "Content-Type header must not be present in redirected request" + ) + assert "Content-Length" not in req2.headers, ( + "Content-Length header must not be present in redirected request" + ) + assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" + + def test_ignore_tags_default(self): + req = Request(url="http://example.org") + body = ( + """""" + ) + rsp = HtmlResponse(req.url, body=body.encode()) + response = self.mw.process_response(req, rsp) + assert isinstance(response, Response) + + def test_ignore_tags_1_x_list(self): + """Test that Scrapy 1.x behavior remains possible""" + settings = {"METAREFRESH_IGNORE_TAGS": ["script", "noscript"]} + crawler = get_crawler(Spider, settings) + mw = MetaRefreshMiddleware.from_crawler(crawler) + req = Request(url="http://example.org") + body = ( + """""" + ) + rsp = HtmlResponse(req.url, body=body.encode()) + response = mw.process_response(req, rsp) + assert isinstance(response, Response) + + +@pytest.mark.parametrize( + SCHEME_PARAMS, + [ + *REDIRECT_SCHEME_CASES, + # data/file/ftp/s3/foo → * does not redirect + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + None, + ) + for input_scheme in NON_HTTP_SCHEMES + for output_scheme in chain(HTTP_SCHEMES, NON_HTTP_SCHEMES) + ), + # data/file/ftp/s3/foo → relative does not redirect + *( + ( + f"{scheme}://example.com/a", + location, + None, + ) + for scheme in NON_HTTP_SCHEMES + for location in ("//example.com/b", "/b") + ), + ], +) +def test_meta_refresh_schemes(url, location, target): + crawler = get_crawler(Spider) + mw = MetaRefreshMiddleware.from_crawler(crawler) + request = Request(url) + response = HtmlResponse(url, body=meta_refresh_body(location)) + redirect = mw.process_response(request, response) + if target is None: + assert redirect == response + else: + assert isinstance(redirect, Request) + + +def test_warning_meta_refresh_middleware(caplog): + crawler = get_crawler() + crawler.get_spider_middleware = MagicMock(return_value=None) + mw = build_from_crawler(MetaRefreshMiddleware, crawler) + with caplog.at_level(logging.WARNING): + mw._engine_started() + assert ( + "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware found no " + "scrapy.spidermiddlewares.referer.RefererMiddleware" + ) in caplog.text + assert ( + "enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)" + in caplog.text + ) + assert ( + "replace scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware " + "with a subclass that overrides the handle_referer() method" + ) in caplog.text diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 661175840..56d21a4d2 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,37 +1,34 @@ import logging -import unittest -import warnings +import pytest from testfixtures import LogCapture -from twisted.internet import defer -from twisted.internet.error import ( - ConnectError, - ConnectionDone, - ConnectionLost, - DNSLookupError, - TCPTimedOutError, -) -from twisted.web.client import ResponseFailed +from twisted.internet.error import ConnectError, ConnectionDone, ConnectionLost from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request -from scrapy.exceptions import IgnoreRequest +from scrapy.exceptions import ( + CannotResolveHostError, + DownloadConnectionRefusedError, + DownloadTimeoutError, + IgnoreRequest, +) from scrapy.http import Request, Response from scrapy.settings.default_settings import RETRY_EXCEPTIONS from scrapy.spiders import Spider +from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -class RetryTest(unittest.TestCase): - def setUp(self): - self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider("foo") +class TestRetry: + def setup_method(self): + self.crawler = get_crawler(DefaultSpider) + self.crawler.spider = self.crawler._create_spider() self.mw = RetryMiddleware.from_crawler(self.crawler) self.mw.max_retry_times = 2 def test_priority_adjust(self): req = Request("http://www.scrapytest.org/503") rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) - req2 = self.mw.process_response(req, rsp, self.spider) + req2 = self.mw.process_response(req, rsp) assert req2.priority < req.priority def test_404(self): @@ -39,14 +36,14 @@ class RetryTest(unittest.TestCase): rsp = Response("http://www.scrapytest.org/404", body=b"", status=404) # dont retry 404s - assert self.mw.process_response(req, rsp, self.spider) is rsp + assert self.mw.process_response(req, rsp) is rsp def test_dont_retry(self): req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True}) rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) # first retry - r = self.mw.process_response(req, rsp, self.spider) + r = self.mw.process_response(req, rsp) assert r is rsp # Test retry when dont_retry set to False @@ -54,13 +51,13 @@ class RetryTest(unittest.TestCase): rsp = Response("http://www.scrapytest.org/503") # first retry - r = self.mw.process_response(req, rsp, self.spider) + r = self.mw.process_response(req, rsp) assert r is rsp def test_dont_retry_exc(self): req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True}) - r = self.mw.process_exception(req, DNSLookupError(), self.spider) + r = self.mw.process_exception(req, CannotResolveHostError()) assert r is None def test_503(self): @@ -68,17 +65,17 @@ class RetryTest(unittest.TestCase): rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) # first retry - req = self.mw.process_response(req, rsp, self.spider) + req = self.mw.process_response(req, rsp) assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 1) + assert req.meta["retry_times"] == 1 # second retry - req = self.mw.process_response(req, rsp, self.spider) + req = self.mw.process_response(req, rsp) assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 2) + assert req.meta["retry_times"] == 2 # discard it - assert self.mw.process_response(req, rsp, self.spider) is rsp + assert self.mw.process_response(req, rsp) is rsp assert self.crawler.stats.get_value("retry/max_reached") == 1 assert ( @@ -87,17 +84,47 @@ class RetryTest(unittest.TestCase): ) assert self.crawler.stats.get_value("retry/count") == 2 + def test_give_up_log_level_setting(self): + crawler = get_crawler( + DefaultSpider, settings_dict={"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"} + ) + crawler.spider = crawler._create_spider() + mw = RetryMiddleware.from_crawler(crawler) + mw.max_retry_times = 0 + req = Request("http://example.com/503") + rsp = Response("http://example.com/503", body=b"", status=503) + with LogCapture() as log: + assert mw.process_response(req, rsp) is rsp + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "WARNING", + f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable", + ) + ) + + def test_give_up_log_level_meta(self): + self.mw.max_retry_times = 0 + req = Request("http://example.com/503", meta={"give_up_log_level": "WARNING"}) + rsp = Response("http://example.com/503", body=b"", status=503) + with LogCapture() as log: + assert self.mw.process_response(req, rsp) is rsp + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "WARNING", + f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable", + ) + ) + def test_twistederrors(self): exceptions = [ ConnectError, ConnectionDone, ConnectionLost, - ConnectionRefusedError, - defer.TimeoutError, - DNSLookupError, - ResponseFailed, - TCPTimedOutError, - TimeoutError, + DownloadTimeoutError, + DownloadConnectionRefusedError, + CannotResolveHostError, ] for exc in exceptions: @@ -108,115 +135,81 @@ class RetryTest(unittest.TestCase): assert stats.get_value("retry/max_reached") == len(exceptions) assert stats.get_value("retry/count") == len(exceptions) * 2 assert ( - stats.get_value("retry/reason_count/twisted.internet.defer.TimeoutError") + stats.get_value("retry/reason_count/scrapy.exceptions.DownloadTimeoutError") == 2 ) def test_exception_to_retry_added(self): exc = ValueError settings_dict = { - "RETRY_EXCEPTIONS": list(RETRY_EXCEPTIONS) + [exc], + "RETRY_EXCEPTIONS": [*RETRY_EXCEPTIONS, exc], } - crawler = get_crawler(Spider, settings_dict=settings_dict) + crawler = get_crawler(DefaultSpider, settings_dict=settings_dict) + crawler.spider = crawler._create_spider() mw = RetryMiddleware.from_crawler(crawler) req = Request(f"http://www.scrapytest.org/{exc.__name__}") self._test_retry_exception(req, exc("foo"), mw) - def test_exception_to_retry_custom_middleware(self): - exc = ValueError - - with warnings.catch_warnings(record=True) as warns: - - class MyRetryMiddleware(RetryMiddleware): - EXCEPTIONS_TO_RETRY = RetryMiddleware.EXCEPTIONS_TO_RETRY + (exc,) - - self.assertEqual(len(warns), 1) - - mw2 = MyRetryMiddleware.from_crawler(self.crawler) - req = Request(f"http://www.scrapytest.org/{exc.__name__}") - req = mw2.process_exception(req, exc("foo"), self.spider) - assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 1) - - def test_exception_to_retry_custom_middleware_self(self): - class MyRetryMiddleware(RetryMiddleware): - def process_exception(self, request, exception, spider): - if isinstance(exception, self.EXCEPTIONS_TO_RETRY): - return self._retry(request, exception, spider) - - exc = OSError - mw2 = MyRetryMiddleware.from_crawler(self.crawler) - req = Request(f"http://www.scrapytest.org/{exc.__name__}") - with warnings.catch_warnings(record=True) as warns: - req = mw2.process_exception(req, exc("foo"), self.spider) - assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 1) - self.assertEqual(len(warns), 1) - def _test_retry_exception(self, req, exception, mw=None): if mw is None: mw = self.mw # first retry - req = mw.process_exception(req, exception, self.spider) + req = mw.process_exception(req, exception) assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 1) + assert req.meta["retry_times"] == 1 # second retry - req = mw.process_exception(req, exception, self.spider) + req = mw.process_exception(req, exception) assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 2) + assert req.meta["retry_times"] == 2 # discard it - req = mw.process_exception(req, exception, self.spider) - self.assertEqual(req, None) + req = mw.process_exception(req, exception) + assert req is None -class MaxRetryTimesTest(unittest.TestCase): +class TestMaxRetryTimes: invalid_url = "http://www.scrapytest.org/invalid_url" - def get_spider_and_middleware(self, settings=None): - crawler = get_crawler(Spider, settings or {}) - spider = crawler._create_spider("foo") - middleware = RetryMiddleware.from_crawler(crawler) - return spider, middleware + def get_middleware(self, settings=None): + crawler = get_crawler(DefaultSpider, settings or {}) + crawler.spider = crawler._create_spider() + return RetryMiddleware.from_crawler(crawler) def test_with_settings_zero(self): max_retry_times = 0 settings = {"RETRY_TIMES": max_retry_times} - spider, middleware = self.get_spider_and_middleware(settings) + middleware = self.get_middleware(settings) req = Request(self.invalid_url) self._test_retry( req, - DNSLookupError("foo"), + CannotResolveHostError("foo"), max_retry_times, - spider=spider, middleware=middleware, ) def test_with_metakey_zero(self): max_retry_times = 0 - spider, middleware = self.get_spider_and_middleware() + middleware = self.get_middleware() meta = {"max_retry_times": max_retry_times} req = Request(self.invalid_url, meta=meta) self._test_retry( req, - DNSLookupError("foo"), + CannotResolveHostError("foo"), max_retry_times, - spider=spider, middleware=middleware, ) def test_without_metakey(self): max_retry_times = 5 settings = {"RETRY_TIMES": max_retry_times} - spider, middleware = self.get_spider_and_middleware(settings) + middleware = self.get_middleware(settings) req = Request(self.invalid_url) self._test_retry( req, - DNSLookupError("foo"), + CannotResolveHostError("foo"), max_retry_times, - spider=spider, middleware=middleware, ) @@ -228,20 +221,18 @@ class MaxRetryTimesTest(unittest.TestCase): req2 = Request(self.invalid_url) settings = {"RETRY_TIMES": middleware_max_retry_times} - spider, middleware = self.get_spider_and_middleware(settings) + middleware = self.get_middleware(settings) self._test_retry( req1, - DNSLookupError("foo"), + CannotResolveHostError("foo"), meta_max_retry_times, - spider=spider, middleware=middleware, ) self._test_retry( req2, - DNSLookupError("foo"), + CannotResolveHostError("foo"), middleware_max_retry_times, - spider=spider, middleware=middleware, ) @@ -253,26 +244,24 @@ class MaxRetryTimesTest(unittest.TestCase): req2 = Request(self.invalid_url) settings = {"RETRY_TIMES": middleware_max_retry_times} - spider, middleware = self.get_spider_and_middleware(settings) + middleware = self.get_middleware(settings) self._test_retry( req1, - DNSLookupError("foo"), + CannotResolveHostError("foo"), meta_max_retry_times, - spider=spider, middleware=middleware, ) self._test_retry( req2, - DNSLookupError("foo"), + CannotResolveHostError("foo"), middleware_max_retry_times, - spider=spider, middleware=middleware, ) def test_with_dont_retry(self): max_retry_times = 4 - spider, middleware = self.get_spider_and_middleware() + middleware = self.get_middleware() meta = { "max_retry_times": max_retry_times, "dont_retry": True, @@ -280,9 +269,8 @@ class MaxRetryTimesTest(unittest.TestCase): req = Request(self.invalid_url, meta=meta) self._test_retry( req, - DNSLookupError("foo"), + CannotResolveHostError("foo"), 0, - spider=spider, middleware=middleware, ) @@ -291,22 +279,20 @@ class MaxRetryTimesTest(unittest.TestCase): req, exception, max_retry_times, - spider=None, middleware=None, ): - spider = spider or self.spider middleware = middleware or self.mw - for i in range(0, max_retry_times): - req = middleware.process_exception(req, exception, spider) + for _ in range(max_retry_times): + req = middleware.process_exception(req, exception) assert isinstance(req, Request) # discard it - req = middleware.process_exception(req, exception, spider) - self.assertEqual(req, None) + req = middleware.process_exception(req, exception) + assert req is None -class GetRetryRequestTest(unittest.TestCase): +class TestGetRetryRequest: def get_spider(self, settings=None): crawler = get_crawler(Spider, settings or {}) return crawler._create_spider("foo") @@ -319,15 +305,15 @@ class GetRetryRequestTest(unittest.TestCase): request, spider=spider, ) - self.assertIsInstance(new_request, Request) - self.assertNotEqual(new_request, request) - self.assertEqual(new_request.dont_filter, True) + assert isinstance(new_request, Request) + assert new_request != request + assert new_request.dont_filter expected_retry_times = 1 - self.assertEqual(new_request.meta["retry_times"], expected_retry_times) - self.assertEqual(new_request.priority, -1) + assert new_request.meta["retry_times"] == expected_retry_times + assert new_request.priority == -1 expected_reason = "unspecified" for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): - self.assertEqual(spider.crawler.stats.get_value(stat), 1) + assert spider.crawler.stats.get_value(stat) == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -347,8 +333,8 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, max_retry_times=max_retry_times, ) - self.assertEqual(new_request, None) - self.assertEqual(spider.crawler.stats.get_value("retry/max_reached"), 1) + assert new_request is None + assert spider.crawler.stats.get_value("retry/max_reached") == 1 failure_count = max_retry_times + 1 expected_reason = "unspecified" log.check_present( @@ -369,15 +355,15 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, max_retry_times=1, ) - self.assertIsInstance(new_request, Request) - self.assertNotEqual(new_request, request) - self.assertEqual(new_request.dont_filter, True) + assert isinstance(new_request, Request) + assert new_request != request + assert new_request.dont_filter expected_retry_times = 1 - self.assertEqual(new_request.meta["retry_times"], expected_retry_times) - self.assertEqual(new_request.priority, -1) + assert new_request.meta["retry_times"] == expected_retry_times + assert new_request.priority == -1 expected_reason = "unspecified" for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): - self.assertEqual(spider.crawler.stats.get_value(stat), 1) + assert spider.crawler.stats.get_value(stat) == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -399,16 +385,16 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, max_retry_times=max_retry_times, ) - self.assertIsInstance(new_request, Request) - self.assertNotEqual(new_request, request) - self.assertEqual(new_request.dont_filter, True) + assert isinstance(new_request, Request) + assert new_request != request + assert new_request.dont_filter expected_retry_times = index + 1 - self.assertEqual(new_request.meta["retry_times"], expected_retry_times) - self.assertEqual(new_request.priority, -expected_retry_times) + assert new_request.meta["retry_times"] == expected_retry_times + assert new_request.priority == -expected_retry_times expected_reason = "unspecified" for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): value = spider.crawler.stats.get_value(stat) - self.assertEqual(value, expected_retry_times) + assert value == expected_retry_times log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -424,8 +410,8 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, max_retry_times=max_retry_times, ) - self.assertEqual(new_request, None) - self.assertEqual(spider.crawler.stats.get_value("retry/max_reached"), 1) + assert new_request is None + assert spider.crawler.stats.get_value("retry/max_reached") == 1 failure_count = max_retry_times + 1 expected_reason = "unspecified" log.check_present( @@ -439,7 +425,7 @@ class GetRetryRequestTest(unittest.TestCase): def test_no_spider(self): request = Request("https://example.com") - with self.assertRaises(TypeError): + with pytest.raises(TypeError): get_retry_request(request) # pylint: disable=missing-kwoa def test_max_retry_times_setting(self): @@ -450,7 +436,7 @@ class GetRetryRequestTest(unittest.TestCase): request, spider=spider, ) - self.assertEqual(new_request, None) + assert new_request is None def test_max_retry_times_meta(self): max_retry_times = 0 @@ -461,7 +447,7 @@ class GetRetryRequestTest(unittest.TestCase): request, spider=spider, ) - self.assertEqual(new_request, None) + assert new_request is None def test_max_retry_times_argument(self): max_retry_times = 0 @@ -473,7 +459,7 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, max_retry_times=max_retry_times, ) - self.assertEqual(new_request, None) + assert new_request is None def test_priority_adjust_setting(self): priority_adjust = 1 @@ -483,7 +469,7 @@ class GetRetryRequestTest(unittest.TestCase): request, spider=spider, ) - self.assertEqual(new_request.priority, priority_adjust) + assert new_request.priority == priority_adjust def test_priority_adjust_argument(self): priority_adjust = 1 @@ -494,7 +480,7 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, priority_adjust=priority_adjust, ) - self.assertEqual(new_request.priority, priority_adjust) + assert new_request.priority == priority_adjust def test_log_extra_retry_success(self): request = Request("https://example.com") @@ -529,7 +515,7 @@ class GetRetryRequestTest(unittest.TestCase): ) expected_retry_times = 1 for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): - self.assertEqual(spider.crawler.stats.get_value(stat), 1) + assert spider.crawler.stats.get_value(stat) == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -554,7 +540,7 @@ class GetRetryRequestTest(unittest.TestCase): stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) - self.assertEqual(stat, 1) + assert stat == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -579,7 +565,7 @@ class GetRetryRequestTest(unittest.TestCase): stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) - self.assertEqual(stat, 1) + assert stat == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -604,7 +590,7 @@ class GetRetryRequestTest(unittest.TestCase): stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) - self.assertEqual(stat, 1) + assert stat == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -629,7 +615,7 @@ class GetRetryRequestTest(unittest.TestCase): stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) - self.assertEqual(stat, 1) + assert stat == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -659,6 +645,87 @@ class GetRetryRequestTest(unittest.TestCase): ) ) + def test_give_up_log_level_default(self): + request = Request("https://example.com") + spider = self.get_spider() + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + max_retry_times=0, + ) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "ERROR", + f"Gave up retrying {request} (failed 1 times): unspecified", + ) + ) + + def test_give_up_log_level_argument_name(self): + request = Request("https://example.com") + spider = self.get_spider() + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + max_retry_times=0, + give_up_log_level="WARNING", + ) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "WARNING", + f"Gave up retrying {request} (failed 1 times): unspecified", + ) + ) + + def test_give_up_log_level_argument_number(self): + request = Request("https://example.com") + spider = self.get_spider() + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + max_retry_times=0, + give_up_log_level=logging.WARNING, + ) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "WARNING", + f"Gave up retrying {request} (failed 1 times): unspecified", + ) + ) + + def test_give_up_log_level_setting(self): + request = Request("https://example.com") + spider = self.get_spider({"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"}) + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + max_retry_times=0, + ) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "WARNING", + f"Gave up retrying {request} (failed 1 times): unspecified", + ) + ) + + def test_give_up_log_level_invalid(self): + request = Request("https://example.com") + spider = self.get_spider() + with pytest.raises(ValueError, match="Invalid give-up log level"): + get_retry_request( + request, + spider=spider, + max_retry_times=0, + give_up_log_level="NOT_A_LEVEL", + ) + def test_custom_stats_key(self): request = Request("https://example.com") spider = self.get_spider() @@ -674,8 +741,4 @@ class GetRetryRequestTest(unittest.TestCase): f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}", ): - self.assertEqual(spider.crawler.stats.get_value(stat), 1) - - -if __name__ == "__main__": - unittest.main() + assert spider.crawler.stats.get_value(stat) == 1 diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 26898a6a1..082fc743e 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,34 +1,43 @@ +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING from unittest import mock -from twisted.internet import error, reactor -from twisted.internet.defer import Deferred, DeferredList, maybeDeferred +import pytest +from twisted.internet.defer import Deferred, DeferredList from twisted.python import failure -from twisted.trial import unittest from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware -from scrapy.downloadermiddlewares.robotstxt import logger as mw_module_logger -from scrapy.exceptions import IgnoreRequest, NotConfigured +from scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings -from tests.test_robotstxt_interface import reppy_available, rerp_available +from scrapy.utils.asyncio import call_later +from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future +from tests.test_robotstxt_interface import rerp_available +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from scrapy.crawler import Crawler -class RobotsTxtMiddlewareTest(unittest.TestCase): - def setUp(self): - self.crawler = mock.MagicMock() +class TestRobotsTxtMiddleware: + def setup_method(self) -> None: + self.crawler: mock.MagicMock = mock.MagicMock() self.crawler.settings = Settings() - self.crawler.engine.download = mock.MagicMock() + self.crawler.engine.download_async = mock.AsyncMock() - def tearDown(self): + def teardown_method(self): del self.crawler def test_robotstxt_settings(self): self.crawler.settings = Settings() self.crawler.settings.set("USER_AGENT", "CustomAgent") - self.assertRaises(NotConfigured, RobotsTxtMiddleware, self.crawler) + with pytest.raises(NotConfigured): + RobotsTxtMiddleware(self.crawler) - def _get_successful_crawler(self): + def _get_successful_crawler(self) -> Crawler: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) ROBOTS = """ @@ -40,171 +49,163 @@ Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: Disallow: /wiki/Käyttäjä: User-Agent: UnicödeBöt Disallow: /some/randome/page.html -""".encode( - "utf-8" - ) +""".encode() response = TextResponse("http://site.local/robots.txt", body=ROBOTS) - def return_response(request): + async def return_response(request): deferred = Deferred() - reactor.callFromThread(deferred.callback, response) - return deferred + call_later(0, deferred.callback, response) + return await maybe_deferred_to_future(deferred) - crawler.engine.download.side_effect = return_response + crawler.engine.download_async.side_effect = return_response return crawler - def test_robotstxt(self): + @coroutine_test + async def test_robotstxt(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) - return DeferredList( - [ - self.assertNotIgnored(Request("http://site.local/allowed"), middleware), - maybeDeferred(self.assertRobotsTxtRequested, "http://site.local"), - self.assertIgnored(Request("http://site.local/admin/main"), middleware), - self.assertIgnored(Request("http://site.local/static/"), middleware), - self.assertIgnored( - Request("http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:"), - middleware, - ), - self.assertIgnored( - Request("http://site.local/wiki/Käyttäjä:"), middleware - ), - ], - fireOnOneErrback=True, + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + self.assertRobotsTxtRequested("http://site.local") + await self.assertIgnored(Request("http://site.local/admin/main"), middleware) + await self.assertIgnored(Request("http://site.local/static/"), middleware) + await self.assertIgnored( + Request("http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:"), middleware + ) + await self.assertIgnored( + Request("http://site.local/wiki/Käyttäjä:"), middleware ) - def test_robotstxt_ready_parser(self): + @coroutine_test + async def test_robotstxt_multiple_reqs(self) -> None: middleware = RobotsTxtMiddleware(self._get_successful_crawler()) - d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware) - d.addCallback( - lambda _: self.assertNotIgnored( - Request("http://site.local/allowed"), middleware - ) + d1 = deferred_from_coro( + middleware.process_request(Request("http://site.local/allowed1")) ) - return d + d2 = deferred_from_coro( + middleware.process_request(Request("http://site.local/allowed2")) + ) + await maybe_deferred_to_future(DeferredList([d1, d2], fireOnOneErrback=True)) - def test_robotstxt_meta(self): + @pytest.mark.only_asyncio + @coroutine_test + async def test_robotstxt_multiple_reqs_asyncio(self) -> None: + middleware = RobotsTxtMiddleware(self._get_successful_crawler()) + c1 = middleware.process_request(Request("http://site.local/allowed1")) + c2 = middleware.process_request(Request("http://site.local/allowed2")) + await asyncio.gather(c1, c2) + + @coroutine_test + async def test_robotstxt_ready_parser(self): + middleware = RobotsTxtMiddleware(self._get_successful_crawler()) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + + @coroutine_test + async def test_robotstxt_meta(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) meta = {"dont_obey_robotstxt": True} - return DeferredList( - [ - self.assertNotIgnored( - Request("http://site.local/allowed", meta=meta), middleware - ), - self.assertNotIgnored( - Request("http://site.local/admin/main", meta=meta), middleware - ), - self.assertNotIgnored( - Request("http://site.local/static/", meta=meta), middleware - ), - ], - fireOnOneErrback=True, + await self.assertNotIgnored( + Request("http://site.local/allowed", meta=meta), middleware + ) + await self.assertNotIgnored( + Request("http://site.local/admin/main", meta=meta), middleware + ) + await self.assertNotIgnored( + Request("http://site.local/static/", meta=meta), middleware ) - def _get_garbage_crawler(self): + def _get_garbage_crawler(self) -> Crawler: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) response = Response( "http://site.local/robots.txt", body=b"GIF89a\xd3\x00\xfe\x00\xa2" ) - def return_response(request): + async def return_response(request): deferred = Deferred() - reactor.callFromThread(deferred.callback, response) - return deferred + call_later(0, deferred.callback, response) + return await maybe_deferred_to_future(deferred) - crawler.engine.download.side_effect = return_response + crawler.engine.download_async.side_effect = return_response return crawler - def test_robotstxt_garbage(self): + @coroutine_test + async def test_robotstxt_garbage(self): # garbage response should be discarded, equal 'allow all' middleware = RobotsTxtMiddleware(self._get_garbage_crawler()) - deferred = DeferredList( - [ - self.assertNotIgnored(Request("http://site.local"), middleware), - self.assertNotIgnored(Request("http://site.local/allowed"), middleware), - self.assertNotIgnored( - Request("http://site.local/admin/main"), middleware - ), - self.assertNotIgnored(Request("http://site.local/static/"), middleware), - ], - fireOnOneErrback=True, - ) - return deferred + await self.assertNotIgnored(Request("http://site.local"), middleware) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware) + await self.assertNotIgnored(Request("http://site.local/static/"), middleware) - def _get_emptybody_crawler(self): + def _get_emptybody_crawler(self) -> Crawler: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) response = Response("http://site.local/robots.txt") - def return_response(request): + async def return_response(request): deferred = Deferred() - reactor.callFromThread(deferred.callback, response) - return deferred + call_later(0, deferred.callback, response) + return await maybe_deferred_to_future(deferred) - crawler.engine.download.side_effect = return_response + crawler.engine.download_async.side_effect = return_response return crawler - def test_robotstxt_empty_response(self): + @coroutine_test + async def test_robotstxt_empty_response(self): # empty response should equal 'allow all' middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) - return DeferredList( - [ - self.assertNotIgnored(Request("http://site.local/allowed"), middleware), - self.assertNotIgnored( - Request("http://site.local/admin/main"), middleware - ), - self.assertNotIgnored(Request("http://site.local/static/"), middleware), - ], - fireOnOneErrback=True, - ) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware) + await self.assertNotIgnored(Request("http://site.local/static/"), middleware) - def test_robotstxt_error(self): + @coroutine_test + async def test_robotstxt_error(self, caplog: pytest.LogCaptureFixture) -> None: self.crawler.settings.set("ROBOTSTXT_OBEY", True) - err = error.DNSLookupError("Robotstxt address not found") + err = CannotResolveHostError("Robotstxt address not found") - def return_failure(request): + async def return_failure(request): deferred = Deferred() - reactor.callFromThread(deferred.errback, failure.Failure(err)) - return deferred + call_later(0, deferred.errback, failure.Failure(err)) + return await maybe_deferred_to_future(deferred) - self.crawler.engine.download.side_effect = return_failure + self.crawler.engine.download_async.side_effect = return_failure middleware = RobotsTxtMiddleware(self.crawler) - middleware._logerror = mock.MagicMock(side_effect=middleware._logerror) - deferred = middleware.process_request(Request("http://site.local"), None) - deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called)) - return deferred + await middleware.process_request(Request("http://site.local")) + assert "Robotstxt address not found" in caplog.text - def test_robotstxt_immediate_error(self): + @coroutine_test + async def test_robotstxt_immediate_error(self): self.crawler.settings.set("ROBOTSTXT_OBEY", True) - err = error.DNSLookupError("Robotstxt address not found") + err = CannotResolveHostError("Robotstxt address not found") - def immediate_failure(request): - deferred = Deferred() - deferred.errback(failure.Failure(err)) - return deferred + async def immediate_failure(request): + raise err - self.crawler.engine.download.side_effect = immediate_failure + self.crawler.engine.download_async.side_effect = immediate_failure middleware = RobotsTxtMiddleware(self.crawler) - return self.assertNotIgnored(Request("http://site.local"), middleware) + await self.assertNotIgnored(Request("http://site.local"), middleware) - def test_ignore_robotstxt_request(self): + @coroutine_test + async def test_ignore_robotstxt_request(self): self.crawler.settings.set("ROBOTSTXT_OBEY", True) - def ignore_request(request): + async def ignore_request(request): deferred = Deferred() - reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest())) - return deferred + call_later(0, deferred.errback, failure.Failure(IgnoreRequest())) + return await maybe_deferred_to_future(deferred) - self.crawler.engine.download.side_effect = ignore_request + self.crawler.engine.download_async.side_effect = ignore_request middleware = RobotsTxtMiddleware(self.crawler) - mw_module_logger.error = mock.MagicMock() - - d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware) - d.addCallback(lambda _: self.assertFalse(mw_module_logger.error.called)) - return d + with mock.patch( + "scrapy.downloadermiddlewares.robotstxt.logger" + ) as mw_module_logger: + await self.assertNotIgnored( + Request("http://site.local/allowed"), middleware + ) + assert not mw_module_logger.error.called def test_robotstxt_user_agent_setting(self): crawler = self._get_successful_crawler() @@ -212,58 +213,50 @@ Disallow: /some/randome/page.html crawler.settings.set("USER_AGENT", "Mozilla/5.0 (X11; Linux x86_64)") middleware = RobotsTxtMiddleware(crawler) rp = mock.MagicMock(return_value=True) - middleware.process_request_2(rp, Request("http://site.local/allowed"), None) + middleware.process_request_2(rp, Request("http://site.local/allowed")) rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot") - def test_robotstxt_local_file(self): + @coroutine_test + async def test_robotstxt_local_file(self): middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) - assert not middleware.process_request( - Request("data:text/plain,Hello World data"), None - ) - assert not middleware.process_request( - Request("file:///tests/sample_data/test_site/nothinghere.html"), None - ) - assert isinstance( - middleware.process_request(Request("http://site.local/allowed"), None), - Deferred, - ) + middleware.process_request_2 = mock.MagicMock() - def assertNotIgnored(self, request, middleware): - spider = None # not actually used - dfd = maybeDeferred(middleware.process_request, request, spider) - dfd.addCallback(self.assertIsNone) - return dfd + await middleware.process_request(Request("data:text/plain,Hello World data")) + assert not middleware.process_request_2.called - def assertIgnored(self, request, middleware): - spider = None # not actually used - return self.assertFailure( - maybeDeferred(middleware.process_request, request, spider), IgnoreRequest + await middleware.process_request( + Request("file:///tests/sample_data/test_site/nothinghere.html") ) + assert not middleware.process_request_2.called - def assertRobotsTxtRequested(self, base_url): - calls = self.crawler.engine.download.call_args_list + await middleware.process_request(Request("http://site.local/allowed")) + assert middleware.process_request_2.called + + async def assertNotIgnored( + self, request: Request, middleware: RobotsTxtMiddleware + ) -> None: + try: + await middleware.process_request(request) + except IgnoreRequest: + pytest.fail("IgnoreRequest was raised unexpectedly") + + async def assertIgnored( + self, request: Request, middleware: RobotsTxtMiddleware + ) -> None: + with pytest.raises(IgnoreRequest): + await middleware.process_request(request) + + def assertRobotsTxtRequested(self, base_url: str) -> None: + calls = self.crawler.engine.download_async.call_args_list request = calls[0][0][0] - self.assertEqual(request.url, f"{base_url}/robots.txt") - self.assertEqual(request.callback, NO_CALLBACK) + assert request.url == f"{base_url}/robots.txt" + assert request.callback == NO_CALLBACK -class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): - if not rerp_available(): - skip = "Rerp parser is not installed" - - def setUp(self): - super().setUp() +@pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed") +class TestRobotsTxtMiddlewareWithRerp(TestRobotsTxtMiddleware): + def setup_method(self): + super().setup_method() self.crawler.settings.set( "ROBOTSTXT_PARSER", "scrapy.robotstxt.RerpRobotParser" ) - - -class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest): - if not reppy_available(): - skip = "Reppy parser is not installed" - - def setUp(self): - super().setUp() - self.crawler.settings.set( - "ROBOTSTXT_PARSER", "scrapy.robotstxt.ReppyRobotParser" - ) diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 5b7181848..67af4264c 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,5 +1,3 @@ -from unittest import TestCase - from scrapy.downloadermiddlewares.stats import DownloaderStats from scrapy.http import Request, Response from scrapy.spiders import Spider @@ -10,39 +8,36 @@ class MyException(Exception): pass -class TestDownloaderStats(TestCase): - def setUp(self): +class TestDownloaderStats: + def setup_method(self): self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider("scrapytest.org") self.mw = DownloaderStats(self.crawler.stats) - self.crawler.stats.open_spider(self.spider) + self.crawler.stats.open_spider() self.req = Request("http://scrapytest.org") self.res = Response("scrapytest.org", status=400) def assertStatsEqual(self, key, value): - self.assertEqual( - self.crawler.stats.get_value(key, spider=self.spider), - value, - str(self.crawler.stats.get_stats(self.spider)), + assert self.crawler.stats.get_value(key) == value, str( + self.crawler.stats.get_stats() ) def test_process_request(self): - self.mw.process_request(self.req, self.spider) + self.mw.process_request(self.req) self.assertStatsEqual("downloader/request_count", 1) def test_process_response(self): - self.mw.process_response(self.req, self.res, self.spider) + self.mw.process_response(self.req, self.res) self.assertStatsEqual("downloader/response_count", 1) def test_process_exception(self): - self.mw.process_exception(self.req, MyException(), self.spider) + self.mw.process_exception(self.req, MyException()) self.assertStatsEqual("downloader/exception_count", 1) self.assertStatsEqual( "downloader/exception_type_count/tests.test_downloadermiddleware_stats.MyException", 1, ) - def tearDown(self): - self.crawler.stats.close_spider(self.spider, "") + def teardown_method(self): + self.crawler.stats.close_spider() diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index cad3dea5c..539183cd6 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -1,54 +1,33 @@ -from unittest import TestCase - from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class UserAgentMiddlewareTest(TestCase): +class TestUserAgentMiddleware: def get_spider_and_mw(self, default_useragent): crawler = get_crawler(Spider, {"USER_AGENT": default_useragent}) spider = crawler._create_spider("foo") return spider, UserAgentMiddleware.from_crawler(crawler) def test_default_agent(self): - spider, mw = self.get_spider_and_mw("default_useragent") + _, mw = self.get_spider_and_mw("default_useragent") req = Request("http://scrapytest.org/") - assert mw.process_request(req, spider) is None - self.assertEqual(req.headers["User-Agent"], b"default_useragent") - - def test_remove_agent(self): - # settings USER_AGENT to None should remove the user agent - spider, mw = self.get_spider_and_mw("default_useragent") - spider.user_agent = None - mw.spider_opened(spider) - req = Request("http://scrapytest.org/") - assert mw.process_request(req, spider) is None - assert req.headers.get("User-Agent") is None - - def test_spider_agent(self): - spider, mw = self.get_spider_and_mw("default_useragent") - spider.user_agent = "spider_useragent" - mw.spider_opened(spider) - req = Request("http://scrapytest.org/") - assert mw.process_request(req, spider) is None - self.assertEqual(req.headers["User-Agent"], b"spider_useragent") + assert mw.process_request(req) is None + assert req.headers["User-Agent"] == b"default_useragent" def test_header_agent(self): spider, mw = self.get_spider_and_mw("default_useragent") - spider.user_agent = "spider_useragent" mw.spider_opened(spider) req = Request( "http://scrapytest.org/", headers={"User-Agent": "header_useragent"} ) - assert mw.process_request(req, spider) is None - self.assertEqual(req.headers["User-Agent"], b"header_useragent") + assert mw.process_request(req) is None + assert req.headers["User-Agent"] == b"header_useragent" def test_no_agent(self): spider, mw = self.get_spider_and_mw(None) - spider.user_agent = None mw.spider_opened(spider) req = Request("http://scrapytest.org/") - assert mw.process_request(req, spider) is None + assert mw.process_request(req) is None assert "User-Agent" not in req.headers diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 9d4072d19..a717b18a6 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -1,12 +1,17 @@ import time +from typing import Any -from twisted.internet import defer -from twisted.trial.unittest import TestCase +import pytest +from scrapy import Request +from scrapy.core.downloader import Downloader, Slot from scrapy.crawler import CrawlerRunner -from scrapy.http import Request -from tests.mockserver import MockServer +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.test import get_crawler +from tests.mockserver.http import MockServer from tests.spiders import MetaSpider +from tests.utils.decorators import coroutine_test, inline_callbacks_test class DownloaderSlotsSettingsTestSpider(MetaSpider): @@ -20,44 +25,52 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): "concurrency": 1, "delay": 2, "randomize_delay": False, + "throttle": False, }, "books.toscrape.com": {"delay": 3, "randomize_delay": False}, }, } - def start_requests(self): - self.times = {None: []} - - slots = list(self.custom_settings.get("DOWNLOAD_SLOTS", {}).keys()) + [None] + def __init__(self, *args: Any, **kwargs: Any): + super().__init__(*args, **kwargs) + assert self.mockserver + self.default_slot = self.mockserver.host + self.times: dict[str, list[float]] = {} + async def start(self): + slots = [*self.custom_settings.get("DOWNLOAD_SLOTS", {}), None] for slot in slots: url = self.mockserver.url(f"/?downloader_slot={slot}") - self.times[slot] = [] + self.times[slot or self.default_slot] = [] yield Request(url, callback=self.parse, meta={"download_slot": slot}) def parse(self, response): - slot = response.meta.get("download_slot", None) + slot = response.meta.get("download_slot", self.default_slot) self.times[slot].append(time.time()) url = self.mockserver.url(f"/?downloader_slot={slot}&req=2") yield Request(url, callback=self.not_parse, meta={"download_slot": slot}) def not_parse(self, response): - slot = response.meta.get("download_slot", None) + slot = response.meta.get("download_slot", self.default_slot) self.times[slot].append(time.time()) -class CrawlTestCase(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() +class TestCrawl: + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) + + def setup_method(self): self.runner = CrawlerRunner() - def tearDown(self): - self.mockserver.__exit__(None, None, None) - - @defer.inlineCallbacks + @inline_callbacks_test def test_delay(self): - crawler = CrawlerRunner().create_crawler(DownloaderSlotsSettingsTestSpider) + crawler = get_crawler(DownloaderSlotsSettingsTestSpider) yield crawler.crawl(mockserver=self.mockserver) slots = crawler.engine.downloader.slots times = crawler.spider.times @@ -69,4 +82,78 @@ class CrawlTestCase(TestCase): for k, v in slots.items() } - self.assertTrue(max(list(error_delta.values())) < tolerance) + assert max(list(error_delta.values())) < tolerance + + +@coroutine_test +async def test_params(): + params = { + "concurrency": 1, + "delay": 2, + "randomize_delay": False, + } + settings = { + "DOWNLOAD_SLOTS": { + "example.com": params, + }, + } + crawler = get_crawler(DefaultSpider, settings_dict=settings) + crawler.spider = crawler._create_spider() + downloader = Downloader(crawler) + request = Request("https://example.com") + _, actual = downloader._get_slot(request) + downloader.close() + expected = Slot(**params) + for param in params: + assert getattr(expected, param) == getattr(actual, param), ( + f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}" + ) + + +@coroutine_test +async def test_get_slot_deprecated_spider_arg(): + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() + downloader = Downloader(crawler) + request = Request("https://example.com") + + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a 'spider' argument to Downloader\._get_slot\(\) is deprecated", + ): + key1, slot1 = downloader._get_slot(request, spider=crawler.spider) + key2, slot2 = downloader._get_slot(request) + downloader.close() + + assert key1 == key2 + assert slot1 == slot2 + + +@pytest.mark.parametrize( + "priority_queue_class", + [ + "scrapy.pqueues.ScrapyPriorityQueue", + "scrapy.pqueues.DownloaderAwarePriorityQueue", + ], +) +@coroutine_test +async def test_none_slot_with_priority_queue( + mockserver: MockServer, priority_queue_class: str +) -> None: + """Test specific cases for None slot handling with different priority queues.""" + crawler = get_crawler( + DownloaderSlotsSettingsTestSpider, + settings_dict={"SCHEDULER_PRIORITY_QUEUE": priority_queue_class}, + ) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, DownloaderSlotsSettingsTestSpider) + + assert hasattr(crawler.spider, "times") + assert None not in crawler.spider.times + assert crawler.spider.default_slot in crawler.spider.times + assert len(crawler.spider.times[crawler.spider.default_slot]) == 2 + + assert crawler.stats + stats = crawler.stats + assert stats.get_value("spider_exceptions", 0) == 0 + assert stats.get_value("downloader/exception_count", 0) == 0 diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index aa0975555..b38bf9570 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -2,25 +2,26 @@ import hashlib import shutil import sys import tempfile -import unittest from pathlib import Path +from warnings import catch_warnings from testfixtures import LogCapture from scrapy.core.scheduler import Scheduler -from scrapy.dupefilters import RFPDupeFilter +from scrapy.dupefilters import BaseDupeFilter, RFPDupeFilter +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider -def _get_dupefilter(*, crawler=None, settings=None, open=True): +def _get_dupefilter(*, crawler=None, settings=None, open_=True): if crawler is None: crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) dupefilter = scheduler.df - if open: + if open_: dupefilter.open() return dupefilter @@ -33,49 +34,28 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter): return df -class FromSettingsRFPDupeFilter(RFPDupeFilter): - @classmethod - def from_settings(cls, settings, *, fingerprinter=None): - df = super().from_settings(settings, fingerprinter=fingerprinter) - df.method = "from_settings" - return df - - class DirectDupeFilter: method = "n/a" -class RFPDupeFilterTest(unittest.TestCase): +class TestRFPDupeFilter: def test_df_from_crawler_scheduler(self): settings = { "DUPEFILTER_DEBUG": True, "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) - self.assertTrue(scheduler.df.debug) - self.assertEqual(scheduler.df.method, "from_crawler") - - def test_df_from_settings_scheduler(self): - settings = { - "DUPEFILTER_DEBUG": True, - "DUPEFILTER_CLASS": FromSettingsRFPDupeFilter, - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", - } - crawler = get_crawler(settings_dict=settings) - scheduler = Scheduler.from_crawler(crawler) - self.assertTrue(scheduler.df.debug) - self.assertEqual(scheduler.df.method, "from_settings") + assert scheduler.df.debug + assert scheduler.df.method == "from_crawler" def test_df_direct_scheduler(self): settings = { "DUPEFILTER_CLASS": DirectDupeFilter, - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) - self.assertEqual(scheduler.df.method, "n/a") + assert scheduler.df.method == "n/a" def test_filter(self): dupefilter = _get_dupefilter() @@ -97,7 +77,7 @@ class RFPDupeFilterTest(unittest.TestCase): path = tempfile.mkdtemp() try: - df = _get_dupefilter(settings={"JOBDIR": path}, open=False) + df = _get_dupefilter(settings={"JOBDIR": path}, open_=False) try: df.open() assert not df.request_seen(r1) @@ -105,7 +85,7 @@ class RFPDupeFilterTest(unittest.TestCase): finally: df.close("finished") - df2 = _get_dupefilter(settings={"JOBDIR": path}, open=False) + df2 = _get_dupefilter(settings={"JOBDIR": path}, open_=False) assert df != df2 try: df2.open() @@ -146,7 +126,7 @@ class RFPDupeFilterTest(unittest.TestCase): case_insensitive_dupefilter.close("finished") def test_seenreq_newlines(self): - """Checks against adding duplicate \r to + r"""Checks against adding duplicate \r to line endings on Windows platforms.""" r1 = Request("http://scrapytest.org/1") @@ -176,7 +156,6 @@ class RFPDupeFilterTest(unittest.TestCase): settings = { "DUPEFILTER_DEBUG": False, "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) @@ -205,7 +184,6 @@ class RFPDupeFilterTest(unittest.TestCase): settings = { "DUPEFILTER_DEBUG": True, "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) @@ -243,7 +221,6 @@ class RFPDupeFilterTest(unittest.TestCase): with LogCapture() as log: settings = { "DUPEFILTER_DEBUG": True, - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) @@ -276,3 +253,18 @@ class RFPDupeFilterTest(unittest.TestCase): ) dupefilter.close("finished") + + +class TestBaseDupeFilter: + def test_log_deprecation(self): + dupefilter = _get_dupefilter( + settings={"DUPEFILTER_CLASS": BaseDupeFilter}, + ) + with catch_warnings(record=True) as warning_list: + dupefilter.log(None, None) + assert len(warning_list) == 1 + assert ( + str(warning_list[0].message) + == "Calling BaseDupeFilter.log() is deprecated." + ) + assert warning_list[0].category == ScrapyDeprecationWarning diff --git a/tests/test_engine.py b/tests/test_engine.py index 8d7afb6a1..2cd583721 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -1,44 +1,52 @@ -""" -Scrapy engine tests - -This starts a testing web server (using twisted.server.Site) and then crawls it -with the Scrapy crawler. - -To view the testing web server in a browser you can start it by running this -module with the ``runserver`` argument:: - - python test_engine.py runserver -""" +from __future__ import annotations +import asyncio import re import subprocess import sys from collections import defaultdict from dataclasses import dataclass -from pathlib import Path -from threading import Timer +from logging import DEBUG +from typing import TYPE_CHECKING, Any, cast +from unittest.mock import Mock, call from urllib.parse import urlparse import attr +import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher -from twisted.internet import defer, reactor -from twisted.trial import unittest -from twisted.web import server, static, util +from testfixtures import LogCapture +from twisted.internet import defer from scrapy import signals -from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import CloseSpider -from scrapy.http import Request +from scrapy.core.engine import ExecutionEngine, _Slot +from scrapy.core.scheduler import BaseScheduler +from scrapy.exceptions import CloseSpider, IgnoreRequest +from scrapy.http import Headers, Request, Response from scrapy.item import Field, Item from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider +from scrapy.statscollectors import MemoryStatsCollector +from scrapy.utils.defer import ( + _schedule_coro, + deferred_from_coro, + maybe_deferred_to_future, +) from scrapy.utils.signal import disconnect_all +from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -from tests import get_testdata, tests_datadir +from tests import get_testdata +from tests.utils.decorators import coroutine_test, inline_callbacks_test + +if TYPE_CHECKING: + from twisted.python.failure import Failure + + from scrapy.core.scheduler import Scheduler + from scrapy.crawler import Crawler + from tests.mockserver.http import MockServer -class TestItem(Item): +class MyItem(Item): name = Field() url = Field() price = Field() @@ -58,15 +66,14 @@ class DataClassItem: price: int = 0 -class TestSpider(Spider): +class MySpider(Spider): name = "scrapytest.org" - allowed_domains = ["scrapytest.org", "localhost"] itemurl_re = re.compile(r"item\d+.html") - name_re = re.compile(r"

(.*?)

", re.M) - price_re = re.compile(r">Price: \$(.*?)<", re.M) + name_re = re.compile(r"

(.*?)

", re.MULTILINE) + price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE) - item_cls: type = TestItem + item_cls: type = MyItem def parse(self, response): xlink = LinkExtractor() @@ -87,24 +94,25 @@ class TestSpider(Spider): return adapter.item -class TestDupeFilterSpider(TestSpider): - def start_requests(self): - return (Request(url) for url in self.start_urls) # no dont_filter=True +class DupeFilterSpider(MySpider): + async def start(self): + for url in self.start_urls: + yield Request(url) # no dont_filter=True -class DictItemsSpider(TestSpider): +class DictItemsSpider(MySpider): item_cls = dict -class AttrsItemsSpider(TestSpider): +class AttrsItemsSpider(MySpider): item_cls = AttrsItem -class DataClassItemsSpider(TestSpider): +class DataClassItemsSpider(MySpider): item_cls = DataClassItem -class ItemZeroDivisionErrorSpider(TestSpider): +class ItemZeroDivisionErrorSpider(MySpider): custom_settings = { "ITEM_PIPELINES": { "tests.pipelines.ProcessWithZeroDivisionErrorPipeline": 300, @@ -112,7 +120,7 @@ class ItemZeroDivisionErrorSpider(TestSpider): } -class ChangeCloseReasonSpider(TestSpider): +class ChangeCloseReasonSpider(MySpider): @classmethod def from_crawler(cls, crawler, *args, **kwargs): spider = cls(*args, **kwargs) @@ -124,45 +132,26 @@ class ChangeCloseReasonSpider(TestSpider): raise CloseSpider(reason="custom_reason") -def start_test_site(debug=False): - root_dir = Path(tests_datadir, "test_site") - r = static.File(str(root_dir)) - r.putChild(b"redirect", util.Redirect(b"/redirected")) - r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) - numbers = [str(x).encode("utf8") for x in range(2**18)] - r.putChild(b"numbers", static.Data(b"".join(numbers), "text/plain")) - - port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") - if debug: - print( - f"Test server running at http://localhost:{port.getHost().port}/ " - "- hit Ctrl-C to finish." - ) - return port - - class CrawlerRun: """A class to run the crawler and keep track of events occurred""" - def __init__(self, spider_class): - self.spider = None - self.respplug = [] - self.reqplug = [] - self.reqdropped = [] - self.reqreached = [] - self.itemerror = [] - self.itemresp = [] - self.headers = {} - self.bytes = defaultdict(list) - self.signals_caught = {} + def __init__(self, spider_class: type[Spider]): + self.respplug: list[tuple[Response, Spider]] = [] + self.reqplug: list[tuple[Request, Spider]] = [] + self.reqdropped: list[tuple[Request, Spider]] = [] + self.reqreached: list[tuple[Request, Spider]] = [] + self.itemerror: list[tuple[Any, Response, Spider, Failure]] = [] + self.itemresp: list[tuple[Any, Response]] = [] + self.headers: dict[Request, Headers] = {} + self.bytes: defaultdict[Request, list[bytes]] = defaultdict(list) + self.signals_caught: dict[Any, dict[str, Any]] = {} self.spider_class = spider_class - def run(self): - self.port = start_test_site() - self.portno = self.port.getHost().port + async def run(self, mockserver: MockServer) -> None: + self.mockserver = mockserver start_urls = [ - self.geturl("/"), + self.geturl("/static/"), self.geturl("/redirect"), self.geturl("/redirect"), # duplicate self.geturl("/numbers"), @@ -186,52 +175,54 @@ class CrawlerRun: self.response_downloaded, signals.response_downloaded ) self.crawler.crawl(start_urls=start_urls) - self.spider = self.crawler.spider - self.deferred = defer.Deferred() + self.deferred: defer.Deferred[None] = defer.Deferred() dispatcher.connect(self.stop, signals.engine_stopped) - return self.deferred + await maybe_deferred_to_future(self.deferred) - def stop(self): - self.port.stopListening() # FIXME: wait for this Deferred + async def stop(self): for name, signal in vars(signals).items(): if not name.startswith("_"): disconnect_all(signal) self.deferred.callback(None) - return self.crawler.stop() + await self.crawler.stop_async() - def geturl(self, path): - return f"http://localhost:{self.portno}{path}" + def geturl(self, path: str) -> str: + return self.mockserver.url(path) - def getpath(self, url): + def getpath(self, url: str) -> str: u = urlparse(url) return u.path - def item_error(self, item, response, spider, failure): + def item_error( + self, item: Any, response: Response, spider: Spider, failure: Failure + ) -> None: self.itemerror.append((item, response, spider, failure)) - def item_scraped(self, item, spider, response): + def item_scraped(self, item: Any, spider: Spider, response: Response) -> None: self.itemresp.append((item, response)) - def headers_received(self, headers, body_length, request, spider): + def headers_received( + self, headers: Headers, body_length: int, request: Request, spider: Spider + ) -> None: self.headers[request] = headers - def bytes_received(self, data, request, spider): + def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None: self.bytes[request].append(data) - def request_scheduled(self, request, spider): + def request_scheduled(self, request: Request, spider: Spider) -> None: self.reqplug.append((request, spider)) - def request_reached(self, request, spider): + def request_reached(self, request: Request, spider: Spider) -> None: self.reqreached.append((request, spider)) - def request_dropped(self, request, spider): + def request_dropped(self, request: Request, spider: Spider) -> None: self.reqdropped.append((request, spider)) - def response_downloaded(self, response, spider): + def response_downloaded(self, response: Response, spider: Spider) -> None: self.respplug.append((response, spider)) - def record_signal(self, *args, **kwargs): + def record_signal(self, *args: Any, **kwargs: Any) -> None: """Record a signal and its parameters""" signalargs = kwargs.copy() sig = signalargs.pop("signal") @@ -239,65 +230,32 @@ class CrawlerRun: self.signals_caught[sig] = signalargs -class EngineTest(unittest.TestCase): - @defer.inlineCallbacks - def test_crawler(self): - for spider in ( - TestSpider, - DictItemsSpider, - AttrsItemsSpider, - DataClassItemsSpider, - ): - run = CrawlerRun(spider) - yield run.run() - self._assert_visited_urls(run) - self._assert_scheduled_requests(run, count=9) - self._assert_downloaded_responses(run, count=9) - self._assert_scraped_items(run) - self._assert_signals_caught(run) - self._assert_bytes_received(run) - - @defer.inlineCallbacks - def test_crawler_dupefilter(self): - run = CrawlerRun(TestDupeFilterSpider) - yield run.run() - self._assert_scheduled_requests(run, count=8) - self._assert_dropped_requests(run) - - @defer.inlineCallbacks - def test_crawler_itemerror(self): - run = CrawlerRun(ItemZeroDivisionErrorSpider) - yield run.run() - self._assert_items_error(run) - - @defer.inlineCallbacks - def test_crawler_change_close_reason_on_idle(self): - run = CrawlerRun(ChangeCloseReasonSpider) - yield run.run() - self.assertEqual( - {"spider": run.spider, "reason": "custom_reason"}, - run.signals_caught[signals.spider_closed], - ) - - def _assert_visited_urls(self, run: CrawlerRun): +class TestEngineBase: + @staticmethod + def _assert_visited_urls(run: CrawlerRun) -> None: must_be_visited = [ - "/", + "/static/", "/redirect", "/redirected", - "/item1.html", - "/item2.html", - "/item999.html", + "/static/item1.html", + "/static/item2.html", + "/static/item999.html", ] urls_visited = {rp[0].url for rp in run.respplug} urls_expected = {run.geturl(p) for p in must_be_visited} - assert ( - urls_expected <= urls_visited - ), f"URLs not visited: {list(urls_expected - urls_visited)}" + assert urls_expected <= urls_visited, ( + f"URLs not visited: {list(urls_expected - urls_visited)}" + ) - def _assert_scheduled_requests(self, run: CrawlerRun, count=None): - self.assertEqual(count, len(run.reqplug)) + @staticmethod + def _assert_scheduled_requests(run: CrawlerRun, count: int) -> None: + assert len(run.reqplug) == count - paths_expected = ["/item999.html", "/item2.html", "/item1.html"] + paths_expected = [ + "/static/item999.html", + "/static/item2.html", + "/static/item1.html", + ] urls_requested = {rq[0].url for rq in run.reqplug} urls_expected = {run.geturl(p) for p in paths_expected} @@ -305,101 +263,105 @@ class EngineTest(unittest.TestCase): scheduled_requests_count = len(run.reqplug) dropped_requests_count = len(run.reqdropped) responses_count = len(run.respplug) - self.assertEqual( - scheduled_requests_count, dropped_requests_count + responses_count - ) - self.assertEqual(len(run.reqreached), responses_count) + assert scheduled_requests_count == dropped_requests_count + responses_count + assert len(run.reqreached) == responses_count - def _assert_dropped_requests(self, run: CrawlerRun): - self.assertEqual(len(run.reqdropped), 1) + @staticmethod + def _assert_dropped_requests(run: CrawlerRun) -> None: + assert len(run.reqdropped) == 1 - def _assert_downloaded_responses(self, run: CrawlerRun, count): + @staticmethod + def _assert_downloaded_responses(run: CrawlerRun, count: int) -> None: # response tests - self.assertEqual(count, len(run.respplug)) - self.assertEqual(count, len(run.reqreached)) + assert len(run.respplug) == count + assert len(run.reqreached) == count for response, _ in run.respplug: - if run.getpath(response.url) == "/item999.html": - self.assertEqual(404, response.status) + if run.getpath(response.url) == "/static/item999.html": + assert response.status == 404 if run.getpath(response.url) == "/redirect": - self.assertEqual(302, response.status) + assert response.status == 302 - def _assert_items_error(self, run: CrawlerRun): - self.assertEqual(2, len(run.itemerror)) + @staticmethod + def _assert_items_error(run: CrawlerRun) -> None: + assert len(run.itemerror) == 2 for item, response, spider, failure in run.itemerror: - self.assertEqual(failure.value.__class__, ZeroDivisionError) - self.assertEqual(spider, run.spider) + assert failure.value.__class__ is ZeroDivisionError + assert spider == run.crawler.spider - self.assertEqual(item["url"], response.url) + assert item["url"] == response.url if "item1.html" in item["url"]: - self.assertEqual("Item 1 name", item["name"]) - self.assertEqual("100", item["price"]) + assert item["name"] == "Item 1 name" + assert item["price"] == "100" if "item2.html" in item["url"]: - self.assertEqual("Item 2 name", item["name"]) - self.assertEqual("200", item["price"]) + assert item["name"] == "Item 2 name" + assert item["price"] == "200" - def _assert_scraped_items(self, run: CrawlerRun): - self.assertEqual(2, len(run.itemresp)) - for item, response in run.itemresp: - item = ItemAdapter(item) - self.assertEqual(item["url"], response.url) + @staticmethod + def _assert_scraped_items(run: CrawlerRun) -> None: + assert len(run.itemresp) == 2 + for item_, response in run.itemresp: + item = ItemAdapter(item_) + assert item["url"] == response.url if "item1.html" in item["url"]: - self.assertEqual("Item 1 name", item["name"]) - self.assertEqual("100", item["price"]) + assert item["name"] == "Item 1 name" + assert item["price"] == "100" if "item2.html" in item["url"]: - self.assertEqual("Item 2 name", item["name"]) - self.assertEqual("200", item["price"]) + assert item["name"] == "Item 2 name" + assert item["price"] == "200" - def _assert_headers_received(self, run: CrawlerRun): + @staticmethod + def _assert_headers_received(run: CrawlerRun) -> None: for headers in run.headers.values(): - self.assertIn(b"Server", headers) - self.assertIn(b"TwistedWeb", headers[b"Server"]) - self.assertIn(b"Date", headers) - self.assertIn(b"Content-Type", headers) + assert b"Server" in headers + assert headers[b"Server"] + assert b"TwistedWeb" in headers[b"Server"] + assert b"Date" in headers + assert b"Content-Type" in headers - def _assert_bytes_received(self, run: CrawlerRun): - self.assertEqual(9, len(run.bytes)) + @staticmethod + def _assert_bytes_received(run: CrawlerRun) -> None: + assert len(run.bytes) == 9 for request, data in run.bytes.items(): joined_data = b"".join(data) - if run.getpath(request.url) == "/": - self.assertEqual(joined_data, get_testdata("test_site", "index.html")) - elif run.getpath(request.url) == "/item1.html": - self.assertEqual(joined_data, get_testdata("test_site", "item1.html")) - elif run.getpath(request.url) == "/item2.html": - self.assertEqual(joined_data, get_testdata("test_site", "item2.html")) + if run.getpath(request.url) == "/static/": + assert joined_data == get_testdata("test_site", "index.html") + elif run.getpath(request.url) == "/static/item1.html": + assert joined_data == get_testdata("test_site", "item1.html") + elif run.getpath(request.url) == "/static/item2.html": + assert joined_data == get_testdata("test_site", "item2.html") elif run.getpath(request.url) == "/redirected": - self.assertEqual(joined_data, b"Redirected here") + assert joined_data == b"Redirected here" elif run.getpath(request.url) == "/redirect": - self.assertEqual( - joined_data, - b"\n\n" + assert ( + joined_data == b"\n\n" b" \n" b' \n' b" \n" b' \n' b' click here\n' b" \n" - b"\n", + b"\n" ) - elif run.getpath(request.url) == "/tem999.html": - self.assertEqual( - joined_data, - b"\n\n" + elif run.getpath(request.url) == "/static/item999.html": + assert ( + joined_data == b"\n\n" b" 404 - No Such Resource\n" b" \n" b"

No Such Resource

\n" b"

File not found.

\n" b" \n" - b"\n", + b"\n" ) elif run.getpath(request.url) == "/numbers": # signal was fired multiple times - self.assertTrue(len(data) > 1) + assert len(data) > 1 # bytes were received in order numbers = [str(x).encode("utf8") for x in range(2**18)] - self.assertEqual(joined_data, b"".join(numbers)) + assert joined_data == b"".join(numbers) - def _assert_signals_caught(self, run: CrawlerRun): + @staticmethod + def _assert_signals_caught(run: CrawlerRun) -> None: assert signals.engine_started in run.signals_caught assert signals.engine_stopped in run.signals_caught assert signals.spider_opened in run.signals_caught @@ -407,33 +369,119 @@ class EngineTest(unittest.TestCase): assert signals.spider_closed in run.signals_caught assert signals.headers_received in run.signals_caught - self.assertEqual( - {"spider": run.spider}, run.signals_caught[signals.spider_opened] - ) - self.assertEqual( - {"spider": run.spider}, run.signals_caught[signals.spider_idle] - ) - self.assertEqual( - {"spider": run.spider, "reason": "finished"}, - run.signals_caught[signals.spider_closed], - ) + assert {"spider": run.crawler.spider} == run.signals_caught[ + signals.spider_opened + ] + assert {"spider": run.crawler.spider} == run.signals_caught[signals.spider_idle] + assert { + "spider": run.crawler.spider, + "reason": "finished", + } == run.signals_caught[signals.spider_closed] - @defer.inlineCallbacks - def test_close_downloader(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.close() - @defer.inlineCallbacks - def test_start_already_running_exception(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) - e.start() - try: - yield self.assertFailure(e.start(), RuntimeError).addBoth( - lambda exc: self.assertEqual(str(exc), "Engine already running") +class TestEngine(TestEngineBase): + @coroutine_test + async def test_crawler(self, mockserver: MockServer) -> None: + for spider in ( + MySpider, + DictItemsSpider, + AttrsItemsSpider, + DataClassItemsSpider, + ): + run = CrawlerRun(spider) + await run.run(mockserver) + self._assert_visited_urls(run) + self._assert_scheduled_requests(run, count=9) + self._assert_downloaded_responses(run, count=9) + self._assert_scraped_items(run) + self._assert_signals_caught(run) + self._assert_bytes_received(run) + + @coroutine_test + async def test_crawler_dupefilter(self, mockserver: MockServer) -> None: + run = CrawlerRun(DupeFilterSpider) + await run.run(mockserver) + self._assert_scheduled_requests(run, count=8) + self._assert_dropped_requests(run) + + @coroutine_test + async def test_crawler_itemerror(self, mockserver: MockServer) -> None: + run = CrawlerRun(ItemZeroDivisionErrorSpider) + await run.run(mockserver) + self._assert_items_error(run) + + @coroutine_test + async def test_crawler_change_close_reason_on_idle( + self, mockserver: MockServer + ) -> None: + run = CrawlerRun(ChangeCloseReasonSpider) + await run.run(mockserver) + assert { + "spider": run.crawler.spider, + "reason": "custom_reason", + } == run.signals_caught[signals.spider_closed] + + @coroutine_test + async def test_close_downloader(self): + e = ExecutionEngine(get_crawler(MySpider), lambda _: None) + await e.close_async() + + def test_close_without_downloader(self): + class CustomException(Exception): + pass + + class BadDownloader: + def __init__(self, crawler): + raise CustomException + + with pytest.raises(CustomException): + ExecutionEngine( + get_crawler(MySpider, {"DOWNLOADER": BadDownloader}), lambda _: None ) - finally: - yield e.stop() + + @inline_callbacks_test + def test_start_already_running_exception(self): + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() + e = ExecutionEngine(crawler, lambda _: None) + crawler.engine = e + yield deferred_from_coro(e.open_spider_async()) + _schedule_coro(e.start_async()) + with pytest.raises(RuntimeError, match="Engine already running"): + yield deferred_from_coro(e.start_async()) + yield deferred_from_coro(e.stop_async()) + + @pytest.mark.only_asyncio + @coroutine_test + async def test_start_already_running_exception_asyncio(self): + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() + e = ExecutionEngine(crawler, lambda _: None) + crawler.engine = e + await e.open_spider_async() + with pytest.raises(RuntimeError, match="Engine already running"): + await asyncio.gather(e.start_async(), e.start_async()) + await e.stop_async() + + @inline_callbacks_test + def test_start_request_processing_exception(self): + class BadRequestFingerprinter: + def fingerprint(self, request): + raise ValueError # to make Scheduler.enqueue_request() fail + + class SimpleSpider(Spider): + name = "simple" + + async def start(self): + yield Request("data:,") + + crawler = get_crawler( + SimpleSpider, {"REQUEST_FINGERPRINTER_CLASS": BadRequestFingerprinter} + ) + with LogCapture() as log: + yield crawler.crawl() + assert "Error while processing requests from start()" in str(log) + assert "Spider closed (shutdown)" in str(log) def test_short_timeout(self): args = ( @@ -449,25 +497,274 @@ class EngineTest(unittest.TestCase): ) p = subprocess.Popen( args, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, ) - def kill_proc(): + try: + _, stderr = p.communicate(timeout=15) + except subprocess.TimeoutExpired: p.kill() p.communicate() - assert False, "Command took too much time to complete" + pytest.fail("Command took too much time to complete") - timer = Timer(15, kill_proc) - try: - timer.start() - _, stderr = p.communicate() - finally: - timer.cancel() - - self.assertNotIn(b"Traceback", stderr) + stderr_str = stderr.decode("utf-8") + assert "AttributeError" not in stderr_str, stderr_str + assert "AssertionError" not in stderr_str, stderr_str -if __name__ == "__main__": - if len(sys.argv) > 1 and sys.argv[1] == "runserver": - start_test_site(debug=True) - reactor.run() +class TestEngineDownloadAsync: + """Test cases for ExecutionEngine.download_async().""" + + @pytest.fixture + def engine(self) -> ExecutionEngine: + crawler = get_crawler(MySpider) + engine = ExecutionEngine(crawler, lambda _: None) + engine.downloader.close() + engine.downloader = Mock() + engine._slot = Mock() + engine._slot.inprogress = set() + return engine + + @staticmethod + async def _download(engine: ExecutionEngine, request: Request) -> Response: + return await engine.download_async(request) + + @coroutine_test + async def test_download_async_success(self, engine): + """Test basic successful async download of a request.""" + request = Request("http://example.com") + response = Response("http://example.com", body=b"test body") + engine.spider = Mock() + engine.downloader.fetch.return_value = defer.succeed(response) + engine._slot.add_request = Mock() + engine._slot.remove_request = Mock() + + result = await self._download(engine, request) + assert result == response + engine._slot.add_request.assert_called_once_with(request) + engine._slot.remove_request.assert_called_once_with(request) + engine.downloader.fetch.assert_called_once_with(request) + + @coroutine_test + async def test_download_async_redirect(self, engine): + """Test async download with a redirect request.""" + original_request = Request("http://example.com") + redirect_request = Request("http://example.com/redirect") + final_response = Response("http://example.com/redirect", body=b"redirected") + + # First call returns redirect request, second call returns final response + engine.downloader.fetch.side_effect = [ + defer.succeed(redirect_request), + defer.succeed(final_response), + ] + engine.spider = Mock() + engine._slot.add_request = Mock() + engine._slot.remove_request = Mock() + + result = await self._download(engine, original_request) + assert result == final_response + assert engine.downloader.fetch.call_count == 2 + engine._slot.add_request.assert_has_calls( + [call(original_request), call(redirect_request)] + ) + engine._slot.remove_request.assert_has_calls( + [call(original_request), call(redirect_request)] + ) + + @coroutine_test + async def test_download_async_no_spider(self, engine): + """Test async download attempt when no spider is available.""" + request = Request("http://example.com") + engine.spider = None + with pytest.raises(RuntimeError, match="No open spider to crawl:"): + await self._download(engine, request) + + @coroutine_test + async def test_download_async_failure(self, engine): + """Test async download when the downloader raises an exception.""" + request = Request("http://example.com") + error = RuntimeError("Download failed") + engine.spider = Mock() + engine.downloader.fetch.return_value = defer.fail(error) + engine._slot.add_request = Mock() + engine._slot.remove_request = Mock() + + with pytest.raises(RuntimeError, match="Download failed"): + await self._download(engine, request) + engine._slot.add_request.assert_called_once_with(request) + engine._slot.remove_request.assert_called_once_with(request) + + +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class TestEngineDownload(TestEngineDownloadAsync): + """Test cases for ExecutionEngine.download().""" + + @staticmethod + async def _download(engine: ExecutionEngine, request: Request) -> Response: + return await maybe_deferred_to_future(engine.download(request)) + + +@coroutine_test +async def test_request_scheduled_signal(caplog): + class TestScheduler(BaseScheduler): + def __init__(self): + self.enqueued = [] + + def enqueue_request(self, request: Request) -> bool: + self.enqueued.append(request) + return True + + def signal_handler(request: Request, spider: Spider) -> None: + if "drop" in request.url: + raise IgnoreRequest + + crawler = get_crawler(MySpider) + engine = ExecutionEngine(crawler, lambda _: None) + scheduler = TestScheduler() + + async def start(): + return + yield + + engine._start = start() + engine._slot = _Slot(False, Mock(), scheduler) + crawler.signals.connect(signal_handler, signals.request_scheduled) + keep_request = Request("https://keep.example") + engine._schedule_request(keep_request) + drop_request = Request("https://drop.example") + caplog.set_level(DEBUG) + engine._schedule_request(drop_request) + assert scheduler.enqueued == [keep_request], ( + f"{scheduler.enqueued!r} != [{keep_request!r}]" + ) + crawler.signals.disconnect(signal_handler, signals.request_scheduled) + + +class TestEngineCloseSpider: + """Tests for exception handling coverage during close_spider_async().""" + + @pytest.fixture + def crawler(self) -> Crawler: + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() + return crawler + + @coroutine_test + async def test_no_slot(self, crawler: Crawler) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + slot = engine._slot + engine._slot = None + with pytest.raises(RuntimeError, match="Engine slot not assigned"): + await engine.close_spider_async() + # close it correctly + engine._slot = slot + await engine.close_spider_async() + + @coroutine_test + async def test_no_spider(self, crawler: Crawler) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + with pytest.raises(RuntimeError, match="Spider not opened"): + await engine.close_spider_async() + engine.downloader.close() # cleanup + + @coroutine_test + async def test_exception_slot( + self, crawler: Crawler, caplog: pytest.LogCaptureFixture + ) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + assert engine._slot + del engine._slot.heartbeat + await engine.close_spider_async() + assert "Slot close failure" in caplog.text + + @coroutine_test + async def test_exception_downloader( + self, crawler: Crawler, caplog: pytest.LogCaptureFixture + ) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + del engine.downloader.slots + await engine.close_spider_async() + assert "Downloader close failure" in caplog.text + + @coroutine_test + async def test_exception_scraper( + self, crawler: Crawler, caplog: pytest.LogCaptureFixture + ) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + engine.scraper.slot = None + await engine.close_spider_async() + assert "Scraper close failure" in caplog.text + + @coroutine_test + async def test_exception_scheduler( + self, crawler: Crawler, caplog: pytest.LogCaptureFixture + ) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + assert engine._slot + del cast("Scheduler", engine._slot.scheduler).dqs + await engine.close_spider_async() + assert "Scheduler close failure" in caplog.text + + @coroutine_test + async def test_exception_signal( + self, crawler: Crawler, caplog: pytest.LogCaptureFixture + ) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + signal_manager = engine.signals + del engine.signals + await engine.close_spider_async() + assert "Error while sending spider_close signal" in caplog.text + # send the spider_closed signal to close various components + await signal_manager.send_catch_log_async( + signal=signals.spider_closed, + spider=engine.spider, + reason="cancelled", + ) + + @coroutine_test + async def test_exception_stats( + self, crawler: Crawler, caplog: pytest.LogCaptureFixture + ) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + assert isinstance(crawler.stats, MemoryStatsCollector) + del crawler.stats.spider_stats + await engine.close_spider_async() + assert "Stats close failure" in caplog.text + + @coroutine_test + async def test_exception_callback( + self, crawler: Crawler, caplog: pytest.LogCaptureFixture + ) -> None: + engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError())) + crawler.engine = engine + await engine.open_spider_async() + await engine.close_spider_async() + assert "Error running spider_closed_callback" in caplog.text + + @coroutine_test + async def test_exception_async_callback( + self, crawler: Crawler, caplog: pytest.LogCaptureFixture + ) -> None: + async def cb(_): + raise ValueError + + engine = ExecutionEngine(crawler, cb) + crawler.engine = engine + await engine.open_spider_async() + await engine.close_spider_async() + assert "Error running spider_closed_callback" in caplog.text diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py new file mode 100644 index 000000000..6cc8c0650 --- /dev/null +++ b/tests/test_engine_loop.py @@ -0,0 +1,346 @@ +from __future__ import annotations + +from collections import deque +from logging import ERROR +from typing import TYPE_CHECKING + +from scrapy import Request, Spider, signals +from scrapy.utils.asyncio import call_later +from scrapy.utils.test import get_crawler +from tests.mockserver.http import MockServer +from tests.test_scheduler import MemoryScheduler +from tests.utils import async_sleep +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + import pytest + + from scrapy.http import Response + + +class TestMain: + @coroutine_test + async def test_sleep(self): + """Neither asynchronous sleeps on Spider.start() nor the equivalent on + the scheduler (returning no requests while also returning True from + the has_pending_requests() method) should cause the spider to miss the + processing of any later requests.""" + seconds = 2 + + class TestSpider(Spider): + name = "test" + + async def start(self): + yield Request("data:,a") + + await async_sleep(seconds) + + self.crawler.engine._slot.scheduler.pause() + self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b")) + + # During this time, the scheduler reports having requests but + # returns None. + await async_sleep(seconds) + + self.crawler.engine._slot.scheduler.unpause() + + # The scheduler request is processed. + await async_sleep(seconds) + + yield Request("data:,c") + + await async_sleep(seconds) + + self.crawler.engine._slot.scheduler.pause() + self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d")) + + # The last start request is processed during the time until the + # delayed call below, proving that the start iteration can + # finish before a scheduler “sleep†without causing the + # scheduler to finish. + call_later(seconds, self.crawler.engine._slot.scheduler.unpause) + + def parse(self, response): + pass + + actual_urls = [] + + def track_url(request, spider): + actual_urls.append(request.url) + + settings = {"SCHEDULER": MemoryScheduler} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_url, signals.request_reached_downloader) + await crawler.crawl_async() + assert crawler.stats.get_value("finish_reason") == "finished" + expected_urls = ["data:,a", "data:,b", "data:,c", "data:,d"] + assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" + + @coroutine_test + async def test_close_during_start_iteration( + self, caplog: pytest.LogCaptureFixture + ) -> None: + class TestSpider(Spider): + name = "test" + + async def start(self): + assert self.crawler.engine is not None + await self.crawler.engine.close_async() + yield Request("data:,a") + + def parse(self, response): + pass + + actual_urls = [] + + def track_url(request, spider): + actual_urls.append(request.url) + + settings = {"SCHEDULER": MemoryScheduler} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_url, signals.request_reached_downloader) + + caplog.clear() + with caplog.at_level(ERROR): + await crawler.crawl_async() + + assert not caplog.records + assert crawler.stats + assert crawler.stats.get_value("finish_reason") == "shutdown" + assert not actual_urls + + +class TestRequestSendOrder: + seconds = 0.1 # increase if flaky + + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) # increase if flaky + + def request(self, num, response_seconds, download_slots, priority=0): + url = self.mockserver.url(f"/delay?n={response_seconds}&{num}") + meta = {"download_slot": str(num % download_slots)} + return Request(url, meta=meta, priority=priority) + + def get_num(self, request_or_response: Request | Response): + return int(request_or_response.url.rsplit("&", maxsplit=1)[1]) + + async def _test_request_order( + self, + start_nums, + cb_nums=None, + settings=None, + response_seconds=None, + download_slots=1, + start_fn=None, + parse_fn=None, + ): + cb_nums = cb_nums or [] + settings = settings or {} + response_seconds = response_seconds or self.seconds + + cb_requests = deque( + [self.request(num, response_seconds, download_slots) for num in cb_nums] + ) + + if start_fn is None: + + async def start_fn(spider): + for num in start_nums: + yield self.request(num, response_seconds, download_slots) + + if parse_fn is None: + + def parse_fn(spider, response): + while cb_requests: + yield cb_requests.popleft() + + class TestSpider(Spider): + name = "test" + start = start_fn + parse = parse_fn + + actual_nums = [] + + def track_num(request, spider): + actual_nums.append(self.get_num(request)) + + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_num, signals.request_reached_downloader) + await crawler.crawl_async() + assert crawler.stats.get_value("finish_reason") == "finished" + expected_nums = sorted(start_nums + cb_nums) + assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}" + + @coroutine_test + async def test_default(self): + """By default, callback requests take priority over start requests and + are sent in order. Priority matters, but given the same priority, a + callback request takes precedence.""" + nums = [1, 2, 3, 4, 5, 6] + response_seconds = 0 + download_slots = 1 + + def _request(num, priority=0): + return self.request( + num, response_seconds, download_slots, priority=priority + ) + + async def start(spider): + # The first CONCURRENT_REQUESTS start requests are sent + # immediately. + yield _request(1) + + for request in ( + _request(2, priority=1), + _request(5), + ): + spider.crawler.engine._slot.scheduler.enqueue_request(request) + yield _request(6) + yield _request(3, priority=1) + yield _request(4, priority=1) + + def parse(spider, response): + return + yield + + await self._test_request_order( + start_nums=nums, + settings={"CONCURRENT_REQUESTS": 1}, + response_seconds=response_seconds, + start_fn=start, + parse_fn=parse, + ) + + @coroutine_test + async def test_lifo_start(self): + """Changing the queues of start requests to LIFO, matching the queues + of non-start requests, does not cause all requests to be stored in the + same queue objects, it only affects the order of start requests.""" + nums = [1, 2, 3, 4, 5, 6] + response_seconds = 0 + download_slots = 1 + + def _request(num, priority=0): + return self.request( + num, response_seconds, download_slots, priority=priority + ) + + async def start(spider): + # The first CONCURRENT_REQUESTS start requests are sent + # immediately. + yield _request(1) + + for request in ( + _request(2, priority=1), + _request(5), + ): + spider.crawler.engine._slot.scheduler.enqueue_request(request) + yield _request(6) + yield _request(4, priority=1) + yield _request(3, priority=1) + + def parse(spider, response): + return + yield + + await self._test_request_order( + start_nums=nums, + settings={ + "CONCURRENT_REQUESTS": 1, + "SCHEDULER_START_MEMORY_QUEUE": "scrapy.squeues.LifoMemoryQueue", + }, + response_seconds=response_seconds, + start_fn=start, + parse_fn=parse, + ) + + @coroutine_test + async def test_shared_queues(self): + """If SCHEDULER_START_*_QUEUE is falsy, start requests and other + requests share the same queue, i.e. start requests are not prioritized + over other requests if their priority matches.""" + nums = list(range(1, 14)) + response_seconds = 0 + download_slots = 1 + + def _request(num, priority=0): + return self.request( + num, response_seconds, download_slots, priority=priority + ) + + async def start(spider): + # The first CONCURRENT_REQUESTS start requests are sent + # immediately. + yield _request(1) + + # Below, priority 1 requests are sent first, and requests are sent + # in LIFO order. + + for request in ( + _request(7, priority=1), + _request(6, priority=1), + _request(13), + _request(12), + ): + spider.crawler.engine._slot.scheduler.enqueue_request(request) + + yield _request(11) + yield _request(10) + yield _request(5, priority=1) + yield _request(4, priority=1) + + for request in ( + _request(3, priority=1), + _request(2, priority=1), + _request(9), + _request(8), + ): + spider.crawler.engine._slot.scheduler.enqueue_request(request) + + def parse(spider, response): + return + yield + + await self._test_request_order( + start_nums=nums, + settings={ + "CONCURRENT_REQUESTS": 1, + "SCHEDULER_START_MEMORY_QUEUE": None, + }, + response_seconds=response_seconds, + start_fn=start, + parse_fn=parse, + ) + + # Examples from the “Start requests†section of the documentation about + # spiders. + + @coroutine_test + async def test_lazy(self): + start_nums = [1, 2, 4] + cb_nums = [3] + response_seconds = self.seconds * 2**1 # increase if flaky + download_slots = 1 + + async def start(spider): + for num in start_nums: + if spider.crawler.engine.needs_backout(): + await spider.crawler.signals.wait_for(signals.scheduler_empty) + request = self.request(num, response_seconds, download_slots) + yield request + + await self._test_request_order( + start_nums=start_nums, + cb_nums=cb_nums, + settings={ + "CONCURRENT_REQUESTS": 1, + }, + response_seconds=response_seconds, + start_fn=start, + ) diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index 8dbb5b7ea..091485781 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -1,5 +1,6 @@ -from testfixtures import LogCapture -from twisted.internet import defer +from __future__ import annotations + +from typing import TYPE_CHECKING from scrapy.exceptions import StopDownload from tests.test_engine import ( @@ -7,9 +8,15 @@ from tests.test_engine import ( CrawlerRun, DataClassItemsSpider, DictItemsSpider, - EngineTest, - TestSpider, + MySpider, + TestEngineBase, ) +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + import pytest + + from tests.mockserver.http import MockServer class BytesReceivedCrawlerRun(CrawlerRun): @@ -18,42 +25,25 @@ class BytesReceivedCrawlerRun(CrawlerRun): raise StopDownload(fail=False) -class BytesReceivedEngineTest(EngineTest): - @defer.inlineCallbacks - def test_crawler(self): +class TestBytesReceivedEngine(TestEngineBase): + @coroutine_test + async def test_crawler( + self, mockserver: MockServer, caplog: pytest.LogCaptureFixture + ) -> None: for spider in ( - TestSpider, + MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider, ): run = BytesReceivedCrawlerRun(spider) - with LogCapture() as log: - yield run.run() - log.check_present( - ( - "scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler BytesReceivedCrawlerRun.bytes_received", - ) - ) - log.check_present( - ( - "scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler BytesReceivedCrawlerRun.bytes_received", - ) - ) - log.check_present( - ( - "scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler BytesReceivedCrawlerRun.bytes_received", - ) - ) + with caplog.at_level("DEBUG"): + await run.run(mockserver) + for url in ("/redirected", "/static/", "/numbers"): + assert ( + f"Download stopped for " + "from signal handler BytesReceivedCrawlerRun.bytes_received" + ) in caplog.text self._assert_visited_urls(run) self._assert_scheduled_requests(run, count=9) self._assert_downloaded_responses(run, count=9) @@ -61,14 +51,15 @@ class BytesReceivedEngineTest(EngineTest): self._assert_headers_received(run) self._assert_bytes_received(run) - def _assert_bytes_received(self, run: CrawlerRun): - self.assertEqual(9, len(run.bytes)) + @staticmethod + def _assert_bytes_received(run: CrawlerRun) -> None: + assert len(run.bytes) == 9 for request, data in run.bytes.items(): joined_data = b"".join(data) - self.assertTrue(len(data) == 1) # signal was fired only once + assert len(data) == 1 # signal was fired only once if run.getpath(request.url) == "/numbers": # Received bytes are not the complete response. The exact amount depends # on the buffer size, which can vary, so we only check that the amount # of received bytes is strictly less than the full response. numbers = [str(x).encode("utf8") for x in range(2**18)] - self.assertTrue(len(joined_data) < len(b"".join(numbers))) + assert len(joined_data) < len(b"".join(numbers)) diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index 0bad5ba55..8ebc948e2 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -1,5 +1,6 @@ -from testfixtures import LogCapture -from twisted.internet import defer +from __future__ import annotations + +from typing import TYPE_CHECKING from scrapy.exceptions import StopDownload from tests.test_engine import ( @@ -7,9 +8,15 @@ from tests.test_engine import ( CrawlerRun, DataClassItemsSpider, DictItemsSpider, - EngineTest, - TestSpider, + MySpider, + TestEngineBase, ) +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + import pytest + + from tests.mockserver.http import MockServer class HeadersReceivedCrawlerRun(CrawlerRun): @@ -18,55 +25,40 @@ class HeadersReceivedCrawlerRun(CrawlerRun): raise StopDownload(fail=False) -class HeadersReceivedEngineTest(EngineTest): - @defer.inlineCallbacks - def test_crawler(self): +class TestHeadersReceivedEngine(TestEngineBase): + @coroutine_test + async def test_crawler( + self, mockserver: MockServer, caplog: pytest.LogCaptureFixture + ) -> None: for spider in ( - TestSpider, + MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider, ): run = HeadersReceivedCrawlerRun(spider) - with LogCapture() as log: - yield run.run() - log.check_present( - ( - "scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for from" - " signal handler HeadersReceivedCrawlerRun.headers_received", - ) - ) - log.check_present( - ( - "scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for from signal" - " handler HeadersReceivedCrawlerRun.headers_received", - ) - ) - log.check_present( - ( - "scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for from" - " signal handler HeadersReceivedCrawlerRun.headers_received", - ) - ) + with caplog.at_level("DEBUG"): + await run.run(mockserver) + for url in ("/redirected", "/static/", "/numbers"): + assert ( + f"Download stopped for " + "from signal handler HeadersReceivedCrawlerRun.headers_received" + ) in caplog.text self._assert_visited_urls(run) self._assert_downloaded_responses(run, count=6) self._assert_signals_caught(run) self._assert_bytes_received(run) self._assert_headers_received(run) - def _assert_bytes_received(self, run: CrawlerRun): - self.assertEqual(0, len(run.bytes)) + @staticmethod + def _assert_bytes_received(run: CrawlerRun) -> None: + assert len(run.bytes) == 0 - def _assert_visited_urls(self, run: CrawlerRun): - must_be_visited = ["/", "/redirect", "/redirected"] + @staticmethod + def _assert_visited_urls(run: CrawlerRun) -> None: + must_be_visited = ["/static/", "/redirect", "/redirected"] urls_visited = {rp[0].url for rp in run.respplug} urls_expected = {run.geturl(p) for p in must_be_visited} - assert ( - urls_expected <= urls_visited - ), f"URLs not visited: {list(urls_expected - urls_visited)}" + assert urls_expected <= urls_visited, ( + f"URLs not visited: {list(urls_expected - urls_visited)}" + ) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index c11913365..2fded613d 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -4,12 +4,13 @@ import marshal import pickle import re import tempfile -import unittest +from abc import ABC, abstractmethod from datetime import datetime from io import BytesIO from typing import Any import lxml.etree +import pytest from itemadapter import ItemAdapter from scrapy.exporters import ( @@ -31,7 +32,7 @@ def custom_serializer(value): return str(int(value) + 2) -class TestItem(Item): +class MyItem(Item): name = Field() age = Field() @@ -42,7 +43,7 @@ class CustomFieldItem(Item): @dataclasses.dataclass -class TestDataClass: +class MyDataClass: name: str age: int @@ -53,25 +54,26 @@ class CustomFieldDataclass: age: int = dataclasses.field(metadata={"serializer": custom_serializer}) -class BaseItemExporterTest(unittest.TestCase): - item_class: type = TestItem +class TestBaseItemExporter(ABC): + item_class: type = MyItem custom_field_item_class: type = CustomFieldItem - def setUp(self): + def setup_method(self): self.i = self.item_class(name="John\xa3", age="22") self.output = BytesIO() self.ie = self._get_exporter() - def _get_exporter(self, **kwargs): - return BaseItemExporter(**kwargs) + @abstractmethod + def _get_exporter(self, **kwargs) -> BaseItemExporter: + raise NotImplementedError - def _check_output(self): + def _check_output(self): # noqa: B027 pass def _assert_expected_item(self, exported_dict): for k, v in exported_dict.items(): exported_dict[k] = to_unicode(v) - self.assertEqual(self.i, self.item_class(**exported_dict)) + assert self.i == self.item_class(**exported_dict) def _get_nonstring_types_item(self): return { @@ -83,11 +85,7 @@ class BaseItemExporterTest(unittest.TestCase): def assertItemExportWorks(self, item): self.ie.start_exporting() - try: - self.ie.export_item(item) - except NotImplementedError: - if self.ie.__class__ is not BaseItemExporter: - raise + self.ie.export_item(item) self.ie.finish_exporting() # Delete the item exporter object, so that if it causes the output # file handle to be closed, which should not be the case, follow-up @@ -104,66 +102,55 @@ class BaseItemExporterTest(unittest.TestCase): def test_serialize_field(self): a = ItemAdapter(self.i) res = self.ie.serialize_field(a.get_field_meta("name"), "name", a["name"]) - self.assertEqual(res, "John\xa3") + assert res == "John\xa3" res = self.ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) - self.assertEqual(res, "22") + assert res == "22" def test_fields_to_export(self): ie = self._get_exporter(fields_to_export=["name"]) - self.assertEqual( - list(ie._get_serialized_fields(self.i)), [("name", "John\xa3")] - ) + assert list(ie._get_serialized_fields(self.i)) == [("name", "John\xa3")] ie = self._get_exporter(fields_to_export=["name"], encoding="latin-1") - _, name = list(ie._get_serialized_fields(self.i))[0] + _, name = next(iter(ie._get_serialized_fields(self.i))) assert isinstance(name, str) - self.assertEqual(name, "John\xa3") + assert name == "John\xa3" ie = self._get_exporter(fields_to_export={"name": "å稱"}) - self.assertEqual(list(ie._get_serialized_fields(self.i)), [("å稱", "John\xa3")]) + assert list(ie._get_serialized_fields(self.i)) == [("å稱", "John\xa3")] def test_field_custom_serializer(self): i = self.custom_field_item_class(name="John\xa3", age="22") a = ItemAdapter(i) ie = self._get_exporter() - self.assertEqual( - ie.serialize_field(a.get_field_meta("name"), "name", a["name"]), "John\xa3" - ) - self.assertEqual( - ie.serialize_field(a.get_field_meta("age"), "age", a["age"]), "24" + assert ( + ie.serialize_field(a.get_field_meta("name"), "name", a["name"]) + == "John\xa3" ) + assert ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) == "24" -class BaseItemExporterDataclassTest(BaseItemExporterTest): - item_class = TestDataClass - custom_field_item_class = CustomFieldDataclass - - -class PythonItemExporterTest(BaseItemExporterTest): +class TestPythonItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): return PythonItemExporter(**kwargs) def test_invalid_option(self): - with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"): + with pytest.raises(TypeError, match="Unexpected options: invalid_option"): PythonItemExporter(invalid_option="something") def test_nested_item(self): i1 = self.item_class(name="Joseph", age="22") - i2 = dict(name="Maria", age=i1) + i2 = {"name": "Maria", "age": i1} i3 = self.item_class(name="Jesus", age=i2) ie = self._get_exporter() exported = ie.export_item(i3) - self.assertEqual(type(exported), dict) - self.assertEqual( - exported, - { - "age": {"age": {"age": "22", "name": "Joseph"}, "name": "Maria"}, - "name": "Jesus", - }, - ) - self.assertEqual(type(exported["age"]), dict) - self.assertEqual(type(exported["age"]["age"]), dict) + assert isinstance(exported, dict) + assert exported == { + "age": {"age": {"age": "22", "name": "Joseph"}, "name": "Maria"}, + "name": "Jesus", + } + assert isinstance(exported["age"], dict) + assert isinstance(exported["age"]["age"], dict) def test_export_list(self): i1 = self.item_class(name="Joseph", age="22") @@ -171,45 +158,39 @@ class PythonItemExporterTest(BaseItemExporterTest): i3 = self.item_class(name="Jesus", age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) - self.assertEqual( - exported, - { - "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}], - "name": "Jesus", - }, - ) - self.assertEqual(type(exported["age"][0]), dict) - self.assertEqual(type(exported["age"][0]["age"][0]), dict) + assert exported == { + "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}], + "name": "Jesus", + } + assert isinstance(exported["age"][0], dict) + assert isinstance(exported["age"][0]["age"][0], dict) def test_export_item_dict_list(self): i1 = self.item_class(name="Joseph", age="22") - i2 = dict(name="Maria", age=[i1]) + i2 = {"name": "Maria", "age": [i1]} i3 = self.item_class(name="Jesus", age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) - self.assertEqual( - exported, - { - "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}], - "name": "Jesus", - }, - ) - self.assertEqual(type(exported["age"][0]), dict) - self.assertEqual(type(exported["age"][0]["age"][0]), dict) + assert exported == { + "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}], + "name": "Jesus", + } + assert isinstance(exported["age"][0], dict) + assert isinstance(exported["age"][0]["age"][0], dict) def test_nonstring_types_item(self): item = self._get_nonstring_types_item() ie = self._get_exporter() exported = ie.export_item(item) - self.assertEqual(exported, item) + assert exported == item -class PythonItemExporterDataclassTest(PythonItemExporterTest): - item_class = TestDataClass +class TestPythonItemExporterDataclass(TestPythonItemExporter): + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class PprintItemExporterTest(BaseItemExporterTest): +class TestPprintItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): return PprintItemExporter(self.output, **kwargs) @@ -217,12 +198,12 @@ class PprintItemExporterTest(BaseItemExporterTest): self._assert_expected_item(eval(self.output.getvalue())) -class PprintItemExporterDataclassTest(PprintItemExporterTest): - item_class = TestDataClass +class TestPprintItemExporterDataclass(TestPprintItemExporter): + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class PickleItemExporterTest(BaseItemExporterTest): +class TestPickleItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): return PickleItemExporter(self.output, **kwargs) @@ -240,8 +221,8 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.finish_exporting() del ie # See the first “del self.ie†in this file for context. f.seek(0) - self.assertEqual(self.item_class(**pickle.load(f)), i1) - self.assertEqual(self.item_class(**pickle.load(f)), i2) + assert self.item_class(**pickle.load(f)) == i1 + assert self.item_class(**pickle.load(f)) == i2 def test_nonstring_types_item(self): item = self._get_nonstring_types_item() @@ -251,15 +232,15 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.export_item(item) ie.finish_exporting() del ie # See the first “del self.ie†in this file for context. - self.assertEqual(pickle.loads(fp.getvalue()), item) + assert pickle.loads(fp.getvalue()) == item -class PickleItemExporterDataclassTest(PickleItemExporterTest): - item_class = TestDataClass +class TestPickleItemExporterDataclass(TestPickleItemExporter): + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class MarshalItemExporterTest(BaseItemExporterTest): +class TestMarshalItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): self.output = tempfile.TemporaryFile() return MarshalItemExporter(self.output, **kwargs) @@ -278,15 +259,15 @@ class MarshalItemExporterTest(BaseItemExporterTest): ie.finish_exporting() del ie # See the first “del self.ie†in this file for context. fp.seek(0) - self.assertEqual(marshal.load(fp), item) + assert marshal.load(fp) == item -class MarshalItemExporterDataclassTest(MarshalItemExporterTest): - item_class = TestDataClass +class TestMarshalItemExporterDataclass(TestMarshalItemExporter): + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class CsvItemExporterTest(BaseItemExporterTest): +class TestCsvItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): self.output = tempfile.TemporaryFile() return CsvItemExporter(self.output, **kwargs) @@ -298,7 +279,7 @@ class CsvItemExporterTest(BaseItemExporterTest): for line in to_unicode(csv).splitlines(True) ] - return self.assertEqual(split_csv(first), split_csv(second), msg=msg) + assert split_csv(first) == split_csv(second), msg def _check_output(self): self.output.seek(0) @@ -371,7 +352,7 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_join_multivalue_not_strings(self): self.assertExportResult( - item=dict(name="John", friends=[4, 8]), + item={"name": "John", "friends": [4, 8]}, include_headers_line=False, expected='"[4, 8]",John\r\n', ) @@ -384,16 +365,16 @@ class CsvItemExporterTest(BaseItemExporterTest): ) def test_errors_default(self): - with self.assertRaises(UnicodeEncodeError): + with pytest.raises(UnicodeEncodeError): self.assertExportResult( - item=dict(text="W\u0275\u200Brd"), + item={"text": "W\u0275\u200brd"}, expected=None, encoding="windows-1251", ) def test_errors_xmlcharrefreplace(self): self.assertExportResult( - item=dict(text="W\u0275\u200Brd"), + item={"text": "W\u0275\u200brd"}, include_headers_line=False, expected="Wɵ​rd\r\n", encoding="windows-1251", @@ -401,12 +382,12 @@ class CsvItemExporterTest(BaseItemExporterTest): ) -class CsvItemExporterDataclassTest(CsvItemExporterTest): - item_class = TestDataClass +class TestCsvItemExporterDataclass(TestCsvItemExporter): + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class XmlItemExporterTest(BaseItemExporterTest): +class TestXmlItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): return XmlItemExporter(self.output, **kwargs) @@ -421,7 +402,7 @@ class XmlItemExporterTest(BaseItemExporterTest): doc = lxml.etree.fromstring(xmlcontent) return xmltuple(doc) - return self.assertEqual(xmlsplit(first), xmlsplit(second), msg) + assert xmlsplit(first) == xmlsplit(second), msg def assertExportResult(self, item, expected_value): fp = BytesIO() @@ -453,8 +434,8 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_item(self): - i1 = dict(name="foo\xa3hoo", age="22") - i2 = dict(name="bar", age=i1) + i1 = {"name": "foo\xa3hoo", "age": "22"} + i2 = {"name": "bar", "age": i1} i3 = self.item_class(name="buz", age=i2) self.assertExportResult( @@ -476,8 +457,8 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_list_item(self): - i1 = dict(name="foo") - i2 = dict(name="bar", v2={"egg": ["spam"]}) + i1 = {"name": "foo"} + i2 = {"name": "bar", "v2": {"egg": ["spam"]}} i3 = self.item_class(name="buz", age=[i1, i2]) self.assertExportResult( @@ -512,12 +493,12 @@ class XmlItemExporterTest(BaseItemExporterTest): ) -class XmlItemExporterDataclassTest(XmlItemExporterTest): - item_class = TestDataClass +class TestXmlItemExporterDataclass(TestXmlItemExporter): + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class JsonLinesItemExporterTest(BaseItemExporterTest): +class TestJsonLinesItemExporter(TestBaseItemExporter): _expected_nested: Any = { "name": "Jesus", "age": {"name": "Maria", "age": {"name": "Joseph", "age": "22"}}, @@ -528,24 +509,25 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): def _check_output(self): exported = json.loads(to_unicode(self.output.getvalue().strip())) - self.assertEqual(exported, ItemAdapter(self.i).asdict()) + assert exported == ItemAdapter(self.i).asdict() def test_nested_item(self): i1 = self.item_class(name="Joseph", age="22") - i2 = dict(name="Maria", age=i1) + i2 = {"name": "Maria", "age": i1} i3 = self.item_class(name="Jesus", age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() del self.ie # See the first “del self.ie†in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) - self.assertEqual(exported, self._expected_nested) + assert exported == self._expected_nested def test_extra_keywords(self): self.ie = self._get_exporter(sort_keys=True) self.test_export_item() self._check_output() - self.assertRaises(TypeError, self._get_exporter, foo_unknown_keyword_bar=True) + with pytest.raises(TypeError): + self._get_exporter(foo_unknown_keyword_bar=True) def test_nonstring_types_item(self): item = self._get_nonstring_types_item() @@ -555,23 +537,23 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): del self.ie # See the first “del self.ie†in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) item["time"] = str(item["time"]) - self.assertEqual(exported, item) + assert exported == item -class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest): - item_class = TestDataClass +class TestJsonLinesItemExporterDataclass(TestJsonLinesItemExporter): + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class JsonItemExporterTest(JsonLinesItemExporterTest): - _expected_nested = [JsonLinesItemExporterTest._expected_nested] +class TestJsonItemExporter(TestJsonLinesItemExporter): + _expected_nested = [TestJsonLinesItemExporter._expected_nested] def _get_exporter(self, **kwargs): return JsonItemExporter(self.output, **kwargs) def _check_output(self): exported = json.loads(to_unicode(self.output.getvalue().strip())) - self.assertEqual(exported, [ItemAdapter(self.i).asdict()]) + assert exported == [ItemAdapter(self.i).asdict()] def assertTwoItemsExported(self, item): self.ie.start_exporting() @@ -580,9 +562,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.finish_exporting() del self.ie # See the first “del self.ie†in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) - self.assertEqual( - exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()] - ) + assert exported == [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()] def test_two_items(self): self.assertTwoItemsExported(self.i) @@ -591,18 +571,19 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertTwoItemsExported(ItemAdapter(self.i).asdict()) def test_two_items_with_failure_between(self): - i1 = TestItem(name="Joseph\xa3", age="22") - i2 = TestItem( + i1 = MyItem(name="Joseph\xa3", age="22") + i2 = MyItem( name="Maria", age=1j ) # Invalid datetimes didn't consistently fail between Python versions - i3 = TestItem(name="Jesus", age="44") + i3 = MyItem(name="Jesus", age="44") self.ie.start_exporting() self.ie.export_item(i1) - self.assertRaises(TypeError, self.ie.export_item, i2) + with pytest.raises(TypeError): + self.ie.export_item(i2) self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - self.assertEqual(exported, [dict(i1), dict(i3)]) + assert exported == [dict(i1), dict(i3)] def test_nested_item(self): i1 = self.item_class(name="Joseph\xa3", age="22") @@ -617,19 +598,19 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): "name": "Jesus", "age": {"name": "Maria", "age": ItemAdapter(i1).asdict()}, } - self.assertEqual(exported, [expected]) + assert exported == [expected] def test_nested_dict_item(self): - i1 = dict(name="Joseph\xa3", age="22") + i1 = {"name": "Joseph\xa3", "age": "22"} i2 = self.item_class(name="Maria", age=i1) - i3 = dict(name="Jesus", age=i2) + i3 = {"name": "Jesus", "age": i2} self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() del self.ie # See the first “del self.ie†in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) expected = {"name": "Jesus", "age": {"name": "Maria", "age": i1}} - self.assertEqual(exported, [expected]) + assert exported == [expected] def test_nonstring_types_item(self): item = self._get_nonstring_types_item() @@ -639,38 +620,39 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): del self.ie # See the first “del self.ie†in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) item["time"] = str(item["time"]) - self.assertEqual(exported, [item]) + assert exported == [item] -class JsonItemExporterToBytesTest(BaseItemExporterTest): +class TestJsonItemExporterToBytes(TestBaseItemExporter): def _get_exporter(self, **kwargs): kwargs["encoding"] = "latin" return JsonItemExporter(self.output, **kwargs) def test_two_items_with_failure_between(self): - i1 = TestItem(name="Joseph", age="22") - i2 = TestItem(name="\u263a", age="11") - i3 = TestItem(name="Jesus", age="44") + i1 = MyItem(name="Joseph", age="22") + i2 = MyItem(name="\u263a", age="11") + i3 = MyItem(name="Jesus", age="44") self.ie.start_exporting() self.ie.export_item(i1) - self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2) + with pytest.raises(UnicodeEncodeError): + self.ie.export_item(i2) self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue(), encoding="latin")) - self.assertEqual(exported, [dict(i1), dict(i3)]) + assert exported == [dict(i1), dict(i3)] -class JsonItemExporterDataclassTest(JsonItemExporterTest): - item_class = TestDataClass +class TestJsonItemExporterDataclass(TestJsonItemExporter): + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class CustomExporterItemTest(unittest.TestCase): - item_class: type = TestItem +class TestCustomExporterItem: + item_class: type = MyItem - def setUp(self): + def setup_method(self): if self.item_class is None: - raise unittest.SkipTest("item class is None") + pytest.skip("item class is None") def test_exporter_custom_serializer(self): class CustomItemExporter(BaseItemExporter): @@ -679,25 +661,20 @@ class CustomExporterItemTest(unittest.TestCase): return str(int(value) + 1) return super().serialize_field(field, name, value) + def export_item(self, item: Any) -> None: + pass + i = self.item_class(name="John", age="22") a = ItemAdapter(i) ie = CustomItemExporter() - self.assertEqual( - ie.serialize_field(a.get_field_meta("name"), "name", a["name"]), "John" - ) - self.assertEqual( - ie.serialize_field(a.get_field_meta("age"), "age", a["age"]), "23" - ) + assert ie.serialize_field(a.get_field_meta("name"), "name", a["name"]) == "John" + assert ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) == "23" i2 = {"name": "John", "age": "22"} - self.assertEqual(ie.serialize_field({}, "name", i2["name"]), "John") - self.assertEqual(ie.serialize_field({}, "age", i2["age"]), "23") + assert ie.serialize_field({}, "name", i2["name"]) == "John" + assert ie.serialize_field({}, "age", i2["age"]) == "23" -class CustomExporterDataclassTest(CustomExporterItemTest): - item_class = TestDataClass - - -if __name__ == "__main__": - unittest.main() +class TestCustomExporterDataclass(TestCustomExporterItem): + item_class = MyDataClass diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index b7312bbcd..18782fb70 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -1,11 +1,17 @@ -import datetime -import typing -import unittest +from __future__ import annotations + +import datetime +from typing import TYPE_CHECKING, Any -from scrapy.crawler import Crawler from scrapy.extensions.periodic_log import PeriodicLog +from scrapy.utils.test import get_crawler from .spiders import MetaSpider +from .utils.decorators import coroutine_test + +if TYPE_CHECKING: + from collections.abc import Callable + stats_dump_1 = { "log_count/INFO": 10, @@ -51,21 +57,20 @@ stats_dump_2 = { } -class TestExtPeriodicLog(PeriodicLog): - def set_a(self): +class CustomPeriodicLog(PeriodicLog): + def set_a(self) -> None: self.stats._stats = stats_dump_1 - def set_b(self): + def set_b(self) -> None: self.stats._stats = stats_dump_2 -def extension(settings=None): - crawler = Crawler(MetaSpider, settings=settings) - crawler._apply_settings() - return TestExtPeriodicLog.from_crawler(crawler) +def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog: + crawler = get_crawler(MetaSpider, settings) + return CustomPeriodicLog.from_crawler(crawler) -class TestPeriodicLog(unittest.TestCase): +class TestPeriodicLog: def test_extension_enabled(self): # Expected that settings for this extension loaded successfully # And on certain conditions - extension raising NotConfigured @@ -82,19 +87,22 @@ class TestPeriodicLog(unittest.TestCase): assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60}) assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60}) - def test_log_delta(self): - def emulate(settings=None): + @coroutine_test + async def test_log_delta(self): + def emulate( + settings: dict[str, Any] | None = None, + ) -> tuple[PeriodicLog, dict[str, Any], dict[str, Any]]: spider = MetaSpider() ext = extension(settings) ext.spider_opened(spider) ext.set_a() a = ext.log_delta() - ext.set_a() + ext.set_b() b = ext.log_delta() ext.spider_closed(spider, reason="finished") return ext, a, b - def check(settings: dict, condition: typing.Callable): + def check(settings: dict[str, Any], condition: Callable) -> None: ext, a, b = emulate(settings) assert list(a["delta"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) @@ -115,8 +123,10 @@ class TestPeriodicLog(unittest.TestCase): # include multiple check( {"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}}, - lambda k, v: isinstance(v, (int, float)) - and ("downloader/" in k or "scheduler/" in k), + lambda k, v: ( + isinstance(v, (int, float)) + and ("downloader/" in k or "scheduler/" in k) + ), ) # exclude @@ -128,30 +138,37 @@ class TestPeriodicLog(unittest.TestCase): # exclude multiple check( {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/", "scheduler/"]}}, - lambda k, v: isinstance(v, (int, float)) - and ("downloader/" not in k and "scheduler/" not in k), + lambda k, v: ( + isinstance(v, (int, float)) + and ("downloader/" not in k and "scheduler/" not in k) + ), ) # include exclude combined check( {"PERIODIC_LOG_DELTA": {"include": ["downloader/"], "exclude": ["bytes"]}}, - lambda k, v: isinstance(v, (int, float)) - and ("downloader/" in k and "bytes" not in k), + lambda k, v: ( + isinstance(v, (int, float)) + and ("downloader/" in k and "bytes" not in k) + ), ) - def test_log_stats(self): - def emulate(settings=None): + @coroutine_test + async def test_log_stats(self): + def emulate( + settings: dict[str, Any] | None = None, + ) -> tuple[PeriodicLog, dict[str, Any], dict[str, Any]]: spider = MetaSpider() ext = extension(settings) ext.spider_opened(spider) ext.set_a() a = ext.log_crawler_stats() - ext.set_a() + ext.set_b() b = ext.log_crawler_stats() ext.spider_closed(spider, reason="finished") return ext, a, b - def check(settings: dict, condition: typing.Callable): + def check(settings: dict[str, Any], condition: Callable) -> None: ext, a, b = emulate(settings) assert list(a["stats"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) @@ -192,4 +209,3 @@ class TestPeriodicLog(unittest.TestCase): {"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}}, lambda k, v: "downloader/" in k and "bytes" not in k, ) - # diff --git a/tests/test_extension_statsmailer.py b/tests/test_extension_statsmailer.py new file mode 100644 index 000000000..28db389c3 --- /dev/null +++ b/tests/test_extension_statsmailer.py @@ -0,0 +1,88 @@ +from unittest.mock import MagicMock + +import pytest + +from scrapy import signals +from scrapy.exceptions import NotConfigured +from scrapy.signalmanager import SignalManager +from scrapy.statscollectors import StatsCollector +from scrapy.utils.spider import DefaultSpider + +pytestmark = pytest.mark.filterwarnings( + "ignore:The scrapy.extensions.statsmailer module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning", + "ignore:The scrapy.mail module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning", +) + +from scrapy.extensions import statsmailer # noqa: E402 +from scrapy.mail import MailSender # noqa: E402 + + +@pytest.fixture +def dummy_stats(): + class DummyStats(StatsCollector): + def __init__(self): + # pylint: disable=super-init-not-called + self._stats = {"global_item_scraped_count": 42} + + def get_stats(self): + return {"item_scraped_count": 10, **self._stats} + + return DummyStats() + + +def test_from_crawler_without_recipients_raises_notconfigured(): + crawler = MagicMock() + crawler.settings.getlist.return_value = [] + crawler.stats = MagicMock() + + with pytest.raises(NotConfigured): + statsmailer.StatsMailer.from_crawler(crawler) + + +def test_from_crawler_with_recipients_initializes_extension(dummy_stats, monkeypatch): + crawler = MagicMock() + crawler.settings.getlist.return_value = ["test@example.com"] + crawler.stats = dummy_stats + crawler.signals = SignalManager(crawler) + + mailer = MagicMock(spec=MailSender) + monkeypatch.setattr(statsmailer.MailSender, "from_crawler", lambda _: mailer) + + ext = statsmailer.StatsMailer.from_crawler(crawler) + + assert isinstance(ext, statsmailer.StatsMailer) + assert ext.recipients == ["test@example.com"] + assert ext.mail is mailer + + +def test_from_crawler_connects_spider_closed_signal(dummy_stats, monkeypatch): + crawler = MagicMock() + crawler.settings.getlist.return_value = ["test@example.com"] + crawler.stats = dummy_stats + crawler.signals = SignalManager(crawler) + + mailer = MagicMock(spec=MailSender) + monkeypatch.setattr(statsmailer.MailSender, "from_crawler", lambda _: mailer) + + statsmailer.StatsMailer.from_crawler(crawler) + + connected = crawler.signals.send_catch_log( + signals.spider_closed, spider=DefaultSpider(name="dummy") + ) + assert connected is not None + + +def test_spider_closed_sends_email(dummy_stats): + recipients = ["test@example.com"] + mail = MagicMock(spec=MailSender) + ext = statsmailer.StatsMailer(dummy_stats, recipients, mail) + + spider = DefaultSpider(name="dummy") + ext.spider_closed(spider) + + args, _ = mail.send.call_args + to, subject, body = args + assert to == recipients + assert "Scrapy stats for: dummy" in subject + assert "global_item_scraped_count" in body + assert "item_scraped_count" in body diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 9fd680e9f..f1c86ce62 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -1,19 +1,21 @@ +import pytest from twisted.conch.telnet import ITelnetProtocol from twisted.cred import credentials -from twisted.internet import defer -from twisted.trial import unittest from scrapy.extensions.telnet import TelnetConsole from scrapy.utils.test import get_crawler +from tests.utils.decorators import inline_callbacks_test + +pytestmark = pytest.mark.requires_reactor # TelnetConsole requires a reactor -class TelnetExtensionTest(unittest.TestCase): +class TestTelnetExtension: def _get_console_and_portal(self, settings=None): crawler = get_crawler(settings_dict=settings) console = TelnetConsole(crawler) # This function has some side effects we don't need for this test - console._get_telnet_vars = lambda: {} + console._get_telnet_vars = dict console.start_listening() protocol = console.protocol() @@ -21,15 +23,16 @@ class TelnetExtensionTest(unittest.TestCase): return console, portal - @defer.inlineCallbacks + @inline_callbacks_test def test_bad_credentials(self): console, portal = self._get_console_and_portal() creds = credentials.UsernamePassword(b"username", b"password") d = portal.login(creds, None, ITelnetProtocol) - yield self.assertFailure(d, ValueError) + with pytest.raises(ValueError, match="Invalid credentials"): + yield d console.stop_listening() - @defer.inlineCallbacks + @inline_callbacks_test def test_good_credentials(self): console, portal = self._get_console_and_portal() creds = credentials.UsernamePassword( @@ -39,7 +42,7 @@ class TelnetExtensionTest(unittest.TestCase): yield d console.stop_listening() - @defer.inlineCallbacks + @inline_callbacks_test def test_custom_credentials(self): settings = { "TELNETCONSOLE_USERNAME": "user", diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py new file mode 100644 index 000000000..4874f284a --- /dev/null +++ b/tests/test_extension_throttle.py @@ -0,0 +1,345 @@ +from logging import INFO +from unittest.mock import Mock + +import pytest + +from scrapy import Request, Spider +from scrapy.exceptions import NotConfigured +from scrapy.extensions.throttle import AutoThrottle +from scrapy.http.response import Response +from scrapy.settings.default_settings import ( + AUTOTHROTTLE_MAX_DELAY, + AUTOTHROTTLE_START_DELAY, + DOWNLOAD_DELAY, +) +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.test import get_crawler as _get_crawler + +UNSET = object() + + +def get_crawler(settings=None, spidercls=None): + settings = settings or {} + settings["AUTOTHROTTLE_ENABLED"] = True + return _get_crawler(settings_dict=settings, spidercls=spidercls) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (UNSET, False), + (False, False), + (True, True), + ], +) +def test_enabled(value, expected): + settings = {} + if value is not UNSET: + settings["AUTOTHROTTLE_ENABLED"] = value + crawler = _get_crawler(settings_dict=settings) + if expected: + build_from_crawler(AutoThrottle, crawler) + else: + with pytest.raises(NotConfigured): + build_from_crawler(AutoThrottle, crawler) + + +@pytest.mark.parametrize( + "value", + [ + 0.0, + -1.0, + ], +) +def test_target_concurrency_invalid(value): + settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": value} + crawler = get_crawler(settings) + with pytest.raises(NotConfigured): + build_from_crawler(AutoThrottle, crawler) + + +@pytest.mark.parametrize( + ("spider", "setting", "expected"), + [ + (UNSET, UNSET, DOWNLOAD_DELAY), + (1.0, UNSET, 1.0), + (UNSET, 1.0, 1.0), + (1.0, 2.0, 1.0), + (3.0, 2.0, 3.0), + ], +) +def test_mindelay_definition(spider, setting, expected): + settings = {} + if setting is not UNSET: + settings["DOWNLOAD_DELAY"] = setting + + class _TestSpider(Spider): + name = "test" + + if spider is not UNSET: + _TestSpider.download_delay = spider + + crawler = get_crawler(settings, _TestSpider) + at = build_from_crawler(AutoThrottle, crawler) + at._spider_opened(_TestSpider()) + assert at.mindelay == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (UNSET, AUTOTHROTTLE_MAX_DELAY), + (1.0, 1.0), + ], +) +def test_maxdelay_definition(value, expected): + settings = {} + if value is not UNSET: + settings["AUTOTHROTTLE_MAX_DELAY"] = value + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + at._spider_opened(DefaultSpider()) + assert at.maxdelay == expected + + +@pytest.mark.parametrize( + ("min_spider", "min_setting", "start_setting", "expected"), + [ + (UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), + (UNSET, AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY), + (UNSET, AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), + (UNSET, UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0), + (UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 1.0), + ( + AUTOTHROTTLE_START_DELAY + 1.0, + AUTOTHROTTLE_START_DELAY + 2.0, + UNSET, + AUTOTHROTTLE_START_DELAY + 1.0, + ), + ( + AUTOTHROTTLE_START_DELAY + 2.0, + UNSET, + AUTOTHROTTLE_START_DELAY + 1.0, + AUTOTHROTTLE_START_DELAY + 2.0, + ), + ( + AUTOTHROTTLE_START_DELAY + 1.0, + UNSET, + AUTOTHROTTLE_START_DELAY + 2.0, + AUTOTHROTTLE_START_DELAY + 2.0, + ), + ], +) +def test_startdelay_definition(min_spider, min_setting, start_setting, expected): + settings = {} + if min_setting is not UNSET: + settings["DOWNLOAD_DELAY"] = min_setting + if start_setting is not UNSET: + settings["AUTOTHROTTLE_START_DELAY"] = start_setting + + class _TestSpider(Spider): + name = "test" + + if min_spider is not UNSET: + _TestSpider.download_delay = min_spider + + crawler = get_crawler(settings, _TestSpider) + at = build_from_crawler(AutoThrottle, crawler) + spider = _TestSpider() + at._spider_opened(spider) + assert spider.download_delay == expected + + +@pytest.mark.parametrize( + ("meta", "slot"), + [ + ({}, None), + ({"download_latency": 1.0}, None), + ({"download_slot": "foo"}, None), + ({"download_slot": "foo"}, "foo"), + ({"download_latency": 1.0, "download_slot": "foo"}, None), + ( + { + "download_latency": 1.0, + "download_slot": "foo", + "autothrottle_dont_adjust_delay": True, + }, + "foo", + ), + ], +) +def test_skipped(meta, slot): + crawler = get_crawler() + at = build_from_crawler(AutoThrottle, crawler) + spider = DefaultSpider() + at._spider_opened(spider) + request = Request("https://example.com", meta=meta) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + if slot is not None: + crawler.engine.downloader.slots[slot] = object() + at._adjust_delay = None # Raise exception if called. + + at._response_downloaded(None, request, spider) + + +@pytest.mark.parametrize( + ("download_latency", "target_concurrency", "slot_delay", "expected"), + [ + (2.0, 2.0, 1.0, 1.0), + (1.0, 2.0, 1.0, 0.75), + (4.0, 2.0, 1.0, 2.0), + (2.0, 1.0, 1.0, 2.0), + (2.0, 4.0, 1.0, 0.75), + (2.0, 2.0, 0.5, 1.0), + (2.0, 2.0, 2.0, 1.5), + ], +) +def test_adjustment(download_latency, target_concurrency, slot_delay, expected): + settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = DefaultSpider() + at._spider_opened(spider) + meta = {"download_latency": download_latency, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = slot_delay + crawler.engine.downloader.slots["foo"] = slot + + at._response_downloaded(response, request, spider) + + assert slot.delay == expected, f"{slot.delay} != {expected}" + + +@pytest.mark.parametrize( + ("mindelay", "maxdelay", "expected"), + [ + (0.5, 2.0, 1.0), + (0.25, 0.5, 0.5), + (2.0, 4.0, 2.0), + ], +) +def test_adjustment_limits(mindelay, maxdelay, expected): + download_latency, target_concurrency, slot_delay = (2.0, 2.0, 1.0) + # expected adjustment without limits with these values: 1.0 + settings = { + "AUTOTHROTTLE_MAX_DELAY": maxdelay, + "AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency, + "DOWNLOAD_DELAY": mindelay, + } + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = DefaultSpider() + at._spider_opened(spider) + meta = {"download_latency": download_latency, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = slot_delay + crawler.engine.downloader.slots["foo"] = slot + + at._response_downloaded(response, request, spider) + + assert slot.delay == expected, f"{slot.delay} != {expected}" + + +@pytest.mark.parametrize( + ("download_latency", "target_concurrency", "slot_delay", "expected"), + [ + (2.0, 2.0, 1.0, 1.0), + (1.0, 2.0, 1.0, 1.0), # Instead of 0.75 + (4.0, 2.0, 1.0, 2.0), + ], +) +def test_adjustment_bad_response( + download_latency, target_concurrency, slot_delay, expected +): + settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = DefaultSpider() + at._spider_opened(spider) + meta = {"download_latency": download_latency, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url, status=400) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = slot_delay + crawler.engine.downloader.slots["foo"] = slot + + at._response_downloaded(response, request, spider) + + assert slot.delay == expected, f"{slot.delay} != {expected}" + + +def test_debug(caplog): + settings = {"AUTOTHROTTLE_DEBUG": True} + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = DefaultSpider() + at._spider_opened(spider) + meta = {"download_latency": 1.0, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url, body=b"foo") + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = 2.0 + slot.transferring = (None, None) + crawler.engine.downloader.slots["foo"] = slot + + caplog.clear() + with caplog.at_level(INFO): + at._response_downloaded(response, request, spider) + + assert caplog.record_tuples == [ + ( + "scrapy.extensions.throttle", + INFO, + "slot: foo | conc: 2 | delay: 1500 ms (-500) | latency: 1000 ms | size: 3 bytes", + ), + ] + + +def test_debug_disabled(caplog): + crawler = get_crawler() + at = build_from_crawler(AutoThrottle, crawler) + spider = DefaultSpider() + at._spider_opened(spider) + meta = {"download_latency": 1.0, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url, body=b"foo") + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = 2.0 + slot.transferring = (None, None) + crawler.engine.downloader.slots["foo"] = slot + + caplog.clear() + with caplog.at_level(INFO): + at._response_downloaded(response, request, spider) + + assert caplog.record_tuples == [] diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 277555608..27e0c6445 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,585 +1,55 @@ -import bz2 +from __future__ import annotations + import csv -import gzip import json -import lzma +import marshal +import pickle import random import shutil -import string -import sys import tempfile -import warnings from abc import ABC, abstractmethod -from collections import defaultdict -from contextlib import ExitStack -from io import BytesIO from logging import getLogger -from os import PathLike from pathlib import Path from string import ascii_letters, digits -from typing import Union +from typing import IO, TYPE_CHECKING, Any from unittest import mock -from urllib.parse import quote, urljoin +from urllib.parse import urljoin from urllib.request import pathname2url import lxml.etree import pytest from testfixtures import LogCapture -from twisted.internet import defer -from twisted.trial import unittest -from w3lib.url import file_uri_to_path, path_to_file_uri +from w3lib.url import file_uri_to_path from zope.interface import implementer -from zope.interface.verify import verifyObject import scrapy -from scrapy import signals -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy import Spider, signals +from scrapy.exceptions import NotConfigured from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.extensions.feedexport import ( - IS_BOTO3_AVAILABLE, BlockingFeedStorage, FeedExporter, FeedSlot, FileFeedStorage, - FTPFeedStorage, - GCSFeedStorage, IFeedStorage, S3FeedStorage, - StdoutFeedStorage, ) -from scrapy.settings import Settings from scrapy.utils.python import to_unicode -from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no_boto -from tests.mockserver import MockFTPServer, MockServer +from scrapy.utils.test import get_crawler +from tests.mockserver.http import MockServer from tests.spiders import ItemSpider +from tests.utils.decorators import coroutine_test, inline_callbacks_test + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable, Iterable -def path_to_url(path): +def path_to_url(path: str | Path) -> str: return urljoin("file:", pathname2url(str(path))) -def printf_escape(string): - return string.replace("%", "%%") - - -def build_url(path: Union[str, PathLike]) -> str: - path_str = str(path) - if path_str[0] != "/": - path_str = "/" + path_str - return urljoin("file:", path_str) - - -class FileFeedStorageTest(unittest.TestCase): - def test_store_file_uri(self): - path = Path(self.mktemp()).resolve() - uri = path_to_file_uri(str(path)) - return self._assert_stores(FileFeedStorage(uri), path) - - def test_store_file_uri_makedirs(self): - path = Path(self.mktemp()).resolve() / "more" / "paths" / "file.txt" - uri = path_to_file_uri(str(path)) - return self._assert_stores(FileFeedStorage(uri), path) - - def test_store_direct_path(self): - path = Path(self.mktemp()).resolve() - return self._assert_stores(FileFeedStorage(str(path)), path) - - def test_store_direct_path_relative(self): - path = Path(self.mktemp()) - return self._assert_stores(FileFeedStorage(str(path)), path) - - def test_interface(self): - path = self.mktemp() - st = FileFeedStorage(path) - verifyObject(IFeedStorage, st) - - def _store(self, feed_options=None) -> Path: - path = Path(self.mktemp()).resolve() - storage = FileFeedStorage(str(path), feed_options=feed_options) - spider = scrapy.Spider("default") - file = storage.open(spider) - file.write(b"content") - storage.store(file) - return path - - def test_append(self): - path = self._store() - return self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent") - - def test_overwrite(self): - path = self._store({"overwrite": True}) - return self._assert_stores( - FileFeedStorage(str(path), feed_options={"overwrite": True}), path - ) - - @defer.inlineCallbacks - def _assert_stores(self, storage, path: Path, expected_content=b"content"): - spider = scrapy.Spider("default") - file = storage.open(spider) - file.write(b"content") - yield storage.store(file) - self.assertTrue(path.exists()) - try: - self.assertEqual(path.read_bytes(), expected_content) - finally: - path.unlink() - - -class FTPFeedStorageTest(unittest.TestCase): - def get_test_spider(self, settings=None): - class TestSpider(scrapy.Spider): - name = "test_spider" - - crawler = get_crawler(settings_dict=settings) - spider = TestSpider.from_crawler(crawler) - return spider - - def _store(self, uri, content, feed_options=None, settings=None): - crawler = get_crawler(settings_dict=settings or {}) - storage = FTPFeedStorage.from_crawler( - crawler, - uri, - feed_options=feed_options, - ) - verifyObject(IFeedStorage, storage) - spider = self.get_test_spider() - file = storage.open(spider) - file.write(content) - return storage.store(file) - - def _assert_stored(self, path: Path, content): - self.assertTrue(path.exists()) - try: - self.assertEqual(path.read_bytes(), content) - finally: - path.unlink() - - @defer.inlineCallbacks - def test_append(self): - with MockFTPServer() as ftp_server: - filename = "file" - url = ftp_server.url(filename) - feed_options = {"overwrite": False} - yield self._store(url, b"foo", feed_options=feed_options) - yield self._store(url, b"bar", feed_options=feed_options) - self._assert_stored(ftp_server.path / filename, b"foobar") - - @defer.inlineCallbacks - def test_overwrite(self): - with MockFTPServer() as ftp_server: - filename = "file" - url = ftp_server.url(filename) - yield self._store(url, b"foo") - yield self._store(url, b"bar") - self._assert_stored(ftp_server.path / filename, b"bar") - - @defer.inlineCallbacks - def test_append_active_mode(self): - with MockFTPServer() as ftp_server: - settings = {"FEED_STORAGE_FTP_ACTIVE": True} - filename = "file" - url = ftp_server.url(filename) - feed_options = {"overwrite": False} - yield self._store(url, b"foo", feed_options=feed_options, settings=settings) - yield self._store(url, b"bar", feed_options=feed_options, settings=settings) - self._assert_stored(ftp_server.path / filename, b"foobar") - - @defer.inlineCallbacks - def test_overwrite_active_mode(self): - with MockFTPServer() as ftp_server: - settings = {"FEED_STORAGE_FTP_ACTIVE": True} - filename = "file" - url = ftp_server.url(filename) - yield self._store(url, b"foo", settings=settings) - yield self._store(url, b"bar", settings=settings) - self._assert_stored(ftp_server.path / filename, b"bar") - - def test_uri_auth_quote(self): - # RFC3986: 3.2.1. User Information - pw_quoted = quote(string.punctuation, safe="") - st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {}) - self.assertEqual(st.password, string.punctuation) - - -class BlockingFeedStorageTest(unittest.TestCase): - def get_test_spider(self, settings=None): - class TestSpider(scrapy.Spider): - name = "test_spider" - - crawler = get_crawler(settings_dict=settings) - spider = TestSpider.from_crawler(crawler) - return spider - - def test_default_temp_dir(self): - b = BlockingFeedStorage() - - tmp = b.open(self.get_test_spider()) - tmp_path = Path(tmp.name).parent - self.assertEqual(str(tmp_path), tempfile.gettempdir()) - - def test_temp_file(self): - b = BlockingFeedStorage() - - tests_path = Path(__file__).resolve().parent - spider = self.get_test_spider({"FEED_TEMPDIR": str(tests_path)}) - tmp = b.open(spider) - tmp_path = Path(tmp.name).parent - self.assertEqual(tmp_path, tests_path) - - def test_invalid_folder(self): - b = BlockingFeedStorage() - - tests_path = Path(__file__).resolve().parent - invalid_path = tests_path / "invalid_path" - spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)}) - - self.assertRaises(OSError, b.open, spider=spider) - - -class S3FeedStorageTest(unittest.TestCase): - def setUp(self): - skip_if_no_boto() - - def test_parse_credentials(self): - aws_credentials = { - "AWS_ACCESS_KEY_ID": "settings_key", - "AWS_SECRET_ACCESS_KEY": "settings_secret", - "AWS_SESSION_TOKEN": "settings_token", - } - crawler = get_crawler(settings_dict=aws_credentials) - # Instantiate with crawler - storage = S3FeedStorage.from_crawler( - crawler, - "s3://mybucket/export.csv", - ) - self.assertEqual(storage.access_key, "settings_key") - self.assertEqual(storage.secret_key, "settings_secret") - self.assertEqual(storage.session_token, "settings_token") - # Instantiate directly - storage = S3FeedStorage( - "s3://mybucket/export.csv", - aws_credentials["AWS_ACCESS_KEY_ID"], - aws_credentials["AWS_SECRET_ACCESS_KEY"], - session_token=aws_credentials["AWS_SESSION_TOKEN"], - ) - self.assertEqual(storage.access_key, "settings_key") - self.assertEqual(storage.secret_key, "settings_secret") - self.assertEqual(storage.session_token, "settings_token") - # URI priority > settings priority - storage = S3FeedStorage( - "s3://uri_key:uri_secret@mybucket/export.csv", - aws_credentials["AWS_ACCESS_KEY_ID"], - aws_credentials["AWS_SECRET_ACCESS_KEY"], - ) - self.assertEqual(storage.access_key, "uri_key") - self.assertEqual(storage.secret_key, "uri_secret") - - @defer.inlineCallbacks - def test_store(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - } - crawler = get_crawler(settings_dict=settings) - bucket = "mybucket" - key = "export.csv" - storage = S3FeedStorage.from_crawler(crawler, f"s3://{bucket}/{key}") - verifyObject(IFeedStorage, storage) - - file = mock.MagicMock() - - if IS_BOTO3_AVAILABLE: - storage.s3_client = mock.MagicMock() - yield storage.store(file) - self.assertEqual( - storage.s3_client.upload_fileobj.call_args, - mock.call(Bucket=bucket, Key=key, Fileobj=file), - ) - else: - from botocore.stub import Stubber - - with Stubber(storage.s3_client) as stub: - stub.add_response( - "put_object", - expected_params={ - "Body": file, - "Bucket": bucket, - "Key": key, - }, - service_response={}, - ) - - yield storage.store(file) - - stub.assert_no_pending_responses() - self.assertEqual( - file.method_calls, - [ - mock.call.seek(0), - # The call to read does not happen with Stubber - mock.call.close(), - ], - ) - - def test_init_without_acl(self): - storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, None) - - def test_init_with_acl(self): - storage = S3FeedStorage( - "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" - ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, "custom-acl") - - def test_init_with_endpoint_url(self): - storage = S3FeedStorage( - "s3://mybucket/export.csv", - "access_key", - "secret_key", - endpoint_url="https://example.com", - ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.endpoint_url, "https://example.com") - - def test_init_with_region_name(self): - region_name = "ap-east-1" - storage = S3FeedStorage( - "s3://mybucket/export.csv", - "access_key", - "secret_key", - region_name=region_name, - ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.region_name, region_name) - self.assertEqual(storage.s3_client._client_config.region_name, region_name) - - def test_from_crawler_without_acl(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler( - crawler, - "s3://mybucket/export.csv", - ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, None) - - def test_without_endpoint_url(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler( - crawler, - "s3://mybucket/export.csv", - ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.endpoint_url, None) - - def test_without_region_name(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler( - crawler, - "s3://mybucket/export.csv", - ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.s3_client._client_config.region_name, "us-east-1") - - def test_from_crawler_with_acl(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - "FEED_STORAGE_S3_ACL": "custom-acl", - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler( - crawler, - "s3://mybucket/export.csv", - ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, "custom-acl") - - def test_from_crawler_with_endpoint_url(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - "AWS_ENDPOINT_URL": "https://example.com", - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.endpoint_url, "https://example.com") - - def test_from_crawler_with_region_name(self): - region_name = "ap-east-1" - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - "AWS_REGION_NAME": region_name, - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.region_name, region_name) - self.assertEqual(storage.s3_client._client_config.region_name, region_name) - - @defer.inlineCallbacks - def test_store_without_acl(self): - storage = S3FeedStorage( - "s3://mybucket/export.csv", - "access_key", - "secret_key", - ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, None) - - storage.s3_client = mock.MagicMock() - yield storage.store(BytesIO(b"test file")) - if IS_BOTO3_AVAILABLE: - acl = ( - storage.s3_client.upload_fileobj.call_args[1] - .get("ExtraArgs", {}) - .get("ACL") - ) - else: - acl = storage.s3_client.put_object.call_args[1].get("ACL") - self.assertIsNone(acl) - - @defer.inlineCallbacks - def test_store_with_acl(self): - storage = S3FeedStorage( - "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" - ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, "custom-acl") - - storage.s3_client = mock.MagicMock() - yield storage.store(BytesIO(b"test file")) - if IS_BOTO3_AVAILABLE: - acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] - else: - acl = storage.s3_client.put_object.call_args[1]["ACL"] - self.assertEqual(acl, "custom-acl") - - def test_overwrite_default(self): - with LogCapture() as log: - S3FeedStorage( - "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" - ) - self.assertNotIn("S3 does not support appending to files", str(log)) - - def test_overwrite_false(self): - with LogCapture() as log: - S3FeedStorage( - "s3://mybucket/export.csv", - "access_key", - "secret_key", - "custom-acl", - feed_options={"overwrite": False}, - ) - self.assertIn("S3 does not support appending to files", str(log)) - - -class GCSFeedStorageTest(unittest.TestCase): - def test_parse_settings(self): - try: - from google.cloud.storage import Client # noqa - except ImportError: - raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") - - settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"} - crawler = get_crawler(settings_dict=settings) - storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") - assert storage.project_id == "123" - assert storage.acl == "publicRead" - assert storage.bucket_name == "mybucket" - assert storage.blob_name == "export.csv" - - def test_parse_empty_acl(self): - try: - from google.cloud.storage import Client # noqa - except ImportError: - raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") - - settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""} - crawler = get_crawler(settings_dict=settings) - storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") - assert storage.acl is None - - settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": None} - crawler = get_crawler(settings_dict=settings) - storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") - assert storage.acl is None - - @defer.inlineCallbacks - def test_store(self): - try: - from google.cloud.storage import Client # noqa - except ImportError: - raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") - - uri = "gs://mybucket/export.csv" - project_id = "myproject-123" - acl = "publicRead" - (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() - with mock.patch("google.cloud.storage.Client") as m: - m.return_value = client_mock - - f = mock.Mock() - storage = GCSFeedStorage(uri, project_id, acl) - yield storage.store(f) - - f.seek.assert_called_once_with(0) - m.assert_called_once_with(project=project_id) - client_mock.get_bucket.assert_called_once_with("mybucket") - bucket_mock.blob.assert_called_once_with("export.csv") - blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) - - -class StdoutFeedStorageTest(unittest.TestCase): - @defer.inlineCallbacks - def test_store(self): - out = BytesIO() - storage = StdoutFeedStorage("stdout:", _stdout=out) - file = storage.open(scrapy.Spider("default")) - file.write(b"content") - yield storage.store(file) - self.assertEqual(out.getvalue(), b"content") - - def test_overwrite_default(self): - with LogCapture() as log: - StdoutFeedStorage("stdout:") - self.assertNotIn( - "Standard output (stdout) storage does not support overwriting", str(log) - ) - - def test_overwrite_true(self): - with LogCapture() as log: - StdoutFeedStorage("stdout:", feed_options={"overwrite": True}) - self.assertIn( - "Standard output (stdout) storage does not support overwriting", str(log) - ) +def printf_escape(s: str) -> str: + return s.replace("%", "%%") class FromCrawlerMixin: @@ -640,8 +110,8 @@ class LogOnStoreFileStorage: file.close() -class FeedExportTestBase(ABC, unittest.TestCase): - __test__ = False +class TestFeedExportBase(ABC): + mockserver: MockServer class MyItem(scrapy.Item): foo = scrapy.Field() @@ -657,14 +127,24 @@ class FeedExportTestBase(ABC, unittest.TestCase): filename = "".join(chars) return Path(self.temp_dir, inter_dir, filename) - def setUp(self): + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) + + def setup_method(self): self.temp_dir = tempfile.mkdtemp() - def tearDown(self): + def teardown_method(self): shutil.rmtree(self.temp_dir, ignore_errors=True) - @defer.inlineCallbacks - def exported_data(self, items, settings): + async def exported_data( + self, items: Iterable[Any], settings: dict[str, Any] + ) -> dict[str, Any]: """ Return exported data which a spider yielding ``items`` would return. """ @@ -675,11 +155,9 @@ class FeedExportTestBase(ABC, unittest.TestCase): def parse(self, response): yield from items - data = yield self.run_and_export(TestSpider, settings) - return data + return await self.run_and_export(TestSpider, settings) - @defer.inlineCallbacks - def exported_no_data(self, settings): + async def exported_no_data(self, settings: dict[str, Any]) -> dict[str, Any]: """ Return exported data which a spider yielding no ``items`` would return. """ @@ -690,24 +168,81 @@ class FeedExportTestBase(ABC, unittest.TestCase): def parse(self, response): pass - data = yield self.run_and_export(TestSpider, settings) - return data + return await self.run_and_export(TestSpider, settings) - @defer.inlineCallbacks - def assertExported(self, items, header, rows, settings=None): - yield self.assertExportedCsv(items, header, rows, settings) - yield self.assertExportedJsonLines(items, rows, settings) - yield self.assertExportedXml(items, rows, settings) - yield self.assertExportedPickle(items, rows, settings) - yield self.assertExportedMarshal(items, rows, settings) - yield self.assertExportedMultiple(items, rows, settings) + async def assertExported( + self, + items: Iterable[Any], + header: Iterable[str], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + await self.assertExportedCsv(items, header, rows, settings) + await self.assertExportedJsonLines(items, rows, settings) + await self.assertExportedXml(items, rows, settings) + await self.assertExportedPickle(items, rows, settings) + await self.assertExportedMarshal(items, rows, settings) + await self.assertExportedMultiple(items, rows, settings) - @abstractmethod - def run_and_export(self, spider_cls, settings): + async def assertExportedCsv( # noqa: B027 + self, + items: Iterable[Any], + header: Iterable[str], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: pass - def _load_until_eof(self, data, load_func): - result = [] + async def assertExportedJsonLines( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedXml( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedMultiple( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedPickle( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedMarshal( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + @abstractmethod + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, Any]: + pass + + def _load_until_eof( + self, data: bytes, load_func: Callable[[IO[bytes]], Any] + ) -> list[Any]: + result: list[Any] = [] with tempfile.TemporaryFile() as temp: temp.write(data) temp.seek(0) @@ -753,21 +288,20 @@ class IsExportingListener: if self.start_without_finish: self.start_without_finish = False else: - self.finish_before_start = True + self.finish_without_start = True class ExceptionJsonItemExporter(JsonItemExporter): """JsonItemExporter that throws an exception every time export_item is called.""" def export_item(self, _): - raise Exception("foo") + raise RuntimeError("foo") -class FeedExportTest(FeedExportTestBase): - __test__ = True - - @defer.inlineCallbacks - def run_and_export(self, spider_cls, settings): +class TestFeedExport(TestFeedExportBase): + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, Any]: """Run spider with specified settings; return exported data.""" FEEDS = settings.get("FEEDS") or {} @@ -776,12 +310,11 @@ class FeedExportTest(FeedExportTestBase): for file_path, feed_options in FEEDS.items() } - content = {} + content: dict[str, Any] = {} try: - with MockServer() as s: - spider_cls.start_urls = [s.url("/")] - crawler = get_crawler(spider_cls, settings) - yield crawler.crawl() + spider_cls.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(spider_cls, settings) + await crawler.crawl_async() for file_path, feed_options in FEEDS.items(): content[feed_options["format"]] = ( @@ -789,7 +322,7 @@ class FeedExportTest(FeedExportTestBase): ) finally: - for file_path in FEEDS.keys(): + for file_path in FEEDS: if not Path(file_path).exists(): continue @@ -797,8 +330,13 @@ class FeedExportTest(FeedExportTestBase): return content - @defer.inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None): + async def assertExportedCsv( + self, + items: Iterable[Any], + header: Iterable[str], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -807,13 +345,17 @@ class FeedExportTest(FeedExportTestBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) reader = csv.DictReader(to_unicode(data["csv"]).splitlines()) - self.assertEqual(reader.fieldnames, list(header)) - self.assertEqual(rows, list(reader)) + assert reader.fieldnames == list(header) + assert rows == list(reader) - @defer.inlineCallbacks - def assertExportedJsonLines(self, items, rows, settings=None): + async def assertExportedJsonLines( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -822,13 +364,17 @@ class FeedExportTest(FeedExportTestBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) parsed = [json.loads(to_unicode(line)) for line in data["jl"].splitlines()] rows = [{k: v for k, v in row.items() if v} for row in rows] - self.assertEqual(rows, parsed) + assert rows == parsed - @defer.inlineCallbacks - def assertExportedXml(self, items, rows, settings=None): + async def assertExportedXml( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -837,14 +383,18 @@ class FeedExportTest(FeedExportTestBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) rows = [{k: v for k, v in row.items() if v} for row in rows] root = lxml.etree.fromstring(data["xml"]) got_rows = [{e.tag: e.text for e in it} for it in root.findall("item")] - self.assertEqual(rows, got_rows) + assert rows == got_rows - @defer.inlineCallbacks - def assertExportedMultiple(self, items, rows, settings=None): + async def assertExportedMultiple( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -854,18 +404,22 @@ class FeedExportTest(FeedExportTestBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) rows = [{k: v for k, v in row.items() if v} for row in rows] # XML root = lxml.etree.fromstring(data["xml"]) xml_rows = [{e.tag: e.text for e in it} for it in root.findall("item")] - self.assertEqual(rows, xml_rows) + assert rows == xml_rows # JSON json_rows = json.loads(to_unicode(data["json"])) - self.assertEqual(rows, json_rows) + assert rows == json_rows - @defer.inlineCallbacks - def assertExportedPickle(self, items, rows, settings=None): + async def assertExportedPickle( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -874,15 +428,18 @@ class FeedExportTest(FeedExportTestBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) expected = [{k: v for k, v in row.items() if v} for row in rows] - import pickle result = self._load_until_eof(data["pickle"], load_func=pickle.load) - self.assertEqual(expected, result) + assert result == expected - @defer.inlineCallbacks - def assertExportedMarshal(self, items, rows, settings=None): + async def assertExportedMarshal( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -891,14 +448,13 @@ class FeedExportTest(FeedExportTestBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) expected = [{k: v for k, v in row.items() if v} for row in rows] - import marshal result = self._load_until_eof(data["marshal"], load_func=marshal.load) - self.assertEqual(expected, result) + assert result == expected - @defer.inlineCallbacks + @inline_callbacks_test def test_stats_file_success(self): settings = { "FEEDS": { @@ -908,16 +464,11 @@ class FeedExportTest(FeedExportTestBase): }, } crawler = get_crawler(ItemSpider, settings) - with MockServer() as mockserver: - yield crawler.crawl(mockserver=mockserver) - self.assertIn( - "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() - ) - self.assertEqual( - crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1 - ) + yield crawler.crawl(mockserver=self.mockserver) + assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() + assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 - @defer.inlineCallbacks + @inline_callbacks_test def test_stats_file_failed(self): settings = { "FEEDS": { @@ -927,23 +478,15 @@ class FeedExportTest(FeedExportTestBase): }, } crawler = get_crawler(ItemSpider, settings) - with ExitStack() as stack: - mockserver = stack.enter_context(MockServer()) - stack.enter_context( - mock.patch( - "scrapy.extensions.feedexport.FileFeedStorage.store", - side_effect=KeyError("foo"), - ) - ) - yield crawler.crawl(mockserver=mockserver) - self.assertIn( - "feedexport/failed_count/FileFeedStorage", crawler.stats.get_stats() - ) - self.assertEqual( - crawler.stats.get_value("feedexport/failed_count/FileFeedStorage"), 1 - ) + with mock.patch( + "scrapy.extensions.feedexport.FileFeedStorage.store", + side_effect=KeyError("foo"), + ): + yield crawler.crawl(mockserver=self.mockserver) + assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats() + assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1 - @defer.inlineCallbacks + @inline_callbacks_test def test_stats_multiple_file(self): settings = { "FEEDS": { @@ -956,23 +499,17 @@ class FeedExportTest(FeedExportTestBase): }, } crawler = get_crawler(ItemSpider, settings) - with MockServer() as mockserver, mock.patch.object(S3FeedStorage, "store"): - yield crawler.crawl(mockserver=mockserver) - self.assertIn( - "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() - ) - self.assertIn( - "feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats() - ) - self.assertEqual( - crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1 - ) - self.assertEqual( - crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1 + with mock.patch.object(S3FeedStorage, "store"): + yield crawler.crawl(mockserver=self.mockserver) + assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() + assert "feedexport/success_count/StdoutFeedStorage" in crawler.stats.get_stats() + assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 + assert ( + crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage") == 1 ) - @defer.inlineCallbacks - def test_export_items(self): + @coroutine_test + async def test_export_items(self): # feed exporters use field names from Item items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -983,10 +520,10 @@ class FeedExportTest(FeedExportTestBase): {"egg": "spam2", "foo": "bar2", "baz": "quux2"}, ] header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows) + await self.assertExported(items, header, rows) - @defer.inlineCallbacks - def test_export_no_items_not_store_empty(self): + @coroutine_test + async def test_export_no_items_not_store_empty(self): for fmt in ("json", "jsonlines", "xml", "csv"): settings = { "FEEDS": { @@ -994,11 +531,11 @@ class FeedExportTest(FeedExportTestBase): }, "FEED_STORE_EMPTY": False, } - data = yield self.exported_no_data(settings) - self.assertEqual(None, data[fmt]) + data = await self.exported_no_data(settings) + assert data[fmt] is None - @defer.inlineCallbacks - def test_start_finish_exporting_items(self): + @coroutine_test + async def test_start_finish_exporting_items(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), ] @@ -1013,12 +550,12 @@ class FeedExportTest(FeedExportTestBase): InstrumentedFeedSlot.subscribe__listener(listener) with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): - _ = yield self.exported_data(items, settings) - self.assertFalse(listener.start_without_finish) - self.assertFalse(listener.finish_without_start) + await self.exported_data(items, settings) + assert not listener.start_without_finish + assert not listener.finish_without_start - @defer.inlineCallbacks - def test_start_finish_exporting_no_items(self): + @coroutine_test + async def test_start_finish_exporting_no_items(self): items = [] settings = { "FEEDS": { @@ -1031,12 +568,12 @@ class FeedExportTest(FeedExportTestBase): InstrumentedFeedSlot.subscribe__listener(listener) with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): - _ = yield self.exported_data(items, settings) - self.assertFalse(listener.start_without_finish) - self.assertFalse(listener.finish_without_start) + await self.exported_data(items, settings) + assert not listener.start_without_finish + assert not listener.finish_without_start - @defer.inlineCallbacks - def test_start_finish_exporting_items_exception(self): + @coroutine_test + async def test_start_finish_exporting_items_exception(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), ] @@ -1052,12 +589,12 @@ class FeedExportTest(FeedExportTestBase): InstrumentedFeedSlot.subscribe__listener(listener) with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): - _ = yield self.exported_data(items, settings) - self.assertFalse(listener.start_without_finish) - self.assertFalse(listener.finish_without_start) + await self.exported_data(items, settings) + assert not listener.start_without_finish + assert not listener.finish_without_start - @defer.inlineCallbacks - def test_start_finish_exporting_no_items_exception(self): + @coroutine_test + async def test_start_finish_exporting_no_items_exception(self): items = [] settings = { "FEEDS": { @@ -1071,12 +608,12 @@ class FeedExportTest(FeedExportTestBase): InstrumentedFeedSlot.subscribe__listener(listener) with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): - _ = yield self.exported_data(items, settings) - self.assertFalse(listener.start_without_finish) - self.assertFalse(listener.finish_without_start) + await self.exported_data(items, settings) + assert not listener.start_without_finish + assert not listener.finish_without_start - @defer.inlineCallbacks - def test_export_no_items_store_empty(self): + @coroutine_test + async def test_export_no_items_store_empty(self): formats = ( ("json", b"[]"), ("jsonlines", b""), @@ -1092,11 +629,11 @@ class FeedExportTest(FeedExportTestBase): "FEED_STORE_EMPTY": True, "FEED_EXPORT_INDENT": None, } - data = yield self.exported_no_data(settings) - self.assertEqual(expctd, data[fmt]) + data = await self.exported_no_data(settings) + assert expctd == data[fmt] - @defer.inlineCallbacks - def test_export_no_items_multiple_feeds(self): + @coroutine_test + async def test_export_no_items_multiple_feeds(self): """Make sure that `storage.store` is called for every feed.""" settings = { "FEEDS": { @@ -1109,12 +646,12 @@ class FeedExportTest(FeedExportTestBase): } with LogCapture() as log: - yield self.exported_no_data(settings) + await self.exported_no_data(settings) - self.assertEqual(str(log).count("Storage.store is called"), 0) + assert str(log).count("Storage.store is called") == 0 - @defer.inlineCallbacks - def test_export_multiple_item_classes(self): + @coroutine_test + async def test_export_multiple_item_classes(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), self.MyItem2({"hello": "world2", "foo": "bar2"}), @@ -1132,53 +669,53 @@ class FeedExportTest(FeedExportTestBase): {"egg": "spam4", "foo": "", "baz": ""}, ] rows_jl = [dict(row) for row in items] - yield self.assertExportedCsv(items, header, rows_csv) - yield self.assertExportedJsonLines(items, rows_jl) + await self.assertExportedCsv(items, header, rows_csv) + await self.assertExportedJsonLines(items, rows_jl) - @defer.inlineCallbacks - def test_export_items_empty_field_list(self): + @coroutine_test + async def test_export_items_empty_field_list(self): # FEED_EXPORT_FIELDS==[] means the same as default None items = [{"foo": "bar"}] header = ["foo"] rows = [{"foo": "bar"}] settings = {"FEED_EXPORT_FIELDS": []} - yield self.assertExportedCsv(items, header, rows) - yield self.assertExportedJsonLines(items, rows, settings) + await self.assertExportedCsv(items, header, rows) + await self.assertExportedJsonLines(items, rows, settings) - @defer.inlineCallbacks - def test_export_items_field_list(self): + @coroutine_test + async def test_export_items_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] rows = [{"foo": "bar", "baz": ""}] settings = {"FEED_EXPORT_FIELDS": header} - yield self.assertExported(items, header, rows, settings=settings) + await self.assertExported(items, header, rows, settings=settings) - @defer.inlineCallbacks - def test_export_items_comma_separated_field_list(self): + @coroutine_test + async def test_export_items_comma_separated_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] rows = [{"foo": "bar", "baz": ""}] settings = {"FEED_EXPORT_FIELDS": ",".join(header)} - yield self.assertExported(items, header, rows, settings=settings) + await self.assertExported(items, header, rows, settings=settings) - @defer.inlineCallbacks - def test_export_items_json_field_list(self): + @coroutine_test + async def test_export_items_json_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] rows = [{"foo": "bar", "baz": ""}] settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} - yield self.assertExported(items, header, rows, settings=settings) + await self.assertExported(items, header, rows, settings=settings) - @defer.inlineCallbacks - def test_export_items_field_names(self): + @coroutine_test + async def test_export_items_field_names(self): items = [{"foo": "bar"}] header = {"foo": "Foo"} rows = [{"Foo": "bar"}] settings = {"FEED_EXPORT_FIELDS": header} - yield self.assertExported(items, list(header.values()), rows, settings=settings) + await self.assertExported(items, list(header.values()), rows, settings=settings) - @defer.inlineCallbacks - def test_export_items_dict_field_names(self): + @coroutine_test + async def test_export_items_dict_field_names(self): items = [{"foo": "bar"}] header = { "baz": "Baz", @@ -1186,18 +723,18 @@ class FeedExportTest(FeedExportTestBase): } rows = [{"Baz": "", "Foo": "bar"}] settings = {"FEED_EXPORT_FIELDS": header} - yield self.assertExported(items, ["Baz", "Foo"], rows, settings=settings) + await self.assertExported(items, ["Baz", "Foo"], rows, settings=settings) - @defer.inlineCallbacks - def test_export_items_json_field_names(self): + @coroutine_test + async def test_export_items_json_field_names(self): items = [{"foo": "bar"}] header = {"foo": "Foo"} rows = [{"Foo": "bar"}] settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} - yield self.assertExported(items, list(header.values()), rows, settings=settings) + await self.assertExported(items, list(header.values()), rows, settings=settings) - @defer.inlineCallbacks - def test_export_based_on_item_classes(self): + @coroutine_test + async def test_export_based_on_item_classes(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), self.MyItem2({"hello": "world2", "foo": "bar2"}), @@ -1208,8 +745,7 @@ class FeedExportTest(FeedExportTestBase): "csv": b"baz,egg,foo\r\n,spam1,bar1\r\n", "json": b'[\n{"hello": "world2", "foo": "bar2"}\n]', "jsonlines": ( - b'{"foo": "bar1", "egg": "spam1"}\n' - b'{"hello": "world2", "foo": "bar2"}\n' + b'{"foo": "bar1", "egg": "spam1"}\n{"hello": "world2", "foo": "bar2"}\n' ), "xml": ( b'\n\n' @@ -1239,12 +775,12 @@ class FeedExportTest(FeedExportTestBase): }, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for fmt, expected in formats.items(): - self.assertEqual(expected, data[fmt]) + assert data[fmt] == expected - @defer.inlineCallbacks - def test_export_based_on_custom_filters(self): + @coroutine_test + async def test_export_based_on_custom_filters(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), self.MyItem2({"hello": "world2", "foo": "bar2"}), @@ -1262,15 +798,13 @@ class FeedExportTest(FeedExportTestBase): class CustomFilter2(scrapy.extensions.feedexport.ItemFilter): def accepts(self, item): - if "foo" not in item.fields: - return False - return True + return "foo" in item.fields class CustomFilter3(scrapy.extensions.feedexport.ItemFilter): def accepts(self, item): - if isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1": - return True - return False + return ( + isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1" + ) formats = { "json": b'[\n{"foo": "bar1", "egg": "spam1"}\n]', @@ -1300,12 +834,12 @@ class FeedExportTest(FeedExportTestBase): }, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for fmt, expected in formats.items(): - self.assertEqual(expected, data[fmt]) + assert data[fmt] == expected - @defer.inlineCallbacks - def test_export_dicts(self): + @coroutine_test + async def test_export_dicts(self): # When dicts are used, only keys from the first row are used as # a header for CSV, and all fields are used for JSON Lines. items = [ @@ -1314,11 +848,11 @@ class FeedExportTest(FeedExportTestBase): ] rows_csv = [{"egg": "spam", "foo": "bar"}, {"egg": "spam", "foo": "bar"}] rows_jl = items - yield self.assertExportedCsv(items, ["foo", "egg"], rows_csv) - yield self.assertExportedJsonLines(items, rows_jl) + await self.assertExportedCsv(items, ["foo", "egg"], rows_csv) + await self.assertExportedJsonLines(items, rows_jl) - @defer.inlineCallbacks - def test_export_tuple(self): + @coroutine_test + async def test_export_tuple(self): items = [ {"foo": "bar1", "egg": "spam1"}, {"foo": "bar2", "egg": "spam2", "baz": "quux"}, @@ -1326,10 +860,10 @@ class FeedExportTest(FeedExportTestBase): settings = {"FEED_EXPORT_FIELDS": ("foo", "baz")} rows = [{"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}] - yield self.assertExported(items, ["foo", "baz"], rows, settings=settings) + await self.assertExported(items, ["foo", "baz"], rows, settings=settings) - @defer.inlineCallbacks - def test_export_feed_export_fields(self): + @coroutine_test + async def test_export_feed_export_fields(self): # FEED_EXPORT_FIELDS option allows to order export fields # and to select a subset of fields to export, both for Items and dicts. @@ -1345,27 +879,27 @@ class FeedExportTest(FeedExportTestBase): {"egg": "spam1", "foo": "bar1", "baz": ""}, {"egg": "spam2", "foo": "bar2", "baz": "quux2"}, ] - yield self.assertExported( + await self.assertExported( items, ["foo", "baz", "egg"], rows, settings=settings ) # export a subset of columns settings = {"FEED_EXPORT_FIELDS": "egg,baz"} rows = [{"egg": "spam1", "baz": ""}, {"egg": "spam2", "baz": "quux2"}] - yield self.assertExported(items, ["egg", "baz"], rows, settings=settings) + await self.assertExported(items, ["egg", "baz"], rows, settings=settings) - @defer.inlineCallbacks - def test_export_encoding(self): - items = [dict({"foo": "Test\xd6"})] + @coroutine_test + async def test_export_encoding(self): + items = [{"foo": "Test\xd6"}] formats = { - "json": '[{"foo": "Test\\u00d6"}]'.encode("utf-8"), - "jsonlines": '{"foo": "Test\\u00d6"}\n'.encode("utf-8"), + "json": b'[{"foo": "Test\\u00d6"}]', + "jsonlines": b'{"foo": "Test\\u00d6"}\n', "xml": ( '\n' "Test\xd6" - ).encode("utf-8"), - "csv": "foo\r\nTest\xd6\r\n".encode("utf-8"), + ).encode(), + "csv": "foo\r\nTest\xd6\r\n".encode(), } for fmt, expected in formats.items(): @@ -1375,17 +909,17 @@ class FeedExportTest(FeedExportTestBase): }, "FEED_EXPORT_INDENT": None, } - data = yield self.exported_data(items, settings) - self.assertEqual(expected, data[fmt]) + data = await self.exported_data(items, settings) + assert data[fmt] == expected formats = { - "json": '[{"foo": "Test\xd6"}]'.encode("latin-1"), - "jsonlines": '{"foo": "Test\xd6"}\n'.encode("latin-1"), + "json": b'[{"foo": "Test\xd6"}]', + "jsonlines": b'{"foo": "Test\xd6"}\n', "xml": ( - '\n' - "Test\xd6" - ).encode("latin-1"), - "csv": "foo\r\nTest\xd6\r\n".encode("latin-1"), + b'\n' + b"Test\xd6" + ), + "csv": b"foo\r\nTest\xd6\r\n", } for fmt, expected in formats.items(): @@ -1396,20 +930,20 @@ class FeedExportTest(FeedExportTestBase): "FEED_EXPORT_INDENT": None, "FEED_EXPORT_ENCODING": "latin-1", } - data = yield self.exported_data(items, settings) - self.assertEqual(expected, data[fmt]) + data = await self.exported_data(items, settings) + assert data[fmt] == expected - @defer.inlineCallbacks - def test_export_multiple_configs(self): - items = [dict({"foo": "FOO", "bar": "BAR"})] + @coroutine_test + async def test_export_multiple_configs(self): + items = [{"foo": "FOO", "bar": "BAR"}] formats = { - "json": '[\n{"bar": "BAR"}\n]'.encode("utf-8"), + "json": b'[\n{"bar": "BAR"}\n]', "xml": ( - '\n' - "\n \n FOO\n \n" - ).encode("latin-1"), - "csv": "bar,foo\r\nBAR,FOO\r\n".encode("utf-8"), + b'\n' + b"\n \n FOO\n \n" + ), + "csv": b"bar,foo\r\nBAR,FOO\r\n", } settings = { @@ -1435,12 +969,12 @@ class FeedExportTest(FeedExportTestBase): }, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for fmt, expected in formats.items(): - self.assertEqual(expected, data[fmt]) + assert data[fmt] == expected - @defer.inlineCallbacks - def test_export_indentation(self): + @coroutine_test + async def test_export_indentation(self): items = [ {"foo": ["bar"]}, {"key": "value"}, @@ -1592,11 +1126,11 @@ class FeedExportTest(FeedExportTestBase): }, }, } - data = yield self.exported_data(items, settings) - self.assertEqual(row["expected"], data[row["format"]]) + data = await self.exported_data(items, settings) + assert data[row["format"]] == row["expected"] - @defer.inlineCallbacks - def test_init_exporters_storages_with_crawler(self): + @coroutine_test + async def test_init_exporters_storages_with_crawler(self): settings = { "FEED_EXPORTERS": {"csv": FromCrawlerCsvItemExporter}, "FEED_STORAGES": {"file": FromCrawlerFileFeedStorage}, @@ -1604,21 +1138,21 @@ class FeedExportTest(FeedExportTestBase): self._random_temp_filename(): {"format": "csv"}, }, } - yield self.exported_data(items=[], settings=settings) - self.assertTrue(FromCrawlerCsvItemExporter.init_with_crawler) - self.assertTrue(FromCrawlerFileFeedStorage.init_with_crawler) + await self.exported_data(items=[], settings=settings) + assert FromCrawlerCsvItemExporter.init_with_crawler + assert FromCrawlerFileFeedStorage.init_with_crawler - @defer.inlineCallbacks - def test_str_uri(self): + @coroutine_test + async def test_str_uri(self): settings = { "FEED_STORE_EMPTY": True, "FEEDS": {str(self._random_temp_filename()): {"format": "csv"}}, } - data = yield self.exported_no_data(settings) - self.assertEqual(data["csv"], b"") + data = await self.exported_no_data(settings) + assert data["csv"] == b"" - @defer.inlineCallbacks - def test_multiple_feeds_success_logs_blocking_feed_storage(self): + @coroutine_test + async def test_multiple_feeds_success_logs_blocking_feed_storage(self): settings = { "FEEDS": { self._random_temp_filename(): {"format": "json"}, @@ -1632,14 +1166,13 @@ class FeedExportTest(FeedExportTestBase): {"foo": "bar2", "baz": "quux"}, ] with LogCapture() as log: - yield self.exported_data(items, settings) + await self.exported_data(items, settings) - print(log) for fmt in ["json", "xml", "csv"]: - self.assertIn(f"Stored {fmt} feed (2 items)", str(log)) + assert f"Stored {fmt} feed (2 items)" in str(log) - @defer.inlineCallbacks - def test_multiple_feeds_failing_logs_blocking_feed_storage(self): + @coroutine_test + async def test_multiple_feeds_failing_logs_blocking_feed_storage(self): settings = { "FEEDS": { self._random_temp_filename(): {"format": "json"}, @@ -1653,18 +1186,17 @@ class FeedExportTest(FeedExportTestBase): {"foo": "bar2", "baz": "quux"}, ] with LogCapture() as log: - yield self.exported_data(items, settings) + await self.exported_data(items, settings) - print(log) for fmt in ["json", "xml", "csv"]: - self.assertIn(f"Error storing {fmt} feed (2 items)", str(log)) + assert f"Error storing {fmt} feed (2 items)" in str(log) - @defer.inlineCallbacks - def test_extend_kwargs(self): + @coroutine_test + async def test_extend_kwargs(self): items = [{"foo": "FOO", "bar": "BAR"}] - expected_with_title_csv = "foo,bar\r\nFOO,BAR\r\n".encode("utf-8") - expected_without_title_csv = "FOO,BAR\r\n".encode("utf-8") + expected_with_title_csv = b"foo,bar\r\nFOO,BAR\r\n" + expected_without_title_csv = b"FOO,BAR\r\n" test_cases = [ # with title { @@ -1693,11 +1225,11 @@ class FeedExportTest(FeedExportTestBase): "FEED_EXPORT_INDENT": None, } - data = yield self.exported_data(items, settings) - self.assertEqual(row["expected"], data[feed_options["format"]]) + data = await self.exported_data(items, settings) + assert data[feed_options["format"]] == row["expected"] - @defer.inlineCallbacks - def test_storage_file_no_postprocessing(self): + @coroutine_test + async def test_storage_file_no_postprocessing(self): @implementer(IFeedStorage) class Storage: def __init__(self, uri, *, feed_options=None): @@ -1715,11 +1247,11 @@ class FeedExportTest(FeedExportTestBase): "FEEDS": {self._random_temp_filename(): {"format": "jsonlines"}}, "FEED_STORAGES": {"file": Storage}, } - yield self.exported_no_data(settings) - self.assertIs(Storage.open_file, Storage.store_file) + await self.exported_no_data(settings) + assert Storage.open_file is Storage.store_file - @defer.inlineCallbacks - def test_storage_file_postprocessing(self): + @coroutine_test + async def test_storage_file_postprocessing(self): @implementer(IFeedStorage) class Storage: def __init__(self, uri, *, feed_options=None): @@ -1731,6 +1263,7 @@ class FeedExportTest(FeedExportTestBase): def store(self, file): Storage.store_file = file + Storage.file_was_closed = file.closed file.close() settings = { @@ -1744,971 +1277,13 @@ class FeedExportTest(FeedExportTestBase): }, "FEED_STORAGES": {"file": Storage}, } - yield self.exported_no_data(settings) - self.assertIs(Storage.open_file, Storage.store_file) - - -class FeedPostProcessedExportsTest(FeedExportTestBase): - __test__ = True - - items = [{"foo": "bar"}] - expected = b"foo\r\nbar\r\n" - - class MyPlugin1: - def __init__(self, file, feed_options): - self.file = file - self.feed_options = feed_options - self.char = self.feed_options.get("plugin1_char", b"") - - def write(self, data): - written_count = self.file.write(data) - written_count += self.file.write(self.char) - return written_count - - def close(self): - self.file.close() - - def _named_tempfile(self, name) -> str: - return str(Path(self.temp_dir, name)) - - @defer.inlineCallbacks - def run_and_export(self, spider_cls, settings): - """Run spider with specified settings; return exported data with filename.""" - - FEEDS = settings.get("FEEDS") or {} - settings["FEEDS"] = { - printf_escape(path_to_url(file_path)): feed_options - for file_path, feed_options in FEEDS.items() - } - - content = {} - try: - with MockServer() as s: - spider_cls.start_urls = [s.url("/")] - crawler = get_crawler(spider_cls, settings) - yield crawler.crawl() - - for file_path, feed_options in FEEDS.items(): - content[str(file_path)] = ( - Path(file_path).read_bytes() if Path(file_path).exists() else None - ) - - finally: - for file_path in FEEDS.keys(): - if not Path(file_path).exists(): - continue - - Path(file_path).unlink() - - return content - - def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=""): - data_stream = BytesIO() - gzipf = gzip.GzipFile( - fileobj=data_stream, - filename=filename, - mtime=mtime, - compresslevel=compresslevel, - mode="wb", - ) - gzipf.write(data) - gzipf.close() - data_stream.seek(0) - return data_stream.read() - - @defer.inlineCallbacks - def test_gzip_plugin(self): - filename = self._named_tempfile("gzip_file") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - }, - }, - } - - data = yield self.exported_data(self.items, settings) - try: - gzip.decompress(data[filename]) - except OSError: - self.fail("Received invalid gzip data.") - - @defer.inlineCallbacks - def test_gzip_plugin_compresslevel(self): - filename_to_compressed = { - self._named_tempfile("compresslevel_0"): self.get_gzip_compressed( - self.expected, compresslevel=0 - ), - self._named_tempfile("compresslevel_9"): self.get_gzip_compressed( - self.expected, compresslevel=9 - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("compresslevel_0"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_compresslevel": 0, - "gzip_mtime": 0, - "gzip_filename": "", - }, - self._named_tempfile("compresslevel_9"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_compresslevel": 9, - "gzip_mtime": 0, - "gzip_filename": "", - }, - }, - } - - data = yield self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = gzip.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) - - @defer.inlineCallbacks - def test_gzip_plugin_mtime(self): - filename_to_compressed = { - self._named_tempfile("mtime_123"): self.get_gzip_compressed( - self.expected, mtime=123 - ), - self._named_tempfile("mtime_123456789"): self.get_gzip_compressed( - self.expected, mtime=123456789 - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("mtime_123"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_mtime": 123, - "gzip_filename": "", - }, - self._named_tempfile("mtime_123456789"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_mtime": 123456789, - "gzip_filename": "", - }, - }, - } - - data = yield self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = gzip.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) - - @defer.inlineCallbacks - def test_gzip_plugin_filename(self): - filename_to_compressed = { - self._named_tempfile("filename_FILE1"): self.get_gzip_compressed( - self.expected, filename="FILE1" - ), - self._named_tempfile("filename_FILE2"): self.get_gzip_compressed( - self.expected, filename="FILE2" - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("filename_FILE1"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_mtime": 0, - "gzip_filename": "FILE1", - }, - self._named_tempfile("filename_FILE2"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_mtime": 0, - "gzip_filename": "FILE2", - }, - }, - } - - data = yield self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = gzip.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) - - @defer.inlineCallbacks - def test_lzma_plugin(self): - filename = self._named_tempfile("lzma_file") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - }, - }, - } - - data = yield self.exported_data(self.items, settings) - try: - lzma.decompress(data[filename]) - except lzma.LZMAError: - self.fail("Received invalid lzma data.") - - @defer.inlineCallbacks - def test_lzma_plugin_format(self): - filename_to_compressed = { - self._named_tempfile("format_FORMAT_XZ"): lzma.compress( - self.expected, format=lzma.FORMAT_XZ - ), - self._named_tempfile("format_FORMAT_ALONE"): lzma.compress( - self.expected, format=lzma.FORMAT_ALONE - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("format_FORMAT_XZ"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_format": lzma.FORMAT_XZ, - }, - self._named_tempfile("format_FORMAT_ALONE"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_format": lzma.FORMAT_ALONE, - }, - }, - } - - data = yield self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = lzma.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) - - @defer.inlineCallbacks - def test_lzma_plugin_check(self): - filename_to_compressed = { - self._named_tempfile("check_CHECK_NONE"): lzma.compress( - self.expected, check=lzma.CHECK_NONE - ), - self._named_tempfile("check_CHECK_CRC256"): lzma.compress( - self.expected, check=lzma.CHECK_SHA256 - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("check_CHECK_NONE"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_check": lzma.CHECK_NONE, - }, - self._named_tempfile("check_CHECK_CRC256"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_check": lzma.CHECK_SHA256, - }, - }, - } - - data = yield self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = lzma.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) - - @defer.inlineCallbacks - def test_lzma_plugin_preset(self): - filename_to_compressed = { - self._named_tempfile("preset_PRESET_0"): lzma.compress( - self.expected, preset=0 - ), - self._named_tempfile("preset_PRESET_9"): lzma.compress( - self.expected, preset=9 - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("preset_PRESET_0"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_preset": 0, - }, - self._named_tempfile("preset_PRESET_9"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_preset": 9, - }, - }, - } - - data = yield self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = lzma.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) - - @defer.inlineCallbacks - def test_lzma_plugin_filters(self): - if "PyPy" in sys.version: - # https://foss.heptapod.net/pypy/pypy/-/issues/3527 - raise unittest.SkipTest("lzma filters doesn't work in PyPy") - - filters = [{"id": lzma.FILTER_LZMA2}] - compressed = lzma.compress(self.expected, filters=filters) - filename = self._named_tempfile("filters") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_filters": filters, - }, - }, - } - - data = yield self.exported_data(self.items, settings) - self.assertEqual(compressed, data[filename]) - result = lzma.decompress(data[filename]) - self.assertEqual(self.expected, result) - - @defer.inlineCallbacks - def test_bz2_plugin(self): - filename = self._named_tempfile("bz2_file") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], - }, - }, - } - - data = yield self.exported_data(self.items, settings) - try: - bz2.decompress(data[filename]) - except OSError: - self.fail("Received invalid bz2 data.") - - @defer.inlineCallbacks - def test_bz2_plugin_compresslevel(self): - filename_to_compressed = { - self._named_tempfile("compresslevel_1"): bz2.compress( - self.expected, compresslevel=1 - ), - self._named_tempfile("compresslevel_9"): bz2.compress( - self.expected, compresslevel=9 - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("compresslevel_1"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], - "bz2_compresslevel": 1, - }, - self._named_tempfile("compresslevel_9"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], - "bz2_compresslevel": 9, - }, - }, - } - - data = yield self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = bz2.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) - - @defer.inlineCallbacks - def test_custom_plugin(self): - filename = self._named_tempfile("csv_file") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": [self.MyPlugin1], - }, - }, - } - - data = yield self.exported_data(self.items, settings) - self.assertEqual(self.expected, data[filename]) - - @defer.inlineCallbacks - def test_custom_plugin_with_parameter(self): - expected = b"foo\r\n\nbar\r\n\n" - filename = self._named_tempfile("newline") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": [self.MyPlugin1], - "plugin1_char": b"\n", - }, - }, - } - - data = yield self.exported_data(self.items, settings) - self.assertEqual(expected, data[filename]) - - @defer.inlineCallbacks - def test_custom_plugin_with_compression(self): - expected = b"foo\r\n\nbar\r\n\n" - - filename_to_decompressor = { - self._named_tempfile("bz2"): bz2.decompress, - self._named_tempfile("lzma"): lzma.decompress, - self._named_tempfile("gzip"): gzip.decompress, - } - - settings = { - "FEEDS": { - self._named_tempfile("bz2"): { - "format": "csv", - "postprocessing": [ - self.MyPlugin1, - "scrapy.extensions.postprocessing.Bz2Plugin", - ], - "plugin1_char": b"\n", - }, - self._named_tempfile("lzma"): { - "format": "csv", - "postprocessing": [ - self.MyPlugin1, - "scrapy.extensions.postprocessing.LZMAPlugin", - ], - "plugin1_char": b"\n", - }, - self._named_tempfile("gzip"): { - "format": "csv", - "postprocessing": [ - self.MyPlugin1, - "scrapy.extensions.postprocessing.GzipPlugin", - ], - "plugin1_char": b"\n", - }, - }, - } - - data = yield self.exported_data(self.items, settings) - - for filename, decompressor in filename_to_decompressor.items(): - result = decompressor(data[filename]) - self.assertEqual(expected, result) - - @defer.inlineCallbacks - def test_exports_compatibility_with_postproc(self): - import marshal - import pickle - - filename_to_expected = { - self._named_tempfile("csv"): b"foo\r\nbar\r\n", - self._named_tempfile("json"): b'[\n{"foo": "bar"}\n]', - self._named_tempfile("jsonlines"): b'{"foo": "bar"}\n', - self._named_tempfile("xml"): b'\n' - b"\nbar\n", - } - - settings = { - "FEEDS": { - self._named_tempfile("csv"): { - "format": "csv", - "postprocessing": [self.MyPlugin1], - # empty plugin to activate postprocessing.PostProcessingManager - }, - self._named_tempfile("json"): { - "format": "json", - "postprocessing": [self.MyPlugin1], - }, - self._named_tempfile("jsonlines"): { - "format": "jsonlines", - "postprocessing": [self.MyPlugin1], - }, - self._named_tempfile("xml"): { - "format": "xml", - "postprocessing": [self.MyPlugin1], - }, - self._named_tempfile("marshal"): { - "format": "marshal", - "postprocessing": [self.MyPlugin1], - }, - self._named_tempfile("pickle"): { - "format": "pickle", - "postprocessing": [self.MyPlugin1], - }, - }, - } - - data = yield self.exported_data(self.items, settings) - - for filename, result in data.items(): - if "pickle" in filename: - expected, result = self.items[0], pickle.loads(result) - elif "marshal" in filename: - expected, result = self.items[0], marshal.loads(result) - else: - expected = filename_to_expected[filename] - self.assertEqual(expected, result) - - -class BatchDeliveriesTest(FeedExportTestBase): - __test__ = True - _file_mark = "_%(batch_time)s_#%(batch_id)02d_" - - @defer.inlineCallbacks - def run_and_export(self, spider_cls, settings): - """Run spider with specified settings; return exported data.""" - - FEEDS = settings.get("FEEDS") or {} - settings["FEEDS"] = { - build_url(file_path): feed for file_path, feed in FEEDS.items() - } - content = defaultdict(list) - try: - with MockServer() as s: - spider_cls.start_urls = [s.url("/")] - crawler = get_crawler(spider_cls, settings) - yield crawler.crawl() - - for path, feed in FEEDS.items(): - dir_name = Path(path).parent - if not dir_name.exists(): - content[feed["format"]] = [] - continue - for file in sorted(dir_name.iterdir()): - content[feed["format"]].append(file.read_bytes()) - finally: - self.tearDown() - return content - - @defer.inlineCallbacks - def assertExportedJsonLines(self, items, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() - / "jl" - / self._file_mark: {"format": "jl"}, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - rows = [{k: v for k, v in row.items() if v} for row in rows] - data = yield self.exported_data(items, settings) - for batch in data["jl"]: - got_batch = [ - json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines() - ] - expected_batch, rows = rows[:batch_size], rows[batch_size:] - self.assertEqual(expected_batch, got_batch) - - @defer.inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() - / "csv" - / self._file_mark: {"format": "csv"}, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - data = yield self.exported_data(items, settings) - for batch in data["csv"]: - got_batch = csv.DictReader(to_unicode(batch).splitlines()) - self.assertEqual(list(header), got_batch.fieldnames) - expected_batch, rows = rows[:batch_size], rows[batch_size:] - self.assertEqual(expected_batch, list(got_batch)) - - @defer.inlineCallbacks - def assertExportedXml(self, items, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() - / "xml" - / self._file_mark: {"format": "xml"}, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - rows = [{k: v for k, v in row.items() if v} for row in rows] - data = yield self.exported_data(items, settings) - for batch in data["xml"]: - root = lxml.etree.fromstring(batch) - got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] - expected_batch, rows = rows[:batch_size], rows[batch_size:] - self.assertEqual(expected_batch, got_batch) - - @defer.inlineCallbacks - def assertExportedMultiple(self, items, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() - / "xml" - / self._file_mark: {"format": "xml"}, - self._random_temp_filename() - / "json" - / self._file_mark: {"format": "json"}, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - rows = [{k: v for k, v in row.items() if v} for row in rows] - data = yield self.exported_data(items, settings) - # XML - xml_rows = rows.copy() - for batch in data["xml"]: - root = lxml.etree.fromstring(batch) - got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] - expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:] - self.assertEqual(expected_batch, got_batch) - # JSON - json_rows = rows.copy() - for batch in data["json"]: - got_batch = json.loads(batch.decode("utf-8")) - expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] - self.assertEqual(expected_batch, got_batch) - - @defer.inlineCallbacks - def assertExportedPickle(self, items, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() - / "pickle" - / self._file_mark: {"format": "pickle"}, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - rows = [{k: v for k, v in row.items() if v} for row in rows] - data = yield self.exported_data(items, settings) - import pickle - - for batch in data["pickle"]: - got_batch = self._load_until_eof(batch, load_func=pickle.load) - expected_batch, rows = rows[:batch_size], rows[batch_size:] - self.assertEqual(expected_batch, got_batch) - - @defer.inlineCallbacks - def assertExportedMarshal(self, items, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() - / "marshal" - / self._file_mark: {"format": "marshal"}, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - rows = [{k: v for k, v in row.items() if v} for row in rows] - data = yield self.exported_data(items, settings) - import marshal - - for batch in data["marshal"]: - got_batch = self._load_until_eof(batch, load_func=marshal.load) - expected_batch, rows = rows[:batch_size], rows[batch_size:] - self.assertEqual(expected_batch, got_batch) - - @defer.inlineCallbacks - def test_export_items(self): - """Test partial deliveries in all supported formats""" - items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), - self.MyItem({"foo": "bar3", "baz": "quux3"}), - ] - rows = [ - {"egg": "spam1", "foo": "bar1", "baz": ""}, - {"egg": "spam2", "foo": "bar2", "baz": "quux2"}, - {"foo": "bar3", "baz": "quux3", "egg": ""}, - ] - settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2} - header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows, settings=settings) - - def test_wrong_path(self): - """If path is without %(batch_time)s and %(batch_id) an exception must be raised""" - settings = { - "FEEDS": { - self._random_temp_filename(): {"format": "xml"}, - }, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - } - crawler = get_crawler(settings_dict=settings) - self.assertRaises(NotConfigured, FeedExporter, crawler) - - @defer.inlineCallbacks - def test_export_no_items_not_store_empty(self): - for fmt in ("json", "jsonlines", "xml", "csv"): - settings = { - "FEEDS": { - self._random_temp_filename() - / fmt - / self._file_mark: {"format": fmt}, - }, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - "FEED_STORE_EMPTY": False, - } - data = yield self.exported_no_data(settings) - data = dict(data) - self.assertEqual(0, len(data[fmt])) - - @defer.inlineCallbacks - def test_export_no_items_store_empty(self): - formats = ( - ("json", b"[]"), - ("jsonlines", b""), - ("xml", b'\n'), - ("csv", b""), - ) - - for fmt, expctd in formats: - settings = { - "FEEDS": { - self._random_temp_filename() - / fmt - / self._file_mark: {"format": fmt}, - }, - "FEED_STORE_EMPTY": True, - "FEED_EXPORT_INDENT": None, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - } - data = yield self.exported_no_data(settings) - data = dict(data) - self.assertEqual(expctd, data[fmt][0]) - - @defer.inlineCallbacks - def test_export_multiple_configs(self): - items = [ - dict({"foo": "FOO", "bar": "BAR"}), - dict({"foo": "FOO1", "bar": "BAR1"}), - ] - - formats = { - "json": [ - '[\n{"bar": "BAR"}\n]'.encode("utf-8"), - '[\n{"bar": "BAR1"}\n]'.encode("utf-8"), - ], - "xml": [ - ( - '\n' - "\n \n FOO\n \n" - ).encode("latin-1"), - ( - '\n' - "\n \n FOO1\n \n" - ).encode("latin-1"), - ], - "csv": [ - "foo,bar\r\nFOO,BAR\r\n".encode("utf-8"), - "foo,bar\r\nFOO1,BAR1\r\n".encode("utf-8"), - ], - } - - settings = { - "FEEDS": { - self._random_temp_filename() - / "json" - / self._file_mark: { - "format": "json", - "indent": 0, - "fields": ["bar"], - "encoding": "utf-8", - }, - self._random_temp_filename() - / "xml" - / self._file_mark: { - "format": "xml", - "indent": 2, - "fields": ["foo"], - "encoding": "latin-1", - }, - self._random_temp_filename() - / "csv" - / self._file_mark: { - "format": "csv", - "indent": None, - "fields": ["foo", "bar"], - "encoding": "utf-8", - }, - }, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - } - data = yield self.exported_data(items, settings) - for fmt, expected in formats.items(): - for expected_batch, got_batch in zip(expected, data[fmt]): - self.assertEqual(expected_batch, got_batch) - - @defer.inlineCallbacks - def test_batch_item_count_feeds_setting(self): - items = [dict({"foo": "FOO"}), dict({"foo": "FOO1"})] - formats = { - "json": [ - '[{"foo": "FOO"}]'.encode("utf-8"), - '[{"foo": "FOO1"}]'.encode("utf-8"), - ], - } - settings = { - "FEEDS": { - self._random_temp_filename() - / "json" - / self._file_mark: { - "format": "json", - "indent": None, - "encoding": "utf-8", - "batch_item_count": 1, - }, - }, - } - data = yield self.exported_data(items, settings) - for fmt, expected in formats.items(): - for expected_batch, got_batch in zip(expected, data[fmt]): - self.assertEqual(expected_batch, got_batch) - - @defer.inlineCallbacks - def test_batch_path_differ(self): - """ - Test that the name of all batch files differ from each other. - So %(batch_id)d replaced with the current id. - """ - items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), - self.MyItem({"foo": "bar3", "baz": "quux3"}), - ] - settings = { - "FEEDS": { - self._random_temp_filename() - / "%(batch_id)d": { - "format": "json", - }, - }, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - } - data = yield self.exported_data(items, settings) - self.assertEqual(len(items), len(data["json"])) - - @defer.inlineCallbacks - def test_stats_batch_file_success(self): - settings = { - "FEEDS": { - build_url( - str(self._random_temp_filename() / "json" / self._file_mark) - ): { - "format": "json", - } - }, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - } - crawler = get_crawler(ItemSpider, settings) - with MockServer() as mockserver: - yield crawler.crawl(total=2, mockserver=mockserver) - self.assertIn( - "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() - ) - self.assertEqual( - crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12 - ) - - @defer.inlineCallbacks - def test_s3_export(self): - skip_if_no_boto() - bucket = "mybucket" - items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), - self.MyItem({"foo": "bar3", "baz": "quux3"}), - ] - - class CustomS3FeedStorage(S3FeedStorage): - stubs = [] - - def open(self, *args, **kwargs): - from botocore.stub import ANY, Stubber - - stub = Stubber(self.s3_client) - stub.activate() - CustomS3FeedStorage.stubs.append(stub) - stub.add_response( - "put_object", - expected_params={ - "Body": ANY, - "Bucket": bucket, - "Key": ANY, - }, - service_response={}, - ) - return super().open(*args, **kwargs) - - key = "export.csv" - uri = f"s3://{bucket}/{key}/%(batch_id)d.json" - batch_item_count = 1 - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - "FEED_EXPORT_BATCH_ITEM_COUNT": batch_item_count, - "FEED_STORAGES": { - "s3": CustomS3FeedStorage, - }, - "FEEDS": { - uri: { - "format": "json", - }, - }, - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler(crawler, uri) - verifyObject(IFeedStorage, storage) - - class TestSpider(scrapy.Spider): - name = "testspider" - - def parse(self, response): - yield from items - - with MockServer() as server: - TestSpider.start_urls = [server.url("/")] - crawler = get_crawler(TestSpider, settings) - yield crawler.crawl() - - self.assertEqual(len(CustomS3FeedStorage.stubs), len(items)) - for stub in CustomS3FeedStorage.stubs[:-1]: - stub.assert_no_pending_responses() + await self.exported_no_data(settings) + assert Storage.open_file is Storage.store_file + assert not Storage.file_was_closed # Test that the FeedExporer sends the feed_exporter_closed and feed_slot_closed signals -class FeedExporterSignalsTest(unittest.TestCase): +class TestFeedExporterSignals: items = [ {"foo": "bar1", "egg": "spam1"}, {"foo": "bar2", "egg": "spam2", "baz": "quux2"}, @@ -2718,7 +1293,7 @@ class FeedExporterSignalsTest(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix="json") as tmp: settings = { "FEEDS": { - f"file:///{tmp.name}": { + printf_escape(path_to_url(tmp.name)): { "format": "json", }, }, @@ -2730,21 +1305,17 @@ class FeedExporterSignalsTest(unittest.TestCase): def feed_slot_closed_signal_handler(self, slot): self.feed_slot_closed_received = True - def feed_exporter_closed_signal_handler_deferred(self): - d = defer.Deferred() - d.addCallback(lambda _: setattr(self, "feed_exporter_closed_received", True)) - d.callback(None) - return d + async def feed_exporter_closed_signal_handler_async(self): + self.feed_exporter_closed_received = True - def feed_slot_closed_signal_handler_deferred(self, slot): - d = defer.Deferred() - d.addCallback(lambda _: setattr(self, "feed_slot_closed_received", True)) - d.callback(None) - return d + async def feed_slot_closed_signal_handler_async(self, slot): + self.feed_slot_closed_received = True - def run_signaled_feed_exporter( - self, feed_exporter_signal_handler, feed_slot_signal_handler - ): + async def run_signaled_feed_exporter( + self, + feed_exporter_signal_handler: Callable[[], Awaitable[None] | None], + feed_slot_signal_handler: Callable[[Any], Awaitable[None] | None], + ) -> None: crawler = get_crawler(settings_dict=self.settings) feed_exporter = FeedExporter.from_crawler(crawler) spider = scrapy.Spider("default") @@ -2759,32 +1330,34 @@ class FeedExporterSignalsTest(unittest.TestCase): feed_exporter.open_spider(spider) for item in self.items: feed_exporter.item_scraped(item, spider) - defer.ensureDeferred(feed_exporter.close_spider(spider)) + await feed_exporter.close_spider(spider) - def test_feed_exporter_signals_sent(self): + @coroutine_test + async def test_feed_exporter_signals_sent(self) -> None: self.feed_exporter_closed_received = False self.feed_slot_closed_received = False - self.run_signaled_feed_exporter( + await self.run_signaled_feed_exporter( self.feed_exporter_closed_signal_handler, self.feed_slot_closed_signal_handler, ) - self.assertTrue(self.feed_slot_closed_received) - self.assertTrue(self.feed_exporter_closed_received) + assert self.feed_slot_closed_received + assert self.feed_exporter_closed_received - def test_feed_exporter_signals_sent_deferred(self): + @coroutine_test + async def test_feed_exporter_signals_sent_async(self) -> None: self.feed_exporter_closed_received = False self.feed_slot_closed_received = False - self.run_signaled_feed_exporter( - self.feed_exporter_closed_signal_handler_deferred, - self.feed_slot_closed_signal_handler_deferred, + await self.run_signaled_feed_exporter( + self.feed_exporter_closed_signal_handler_async, + self.feed_slot_closed_signal_handler_async, ) - self.assertTrue(self.feed_slot_closed_received) - self.assertTrue(self.feed_exporter_closed_received) + assert self.feed_slot_closed_received + assert self.feed_exporter_closed_received -class FeedExportInitTest(unittest.TestCase): +class TestFeedExportInit: def test_unsupported_storage(self): settings = { "FEEDS": { @@ -2792,7 +1365,7 @@ class FeedExportInitTest(unittest.TestCase): }, } crawler = get_crawler(settings_dict=settings) - with self.assertRaises(NotConfigured): + with pytest.raises(NotConfigured): FeedExporter.from_crawler(crawler) def test_unsupported_format(self): @@ -2804,7 +1377,7 @@ class FeedExportInitTest(unittest.TestCase): }, } crawler = get_crawler(settings_dict=settings) - with self.assertRaises(NotConfigured): + with pytest.raises(NotConfigured): FeedExporter.from_crawler(crawler) def test_absolute_pathlib_as_uri(self): @@ -2818,7 +1391,7 @@ class FeedExportInitTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings) exporter = FeedExporter.from_crawler(crawler) - self.assertIsInstance(exporter, FeedExporter) + assert isinstance(exporter, FeedExporter) def test_relative_pathlib_as_uri(self): settings = { @@ -2830,135 +1403,4 @@ class FeedExportInitTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings) exporter = FeedExporter.from_crawler(crawler) - self.assertIsInstance(exporter, FeedExporter) - - -class URIParamsTest: - spider_name = "uri_params_spider" - deprecated_options = False - - def build_settings(self, uri="file:///tmp/foobar", uri_params=None): - raise NotImplementedError - - def _crawler_feed_exporter(self, settings): - if self.deprecated_options: - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings) - feed_exporter = FeedExporter.from_crawler(crawler) - else: - crawler = get_crawler(settings_dict=settings) - feed_exporter = FeedExporter.from_crawler(crawler) - return crawler, feed_exporter - - def test_default(self): - settings = self.build_settings( - uri="file:///tmp/%(name)s", - ) - crawler, feed_exporter = self._crawler_feed_exporter(settings) - spider = scrapy.Spider(self.spider_name) - spider.crawler = crawler - - with warnings.catch_warnings(): - warnings.simplefilter("error", ScrapyDeprecationWarning) - feed_exporter.open_spider(spider) - - self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") - - def test_none(self): - def uri_params(params, spider): - pass - - settings = self.build_settings( - uri="file:///tmp/%(name)s", - uri_params=uri_params, - ) - crawler, feed_exporter = self._crawler_feed_exporter(settings) - spider = scrapy.Spider(self.spider_name) - spider.crawler = crawler - - feed_exporter.open_spider(spider) - - self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") - - def test_empty_dict(self): - def uri_params(params, spider): - return {} - - settings = self.build_settings( - uri="file:///tmp/%(name)s", - uri_params=uri_params, - ) - crawler, feed_exporter = self._crawler_feed_exporter(settings) - spider = scrapy.Spider(self.spider_name) - spider.crawler = crawler - - with warnings.catch_warnings(): - warnings.simplefilter("error", ScrapyDeprecationWarning) - with self.assertRaises(KeyError): - feed_exporter.open_spider(spider) - - def test_params_as_is(self): - def uri_params(params, spider): - return params - - settings = self.build_settings( - uri="file:///tmp/%(name)s", - uri_params=uri_params, - ) - crawler, feed_exporter = self._crawler_feed_exporter(settings) - spider = scrapy.Spider(self.spider_name) - spider.crawler = crawler - with warnings.catch_warnings(): - warnings.simplefilter("error", ScrapyDeprecationWarning) - feed_exporter.open_spider(spider) - - self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") - - def test_custom_param(self): - def uri_params(params, spider): - return {**params, "foo": self.spider_name} - - settings = self.build_settings( - uri="file:///tmp/%(foo)s", - uri_params=uri_params, - ) - crawler, feed_exporter = self._crawler_feed_exporter(settings) - spider = scrapy.Spider(self.spider_name) - spider.crawler = crawler - with warnings.catch_warnings(): - warnings.simplefilter("error", ScrapyDeprecationWarning) - feed_exporter.open_spider(spider) - - self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") - - -class URIParamsSettingTest(URIParamsTest, unittest.TestCase): - deprecated_options = True - - def build_settings(self, uri="file:///tmp/foobar", uri_params=None): - extra_settings = {} - if uri_params: - extra_settings["FEED_URI_PARAMS"] = uri_params - return { - "FEED_URI": uri, - **extra_settings, - } - - -class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase): - deprecated_options = False - - def build_settings(self, uri="file:///tmp/foobar", uri_params=None): - options = { - "format": "jl", - } - if uri_params: - options["uri_params"] = uri_params - return { - "FEEDS": { - uri: options, - }, - } + assert isinstance(exporter, FeedExporter) diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py new file mode 100644 index 000000000..d855d0f74 --- /dev/null +++ b/tests/test_feedexport_batch.py @@ -0,0 +1,457 @@ +from __future__ import annotations + +import csv +import json +import marshal +import pickle +from collections import defaultdict +from pathlib import Path +from typing import TYPE_CHECKING, Any +from urllib.parse import urljoin + +import lxml.etree +import pytest +from packaging.version import Version +from zope.interface.verify import verifyObject + +import scrapy +from scrapy import Spider +from scrapy.exceptions import NotConfigured +from scrapy.extensions.feedexport import FeedExporter, IFeedStorage, S3FeedStorage +from scrapy.settings import Settings +from scrapy.utils.python import to_unicode +from scrapy.utils.test import get_crawler +from tests.spiders import ItemSpider +from tests.test_feedexport import TestFeedExportBase +from tests.utils.decorators import coroutine_test, inline_callbacks_test + +if TYPE_CHECKING: + from os import PathLike + + +def build_url(path: str | PathLike[str]) -> str: + path_str = str(path) + if path_str[0] != "/": + path_str = "/" + path_str + return urljoin("file:", path_str) + + +class TestBatchDeliveries(TestFeedExportBase): + _file_mark = "_%(batch_time)s_#%(batch_id)02d_" + + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, list[bytes]]: + """Run spider with specified settings; return exported data.""" + + FEEDS = settings.get("FEEDS") or {} + settings["FEEDS"] = { + build_url(file_path): feed for file_path, feed in FEEDS.items() + } + content: defaultdict[str, list[bytes]] = defaultdict(list) + spider_cls.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(spider_cls, settings) + await crawler.crawl_async() + + for path, feed in FEEDS.items(): + dir_name = Path(path).parent + if not dir_name.exists(): + content[feed["format"]] = [] + continue + for file in sorted(dir_name.iterdir()): + content[feed["format"]].append(file.read_bytes()) + return content + + async def assertExportedJsonLines(self, items, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "jl" / self._file_mark: { + "format": "jl" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = await self.exported_data(items, settings) + for batch in data["jl"]: + got_batch = [ + json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines() + ] + expected_batch, rows = rows[:batch_size], rows[batch_size:] + assert got_batch == expected_batch + + async def assertExportedCsv(self, items, header, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "csv" / self._file_mark: { + "format": "csv" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + data = await self.exported_data(items, settings) + for batch in data["csv"]: + got_batch = csv.DictReader(to_unicode(batch).splitlines()) + assert list(header) == got_batch.fieldnames + expected_batch, rows = rows[:batch_size], rows[batch_size:] + assert list(got_batch) == expected_batch + + async def assertExportedXml(self, items, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "xml" / self._file_mark: { + "format": "xml" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = await self.exported_data(items, settings) + for batch in data["xml"]: + root = lxml.etree.fromstring(batch) + got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] + expected_batch, rows = rows[:batch_size], rows[batch_size:] + assert got_batch == expected_batch + + async def assertExportedMultiple(self, items, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "xml" / self._file_mark: { + "format": "xml" + }, + self._random_temp_filename() / "json" / self._file_mark: { + "format": "json" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = await self.exported_data(items, settings) + # XML + xml_rows = rows.copy() + for batch in data["xml"]: + root = lxml.etree.fromstring(batch) + got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] + expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:] + assert got_batch == expected_batch + # JSON + json_rows = rows.copy() + for batch in data["json"]: + got_batch = json.loads(batch.decode("utf-8")) + expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] + assert got_batch == expected_batch + + async def assertExportedPickle(self, items, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "pickle" / self._file_mark: { + "format": "pickle" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = await self.exported_data(items, settings) + + for batch in data["pickle"]: + got_batch = self._load_until_eof(batch, load_func=pickle.load) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + assert got_batch == expected_batch + + async def assertExportedMarshal(self, items, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "marshal" / self._file_mark: { + "format": "marshal" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = await self.exported_data(items, settings) + + for batch in data["marshal"]: + got_batch = self._load_until_eof(batch, load_func=marshal.load) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + assert got_batch == expected_batch + + @coroutine_test + async def test_export_items(self): + """Test partial deliveries in all supported formats""" + items = [ + self.MyItem({"foo": "bar1", "egg": "spam1"}), + self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + self.MyItem({"foo": "bar3", "baz": "quux3"}), + ] + rows = [ + {"egg": "spam1", "foo": "bar1", "baz": ""}, + {"egg": "spam2", "foo": "bar2", "baz": "quux2"}, + {"foo": "bar3", "baz": "quux3", "egg": ""}, + ] + settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2} + header = self.MyItem.fields.keys() + await self.assertExported(items, header, rows, settings=settings) + + def test_wrong_path(self): + """If path is without %(batch_time)s and %(batch_id) an exception must be raised""" + settings = { + "FEEDS": { + self._random_temp_filename(): {"format": "xml"}, + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + crawler = get_crawler(settings_dict=settings) + with pytest.raises(NotConfigured): + FeedExporter(crawler) + + @coroutine_test + async def test_export_no_items_not_store_empty(self): + for fmt in ("json", "jsonlines", "xml", "csv"): + settings = { + "FEEDS": { + self._random_temp_filename() / fmt / self._file_mark: { + "format": fmt + }, + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + "FEED_STORE_EMPTY": False, + } + data = await self.exported_no_data(settings) + data = dict(data) + assert len(data[fmt]) == 0 + + @coroutine_test + async def test_export_no_items_store_empty(self): + formats = ( + ("json", b"[]"), + ("jsonlines", b""), + ("xml", b'\n'), + ("csv", b""), + ) + + for fmt, expctd in formats: + settings = { + "FEEDS": { + self._random_temp_filename() / fmt / self._file_mark: { + "format": fmt + }, + }, + "FEED_STORE_EMPTY": True, + "FEED_EXPORT_INDENT": None, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + data = await self.exported_no_data(settings) + data = dict(data) + assert data[fmt][0] == expctd + + @coroutine_test + async def test_export_multiple_configs(self): + items = [ + {"foo": "FOO", "bar": "BAR"}, + {"foo": "FOO1", "bar": "BAR1"}, + ] + + formats = { + "json": [ + b'[\n{"bar": "BAR"}\n]', + b'[\n{"bar": "BAR1"}\n]', + ], + "xml": [ + ( + b'\n' + b"\n \n FOO\n \n" + ), + ( + b'\n' + b"\n \n FOO1\n \n" + ), + ], + "csv": [ + b"foo,bar\r\nFOO,BAR\r\n", + b"foo,bar\r\nFOO1,BAR1\r\n", + ], + } + + settings = { + "FEEDS": { + self._random_temp_filename() / "json" / self._file_mark: { + "format": "json", + "indent": 0, + "fields": ["bar"], + "encoding": "utf-8", + }, + self._random_temp_filename() / "xml" / self._file_mark: { + "format": "xml", + "indent": 2, + "fields": ["foo"], + "encoding": "latin-1", + }, + self._random_temp_filename() / "csv" / self._file_mark: { + "format": "csv", + "indent": None, + "fields": ["foo", "bar"], + "encoding": "utf-8", + }, + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + data = await self.exported_data(items, settings) + for fmt, expected in formats.items(): + for expected_batch, got_batch in zip(expected, data[fmt], strict=False): + assert got_batch == expected_batch + + @coroutine_test + async def test_batch_item_count_feeds_setting(self): + items = [{"foo": "FOO"}, {"foo": "FOO1"}] + formats = { + "json": [ + b'[{"foo": "FOO"}]', + b'[{"foo": "FOO1"}]', + ], + } + settings = { + "FEEDS": { + self._random_temp_filename() / "json" / self._file_mark: { + "format": "json", + "indent": None, + "encoding": "utf-8", + "batch_item_count": 1, + }, + }, + } + data = await self.exported_data(items, settings) + for fmt, expected in formats.items(): + for expected_batch, got_batch in zip(expected, data[fmt], strict=False): + assert got_batch == expected_batch + + @coroutine_test + async def test_batch_path_differ(self): + """ + Test that the name of all batch files differ from each other. + So %(batch_id)d replaced with the current id. + """ + items = [ + self.MyItem({"foo": "bar1", "egg": "spam1"}), + self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + self.MyItem({"foo": "bar3", "baz": "quux3"}), + ] + settings = { + "FEEDS": { + self._random_temp_filename() / "%(batch_id)d": { + "format": "json", + }, + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + data = await self.exported_data(items, settings) + assert len(items) == len(data["json"]) + + @inline_callbacks_test + def test_stats_batch_file_success(self): + settings = { + "FEEDS": { + build_url( + str(self._random_temp_filename() / "json" / self._file_mark) + ): { + "format": "json", + } + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + crawler = get_crawler(ItemSpider, settings) + yield crawler.crawl(total=2, mockserver=self.mockserver) + assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() + assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12 + + @pytest.mark.requires_boto3 + @inline_callbacks_test + def test_s3_export(self): + bucket = "mybucket" + items = [ + self.MyItem({"foo": "bar1", "egg": "spam1"}), + self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + self.MyItem({"foo": "bar3", "baz": "quux3"}), + ] + + class CustomS3FeedStorage(S3FeedStorage): + stubs = [] + + def open(self, *args, **kwargs): + from botocore import __version__ as botocore_version # noqa: PLC0415 + from botocore.stub import ANY, Stubber # noqa: PLC0415 + + expected_params = { + "Body": ANY, + "Bucket": bucket, + "Key": ANY, + } + if Version(botocore_version) >= Version("1.36.0"): + expected_params["ChecksumAlgorithm"] = ANY + + stub = Stubber(self.s3_client) + stub.activate() + CustomS3FeedStorage.stubs.append(stub) + stub.add_response( + "put_object", + expected_params=expected_params, + service_response={}, + ) + return super().open(*args, **kwargs) + + key = "export.csv" + uri = f"s3://{bucket}/{key}/%(batch_id)d.json" + batch_item_count = 1 + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "FEED_EXPORT_BATCH_ITEM_COUNT": batch_item_count, + "FEED_STORAGES": { + "s3": CustomS3FeedStorage, + }, + "FEEDS": { + uri: { + "format": "json", + }, + }, + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, uri) + verifyObject(IFeedStorage, storage) + + class TestSpider(scrapy.Spider): + name = "testspider" + + def parse(self, response): + yield from items + + TestSpider.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(TestSpider, settings) + yield crawler.crawl() + + assert len(CustomS3FeedStorage.stubs) == len(items) + for stub in CustomS3FeedStorage.stubs[:-1]: + stub.assert_no_pending_responses() + assert ( + "feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats() + ) + assert ( + crawler.stats.get_value("feedexport/success_count/CustomS3FeedStorage") == 3 + ) diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py new file mode 100644 index 000000000..fa1c0586a --- /dev/null +++ b/tests/test_feedexport_postprocess.py @@ -0,0 +1,538 @@ +from __future__ import annotations + +import bz2 +import gzip +import lzma +import marshal +import pickle +import sys +from io import BytesIO +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest + +from scrapy.utils.test import get_crawler +from tests.test_feedexport import TestFeedExportBase, path_to_url, printf_escape +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from scrapy import Spider + + +class TestFeedPostProcessedExports(TestFeedExportBase): + items = [{"foo": "bar"}] + expected = b"foo\r\nbar\r\n" + + class MyPlugin1: + def __init__(self, file, feed_options): + self.file = file + self.feed_options = feed_options + self.char = self.feed_options.get("plugin1_char", b"") + + def write(self, data): + written_count = self.file.write(data) + written_count += self.file.write(self.char) + return written_count + + def close(self): + self.file.close() + + def _named_tempfile(self, name) -> str: + return str(Path(self.temp_dir, name)) + + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, bytes | None]: + """Run spider with specified settings; return exported data with filename.""" + + FEEDS = settings.get("FEEDS") or {} + settings["FEEDS"] = { + printf_escape(path_to_url(file_path)): feed_options + for file_path, feed_options in FEEDS.items() + } + + content: dict[str, bytes | None] = {} + try: + spider_cls.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(spider_cls, settings) + await crawler.crawl_async() + + for file_path in FEEDS: + content[str(file_path)] = ( + Path(file_path).read_bytes() if Path(file_path).exists() else None + ) + + finally: + for file_path in FEEDS: + if not Path(file_path).exists(): + continue + + Path(file_path).unlink() + + return content + + def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=""): + data_stream = BytesIO() + gzipf = gzip.GzipFile( + fileobj=data_stream, + filename=filename, + mtime=mtime, + compresslevel=compresslevel, + mode="wb", + ) + gzipf.write(data) + gzipf.close() + data_stream.seek(0) + return data_stream.read() + + @coroutine_test + async def test_gzip_plugin(self): + filename = self._named_tempfile("gzip_file") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + }, + }, + } + + data = await self.exported_data(self.items, settings) + try: + gzip.decompress(data[filename]) + except OSError: + pytest.fail("Received invalid gzip data.") + + @coroutine_test + async def test_gzip_plugin_compresslevel(self): + filename_to_compressed = { + self._named_tempfile("compresslevel_0"): self.get_gzip_compressed( + self.expected, compresslevel=0 + ), + self._named_tempfile("compresslevel_9"): self.get_gzip_compressed( + self.expected, compresslevel=9 + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("compresslevel_0"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_compresslevel": 0, + "gzip_mtime": 0, + "gzip_filename": "", + }, + self._named_tempfile("compresslevel_9"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_compresslevel": 9, + "gzip_mtime": 0, + "gzip_filename": "", + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_gzip_plugin_mtime(self): + filename_to_compressed = { + self._named_tempfile("mtime_123"): self.get_gzip_compressed( + self.expected, mtime=123 + ), + self._named_tempfile("mtime_123456789"): self.get_gzip_compressed( + self.expected, mtime=123456789 + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("mtime_123"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_mtime": 123, + "gzip_filename": "", + }, + self._named_tempfile("mtime_123456789"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_mtime": 123456789, + "gzip_filename": "", + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_gzip_plugin_filename(self): + filename_to_compressed = { + self._named_tempfile("filename_FILE1"): self.get_gzip_compressed( + self.expected, filename="FILE1" + ), + self._named_tempfile("filename_FILE2"): self.get_gzip_compressed( + self.expected, filename="FILE2" + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("filename_FILE1"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_mtime": 0, + "gzip_filename": "FILE1", + }, + self._named_tempfile("filename_FILE2"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_mtime": 0, + "gzip_filename": "FILE2", + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_lzma_plugin(self): + filename = self._named_tempfile("lzma_file") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + }, + }, + } + + data = await self.exported_data(self.items, settings) + try: + lzma.decompress(data[filename]) + except lzma.LZMAError: + pytest.fail("Received invalid lzma data.") + + @coroutine_test + async def test_lzma_plugin_format(self): + filename_to_compressed = { + self._named_tempfile("format_FORMAT_XZ"): lzma.compress( + self.expected, format=lzma.FORMAT_XZ + ), + self._named_tempfile("format_FORMAT_ALONE"): lzma.compress( + self.expected, format=lzma.FORMAT_ALONE + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("format_FORMAT_XZ"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_format": lzma.FORMAT_XZ, + }, + self._named_tempfile("format_FORMAT_ALONE"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_format": lzma.FORMAT_ALONE, + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_lzma_plugin_check(self): + filename_to_compressed = { + self._named_tempfile("check_CHECK_NONE"): lzma.compress( + self.expected, check=lzma.CHECK_NONE + ), + self._named_tempfile("check_CHECK_CRC256"): lzma.compress( + self.expected, check=lzma.CHECK_SHA256 + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("check_CHECK_NONE"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_check": lzma.CHECK_NONE, + }, + self._named_tempfile("check_CHECK_CRC256"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_check": lzma.CHECK_SHA256, + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_lzma_plugin_preset(self): + filename_to_compressed = { + self._named_tempfile("preset_PRESET_0"): lzma.compress( + self.expected, preset=0 + ), + self._named_tempfile("preset_PRESET_9"): lzma.compress( + self.expected, preset=9 + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("preset_PRESET_0"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_preset": 0, + }, + self._named_tempfile("preset_PRESET_9"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_preset": 9, + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_lzma_plugin_filters(self): + if "PyPy" in sys.version: + # https://foss.heptapod.net/pypy/pypy/-/issues/3527 + pytest.skip("lzma filters doesn't work in PyPy") + + filters = [{"id": lzma.FILTER_LZMA2}] + compressed = lzma.compress(self.expected, filters=filters) + filename = self._named_tempfile("filters") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_filters": filters, + }, + }, + } + + data = await self.exported_data(self.items, settings) + assert compressed == data[filename] + result = lzma.decompress(data[filename]) + assert result == self.expected + + @coroutine_test + async def test_bz2_plugin(self): + filename = self._named_tempfile("bz2_file") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], + }, + }, + } + + data = await self.exported_data(self.items, settings) + try: + bz2.decompress(data[filename]) + except OSError: + pytest.fail("Received invalid bz2 data.") + + @coroutine_test + async def test_bz2_plugin_compresslevel(self): + filename_to_compressed = { + self._named_tempfile("compresslevel_1"): bz2.compress( + self.expected, compresslevel=1 + ), + self._named_tempfile("compresslevel_9"): bz2.compress( + self.expected, compresslevel=9 + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("compresslevel_1"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], + "bz2_compresslevel": 1, + }, + self._named_tempfile("compresslevel_9"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], + "bz2_compresslevel": 9, + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = bz2.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_custom_plugin(self): + filename = self._named_tempfile("csv_file") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": [self.MyPlugin1], + }, + }, + } + + data = await self.exported_data(self.items, settings) + assert data[filename] == self.expected + + @coroutine_test + async def test_custom_plugin_with_parameter(self): + expected = b"foo\r\n\nbar\r\n\n" + filename = self._named_tempfile("newline") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": [self.MyPlugin1], + "plugin1_char": b"\n", + }, + }, + } + + data = await self.exported_data(self.items, settings) + assert data[filename] == expected + + @coroutine_test + async def test_custom_plugin_with_compression(self): + expected = b"foo\r\n\nbar\r\n\n" + + filename_to_decompressor = { + self._named_tempfile("bz2"): bz2.decompress, + self._named_tempfile("lzma"): lzma.decompress, + self._named_tempfile("gzip"): gzip.decompress, + } + + settings = { + "FEEDS": { + self._named_tempfile("bz2"): { + "format": "csv", + "postprocessing": [ + self.MyPlugin1, + "scrapy.extensions.postprocessing.Bz2Plugin", + ], + "plugin1_char": b"\n", + }, + self._named_tempfile("lzma"): { + "format": "csv", + "postprocessing": [ + self.MyPlugin1, + "scrapy.extensions.postprocessing.LZMAPlugin", + ], + "plugin1_char": b"\n", + }, + self._named_tempfile("gzip"): { + "format": "csv", + "postprocessing": [ + self.MyPlugin1, + "scrapy.extensions.postprocessing.GzipPlugin", + ], + "plugin1_char": b"\n", + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, decompressor in filename_to_decompressor.items(): + result = decompressor(data[filename]) + assert result == expected + + @coroutine_test + async def test_exports_compatibility_with_postproc(self): + filename_to_expected = { + self._named_tempfile("csv"): b"foo\r\nbar\r\n", + self._named_tempfile("json"): b'[\n{"foo": "bar"}\n]', + self._named_tempfile("jsonlines"): b'{"foo": "bar"}\n', + self._named_tempfile("xml"): b'\n' + b"\nbar\n", + } + + settings = { + "FEEDS": { + self._named_tempfile("csv"): { + "format": "csv", + "postprocessing": [self.MyPlugin1], + # empty plugin to activate postprocessing.PostProcessingManager + }, + self._named_tempfile("json"): { + "format": "json", + "postprocessing": [self.MyPlugin1], + }, + self._named_tempfile("jsonlines"): { + "format": "jsonlines", + "postprocessing": [self.MyPlugin1], + }, + self._named_tempfile("xml"): { + "format": "xml", + "postprocessing": [self.MyPlugin1], + }, + self._named_tempfile("marshal"): { + "format": "marshal", + "postprocessing": [self.MyPlugin1], + }, + self._named_tempfile("pickle"): { + "format": "pickle", + "postprocessing": [self.MyPlugin1], + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, result in data.items(): + if "pickle" in filename: + expected, result = self.items[0], pickle.loads(result) + elif "marshal" in filename: + expected, result = self.items[0], marshal.loads(result) + else: + expected = filename_to_expected[filename] + assert result == expected diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py new file mode 100644 index 000000000..11e79775f --- /dev/null +++ b/tests/test_feedexport_storages.py @@ -0,0 +1,583 @@ +from __future__ import annotations + +import os +import string +import tempfile +from io import BytesIO +from pathlib import Path +from typing import IO, Any +from unittest import mock +from urllib.parse import quote + +import pytest +from testfixtures import LogCapture +from w3lib.url import path_to_file_uri +from zope.interface.verify import verifyObject + +import scrapy +from scrapy.extensions.feedexport import ( + BlockingFeedStorage, + FileFeedStorage, + FTPFeedStorage, + GCSFeedStorage, + IFeedStorage, + S3FeedStorage, + StdoutFeedStorage, +) +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.test import get_crawler +from tests.mockserver.ftp import MockFTPServer +from tests.utils.decorators import coroutine_test + + +def mock_google_cloud_storage() -> tuple[Any, Any, Any]: + """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob + classes and set their proper return values. + """ + from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415 + + client_mock = mock.create_autospec(Client) + + bucket_mock = mock.create_autospec(Bucket) + client_mock.get_bucket.return_value = bucket_mock + + blob_mock = mock.create_autospec(Blob) + bucket_mock.blob.return_value = blob_mock + + return (client_mock, bucket_mock, blob_mock) + + +class TestFileFeedStorage: + def test_store_file_uri(self, tmp_path): + path = tmp_path / "file.txt" + uri = path_to_file_uri(str(path)) + self._assert_stores(FileFeedStorage(uri), path) + + def test_store_file_uri_makedirs(self, tmp_path): + path = tmp_path / "more" / "paths" / "file.txt" + uri = path_to_file_uri(str(path)) + self._assert_stores(FileFeedStorage(uri), path) + + def test_store_direct_path(self, tmp_path): + path = tmp_path / "file.txt" + self._assert_stores(FileFeedStorage(str(path)), path) + + def test_store_direct_path_relative(self, tmp_path): + old_cwd = Path.cwd() + try: + os.chdir(tmp_path) + path = Path("foo", "bar") + self._assert_stores(FileFeedStorage(str(path)), path) + finally: + os.chdir(old_cwd) + + def test_interface(self, tmp_path): + path = tmp_path / "file.txt" + st = FileFeedStorage(str(path)) + verifyObject(IFeedStorage, st) + + @staticmethod + def _store(path: Path, feed_options: dict[str, Any] | None = None) -> None: + storage = FileFeedStorage(str(path), feed_options=feed_options) + spider = scrapy.Spider("default") + file = storage.open(spider) + file.write(b"content") + storage.store(file) + + def test_append(self, tmp_path): + path = tmp_path / "file.txt" + self._store(path) + self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent") + + def test_overwrite(self, tmp_path): + path = tmp_path / "file.txt" + self._store(path, {"overwrite": True}) + self._assert_stores( + FileFeedStorage(str(path), feed_options={"overwrite": True}), path + ) + + @staticmethod + def _assert_stores( + storage: FileFeedStorage, path: Path, expected_content: bytes = b"content" + ) -> None: + spider = scrapy.Spider("default") + file = storage.open(spider) + file.write(b"content") + storage.store(file) + assert path.exists() + try: + assert path.read_bytes() == expected_content + finally: + path.unlink() + + def test_preserves_windows_path_without_file_scheme(self): + path = r"C:\Users\user\Desktop\test.txt" + storage = FileFeedStorage(path) + assert storage.path == path + + +class TestFTPFeedStorage: + def get_test_spider(self, settings=None): + class TestSpider(scrapy.Spider): + name = "test_spider" + + crawler = get_crawler(settings_dict=settings) + return TestSpider.from_crawler(crawler) + + async def _store(self, uri, content, feed_options=None, settings=None): + crawler = get_crawler(settings_dict=settings or {}) + storage = FTPFeedStorage.from_crawler( + crawler, + uri, + feed_options=feed_options, + ) + verifyObject(IFeedStorage, storage) + spider = self.get_test_spider() + file = storage.open(spider) + file.write(content) + await maybe_deferred_to_future(storage.store(file)) + + def _assert_stored(self, path: Path, content): + assert path.exists() + try: + assert path.read_bytes() == content + finally: + path.unlink() + + @coroutine_test + async def test_append(self): + with MockFTPServer() as ftp_server: + filename = "file" + url = ftp_server.url(filename) + feed_options = {"overwrite": False} + await self._store(url, b"foo", feed_options=feed_options) + await self._store(url, b"bar", feed_options=feed_options) + self._assert_stored(ftp_server.path / filename, b"foobar") + + @coroutine_test + async def test_overwrite(self): + with MockFTPServer() as ftp_server: + filename = "file" + url = ftp_server.url(filename) + await self._store(url, b"foo") + await self._store(url, b"bar") + self._assert_stored(ftp_server.path / filename, b"bar") + + @coroutine_test + async def test_append_active_mode(self): + with MockFTPServer() as ftp_server: + settings = {"FEED_STORAGE_FTP_ACTIVE": True} + filename = "file" + url = ftp_server.url(filename) + feed_options = {"overwrite": False} + await self._store(url, b"foo", feed_options=feed_options, settings=settings) + await self._store(url, b"bar", feed_options=feed_options, settings=settings) + self._assert_stored(ftp_server.path / filename, b"foobar") + + @coroutine_test + async def test_overwrite_active_mode(self): + with MockFTPServer() as ftp_server: + settings = {"FEED_STORAGE_FTP_ACTIVE": True} + filename = "file" + url = ftp_server.url(filename) + await self._store(url, b"foo", settings=settings) + await self._store(url, b"bar", settings=settings) + self._assert_stored(ftp_server.path / filename, b"bar") + + def test_uri_auth_quote(self): + # RFC3986: 3.2.1. User Information + pw_quoted = quote(string.punctuation, safe="") + st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {}) + assert st.password == string.punctuation + + +class MyBlockingFeedStorage(BlockingFeedStorage): + def _store_in_thread(self, file: IO[bytes]) -> None: + return + + +class TestBlockingFeedStorage: + def get_test_spider(self, settings=None): + class TestSpider(scrapy.Spider): + name = "test_spider" + + crawler = get_crawler(settings_dict=settings) + return TestSpider.from_crawler(crawler) + + def test_default_temp_dir(self): + b = MyBlockingFeedStorage() + + storage_file = b.open(self.get_test_spider()) + storage_dir = Path(storage_file.name).parent + assert str(storage_dir) == tempfile.gettempdir() + + def test_temp_file(self, tmp_path): + b = MyBlockingFeedStorage() + + spider = self.get_test_spider({"FEED_TEMPDIR": str(tmp_path)}) + storage_file = b.open(spider) + storage_dir = Path(storage_file.name).parent + assert storage_dir == tmp_path + + def test_invalid_folder(self, tmp_path): + b = MyBlockingFeedStorage() + + invalid_path = tmp_path / "invalid_path" + spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)}) + + with pytest.raises(OSError, match="Not a Directory:"): + b.open(spider=spider) + + +@pytest.mark.requires_boto3 +class TestS3FeedStorage: + def test_parse_credentials(self): + aws_credentials = { + "AWS_ACCESS_KEY_ID": "settings_key", + "AWS_SECRET_ACCESS_KEY": "settings_secret", + "AWS_SESSION_TOKEN": "settings_token", + } + crawler = get_crawler(settings_dict=aws_credentials) + # Instantiate with crawler + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + assert storage.access_key == "settings_key" + assert storage.secret_key == "settings_secret" + assert storage.session_token == "settings_token" + # Instantiate directly + storage = S3FeedStorage( + "s3://mybucket/export.csv", + aws_credentials["AWS_ACCESS_KEY_ID"], + aws_credentials["AWS_SECRET_ACCESS_KEY"], + session_token=aws_credentials["AWS_SESSION_TOKEN"], + ) + assert storage.access_key == "settings_key" + assert storage.secret_key == "settings_secret" + assert storage.session_token == "settings_token" + # URI priority > settings priority + storage = S3FeedStorage( + "s3://uri_key:uri_secret@mybucket/export.csv", + aws_credentials["AWS_ACCESS_KEY_ID"], + aws_credentials["AWS_SECRET_ACCESS_KEY"], + ) + assert storage.access_key == "uri_key" + assert storage.secret_key == "uri_secret" + + @coroutine_test + async def test_store(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + } + crawler = get_crawler(settings_dict=settings) + bucket = "mybucket" + key = "export.csv" + storage = S3FeedStorage.from_crawler(crawler, f"s3://{bucket}/{key}") + verifyObject(IFeedStorage, storage) + + file = mock.MagicMock() + + storage.s3_client = mock.MagicMock() + await maybe_deferred_to_future(storage.store(file)) + assert storage.s3_client.upload_fileobj.call_args == mock.call( + Bucket=bucket, Key=key, Fileobj=file + ) + + def test_init_without_acl(self): + storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl is None + + def test_init_with_acl(self): + storage = S3FeedStorage( + "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl == "custom-acl" + + def test_init_with_endpoint_url(self): + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + endpoint_url="https://example.com", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.endpoint_url == "https://example.com" + + def test_init_with_region_name(self): + region_name = "ap-east-1" + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + region_name=region_name, + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.region_name == region_name + assert storage.s3_client._client_config.region_name == region_name + + def test_from_crawler_without_acl(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl is None + + def test_without_endpoint_url(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.endpoint_url is None + + def test_without_region_name(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.s3_client._client_config.region_name == "us-east-1" + + def test_from_crawler_with_acl(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "FEED_STORAGE_S3_ACL": "custom-acl", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl == "custom-acl" + + def test_from_crawler_with_endpoint_url(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "AWS_ENDPOINT_URL": "https://example.com", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.endpoint_url == "https://example.com" + + def test_from_crawler_with_region_name(self): + region_name = "ap-east-1" + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "AWS_REGION_NAME": region_name, + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.region_name == region_name + assert storage.s3_client._client_config.region_name == region_name + + @coroutine_test + async def test_store_without_acl(self): + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl is None + + storage.s3_client = mock.MagicMock() + await maybe_deferred_to_future(storage.store(BytesIO(b"test file"))) + acl = ( + storage.s3_client.upload_fileobj.call_args[1] + .get("ExtraArgs", {}) + .get("ACL") + ) + assert acl is None + + @coroutine_test + async def test_store_with_acl(self): + storage = S3FeedStorage( + "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl == "custom-acl" + + storage.s3_client = mock.MagicMock() + await maybe_deferred_to_future(storage.store(BytesIO(b"test file"))) + acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] + assert acl == "custom-acl" + + def test_overwrite_default(self): + with LogCapture() as log: + S3FeedStorage( + "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" + ) + assert "S3 does not support appending to files" not in str(log) + + def test_overwrite_false(self): + with LogCapture() as log: + S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + "custom-acl", + feed_options={"overwrite": False}, + ) + assert "S3 does not support appending to files" in str(log) + + +class TestGCSFeedStorage: + def test_parse_settings(self): + try: + from google.cloud.storage import Client # noqa: F401,PLC0415 + except ImportError: + pytest.skip("GCSFeedStorage requires google-cloud-storage") + + settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"} + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") + assert storage.project_id == "123" + assert storage.acl == "publicRead" + assert storage.bucket_name == "mybucket" + assert storage.blob_name == "export.csv" + + def test_parse_empty_acl(self): + try: + from google.cloud.storage import Client # noqa: F401,PLC0415 + except ImportError: + pytest.skip("GCSFeedStorage requires google-cloud-storage") + + settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""} + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") + assert storage.acl is None + + settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": None} + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") + assert storage.acl is None + + @coroutine_test + async def test_store(self): + try: + from google.cloud.storage import Client # noqa: F401,PLC0415 + except ImportError: + pytest.skip("GCSFeedStorage requires google-cloud-storage") + + uri = "gs://mybucket/export.csv" + project_id = "myproject-123" + acl = "publicRead" + (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() + with mock.patch("google.cloud.storage.Client") as m: + m.return_value = client_mock + + f = mock.Mock() + storage = GCSFeedStorage(uri, project_id, acl) + await maybe_deferred_to_future(storage.store(f)) + + f.seek.assert_called_once_with(0) + m.assert_called_once_with(project=project_id) + client_mock.get_bucket.assert_called_once_with("mybucket") + bucket_mock.blob.assert_called_once_with("export.csv") + blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) + f.close.assert_called_once_with() + + @coroutine_test + async def test_store_closes_file_on_upload_error(self): + try: + from google.cloud.storage import Client # noqa: F401,PLC0415 + except ImportError: + pytest.skip("GCSFeedStorage requires google-cloud-storage") + + uri = "gs://mybucket/export.csv" + project_id = "myproject-123" + acl = "publicRead" + (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() + blob_mock.upload_from_file.side_effect = OSError("Upload failed") + with mock.patch("google.cloud.storage.Client") as m: + m.return_value = client_mock + + f = mock.Mock() + storage = GCSFeedStorage(uri, project_id, acl) + with pytest.raises(OSError, match="Upload failed"): + await maybe_deferred_to_future(storage.store(f)) + + f.seek.assert_called_once_with(0) + m.assert_called_once_with(project=project_id) + client_mock.get_bucket.assert_called_once_with("mybucket") + bucket_mock.blob.assert_called_once_with("export.csv") + blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) + f.close.assert_called_once_with() + + def test_overwrite_default(self): + with LogCapture() as log: + GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl") + assert "GCS does not support appending to files" not in str(log) + + def test_overwrite_false(self): + with LogCapture() as log: + GCSFeedStorage( + "gs://mybucket/export.csv", + "myproject-123", + "custom-acl", + feed_options={"overwrite": False}, + ) + assert "GCS does not support appending to files" in str(log) + + +class TestStdoutFeedStorage: + def test_store(self): + out = BytesIO() + storage = StdoutFeedStorage("stdout:", _stdout=out) + file = storage.open(scrapy.Spider("default")) + file.write(b"content") + storage.store(file) + assert out.getvalue() == b"content" + + def test_overwrite_default(self): + with LogCapture() as log: + StdoutFeedStorage("stdout:") + assert ( + "Standard output (stdout) storage does not support overwriting" + not in str(log) + ) + + def test_overwrite_true(self): + with LogCapture() as log: + StdoutFeedStorage("stdout:", feed_options={"overwrite": True}) + assert "Standard output (stdout) storage does not support overwriting" in str( + log + ) diff --git a/tests/test_feedexport_uri_params.py b/tests/test_feedexport_uri_params.py new file mode 100644 index 000000000..150d8449f --- /dev/null +++ b/tests/test_feedexport_uri_params.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import warnings +from abc import ABC, abstractmethod + +import pytest + +import scrapy +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.extensions.feedexport import FeedExporter +from scrapy.utils.test import get_crawler + + +class TestURIParams(ABC): + spider_name = "uri_params_spider" + deprecated_options = False + + @abstractmethod + def build_settings(self, uri="file:///tmp/foobar", uri_params=None): + raise NotImplementedError + + def _crawler_feed_exporter(self, settings): + if self.deprecated_options: + with pytest.warns( + ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", + ): + crawler = get_crawler(settings_dict=settings) + else: + crawler = get_crawler(settings_dict=settings) + feed_exporter = crawler.get_extension(FeedExporter) + return crawler, feed_exporter + + def test_default(self): + settings = self.build_settings( + uri="file:///tmp/%(name)s", + ) + crawler, feed_exporter = self._crawler_feed_exporter(settings) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) + feed_exporter.open_spider(spider) + + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" + + def test_none(self): + def uri_params(params, spider): + pass + + settings = self.build_settings( + uri="file:///tmp/%(name)s", + uri_params=uri_params, + ) + crawler, feed_exporter = self._crawler_feed_exporter(settings) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + + feed_exporter.open_spider(spider) + + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" + + def test_empty_dict(self): + def uri_params(params, spider): + return {} + + settings = self.build_settings( + uri="file:///tmp/%(name)s", + uri_params=uri_params, + ) + crawler, feed_exporter = self._crawler_feed_exporter(settings) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) + with pytest.raises(KeyError): + feed_exporter.open_spider(spider) + + def test_params_as_is(self): + def uri_params(params, spider): + return params + + settings = self.build_settings( + uri="file:///tmp/%(name)s", + uri_params=uri_params, + ) + crawler, feed_exporter = self._crawler_feed_exporter(settings) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) + feed_exporter.open_spider(spider) + + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" + + def test_custom_param(self): + def uri_params(params, spider): + return {**params, "foo": self.spider_name} + + settings = self.build_settings( + uri="file:///tmp/%(foo)s", + uri_params=uri_params, + ) + crawler, feed_exporter = self._crawler_feed_exporter(settings) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) + feed_exporter.open_spider(spider) + + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" + + +class TestURIParamsSetting(TestURIParams): + deprecated_options = True + + def build_settings(self, uri="file:///tmp/foobar", uri_params=None): + extra_settings = {} + if uri_params: + extra_settings["FEED_URI_PARAMS"] = uri_params + return { + "FEED_URI": uri, + **extra_settings, + } + + +class TestURIParamsFeedOption(TestURIParams): + deprecated_options = False + + def build_settings(self, uri="file:///tmp/foobar", uri_params=None): + options = { + "format": "jl", + } + if uri_params: + options["uri_params"] = uri_params + return { + "FEEDS": { + uri: options, + }, + } diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 8fdf3d56f..cec5d728b 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,43 +1,57 @@ +from __future__ import annotations + import json import random import re -import shutil import string from ipaddress import IPv4Address from pathlib import Path -from typing import Dict -from unittest import mock, skipIf +from typing import TYPE_CHECKING, Any, cast +from unittest import mock from urllib.parse import urlencode -from twisted.internet import reactor -from twisted.internet.defer import ( - CancelledError, - Deferred, - DeferredList, - inlineCallbacks, -) +import pytest +from pytest_twisted import async_yield_fixture +from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint -from twisted.internet.error import TimeoutError from twisted.internet.ssl import Certificate, PrivateCertificate, optionsForClientTLS -from twisted.python.failure import Failure -from twisted.trial.unittest import TestCase from twisted.web.client import URI, ResponseFailed from twisted.web.http import H2_ENABLED from twisted.web.http import Request as TxRequest from twisted.web.server import NOT_DONE_YET, Site from twisted.web.static import File +from scrapy.exceptions import DownloadCancelledError, DownloadTimeoutError from scrapy.http import JsonRequest, Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider -from tests.mockserver import LeafResource, Status, ssl_context_factory +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + deferred_from_coro, + maybe_deferred_to_future, +) +from tests.mockserver.http_resources import LeafResource, Status +from tests.mockserver.utils import ssl_context_factory + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Callable, Coroutine, Generator + + from scrapy.core.http2.protocol import H2ClientProtocol -def generate_random_string(size): +pytestmark = [ + pytest.mark.requires_reactor, # H2ClientProtocol requires a reactor + pytest.mark.skipif( + not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled" + ), +] + + +def generate_random_string(size: int) -> str: return "".join(random.choices(string.ascii_uppercase + string.digits, k=size)) -def make_html_body(val): +def make_html_body(val: str) -> bytes: response = f"""

Hello from HTTP2

{val}

@@ -47,7 +61,7 @@ def make_html_body(val): class DummySpider(Spider): name = "dummy" - start_urls: list = [] + start_urls = [] def parse(self, response): print(response) @@ -87,7 +101,7 @@ class GetDataHtmlLarge(LeafResource): class PostDataJsonMixin: @staticmethod - def make_response(request: TxRequest, extra_data: str): + def make_response(request: TxRequest, extra_data: str) -> bytes: assert request.content is not None response = { "request-headers": {}, @@ -147,7 +161,7 @@ class QueryParams(LeafResource): request.setHeader("Content-Type", "application/json; charset=UTF-8") request.setHeader("Content-Encoding", "UTF-8") - query_params: Dict[str, str] = {} + query_params: dict[str, str] = {} assert request.args is not None for k, v in request.args.items(): query_params[str(k, "utf-8")] = str(v[0], "utf-8") @@ -168,26 +182,23 @@ class RequestHeaders(LeafResource): return bytes(json.dumps(headers), "utf-8") -def get_client_certificate( - key_file: Path, certificate_file: Path -) -> PrivateCertificate: - pem = key_file.read_text(encoding="utf-8") + certificate_file.read_text( - encoding="utf-8" - ) - - return PrivateCertificate.loadPEM(pem) +def make_request_dfd(client: H2ClientProtocol, request: Request) -> Deferred[Response]: + return client.request(request, DummySpider()) -@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") -class Https2ClientProtocolTestCase(TestCase): +async def make_request(client: H2ClientProtocol, request: Request) -> Response: + return await maybe_deferred_to_future(make_request_dfd(client, request)) + + +class TestHttps2ClientProtocol: scheme = "https" + host = "localhost" key_file = Path(__file__).parent / "keys" / "localhost.key" certificate_file = Path(__file__).parent / "keys" / "localhost.crt" - def _init_resource(self): - self.temp_directory = self.mktemp() - Path(self.temp_directory).mkdir() - r = File(self.temp_directory) + @pytest.fixture + def site(self, tmp_path): + r = File(str(tmp_path)) r.putChild(b"get-data-html-small", GetDataHtmlSmall()) r.putChild(b"get-data-html-large", GetDataHtmlLarge()) @@ -200,502 +211,556 @@ class Https2ClientProtocolTestCase(TestCase): r.putChild(b"query-params", QueryParams()) r.putChild(b"timeout", TimeoutResponse()) r.putChild(b"request-headers", RequestHeaders()) - return r + return Site(r, timeout=None) - @inlineCallbacks - def setUp(self): - # Initialize resource tree - root = self._init_resource() - self.site = Site(root, timeout=None) + @async_yield_fixture # type: ignore[untyped-decorator] + async def server_port(self, site: Site) -> AsyncGenerator[int]: + from twisted.internet import reactor - # Start server for testing - self.hostname = "localhost" context_factory = ssl_context_factory( str(self.key_file), str(self.certificate_file) ) - server_endpoint = SSL4ServerEndpoint( - reactor, 0, context_factory, interface=self.hostname + reactor, 0, context_factory, interface=self.host ) - self.server = yield server_endpoint.listen(self.site) - self.port_number = self.server.getHost().port + server = await server_endpoint.listen(site) + + yield server.getHost().port + + await server.stopListening() + + @pytest.fixture + def client_certificate(self) -> PrivateCertificate: + pem = self.key_file.read_text( + encoding="utf-8" + ) + self.certificate_file.read_text(encoding="utf-8") + return PrivateCertificate.loadPEM(pem) # type: ignore[no-any-return] + + @async_yield_fixture # type: ignore[untyped-decorator] + async def client( + self, server_port: int, client_certificate: PrivateCertificate + ) -> AsyncGenerator[H2ClientProtocol]: + from twisted.internet import reactor + + from scrapy.core.http2.protocol import H2ClientFactory # noqa: PLC0415 - # Connect H2 client with server - self.client_certificate = get_client_certificate( - self.key_file, self.certificate_file - ) client_options = optionsForClientTLS( - hostname=self.hostname, - trustRoot=self.client_certificate, + hostname=self.host, + trustRoot=client_certificate, acceptableProtocols=[b"h2"], ) - uri = URI.fromBytes(bytes(self.get_url("/"), "utf-8")) - - self.conn_closed_deferred = Deferred() - from scrapy.core.http2.protocol import H2ClientFactory - - h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred) + uri = URI.fromBytes(bytes(self.get_url(server_port, "/"), "utf-8")) + h2_client_factory = H2ClientFactory(uri, Settings(), Deferred()) client_endpoint = SSL4ClientEndpoint( - reactor, self.hostname, self.port_number, client_options + reactor, self.host, server_port, client_options ) - self.client = yield client_endpoint.connect(h2_client_factory) + client = await client_endpoint.connect(h2_client_factory) - @inlineCallbacks - def tearDown(self): - if self.client.connected: - yield self.client.transport.loseConnection() - yield self.client.transport.abortConnection() - yield self.server.stopListening() - shutil.rmtree(self.temp_directory) - self.conn_closed_deferred = None + yield client - def get_url(self, path): + if client.connected: + client.transport.loseConnection() + client.transport.abortConnection() + + def get_url(self, portno: int, path: str) -> str: """ :param path: Should have / at the starting compulsorily if not empty :return: Complete url """ - assert len(path) > 0 and (path[0] == "/" or path[0] == "&") - return f"{self.scheme}://{self.hostname}:{self.port_number}{path}" - - def make_request(self, request: Request) -> Deferred: - return self.client.request(request, DummySpider()) + assert len(path) > 0 + assert path[0] == "/" or path[0] == "&" + return f"{self.scheme}://{self.host}:{portno}{path}" @staticmethod - def _check_repeat(get_deferred, count): + async def _check_repeat( + get_coro: Callable[[], Coroutine[Any, Any, None]], count: int + ) -> None: d_list = [] for _ in range(count): - d = get_deferred() + d = deferred_from_coro(get_coro()) d_list.append(d) - return DeferredList(d_list, fireOnOneErrback=True) + await maybe_deferred_to_future(DeferredList(d_list, fireOnOneErrback=True)) - def _check_GET(self, request: Request, expected_body, expected_status): - def check_response(response: Response): - self.assertEqual(response.status, expected_status) - self.assertEqual(response.body, expected_body) - self.assertEqual(response.request, request) - - content_length_header = response.headers.get("Content-Length") - assert content_length_header is not None - content_length = int(content_length_header) - self.assertEqual(len(response.body), content_length) - - d = self.make_request(request) - d.addCallback(check_response) - d.addErrback(self.fail) - return d - - def test_GET_small_body(self): - request = Request(self.get_url("/get-data-html-small")) - return self._check_GET(request, Data.HTML_SMALL, 200) - - def test_GET_large_body(self): - request = Request(self.get_url("/get-data-html-large")) - return self._check_GET(request, Data.HTML_LARGE, 200) - - def _check_GET_x10(self, *args, **kwargs): - def get_deferred(): - return self._check_GET(*args, **kwargs) - - return self._check_repeat(get_deferred, 10) - - def test_GET_small_body_x10(self): - return self._check_GET_x10( - Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200 - ) - - def test_GET_large_body_x10(self): - return self._check_GET_x10( - Request(self.get_url("/get-data-html-large")), Data.HTML_LARGE, 200 - ) - - def _check_POST_json( + async def _check_GET( self, + client: H2ClientProtocol, request: Request, - expected_request_body, - expected_extra_data, + expected_body: bytes, expected_status: int, - ): - d = self.make_request(request) + ) -> None: + response = await make_request(client, request) + assert response.status == expected_status + assert response.body == expected_body - def assert_response(response: Response): - self.assertEqual(response.status, expected_status) - self.assertEqual(response.request, request) + content_length_header = response.headers.get("Content-Length") + assert content_length_header is not None + content_length = int(content_length_header) + assert len(response.body) == content_length - content_length_header = response.headers.get("Content-Length") - assert content_length_header is not None - content_length = int(content_length_header) - self.assertEqual(len(response.body), content_length) + @deferred_f_from_coro_f + async def test_GET_small_body( + self, server_port: int, client: H2ClientProtocol + ) -> None: + request = Request(self.get_url(server_port, "/get-data-html-small")) + await self._check_GET(client, request, Data.HTML_SMALL, 200) - # Parse the body - content_encoding_header = response.headers[b"Content-Encoding"] - assert content_encoding_header is not None - content_encoding = str(content_encoding_header, "utf-8") - body = json.loads(str(response.body, content_encoding)) - self.assertIn("request-body", body) - self.assertIn("extra-data", body) - self.assertIn("request-headers", body) + @deferred_f_from_coro_f + async def test_GET_large_body( + self, server_port: int, client: H2ClientProtocol + ) -> None: + request = Request(self.get_url(server_port, "/get-data-html-large")) + await self._check_GET(client, request, Data.HTML_LARGE, 200) - request_body = body["request-body"] - self.assertEqual(request_body, expected_request_body) + async def _check_GET_x10( + self, + client: H2ClientProtocol, + request: Request, + expected_body: bytes, + expected_status: int, + ) -> None: + async def get_coro() -> None: + await self._check_GET(client, request, expected_body, expected_status) - extra_data = body["extra-data"] - self.assertEqual(extra_data, expected_extra_data) + await self._check_repeat(get_coro, 10) - # Check if headers were sent successfully - request_headers = body["request-headers"] - for k, v in request.headers.items(): - k_str = str(k, "utf-8") - self.assertIn(k_str, request_headers) - self.assertEqual(request_headers[k_str], str(v[0], "utf-8")) + @deferred_f_from_coro_f + async def test_GET_small_body_x10( + self, server_port: int, client: H2ClientProtocol + ) -> None: + await self._check_GET_x10( + client, + Request(self.get_url(server_port, "/get-data-html-small")), + Data.HTML_SMALL, + 200, + ) - d.addCallback(assert_response) - d.addErrback(self.fail) - return d + @deferred_f_from_coro_f + async def test_GET_large_body_x10( + self, server_port: int, client: H2ClientProtocol + ) -> None: + await self._check_GET_x10( + client, + Request(self.get_url(server_port, "/get-data-html-large")), + Data.HTML_LARGE, + 200, + ) - def test_POST_small_json(self): + @staticmethod + async def _check_POST_json( + client: H2ClientProtocol, + request: Request, + expected_request_body: dict[str, str], + expected_extra_data: str, + expected_status: int, + ) -> None: + response = await make_request(client, request) + + assert response.status == expected_status + + content_length_header = response.headers.get("Content-Length") + assert content_length_header is not None + content_length = int(content_length_header) + assert len(response.body) == content_length + + # Parse the body + content_encoding_header = response.headers[b"Content-Encoding"] + assert content_encoding_header is not None + content_encoding = str(content_encoding_header, "utf-8") + body = json.loads(str(response.body, content_encoding)) + assert "request-body" in body + assert "extra-data" in body + assert "request-headers" in body + + request_body = body["request-body"] + assert request_body == expected_request_body + + extra_data = body["extra-data"] + assert extra_data == expected_extra_data + + # Check if headers were sent successfully + request_headers = body["request-headers"] + for k, v in request.headers.items(): + k_str = str(k, "utf-8") + assert k_str in request_headers + assert request_headers[k_str] == str(v[0], "utf-8") + + @deferred_f_from_coro_f + async def test_POST_small_json( + self, server_port: int, client: H2ClientProtocol + ) -> None: request = JsonRequest( - url=self.get_url("/post-data-json-small"), + url=self.get_url(server_port, "/post-data-json-small"), method="POST", data=Data.JSON_SMALL, ) - return self._check_POST_json(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200) + await self._check_POST_json( + client, request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200 + ) - def test_POST_large_json(self): + @deferred_f_from_coro_f + async def test_POST_large_json( + self, server_port: int, client: H2ClientProtocol + ) -> None: request = JsonRequest( - url=self.get_url("/post-data-json-large"), + url=self.get_url(server_port, "/post-data-json-large"), method="POST", data=Data.JSON_LARGE, ) - return self._check_POST_json(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200) + await self._check_POST_json( + client, request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200 + ) - def _check_POST_json_x10(self, *args, **kwargs): - def get_deferred(): - return self._check_POST_json(*args, **kwargs) + async def _check_POST_json_x10( + self, + client: H2ClientProtocol, + request: Request, + expected_request_body: dict[str, str], + expected_extra_data: str, + expected_status: int, + ) -> None: + async def get_coro() -> None: + await self._check_POST_json( + client, + request, + expected_request_body, + expected_extra_data, + expected_status, + ) - return self._check_repeat(get_deferred, 10) + await self._check_repeat(get_coro, 10) - def test_POST_small_json_x10(self): + @deferred_f_from_coro_f + async def test_POST_small_json_x10( + self, server_port: int, client: H2ClientProtocol + ) -> None: request = JsonRequest( - url=self.get_url("/post-data-json-small"), + url=self.get_url(server_port, "/post-data-json-small"), method="POST", data=Data.JSON_SMALL, ) - return self._check_POST_json_x10( - request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200 + await self._check_POST_json_x10( + client, request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200 ) - def test_POST_large_json_x10(self): + @deferred_f_from_coro_f + async def test_POST_large_json_x10( + self, server_port: int, client: H2ClientProtocol + ) -> None: request = JsonRequest( - url=self.get_url("/post-data-json-large"), + url=self.get_url(server_port, "/post-data-json-large"), method="POST", data=Data.JSON_LARGE, ) - return self._check_POST_json_x10( - request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200 + await self._check_POST_json_x10( + client, request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200 ) @inlineCallbacks - def test_invalid_negotiated_protocol(self): + def test_invalid_negotiated_protocol( + self, server_port: int, client: H2ClientProtocol + ) -> Generator[Deferred[Any], Any, None]: with mock.patch( "scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2" ): - request = Request(url=self.get_url("/status?n=200")) - with self.assertRaises(ResponseFailed): - yield self.make_request(request) + request = Request(url=self.get_url(server_port, "/status?n=200")) + with pytest.raises(ResponseFailed): + yield make_request_dfd(client, request) - def test_cancel_request(self): - request = Request(url=self.get_url("/get-data-html-large")) - - def assert_response(response: Response): - self.assertEqual(response.status, 499) - self.assertEqual(response.request, request) - - d = self.make_request(request) - d.addCallback(assert_response) - d.addErrback(self.fail) + @inlineCallbacks + def test_cancel_request( + self, server_port: int, client: H2ClientProtocol + ) -> Generator[Deferred[Any], Any, None]: + request = Request(url=self.get_url(server_port, "/get-data-html-large")) + d = make_request_dfd(client, request) d.cancel() + response = cast("Response", (yield d)) + assert response.status == 499 - return d - - def test_download_maxsize_exceeded(self): + @deferred_f_from_coro_f + async def test_download_maxsize_exceeded( + self, + caplog: pytest.LogCaptureFixture, + server_port: int, + client: H2ClientProtocol, + ) -> None: request = Request( - url=self.get_url("/get-data-html-large"), meta={"download_maxsize": 1000} + url=self.get_url(server_port, "/get-data-html-large"), + meta={"download_maxsize": 1000}, ) + with pytest.raises( + DownloadCancelledError, + match=r"Expected to receive \d+ bytes which is larger than download max size \(1000\)", + ): + await make_request(client, request) - def assert_cancelled_error(failure): - self.assertIsInstance(failure.value, CancelledError) - error_pattern = re.compile( - rf"Cancelling download of {request.url}: received response " - rf"size \(\d*\) larger than download max size \(1000\)" - ) - self.assertEqual(len(re.findall(error_pattern, str(failure.value))), 1) - - d = self.make_request(request) - d.addCallback(self.fail) - d.addErrback(assert_cancelled_error) - return d - - def test_received_dataloss_response(self): + @inlineCallbacks + def test_received_dataloss_response( + self, server_port: int, client: H2ClientProtocol + ) -> Generator[Deferred[Any], Any, None]: """In case when value of Header Content-Length != len(Received Data) ProtocolError is raised""" - request = Request(url=self.get_url("/dataloss")) + from h2.exceptions import InvalidBodyLengthError # noqa: PLC0415 - def assert_failure(failure: Failure): - self.assertTrue(len(failure.value.reasons) > 0) - from h2.exceptions import InvalidBodyLengthError + request = Request(url=self.get_url(server_port, "/dataloss")) + with pytest.raises(ResponseFailed) as exc_info: + yield make_request_dfd(client, request) + assert len(exc_info.value.reasons) > 0 + assert any( + isinstance(error, InvalidBodyLengthError) + for error in exc_info.value.reasons + ) - self.assertTrue( - any( - isinstance(error, InvalidBodyLengthError) - for error in failure.value.reasons - ) - ) + @deferred_f_from_coro_f + async def test_missing_content_length_header( + self, server_port: int, client: H2ClientProtocol + ) -> None: + request = Request(url=self.get_url(server_port, "/no-content-length-header")) + response = await make_request(client, request) + assert response.status == 200 + assert response.body == Data.NO_CONTENT_LENGTH + assert "Content-Length" not in response.headers - d = self.make_request(request) - d.addCallback(self.fail) - d.addErrback(assert_failure) - return d + async def _check_log_warnsize( + self, + client: H2ClientProtocol, + request: Request, + warn_pattern: re.Pattern[str], + expected_body: bytes, + caplog: pytest.LogCaptureFixture, + ) -> None: + with caplog.at_level("WARNING", "scrapy.core.http2.stream"): + response = await make_request(client, request) + assert response.status == 200 + assert response.body == expected_body - def test_missing_content_length_header(self): - request = Request(url=self.get_url("/no-content-length-header")) + # Check the warning is raised only once for this request + assert len(re.findall(warn_pattern, caplog.text)) == 1 - def assert_content_length(response: Response): - self.assertEqual(response.status, 200) - self.assertEqual(response.body, Data.NO_CONTENT_LENGTH) - self.assertEqual(response.request, request) - self.assertNotIn("Content-Length", response.headers) - - d = self.make_request(request) - d.addCallback(assert_content_length) - d.addErrback(self.fail) - return d - - @inlineCallbacks - def _check_log_warnsize(self, request, warn_pattern, expected_body): - with self.assertLogs("scrapy.core.http2.stream", level="WARNING") as cm: - response = yield self.make_request(request) - self.assertEqual(response.status, 200) - self.assertEqual(response.request, request) - self.assertEqual(response.body, expected_body) - - # Check the warning is raised only once for this request - self.assertEqual( - sum(len(re.findall(warn_pattern, log)) for log in cm.output), 1 - ) - - @inlineCallbacks - def test_log_expected_warnsize(self): + @deferred_f_from_coro_f + async def test_log_expected_warnsize( + self, + server_port: int, + client: H2ClientProtocol, + caplog: pytest.LogCaptureFixture, + ) -> None: request = Request( - url=self.get_url("/get-data-html-large"), meta={"download_warnsize": 1000} + url=self.get_url(server_port, "/get-data-html-large"), + meta={"download_warnsize": 1000}, ) warn_pattern = re.compile( - rf"Expected response size \(\d*\) larger than " + rf"Expected to receive \d+ bytes which is larger than " rf"download warn size \(1000\) in request {request}" ) - yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE) + await self._check_log_warnsize( + client, request, warn_pattern, Data.HTML_LARGE, caplog + ) - @inlineCallbacks - def test_log_received_warnsize(self): + @deferred_f_from_coro_f + async def test_log_received_warnsize( + self, + server_port: int, + client: H2ClientProtocol, + caplog: pytest.LogCaptureFixture, + ) -> None: request = Request( - url=self.get_url("/no-content-length-header"), + url=self.get_url(server_port, "/no-content-length-header"), meta={"download_warnsize": 10}, ) warn_pattern = re.compile( - rf"Received more \(\d*\) bytes than download " - rf"warn size \(10\) in request {request}" + rf"Received \d+ bytes which is larger than " + rf"download warn size \(10\) in request {request}" ) - yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) + await self._check_log_warnsize( + client, request, warn_pattern, Data.NO_CONTENT_LENGTH, caplog + ) - def test_max_concurrent_streams(self): + @deferred_f_from_coro_f + async def test_max_concurrent_streams( + self, server_port: int, client: H2ClientProtocol + ) -> None: """Send 500 requests at one to check if we can handle very large number of request. """ - def get_deferred(): - return self._check_GET( - Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200 + async def get_coro() -> None: + await self._check_GET( + client, + Request(self.get_url(server_port, "/get-data-html-small")), + Data.HTML_SMALL, + 200, ) - return self._check_repeat(get_deferred, 500) + await self._check_repeat(get_coro, 500) - def test_inactive_stream(self): + @inlineCallbacks + def test_inactive_stream( + self, server_port: int, client: H2ClientProtocol + ) -> Generator[Deferred[Any], Any, None]: """Here we send 110 requests considering the MAX_CONCURRENT_STREAMS by default is 100. After sending the first 100 requests we close the connection.""" d_list = [] def assert_inactive_stream(failure): - self.assertIsNotNone(failure.check(ResponseFailed)) - from scrapy.core.http2.stream import InactiveStreamClosed + assert failure.check(ResponseFailed) is not None - self.assertTrue( - any(isinstance(e, InactiveStreamClosed) for e in failure.value.reasons) + from scrapy.core.http2.stream import InactiveStreamClosed # noqa: PLC0415 + + assert any( + isinstance(e, InactiveStreamClosed) for e in failure.value.reasons ) # Send 100 request (we do not check the result) for _ in range(100): - d = self.make_request(Request(self.get_url("/get-data-html-small"))) + d = make_request_dfd( + client, Request(self.get_url(server_port, "/get-data-html-small")) + ) d.addBoth(lambda _: None) d_list.append(d) # Now send 10 extra request and save the response deferred in a list for _ in range(10): - d = self.make_request(Request(self.get_url("/get-data-html-small"))) - d.addCallback(self.fail) + d = make_request_dfd( + client, Request(self.get_url(server_port, "/get-data-html-small")) + ) + d.addCallback(lambda _: pytest.fail("This request should have failed")) d.addErrback(assert_inactive_stream) d_list.append(d) # Close the connection now to fire all the extra 10 requests errback # with InactiveStreamClosed - self.client.transport.loseConnection() + assert client.transport + client.transport.loseConnection() - return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) + yield DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) - def test_invalid_request_type(self): - with self.assertRaises(TypeError): - self.make_request("https://InvalidDataTypePassed.com") + @deferred_f_from_coro_f + async def test_invalid_request_type(self, client: H2ClientProtocol): + with pytest.raises(TypeError): + await make_request(client, "https://InvalidDataTypePassed.com") # type: ignore[arg-type] - def test_query_parameters(self): + @deferred_f_from_coro_f + async def test_query_parameters( + self, server_port: int, client: H2ClientProtocol + ) -> None: params = { "a": generate_random_string(20), "b": generate_random_string(20), "c": generate_random_string(20), "d": generate_random_string(20), } - request = Request(self.get_url(f"/query-params?{urlencode(params)}")) + request = Request( + self.get_url(server_port, f"/query-params?{urlencode(params)}") + ) + response = await make_request(client, request) + content_encoding_header = response.headers[b"Content-Encoding"] + assert content_encoding_header is not None + content_encoding = str(content_encoding_header, "utf-8") + data = json.loads(str(response.body, content_encoding)) + assert data == params - def assert_query_params(response: Response): - content_encoding_header = response.headers[b"Content-Encoding"] - assert content_encoding_header is not None - content_encoding = str(content_encoding_header, "utf-8") - data = json.loads(str(response.body, content_encoding)) - self.assertEqual(data, params) - - d = self.make_request(request) - d.addCallback(assert_query_params) - d.addErrback(self.fail) - - return d - - def test_status_codes(self): - def assert_response_status(response: Response, expected_status: int): - self.assertEqual(response.status, expected_status) - - d_list = [] + @deferred_f_from_coro_f + async def test_status_codes( + self, server_port: int, client: H2ClientProtocol + ) -> None: for status in [200, 404]: - request = Request(self.get_url(f"/status?n={status}")) - d = self.make_request(request) - d.addCallback(assert_response_status, status) - d.addErrback(self.fail) - d_list.append(d) + request = Request(self.get_url(server_port, f"/status?n={status}")) + response = await make_request(client, request) + assert response.status == status - return DeferredList(d_list, fireOnOneErrback=True) + @deferred_f_from_coro_f + async def test_response_has_correct_certificate_ip_address( + self, + server_port: int, + client: H2ClientProtocol, + client_certificate: PrivateCertificate, + ) -> None: + request = Request(self.get_url(server_port, "/status?n=200")) + response = await make_request(client, request) + assert isinstance(response.certificate, Certificate) + assert response.certificate.original is not None + assert response.certificate.getIssuer() == client_certificate.getIssuer() + assert response.certificate.getPublicKey().matches( + client_certificate.getPublicKey() + ) + assert isinstance(response.ip_address, IPv4Address) + assert str(response.ip_address) == "127.0.0.1" - def test_response_has_correct_certificate_ip_address(self): - request = Request(self.get_url("/status?n=200")) + @staticmethod + async def _check_invalid_netloc(client: H2ClientProtocol, url: str) -> None: + from scrapy.core.http2.stream import InvalidHostname # noqa: PLC0415 - def assert_metadata(response: Response): - self.assertEqual(response.request, request) - self.assertIsInstance(response.certificate, Certificate) - assert response.certificate # typing - self.assertIsNotNone(response.certificate.original) - self.assertEqual( - response.certificate.getIssuer(), self.client_certificate.getIssuer() - ) - self.assertTrue( - response.certificate.getPublicKey().matches( - self.client_certificate.getPublicKey() - ) - ) - - self.assertIsInstance(response.ip_address, IPv4Address) - self.assertEqual(str(response.ip_address), "127.0.0.1") - - d = self.make_request(request) - d.addCallback(assert_metadata) - d.addErrback(self.fail) - - return d - - def _check_invalid_netloc(self, url): request = Request(url) + with pytest.raises(InvalidHostname) as exc_info: + await make_request(client, request) + error_msg = str(exc_info.value) + assert "localhost" in error_msg + assert "127.0.0.1" in error_msg + assert str(request) in error_msg - def assert_invalid_hostname(failure: Failure): - from scrapy.core.http2.stream import InvalidHostname + @deferred_f_from_coro_f + async def test_invalid_hostname(self, client: H2ClientProtocol) -> None: + await self._check_invalid_netloc( + client, "https://notlocalhost.notlocalhostdomain" + ) - self.assertIsNotNone(failure.check(InvalidHostname)) - error_msg = str(failure.value) - self.assertIn("localhost", error_msg) - self.assertIn("127.0.0.1", error_msg) - self.assertIn(str(request), error_msg) + @deferred_f_from_coro_f + async def test_invalid_host_port( + self, server_port: int, client: H2ClientProtocol + ) -> None: + port = server_port + 1 + await self._check_invalid_netloc(client, f"https://127.0.0.1:{port}") - d = self.make_request(request) - d.addCallback(self.fail) - d.addErrback(assert_invalid_hostname) - return d + @deferred_f_from_coro_f + async def test_connection_stays_with_invalid_requests( + self, server_port: int, client: H2ClientProtocol + ): + await maybe_deferred_to_future(self.test_invalid_hostname(client)) + await maybe_deferred_to_future(self.test_invalid_host_port(server_port, client)) + await maybe_deferred_to_future(self.test_GET_small_body(server_port, client)) + await maybe_deferred_to_future(self.test_POST_small_json(server_port, client)) - def test_invalid_hostname(self): - return self._check_invalid_netloc("https://notlocalhost.notlocalhostdomain") - - def test_invalid_host_port(self): - port = self.port_number + 1 - return self._check_invalid_netloc(f"https://127.0.0.1:{port}") - - def test_connection_stays_with_invalid_requests(self): - d_list = [ - self.test_invalid_hostname(), - self.test_invalid_host_port(), - self.test_GET_small_body(), - self.test_POST_small_json(), - ] - - return DeferredList(d_list, fireOnOneErrback=True) - - def test_connection_timeout(self): - request = Request(self.get_url("/timeout")) - d = self.make_request(request) + @inlineCallbacks + def test_connection_timeout( + self, server_port: int, client: H2ClientProtocol + ) -> Generator[Deferred[Any], Any, None]: + request = Request(self.get_url(server_port, "/timeout")) # Update the timer to 1s to test connection timeout - self.client.setTimeout(1) + client.setTimeout(1) # type: ignore[no-untyped-call] - def assert_timeout_error(failure: Failure): - for err in failure.value.reasons: - from scrapy.core.http2.protocol import H2ClientProtocol + with pytest.raises(ResponseFailed) as exc_info: + yield make_request_dfd(client, request) - if isinstance(err, TimeoutError): - self.assertIn( - f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", - str(err), - ) - break - else: - self.fail() + for err in exc_info.value.reasons: + from scrapy.core.http2.protocol import H2ClientProtocol # noqa: PLC0415 - d.addCallback(self.fail) - d.addErrback(assert_timeout_error) - return d + if isinstance(err, DownloadTimeoutError): + assert ( + f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s" + in str(err) + ) + break + else: + pytest.fail("No TimeoutError raised.") - def test_request_headers_received(self): + @deferred_f_from_coro_f + async def test_request_headers_received( + self, server_port: int, client: H2ClientProtocol + ) -> None: request = Request( - self.get_url("/request-headers"), + self.get_url(server_port, "/request-headers"), headers={"header-1": "header value 1", "header-2": "header value 2"}, ) - d = self.make_request(request) + response = await make_request(client, request) + assert response.status == 200 - def assert_request_headers(response: Response): - self.assertEqual(response.status, 200) - self.assertEqual(response.request, request) - - response_headers = json.loads(str(response.body, "utf-8")) - self.assertIsInstance(response_headers, dict) - for k, v in request.headers.items(): - k, v = str(k, "utf-8"), str(v[0], "utf-8") - self.assertIn(k, response_headers) - self.assertEqual(v, response_headers[k]) - - d.addErrback(self.fail) - d.addCallback(assert_request_headers) - return d + response_headers = json.loads(str(response.body, "utf-8")) + assert isinstance(response_headers, dict) + for k, v in request.headers.items(): + k_decoded, v_decoded = str(k, "utf-8"), str(v[0], "utf-8") + assert k_decoded in response_headers + assert v_decoded == response_headers[k_decoded] diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 9e43b72b0..660b76d08 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -1,67 +1,72 @@ -from unittest import TestCase -from urllib.parse import urlparse - from scrapy.http import Request, Response from scrapy.http.cookies import WrappedRequest, WrappedResponse +from scrapy.utils.httpobj import urlparse_cached -class WrappedRequestTest(TestCase): - def setUp(self): +class TestWrappedRequest: + def setup_method(self): self.request = Request( "http://www.example.com/page.html", headers={"Content-Type": "text/html"} ) self.wrapped = WrappedRequest(self.request) def test_get_full_url(self): - self.assertEqual(self.wrapped.get_full_url(), self.request.url) - self.assertEqual(self.wrapped.full_url, self.request.url) + assert self.wrapped.get_full_url() == self.request.url + assert self.wrapped.full_url == self.request.url def test_get_host(self): - self.assertEqual(self.wrapped.get_host(), urlparse(self.request.url).netloc) - self.assertEqual(self.wrapped.host, urlparse(self.request.url).netloc) + assert self.wrapped.get_host() == urlparse_cached(self.request).netloc + assert self.wrapped.host == urlparse_cached(self.request).netloc def test_get_type(self): - self.assertEqual(self.wrapped.get_type(), urlparse(self.request.url).scheme) - self.assertEqual(self.wrapped.type, urlparse(self.request.url).scheme) + assert self.wrapped.get_type() == urlparse_cached(self.request).scheme + assert self.wrapped.type == urlparse_cached(self.request).scheme def test_is_unverifiable(self): - self.assertFalse(self.wrapped.is_unverifiable()) - self.assertFalse(self.wrapped.unverifiable) + assert not self.wrapped.is_unverifiable() + assert not self.wrapped.unverifiable def test_is_unverifiable2(self): self.request.meta["is_unverifiable"] = True - self.assertTrue(self.wrapped.is_unverifiable()) - self.assertTrue(self.wrapped.unverifiable) + assert self.wrapped.is_unverifiable() + assert self.wrapped.unverifiable def test_get_origin_req_host(self): - self.assertEqual(self.wrapped.origin_req_host, "www.example.com") + assert self.wrapped.origin_req_host == "www.example.com" def test_has_header(self): - self.assertTrue(self.wrapped.has_header("content-type")) - self.assertFalse(self.wrapped.has_header("xxxxx")) + assert self.wrapped.has_header("content-type") + assert not self.wrapped.has_header("xxxxx") def test_get_header(self): - self.assertEqual(self.wrapped.get_header("content-type"), "text/html") - self.assertEqual(self.wrapped.get_header("xxxxx", "def"), "def") + assert self.wrapped.get_header("content-type") == "text/html" + assert self.wrapped.get_header("xxxxx", "def") == "def" + assert self.wrapped.get_header("xxxxx") is None + wrapped = WrappedRequest( + Request( + "http://www.example.com/page.html", headers={"empty-binary-header": b""} + ) + ) + assert wrapped.get_header("empty-binary-header") == "" def test_header_items(self): - self.assertEqual(self.wrapped.header_items(), [("Content-Type", ["text/html"])]) + assert self.wrapped.header_items() == [("Content-Type", ["text/html"])] def test_add_unredirected_header(self): self.wrapped.add_unredirected_header("hello", "world") - self.assertEqual(self.request.headers["hello"], b"world") + assert self.request.headers["hello"] == b"world" -class WrappedResponseTest(TestCase): - def setUp(self): +class TestWrappedResponse: + def setup_method(self): self.response = Response( "http://www.example.com/page.html", headers={"Content-TYpe": "text/html"} ) self.wrapped = WrappedResponse(self.response) def test_info(self): - self.assertIs(self.wrapped.info(), self.wrapped) + assert self.wrapped.info() is self.wrapped def test_get_all(self): # get_all result must be native string - self.assertEqual(self.wrapped.get_all("content-type"), ["text/html"]) + assert self.wrapped.get_all("content-type") == ["text/html"] diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 7db1eb8c5..243aa6afe 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -1,66 +1,68 @@ import copy -import unittest + +import pytest from scrapy.http import Headers -class HeadersTest(unittest.TestCase): +class TestHeaders: def assertSortedEqual(self, first, second, msg=None): - return self.assertEqual(sorted(first), sorted(second), msg) + assert sorted(first) == sorted(second), msg def test_basics(self): h = Headers({"Content-Type": "text/html", "Content-Length": 1234}) assert h["Content-Type"] assert h["Content-Length"] - self.assertRaises(KeyError, h.__getitem__, "Accept") - self.assertEqual(h.get("Accept"), None) - self.assertEqual(h.getlist("Accept"), []) + with pytest.raises(KeyError): + h["Accept"] + assert h.get("Accept") is None + assert h.getlist("Accept") == [] - self.assertEqual(h.get("Accept", "*/*"), b"*/*") - self.assertEqual(h.getlist("Accept", "*/*"), [b"*/*"]) - self.assertEqual( - h.getlist("Accept", ["text/html", "images/jpeg"]), - [b"text/html", b"images/jpeg"], - ) + assert h.get("Accept", "*/*") == b"*/*" + assert h.getlist("Accept", "*/*") == [b"*/*"] + assert h.getlist("Accept", ["text/html", "images/jpeg"]) == [ + b"text/html", + b"images/jpeg", + ] def test_single_value(self): h = Headers() h["Content-Type"] = "text/html" - self.assertEqual(h["Content-Type"], b"text/html") - self.assertEqual(h.get("Content-Type"), b"text/html") - self.assertEqual(h.getlist("Content-Type"), [b"text/html"]) + assert h["Content-Type"] == b"text/html" + assert h.get("Content-Type") == b"text/html" + assert h.getlist("Content-Type") == [b"text/html"] def test_multivalue(self): h = Headers() h["X-Forwarded-For"] = hlist = ["ip1", "ip2"] - self.assertEqual(h["X-Forwarded-For"], b"ip2") - self.assertEqual(h.get("X-Forwarded-For"), b"ip2") - self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1", b"ip2"]) + assert h["X-Forwarded-For"] == b"ip2" + assert h.get("X-Forwarded-For") == b"ip2" + assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"] assert h.getlist("X-Forwarded-For") is not hlist def test_multivalue_for_one_header(self): h = Headers((("a", "b"), ("a", "c"))) - self.assertEqual(h["a"], b"c") - self.assertEqual(h.get("a"), b"c") - self.assertEqual(h.getlist("a"), [b"b", b"c"]) + assert h["a"] == b"c" + assert h.get("a") == b"c" + assert h.getlist("a") == [b"b", b"c"] def test_encode_utf8(self): h = Headers({"key": "\xa3"}, encoding="utf-8") key, val = dict(h).popitem() assert isinstance(key, bytes), key assert isinstance(val[0], bytes), val[0] - self.assertEqual(val[0], b"\xc2\xa3") + assert val[0] == b"\xc2\xa3" def test_encode_latin1(self): h = Headers({"key": "\xa3"}, encoding="latin1") - key, val = dict(h).popitem() - self.assertEqual(val[0], b"\xa3") + _, val = dict(h).popitem() + assert val[0] == b"\xa3" def test_encode_multiple(self): h = Headers({"key": ["\xa3"]}, encoding="utf-8") - key, val = dict(h).popitem() - self.assertEqual(val[0], b"\xc2\xa3") + _, val = dict(h).popitem() + assert val[0] == b"\xc2\xa3" def test_delete_and_contains(self): h = Headers() @@ -78,17 +80,17 @@ class HeadersTest(unittest.TestCase): h = Headers() olist = h.setdefault("X-Forwarded-For", "ip1") - self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1"]) + assert h.getlist("X-Forwarded-For") == [b"ip1"] assert h.getlist("X-Forwarded-For") is olist def test_iterables(self): idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]} h = Headers(idict) - self.assertDictEqual( - dict(h), - {b"Content-Type": [b"text/html"], b"X-Forwarded-For": [b"ip1", b"ip2"]}, - ) + assert dict(h) == { + b"Content-Type": [b"text/html"], + b"X-Forwarded-For": [b"ip1", b"ip2"], + } self.assertSortedEqual(h.keys(), [b"X-Forwarded-For", b"Content-Type"]) self.assertSortedEqual( h.items(), @@ -99,68 +101,64 @@ class HeadersTest(unittest.TestCase): def test_update(self): h = Headers() h.update({"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]}) - self.assertEqual(h.getlist("Content-Type"), [b"text/html"]) - self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1", b"ip2"]) + assert h.getlist("Content-Type") == [b"text/html"] + assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"] def test_copy(self): h1 = Headers({"header1": ["value1", "value2"]}) h2 = copy.copy(h1) - self.assertEqual(h1, h2) - self.assertEqual(h1.getlist("header1"), h2.getlist("header1")) + assert h1 == h2 + assert h1.getlist("header1") == h2.getlist("header1") assert h1.getlist("header1") is not h2.getlist("header1") assert isinstance(h2, Headers) def test_appendlist(self): h1 = Headers({"header1": "value1"}) h1.appendlist("header1", "value3") - self.assertEqual(h1.getlist("header1"), [b"value1", b"value3"]) + assert h1.getlist("header1") == [b"value1", b"value3"] h1 = Headers() h1.appendlist("header1", "value1") h1.appendlist("header1", "value3") - self.assertEqual(h1.getlist("header1"), [b"value1", b"value3"]) + assert h1.getlist("header1") == [b"value1", b"value3"] def test_setlist(self): h1 = Headers({"header1": "value1"}) - self.assertEqual(h1.getlist("header1"), [b"value1"]) + assert h1.getlist("header1") == [b"value1"] h1.setlist("header1", [b"value2", b"value3"]) - self.assertEqual(h1.getlist("header1"), [b"value2", b"value3"]) + assert h1.getlist("header1") == [b"value2", b"value3"] def test_setlistdefault(self): h1 = Headers({"header1": "value1"}) h1.setlistdefault("header1", ["value2", "value3"]) h1.setlistdefault("header2", ["value2", "value3"]) - self.assertEqual(h1.getlist("header1"), [b"value1"]) - self.assertEqual(h1.getlist("header2"), [b"value2", b"value3"]) + assert h1.getlist("header1") == [b"value1"] + assert h1.getlist("header2") == [b"value2", b"value3"] def test_none_value(self): h1 = Headers() h1["foo"] = "bar" h1["foo"] = None h1.setdefault("foo", "bar") - self.assertEqual(h1.get("foo"), None) - self.assertEqual(h1.getlist("foo"), []) + assert h1.get("foo") is None + assert h1.getlist("foo") == [] def test_int_value(self): h1 = Headers({"hey": 5}) h1["foo"] = 1 h1.setdefault("bar", 2) h1.setlist("buz", [1, "dos", 3]) - self.assertEqual(h1.getlist("foo"), [b"1"]) - self.assertEqual(h1.getlist("bar"), [b"2"]) - self.assertEqual(h1.getlist("buz"), [b"1", b"dos", b"3"]) - self.assertEqual(h1.getlist("hey"), [b"5"]) + assert h1.getlist("foo") == [b"1"] + assert h1.getlist("bar") == [b"2"] + assert h1.getlist("buz") == [b"1", b"dos", b"3"] + assert h1.getlist("hey") == [b"5"] def test_invalid_value(self): - self.assertRaisesRegex( - TypeError, "Unsupported value type", Headers, {"foo": object()} - ) - self.assertRaisesRegex( - TypeError, "Unsupported value type", Headers().__setitem__, "foo", object() - ) - self.assertRaisesRegex( - TypeError, "Unsupported value type", Headers().setdefault, "foo", object() - ) - self.assertRaisesRegex( - TypeError, "Unsupported value type", Headers().setlist, "foo", [object()] - ) + with pytest.raises(TypeError, match="Unsupported value type"): + Headers({"foo": object()}) + with pytest.raises(TypeError, match="Unsupported value type"): + Headers()["foo"] = object() + with pytest.raises(TypeError, match="Unsupported value type"): + Headers().setdefault("foo", object()) + with pytest.raises(TypeError, match="Unsupported value type"): + Headers().setlist("foo", [object()]) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 6dc9ec8b7..fed5dbab7 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,46 +1,38 @@ -import json -import re -import unittest import warnings import xmlrpc.client -from typing import Any, Dict, List -from unittest import mock -from urllib.parse import parse_qs, unquote_to_bytes, urlparse +from typing import Any -from scrapy.http import ( - FormRequest, - Headers, - HtmlResponse, - JsonRequest, - Request, - XmlRpcRequest, -) +import pytest + +from scrapy.http import Headers, Request, XmlRpcRequest from scrapy.http.request import NO_CALLBACK -from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.python import to_bytes -class RequestTest(unittest.TestCase): +class TestRequest: request_class = Request default_method = "GET" - default_headers: Dict[bytes, List[bytes]] = {} - default_meta: Dict[str, Any] = {} + default_headers: dict[bytes, list[bytes]] = {} + default_meta: dict[str, Any] = {} def test_init(self): # Request requires url in the __init__ method - self.assertRaises(Exception, self.request_class) + with pytest.raises(TypeError): + self.request_class() # url argument must be basestring - self.assertRaises(TypeError, self.request_class, 123) + with pytest.raises(TypeError): + self.request_class(123) r = self.request_class("http://www.example.com") r = self.request_class("http://www.example.com") assert isinstance(r.url, str) - self.assertEqual(r.url, "http://www.example.com") - self.assertEqual(r.method, self.default_method) + assert r.url == "http://www.example.com" + assert r.method == self.default_method assert isinstance(r.headers, Headers) - self.assertEqual(r.headers, self.default_headers) - self.assertEqual(r.meta, self.default_meta) + assert r.headers == self.default_headers + assert r.meta == self.default_meta meta = {"lala": "lolo"} headers = {b"caca": b"coco"} @@ -49,9 +41,9 @@ class RequestTest(unittest.TestCase): ) assert r.meta is not meta - self.assertEqual(r.meta, meta) + assert r.meta == meta assert r.headers is not headers - self.assertEqual(r.headers[b"caca"], b"coco") + assert r.headers[b"caca"] == b"coco" def test_url_scheme(self): # This test passes by not raising any (ValueError) exception @@ -63,9 +55,13 @@ class RequestTest(unittest.TestCase): self.request_class("data:,Hello%2C%20World!") def test_url_no_scheme(self): - self.assertRaises(ValueError, self.request_class, "foo") - self.assertRaises(ValueError, self.request_class, "/foo/") - self.assertRaises(ValueError, self.request_class, "/foo:bar") + msg = "Missing scheme in request url:" + with pytest.raises(ValueError, match=msg): + self.request_class("foo") + with pytest.raises(ValueError, match=msg): + self.request_class("/foo/") + with pytest.raises(ValueError, match=msg): + self.request_class("/foo:bar") def test_headers(self): # Different ways of setting headers attribute @@ -74,61 +70,61 @@ class RequestTest(unittest.TestCase): r = self.request_class(url=url, headers=headers) p = self.request_class(url=url, headers=r.headers) - self.assertEqual(r.headers, p.headers) - self.assertFalse(r.headers is headers) - self.assertFalse(p.headers is r.headers) + assert r.headers == p.headers + assert r.headers is not headers + assert p.headers is not r.headers # headers must not be unicode h = Headers({"key1": "val1", "key2": "val2"}) h["newkey"] = "newval" for k, v in h.items(): - self.assertIsInstance(k, bytes) + assert isinstance(k, bytes) for s in v: - self.assertIsInstance(s, bytes) + assert isinstance(s, bytes) def test_eq(self): url = "http://www.scrapy.org" r1 = self.request_class(url=url) r2 = self.request_class(url=url) - self.assertNotEqual(r1, r2) + assert r1 != r2 set_ = set() set_.add(r1) set_.add(r2) - self.assertEqual(len(set_), 2) + assert len(set_) == 2 def test_url(self): r = self.request_class(url="http://www.scrapy.org/path") - self.assertEqual(r.url, "http://www.scrapy.org/path") + assert r.url == "http://www.scrapy.org/path" def test_url_quoting(self): r = self.request_class(url="http://www.scrapy.org/blank%20space") - self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") + assert r.url == "http://www.scrapy.org/blank%20space" r = self.request_class(url="http://www.scrapy.org/blank space") - self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") + assert r.url == "http://www.scrapy.org/blank%20space" def test_url_encoding(self): r = self.request_class(url="http://www.scrapy.org/price/£") - self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + assert r.url == "http://www.scrapy.org/price/%C2%A3" def test_url_encoding_other(self): # encoding affects only query part of URI, not path # path part should always be UTF-8 encoded before percent-escaping r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8") - self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + assert r.url == "http://www.scrapy.org/price/%C2%A3" r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1") - self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + assert r.url == "http://www.scrapy.org/price/%C2%A3" def test_url_encoding_query(self): r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ") - self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") + assert r1.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" # should be same as above r2 = self.request_class( url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8" ) - self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") + assert r2.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" def test_url_encoding_query_latin1(self): # encoding is used for encoding query-string before percent-escaping; @@ -136,13 +132,13 @@ class RequestTest(unittest.TestCase): r3 = self.request_class( url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1" ) - self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3") + assert r3.url == "http://www.scrapy.org/price/%C2%B5?currency=%A3" def test_url_encoding_nonutf8_untouched(self): # percent-escaping sequences that do not match valid UTF-8 sequences # should be kept untouched (just upper-cased perhaps) # - # See https://tools.ietf.org/html/rfc3987#section-3.2 + # See https://datatracker.ietf.org/doc/html/rfc3987#section-3.2 # # "Conversions from URIs to IRIs MUST NOT use any character encoding # other than UTF-8 in steps 3 and 4, even if it might be possible to @@ -155,16 +151,29 @@ class RequestTest(unittest.TestCase): # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different # URI from "http://www.example.org/r%E9sum%E9.html". r1 = self.request_class(url="http://www.scrapy.org/price/%a3") - self.assertEqual(r1.url, "http://www.scrapy.org/price/%a3") + assert r1.url == "http://www.scrapy.org/price/%a3" r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") - self.assertEqual(r2.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + assert r2.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3") - self.assertEqual(r3.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + assert r3.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") - self.assertEqual(r4.url, "http://www.example.org/r%E9sum%E9.html") + assert r4.url == "http://www.example.org/r%E9sum%E9.html" + + def test_url_verbatim(self): + r = self.request_class( + url="http://www.scrapy.org/price/£", + meta={"verbatim_url": True}, + ) + assert r.url == "http://www.scrapy.org/price/£" + + r = self.request_class( + url="http://www.scrapy.org/blank space", + meta={"verbatim_url": True}, + ) + assert r.url == "http://www.scrapy.org/blank space" def test_body(self): r1 = self.request_class(url="http://www.example.com/") @@ -172,31 +181,19 @@ class RequestTest(unittest.TestCase): r2 = self.request_class(url="http://www.example.com/", body=b"") assert isinstance(r2.body, bytes) - self.assertEqual(r2.encoding, "utf-8") # default encoding + assert r2.encoding == "utf-8" # default encoding r3 = self.request_class( url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8" ) assert isinstance(r3.body, bytes) - self.assertEqual(r3.body, b"Price: \xc2\xa3100") + assert r3.body == b"Price: \xc2\xa3100" r4 = self.request_class( url="http://www.example.com/", body="Price: \xa3100", encoding="latin1" ) assert isinstance(r4.body, bytes) - self.assertEqual(r4.body, b"Price: \xa3100") - - def test_ajax_url(self): - # ascii url - r = self.request_class(url="http://www.example.com/ajax.html#!key=value") - self.assertEqual( - r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue" - ) - # unicode url - r = self.request_class(url="http://www.example.com/ajax.html#!key=value") - self.assertEqual( - r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue" - ) + assert r4.body == b"Price: \xa3100" def test_copy(self): """Test Request copy""" @@ -222,25 +219,25 @@ class RequestTest(unittest.TestCase): # make sure flags list is shallow copied assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" - self.assertEqual(r1.flags, r2.flags) + assert r1.flags == r2.flags # make sure cb_kwargs dict is shallow copied - assert ( - r1.cb_kwargs is not r2.cb_kwargs - ), "cb_kwargs must be a shallow copy, not identical" - self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) + assert r1.cb_kwargs is not r2.cb_kwargs, ( + "cb_kwargs must be a shallow copy, not identical" + ) + assert r1.cb_kwargs == r2.cb_kwargs # make sure meta dict is shallow copied assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" - self.assertEqual(r1.meta, r2.meta) + assert r1.meta == r2.meta # make sure headers attribute is shallow copied - assert ( - r1.headers is not r2.headers - ), "headers must be a shallow copy, not identical" - self.assertEqual(r1.headers, r2.headers) - self.assertEqual(r1.encoding, r2.encoding) - self.assertEqual(r1.dont_filter, r2.dont_filter) + assert r1.headers is not r2.headers, ( + "headers must be a shallow copy, not identical" + ) + assert r1.headers == r2.headers + assert r1.encoding == r2.encoding + assert r1.dont_filter == r2.dont_filter # Request.body can be identical since it's an immutable object (str) @@ -261,10 +258,10 @@ class RequestTest(unittest.TestCase): hdrs = Headers(r1.headers) hdrs[b"key"] = b"value" r2 = r1.replace(method="POST", body="New body", headers=hdrs) - self.assertEqual(r1.url, r2.url) - self.assertEqual((r1.method, r2.method), ("GET", "POST")) - self.assertEqual((r1.body, r2.body), (b"", b"New body")) - self.assertEqual((r1.headers, r2.headers), (self.default_headers, hdrs)) + assert r1.url == r2.url + assert (r1.method, r2.method) == ("GET", "POST") + assert (r1.body, r2.body) == (b"", b"New body") + assert (r1.headers, r2.headers) == (self.default_headers, hdrs) # Empty attributes (which may fail if not compared properly) r3 = self.request_class( @@ -273,9 +270,9 @@ class RequestTest(unittest.TestCase): r4 = r3.replace( url="http://www.example.com/2", body=b"", meta={}, dont_filter=False ) - self.assertEqual(r4.url, "http://www.example.com/2") - self.assertEqual(r4.body, b"") - self.assertEqual(r4.meta, {}) + assert r4.url == "http://www.example.com/2" + assert r4.body == b"" + assert r4.meta == {} assert r4.dont_filter is False def test_method_always_str(self): @@ -284,55 +281,120 @@ class RequestTest(unittest.TestCase): def test_immutable_attributes(self): r = self.request_class("http://example.com") - self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com") - self.assertRaises(AttributeError, setattr, r, "body", "xxx") + with pytest.raises(AttributeError): + r.url = "http://example2.com" + with pytest.raises(AttributeError): + r.body = "xxx" def test_callback_and_errback(self): def a_function(): pass r1 = self.request_class("http://example.com") - self.assertIsNone(r1.callback) - self.assertIsNone(r1.errback) + assert r1.callback is None + assert r1.errback is None r2 = self.request_class("http://example.com", callback=a_function) - self.assertIs(r2.callback, a_function) - self.assertIsNone(r2.errback) + assert r2.callback is a_function + assert r2.errback is None r3 = self.request_class("http://example.com", errback=a_function) - self.assertIsNone(r3.callback) - self.assertIs(r3.errback, a_function) + assert r3.callback is None + assert r3.errback is a_function r4 = self.request_class( url="http://example.com", callback=a_function, errback=a_function, ) - self.assertIs(r4.callback, a_function) - self.assertIs(r4.errback, a_function) + assert r4.callback is a_function + assert r4.errback is a_function r5 = self.request_class( url="http://example.com", callback=NO_CALLBACK, errback=NO_CALLBACK, ) - self.assertIs(r5.callback, NO_CALLBACK) - self.assertIs(r5.errback, NO_CALLBACK) + assert r5.callback is NO_CALLBACK + assert r5.errback is NO_CALLBACK def test_callback_and_errback_type(self): - with self.assertRaises(TypeError): + with pytest.raises(TypeError): self.request_class("http://example.com", callback="a_function") - with self.assertRaises(TypeError): + with pytest.raises(TypeError): self.request_class("http://example.com", errback="a_function") - with self.assertRaises(TypeError): + with pytest.raises(TypeError): self.request_class( url="http://example.com", callback="a_function", errback="a_function", ) + def test_setters(self): + request = self.request_class("http://example.com") + + request.flags = ["f1"] + assert request.flags == ["f1"] + + request.cookies = {"sid": "1"} + assert request.cookies == {"sid": "1"} + + headers = Headers({b"X-Test": b"1"}) + request.headers = headers + assert request._headers is headers + request.headers = {b"A": b"b"} + assert isinstance(request.headers, Headers) + assert request._headers[b"A"] == b"b" + + def test_setter_mutable_lazy_loading(self): + """Mutable attributes are set internally to None only until they are + read, then they always return the same falsy instance of the + corresponding mutable structure. + + Setting them to None causes the next read to return a different object. + """ + + request = self.request_class("http://example.com") + + assert request._flags is None + assert request.flags == [] + assert request.flags is request.flags + assert request._flags == [] + original_flags = request.flags + request.flags = None + assert request._flags is None + assert request.flags == [] + assert request.flags is not original_flags + + assert request._cookies is None + assert request.cookies == {} + assert request.cookies is request.cookies + assert request._cookies == {} + original_cookies = request.cookies + request.cookies = None + assert request._cookies is None + assert request.cookies == {} + assert request.cookies is not original_cookies + + if self.default_headers: + assert request._headers == self.default_headers + assert request._headers is not self.default_headers + assert request.headers == self.default_headers + else: + assert request._headers is None + assert request.headers == {} + assert request.headers is request.headers + assert isinstance(request.headers, Headers) + assert isinstance(request._headers, Headers) + original_headers = request.headers + request.headers = None + assert request._headers is None + assert request.headers == {} + assert request._headers == {} + assert request.headers is not original_headers + def test_no_callback(self): - with self.assertRaises(RuntimeError): + with pytest.raises(RuntimeError): NO_CALLBACK() def test_from_curl(self): @@ -355,53 +417,46 @@ class RequestTest(unittest.TestCase): "2%3A15&comments=' --compressed" ) r = self.request_class.from_curl(curl_command) - self.assertEqual(r.method, "POST") - self.assertEqual(r.url, "http://httpbin.org/post") - self.assertEqual( - r.body, - b"custname=John+Smith&custtel=500&custemail=jsmith%40" + assert r.method == "POST" + assert r.url == "http://httpbin.org/post" + assert ( + r.body == b"custname=John+Smith&custtel=500&custemail=jsmith%40" b"example.org&size=small&topping=cheese&topping=onion" - b"&delivery=12%3A15&comments=", - ) - self.assertEqual( - r.cookies, - { - "_gauges_unique_year": "1", - "_gauges_unique": "1", - "_gauges_unique_month": "1", - "_gauges_unique_hour": "1", - "_gauges_unique_day": "1", - }, - ) - self.assertEqual( - r.headers, - { - b"Origin": [b"http://httpbin.org"], - b"Accept-Encoding": [b"gzip, deflate"], - b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"], - b"Upgrade-Insecure-Requests": [b"1"], - b"User-Agent": [ - b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537." - b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202" - b".75 Chrome/62.0.3202.75 Safari/537.36" - ], - b"Content-Type": [b"application /x-www-form-urlencoded"], - b"Accept": [ - b"text/html,application/xhtml+xml,application/xml;q=0." - b"9,image/webp,image/apng,*/*;q=0.8" - ], - b"Cache-Control": [b"max-age=0"], - b"Referer": [b"http://httpbin.org/forms/post"], - b"Connection": [b"keep-alive"], - }, + b"&delivery=12%3A15&comments=" ) + assert r.cookies == { + "_gauges_unique_year": "1", + "_gauges_unique": "1", + "_gauges_unique_month": "1", + "_gauges_unique_hour": "1", + "_gauges_unique_day": "1", + } + assert r.headers == { + b"Origin": [b"http://httpbin.org"], + b"Accept-Encoding": [b"gzip, deflate"], + b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"], + b"Upgrade-Insecure-Requests": [b"1"], + b"User-Agent": [ + b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537." + b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202" + b".75 Chrome/62.0.3202.75 Safari/537.36" + ], + b"Content-Type": [b"application /x-www-form-urlencoded"], + b"Accept": [ + b"text/html,application/xhtml+xml,application/xml;q=0." + b"9,image/webp,image/apng,*/*;q=0.8" + ], + b"Cache-Control": [b"max-age=0"], + b"Referer": [b"http://httpbin.org/forms/post"], + b"Connection": [b"keep-alive"], + } def test_from_curl_with_kwargs(self): r = self.request_class.from_curl( 'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"} ) - self.assertEqual(r.method, "POST") - self.assertEqual(r.meta, {"key": "value"}) + assert r.method == "POST" + assert r.meta == {"key": "value"} def test_from_curl_ignore_unknown_options(self): # By default: it works and ignores the unknown options: --foo and -z @@ -410,1058 +465,31 @@ class RequestTest(unittest.TestCase): r = self.request_class.from_curl( 'curl -X DELETE "http://example.org" --foo -z', ) - self.assertEqual(r.method, "DELETE") + assert r.method == "DELETE" # If `ignore_unknown_options` is set to `False` it raises an error with # the unknown options: --foo and -z - self.assertRaises( - ValueError, - lambda: self.request_class.from_curl( + with pytest.raises(ValueError, match="Unrecognized options:"): + self.request_class.from_curl( 'curl -X PATCH "http://example.org" --foo -z', ignore_unknown_options=False, - ), - ) - - -class FormRequestTest(RequestTest): - request_class = FormRequest - - def assertQueryEqual(self, first, second, msg=None): - first = to_unicode(first).split("&") - second = to_unicode(second).split("&") - return self.assertEqual(sorted(first), sorted(second), msg) - - def test_empty_formdata(self): - r1 = self.request_class("http://www.example.com", formdata={}) - self.assertEqual(r1.body, b"") - - def test_formdata_overrides_querystring(self): - data = (("a", "one"), ("a", "two"), ("b", "2")) - url = self.request_class( - "http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data - ).url.split("#")[0] - fs = _qs(self.request_class(url, method="GET", formdata=data)) - self.assertEqual(set(fs[b"a"]), {b"one", b"two"}) - self.assertEqual(fs[b"b"], [b"2"]) - self.assertIsNone(fs.get(b"c")) - - data = {"a": "1", "b": "2"} - fs = _qs( - self.request_class("http://www.example.com/", method="GET", formdata=data) - ) - self.assertEqual(fs[b"a"], [b"1"]) - self.assertEqual(fs[b"b"], [b"2"]) - - def test_default_encoding_bytes(self): - # using default encoding (utf-8) - data = {b"one": b"two", b"price": b"\xc2\xa3 100"} - r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, "POST") - self.assertEqual(r2.encoding, "utf-8") - self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two") - self.assertEqual( - r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" - ) - - def test_default_encoding_textual_data(self): - # using default encoding (utf-8) - data = {"µ one": "two", "price": "£ 100"} - r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, "POST") - self.assertEqual(r2.encoding, "utf-8") - self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") - self.assertEqual( - r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" - ) - - def test_default_encoding_mixed_data(self): - # using default encoding (utf-8) - data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} - r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, "POST") - self.assertEqual(r2.encoding, "utf-8") - self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") - self.assertEqual( - r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" - ) - - def test_custom_encoding_bytes(self): - data = {b"\xb5 one": b"two", b"price": b"\xa3 100"} - r2 = self.request_class( - "http://www.example.com", formdata=data, encoding="latin1" - ) - self.assertEqual(r2.method, "POST") - self.assertEqual(r2.encoding, "latin1") - self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two") - self.assertEqual( - r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" - ) - - def test_custom_encoding_textual_data(self): - data = {"price": "£ 100"} - r3 = self.request_class( - "http://www.example.com", formdata=data, encoding="latin1" - ) - self.assertEqual(r3.encoding, "latin1") - self.assertEqual(r3.body, b"price=%A3+100") - - def test_multi_key_values(self): - # using multiples values for a single key - data = {"price": "\xa3 100", "colours": ["red", "blue", "green"]} - r3 = self.request_class("http://www.example.com", formdata=data) - self.assertQueryEqual( - r3.body, b"colours=red&colours=blue&colours=green&price=%C2%A3+100" - ) - - def test_from_response_post(self): - response = _buildresponse( - b""" - - - - """, - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response( - response, formdata={"one": ["two", "three"], "six": "seven"} - ) - - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers[b"Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") - fs = _qs(req) - self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"}) - self.assertEqual(set(fs[b"one"]), {b"two", b"three"}) - self.assertEqual(fs[b"test2"], [b"xxx"]) - self.assertEqual(fs[b"six"], [b"seven"]) - - def test_from_response_post_nonascii_bytes_utf8(self): - response = _buildresponse( - b"""
- - - -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response( - response, formdata={"one": ["two", "three"], "six": "seven"} - ) - - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers[b"Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") - fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs["test £"]), {"val1", "val2"}) - self.assertEqual(set(fs["one"]), {"two", "three"}) - self.assertEqual(fs["test2"], ["xxx µ"]) - self.assertEqual(fs["six"], ["seven"]) - - def test_from_response_post_nonascii_bytes_latin1(self): - response = _buildresponse( - b"""
- - - -
""", - url="http://www.example.com/this/list.html", - encoding="latin1", - ) - req = self.request_class.from_response( - response, formdata={"one": ["two", "three"], "six": "seven"} - ) - - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers[b"Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") - fs = _qs(req, to_unicode=True, encoding="latin1") - self.assertEqual(set(fs["test £"]), {"val1", "val2"}) - self.assertEqual(set(fs["one"]), {"two", "three"}) - self.assertEqual(fs["test2"], ["xxx µ"]) - self.assertEqual(fs["six"], ["seven"]) - - def test_from_response_post_nonascii_unicode(self): - response = _buildresponse( - """
- - - -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response( - response, formdata={"one": ["two", "three"], "six": "seven"} - ) - - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers[b"Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") - fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs["test £"]), {"val1", "val2"}) - self.assertEqual(set(fs["one"]), {"two", "three"}) - self.assertEqual(fs["test2"], ["xxx µ"]) - self.assertEqual(fs["six"], ["seven"]) - - def test_from_response_duplicate_form_key(self): - response = _buildresponse("
", url="http://www.example.com") - req = self.request_class.from_response( - response=response, - method="GET", - formdata=(("foo", "bar"), ("foo", "baz")), - ) - self.assertEqual(urlparse(req.url).hostname, "www.example.com") - self.assertEqual(urlparse(req.url).query, "foo=bar&foo=baz") - - def test_from_response_override_duplicate_form_key(self): - response = _buildresponse( - """
- - -
""" - ) - req = self.request_class.from_response( - response, formdata=(("two", "2"), ("two", "4")) - ) - fs = _qs(req) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertEqual(fs[b"two"], [b"2", b"4"]) - - def test_from_response_extra_headers(self): - response = _buildresponse( - """
- - - -
""" - ) - req = self.request_class.from_response( - response=response, - formdata={"one": ["two", "three"], "six": "seven"}, - headers={"Accept-Encoding": "gzip,deflate"}, - ) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.headers["Accept-Encoding"], b"gzip,deflate") - - def test_from_response_get(self): - response = _buildresponse( - """
- - - -
""", - url="http://www.example.com/this/list.html", - ) - r1 = self.request_class.from_response( - response, formdata={"one": ["two", "three"], "six": "seven"} - ) - self.assertEqual(r1.method, "GET") - self.assertEqual(urlparse(r1.url).hostname, "www.example.com") - self.assertEqual(urlparse(r1.url).path, "/this/get.php") - fs = _qs(r1) - self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"}) - self.assertEqual(set(fs[b"one"]), {b"two", b"three"}) - self.assertEqual(fs[b"test2"], [b"xxx"]) - self.assertEqual(fs[b"six"], [b"seven"]) - - def test_from_response_override_params(self): - response = _buildresponse( - """
- - -
""" - ) - req = self.request_class.from_response(response, formdata={"two": "2"}) - fs = _qs(req) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertEqual(fs[b"two"], [b"2"]) - - def test_from_response_drop_params(self): - response = _buildresponse( - """
- - -
""" - ) - req = self.request_class.from_response(response, formdata={"two": None}) - fs = _qs(req) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertNotIn(b"two", fs) - - def test_from_response_override_method(self): - response = _buildresponse( - """ -
- """ - ) - request = FormRequest.from_response(response) - self.assertEqual(request.method, "GET") - request = FormRequest.from_response(response, method="POST") - self.assertEqual(request.method, "POST") - - def test_from_response_override_url(self): - response = _buildresponse( - """ -
- """ - ) - request = FormRequest.from_response(response) - self.assertEqual(request.url, "http://example.com/app") - request = FormRequest.from_response(response, url="http://foo.bar/absolute") - self.assertEqual(request.url, "http://foo.bar/absolute") - request = FormRequest.from_response(response, url="/relative") - self.assertEqual(request.url, "http://example.com/relative") - - def test_from_response_case_insensitive(self): - response = _buildresponse( - """
- - - -
""" - ) - req = self.request_class.from_response(response) - fs = _qs(req) - self.assertEqual(fs[b"clickable1"], [b"clicked1"]) - self.assertFalse(b"i1" in fs, fs) # xpath in _get_inputs() - self.assertFalse(b"clickable2" in fs, fs) # xpath in _get_clickable() - - def test_from_response_submit_first_clickable(self): - response = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response(response, formdata={"two": "2"}) - fs = _qs(req) - self.assertEqual(fs[b"clickable1"], [b"clicked1"]) - self.assertFalse(b"clickable2" in fs, fs) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertEqual(fs[b"two"], [b"2"]) - - def test_from_response_submit_not_first_clickable(self): - response = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response( - response, formdata={"two": "2"}, clickdata={"name": "clickable2"} - ) - fs = _qs(req) - self.assertEqual(fs[b"clickable2"], [b"clicked2"]) - self.assertFalse(b"clickable1" in fs, fs) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertEqual(fs[b"two"], [b"2"]) - - def test_from_response_dont_submit_image_as_input(self): - response = _buildresponse( - """
- - - -
""" - ) - req = self.request_class.from_response(response, dont_click=True) - fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v"]}) - - def test_from_response_dont_submit_reset_as_input(self): - response = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response(response, dont_click=True) - fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b"i2v"]}) - - def test_from_response_clickdata_does_not_ignore_image(self): - response = _buildresponse( - """
- - -
""" - ) - req = self.request_class.from_response(response) - fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b"i2v"]}) - - def test_from_response_multiple_clickdata(self): - response = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response( - response, clickdata={"name": "clickable", "value": "clicked2"} - ) - fs = _qs(req) - self.assertEqual(fs[b"clickable"], [b"clicked2"]) - self.assertEqual(fs[b"one"], [b"clicked1"]) - self.assertEqual(fs[b"two"], [b"clicked2"]) - - def test_from_response_unicode_clickdata(self): - response = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response( - response, clickdata={"name": "price in \u00a3"} - ) - fs = _qs(req, to_unicode=True) - self.assertTrue(fs["price in \u00a3"]) - - def test_from_response_unicode_clickdata_latin1(self): - response = _buildresponse( - """
- - - - -
""", - encoding="latin1", - ) - req = self.request_class.from_response( - response, clickdata={"name": "price in \u00a5"} - ) - fs = _qs(req, to_unicode=True, encoding="latin1") - self.assertTrue(fs["price in \u00a5"]) - - def test_from_response_multiple_forms_clickdata(self): - response = _buildresponse( - """
- - -
-
- - -
- """ - ) - req = self.request_class.from_response( - response, formname="form2", clickdata={"name": "clickable"} - ) - fs = _qs(req) - self.assertEqual(fs[b"clickable"], [b"clicked2"]) - self.assertEqual(fs[b"field2"], [b"value2"]) - self.assertFalse(b"field1" in fs, fs) - - def test_from_response_override_clickable(self): - response = _buildresponse( - """
""" - ) - req = self.request_class.from_response( - response, formdata={"clickme": "two"}, clickdata={"name": "clickme"} - ) - fs = _qs(req) - self.assertEqual(fs[b"clickme"], [b"two"]) - - def test_from_response_dont_click(self): - response = _buildresponse( - """
- - - - -
""" - ) - r1 = self.request_class.from_response(response, dont_click=True) - fs = _qs(r1) - self.assertFalse(b"clickable1" in fs, fs) - self.assertFalse(b"clickable2" in fs, fs) - - def test_from_response_ambiguous_clickdata(self): - response = _buildresponse( - """ -
- - - - -
""" - ) - self.assertRaises( - ValueError, - self.request_class.from_response, - response, - clickdata={"type": "submit"}, - ) - - def test_from_response_non_matching_clickdata(self): - response = _buildresponse( - """
- -
""" - ) - self.assertRaises( - ValueError, - self.request_class.from_response, - response, - clickdata={"nonexistent": "notme"}, - ) - - def test_from_response_nr_index_clickdata(self): - response = _buildresponse( - """
- - -
- """ - ) - req = self.request_class.from_response(response, clickdata={"nr": 1}) - fs = _qs(req) - self.assertIn(b"clickable2", fs) - self.assertNotIn(b"clickable1", fs) - - def test_from_response_invalid_nr_index_clickdata(self): - response = _buildresponse( - """
- -
- """ - ) - self.assertRaises( - ValueError, self.request_class.from_response, response, clickdata={"nr": 1} - ) - - def test_from_response_errors_noform(self): - response = _buildresponse("""""") - self.assertRaises(ValueError, self.request_class.from_response, response) - - def test_from_response_invalid_html5(self): - response = _buildresponse( - """
""" - """""" - """
""" - ) - req = self.request_class.from_response(response, formdata={"bar": "buz"}) - fs = _qs(req) - self.assertEqual(fs, {b"foo": [b"xxx"], b"bar": [b"buz"]}) - - def test_from_response_errors_formnumber(self): - response = _buildresponse( - """
- - - -
""" - ) - self.assertRaises( - IndexError, self.request_class.from_response, response, formnumber=1 - ) - - def test_from_response_noformname(self): - response = _buildresponse( - """
- - -
""" - ) - r1 = self.request_class.from_response(response, formdata={"two": "3"}) - self.assertEqual(r1.method, "POST") - self.assertEqual( - r1.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - fs = _qs(r1) - self.assertEqual(fs, {b"one": [b"1"], b"two": [b"3"]}) - - def test_from_response_formname_exists(self): - response = _buildresponse( - """
- - -
-
- - -
""" - ) - r1 = self.request_class.from_response(response, formname="form2") - self.assertEqual(r1.method, "POST") - fs = _qs(r1) - self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) - - def test_from_response_formname_nonexistent(self): - response = _buildresponse( - """
- -
-
- -
""" - ) - r1 = self.request_class.from_response(response, formname="form3") - self.assertEqual(r1.method, "POST") - fs = _qs(r1) - self.assertEqual(fs, {b"one": [b"1"]}) - - def test_from_response_formname_errors_formnumber(self): - response = _buildresponse( - """
- -
-
- -
""" - ) - self.assertRaises( - IndexError, - self.request_class.from_response, - response, - formname="form3", - formnumber=2, - ) - - def test_from_response_formid_exists(self): - response = _buildresponse( - """
- - -
-
- - -
""" - ) - r1 = self.request_class.from_response(response, formid="form2") - self.assertEqual(r1.method, "POST") - fs = _qs(r1) - self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) - - def test_from_response_formname_nonexistent_fallback_formid(self): - response = _buildresponse( - """
- - -
-
- - -
""" - ) - r1 = self.request_class.from_response( - response, formname="form3", formid="form2" - ) - self.assertEqual(r1.method, "POST") - fs = _qs(r1) - self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) - - def test_from_response_formid_nonexistent(self): - response = _buildresponse( - """
- -
-
- -
""" - ) - r1 = self.request_class.from_response(response, formid="form3") - self.assertEqual(r1.method, "POST") - fs = _qs(r1) - self.assertEqual(fs, {b"one": [b"1"]}) - - def test_from_response_formid_errors_formnumber(self): - response = _buildresponse( - """
- -
-
- -
""" - ) - self.assertRaises( - IndexError, - self.request_class.from_response, - response, - formid="form3", - formnumber=2, - ) - - def test_from_response_select(self): - res = _buildresponse( - """
- - - - - - - - - - - - - - -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"iv2"], b"i2": [b"on"]}) - - def test_from_response_checkbox(self): - res = _buildresponse( - """
- - - - - - - - -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"iv2"], b"i2": [b"on"]}) - - def test_from_response_input_text(self): - res = _buildresponse( - """
- - - - - -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v1"], b"i2": [b""], b"i4": [b"i4v1"]}) - - def test_from_response_input_hidden(self): - res = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v1"], b"i2": [b""]}) - - def test_from_response_input_textarea(self): - res = _buildresponse( - """
- - - -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b""], b"i3": [b""]}) - - def test_from_response_descendants(self): - res = _buildresponse( - """
-
-
- - -
- - - - -
- -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - self.assertEqual(set(fs), {b"h2", b"i2", b"i1", b"i3", b"h1", b"i5", b"i4"}) - - def test_from_response_xpath(self): - response = _buildresponse( - """
- - -
-
- - -
""" - ) - r1 = self.request_class.from_response( - response, formxpath="//form[@action='post.php']" - ) - fs = _qs(r1) - self.assertEqual(fs[b"one"], [b"1"]) - - r1 = self.request_class.from_response( - response, formxpath="//form/input[@name='four']" - ) - fs = _qs(r1) - self.assertEqual(fs[b"three"], [b"3"]) - - self.assertRaises( - ValueError, - self.request_class.from_response, - response, - formxpath="//form/input[@name='abc']", - ) - - def test_from_response_unicode_xpath(self): - response = _buildresponse(b'
') - r = self.request_class.from_response( - response, formxpath="//form[@name='\u044a']" - ) - fs = _qs(r) - self.assertEqual(fs, {}) - - xpath = "//form[@name='\u03b1']" - self.assertRaisesRegex( - ValueError, - re.escape(xpath), - self.request_class.from_response, - response, - formxpath=xpath, - ) - - def test_from_response_button_submit(self): - response = _buildresponse( - """
- - - -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response(response) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") - fs = _qs(req) - self.assertEqual(fs[b"test1"], [b"val1"]) - self.assertEqual(fs[b"test2"], [b"val2"]) - self.assertEqual(fs[b"button1"], [b"submit1"]) - - def test_from_response_button_notype(self): - response = _buildresponse( - """
- - - -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response(response) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") - fs = _qs(req) - self.assertEqual(fs[b"test1"], [b"val1"]) - self.assertEqual(fs[b"test2"], [b"val2"]) - self.assertEqual(fs[b"button1"], [b"submit1"]) - - def test_from_response_submit_novalue(self): - response = _buildresponse( - """
- - - Submit -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response(response) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") - fs = _qs(req) - self.assertEqual(fs[b"test1"], [b"val1"]) - self.assertEqual(fs[b"test2"], [b"val2"]) - self.assertEqual(fs[b"button1"], [b""]) - - def test_from_response_button_novalue(self): - response = _buildresponse( - """
- - - -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response(response) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") - fs = _qs(req) - self.assertEqual(fs[b"test1"], [b"val1"]) - self.assertEqual(fs[b"test2"], [b"val2"]) - self.assertEqual(fs[b"button1"], [b""]) - - def test_html_base_form_action(self): - response = _buildresponse( - """ - - - - - -
-
- - - """, - url="http://a.com/", - ) - req = self.request_class.from_response(response) - self.assertEqual(req.url, "http://b.com/test_form") - - def test_spaces_in_action(self): - resp = _buildresponse('
') - req = self.request_class.from_response(resp) - self.assertEqual(req.url, "http://example.com/path") - - def test_from_response_css(self): - response = _buildresponse( - """
- - -
-
- - -
""" - ) - r1 = self.request_class.from_response( - response, formcss="form[action='post.php']" - ) - fs = _qs(r1) - self.assertEqual(fs[b"one"], [b"1"]) - - r1 = self.request_class.from_response(response, formcss="input[name='four']") - fs = _qs(r1) - self.assertEqual(fs[b"three"], [b"3"]) - - self.assertRaises( - ValueError, - self.request_class.from_response, - response, - formcss="input[name='abc']", - ) - - def test_from_response_valid_form_methods(self): - form_methods = [ - [method, method] for method in self.request_class.valid_form_methods - ] - form_methods.append(["UNKNOWN", "GET"]) - - for method, expected in form_methods: - response = _buildresponse( - f'
' - '' - "
" ) - r = self.request_class.from_response(response) - self.assertEqual(r.method, expected) -def _buildresponse(body, **kwargs): - kwargs.setdefault("body", body) - kwargs.setdefault("url", "http://example.com") - kwargs.setdefault("encoding", "utf-8") - return HtmlResponse(**kwargs) - - -def _qs(req, encoding="utf-8", to_unicode=False): - if req.method == "POST": - qs = req.body - else: - qs = req.url.partition("?")[2] - uqs = unquote_to_bytes(qs) - if to_unicode: - uqs = uqs.decode(encoding) - return parse_qs(uqs, True) - - -class XmlRpcRequestTest(RequestTest): +class TestXmlRpcRequest(TestRequest): request_class = XmlRpcRequest default_method = "POST" default_headers = {b"Content-Type": [b"text/xml"]} def _test_request(self, **kwargs): r = self.request_class("http://scrapytest.org/rpc2", **kwargs) - self.assertEqual(r.headers[b"Content-Type"], b"text/xml") - self.assertEqual( - r.body, - to_bytes( - xmlrpc.client.dumps(**kwargs), encoding=kwargs.get("encoding", "utf-8") - ), + assert r.headers[b"Content-Type"] == b"text/xml" + assert r.body == to_bytes( + xmlrpc.client.dumps(**kwargs), encoding=kwargs.get("encoding", "utf-8") ) - self.assertEqual(r.method, "POST") - self.assertEqual(r.encoding, kwargs.get("encoding", "utf-8")) - self.assertTrue(r.dont_filter, True) + assert r.method == "POST" + assert r.encoding == kwargs.get("encoding", "utf-8") + assert r.dont_filter def test_xmlrpc_dumps(self): self._test_request(params=("value",)) @@ -1469,183 +497,10 @@ class XmlRpcRequestTest(RequestTest): self._test_request(params=("response",), methodresponse="login") self._test_request(params=("pas£",), encoding="utf-8") self._test_request(params=(None,), allow_none=1) - self.assertRaises(TypeError, self._test_request) - self.assertRaises(TypeError, self._test_request, params=(None,)) + with pytest.raises(TypeError): + self._test_request() + with pytest.raises(TypeError): + self._test_request(params=(None,)) def test_latin1(self): self._test_request(params=("pas£",), encoding="latin1") - - -class JsonRequestTest(RequestTest): - request_class = JsonRequest - default_method = "GET" - default_headers = { - b"Content-Type": [b"application/json"], - b"Accept": [b"application/json, text/javascript, */*; q=0.01"], - } - - def setUp(self): - warnings.simplefilter("always") - super().setUp() - - def test_data(self): - r1 = self.request_class(url="http://www.example.com/") - self.assertEqual(r1.body, b"") - - body = b"body" - r2 = self.request_class(url="http://www.example.com/", body=body) - self.assertEqual(r2.body, body) - - data = { - "name": "value", - } - r3 = self.request_class(url="http://www.example.com/", data=data) - self.assertEqual(r3.body, to_bytes(json.dumps(data))) - - # empty data - r4 = self.request_class(url="http://www.example.com/", data=[]) - self.assertEqual(r4.body, to_bytes(json.dumps([]))) - - def test_data_method(self): - # data is not passed - r1 = self.request_class(url="http://www.example.com/") - self.assertEqual(r1.method, "GET") - - body = b"body" - r2 = self.request_class(url="http://www.example.com/", body=body) - self.assertEqual(r2.method, "GET") - - data = { - "name": "value", - } - r3 = self.request_class(url="http://www.example.com/", data=data) - self.assertEqual(r3.method, "POST") - - # method passed explicitly - r4 = self.request_class(url="http://www.example.com/", data=data, method="GET") - self.assertEqual(r4.method, "GET") - - r5 = self.request_class(url="http://www.example.com/", data=[]) - self.assertEqual(r5.method, "POST") - - def test_body_data(self): - """passing both body and data should result a warning""" - body = b"body" - data = { - "name": "value", - } - with warnings.catch_warnings(record=True) as _warnings: - r5 = self.request_class(url="http://www.example.com/", body=body, data=data) - self.assertEqual(r5.body, body) - self.assertEqual(r5.method, "GET") - self.assertEqual(len(_warnings), 1) - self.assertIn("data will be ignored", str(_warnings[0].message)) - - def test_empty_body_data(self): - """passing any body value and data should result a warning""" - data = { - "name": "value", - } - with warnings.catch_warnings(record=True) as _warnings: - r6 = self.request_class(url="http://www.example.com/", body=b"", data=data) - self.assertEqual(r6.body, b"") - self.assertEqual(r6.method, "GET") - self.assertEqual(len(_warnings), 1) - self.assertIn("data will be ignored", str(_warnings[0].message)) - - def test_body_none_data(self): - data = { - "name": "value", - } - with warnings.catch_warnings(record=True) as _warnings: - r7 = self.request_class(url="http://www.example.com/", body=None, data=data) - self.assertEqual(r7.body, to_bytes(json.dumps(data))) - self.assertEqual(r7.method, "POST") - self.assertEqual(len(_warnings), 0) - - def test_body_data_none(self): - with warnings.catch_warnings(record=True) as _warnings: - r8 = self.request_class(url="http://www.example.com/", body=None, data=None) - self.assertEqual(r8.method, "GET") - self.assertEqual(len(_warnings), 0) - - def test_dumps_sort_keys(self): - """Test that sort_keys=True is passed to json.dumps by default""" - data = { - "name": "value", - } - with mock.patch("json.dumps", return_value=b"") as mock_dumps: - self.request_class(url="http://www.example.com/", data=data) - kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs["sort_keys"], True) - - def test_dumps_kwargs(self): - """Test that dumps_kwargs are passed to json.dumps""" - data = { - "name": "value", - } - dumps_kwargs = { - "ensure_ascii": True, - "allow_nan": True, - } - with mock.patch("json.dumps", return_value=b"") as mock_dumps: - self.request_class( - url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs - ) - kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs["ensure_ascii"], True) - self.assertEqual(kwargs["allow_nan"], True) - - def test_replace_data(self): - data1 = { - "name1": "value1", - } - data2 = { - "name2": "value2", - } - r1 = self.request_class(url="http://www.example.com/", data=data1) - r2 = r1.replace(data=data2) - self.assertEqual(r2.body, to_bytes(json.dumps(data2))) - - def test_replace_sort_keys(self): - """Test that replace provides sort_keys=True to json.dumps""" - data1 = { - "name1": "value1", - } - data2 = { - "name2": "value2", - } - r1 = self.request_class(url="http://www.example.com/", data=data1) - with mock.patch("json.dumps", return_value=b"") as mock_dumps: - r1.replace(data=data2) - kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs["sort_keys"], True) - - def test_replace_dumps_kwargs(self): - """Test that dumps_kwargs are provided to json.dumps when replace is called""" - data1 = { - "name1": "value1", - } - data2 = { - "name2": "value2", - } - dumps_kwargs = { - "ensure_ascii": True, - "allow_nan": True, - } - r1 = self.request_class( - url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs - ) - with mock.patch("json.dumps", return_value=b"") as mock_dumps: - r1.replace(data=data2) - kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs["ensure_ascii"], True) - self.assertEqual(kwargs["allow_nan"], True) - - def tearDown(self): - warnings.resetwarnings() - super().tearDown() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py new file mode 100644 index 000000000..a4f87d50d --- /dev/null +++ b/tests/test_http_request_form.py @@ -0,0 +1,1028 @@ +from __future__ import annotations + +import re +from urllib.parse import parse_qs, unquote_to_bytes + +import pytest + +from scrapy.http import FormRequest, HtmlResponse +from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_unicode +from tests.test_http_request import TestRequest + + +def _buildresponse(body, **kwargs): + kwargs.setdefault("body", body) + kwargs.setdefault("url", "http://example.com") + kwargs.setdefault("encoding", "utf-8") + return HtmlResponse(**kwargs) + + +def _qs(req, encoding="utf-8", to_unicode=False): + qs = req.body if req.method == "POST" else req.url.partition("?")[2] + uqs = unquote_to_bytes(qs) + if to_unicode: + uqs = uqs.decode(encoding) + return parse_qs(uqs, True) + + +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class TestFormRequest(TestRequest): + request_class = FormRequest # type: ignore[assignment] + + def assertQueryEqual(self, first, second, msg=None): + first = to_unicode(first).split("&") + second = to_unicode(second).split("&") + assert sorted(first) == sorted(second), msg + + def test_empty_formdata(self): + r1 = self.request_class("http://www.example.com", formdata={}) + assert r1.body == b"" + + def test_formdata_overrides_querystring(self): + data = (("a", "one"), ("a", "two"), ("b", "2")) + url = self.request_class( + "http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data + ).url.split("#", maxsplit=1)[0] + fs = _qs(self.request_class(url, method="GET", formdata=data)) + assert set(fs[b"a"]) == {b"one", b"two"} + assert fs[b"b"] == [b"2"] + assert fs.get(b"c") is None + + data = {"a": "1", "b": "2"} + fs = _qs( + self.request_class("http://www.example.com/", method="GET", formdata=data) + ) + assert fs[b"a"] == [b"1"] + assert fs[b"b"] == [b"2"] + + def test_default_encoding_bytes(self): + # using default encoding (utf-8) + data = {b"one": b"two", b"price": b"\xc2\xa3 100"} + r2 = self.request_class("http://www.example.com", formdata=data) + assert r2.method == "POST" + assert r2.encoding == "utf-8" + self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two") + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" + + def test_default_encoding_textual_data(self): + # using default encoding (utf-8) + data = {"µ one": "two", "price": "£ 100"} + r2 = self.request_class("http://www.example.com", formdata=data) + assert r2.method == "POST" + assert r2.encoding == "utf-8" + self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" + + def test_default_encoding_mixed_data(self): + # using default encoding (utf-8) + data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} + r2 = self.request_class("http://www.example.com", formdata=data) + assert r2.method == "POST" + assert r2.encoding == "utf-8" + self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" + + def test_custom_encoding_bytes(self): + data = {b"\xb5 one": b"two", b"price": b"\xa3 100"} + r2 = self.request_class( + "http://www.example.com", formdata=data, encoding="latin1" + ) + assert r2.method == "POST" + assert r2.encoding == "latin1" + self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two") + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" + + def test_custom_encoding_textual_data(self): + data = {"price": "£ 100"} + r3 = self.request_class( + "http://www.example.com", formdata=data, encoding="latin1" + ) + assert r3.encoding == "latin1" + assert r3.body == b"price=%A3+100" + + def test_multi_key_values(self): + # using multiples values for a single key + data = {"price": "\xa3 100", "colours": ["red", "blue", "green"]} + r3 = self.request_class("http://www.example.com", formdata=data) + self.assertQueryEqual( + r3.body, b"colours=red&colours=blue&colours=green&price=%C2%A3+100" + ) + + def test_from_response_post(self): + response = _buildresponse( + b"""
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) + + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req) + assert set(fs[b"test"]) == {b"val1", b"val2"} + assert set(fs[b"one"]) == {b"two", b"three"} + assert fs[b"test2"] == [b"xxx"] + assert fs[b"six"] == [b"seven"] + + def test_from_response_post_nonascii_bytes_utf8(self): + response = _buildresponse( + b"""
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) + + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req, to_unicode=True) + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] + + def test_from_response_post_nonascii_bytes_latin1(self): + response = _buildresponse( + b"""
+ + + +
""", + url="http://www.example.com/this/list.html", + encoding="latin1", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) + + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req, to_unicode=True, encoding="latin1") + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] + + def test_from_response_post_nonascii_unicode(self): + response = _buildresponse( + """
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) + + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req, to_unicode=True) + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] + + def test_from_response_duplicate_form_key(self): + response = _buildresponse("
", url="http://www.example.com") + req = self.request_class.from_response( + response=response, + method="GET", + formdata=(("foo", "bar"), ("foo", "baz")), + ) + assert urlparse_cached(req).hostname == "www.example.com" + assert urlparse_cached(req).query == "foo=bar&foo=baz" + + def test_from_response_override_duplicate_form_key(self): + response = _buildresponse( + """
+ + +
""" + ) + req = self.request_class.from_response( + response, formdata=(("two", "2"), ("two", "4")) + ) + fs = _qs(req) + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2", b"4"] + + def test_from_response_extra_headers(self): + response = _buildresponse( + """
+ + + +
""" + ) + req = self.request_class.from_response( + response=response, + formdata={"one": ["two", "three"], "six": "seven"}, + headers={"Accept-Encoding": "gzip,deflate"}, + ) + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.headers["Accept-Encoding"] == b"gzip,deflate" + + def test_from_response_get(self): + response = _buildresponse( + """
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + r1 = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) + assert r1.method == "GET" + assert urlparse_cached(r1).hostname == "www.example.com" + assert urlparse_cached(r1).path == "/this/get.php" + fs = _qs(r1) + assert set(fs[b"test"]) == {b"val1", b"val2"} + assert set(fs[b"one"]) == {b"two", b"three"} + assert fs[b"test2"] == [b"xxx"] + assert fs[b"six"] == [b"seven"] + + def test_from_response_override_params(self): + response = _buildresponse( + """
+ + +
""" + ) + req = self.request_class.from_response(response, formdata={"two": "2"}) + fs = _qs(req) + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] + + def test_from_response_drop_params(self): + response = _buildresponse( + """
+ + +
""" + ) + req = self.request_class.from_response(response, formdata={"two": None}) + fs = _qs(req) + assert fs[b"one"] == [b"1"] + assert b"two" not in fs + + def test_from_response_override_method(self): + response = _buildresponse( + """ +
+ """ + ) + request = FormRequest.from_response(response) + assert request.method == "GET" + request = FormRequest.from_response(response, method="POST") + assert request.method == "POST" + + def test_from_response_override_url(self): + response = _buildresponse( + """ +
+ """ + ) + request = FormRequest.from_response(response) + assert request.url == "http://example.com/app" + request = FormRequest.from_response(response, url="http://foo.bar/absolute") + assert request.url == "http://foo.bar/absolute" + request = FormRequest.from_response(response, url="/relative") + assert request.url == "http://example.com/relative" + + def test_from_response_case_insensitive(self): + response = _buildresponse( + """
+ + + +
""" + ) + req = self.request_class.from_response(response) + fs = _qs(req) + assert fs[b"clickable1"] == [b"clicked1"] + assert b"i1" not in fs, fs # xpath in _get_inputs() + assert b"clickable2" not in fs, fs # xpath in _get_clickable() + + def test_from_response_submit_first_clickable(self): + response = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response(response, formdata={"two": "2"}) + fs = _qs(req) + assert fs[b"clickable1"] == [b"clicked1"] + assert b"clickable2" not in fs, fs + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] + + def test_from_response_submit_not_first_clickable(self): + response = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response( + response, formdata={"two": "2"}, clickdata={"name": "clickable2"} + ) + fs = _qs(req) + assert fs[b"clickable2"] == [b"clicked2"] + assert b"clickable1" not in fs, fs + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] + + def test_from_response_dont_submit_image_as_input(self): + response = _buildresponse( + """
+ + + +
""" + ) + req = self.request_class.from_response(response, dont_click=True) + fs = _qs(req) + assert fs == {b"i1": [b"i1v"]} + + def test_from_response_dont_submit_reset_as_input(self): + response = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response(response, dont_click=True) + fs = _qs(req) + assert fs == {b"i1": [b"i1v"], b"i2": [b"i2v"]} + + def test_from_response_clickdata_does_not_ignore_image(self): + response = _buildresponse( + """
+ + +
""" + ) + req = self.request_class.from_response(response) + fs = _qs(req) + assert fs == {b"i1": [b"i1v"], b"i2": [b"i2v"]} + + def test_from_response_multiple_clickdata(self): + response = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response( + response, clickdata={"name": "clickable", "value": "clicked2"} + ) + fs = _qs(req) + assert fs[b"clickable"] == [b"clicked2"] + assert fs[b"one"] == [b"clicked1"] + assert fs[b"two"] == [b"clicked2"] + + def test_from_response_unicode_clickdata(self): + response = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response( + response, clickdata={"name": "price in \u00a3"} + ) + fs = _qs(req, to_unicode=True) + assert fs["price in \u00a3"] + + def test_from_response_unicode_clickdata_latin1(self): + response = _buildresponse( + """
+ + + + +
""", + encoding="latin1", + ) + req = self.request_class.from_response( + response, clickdata={"name": "price in \u00a5"} + ) + fs = _qs(req, to_unicode=True, encoding="latin1") + assert fs["price in \u00a5"] + + def test_from_response_multiple_forms_clickdata(self): + response = _buildresponse( + """
+ + +
+
+ + +
+ """ + ) + req = self.request_class.from_response( + response, formname="form2", clickdata={"name": "clickable"} + ) + fs = _qs(req) + assert fs[b"clickable"] == [b"clicked2"] + assert fs[b"field2"] == [b"value2"] + assert b"field1" not in fs, fs + + def test_from_response_override_clickable(self): + response = _buildresponse( + """
""" + ) + req = self.request_class.from_response( + response, formdata={"clickme": "two"}, clickdata={"name": "clickme"} + ) + fs = _qs(req) + assert fs[b"clickme"] == [b"two"] + + def test_from_response_dont_click(self): + response = _buildresponse( + """
+ + + + +
""" + ) + r1 = self.request_class.from_response(response, dont_click=True) + fs = _qs(r1) + assert b"clickable1" not in fs, fs + assert b"clickable2" not in fs, fs + + def test_from_response_ambiguous_clickdata(self): + response = _buildresponse( + """ +
+ + + + +
""" + ) + with pytest.raises( + ValueError, + match=r"Multiple elements found .* matching the criteria in clickdata", + ): + self.request_class.from_response(response, clickdata={"type": "submit"}) + + def test_from_response_non_matching_clickdata(self): + response = _buildresponse( + """
+ +
""" + ) + with pytest.raises( + ValueError, match="No clickable element matching clickdata:" + ): + self.request_class.from_response( + response, clickdata={"nonexistent": "notme"} + ) + + def test_from_response_nr_index_clickdata(self): + response = _buildresponse( + """
+ + +
+ """ + ) + req = self.request_class.from_response(response, clickdata={"nr": 1}) + fs = _qs(req) + assert b"clickable2" in fs + assert b"clickable1" not in fs + + def test_from_response_invalid_nr_index_clickdata(self): + response = _buildresponse( + """
+ +
+ """ + ) + with pytest.raises( + ValueError, match="No clickable element matching clickdata:" + ): + self.request_class.from_response(response, clickdata={"nr": 1}) + + def test_from_response_errors_noform(self): + response = _buildresponse("""""") + with pytest.raises(ValueError, match="No
element found in"): + self.request_class.from_response(response) + + def test_from_response_invalid_html5(self): + response = _buildresponse( + """""" + """""" + """
""" + ) + req = self.request_class.from_response(response, formdata={"bar": "buz"}) + fs = _qs(req) + assert fs == {b"foo": [b"xxx"], b"bar": [b"buz"]} + + def test_from_response_errors_formnumber(self): + response = _buildresponse( + """
+ + + +
""" + ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formnumber=1) + + def test_from_response_noformname(self): + response = _buildresponse( + """
+ + +
""" + ) + r1 = self.request_class.from_response(response, formdata={"two": "3"}) + assert r1.method == "POST" + assert r1.headers["Content-type"] == b"application/x-www-form-urlencoded" + fs = _qs(r1) + assert fs == {b"one": [b"1"], b"two": [b"3"]} + + def test_from_response_formname_exists(self): + response = _buildresponse( + """
+ + +
+
+ + +
""" + ) + r1 = self.request_class.from_response(response, formname="form2") + assert r1.method == "POST" + fs = _qs(r1) + assert fs == {b"four": [b"4"], b"three": [b"3"]} + + def test_from_response_formname_nonexistent(self): + response = _buildresponse( + """
+ +
+
+ +
""" + ) + r1 = self.request_class.from_response(response, formname="form3") + assert r1.method == "POST" + fs = _qs(r1) + assert fs == {b"one": [b"1"]} + + def test_from_response_formname_errors_formnumber(self): + response = _buildresponse( + """
+ +
+
+ +
""" + ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formname="form3", formnumber=2) + + def test_from_response_formid_exists(self): + response = _buildresponse( + """
+ + +
+
+ + +
""" + ) + r1 = self.request_class.from_response(response, formid="form2") + assert r1.method == "POST" + fs = _qs(r1) + assert fs == {b"four": [b"4"], b"three": [b"3"]} + + def test_from_response_formname_nonexistent_fallback_formid(self): + response = _buildresponse( + """
+ + +
+
+ + +
""" + ) + r1 = self.request_class.from_response( + response, formname="form3", formid="form2" + ) + assert r1.method == "POST" + fs = _qs(r1) + assert fs == {b"four": [b"4"], b"three": [b"3"]} + + def test_from_response_formid_nonexistent(self): + response = _buildresponse( + """
+ +
+
+ +
""" + ) + r1 = self.request_class.from_response(response, formid="form3") + assert r1.method == "POST" + fs = _qs(r1) + assert fs == {b"one": [b"1"]} + + def test_from_response_formid_errors_formnumber(self): + response = _buildresponse( + """
+ +
+
+ +
""" + ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formid="form3", formnumber=2) + + def test_from_response_select(self): + res = _buildresponse( + """
+ + + + + + + + + + + + + + +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert fs == {b"i1": [b"iv2"], b"i2": [b"on"]} + + def test_from_response_checkbox(self): + res = _buildresponse( + """
+ + + + + + + + +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert fs == {b"i1": [b"iv2"], b"i2": [b"on"]} + + def test_from_response_input_text(self): + res = _buildresponse( + """
+ + + + + +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert fs == {b"i1": [b"i1v1"], b"i2": [b""], b"i4": [b"i4v1"]} + + def test_from_response_input_hidden(self): + res = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert fs == {b"i1": [b"i1v1"], b"i2": [b""]} + + def test_from_response_input_textarea(self): + res = _buildresponse( + """
+ + + +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert fs == {b"i1": [b"i1v"], b"i2": [b""], b"i3": [b""]} + + def test_from_response_descendants(self): + res = _buildresponse( + """
+
+
+ + +
+ + + + +
+ +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert set(fs) == {b"h2", b"i2", b"i1", b"i3", b"h1", b"i5", b"i4"} + + def test_from_response_xpath(self): + response = _buildresponse( + """
+ + +
+
+ + +
""" + ) + r1 = self.request_class.from_response( + response, formxpath="//form[@action='post.php']" + ) + fs = _qs(r1) + assert fs[b"one"] == [b"1"] + + r1 = self.request_class.from_response( + response, formxpath="//form/input[@name='four']" + ) + fs = _qs(r1) + assert fs[b"three"] == [b"3"] + + with pytest.raises(ValueError, match="No
element found with"): + self.request_class.from_response( + response, formxpath="//form/input[@name='abc']" + ) + + def test_from_response_unicode_xpath(self): + response = _buildresponse(b'
') + r = self.request_class.from_response( + response, formxpath="//form[@name='\u044a']" + ) + fs = _qs(r) + assert not fs + + xpath = "//form[@name='\u03b1']" + with pytest.raises(ValueError, match=re.escape(xpath)): + self.request_class.from_response(response, formxpath=xpath) + + def test_from_response_button_submit(self): + response = _buildresponse( + """
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response(response) + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b"submit1"] + + def test_from_response_button_notype(self): + response = _buildresponse( + """
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response(response) + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b"submit1"] + + def test_from_response_submit_novalue(self): + response = _buildresponse( + """
+ + + Submit +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response(response) + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b""] + + def test_from_response_button_novalue(self): + response = _buildresponse( + """
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response(response) + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b""] + + def test_html_base_form_action(self): + response = _buildresponse( + """ + + + + + +
+
+ + + """, + url="http://a.com/", + ) + req = self.request_class.from_response(response) + assert req.url == "http://b.com/test_form" + + def test_spaces_in_action(self): + resp = _buildresponse('
') + req = self.request_class.from_response(resp) + assert req.url == "http://example.com/path" + + def test_from_response_css(self): + response = _buildresponse( + """
+ + +
+
+ + +
""" + ) + r1 = self.request_class.from_response( + response, formcss="form[action='post.php']" + ) + fs = _qs(r1) + assert fs[b"one"] == [b"1"] + + r1 = self.request_class.from_response(response, formcss="input[name='four']") + fs = _qs(r1) + assert fs[b"three"] == [b"3"] + + with pytest.raises(ValueError, match="No
element found with"): + self.request_class.from_response(response, formcss="input[name='abc']") + + def test_from_response_valid_form_methods(self): + form_methods = [ + [method, method] for method in self.request_class.valid_form_methods + ] + form_methods.append(["UNKNOWN", "GET"]) + + for method, expected in form_methods: + response = _buildresponse( + f'' + '' + "
" + ) + r = self.request_class.from_response(response) + assert r.method == expected + + def test_form_response_with_invalid_formdata_type_error(self): + """Test that a ValueError is raised for non-iterable and non-dict formdata input""" + response = _buildresponse( + """ +
+ +
+ """ + ) + with pytest.raises( + ValueError, match="formdata should be a dict or iterable of tuples" + ): + FormRequest.from_response(response, formdata=123) + + def test_form_response_with_custom_invalid_formdata_value_error(self): + """Test that a ValueError is raised for fault-inducing iterable formdata input""" + response = _buildresponse( + """ +
+ +
+ """ + ) + + with pytest.raises( + ValueError, match="formdata should be a dict or iterable of tuples" + ): + FormRequest.from_response(response, formdata=("a",)) + + def test_get_form_with_xpath_no_form_parent(self): + """Test that _get_from raised a ValueError when an XPath selects an element + not nested within a
and no parent is found""" + response = _buildresponse( + """ +
+

This paragraph is not inside a form.

+
+ + +
+ """ + ) + + with pytest.raises(ValueError, match="No
element found with"): + FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') diff --git a/tests/test_http_request_json.py b/tests/test_http_request_json.py new file mode 100644 index 000000000..fcfe78365 --- /dev/null +++ b/tests/test_http_request_json.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +import json +import warnings +from unittest import mock + +from scrapy.http import JsonRequest +from scrapy.utils.python import to_bytes +from tests.test_http_request import TestRequest + + +class TestJsonRequest(TestRequest): + request_class = JsonRequest + default_method = "GET" + default_headers = { + b"Content-Type": [b"application/json"], + b"Accept": [b"application/json, text/javascript, */*; q=0.01"], + } + + def test_data(self): + r1 = self.request_class(url="http://www.example.com/") + assert r1.body == b"" + + body = b"body" + r2 = self.request_class(url="http://www.example.com/", body=body) + assert r2.body == body + + data = { + "name": "value", + } + r3 = self.request_class(url="http://www.example.com/", data=data) + assert r3.body == to_bytes(json.dumps(data)) + + # empty data + r4 = self.request_class(url="http://www.example.com/", data=[]) + assert r4.body == to_bytes(json.dumps([])) + + def test_data_method(self): + # data is not passed + r1 = self.request_class(url="http://www.example.com/") + assert r1.method == "GET" + + body = b"body" + r2 = self.request_class(url="http://www.example.com/", body=body) + assert r2.method == "GET" + + data = { + "name": "value", + } + r3 = self.request_class(url="http://www.example.com/", data=data) + assert r3.method == "POST" + + # method passed explicitly + r4 = self.request_class(url="http://www.example.com/", data=data, method="GET") + assert r4.method == "GET" + + r5 = self.request_class(url="http://www.example.com/", data=[]) + assert r5.method == "POST" + + def test_body_data(self): + """passing both body and data should result a warning""" + body = b"body" + data = { + "name": "value", + } + with warnings.catch_warnings(record=True) as _warnings: + r5 = self.request_class(url="http://www.example.com/", body=body, data=data) + assert r5.body == body + assert r5.method == "GET" + assert len(_warnings) == 1 + assert "data will be ignored" in str(_warnings[0].message) + + def test_empty_body_data(self): + """passing any body value and data should result a warning""" + data = { + "name": "value", + } + with warnings.catch_warnings(record=True) as _warnings: + r6 = self.request_class(url="http://www.example.com/", body=b"", data=data) + assert r6.body == b"" + assert r6.method == "GET" + assert len(_warnings) == 1 + assert "data will be ignored" in str(_warnings[0].message) + + def test_body_none_data(self): + data = { + "name": "value", + } + with warnings.catch_warnings(record=True) as _warnings: + r7 = self.request_class(url="http://www.example.com/", body=None, data=data) + assert r7.body == to_bytes(json.dumps(data)) + assert r7.method == "POST" + assert len(_warnings) == 0 + + def test_body_data_none(self): + with warnings.catch_warnings(record=True) as _warnings: + r8 = self.request_class(url="http://www.example.com/", body=None, data=None) + assert r8.method == "GET" + assert len(_warnings) == 0 + + def test_dumps_sort_keys(self): + """Test that sort_keys=True is passed to json.dumps by default""" + data = { + "name": "value", + } + with mock.patch("json.dumps", return_value=b"") as mock_dumps: + self.request_class(url="http://www.example.com/", data=data) + kwargs = mock_dumps.call_args[1] + assert kwargs["sort_keys"] is True + + def test_dumps_kwargs(self): + """Test that dumps_kwargs are passed to json.dumps""" + data = { + "name": "value", + } + dumps_kwargs = { + "ensure_ascii": True, + "allow_nan": True, + } + with mock.patch("json.dumps", return_value=b"") as mock_dumps: + self.request_class( + url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs + ) + kwargs = mock_dumps.call_args[1] + assert kwargs["ensure_ascii"] is True + assert kwargs["allow_nan"] is True + + def test_replace_data(self): + data1 = { + "name1": "value1", + } + data2 = { + "name2": "value2", + } + r1 = self.request_class(url="http://www.example.com/", data=data1) + r2 = r1.replace(data=data2) + assert r2.body == to_bytes(json.dumps(data2)) + + def test_replace_sort_keys(self): + """Test that replace provides sort_keys=True to json.dumps""" + data1 = { + "name1": "value1", + } + data2 = { + "name2": "value2", + } + r1 = self.request_class(url="http://www.example.com/", data=data1) + with mock.patch("json.dumps", return_value=b"") as mock_dumps: + r1.replace(data=data2) + kwargs = mock_dumps.call_args[1] + assert kwargs["sort_keys"] is True + + def test_replace_dumps_kwargs(self): + """Test that dumps_kwargs are provided to json.dumps when replace is called""" + data1 = { + "name1": "value1", + } + data2 = { + "name2": "value2", + } + dumps_kwargs = { + "ensure_ascii": True, + "allow_nan": True, + } + r1 = self.request_class( + url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs + ) + with mock.patch("json.dumps", return_value=b"") as mock_dumps: + r1.replace(data=data2) + kwargs = mock_dumps.call_args[1] + assert kwargs["ensure_ascii"] is True + assert kwargs["allow_nan"] is True + + def test_replacement_both_body_and_data_warns(self): + """Test that we get a warning if both body and data are passed""" + body1 = None + body2 = b"body" + data1 = { + "name1": "value1", + } + data2 = { + "name2": "value2", + } + r1 = self.request_class(url="http://www.example.com/", data=data1, body=body1) + + with warnings.catch_warnings(record=True) as _warnings: + r1.replace(data=data2, body=body2) + assert "Both body and data passed. data will be ignored" in str( + _warnings[0].message + ) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index bfa08136f..09c95dc29 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,88 +1,66 @@ -import codecs -import unittest -from unittest import mock - +import pytest from packaging.version import Version as parse_version -from pytest import mark from w3lib import __version__ as w3lib_version from w3lib.encoding import resolve_encoding from scrapy.exceptions import NotSupported -from scrapy.http import ( - Headers, - HtmlResponse, - Request, - Response, - TextResponse, - XmlResponse, -) +from scrapy.http import Headers, Request, Response from scrapy.link import Link -from scrapy.selector import Selector -from scrapy.utils.python import to_unicode from tests import get_testdata -class BaseResponseTest(unittest.TestCase): +class TestResponse: response_class = Response def test_init(self): # Response requires url in the constructor - self.assertRaises(Exception, self.response_class) - self.assertTrue( - isinstance(self.response_class("http://example.com/"), self.response_class) - ) - self.assertRaises(TypeError, self.response_class, b"http://example.com") - self.assertRaises( - TypeError, self.response_class, url="http://example.com", body={} + with pytest.raises(TypeError): + self.response_class() + assert isinstance( + self.response_class("http://example.com/"), self.response_class ) + with pytest.raises(TypeError): + self.response_class(b"http://example.com") + with pytest.raises(TypeError): + self.response_class(url="http://example.com", body={}) # body can be str or None - self.assertTrue( - isinstance( - self.response_class("http://example.com/", body=b""), - self.response_class, - ) + assert isinstance( + self.response_class("http://example.com/", body=b""), + self.response_class, ) - self.assertTrue( - isinstance( - self.response_class("http://example.com/", body=b"body"), - self.response_class, - ) + assert isinstance( + self.response_class("http://example.com/", body=b"body"), + self.response_class, ) # test presence of all optional parameters - self.assertTrue( - isinstance( - self.response_class( - "http://example.com/", body=b"", headers={}, status=200 - ), - self.response_class, - ) + assert isinstance( + self.response_class( + "http://example.com/", body=b"", headers={}, status=200 + ), + self.response_class, ) r = self.response_class("http://www.example.com") assert isinstance(r.url, str) - self.assertEqual(r.url, "http://www.example.com") - self.assertEqual(r.status, 200) + assert r.url == "http://www.example.com" + assert r.status == 200 assert isinstance(r.headers, Headers) - self.assertEqual(r.headers, {}) + assert not r.headers headers = {"foo": "bar"} body = b"a body" r = self.response_class("http://www.example.com", headers=headers, body=body) assert r.headers is not headers - self.assertEqual(r.headers[b"foo"], b"bar") + assert r.headers[b"foo"] == b"bar" r = self.response_class("http://www.example.com", status=301) - self.assertEqual(r.status, 301) + assert r.status == 301 r = self.response_class("http://www.example.com", status="301") - self.assertEqual(r.status, 301) - self.assertRaises( - ValueError, - self.response_class, - "http://example.com", - status="lala200", - ) + assert r.status == 301 + with pytest.raises(ValueError, match=r"invalid literal for int\(\)"): + self.response_class("http://example.com", status="lala200") def test_copy(self): """Test Response copy""" @@ -91,18 +69,18 @@ class BaseResponseTest(unittest.TestCase): r1.flags.append("cached") r2 = r1.copy() - self.assertEqual(r1.status, r2.status) - self.assertEqual(r1.body, r2.body) + assert r1.status == r2.status + assert r1.body == r2.body # make sure flags list is shallow copied assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" - self.assertEqual(r1.flags, r2.flags) + assert r1.flags == r2.flags # make sure headers attribute is shallow copied - assert ( - r1.headers is not r2.headers - ), "headers must be a shallow copy, not identical" - self.assertEqual(r1.headers, r2.headers) + assert r1.headers is not r2.headers, ( + "headers must be a shallow copy, not identical" + ) + assert r1.headers == r2.headers def test_copy_meta(self): req = Request("http://www.example.com") @@ -122,14 +100,12 @@ class BaseResponseTest(unittest.TestCase): def test_unavailable_meta(self): r1 = self.response_class("http://www.example.com", body=b"Some body") - with self.assertRaisesRegex(AttributeError, r"Response\.meta not available"): + with pytest.raises(AttributeError, match=r"Response\.meta not available"): r1.meta def test_unavailable_cb_kwargs(self): r1 = self.response_class("http://www.example.com", body=b"Some body") - with self.assertRaisesRegex( - AttributeError, r"Response\.cb_kwargs not available" - ): + with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"): r1.cb_kwargs def test_copy_inherited_classes(self): @@ -149,16 +125,16 @@ class BaseResponseTest(unittest.TestCase): r1 = self.response_class("http://www.example.com") r2 = r1.replace(status=301, body=b"New body", headers=hdrs) assert r1.body == b"" - self.assertEqual(r1.url, r2.url) - self.assertEqual((r1.status, r2.status), (200, 301)) - self.assertEqual((r1.body, r2.body), (b"", b"New body")) - self.assertEqual((r1.headers, r2.headers), ({}, hdrs)) + assert r1.url == r2.url + assert (r1.status, r2.status) == (200, 301) + assert (r1.body, r2.body) == (b"", b"New body") + assert (r1.headers, r2.headers) == ({}, hdrs) # Empty attributes (which may fail if not compared properly) r3 = self.response_class("http://www.example.com", flags=["cached"]) r4 = r3.replace(body=b"", flags=[]) - self.assertEqual(r4.body, b"") - self.assertEqual(r4.flags, []) + assert r4.body == b"" + assert not r4.flags def _assert_response_values(self, response, encoding, body): if isinstance(body, str): @@ -171,31 +147,84 @@ class BaseResponseTest(unittest.TestCase): assert isinstance(response.body, bytes) assert isinstance(response.text, str) self._assert_response_encoding(response, encoding) - self.assertEqual(response.body, body_bytes) - self.assertEqual(response.text, body_unicode) + assert response.body == body_bytes + assert response.text == body_unicode def _assert_response_encoding(self, response, encoding): - self.assertEqual(response.encoding, resolve_encoding(encoding)) + assert response.encoding == resolve_encoding(encoding) def test_immutable_attributes(self): r = self.response_class("http://example.com") - self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com") - self.assertRaises(AttributeError, setattr, r, "body", "xxx") + with pytest.raises(AttributeError): + r.url = "http://example2.com" + with pytest.raises(AttributeError): + r.body = "xxx" + + def test_setter_mutable_lazy_loading(self): + """Mutable attributes are set internally to None only until they are + read, then they always return the same falsy instance of the + corresponding mutable structure. + + Setting them to None causes the next read to return a different object. + """ + + response = self.response_class("http://example.com") + + response.request = Request("http://example.com") + + assert response._flags is None + assert response.flags == [] + assert response.flags is response.flags + assert response._flags == [] + original_flags = response.flags + response.flags = None + assert response._flags is None + assert response.flags == [] + assert response.flags is not original_flags + + assert response._headers is None + assert response.headers == {} + assert response.headers is response.headers + assert isinstance(response.headers, Headers) + assert isinstance(response._headers, Headers) + original_headers = response.headers + response.headers = None + assert response._headers is None + assert response.headers == {} + assert response._headers == {} + assert response.headers is not original_headers + + def test_setters(self): + response = self.response_class("http://example.com") + + response.flags = ["f1"] + assert response.flags == ["f1"] + + headers = Headers({b"X-Test": b"1"}) + response.headers = headers + assert response._headers is headers + response.headers = {b"A": b"b"} + assert isinstance(response.headers, Headers) + assert response._headers[b"A"] == b"b" def test_urljoin(self): """Test urljoin shortcut (only for existence, since behavior equals urljoin)""" joined = self.response_class("http://www.example.com").urljoin("/test") absolute = "http://www.example.com/test" - self.assertEqual(joined, absolute) + assert joined == absolute def test_shortcut_attributes(self): r = self.response_class("http://example.com", body=b"hello") if self.response_class == Response: msg = "Response content isn't text" - self.assertRaisesRegex(AttributeError, msg, getattr, r, "text") - self.assertRaisesRegex(NotSupported, msg, r.css, "body") - self.assertRaisesRegex(NotSupported, msg, r.xpath, "//body") - self.assertRaisesRegex(NotSupported, msg, r.jmespath, "body") + with pytest.raises(AttributeError, match=msg): + r.text + with pytest.raises(NotSupported, match=msg): + r.css("body") + with pytest.raises(NotSupported, match=msg): + r.xpath("//body") + with pytest.raises(NotSupported, match=msg): + r.jmespath("body") else: r.text r.css("body") @@ -216,9 +245,10 @@ class BaseResponseTest(unittest.TestCase): def test_follow_None_url(self): r = self.response_class("http://example.com") - self.assertRaises(ValueError, r.follow, None) + with pytest.raises(ValueError, match="url can't be None"): + r.follow(None) - @mark.xfail( + @pytest.mark.xfail( parse_version(w3lib_version) < parse_version("2.1.1"), reason="https://github.com/scrapy/w3lib/pull/207", strict=True, @@ -226,7 +256,7 @@ class BaseResponseTest(unittest.TestCase): def test_follow_whitespace_url(self): self._assert_followed_url("foo ", "http://example.com/foo") - @mark.xfail( + @pytest.mark.xfail( parse_version(w3lib_version) < parse_version("2.1.1"), reason="https://github.com/scrapy/w3lib/pull/207", strict=True, @@ -239,7 +269,7 @@ class BaseResponseTest(unittest.TestCase): def test_follow_flags(self): res = self.response_class("http://example.com/") fol = res.follow("http://example.com/", flags=["cached", "allowed"]) - self.assertEqual(fol.flags, ["cached", "allowed"]) + assert fol.flags == ["cached", "allowed"] # Response.follow_all @@ -274,23 +304,25 @@ class BaseResponseTest(unittest.TestCase): def test_follow_all_empty(self): r = self.response_class("http://example.com") - self.assertEqual([], list(r.follow_all([]))) + assert not list(r.follow_all([])) def test_follow_all_invalid(self): r = self.response_class("http://example.com") if self.response_class == Response: - with self.assertRaises(TypeError): + with pytest.raises(TypeError): list(r.follow_all(urls=None)) - with self.assertRaises(TypeError): + with pytest.raises(TypeError): list(r.follow_all(urls=12345)) - with self.assertRaises(ValueError): + with pytest.raises(ValueError, match="url can't be None"): list(r.follow_all(urls=[None])) else: - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, match="Please supply exactly one of the following arguments" + ): list(r.follow_all(urls=None)) - with self.assertRaises(TypeError): + with pytest.raises(TypeError): list(r.follow_all(urls=12345)) - with self.assertRaises(ValueError): + with pytest.raises(ValueError, match="url can't be None"): list(r.follow_all(urls=[None])) def test_follow_all_whitespace(self): @@ -323,734 +355,27 @@ class BaseResponseTest(unittest.TestCase): ] fol = re.follow_all(urls, flags=["cached", "allowed"]) for req in fol: - self.assertEqual(req.flags, ["cached", "allowed"]) + assert req.flags == ["cached", "allowed"] def _assert_followed_url(self, follow_obj, target_url, response=None): if response is None: response = self._links_response() req = response.follow(follow_obj) - self.assertEqual(req.url, target_url) + assert req.url == target_url return req def _assert_followed_all_urls(self, follow_obj, target_urls, response=None): if response is None: response = self._links_response() followed = response.follow_all(follow_obj) - for req, target in zip(followed, target_urls): - self.assertEqual(req.url, target) + for req, target in zip(followed, target_urls, strict=False): + assert req.url == target yield req def _links_response(self): body = get_testdata("link_extractor", "linkextractor.html") - resp = self.response_class("http://example.com/index", body=body) - return resp + return self.response_class("http://example.com/index", body=body) def _links_response_no_href(self): body = get_testdata("link_extractor", "linkextractor_no_href.html") - resp = self.response_class("http://example.com/index", body=body) - return resp - - -class TextResponseTest(BaseResponseTest): - response_class = TextResponse - - def test_replace(self): - super().test_replace() - r1 = self.response_class( - "http://www.example.com", body="hello", encoding="cp852" - ) - r2 = r1.replace(url="http://www.example.com/other") - r3 = r1.replace(url="http://www.example.com/other", encoding="latin1") - - assert isinstance(r2, self.response_class) - self.assertEqual(r2.url, "http://www.example.com/other") - self._assert_response_encoding(r2, "cp852") - self.assertEqual(r3.url, "http://www.example.com/other") - self.assertEqual(r3._declared_encoding(), "latin1") - - def test_unicode_url(self): - # instantiate with unicode url without encoding (should set default encoding) - resp = self.response_class("http://www.example.com/") - self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING) - - # make sure urls are converted to str - resp = self.response_class(url="http://www.example.com/", encoding="utf-8") - assert isinstance(resp.url, str) - - resp = self.response_class( - url="http://www.example.com/price/\xa3", encoding="utf-8" - ) - self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3")) - resp = self.response_class( - url="http://www.example.com/price/\xa3", encoding="latin-1" - ) - self.assertEqual(resp.url, "http://www.example.com/price/\xa3") - resp = self.response_class( - "http://www.example.com/price/\xa3", - headers={"Content-type": ["text/html; charset=utf-8"]}, - ) - self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3")) - resp = self.response_class( - "http://www.example.com/price/\xa3", - headers={"Content-type": ["text/html; charset=iso-8859-1"]}, - ) - self.assertEqual(resp.url, "http://www.example.com/price/\xa3") - - def test_unicode_body(self): - unicode_string = ( - "\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 " - "\u0442\u0435\u043a\u0441\u0442" - ) - self.assertRaises( - TypeError, - self.response_class, - "http://www.example.com", - body="unicode body", - ) - - original_string = unicode_string.encode("cp1251") - r1 = self.response_class( - "http://www.example.com", body=original_string, encoding="cp1251" - ) - - # check response.text - self.assertTrue(isinstance(r1.text, str)) - self.assertEqual(r1.text, unicode_string) - - def test_encoding(self): - r1 = self.response_class( - "http://www.example.com", - body=b"\xc2\xa3", - headers={"Content-type": ["text/html; charset=utf-8"]}, - ) - r2 = self.response_class( - "http://www.example.com", encoding="utf-8", body="\xa3" - ) - r3 = self.response_class( - "http://www.example.com", - body=b"\xa3", - headers={"Content-type": ["text/html; charset=iso-8859-1"]}, - ) - r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3") - r5 = self.response_class( - "http://www.example.com", - body=b"\xc2\xa3", - headers={"Content-type": ["text/html; charset=None"]}, - ) - r6 = self.response_class( - "http://www.example.com", - body=b"\xa8D", - headers={"Content-type": ["text/html; charset=gb2312"]}, - ) - r7 = self.response_class( - "http://www.example.com", - body=b"\xa8D", - headers={"Content-type": ["text/html; charset=gbk"]}, - ) - r8 = self.response_class( - "http://www.example.com", - body=codecs.BOM_UTF8 + b"\xc2\xa3", - headers={"Content-type": ["text/html; charset=cp1251"]}, - ) - r9 = self.response_class( - "http://www.example.com", - body=b"\x80", - headers={ - "Content-type": [b"application/x-download; filename=\x80dummy.txt"] - }, - ) - - self.assertEqual(r1._headers_encoding(), "utf-8") - self.assertEqual(r2._headers_encoding(), None) - self.assertEqual(r2._declared_encoding(), "utf-8") - self._assert_response_encoding(r2, "utf-8") - self.assertEqual(r3._headers_encoding(), "cp1252") - self.assertEqual(r3._declared_encoding(), "cp1252") - self.assertEqual(r4._headers_encoding(), None) - self.assertEqual(r5._headers_encoding(), None) - self.assertEqual(r8._headers_encoding(), "cp1251") - self.assertEqual(r9._headers_encoding(), None) - self.assertEqual(r8._declared_encoding(), "utf-8") - self.assertEqual(r9._declared_encoding(), None) - self._assert_response_encoding(r5, "utf-8") - self._assert_response_encoding(r8, "utf-8") - self._assert_response_encoding(r9, "cp1252") - assert ( - r4._body_inferred_encoding() is not None - and r4._body_inferred_encoding() != "ascii" - ) - self._assert_response_values(r1, "utf-8", "\xa3") - self._assert_response_values(r2, "utf-8", "\xa3") - self._assert_response_values(r3, "iso-8859-1", "\xa3") - self._assert_response_values(r6, "gb18030", "\u2015") - self._assert_response_values(r7, "gb18030", "\u2015") - self._assert_response_values(r9, "cp1252", "€") - - # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies - self.assertRaises( - TypeError, - self.response_class, - "http://www.example.com", - body="\xa3", - ) - - def test_declared_encoding_invalid(self): - """Check that unknown declared encodings are ignored""" - r = self.response_class( - "http://www.example.com", - headers={"Content-type": ["text/html; charset=UNKNOWN"]}, - body=b"\xc2\xa3", - ) - self.assertEqual(r._declared_encoding(), None) - self._assert_response_values(r, "utf-8", "\xa3") - - def test_utf16(self): - """Test utf-16 because UnicodeDammit is known to have problems with""" - r = self.response_class( - "http://www.example.com", - body=b"\xff\xfeh\x00i\x00", - encoding="utf-16", - ) - self._assert_response_values(r, "utf-16", "hi") - - def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self): - r6 = self.response_class( - "http://www.example.com", - headers={"Content-type": ["text/html; charset=utf-8"]}, - body=b"\xef\xbb\xbfWORD\xe3\xab", - ) - self.assertEqual(r6.encoding, "utf-8") - self.assertIn( - r6.text, - { - "WORD\ufffd\ufffd", # w3lib < 1.19.0 - "WORD\ufffd", # w3lib >= 1.19.0 - }, - ) - - def test_bom_is_removed_from_body(self): - # Inferring encoding from body also cache decoded body as sideeffect, - # this test tries to ensure that calling response.encoding and - # response.text in indistinct order doesn't affect final - # response.text in indistinct order doesn't affect final - # values for encoding and decoded body. - url = "http://example.com" - body = b"\xef\xbb\xbfWORD" - headers = {"Content-type": ["text/html; charset=utf-8"]} - - # Test response without content-type and BOM encoding - response = self.response_class(url, body=body) - self.assertEqual(response.encoding, "utf-8") - self.assertEqual(response.text, "WORD") - response = self.response_class(url, body=body) - self.assertEqual(response.text, "WORD") - self.assertEqual(response.encoding, "utf-8") - - # Body caching sideeffect isn't triggered when encoding is declared in - # content-type header but BOM still need to be removed from decoded - # body - response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.encoding, "utf-8") - self.assertEqual(response.text, "WORD") - response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.text, "WORD") - self.assertEqual(response.encoding, "utf-8") - - def test_replace_wrong_encoding(self): - """Test invalid chars are replaced properly""" - r = self.response_class( - "http://www.example.com", - encoding="utf-8", - body=b"PREFIX\xe3\xabSUFFIX", - ) - # XXX: Policy for replacing invalid chars may suffer minor variations - # but it should always contain the unicode replacement char ('\ufffd') - assert "\ufffd" in r.text, repr(r.text) - assert "PREFIX" in r.text, repr(r.text) - assert "SUFFIX" in r.text, repr(r.text) - - # Do not destroy html tags due to encoding bugs - r = self.response_class( - "http://example.com", - encoding="utf-8", - body=b"\xf0value", - ) - assert "value" in r.text, repr(r.text) - - # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse - # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX') - # assert '\ufffd' in r.text, repr(r.text) - - def test_selector(self): - body = b"Some page" - response = self.response_class("http://www.example.com", body=body) - - self.assertIsInstance(response.selector, Selector) - self.assertEqual(response.selector.type, "html") - self.assertIs(response.selector, response.selector) # property is cached - self.assertIs(response.selector.response, response) - - self.assertEqual( - response.selector.xpath("//title/text()").getall(), ["Some page"] - ) - self.assertEqual(response.selector.css("title::text").getall(), ["Some page"]) - self.assertEqual(response.selector.re("Some (.*)"), ["page"]) - - def test_selector_shortcuts(self): - body = b"Some page" - response = self.response_class("http://www.example.com", body=body) - - self.assertEqual( - response.xpath("//title/text()").getall(), - response.selector.xpath("//title/text()").getall(), - ) - self.assertEqual( - response.css("title::text").getall(), - response.selector.css("title::text").getall(), - ) - - def test_selector_shortcuts_kwargs(self): - body = b'Some page

A nice paragraph.

' - response = self.response_class("http://www.example.com", body=body) - - self.assertEqual( - response.xpath( - "normalize-space(//p[@class=$pclass])", pclass="content" - ).getall(), - response.xpath('normalize-space(//p[@class="content"])').getall(), - ) - self.assertEqual( - response.xpath( - "//title[count(following::p[@class=$pclass])=$pcount]/text()", - pclass="content", - pcount=1, - ).getall(), - response.xpath( - '//title[count(following::p[@class="content"])=1]/text()' - ).getall(), - ) - - def test_urljoin_with_base_url(self): - """Test urljoin shortcut which also evaluates base-url.""" - body = b'' - joined = self.response_class("http://www.example.com", body=body).urljoin( - "/test" - ) - absolute = "https://example.net/test" - self.assertEqual(joined, absolute) - - body = b'' - joined = self.response_class("http://www.example.com", body=body).urljoin( - "test" - ) - absolute = "http://www.example.com/test" - self.assertEqual(joined, absolute) - - body = b'' - joined = self.response_class("http://www.example.com", body=body).urljoin( - "test" - ) - absolute = "http://www.example.com/elsewhere/test" - self.assertEqual(joined, absolute) - - def test_follow_selector(self): - resp = self._links_response() - urls = [ - "http://example.com/sample2.html", - "http://example.com/sample3.html", - "http://example.com/sample3.html", - "http://example.com/sample3.html", - "http://example.com/sample3.html#foo", - "http://www.google.com/something", - "http://example.com/innertag.html", - ] - - # select elements - for sellist in [resp.css("a"), resp.xpath("//a")]: - for sel, url in zip(sellist, urls): - self._assert_followed_url(sel, url, response=resp) - - # select elements - self._assert_followed_url( - Selector(text='').css("link")[0], - "http://example.com/foo", - response=resp, - ) - - # href attributes should work - for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]: - for sel, url in zip(sellist, urls): - self._assert_followed_url(sel, url, response=resp) - - # non-a elements are not supported - self.assertRaises(ValueError, resp.follow, resp.css("div")[0]) - - def test_follow_selector_list(self): - resp = self._links_response() - self.assertRaisesRegex(ValueError, "SelectorList", resp.follow, resp.css("a")) - - def test_follow_selector_invalid(self): - resp = self._links_response() - self.assertRaisesRegex( - ValueError, - "Unsupported", - resp.follow, - resp.xpath("count(//div)")[0], - ) - - def test_follow_selector_attribute(self): - resp = self._links_response() - for src in resp.css("img::attr(src)"): - self._assert_followed_url(src, "http://example.com/sample2.jpg") - - def test_follow_selector_no_href(self): - resp = self.response_class( - url="http://example.com", - body=b"click me", - ) - self.assertRaisesRegex(ValueError, "no href", resp.follow, resp.css("a")[0]) - - def test_follow_whitespace_selector(self): - resp = self.response_class( - "http://example.com", - body=b"""click me""", - ) - self._assert_followed_url( - resp.css("a")[0], "http://example.com/foo", response=resp - ) - self._assert_followed_url( - resp.css("a::attr(href)")[0], - "http://example.com/foo", - response=resp, - ) - - def test_follow_encoding(self): - resp1 = self.response_class( - "http://example.com", - encoding="utf8", - body='click me'.encode( - "utf8" - ), - ) - req = self._assert_followed_url( - resp1.css("a")[0], - "http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82", - response=resp1, - ) - self.assertEqual(req.encoding, "utf8") - - resp2 = self.response_class( - "http://example.com", - encoding="cp1251", - body='click me'.encode( - "cp1251" - ), - ) - req = self._assert_followed_url( - resp2.css("a")[0], - "http://example.com/foo?%EF%F0%E8%E2%E5%F2", - response=resp2, - ) - self.assertEqual(req.encoding, "cp1251") - - def test_follow_flags(self): - res = self.response_class("http://example.com/") - fol = res.follow("http://example.com/", flags=["cached", "allowed"]) - self.assertEqual(fol.flags, ["cached", "allowed"]) - - def test_follow_all_flags(self): - re = self.response_class("http://www.example.com/") - urls = [ - "http://www.example.com/", - "http://www.example.com/2", - "http://www.example.com/foo", - ] - fol = re.follow_all(urls, flags=["cached", "allowed"]) - for req in fol: - self.assertEqual(req.flags, ["cached", "allowed"]) - - def test_follow_all_css(self): - expected = [ - "http://example.com/sample3.html", - "http://example.com/innertag.html", - ] - response = self._links_response() - extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')] - self.assertEqual(expected, extracted) - - def test_follow_all_css_skip_invalid(self): - expected = [ - "http://example.com/page/1/", - "http://example.com/page/3/", - "http://example.com/page/4/", - ] - response = self._links_response_no_href() - extracted1 = [r.url for r in response.follow_all(css=".pagination a")] - self.assertEqual(expected, extracted1) - extracted2 = [r.url for r in response.follow_all(response.css(".pagination a"))] - self.assertEqual(expected, extracted2) - - def test_follow_all_xpath(self): - expected = [ - "http://example.com/sample3.html", - "http://example.com/innertag.html", - ] - response = self._links_response() - extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]') - self.assertEqual(expected, [r.url for r in extracted]) - - def test_follow_all_xpath_skip_invalid(self): - expected = [ - "http://example.com/page/1/", - "http://example.com/page/3/", - "http://example.com/page/4/", - ] - response = self._links_response_no_href() - extracted1 = [ - r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a') - ] - self.assertEqual(expected, extracted1) - extracted2 = [ - r.url - for r in response.follow_all(response.xpath('//div[@id="pagination"]/a')) - ] - self.assertEqual(expected, extracted2) - - def test_follow_all_too_many_arguments(self): - response = self._links_response() - with self.assertRaises(ValueError): - response.follow_all( - css='a[href*="example.com"]', - xpath='//a[contains(@href, "example.com")]', - ) - - def test_json_response(self): - json_body = b"""{"ip": "109.187.217.200"}""" - json_response = self.response_class("http://www.example.com", body=json_body) - self.assertEqual(json_response.json(), {"ip": "109.187.217.200"}) - - text_body = b"""text""" - text_response = self.response_class("http://www.example.com", body=text_body) - with self.assertRaises(ValueError): - text_response.json() - - def test_cache_json_response(self): - json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""] - for json_body in json_valid_bodies: - json_response = self.response_class( - "http://www.example.com", body=json_body - ) - - with mock.patch("json.loads") as mock_json: - for _ in range(2): - json_response.json() - mock_json.assert_called_once_with(json_body) - - -class HtmlResponseTest(TextResponseTest): - response_class = HtmlResponse - - def test_base_url(self): - resp = HtmlResponse( - "http://www.example.com", - body=b""" - - - blahablsdfsal& - """, - ) - self.assertEqual(resp.base_url, "http://www.example.com/img/") - - resp2 = HtmlResponse( - "http://www.example.com", - body=b""" - blahablsdfsal&""", - ) - self.assertEqual(resp2.base_url, "http://www.example.com") - - def test_html_encoding(self): - body = b"""Some page - - Price: \xa3100' - """ - r1 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r1, "iso-8859-1", body) - - body = b""" - - Price: \xa3100 - """ - r2 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r2, "iso-8859-1", body) - - # for conflicting declarations headers must take precedence - body = b"""Some page - - Price: \xa3100' - """ - r3 = self.response_class( - "http://www.example.com", - body=body, - headers={"Content-type": ["text/html; charset=iso-8859-1"]}, - ) - self._assert_response_values(r3, "iso-8859-1", body) - - # make sure replace() preserves the encoding of the original response - body = b"New body \xa3" - r4 = r3.replace(body=body) - self._assert_response_values(r4, "iso-8859-1", body) - - def test_html5_meta_charset(self): - body = b"""Some pagebla bla""" - r1 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r1, "gb2312", body) - - -class XmlResponseTest(TextResponseTest): - response_class = XmlResponse - - def test_xml_encoding(self): - body = b"" - r1 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r1, self.response_class._DEFAULT_ENCODING, body) - - body = b"""""" - r2 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r2, "iso-8859-1", body) - - # make sure replace() preserves the explicit encoding passed in the __init__ method - body = b"""""" - r3 = self.response_class("http://www.example.com", body=body, encoding="utf-8") - body2 = b"New body" - r4 = r3.replace(body=body2) - self._assert_response_values(r4, "utf-8", body2) - - def test_replace_encoding(self): - # make sure replace() keeps the previous encoding unless overridden explicitly - body = b"""""" - body2 = b"""""" - r5 = self.response_class("http://www.example.com", body=body) - r6 = r5.replace(body=body2) - r7 = r5.replace(body=body2, encoding="utf-8") - self._assert_response_values(r5, "iso-8859-1", body) - self._assert_response_values(r6, "iso-8859-1", body2) - self._assert_response_values(r7, "utf-8", body2) - - def test_selector(self): - body = b'value' - response = self.response_class("http://www.example.com", body=body) - - self.assertIsInstance(response.selector, Selector) - self.assertEqual(response.selector.type, "xml") - self.assertIs(response.selector, response.selector) # property is cached - self.assertIs(response.selector.response, response) - - self.assertEqual(response.selector.xpath("//elem/text()").getall(), ["value"]) - - def test_selector_shortcuts(self): - body = b'value' - response = self.response_class("http://www.example.com", body=body) - - self.assertEqual( - response.xpath("//elem/text()").getall(), - response.selector.xpath("//elem/text()").getall(), - ) - - def test_selector_shortcuts_kwargs(self): - body = b""" - - value - """ - response = self.response_class("http://www.example.com", body=body) - - self.assertEqual( - response.xpath( - "//s:elem/text()", namespaces={"s": "http://scrapy.org"} - ).getall(), - response.selector.xpath( - "//s:elem/text()", namespaces={"s": "http://scrapy.org"} - ).getall(), - ) - - response.selector.register_namespace("s2", "http://scrapy.org") - self.assertEqual( - response.xpath( - "//s1:elem/text()", namespaces={"s1": "http://scrapy.org"} - ).getall(), - response.selector.xpath("//s2:elem/text()").getall(), - ) - - -class CustomResponse(TextResponse): - attributes = TextResponse.attributes + ("foo", "bar") - - def __init__(self, *args, **kwargs) -> None: - self.foo = kwargs.pop("foo", None) - self.bar = kwargs.pop("bar", None) - self.lost = kwargs.pop("lost", None) - super().__init__(*args, **kwargs) - - -class CustomResponseTest(TextResponseTest): - response_class = CustomResponse - - def test_copy(self): - super().test_copy() - r1 = self.response_class( - url="https://example.org", - status=200, - foo="foo", - bar="bar", - lost="lost", - ) - r2 = r1.copy() - self.assertIsInstance(r2, self.response_class) - self.assertEqual(r1.foo, r2.foo) - self.assertEqual(r1.bar, r2.bar) - self.assertEqual(r1.lost, "lost") - self.assertIsNone(r2.lost) - - def test_replace(self): - super().test_replace() - r1 = self.response_class( - url="https://example.org", - status=200, - foo="foo", - bar="bar", - lost="lost", - ) - - r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost") - self.assertIsInstance(r2, self.response_class) - self.assertEqual(r1.foo, "foo") - self.assertEqual(r1.bar, "bar") - self.assertEqual(r1.lost, "lost") - self.assertEqual(r2.foo, "new-foo") - self.assertEqual(r2.bar, "new-bar") - self.assertEqual(r2.lost, "new-lost") - - r3 = r1.replace(foo="new-foo", bar="new-bar") - self.assertIsInstance(r3, self.response_class) - self.assertEqual(r1.foo, "foo") - self.assertEqual(r1.bar, "bar") - self.assertEqual(r1.lost, "lost") - self.assertEqual(r3.foo, "new-foo") - self.assertEqual(r3.bar, "new-bar") - self.assertIsNone(r3.lost) - - r4 = r1.replace(foo="new-foo") - self.assertIsInstance(r4, self.response_class) - self.assertEqual(r1.foo, "foo") - self.assertEqual(r1.bar, "bar") - self.assertEqual(r1.lost, "lost") - self.assertEqual(r4.foo, "new-foo") - self.assertEqual(r4.bar, "bar") - self.assertIsNone(r4.lost) - - with self.assertRaises(TypeError) as ctx: - r1.replace(unknown="unknown") - self.assertTrue( - str(ctx.exception).endswith( - "__init__() got an unexpected keyword argument 'unknown'" - ) - ) + return self.response_class("http://example.com/index", body=body) diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py new file mode 100644 index 000000000..c16af52b9 --- /dev/null +++ b/tests/test_http_response_text.py @@ -0,0 +1,684 @@ +from __future__ import annotations + +import codecs +from unittest import mock + +import pytest + +from scrapy.http import HtmlResponse, TextResponse, XmlResponse +from scrapy.selector import Selector +from scrapy.utils.python import to_unicode +from tests.test_http_response import TestResponse + + +class TestTextResponse(TestResponse): + response_class = TextResponse + + def test_replace(self): + super().test_replace() + r1 = self.response_class( + "http://www.example.com", body="hello", encoding="cp852" + ) + r2 = r1.replace(url="http://www.example.com/other") + r3 = r1.replace(url="http://www.example.com/other", encoding="latin1") + + assert isinstance(r2, self.response_class) + assert r2.url == "http://www.example.com/other" + self._assert_response_encoding(r2, "cp852") + assert r3.url == "http://www.example.com/other" + assert r3._declared_encoding() == "latin1" + + def test_unicode_url(self): + # instantiate with unicode url without encoding (should set default encoding) + resp = self.response_class("http://www.example.com/") + self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING) + + # make sure urls are converted to str + resp = self.response_class(url="http://www.example.com/", encoding="utf-8") + assert isinstance(resp.url, str) + + resp = self.response_class( + url="http://www.example.com/price/\xa3", encoding="utf-8" + ) + assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3") + resp = self.response_class( + url="http://www.example.com/price/\xa3", encoding="latin-1" + ) + assert resp.url == "http://www.example.com/price/\xa3" + resp = self.response_class( + "http://www.example.com/price/\xa3", + headers={"Content-type": ["text/html; charset=utf-8"]}, + ) + assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3") + resp = self.response_class( + "http://www.example.com/price/\xa3", + headers={"Content-type": ["text/html; charset=iso-8859-1"]}, + ) + assert resp.url == "http://www.example.com/price/\xa3" + + def test_unicode_body(self): + unicode_string = ( + "\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 " + "\u0442\u0435\u043a\u0441\u0442" + ) + with pytest.raises(TypeError): + self.response_class("http://www.example.com", body="unicode body") + + original_string = unicode_string.encode("cp1251") + r1 = self.response_class( + "http://www.example.com", body=original_string, encoding="cp1251" + ) + + # check response.text + assert isinstance(r1.text, str) + assert r1.text == unicode_string + + def test_encoding(self): + r1 = self.response_class( + "http://www.example.com", + body=b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=utf-8"]}, + ) + r2 = self.response_class( + "http://www.example.com", encoding="utf-8", body="\xa3" + ) + r3 = self.response_class( + "http://www.example.com", + body=b"\xa3", + headers={"Content-type": ["text/html; charset=iso-8859-1"]}, + ) + r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3") + r5 = self.response_class( + "http://www.example.com", + body=b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=None"]}, + ) + r6 = self.response_class( + "http://www.example.com", + body=b"\xa8D", + headers={"Content-type": ["text/html; charset=gb2312"]}, + ) + r7 = self.response_class( + "http://www.example.com", + body=b"\xa8D", + headers={"Content-type": ["text/html; charset=gbk"]}, + ) + r8 = self.response_class( + "http://www.example.com", + body=codecs.BOM_UTF8 + b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=cp1251"]}, + ) + r9 = self.response_class( + "http://www.example.com", + body=b"\x80", + headers={ + "Content-type": [b"application/x-download; filename=\x80dummy.txt"] + }, + ) + + assert r1._headers_encoding() == "utf-8" + assert r2._headers_encoding() is None + assert r2._declared_encoding() == "utf-8" + self._assert_response_encoding(r2, "utf-8") + assert r3._headers_encoding() == "cp1252" + assert r3._declared_encoding() == "cp1252" + assert r4._headers_encoding() is None + assert r5._headers_encoding() is None + assert r8._headers_encoding() == "cp1251" + assert r9._headers_encoding() is None + assert r8._declared_encoding() == "utf-8" + assert r9._declared_encoding() is None + self._assert_response_encoding(r5, "utf-8") + self._assert_response_encoding(r8, "utf-8") + self._assert_response_encoding(r9, "cp1252") + assert r4._body_inferred_encoding() is not None + assert r4._body_inferred_encoding() != "ascii" + self._assert_response_values(r1, "utf-8", "\xa3") + self._assert_response_values(r2, "utf-8", "\xa3") + self._assert_response_values(r3, "iso-8859-1", "\xa3") + self._assert_response_values(r6, "gb18030", "\u2015") + self._assert_response_values(r7, "gb18030", "\u2015") + self._assert_response_values(r9, "cp1252", "€") + + # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies + with pytest.raises(TypeError): + self.response_class("http://www.example.com", body="\xa3") + + def test_declared_encoding_invalid(self): + """Check that unknown declared encodings are ignored""" + r = self.response_class( + "http://www.example.com", + headers={"Content-type": ["text/html; charset=UNKNOWN"]}, + body=b"\xc2\xa3", + ) + assert r._declared_encoding() is None + self._assert_response_values(r, "utf-8", "\xa3") + + def test_utf16(self): + """Test utf-16 because UnicodeDammit is known to have problems with""" + r = self.response_class( + "http://www.example.com", + body=b"\xff\xfeh\x00i\x00", + encoding="utf-16", + ) + self._assert_response_values(r, "utf-16", "hi") + + def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self): + r6 = self.response_class( + "http://www.example.com", + headers={"Content-type": ["text/html; charset=utf-8"]}, + body=b"\xef\xbb\xbfWORD\xe3\xab", + ) + assert r6.encoding == "utf-8" + assert r6.text in { + "WORD\ufffd\ufffd", # w3lib < 1.19.0 + "WORD\ufffd", # w3lib >= 1.19.0 + } + + def test_bom_is_removed_from_body(self): + # Inferring encoding from body also cache decoded body as sideeffect, + # this test tries to ensure that calling response.encoding and + # response.text in indistinct order doesn't affect final + # response.text in indistinct order doesn't affect final + # values for encoding and decoded body. + url = "http://example.com" + body = b"\xef\xbb\xbfWORD" + headers = {"Content-type": ["text/html; charset=utf-8"]} + + # Test response without content-type and BOM encoding + response = self.response_class(url, body=body) + assert response.encoding == "utf-8" + assert response.text == "WORD" + response = self.response_class(url, body=body) + assert response.text == "WORD" + assert response.encoding == "utf-8" + + # Body caching sideeffect isn't triggered when encoding is declared in + # content-type header but BOM still need to be removed from decoded + # body + response = self.response_class(url, headers=headers, body=body) + assert response.encoding == "utf-8" + assert response.text == "WORD" + response = self.response_class(url, headers=headers, body=body) + assert response.text == "WORD" + assert response.encoding == "utf-8" + + def test_replace_wrong_encoding(self): + """Test invalid chars are replaced properly""" + r = self.response_class( + "http://www.example.com", + encoding="utf-8", + body=b"PREFIX\xe3\xabSUFFIX", + ) + # XXX: Policy for replacing invalid chars may suffer minor variations + # but it should always contain the unicode replacement char ('\ufffd') + assert "\ufffd" in r.text, repr(r.text) + assert "PREFIX" in r.text, repr(r.text) + assert "SUFFIX" in r.text, repr(r.text) + + # Do not destroy html tags due to encoding bugs + r = self.response_class( + "http://example.com", + encoding="utf-8", + body=b"\xf0value", + ) + assert "value" in r.text, repr(r.text) + + # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse + # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX') + # assert '\ufffd' in r.text, repr(r.text) + + def test_selector(self): + body = b"Some page" + response = self.response_class("http://www.example.com", body=body) + + assert isinstance(response.selector, Selector) + assert response.selector.type == "html" + assert response.selector is response.selector # property is cached + assert response.selector.response is response + + assert response.selector.xpath("//title/text()").getall() == ["Some page"] + assert response.selector.css("title::text").getall() == ["Some page"] + assert response.selector.re("Some (.*)") == ["page"] + + def test_selector_shortcuts(self): + body = b"Some page" + response = self.response_class("http://www.example.com", body=body) + + assert ( + response.xpath("//title/text()").getall() + == response.selector.xpath("//title/text()").getall() + ) + assert ( + response.css("title::text").getall() + == response.selector.css("title::text").getall() + ) + + def test_selector_shortcuts_kwargs(self): + body = b'Some page

A nice paragraph.

' + response = self.response_class("http://www.example.com", body=body) + + assert ( + response.xpath( + "normalize-space(//p[@class=$pclass])", pclass="content" + ).getall() + == response.xpath('normalize-space(//p[@class="content"])').getall() + ) + assert ( + response.xpath( + "//title[count(following::p[@class=$pclass])=$pcount]/text()", + pclass="content", + pcount=1, + ).getall() + == response.xpath( + '//title[count(following::p[@class="content"])=1]/text()' + ).getall() + ) + + def test_urljoin_with_base_url(self): + """Test urljoin shortcut which also evaluates base-url through get_base_url().""" + body = b'' + joined = self.response_class("http://www.example.com", body=body).urljoin( + "/test" + ) + absolute = "https://example.net/test" + assert joined == absolute + + body = b'' + joined = self.response_class("http://www.example.com", body=body).urljoin( + "test" + ) + absolute = "http://www.example.com/test" + assert joined == absolute + + body = b'' + joined = self.response_class("http://www.example.com", body=body).urljoin( + "test" + ) + absolute = "http://www.example.com/elsewhere/test" + assert joined == absolute + + def test_follow_selector(self): + resp = self._links_response() + urls = [ + "http://example.com/sample2.html", + "http://example.com/sample3.html", + "http://example.com/sample3.html", + "http://example.com/sample3.html", + "http://example.com/sample3.html#foo", + "http://www.google.com/something", + "http://example.com/innertag.html", + ] + + # select elements + for sellist in [resp.css("a"), resp.xpath("//a")]: + for sel, url in zip(sellist, urls, strict=False): + self._assert_followed_url(sel, url, response=resp) + + # select elements + self._assert_followed_url( + Selector(text='').css("link")[0], + "http://example.com/foo", + response=resp, + ) + + # href attributes should work + for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]: + for sel, url in zip(sellist, urls, strict=False): + self._assert_followed_url(sel, url, response=resp) + + # non-a elements are not supported + with pytest.raises( + ValueError, match="Only and elements are supported" + ): + resp.follow(resp.css("div")[0]) + + def test_follow_selector_list(self): + resp = self._links_response() + with pytest.raises(ValueError, match="SelectorList"): + resp.follow(resp.css("a")) + + def test_follow_selector_invalid(self): + resp = self._links_response() + with pytest.raises(ValueError, match="Unsupported"): + resp.follow(resp.xpath("count(//div)")[0]) + + def test_follow_selector_attribute(self): + resp = self._links_response() + for src in resp.css("img::attr(src)"): + self._assert_followed_url(src, "http://example.com/sample2.jpg") + + def test_follow_selector_no_href(self): + resp = self.response_class( + url="http://example.com", + body=b"click me", + ) + with pytest.raises(ValueError, match="no href"): + resp.follow(resp.css("a")[0]) + + def test_follow_whitespace_selector(self): + resp = self.response_class( + "http://example.com", + body=b"""click me""", + ) + self._assert_followed_url( + resp.css("a")[0], "http://example.com/foo", response=resp + ) + self._assert_followed_url( + resp.css("a::attr(href)")[0], + "http://example.com/foo", + response=resp, + ) + + def test_follow_encoding(self): + resp1 = self.response_class( + "http://example.com", + encoding="utf8", + body='click me'.encode(), + ) + req = self._assert_followed_url( + resp1.css("a")[0], + "http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82", + response=resp1, + ) + assert req.encoding == "utf8" + + resp2 = self.response_class( + "http://example.com", + encoding="cp1251", + body='click me'.encode( + "cp1251" + ), + ) + req = self._assert_followed_url( + resp2.css("a")[0], + "http://example.com/foo?%EF%F0%E8%E2%E5%F2", + response=resp2, + ) + assert req.encoding == "cp1251" + + def test_follow_flags(self): + res = self.response_class("http://example.com/") + fol = res.follow("http://example.com/", flags=["cached", "allowed"]) + assert fol.flags == ["cached", "allowed"] + + def test_follow_all_flags(self): + re = self.response_class("http://www.example.com/") + urls = [ + "http://www.example.com/", + "http://www.example.com/2", + "http://www.example.com/foo", + ] + fol = re.follow_all(urls, flags=["cached", "allowed"]) + for req in fol: + assert req.flags == ["cached", "allowed"] + + def test_follow_all_css(self): + expected = [ + "http://example.com/sample3.html", + "http://example.com/innertag.html", + ] + response = self._links_response() + extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')] + assert expected == extracted + + def test_follow_all_css_skip_invalid(self): + expected = [ + "http://example.com/page/1/", + "http://example.com/page/3/", + "http://example.com/page/4/", + ] + response = self._links_response_no_href() + extracted1 = [r.url for r in response.follow_all(css=".pagination a")] + assert expected == extracted1 + extracted2 = [r.url for r in response.follow_all(response.css(".pagination a"))] + assert expected == extracted2 + + def test_follow_all_xpath(self): + expected = [ + "http://example.com/sample3.html", + "http://example.com/innertag.html", + ] + response = self._links_response() + extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]') + assert expected == [r.url for r in extracted] + + def test_follow_all_xpath_skip_invalid(self): + expected = [ + "http://example.com/page/1/", + "http://example.com/page/3/", + "http://example.com/page/4/", + ] + response = self._links_response_no_href() + extracted1 = [ + r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a') + ] + assert expected == extracted1 + extracted2 = [ + r.url + for r in response.follow_all(response.xpath('//div[@id="pagination"]/a')) + ] + assert expected == extracted2 + + def test_follow_all_too_many_arguments(self): + response = self._links_response() + with pytest.raises( + ValueError, match="Please supply exactly one of the following arguments" + ): + response.follow_all( + css='a[href*="example.com"]', + xpath='//a[contains(@href, "example.com")]', + ) + + def test_json_response(self): + json_body = b"""{"ip": "109.187.217.200"}""" + json_response = self.response_class("http://www.example.com", body=json_body) + assert json_response.json() == {"ip": "109.187.217.200"} + + text_body = b"""text""" + text_response = self.response_class("http://www.example.com", body=text_body) + with pytest.raises( + ValueError, match=r"(Expecting value|Unexpected '<'): line 1" + ): + text_response.json() + + def test_cache_json_response(self): + json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""] + for json_body in json_valid_bodies: + json_response = self.response_class( + "http://www.example.com", body=json_body + ) + + with mock.patch("json.loads") as mock_json: + for _ in range(2): + json_response.json() + mock_json.assert_called_once_with(json_body) + + +class TestHtmlResponse(TestTextResponse): + response_class = HtmlResponse + + def test_html_encoding(self): + body = b"""Some page + + Price: \xa3100' + """ + r1 = self.response_class("http://www.example.com", body=body) + self._assert_response_values(r1, "iso-8859-1", body) + + body = b""" + + Price: \xa3100 + """ + r2 = self.response_class("http://www.example.com", body=body) + self._assert_response_values(r2, "iso-8859-1", body) + + # for conflicting declarations headers must take precedence + body = b"""Some page + + Price: \xa3100' + """ + r3 = self.response_class( + "http://www.example.com", + body=body, + headers={"Content-type": ["text/html; charset=iso-8859-1"]}, + ) + self._assert_response_values(r3, "iso-8859-1", body) + + # make sure replace() preserves the encoding of the original response + body = b"New body \xa3" + r4 = r3.replace(body=body) + self._assert_response_values(r4, "iso-8859-1", body) + + def test_html5_meta_charset(self): + body = b"""Some pagebla bla""" + r1 = self.response_class("http://www.example.com", body=body) + self._assert_response_values(r1, "gb2312", body) + + +class TestXmlResponse(TestTextResponse): + response_class = XmlResponse + + def test_xml_encoding(self): + body = b"" + r1 = self.response_class("http://www.example.com", body=body) + self._assert_response_values(r1, self.response_class._DEFAULT_ENCODING, body) + + body = b"""""" + r2 = self.response_class("http://www.example.com", body=body) + self._assert_response_values(r2, "iso-8859-1", body) + + # make sure replace() preserves the explicit encoding passed in the __init__ method + body = b"""""" + r3 = self.response_class("http://www.example.com", body=body, encoding="utf-8") + body2 = b"New body" + r4 = r3.replace(body=body2) + self._assert_response_values(r4, "utf-8", body2) + + def test_replace_encoding(self): + # make sure replace() keeps the previous encoding unless overridden explicitly + body = b"""""" + body2 = b"""""" + r5 = self.response_class("http://www.example.com", body=body) + r6 = r5.replace(body=body2) + r7 = r5.replace(body=body2, encoding="utf-8") + self._assert_response_values(r5, "iso-8859-1", body) + self._assert_response_values(r6, "iso-8859-1", body2) + self._assert_response_values(r7, "utf-8", body2) + + def test_selector(self): + body = b'value' + response = self.response_class("http://www.example.com", body=body) + + assert isinstance(response.selector, Selector) + assert response.selector.type == "xml" + assert response.selector is response.selector # property is cached + assert response.selector.response is response + + assert response.selector.xpath("//elem/text()").getall() == ["value"] + + def test_selector_shortcuts(self): + body = b'value' + response = self.response_class("http://www.example.com", body=body) + + assert ( + response.xpath("//elem/text()").getall() + == response.selector.xpath("//elem/text()").getall() + ) + + def test_selector_shortcuts_kwargs(self): + body = b""" + + value + """ + response = self.response_class("http://www.example.com", body=body) + + assert ( + response.xpath( + "//s:elem/text()", namespaces={"s": "http://scrapy.org"} + ).getall() + == response.selector.xpath( + "//s:elem/text()", namespaces={"s": "http://scrapy.org"} + ).getall() + ) + + response.selector.register_namespace("s2", "http://scrapy.org") + assert ( + response.xpath( + "//s1:elem/text()", namespaces={"s1": "http://scrapy.org"} + ).getall() + == response.selector.xpath("//s2:elem/text()").getall() + ) + + +class CustomResponse(TextResponse): + attributes = (*TextResponse.attributes, "foo", "bar") + + def __init__(self, *args, **kwargs) -> None: + self.foo = kwargs.pop("foo", None) + self.bar = kwargs.pop("bar", None) + self.lost = kwargs.pop("lost", None) + super().__init__(*args, **kwargs) + + +class TestCustomResponse(TestTextResponse): + response_class = CustomResponse + + def test_copy(self): + super().test_copy() + r1 = self.response_class( + url="https://example.org", + status=200, + foo="foo", + bar="bar", + lost="lost", + ) + r2 = r1.copy() + assert isinstance(r2, self.response_class) + assert r1.foo == r2.foo + assert r1.bar == r2.bar + assert r1.lost == "lost" + assert r2.lost is None + + def test_replace(self): + super().test_replace() + r1 = self.response_class( + url="https://example.org", + status=200, + foo="foo", + bar="bar", + lost="lost", + ) + + r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost") + assert isinstance(r2, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r2.foo == "new-foo" + assert r2.bar == "new-bar" + assert r2.lost == "new-lost" + + r3 = r1.replace(foo="new-foo", bar="new-bar") + assert isinstance(r3, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r3.foo == "new-foo" + assert r3.bar == "new-bar" + assert r3.lost is None + + r4 = r1.replace(foo="new-foo") + assert isinstance(r4, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r4.foo == "new-foo" + assert r4.bar == "bar" + assert r4.lost is None + + with pytest.raises( + TypeError, + match=r"__init__\(\) got an unexpected keyword argument 'unknown'", + ): + r1.replace(unknown="unknown") diff --git a/tests/test_item.py b/tests/test_item.py index ce2b4fd15..4eb37a344 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,12 +1,14 @@ -import unittest +from abc import ABCMeta from unittest import mock -from scrapy.item import ABCMeta, Field, Item, ItemMeta +import pytest + +from scrapy.item import Field, Item, ItemMeta -class ItemTest(unittest.TestCase): +class TestItem: def assertSortedEqual(self, first, second, msg=None): - return self.assertEqual(sorted(first), sorted(second), msg) + assert sorted(first) == sorted(second), msg def test_simple(self): class TestItem(Item): @@ -14,33 +16,37 @@ class ItemTest(unittest.TestCase): i = TestItem() i["name"] = "name" - self.assertEqual(i["name"], "name") + assert i["name"] == "name" def test_init(self): class TestItem(Item): name = Field() i = TestItem() - self.assertRaises(KeyError, i.__getitem__, "name") + with pytest.raises(KeyError): + i["name"] i2 = TestItem(name="john doe") - self.assertEqual(i2["name"], "john doe") + assert i2["name"] == "john doe" i3 = TestItem({"name": "john doe"}) - self.assertEqual(i3["name"], "john doe") + assert i3["name"] == "john doe" i4 = TestItem(i3) - self.assertEqual(i4["name"], "john doe") + assert i4["name"] == "john doe" - self.assertRaises(KeyError, TestItem, {"name": "john doe", "other": "foo"}) + with pytest.raises(KeyError): + TestItem({"name": "john doe", "other": "foo"}) def test_invalid_field(self): class TestItem(Item): pass i = TestItem() - self.assertRaises(KeyError, i.__setitem__, "field", "text") - self.assertRaises(KeyError, i.__getitem__, "field") + with pytest.raises(KeyError): + i["field"] = "text" + with pytest.raises(KeyError): + i["field"] def test_repr(self): class TestItem(Item): @@ -52,11 +58,11 @@ class ItemTest(unittest.TestCase): i["number"] = 123 itemrepr = repr(i) - self.assertEqual(itemrepr, "{'name': 'John Doe', 'number': 123}") + assert itemrepr == "{'name': 'John Doe', 'number': 123}" i2 = eval(itemrepr) - self.assertEqual(i2["name"], "John Doe") - self.assertEqual(i2["number"], 123) + assert i2["name"] == "John Doe" + assert i2["number"] == 123 def test_private_attr(self): class TestItem(Item): @@ -64,21 +70,23 @@ class ItemTest(unittest.TestCase): i = TestItem() i._private = "test" - self.assertEqual(i._private, "test") + assert i._private == "test" def test_raise_getattr(self): class TestItem(Item): name = Field() i = TestItem() - self.assertRaises(AttributeError, getattr, i, "name") + with pytest.raises(AttributeError): + i.name def test_raise_setattr(self): class TestItem(Item): name = Field() i = TestItem() - self.assertRaises(AttributeError, setattr, i, "name", "john") + with pytest.raises(AttributeError): + i.name = "john" def test_custom_methods(self): class TestItem(Item): @@ -91,11 +99,12 @@ class ItemTest(unittest.TestCase): self["name"] = name i = TestItem() - self.assertRaises(KeyError, i.get_name) + with pytest.raises(KeyError): + i.get_name() i["name"] = "lala" - self.assertEqual(i.get_name(), "lala") + assert i.get_name() == "lala" i.change_name("other") - self.assertEqual(i.get_name(), "other") + assert i.get_name() == "other" def test_metaclass(self): class TestItem(Item): @@ -105,8 +114,8 @@ class ItemTest(unittest.TestCase): i = TestItem() i["name"] = "John" - self.assertEqual(list(i.keys()), ["name"]) - self.assertEqual(list(i.values()), ["John"]) + assert list(i.keys()) == ["name"] + assert list(i.values()) == ["John"] i["keys"] = "Keys" i["values"] = "Values" @@ -132,8 +141,8 @@ class ItemTest(unittest.TestCase): i = TestItem() i["keys"] = 3 - self.assertEqual(list(i.keys()), ["keys"]) - self.assertEqual(list(i.values()), [3]) + assert list(i.keys()) == ["keys"] + assert list(i.values()) == [3] def test_metaclass_multiple_inheritance_simple(self): class A(Item): @@ -151,17 +160,17 @@ class ItemTest(unittest.TestCase): pass item = D(save="X", load="Y") - self.assertEqual(item["save"], "X") - self.assertEqual(item["load"], "Y") - self.assertEqual(D.fields, {"load": {"default": "A"}, "save": {"default": "A"}}) + assert item["save"] == "X" + assert item["load"] == "Y" + assert D.fields == {"load": {"default": "A"}, "save": {"default": "A"}} # D class inverted class E(C, B): pass - self.assertEqual(E(save="X")["save"], "X") - self.assertEqual(E(load="X")["load"], "X") - self.assertEqual(E.fields, {"load": {"default": "C"}, "save": {"default": "C"}}) + assert E(save="X")["save"] == "X" + assert E(load="X")["load"] == "X" + assert E.fields == {"load": {"default": "C"}, "save": {"default": "C"}} def test_metaclass_multiple_inheritance_diamond(self): class A(Item): @@ -180,31 +189,25 @@ class ItemTest(unittest.TestCase): fields = {"update": Field(default="D")} load = Field(default="D") - self.assertEqual(D(save="X")["save"], "X") - self.assertEqual(D(load="X")["load"], "X") - self.assertEqual( - D.fields, - { - "save": {"default": "C"}, - "load": {"default": "D"}, - "update": {"default": "D"}, - }, - ) + assert D(save="X")["save"] == "X" + assert D(load="X")["load"] == "X" + assert D.fields == { + "save": {"default": "C"}, + "load": {"default": "D"}, + "update": {"default": "D"}, + } # D class inverted class E(C, B): load = Field(default="E") - self.assertEqual(E(save="X")["save"], "X") - self.assertEqual(E(load="X")["load"], "X") - self.assertEqual( - E.fields, - { - "save": {"default": "C"}, - "load": {"default": "E"}, - "update": {"default": "C"}, - }, - ) + assert E(save="X")["save"] == "X" + assert E(load="X")["load"] == "X" + assert E.fields == { + "save": {"default": "C"}, + "load": {"default": "E"}, + "update": {"default": "C"}, + } def test_metaclass_multiple_inheritance_without_metaclass(self): class A(Item): @@ -222,17 +225,19 @@ class ItemTest(unittest.TestCase): class D(B, C): pass - self.assertRaises(KeyError, D, not_allowed="value") - self.assertEqual(D(save="X")["save"], "X") - self.assertEqual(D.fields, {"save": {"default": "A"}, "load": {"default": "A"}}) + with pytest.raises(KeyError): + D(not_allowed="value") + assert D(save="X")["save"] == "X" + assert D.fields == {"save": {"default": "A"}, "load": {"default": "A"}} # D class inverted class E(C, B): pass - self.assertRaises(KeyError, E, not_allowed="value") - self.assertEqual(E(save="X")["save"], "X") - self.assertEqual(E.fields, {"save": {"default": "A"}, "load": {"default": "A"}}) + with pytest.raises(KeyError): + E(not_allowed="value") + assert E(save="X")["save"] == "X" + assert E.fields == {"save": {"default": "A"}, "load": {"default": "A"}} def test_to_dict(self): class TestItem(Item): @@ -240,7 +245,7 @@ class ItemTest(unittest.TestCase): i = TestItem() i["name"] = "John" - self.assertEqual(dict(i), {"name": "John"}) + assert dict(i) == {"name": "John"} def test_copy(self): class TestItem(Item): @@ -248,9 +253,9 @@ class ItemTest(unittest.TestCase): item = TestItem({"name": "lower"}) copied_item = item.copy() - self.assertNotEqual(id(item), id(copied_item)) + assert id(item) != id(copied_item) copied_item["name"] = copied_item["name"].upper() - self.assertNotEqual(item["name"], copied_item["name"]) + assert item["name"] != copied_item["name"] def test_deepcopy(self): class TestItem(Item): @@ -262,7 +267,7 @@ class ItemTest(unittest.TestCase): assert item["tags"] != copied_item["tags"] -class ItemMetaTest(unittest.TestCase): +class TestItemMeta: def test_new_method_propagates_classcell(self): new_mock = mock.Mock(side_effect=ABCMeta.__new__) base = ItemMeta.__bases__[0] @@ -273,30 +278,24 @@ class ItemMetaTest(unittest.TestCase): def f(self): # For rationale of this see: # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222 - return ( - __class__ # noqa https://github.com/scrapy/scrapy/issues/2836 - ) + return __class__ MyItem() (first_call, second_call) = new_mock.call_args_list[-2:] - mcs, class_name, bases, attrs = first_call[0] + *_, attrs = first_call[0] assert "__classcell__" not in attrs - mcs, class_name, bases, attrs = second_call[0] + *_, attrs = second_call[0] assert "__classcell__" in attrs -class ItemMetaClassCellRegression(unittest.TestCase): +class TestItemMetaClassCellRegression: def test_item_meta_classcell_regression(self): class MyItem(Item, metaclass=ItemMeta): - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs): # pylint: disable=useless-parent-delegation # This call to super() trigger the __classcell__ propagation # requirement. When not done properly raises an error: # TypeError: __class__ set to # defining 'MyItem' as super().__init__(*args, **kwargs) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_link.py b/tests/test_link.py index 7ba0851ae..c49e5c090 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -1,16 +1,16 @@ -import unittest +import pytest from scrapy.link import Link -class LinkTest(unittest.TestCase): +class TestLink: def _assert_same_links(self, link1, link2): - self.assertEqual(link1, link2) - self.assertEqual(hash(link1), hash(link2)) + assert link1 == link2 + assert hash(link1) == hash(link2) def _assert_different_links(self, link1, link2): - self.assertNotEqual(link1, link2) - self.assertNotEqual(hash(link1), hash(link2)) + assert link1 != link2 + assert hash(link1) != hash(link2) def test_eq_and_hash(self): l1 = Link("http://www.example.com") @@ -53,5 +53,5 @@ class LinkTest(unittest.TestCase): self._assert_same_links(l1, l2) def test_bytes_url(self): - with self.assertRaises(TypeError): + with pytest.raises(TypeError): Link(b"http://www.example.com/\xc2\xa3") diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 18e9608c1..15d358d2a 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,10 +1,10 @@ +from __future__ import annotations + import pickle import re -import unittest -from typing import Optional +import pytest from packaging.version import Version -from pytest import mark from w3lib import __version__ as w3lib_version from scrapy.http import HtmlResponse, XmlResponse @@ -15,180 +15,144 @@ from tests import get_testdata # a hack to skip base class tests in pytest class Base: - class LinkExtractorTestCase(unittest.TestCase): - extractor_cls: Optional[type] = None + class TestLinkExtractorBase: + extractor_cls: type | None = None - def setUp(self): + def setup_method(self): body = get_testdata("link_extractor", "linkextractor.html") self.response = HtmlResponse(url="http://example.com/index", body=body) def test_urls_type(self): """Test that the resulting urls are str objects""" lx = self.extractor_cls() - self.assertTrue( - all( - isinstance(link.url, str) - for link in lx.extract_links(self.response) - ) + assert all( + isinstance(link.url, str) for link in lx.extract_links(self.response) ) def test_extract_all_links(self): lx = self.extractor_cls() page4_url = "http://example.com/page%204.html" - self.assertEqual( - [link for link in lx.extract_links(self.response)], - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - Link(url="http://www.google.com/something", text=""), - Link(url="http://example.com/innertag.html", text="inner tag"), - Link(url=page4_url, text="href with whitespaces"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + Link(url="http://www.google.com/something", text=""), + Link(url="http://example.com/innertag.html", text="inner tag"), + Link(url=page4_url, text="href with whitespaces"), + ] def test_extract_filter_allow(self): lx = self.extractor_cls(allow=("sample",)) - self.assertEqual( - [link for link in lx.extract_links(self.response)], - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + ] def test_extract_filter_allow_with_duplicates(self): lx = self.extractor_cls(allow=("sample",), unique=False) - self.assertEqual( - [link for link in lx.extract_links(self.response)], - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html", - text="sample 3 repetition", - ), - Link( - url="http://example.com/sample3.html", - text="sample 3 repetition", - ), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition", + ), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition", + ), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + ] def test_extract_filter_allow_with_duplicates_canonicalize(self): lx = self.extractor_cls(allow=("sample",), unique=False, canonicalize=True) - self.assertEqual( - [link for link in lx.extract_links(self.response)], - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html", - text="sample 3 repetition", - ), - Link( - url="http://example.com/sample3.html", - text="sample 3 repetition", - ), - Link( - url="http://example.com/sample3.html", - text="sample 3 repetition with fragment", - ), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition", + ), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition", + ), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition with fragment", + ), + ] def test_extract_filter_allow_no_duplicates_canonicalize(self): lx = self.extractor_cls(allow=("sample",), unique=True, canonicalize=True) - self.assertEqual( - [link for link in lx.extract_links(self.response)], - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + ] def test_extract_filter_allow_and_deny(self): lx = self.extractor_cls(allow=("sample",), deny=("3",)) - self.assertEqual( - [link for link in lx.extract_links(self.response)], - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + ] def test_extract_filter_allowed_domains(self): lx = self.extractor_cls(allow_domains=("google.com",)) - self.assertEqual( - [link for link in lx.extract_links(self.response)], - [ - Link(url="http://www.google.com/something", text=""), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://www.google.com/something", text=""), + ] def test_extraction_using_single_values(self): """Test the extractor's behaviour among different situations""" lx = self.extractor_cls(allow="sample") - self.assertEqual( - [link for link in lx.extract_links(self.response)], - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + ] lx = self.extractor_cls(allow="sample", deny="3") - self.assertEqual( - [link for link in lx.extract_links(self.response)], - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + ] lx = self.extractor_cls(allow_domains="google.com") - self.assertEqual( - [link for link in lx.extract_links(self.response)], - [ - Link(url="http://www.google.com/something", text=""), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://www.google.com/something", text=""), + ] lx = self.extractor_cls(deny_domains="example.com") - self.assertEqual( - [link for link in lx.extract_links(self.response)], - [ - Link(url="http://www.google.com/something", text=""), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://www.google.com/something", text=""), + ] def test_nofollow(self): - '''Test the extractor's behaviour for links with rel="nofollow"''' + """Test the extractor's behaviour for links with rel='nofollow'""" - html = b"""Page title<title> + html = b"""<html><head><title>Page title

About us

@@ -209,47 +173,44 @@ class Base: response = HtmlResponse("http://example.org/somepage/index.html", body=html) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.org/about.html", text="About us"), - Link(url="http://example.org/follow.html", text="Follow this link"), - Link( - url="http://example.org/nofollow.html", - text="Dont follow this one", - nofollow=True, - ), - Link( - url="http://example.org/nofollow2.html", - text="Choose to follow or not", - ), - Link( - url="http://google.com/something", - text="External link not to follow", - nofollow=True, - ), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.org/about.html", text="About us"), + Link(url="http://example.org/follow.html", text="Follow this link"), + Link( + url="http://example.org/nofollow.html", + text="Dont follow this one", + nofollow=True, + ), + Link( + url="http://example.org/nofollow2.html", + text="Choose to follow or not", + ), + Link( + url="http://google.com/something", + text="External link not to follow", + nofollow=True, + ), + ] def test_matches(self): url1 = "http://lotsofstuff.com/stuff1/index" url2 = "http://evenmorestuff.com/uglystuff/index" lx = self.extractor_cls(allow=(r"stuff1",)) - self.assertTrue(lx.matches(url1)) - self.assertFalse(lx.matches(url2)) + assert lx.matches(url1) + assert not lx.matches(url2) lx = self.extractor_cls(deny=(r"uglystuff",)) - self.assertTrue(lx.matches(url1)) - self.assertFalse(lx.matches(url2)) + assert lx.matches(url1) + assert not lx.matches(url2) lx = self.extractor_cls(allow_domains=("evenmorestuff.com",)) - self.assertFalse(lx.matches(url1)) - self.assertTrue(lx.matches(url2)) + assert not lx.matches(url1) + assert lx.matches(url2) lx = self.extractor_cls(deny_domains=("lotsofstuff.com",)) - self.assertFalse(lx.matches(url1)) - self.assertTrue(lx.matches(url2)) + assert not lx.matches(url1) + assert lx.matches(url2) lx = self.extractor_cls( allow=["blah1"], @@ -257,24 +218,21 @@ class Base: allow_domains=["blah1.com"], deny_domains=["blah2.com"], ) - self.assertTrue(lx.matches("http://blah1.com/blah1")) - self.assertFalse(lx.matches("http://blah1.com/blah2")) - self.assertFalse(lx.matches("http://blah2.com/blah1")) - self.assertFalse(lx.matches("http://blah2.com/blah2")) + assert lx.matches("http://blah1.com/blah1") + assert not lx.matches("http://blah1.com/blah2") + assert not lx.matches("http://blah2.com/blah1") + assert not lx.matches("http://blah2.com/blah2") def test_restrict_xpaths(self): lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]',)) - self.assertEqual( - [link for link in lx.extract_links(self.response)], - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + ] def test_restrict_xpaths_encoding(self): """Test restrict_xpaths with encodings""" - html = b"""Page title<title> + html = b"""<html><head><title>Page title

Item 12

About us\xa3

@@ -290,10 +248,9 @@ class Base: ) lx = self.extractor_cls(restrict_xpaths="//div[@class='links']") - self.assertEqual( - lx.extract_links(response), - [Link(url="http://example.org/about.html", text="About us\xa3")], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.org/about.html", text="About us\xa3") + ] def test_restrict_xpaths_with_html_entities(self): html = b'

text

' @@ -303,47 +260,40 @@ class Base: encoding="iso8859-15", ) links = self.extractor_cls(restrict_xpaths="//p").extract_links(response) - self.assertEqual( - links, [Link(url="http://example.org/%E2%99%A5/you?c=%A4", text="text")] - ) + assert links == [ + Link(url="http://example.org/%E2%99%A5/you?c=%A4", text="text") + ] def test_restrict_xpaths_concat_in_handle_data(self): """html entities cause SGMLParser to call handle_data hook twice""" body = b"""
>\xbe\xa9<\xb6\xab""" response = HtmlResponse("http://example.org", body=body, encoding="gb18030") lx = self.extractor_cls(restrict_xpaths="//div") - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.org/foo", - text=">\u4eac<\u4e1c", - fragment="", - nofollow=False, - ) - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.org/foo", + text=">\u4eac<\u4e1c", + fragment="", + nofollow=False, + ) + ] def test_restrict_css(self): lx = self.extractor_cls(restrict_css=("#subwrapper a",)) - self.assertEqual( - lx.extract_links(self.response), - [Link(url="http://example.com/sample2.html", text="sample 2")], - ) + assert lx.extract_links(self.response) == [ + Link(url="http://example.com/sample2.html", text="sample 2") + ] def test_restrict_css_and_restrict_xpaths_together(self): lx = self.extractor_cls( restrict_xpaths=('//div[@id="subwrapper"]',), restrict_css=("#subwrapper + a",), ) - self.assertEqual( - [link for link in lx.extract_links(self.response)], - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + ] def test_area_tag_with_unicode_present(self): body = b"""\xbe\xa9""" @@ -352,17 +302,14 @@ class Base: lx.extract_links(response) lx.extract_links(response) lx.extract_links(response) - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.org/foo", - text="", - fragment="", - nofollow=False, - ) - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.org/foo", + text="", + fragment="", + nofollow=False, + ) + ] def test_encoded_url(self): body = b"""
BinB""" @@ -370,17 +317,14 @@ class Base: "http://known.fm/AC%2FDC/", body=body, encoding="utf8" ) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://known.fm/AC%2FDC/?page=2", - text="BinB", - fragment="", - nofollow=False, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://known.fm/AC%2FDC/?page=2", + text="BinB", + fragment="", + nofollow=False, + ), + ] def test_encoded_url_in_restricted_xpath(self): body = b"""
BinB""" @@ -388,38 +332,29 @@ class Base: "http://known.fm/AC%2FDC/", body=body, encoding="utf8" ) lx = self.extractor_cls(restrict_xpaths="//div") - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://known.fm/AC%2FDC/?page=2", - text="BinB", - fragment="", - nofollow=False, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://known.fm/AC%2FDC/?page=2", + text="BinB", + fragment="", + nofollow=False, + ), + ] def test_ignored_extensions(self): # jpg is ignored by default html = b"""asd and """ response = HtmlResponse("http://example.org/", body=html) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.org/page.html", text="asd"), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.org/page.html", text="asd"), + ] # override denied extensions lx = self.extractor_cls(deny_extensions=["html"]) - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.org/photo.jpg"), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.org/photo.jpg"), + ] def test_process_value(self): """Test restrict_xpaths with encodings""" @@ -435,68 +370,59 @@ class Base: def process_value(value): m = re.search(r"javascript:goToPage\('(.*?)'", value) - if m: - return m.group(1) + return m.group(1) if m else None lx = self.extractor_cls(process_value=process_value) - self.assertEqual( - lx.extract_links(response), - [Link(url="http://example.org/other/page.html", text="Text")], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.org/other/page.html", text="Text") + ] def test_base_url_with_restrict_xpaths(self): - html = b"""Page title<title><base href="http://otherdomain.com/base/" /> + html = b"""<html><head><title>Page title

Item 12

""" response = HtmlResponse("http://example.org/somepage/index.html", body=html) lx = self.extractor_cls(restrict_xpaths="//p") - self.assertEqual( - lx.extract_links(response), - [Link(url="http://otherdomain.com/base/item/12.html", text="Item 12")], - ) + assert lx.extract_links(response) == [ + Link(url="http://otherdomain.com/base/item/12.html", text="Item 12") + ] def test_attrs(self): lx = self.extractor_cls(attrs="href") page4_url = "http://example.com/page%204.html" - self.assertEqual( - lx.extract_links(self.response), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - Link(url="http://www.google.com/something", text=""), - Link(url="http://example.com/innertag.html", text="inner tag"), - Link(url=page4_url, text="href with whitespaces"), - ], - ) + assert lx.extract_links(self.response) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + Link(url="http://www.google.com/something", text=""), + Link(url="http://example.com/innertag.html", text="inner tag"), + Link(url=page4_url, text="href with whitespaces"), + ] lx = self.extractor_cls( attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=() ) - self.assertEqual( - lx.extract_links(self.response), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample2.jpg", text=""), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - Link(url="http://www.google.com/something", text=""), - Link(url="http://example.com/innertag.html", text="inner tag"), - Link(url=page4_url, text="href with whitespaces"), - ], - ) + assert lx.extract_links(self.response) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample2.jpg", text=""), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + Link(url="http://www.google.com/something", text=""), + Link(url="http://example.com/innertag.html", text="inner tag"), + Link(url=page4_url, text="href with whitespaces"), + ] lx = self.extractor_cls(attrs=None) - self.assertEqual(lx.extract_links(self.response), []) + assert lx.extract_links(self.response) == [] def test_tags(self): html = ( @@ -506,43 +432,31 @@ class Base: response = HtmlResponse("http://example.com/index.html", body=html) lx = self.extractor_cls(tags=None) - self.assertEqual(lx.extract_links(response), []) + assert lx.extract_links(response) == [] lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + ] lx = self.extractor_cls(tags="area") - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.com/sample1.html", text=""), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.com/sample1.html", text=""), + ] lx = self.extractor_cls(tags="a") - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.com/sample2.html", text="sample 2"), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.com/sample2.html", text="sample 2"), + ] lx = self.extractor_cls( tags=("a", "img"), attrs=("href", "src"), deny_extensions=() ) - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample2.jpg", text=""), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample2.jpg", text=""), + ] def test_tags_attrs(self): html = b""" @@ -554,42 +468,36 @@ class Base: response = HtmlResponse("http://example.com/index.html", body=html) lx = self.extractor_cls(tags="div", attrs="data-url") - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.com/get?id=1", - text="Item 1", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/get?id=2", - text="Item 2", - fragment="", - nofollow=False, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.com/get?id=1", + text="Item 1", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/get?id=2", + text="Item 2", + fragment="", + nofollow=False, + ), + ] lx = self.extractor_cls(tags=("div",), attrs=("data-url",)) - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.com/get?id=1", - text="Item 1", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/get?id=2", - text="Item 2", - fragment="", - nofollow=False, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.com/get?id=1", + text="Item 1", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/get?id=2", + text="Item 2", + fragment="", + nofollow=False, + ), + ] def test_xhtml(self): xhtml = b""" @@ -623,78 +531,72 @@ class Base: response = HtmlResponse("http://example.com/index.xhtml", body=xhtml) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.com/about.html", - text="About us", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/follow.html", - text="Follow this link", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/nofollow.html", - text="Dont follow this one", - fragment="", - nofollow=True, - ), - Link( - url="http://example.com/nofollow2.html", - text="Choose to follow or not", - fragment="", - nofollow=False, - ), - Link( - url="http://google.com/something", - text="External link not to follow", - nofollow=True, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.com/about.html", + text="About us", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/follow.html", + text="Follow this link", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/nofollow.html", + text="Dont follow this one", + fragment="", + nofollow=True, + ), + Link( + url="http://example.com/nofollow2.html", + text="Choose to follow or not", + fragment="", + nofollow=False, + ), + Link( + url="http://google.com/something", + text="External link not to follow", + nofollow=True, + ), + ] response = XmlResponse("http://example.com/index.xhtml", body=xhtml) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.com/about.html", - text="About us", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/follow.html", - text="Follow this link", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/nofollow.html", - text="Dont follow this one", - fragment="", - nofollow=True, - ), - Link( - url="http://example.com/nofollow2.html", - text="Choose to follow or not", - fragment="", - nofollow=False, - ), - Link( - url="http://google.com/something", - text="External link not to follow", - nofollow=True, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.com/about.html", + text="About us", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/follow.html", + text="Follow this link", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/nofollow.html", + text="Dont follow this one", + fragment="", + nofollow=True, + ), + Link( + url="http://example.com/nofollow2.html", + text="Choose to follow or not", + fragment="", + nofollow=False, + ), + Link( + url="http://google.com/something", + text="External link not to follow", + nofollow=True, + ), + ] def test_link_wrong_href(self): html = b""" @@ -704,21 +606,18 @@ class Base: """ response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() - self.assertEqual( - [link for link in lx.extract_links(response)], - [ - Link( - url="http://example.org/item1.html", - text="Item 1", - nofollow=False, - ), - Link( - url="http://example.org/item3.html", - text="Item 3", - nofollow=False, - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link( + url="http://example.org/item1.html", + text="Item 1", + nofollow=False, + ), + Link( + url="http://example.org/item3.html", + text="Item 3", + nofollow=False, + ), + ] def test_ftp_links(self): body = b""" @@ -729,24 +628,121 @@ class Base: "http://www.example.com/index.html", body=body, encoding="utf8" ) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="ftp://www.external.com/", - text="An Item", - fragment="", - nofollow=False, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="ftp://www.external.com/", + text="An Item", + fragment="", + nofollow=False, + ), + ] def test_pickle_extractor(self): lx = self.extractor_cls() - self.assertIsInstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls) + assert isinstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls) + + def test_link_extractor_aggregation(self): + """When a parameter like restrict_css is used, the underlying + implementation calls its internal link extractor once per selector + matching the specified restrictions, and then aggregates the + extracted links. + + Test that aggregation respects the unique and canonicalize + parameters. + """ + # unique=True (default), canonicalize=False (default) + lx = self.extractor_cls(restrict_css=("div",)) + response = HtmlResponse( + "https://example.com", + body=b""" +
+ a1 + b1 +
+
+ a2 + b2 +
+ """, + ) + actual = lx.extract_links(response) + assert actual == [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/b?b=2&a=1", text="b2"), + ] + + # unique=True (default), canonicalize=True + lx = self.extractor_cls(restrict_css=("div",), canonicalize=True) + response = HtmlResponse( + "https://example.com", + body=b""" +
+ a1 + b1 +
+
+ a2 + b2 +
+ """, + ) + actual = lx.extract_links(response) + assert actual == [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + ] + + # unique=False, canonicalize=False (default) + lx = self.extractor_cls(restrict_css=("div",), unique=False) + response = HtmlResponse( + "https://example.com", + body=b""" +
+ a1 + b1 +
+
+ a2 + b2 +
+ """, + ) + actual = lx.extract_links(response) + assert actual == [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/a", text="a2"), + Link(url="https://example.com/b?b=2&a=1", text="b2"), + ] + + # unique=False, canonicalize=True + lx = self.extractor_cls( + restrict_css=("div",), unique=False, canonicalize=True + ) + response = HtmlResponse( + "https://example.com", + body=b""" +
+ a1 + b1 +
+
+ a2 + b2 +
+ """, + ) + actual = lx.extract_links(response) + assert actual == [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/a", text="a2"), + Link(url="https://example.com/b?a=1&b=2", text="b2"), + ] -class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): +class TestLxmlLinkExtractor(Base.TestLinkExtractorBase): extractor_cls = LxmlLinkExtractor def test_link_wrong_href(self): @@ -757,17 +753,10 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): """ response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() - self.assertEqual( - [link for link in lx.extract_links(response)], - [ - Link( - url="http://example.org/item1.html", text="Item 1", nofollow=False - ), - Link( - url="http://example.org/item3.html", text="Item 3", nofollow=False - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link(url="http://example.org/item1.html", text="Item 1", nofollow=False), + Link(url="http://example.org/item3.html", text="Item 3", nofollow=False), + ] def test_link_restrict_text(self): html = b""" @@ -778,50 +767,38 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): response = HtmlResponse("http://example.org/index.html", body=html) # Simple text inclusion test lx = self.extractor_cls(restrict_text="dog") - self.assertEqual( - [link for link in lx.extract_links(response)], - [ - Link( - url="http://example.org/item2.html", - text="Pic of a dog", - nofollow=False, - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link( + url="http://example.org/item2.html", + text="Pic of a dog", + nofollow=False, + ), + ] # Unique regex test lx = self.extractor_cls(restrict_text=r"of.*dog") - self.assertEqual( - [link for link in lx.extract_links(response)], - [ - Link( - url="http://example.org/item2.html", - text="Pic of a dog", - nofollow=False, - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link( + url="http://example.org/item2.html", + text="Pic of a dog", + nofollow=False, + ), + ] # Multiple regex test lx = self.extractor_cls(restrict_text=[r"of.*dog", r"of.*cat"]) - self.assertEqual( - [link for link in lx.extract_links(response)], - [ - Link( - url="http://example.org/item1.html", - text="Pic of a cat", - nofollow=False, - ), - Link( - url="http://example.org/item2.html", - text="Pic of a dog", - nofollow=False, - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link( + url="http://example.org/item1.html", + text="Pic of a cat", + nofollow=False, + ), + Link( + url="http://example.org/item2.html", + text="Pic of a dog", + nofollow=False, + ), + ] - def test_restrict_xpaths_with_html_entities(self): - super().test_restrict_xpaths_with_html_entities() - - @mark.skipif( + @pytest.mark.skipif( Version(w3lib_version) < Version("2.0.0"), reason=( "Before w3lib 2.0.0, w3lib.url.safe_url_string would not complain " @@ -836,18 +813,27 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): """ response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() - self.assertEqual( - [link for link in lx.extract_links(response)], - [ - Link( - url="http://example.org/item2.html", - text="Good Link", - nofollow=False, - ), - Link( - url="http://example.org/item3.html", - text="Good Link 2", - nofollow=False, - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link( + url="http://example.org/item2.html", + text="Good Link", + nofollow=False, + ), + Link( + url="http://example.org/item3.html", + text="Good Link 2", + nofollow=False, + ), + ] + + def test_link_allowed_is_false_with_empty_url(self): + bad_link = Link("") + assert not LxmlLinkExtractor()._link_allowed(bad_link) + + def test_link_allowed_is_false_with_bad_url_prefix(self): + bad_link = Link("htp://should_be_http.example") + assert not LxmlLinkExtractor()._link_allowed(bad_link) + + def test_link_allowed_is_false_with_missing_url_prefix(self): + bad_link = Link("should_have_prefix.example") + assert not LxmlLinkExtractor()._link_allowed(bad_link) diff --git a/tests/test_loader.py b/tests/test_loader.py index b0b7f8723..c094d25d8 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,8 +1,9 @@ +from __future__ import annotations + import dataclasses -import unittest -from typing import Optional import attr +import pytest from itemadapter import ItemAdapter from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst @@ -17,12 +18,12 @@ class NameItem(Item): name = Field() -class TestItem(NameItem): +class SummaryItem(NameItem): url = Field() summary = Field() -class TestNestedItem(Item): +class NestedItem(Item): name = Field() name_div = Field() name_value = Field() @@ -37,74 +38,64 @@ class AttrsNameItem: @dataclasses.dataclass -class TestDataClass: - name: list = dataclasses.field(default_factory=list) +class NameDataClass: + name: list[str] = dataclasses.field(default_factory=list) # test item loaders class NameItemLoader(ItemLoader): - default_item_class = TestItem + default_item_class = SummaryItem class NestedItemLoader(ItemLoader): - default_item_class = TestNestedItem + default_item_class = NestedItem -class TestItemLoader(NameItemLoader): +class ProcessorItemLoader(NameItemLoader): name_in = MapCompose(lambda v: v.title()) -class DefaultedItemLoader(NameItemLoader): - default_input_processor = MapCompose(lambda v: v[:-1]) - - -# test processors -def processor_with_args(value, other=None, loader_context=None): - if "key" in loader_context: - return loader_context["key"] - return value - - -class BasicItemLoaderTest(unittest.TestCase): +class TestBasicItemLoader: def test_add_value_on_unknown_field(self): - il = TestItemLoader() - self.assertRaises(KeyError, il.add_value, "wrong_field", ["lala", "lolo"]) + il = ProcessorItemLoader() + with pytest.raises(KeyError): + il.add_value("wrong_field", ["lala", "lolo"]) def test_load_item_using_default_loader(self): - i = TestItem() + i = SummaryItem() i["summary"] = "lala" il = ItemLoader(item=i) il.add_value("name", "marta") item = il.load_item() assert item is i - self.assertEqual(item["summary"], ["lala"]) - self.assertEqual(item["name"], ["marta"]) + assert item["summary"] == ["lala"] + assert item["name"] == ["marta"] def test_load_item_using_custom_loader(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", "marta") item = il.load_item() - self.assertEqual(item["name"], ["Marta"]) + assert item["name"] == ["Marta"] class InitializationTestMixin: - item_class: Optional[type] = None + item_class: type | None = None def test_keep_single_value(self): """Loaded item should contain values from the initial item""" input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo"]} def test_keep_list(self): """Loaded item should contain values from the initial item""" input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar"]} def test_add_value_singlevalue_singlevalue(self): """Values added after initialization should be appended""" @@ -112,8 +103,8 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) il.add_value("name", "bar") loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar"]} def test_add_value_singlevalue_list(self): """Values added after initialization should be appended""" @@ -121,10 +112,8 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) il.add_value("name", ["item", "loader"]) loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual( - ItemAdapter(loaded_item).asdict(), {"name": ["foo", "item", "loader"]} - ) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "item", "loader"]} def test_add_value_list_singlevalue(self): """Values added after initialization should be appended""" @@ -132,10 +121,8 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) il.add_value("name", "qwerty") loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual( - ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar", "qwerty"]} - ) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar", "qwerty"]} def test_add_value_list_list(self): """Values added after initialization should be appended""" @@ -143,59 +130,56 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) il.add_value("name", ["item", "loader"]) loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual( - ItemAdapter(loaded_item).asdict(), - {"name": ["foo", "bar", "item", "loader"]}, - ) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == { + "name": ["foo", "bar", "item", "loader"] + } def test_get_output_value_singlevalue(self): """Getting output value must not remove value from item""" input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value("name"), ["foo"]) + assert il.get_output_value("name") == ["foo"] loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({"name": ["foo"]})) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo"]} def test_get_output_value_list(self): """Getting output value must not remove value from item""" input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value("name"), ["foo", "bar"]) + assert il.get_output_value("name") == ["foo", "bar"] loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual( - ItemAdapter(loaded_item).asdict(), dict({"name": ["foo", "bar"]}) - ) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar"]} def test_values_single(self): """Values from initial item must be added to loader._values""" input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) - self.assertEqual(il._values.get("name"), ["foo"]) + assert il._values.get("name") == ["foo"] def test_values_list(self): """Values from initial item must be added to loader._values""" input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) - self.assertEqual(il._values.get("name"), ["foo", "bar"]) + assert il._values.get("name") == ["foo", "bar"] -class InitializationFromDictTest(InitializationTestMixin, unittest.TestCase): +class TestInitializationFromDict(InitializationTestMixin): item_class = dict -class InitializationFromItemTest(InitializationTestMixin, unittest.TestCase): +class TestInitializationFromItem(InitializationTestMixin): item_class = NameItem -class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase): +class TestInitializationFromAttrsItem(InitializationTestMixin): item_class = AttrsNameItem -class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase): - item_class = TestDataClass +class TestInitializationFromDataClass(InitializationTestMixin): + item_class = NameDataClass class BaseNoInputReprocessingLoader(ItemLoader): @@ -211,7 +195,7 @@ class NoInputReprocessingItemLoader(BaseNoInputReprocessingLoader): default_item_class = NoInputReprocessingItem -class NoInputReprocessingFromItemTest(unittest.TestCase): +class TestNoInputReprocessingFromItem: """ Loaders initialized from loaded items must not reprocess fields (Item instances) """ @@ -219,41 +203,41 @@ class NoInputReprocessingFromItemTest(unittest.TestCase): def test_avoid_reprocessing_with_initial_values_single(self): il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title="foo")) il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "foo"}) - self.assertEqual( - NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "foo"} - ) + assert il_loaded == {"title": "foo"} + assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == { + "title": "foo" + } def test_avoid_reprocessing_with_initial_values_list(self): il = NoInputReprocessingItemLoader( item=NoInputReprocessingItem(title=["foo", "bar"]) ) il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "foo"}) - self.assertEqual( - NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "foo"} - ) + assert il_loaded == {"title": "foo"} + assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == { + "title": "foo" + } def test_avoid_reprocessing_without_initial_values_single(self): il = NoInputReprocessingItemLoader() il.add_value("title", "FOO") il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "FOO"}) - self.assertEqual( - NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "FOO"} - ) + assert il_loaded == {"title": "FOO"} + assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == { + "title": "FOO" + } def test_avoid_reprocessing_without_initial_values_list(self): il = NoInputReprocessingItemLoader() il.add_value("title", ["foo", "bar"]) il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "FOO"}) - self.assertEqual( - NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "FOO"} - ) + assert il_loaded == {"title": "FOO"} + assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == { + "title": "FOO" + } -class TestOutputProcessorItem(unittest.TestCase): +class TestOutputProcessorItem: def test_output_processor(self): class TempItem(Item): temp = Field() @@ -269,11 +253,11 @@ class TestOutputProcessorItem(unittest.TestCase): loader = TempLoader() item = loader.load_item() - self.assertIsInstance(item, TempItem) - self.assertEqual(dict(item), {"temp": 0.3}) + assert isinstance(item, TempItem) + assert dict(item) == {"temp": 0.3} -class SelectortemLoaderTest(unittest.TestCase): +class TestSelectortemLoader: response = HtmlResponse( url="", encoding="utf-8", @@ -290,165 +274,170 @@ class SelectortemLoaderTest(unittest.TestCase): ) def test_init_method(self): - l = TestItemLoader() - self.assertEqual(l.selector, None) + l = ProcessorItemLoader() + assert l.selector is None def test_init_method_errors(self): - l = TestItemLoader() - self.assertRaises(RuntimeError, l.add_xpath, "url", "//a/@href") - self.assertRaises(RuntimeError, l.replace_xpath, "url", "//a/@href") - self.assertRaises(RuntimeError, l.get_xpath, "//a/@href") - self.assertRaises(RuntimeError, l.add_css, "name", "#name::text") - self.assertRaises(RuntimeError, l.replace_css, "name", "#name::text") - self.assertRaises(RuntimeError, l.get_css, "#name::text") + l = ProcessorItemLoader() + with pytest.raises(RuntimeError): + l.add_xpath("url", "//a/@href") + with pytest.raises(RuntimeError): + l.replace_xpath("url", "//a/@href") + with pytest.raises(RuntimeError): + l.get_xpath("//a/@href") + with pytest.raises(RuntimeError): + l.add_css("name", "#name::text") + with pytest.raises(RuntimeError): + l.replace_css("name", "#name::text") + with pytest.raises(RuntimeError): + l.get_css("#name::text") def test_init_method_with_selector(self): sel = Selector(text="
marta
") - l = TestItemLoader(selector=sel) - self.assertIs(l.selector, sel) + l = ProcessorItemLoader(selector=sel) + assert l.selector is sel l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] def test_init_method_with_selector_css(self): sel = Selector(text="
marta
") - l = TestItemLoader(selector=sel) - self.assertIs(l.selector, sel) + l = ProcessorItemLoader(selector=sel) + assert l.selector is sel l.add_css("name", "div::text") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] def test_init_method_with_base_response(self): """Selector should be None after initialization""" response = Response("https://scrapy.org") - l = TestItemLoader(response=response) - self.assertIs(l.selector, None) + l = ProcessorItemLoader(response=response) + assert l.selector is None def test_init_method_with_response(self): - l = TestItemLoader(response=self.response) - self.assertTrue(l.selector) + l = ProcessorItemLoader(response=self.response) + assert l.selector l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] def test_init_method_with_response_css(self): - l = TestItemLoader(response=self.response) - self.assertTrue(l.selector) + l = ProcessorItemLoader(response=self.response) + assert l.selector l.add_css("name", "div::text") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.add_css("url", "a::attr(href)") - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] # combining/accumulating CSS selectors and XPath expressions l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta", "Marta"]) + assert l.get_output_value("name") == ["Marta", "Marta"] l.add_xpath("url", "//img/@src") - self.assertEqual( - l.get_output_value("url"), ["http://www.scrapy.org", "/images/logo.png"] - ) + assert l.get_output_value("url") == [ + "http://www.scrapy.org", + "/images/logo.png", + ] def test_add_xpath_re(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) l.add_xpath("name", "//div/text()", re="ma") - self.assertEqual(l.get_output_value("name"), ["Ma"]) + assert l.get_output_value("name") == ["Ma"] def test_replace_xpath(self): - l = TestItemLoader(response=self.response) - self.assertTrue(l.selector) + l = ProcessorItemLoader(response=self.response) + assert l.selector l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_xpath("name", "//p/text()") - self.assertEqual(l.get_output_value("name"), ["Paragraph"]) + assert l.get_output_value("name") == ["Paragraph"] l.replace_xpath("name", ["//p/text()", "//div/text()"]) - self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"]) + assert l.get_output_value("name") == ["Paragraph", "Marta"] def test_get_xpath(self): - l = TestItemLoader(response=self.response) - self.assertEqual(l.get_xpath("//p/text()"), ["paragraph"]) - self.assertEqual(l.get_xpath("//p/text()", TakeFirst()), "paragraph") - self.assertEqual(l.get_xpath("//p/text()", TakeFirst(), re="pa"), "pa") + l = ProcessorItemLoader(response=self.response) + assert l.get_xpath("//p/text()") == ["paragraph"] + assert l.get_xpath("//p/text()", TakeFirst()) == "paragraph" + assert l.get_xpath("//p/text()", TakeFirst(), re="pa") == "pa" - self.assertEqual( - l.get_xpath(["//p/text()", "//div/text()"]), ["paragraph", "marta"] - ) + assert l.get_xpath(["//p/text()", "//div/text()"]) == ["paragraph", "marta"] def test_replace_xpath_multi_fields(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) l.add_xpath(None, "//div/text()", TakeFirst(), lambda x: {"name": x}) - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_xpath(None, "//p/text()", TakeFirst(), lambda x: {"name": x}) - self.assertEqual(l.get_output_value("name"), ["Paragraph"]) + assert l.get_output_value("name") == ["Paragraph"] def test_replace_xpath_re(self): - l = TestItemLoader(response=self.response) - self.assertTrue(l.selector) + l = ProcessorItemLoader(response=self.response) + assert l.selector l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_xpath("name", "//div/text()", re="ma") - self.assertEqual(l.get_output_value("name"), ["Ma"]) + assert l.get_output_value("name") == ["Ma"] def test_add_css_re(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) l.add_css("name", "div::text", re="ma") - self.assertEqual(l.get_output_value("name"), ["Ma"]) + assert l.get_output_value("name") == ["Ma"] l.add_css("url", "a::attr(href)", re="http://(.+)") - self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"]) + assert l.get_output_value("url") == ["www.scrapy.org"] def test_replace_css(self): - l = TestItemLoader(response=self.response) - self.assertTrue(l.selector) + l = ProcessorItemLoader(response=self.response) + assert l.selector l.add_css("name", "div::text") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_css("name", "p::text") - self.assertEqual(l.get_output_value("name"), ["Paragraph"]) + assert l.get_output_value("name") == ["Paragraph"] l.replace_css("name", ["p::text", "div::text"]) - self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"]) + assert l.get_output_value("name") == ["Paragraph", "Marta"] l.add_css("url", "a::attr(href)", re="http://(.+)") - self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"]) + assert l.get_output_value("url") == ["www.scrapy.org"] l.replace_css("url", "img::attr(src)") - self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) + assert l.get_output_value("url") == ["/images/logo.png"] def test_get_css(self): - l = TestItemLoader(response=self.response) - self.assertEqual(l.get_css("p::text"), ["paragraph"]) - self.assertEqual(l.get_css("p::text", TakeFirst()), "paragraph") - self.assertEqual(l.get_css("p::text", TakeFirst(), re="pa"), "pa") + l = ProcessorItemLoader(response=self.response) + assert l.get_css("p::text") == ["paragraph"] + assert l.get_css("p::text", TakeFirst()) == "paragraph" + assert l.get_css("p::text", TakeFirst(), re="pa") == "pa" - self.assertEqual(l.get_css(["p::text", "div::text"]), ["paragraph", "marta"]) - self.assertEqual( - l.get_css(["a::attr(href)", "img::attr(src)"]), - ["http://www.scrapy.org", "/images/logo.png"], - ) + assert l.get_css(["p::text", "div::text"]) == ["paragraph", "marta"] + assert l.get_css(["a::attr(href)", "img::attr(src)"]) == [ + "http://www.scrapy.org", + "/images/logo.png", + ] def test_replace_css_multi_fields(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) l.add_css(None, "div::text", TakeFirst(), lambda x: {"name": x}) - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_css(None, "p::text", TakeFirst(), lambda x: {"name": x}) - self.assertEqual(l.get_output_value("name"), ["Paragraph"]) + assert l.get_output_value("name") == ["Paragraph"] l.add_css(None, "a::attr(href)", TakeFirst(), lambda x: {"url": x}) - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] l.replace_css(None, "img::attr(src)", TakeFirst(), lambda x: {"url": x}) - self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) + assert l.get_output_value("url") == ["/images/logo.png"] def test_replace_css_re(self): - l = TestItemLoader(response=self.response) - self.assertTrue(l.selector) + l = ProcessorItemLoader(response=self.response) + assert l.selector l.add_css("url", "a::attr(href)") - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] l.replace_css("url", "a::attr(href)", re=r"http://www\.(.+)") - self.assertEqual(l.get_output_value("url"), ["scrapy.org"]) + assert l.get_output_value("url") == ["scrapy.org"] -class SubselectorLoaderTest(unittest.TestCase): +class TestSubselectorLoader: response = HtmlResponse( url="", encoding="utf-8", @@ -476,17 +465,13 @@ class SubselectorLoaderTest(unittest.TestCase): nl.add_css("name_div", "#id") nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value("name"), ["marta"]) - self.assertEqual(l.get_output_value("name_div"), ['
marta
']) - self.assertEqual(l.get_output_value("name_value"), ["marta"]) + assert l.get_output_value("name") == ["marta"] + assert l.get_output_value("name_div") == ['
marta
'] + assert l.get_output_value("name_value") == ["marta"] - self.assertEqual(l.get_output_value("name"), nl.get_output_value("name")) - self.assertEqual( - l.get_output_value("name_div"), nl.get_output_value("name_div") - ) - self.assertEqual( - l.get_output_value("name_value"), nl.get_output_value("name_value") - ) + assert l.get_output_value("name") == nl.get_output_value("name") + assert l.get_output_value("name_div") == nl.get_output_value("name_div") + assert l.get_output_value("name_value") == nl.get_output_value("name_value") def test_nested_css(self): l = NestedItemLoader(response=self.response) @@ -495,17 +480,13 @@ class SubselectorLoaderTest(unittest.TestCase): nl.add_css("name_div", "#id") nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value("name"), ["marta"]) - self.assertEqual(l.get_output_value("name_div"), ['
marta
']) - self.assertEqual(l.get_output_value("name_value"), ["marta"]) + assert l.get_output_value("name") == ["marta"] + assert l.get_output_value("name_div") == ['
marta
'] + assert l.get_output_value("name_value") == ["marta"] - self.assertEqual(l.get_output_value("name"), nl.get_output_value("name")) - self.assertEqual( - l.get_output_value("name_div"), nl.get_output_value("name_div") - ) - self.assertEqual( - l.get_output_value("name_value"), nl.get_output_value("name_value") - ) + assert l.get_output_value("name") == nl.get_output_value("name") + assert l.get_output_value("name_div") == nl.get_output_value("name_div") + assert l.get_output_value("name_value") == nl.get_output_value("name_value") def test_nested_replace(self): l = NestedItemLoader(response=self.response) @@ -513,11 +494,11 @@ class SubselectorLoaderTest(unittest.TestCase): nl2 = nl1.nested_xpath("a") l.add_xpath("url", "//footer/a/@href") - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] nl1.replace_xpath("url", "img/@src") - self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) + assert l.get_output_value("url") == ["/images/logo.png"] nl2.replace_xpath("url", "@href") - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] def test_nested_ordering(self): l = NestedItemLoader(response=self.response) @@ -529,15 +510,12 @@ class SubselectorLoaderTest(unittest.TestCase): nl2.add_xpath("url", "text()") l.add_xpath("url", "//footer/a/@href") - self.assertEqual( - l.get_output_value("url"), - [ - "/images/logo.png", - "http://www.scrapy.org", - "homepage", - "http://www.scrapy.org", - ], - ) + assert l.get_output_value("url") == [ + "/images/logo.png", + "http://www.scrapy.org", + "homepage", + "http://www.scrapy.org", + ] def test_nested_load_item(self): l = NestedItemLoader(response=self.response) @@ -554,9 +532,9 @@ class SubselectorLoaderTest(unittest.TestCase): assert item is nl1.item assert item is nl2.item - self.assertEqual(item["name"], ["marta"]) - self.assertEqual(item["url"], ["http://www.scrapy.org"]) - self.assertEqual(item["image"], ["/images/logo.png"]) + assert item["name"] == ["marta"] + assert item["url"] == ["http://www.scrapy.org"] + assert item["image"] == ["/images/logo.png"] # Functions as processors @@ -581,13 +559,9 @@ class FunctionProcessorItemLoader(ItemLoader): default_item_class = FunctionProcessorItem -class FunctionProcessorTestCase(unittest.TestCase): +class TestFunctionProcessor: def test_processor_defined_in_item(self): lo = FunctionProcessorItemLoader() lo.add_value("foo", " bar ") lo.add_value("foo", [" asdf ", " qwerty "]) - self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]}) - - -if __name__ == "__main__": - unittest.main() + assert dict(lo.load_item()) == {"foo": ["BAR", "ASDF", "QWERTY"]} diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py deleted file mode 100644 index d7f773d5c..000000000 --- a/tests/test_loader_deprecated.py +++ /dev/null @@ -1,722 +0,0 @@ -""" -These tests are kept as references from the ones that were ported to a itemloaders library. -Once we remove the references from scrapy, we can remove these tests. -""" - -import unittest -from functools import partial - -from itemloaders.processors import ( - Compose, - Identity, - Join, - MapCompose, - SelectJmes, - TakeFirst, -) - -from scrapy.item import Field, Item -from scrapy.loader import ItemLoader - - -# test items -class NameItem(Item): - name = Field() - - -class TestItem(NameItem): - url = Field() - summary = Field() - - -# test item loaders -class NameItemLoader(ItemLoader): - default_item_class = TestItem - - -class TestItemLoader(NameItemLoader): - name_in = MapCompose(lambda v: v.title()) - - -class DefaultedItemLoader(NameItemLoader): - default_input_processor = MapCompose(lambda v: v[:-1]) - - -# test processors -def processor_with_args(value, other=None, loader_context=None): - if "key" in loader_context: - return loader_context["key"] - return value - - -class BasicItemLoaderTest(unittest.TestCase): - def test_load_item_using_default_loader(self): - i = TestItem() - i["summary"] = "lala" - il = ItemLoader(item=i) - il.add_value("name", "marta") - item = il.load_item() - assert item is i - self.assertEqual(item["summary"], ["lala"]) - self.assertEqual(item["name"], ["marta"]) - - def test_load_item_using_custom_loader(self): - il = TestItemLoader() - il.add_value("name", "marta") - item = il.load_item() - self.assertEqual(item["name"], ["Marta"]) - - def test_load_item_ignore_none_field_values(self): - def validate_sku(value): - # Let's assume a SKU is only digits. - if value.isdigit(): - return value - - class MyLoader(ItemLoader): - name_out = Compose(lambda vs: vs[0]) # take first which allows empty values - price_out = Compose(TakeFirst(), float) - sku_out = Compose(TakeFirst(), validate_sku) - - valid_fragment = "SKU: 1234" - invalid_fragment = "SKU: not available" - sku_re = "SKU: (.+)" - - il = MyLoader(item={}) - # Should not return "sku: None". - il.add_value("sku", [invalid_fragment], re=sku_re) - # Should not ignore empty values. - il.add_value("name", "") - il.add_value("price", ["0"]) - self.assertEqual( - il.load_item(), - { - "name": "", - "price": 0.0, - }, - ) - - il.replace_value("sku", [valid_fragment], re=sku_re) - self.assertEqual(il.load_item()["sku"], "1234") - - def test_self_referencing_loader(self): - class MyLoader(ItemLoader): - url_out = TakeFirst() - - def img_url_out(self, values): - return (self.get_output_value("url") or "") + values[0] - - il = MyLoader(item={}) - il.add_value("url", "http://example.com/") - il.add_value("img_url", "1234.png") - self.assertEqual( - il.load_item(), - { - "url": "http://example.com/", - "img_url": "http://example.com/1234.png", - }, - ) - - il = MyLoader(item={}) - il.add_value("img_url", "1234.png") - self.assertEqual( - il.load_item(), - { - "img_url": "1234.png", - }, - ) - - def test_add_value(self): - il = TestItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_collected_values("name"), ["Marta"]) - self.assertEqual(il.get_output_value("name"), ["Marta"]) - il.add_value("name", "pepe") - self.assertEqual(il.get_collected_values("name"), ["Marta", "Pepe"]) - self.assertEqual(il.get_output_value("name"), ["Marta", "Pepe"]) - - # test add object value - il.add_value("summary", {"key": 1}) - self.assertEqual(il.get_collected_values("summary"), [{"key": 1}]) - - il.add_value(None, "Jim", lambda x: {"name": x}) - self.assertEqual(il.get_collected_values("name"), ["Marta", "Pepe", "Jim"]) - - def test_add_zero(self): - il = NameItemLoader() - il.add_value("name", 0) - self.assertEqual(il.get_collected_values("name"), [0]) - - def test_replace_value(self): - il = TestItemLoader() - il.replace_value("name", "marta") - self.assertEqual(il.get_collected_values("name"), ["Marta"]) - self.assertEqual(il.get_output_value("name"), ["Marta"]) - il.replace_value("name", "pepe") - self.assertEqual(il.get_collected_values("name"), ["Pepe"]) - self.assertEqual(il.get_output_value("name"), ["Pepe"]) - - il.replace_value(None, "Jim", lambda x: {"name": x}) - self.assertEqual(il.get_collected_values("name"), ["Jim"]) - - def test_get_value(self): - il = NameItemLoader() - self.assertEqual("FOO", il.get_value(["foo", "bar"], TakeFirst(), str.upper)) - self.assertEqual( - ["foo", "bar"], il.get_value(["name:foo", "name:bar"], re="name:(.*)$") - ) - self.assertEqual( - "foo", il.get_value(["name:foo", "name:bar"], TakeFirst(), re="name:(.*)$") - ) - - il.add_value("name", ["name:foo", "name:bar"], TakeFirst(), re="name:(.*)$") - self.assertEqual(["foo"], il.get_collected_values("name")) - il.replace_value("name", "name:bar", re="name:(.*)$") - self.assertEqual(["bar"], il.get_collected_values("name")) - - def test_iter_on_input_processor_input(self): - class NameFirstItemLoader(NameItemLoader): - name_in = TakeFirst() - - il = NameFirstItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_collected_values("name"), ["marta"]) - il = NameFirstItemLoader() - il.add_value("name", ["marta", "jose"]) - self.assertEqual(il.get_collected_values("name"), ["marta"]) - - il = NameFirstItemLoader() - il.replace_value("name", "marta") - self.assertEqual(il.get_collected_values("name"), ["marta"]) - il = NameFirstItemLoader() - il.replace_value("name", ["marta", "jose"]) - self.assertEqual(il.get_collected_values("name"), ["marta"]) - - il = NameFirstItemLoader() - il.add_value("name", "marta") - il.add_value("name", ["jose", "pedro"]) - self.assertEqual(il.get_collected_values("name"), ["marta", "jose"]) - - def test_map_compose_filter(self): - def filter_world(x): - return None if x == "world" else x - - proc = MapCompose(filter_world, str.upper) - self.assertEqual( - proc(["hello", "world", "this", "is", "scrapy"]), - ["HELLO", "THIS", "IS", "SCRAPY"], - ) - - def test_map_compose_filter_multil(self): - class TestItemLoader(NameItemLoader): - name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1]) - - il = TestItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["Mart"]) - item = il.load_item() - self.assertEqual(item["name"], ["Mart"]) - - def test_default_input_processor(self): - il = DefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["mart"]) - - def test_inherited_default_input_processor(self): - class InheritDefaultedItemLoader(DefaultedItemLoader): - pass - - il = InheritDefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["mart"]) - - def test_input_processor_inheritance(self): - class ChildItemLoader(TestItemLoader): - url_in = MapCompose(lambda v: v.lower()) - - il = ChildItemLoader() - il.add_value("url", "HTTP://scrapy.ORG") - self.assertEqual(il.get_output_value("url"), ["http://scrapy.org"]) - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["Marta"]) - - class ChildChildItemLoader(ChildItemLoader): - url_in = MapCompose(lambda v: v.upper()) - summary_in = MapCompose(lambda v: v) - - il = ChildChildItemLoader() - il.add_value("url", "http://scrapy.org") - self.assertEqual(il.get_output_value("url"), ["HTTP://SCRAPY.ORG"]) - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["Marta"]) - - def test_empty_map_compose(self): - class IdentityDefaultedItemLoader(DefaultedItemLoader): - name_in = MapCompose() - - il = IdentityDefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["marta"]) - - def test_identity_input_processor(self): - class IdentityDefaultedItemLoader(DefaultedItemLoader): - name_in = Identity() - - il = IdentityDefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["marta"]) - - def test_extend_custom_input_processors(self): - class ChildItemLoader(TestItemLoader): - name_in = MapCompose(TestItemLoader.name_in, str.swapcase) - - il = ChildItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["mARTA"]) - - def test_extend_default_input_processors(self): - class ChildDefaultedItemLoader(DefaultedItemLoader): - name_in = MapCompose( - DefaultedItemLoader.default_input_processor, str.swapcase - ) - - il = ChildDefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["MART"]) - - def test_output_processor_using_function(self): - il = TestItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - - class TakeFirstItemLoader(TestItemLoader): - name_out = " ".join - - il = TakeFirstItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), "Mar Ta") - - def test_output_processor_error(self): - class TestItemLoader(ItemLoader): - default_item_class = TestItem - name_out = MapCompose(float) - - il = TestItemLoader() - il.add_value("name", ["$10"]) - try: - float("$10") - except Exception as e: - expected_exc_str = str(e) - - exc = None - try: - il.load_item() - except Exception as e: - exc = e - assert isinstance(exc, ValueError) - s = str(exc) - assert "name" in s, s - assert "$10" in s, s - assert "ValueError" in s, s - assert expected_exc_str in s, s - - def test_output_processor_using_classes(self): - il = TestItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - - class TakeFirstItemLoader(TestItemLoader): - name_out = Join() - - il = TakeFirstItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), "Mar Ta") - - class TakeFirstItemLoader(TestItemLoader): - name_out = Join("
") - - il = TakeFirstItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), "Mar
Ta") - - def test_default_output_processor(self): - il = TestItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - - class LalaItemLoader(TestItemLoader): - default_output_processor = Identity() - - il = LalaItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - - def test_loader_context_on_declaration(self): - class ChildItemLoader(TestItemLoader): - url_in = MapCompose(processor_with_args, key="val") - - il = ChildItemLoader() - il.add_value("url", "text") - self.assertEqual(il.get_output_value("url"), ["val"]) - il.replace_value("url", "text2") - self.assertEqual(il.get_output_value("url"), ["val"]) - - def test_loader_context_on_instantiation(self): - class ChildItemLoader(TestItemLoader): - url_in = MapCompose(processor_with_args) - - il = ChildItemLoader(key="val") - il.add_value("url", "text") - self.assertEqual(il.get_output_value("url"), ["val"]) - il.replace_value("url", "text2") - self.assertEqual(il.get_output_value("url"), ["val"]) - - def test_loader_context_on_assign(self): - class ChildItemLoader(TestItemLoader): - url_in = MapCompose(processor_with_args) - - il = ChildItemLoader() - il.context["key"] = "val" - il.add_value("url", "text") - self.assertEqual(il.get_output_value("url"), ["val"]) - il.replace_value("url", "text2") - self.assertEqual(il.get_output_value("url"), ["val"]) - - def test_item_passed_to_input_processor_functions(self): - def processor(value, loader_context): - return loader_context["item"]["name"] - - class ChildItemLoader(TestItemLoader): - url_in = MapCompose(processor) - - it = TestItem(name="marta") - il = ChildItemLoader(item=it) - il.add_value("url", "text") - self.assertEqual(il.get_output_value("url"), ["marta"]) - il.replace_value("url", "text2") - self.assertEqual(il.get_output_value("url"), ["marta"]) - - def test_compose_processor(self): - class TestItemLoader(NameItemLoader): - name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1]) - - il = TestItemLoader() - il.add_value("name", ["marta", "other"]) - self.assertEqual(il.get_output_value("name"), "Mart") - item = il.load_item() - self.assertEqual(item["name"], "Mart") - - def test_partial_processor(self): - def join(values, sep=None, loader_context=None, ignored=None): - if sep is not None: - return sep.join(values) - if loader_context and "sep" in loader_context: - return loader_context["sep"].join(values) - return "".join(values) - - class TestItemLoader(NameItemLoader): - name_out = Compose(partial(join, sep="+")) - url_out = Compose(partial(join, loader_context={"sep": "."})) - summary_out = Compose(partial(join, ignored="foo")) - - il = TestItemLoader() - il.add_value("name", ["rabbit", "hole"]) - il.add_value("url", ["rabbit", "hole"]) - il.add_value("summary", ["rabbit", "hole"]) - item = il.load_item() - self.assertEqual(item["name"], "rabbit+hole") - self.assertEqual(item["url"], "rabbit.hole") - self.assertEqual(item["summary"], "rabbithole") - - def test_error_input_processor(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - name_in = MapCompose(float) - - il = TestItemLoader() - self.assertRaises(ValueError, il.add_value, "name", ["marta", "other"]) - - def test_error_output_processor(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - name_out = Compose(Join(), float) - - il = TestItemLoader() - il.add_value("name", "marta") - with self.assertRaises(ValueError): - il.load_item() - - def test_error_processor_as_argument(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - - il = TestItemLoader() - self.assertRaises( - ValueError, il.add_value, "name", ["marta", "other"], Compose(float) - ) - - -class InitializationFromDictTest(unittest.TestCase): - item_class = dict - - def test_keep_single_value(self): - """Loaded item should contain values from the initial item""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo"]}) - - def test_keep_list(self): - """Loaded item should contain values from the initial item""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "bar"]}) - - def test_add_value_singlevalue_singlevalue(self): - """Values added after initialization should be appended""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - il.add_value("name", "bar") - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "bar"]}) - - def test_add_value_singlevalue_list(self): - """Values added after initialization should be appended""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - il.add_value("name", ["item", "loader"]) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "item", "loader"]}) - - def test_add_value_list_singlevalue(self): - """Values added after initialization should be appended""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - il.add_value("name", "qwerty") - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "bar", "qwerty"]}) - - def test_add_value_list_list(self): - """Values added after initialization should be appended""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - il.add_value("name", ["item", "loader"]) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "bar", "item", "loader"]}) - - def test_get_output_value_singlevalue(self): - """Getting output value must not remove value from item""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value("name"), ["foo"]) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, dict({"name": ["foo"]})) - - def test_get_output_value_list(self): - """Getting output value must not remove value from item""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value("name"), ["foo", "bar"]) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, dict({"name": ["foo", "bar"]})) - - def test_values_single(self): - """Values from initial item must be added to loader._values""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - self.assertEqual(il._values.get("name"), ["foo"]) - - def test_values_list(self): - """Values from initial item must be added to loader._values""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - self.assertEqual(il._values.get("name"), ["foo", "bar"]) - - -class BaseNoInputReprocessingLoader(ItemLoader): - title_in = MapCompose(str.upper) - title_out = TakeFirst() - - -class NoInputReprocessingDictLoader(BaseNoInputReprocessingLoader): - default_item_class = dict - - -class NoInputReprocessingFromDictTest(unittest.TestCase): - """ - Loaders initialized from loaded items must not reprocess fields (dict instances) - """ - - def test_avoid_reprocessing_with_initial_values_single(self): - il = NoInputReprocessingDictLoader(item=dict(title="foo")) - il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="foo")) - self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo") - ) - - def test_avoid_reprocessing_with_initial_values_list(self): - il = NoInputReprocessingDictLoader(item=dict(title=["foo", "bar"])) - il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="foo")) - self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo") - ) - - def test_avoid_reprocessing_without_initial_values_single(self): - il = NoInputReprocessingDictLoader() - il.add_value("title", "foo") - il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="FOO")) - self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO") - ) - - def test_avoid_reprocessing_without_initial_values_list(self): - il = NoInputReprocessingDictLoader() - il.add_value("title", ["foo", "bar"]) - il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="FOO")) - self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO") - ) - - -class TestOutputProcessorDict(unittest.TestCase): - def test_output_processor(self): - class TempDict(dict): - def __init__(self, *args, **kwargs): - super().__init__(self, *args, **kwargs) - self.setdefault("temp", 0.3) - - class TempLoader(ItemLoader): - default_item_class = TempDict - default_input_processor = Identity() - default_output_processor = Compose(TakeFirst()) - - loader = TempLoader() - item = loader.load_item() - self.assertIsInstance(item, TempDict) - self.assertEqual(dict(item), {"temp": 0.3}) - - -class ProcessorsTest(unittest.TestCase): - def test_take_first(self): - proc = TakeFirst() - self.assertEqual(proc([None, "", "hello", "world"]), "hello") - self.assertEqual(proc([None, "", 0, "hello", "world"]), 0) - - def test_identity(self): - proc = Identity() - self.assertEqual( - proc([None, "", "hello", "world"]), [None, "", "hello", "world"] - ) - - def test_join(self): - proc = Join() - self.assertRaises(TypeError, proc, [None, "", "hello", "world"]) - self.assertEqual(proc(["", "hello", "world"]), " hello world") - self.assertEqual(proc(["hello", "world"]), "hello world") - self.assertIsInstance(proc(["hello", "world"]), str) - - def test_compose(self): - proc = Compose(lambda v: v[0], str.upper) - self.assertEqual(proc(["hello", "world"]), "HELLO") - proc = Compose(str.upper) - self.assertEqual(proc(None), None) - proc = Compose(str.upper, stop_on_none=False) - self.assertRaises(ValueError, proc, None) - proc = Compose(str.upper, lambda x: x + 1) - self.assertRaises(ValueError, proc, "hello") - - def test_mapcompose(self): - def filter_world(x): - return None if x == "world" else x - - proc = MapCompose(filter_world, str.upper) - self.assertEqual( - proc(["hello", "world", "this", "is", "scrapy"]), - ["HELLO", "THIS", "IS", "SCRAPY"], - ) - proc = MapCompose(filter_world, str.upper) - self.assertEqual(proc(None), []) - proc = MapCompose(filter_world, str.upper) - self.assertRaises(ValueError, proc, [1]) - proc = MapCompose(filter_world, lambda x: x + 1) - self.assertRaises(ValueError, proc, "hello") - - -class SelectJmesTestCase(unittest.TestCase): - test_list_equals = { - "simple": ("foo.bar", {"foo": {"bar": "baz"}}, "baz"), - "invalid": ("foo.bar.baz", {"foo": {"bar": "baz"}}, None), - "top_level": ("foo", {"foo": {"bar": "baz"}}, {"bar": "baz"}), - "double_vs_single_quote_string": ("foo.bar", {"foo": {"bar": "baz"}}, "baz"), - "dict": ( - "foo.bar[*].name", - {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, - ["one", "two"], - ), - "list": ("[1]", [1, 2], 2), - } - - def test_output(self): - for tl in self.test_list_equals: - expr, test_list, expected = self.test_list_equals[tl] - test = SelectJmes(expr)(test_list) - self.assertEqual( - test, expected, msg=f'test "{tl}" got {test} expected {expected}' - ) - - -# Functions as processors - - -def function_processor_strip(iterable): - return [x.strip() for x in iterable] - - -def function_processor_upper(iterable): - return [x.upper() for x in iterable] - - -class FunctionProcessorItem(Item): - foo = Field( - input_processor=function_processor_strip, - output_processor=function_processor_upper, - ) - - -class FunctionProcessorDictLoader(ItemLoader): - default_item_class = dict - foo_in = function_processor_strip - foo_out = function_processor_upper - - -class FunctionProcessorTestCase(unittest.TestCase): - def test_processor_defined_in_item_loader(self): - lo = FunctionProcessorDictLoader() - lo.add_value("foo", " bar ") - lo.add_value("foo", [" asdf ", " qwerty "]) - self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]}) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 56810f2ff..9806315b4 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -1,9 +1,8 @@ -import unittest +import logging +import pytest from testfixtures import LogCapture -from twisted.internet import defer from twisted.python.failure import Failure -from twisted.trial.unittest import TestCase as TwistedTestCase from scrapy.exceptions import DropItem from scrapy.http import Request, Response @@ -11,8 +10,9 @@ from scrapy.item import Field, Item from scrapy.logformatter import LogFormatter from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer +from tests.mockserver.http import MockServer from tests.spiders import ItemSpider +from tests.utils.decorators import inline_callbacks_test class CustomItem(Item): @@ -22,19 +22,18 @@ class CustomItem(Item): return f"name: {self['name']}" -class LogFormatterTestCase(unittest.TestCase): - def setUp(self): +class TestLogFormatter: + def setup_method(self): self.formatter = LogFormatter() self.spider = Spider("default") + self.spider.crawler = get_crawler() def test_crawled_with_referer(self): req = Request("http://www.example.com") res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, "Crawled (200) (referer: None)" - ) + assert logline == "Crawled (200) (referer: None)" def test_crawled_without_referer(self): req = Request( @@ -43,9 +42,9 @@ class LogFormatterTestCase(unittest.TestCase): res = Response("http://www.example.com", flags=["cached"]) logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, - "Crawled (200) (referer: http://example.com) ['cached']", + assert ( + logline + == "Crawled (200) (referer: http://example.com) ['cached']" ) def test_flags_in_request(self): @@ -53,9 +52,9 @@ class LogFormatterTestCase(unittest.TestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, - "Crawled (200) ['test', 'flag'] (referer: None)", + assert ( + logline + == "Crawled (200) ['test', 'flag'] (referer: None)" ) def test_dropped(self): @@ -66,7 +65,63 @@ class LogFormatterTestCase(unittest.TestCase): logline = logkws["msg"] % logkws["args"] lines = logline.splitlines() assert all(isinstance(x, str) for x in lines) - self.assertEqual(lines, ["Dropped: \u2018", "{}"]) + assert lines == ["Dropped: \u2018", "{}"] + + def test_dropitem_default_log_level(self): + item = {} + exception = DropItem("Test drop") + response = Response("http://www.example.com") + spider = Spider("foo") + spider.crawler = get_crawler(Spider) + + logkws = self.formatter.dropped(item, exception, response, spider) + assert logkws["level"] == logging.WARNING + + spider.crawler.settings.frozen = False + spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = logging.INFO + spider.crawler.settings.frozen = True + logkws = self.formatter.dropped(item, exception, response, spider) + assert logkws["level"] == logging.INFO + + spider.crawler.settings.frozen = False + spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = "INFO" + spider.crawler.settings.frozen = True + logkws = self.formatter.dropped(item, exception, response, spider) + assert logkws["level"] == logging.INFO + + spider.crawler.settings.frozen = False + spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = 10 + spider.crawler.settings.frozen = True + logkws = self.formatter.dropped(item, exception, response, spider) + assert logkws["level"] == logging.DEBUG + + spider.crawler.settings.frozen = False + spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = 0 + spider.crawler.settings.frozen = True + logkws = self.formatter.dropped(item, exception, response, spider) + assert logkws["level"] == logging.NOTSET + + unsupported_value = object() + spider.crawler.settings.frozen = False + spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = unsupported_value + spider.crawler.settings.frozen = True + logkws = self.formatter.dropped(item, exception, response, spider) + assert logkws["level"] == unsupported_value + + with pytest.raises(TypeError): + logging.log(logkws["level"], "message") # noqa: LOG015 + + def test_dropitem_custom_log_level(self): + item = {} + response = Response("http://www.example.com") + + exception = DropItem("Test drop", log_level="INFO") + logkws = self.formatter.dropped(item, exception, response, self.spider) + assert logkws["level"] == logging.INFO + + exception = DropItem("Test drop", log_level="ERROR") + logkws = self.formatter.dropped(item, exception, response, self.spider) + assert logkws["level"] == logging.ERROR def test_item_error(self): # In practice, the complete traceback is shown by passing the @@ -76,7 +131,7 @@ class LogFormatterTestCase(unittest.TestCase): response = Response("http://www.example.com") logkws = self.formatter.item_error(item, exception, response, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual(logline, "Error processing {'key': 'value'}") + assert logline == "Error processing {'key': 'value'}" def test_spider_error(self): # In practice, the complete traceback is shown by passing the @@ -88,9 +143,9 @@ class LogFormatterTestCase(unittest.TestCase): response = Response("http://www.example.com", request=request) logkws = self.formatter.spider_error(failure, request, response, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, - "Spider error processing (referer: http://example.org)", + assert ( + logline + == "Spider error processing (referer: http://example.org)" ) def test_download_error_short(self): @@ -100,7 +155,7 @@ class LogFormatterTestCase(unittest.TestCase): request = Request("http://www.example.com") logkws = self.formatter.download_error(failure, request, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual(logline, "Error downloading ") + assert logline == "Error downloading " def test_download_error_long(self): # In practice, the complete traceback is shown by passing the @@ -111,9 +166,7 @@ class LogFormatterTestCase(unittest.TestCase): failure, request, self.spider, "Some message" ) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, "Error downloading : Some message" - ) + assert logline == "Error downloading : Some message" def test_scraped(self): item = CustomItem() @@ -123,9 +176,7 @@ class LogFormatterTestCase(unittest.TestCase): logline = logkws["msg"] % logkws["args"] lines = logline.splitlines() assert all(isinstance(x, str) for x in lines) - self.assertEqual( - lines, ["Scraped from <200 http://www.example.com>", "name: \xa3"] - ) + assert lines == ["Scraped from <200 http://www.example.com>", "name: \xa3"] class LogFormatterSubclass(LogFormatter): @@ -141,18 +192,19 @@ class LogFormatterSubclass(LogFormatter): } -class LogformatterSubclassTest(LogFormatterTestCase): - def setUp(self): +class TestLogformatterSubclass(TestLogFormatter): + def setup_method(self): self.formatter = LogFormatterSubclass() self.spider = Spider("default") + self.spider.crawler = get_crawler(Spider) def test_crawled_with_referer(self): req = Request("http://www.example.com") res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, "Crawled (200) (referer: None) []" + assert ( + logline == "Crawled (200) (referer: None) []" ) def test_crawled_without_referer(self): @@ -164,9 +216,9 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, - "Crawled (200) (referer: http://example.com) ['cached']", + assert ( + logline + == "Crawled (200) (referer: http://example.com) ['cached']" ) def test_flags_in_request(self): @@ -174,9 +226,9 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, - "Crawled (200) (referer: None) ['test', 'flag']", + assert ( + logline + == "Crawled (200) (referer: None) ['test', 'flag']" ) @@ -194,18 +246,24 @@ class SkipMessagesLogFormatter(LogFormatter): class DropSomeItemsPipeline: drop = True - def process_item(self, item, spider): + def process_item(self, item): if self.drop: self.drop = False raise DropItem("Ignoring item") - else: - self.drop = True + self.drop = True -class ShowOrSkipMessagesTestCase(TwistedTestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() +class TestShowOrSkipMessages: + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) + + def setup_method(self): self.base_settings = { "LOG_LEVEL": "DEBUG", "ITEM_PIPELINES": { @@ -213,29 +271,22 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): }, } - def tearDown(self): - self.mockserver.__exit__(None, None, None) - - @defer.inlineCallbacks + @inline_callbacks_test def test_show_messages(self): crawler = get_crawler(ItemSpider, self.base_settings) with LogCapture() as lc: yield crawler.crawl(mockserver=self.mockserver) - self.assertIn("Scraped from <200 http://127.0.0.1:", str(lc)) - self.assertIn("Crawled (200) body

") - self.assertEqual(msg.get("Content-Type"), "text/html") + assert msg.get_payload() == "

body

" + assert msg.get("Content-Type") == "text/html" def test_send_attach(self): attach = BytesIO() @@ -73,25 +78,22 @@ class MailSenderTest(unittest.TestCase): ) assert self.catched_msg - self.assertEqual(self.catched_msg["to"], ["test@scrapy.org"]) - self.assertEqual(self.catched_msg["subject"], "subject") - self.assertEqual(self.catched_msg["body"], "body") + assert self.catched_msg["to"] == ["test@scrapy.org"] + assert self.catched_msg["subject"] == "subject" + assert self.catched_msg["body"] == "body" msg = self.catched_msg["msg"] - self.assertEqual(msg["to"], "test@scrapy.org") - self.assertEqual(msg["subject"], "subject") + assert msg["to"] == "test@scrapy.org" + assert msg["subject"] == "subject" payload = msg.get_payload() assert isinstance(payload, list) - self.assertEqual(len(payload), 2) + assert len(payload) == 2 text, attach = payload - self.assertEqual(text.get_payload(decode=True), b"body") - self.assertEqual(text.get_charset(), Charset("us-ascii")) - self.assertEqual(attach.get_payload(decode=True), b"content") - - def _catch_mail_sent(self, **kwargs): - self.catched_msg = dict(**kwargs) + assert text.get_payload(decode=True) == b"body" + assert text.get_charset() == Charset("us-ascii") + assert attach.get_payload(decode=True) == b"content" def test_send_utf8(self): subject = "sübjèçt" @@ -106,14 +108,14 @@ class MailSenderTest(unittest.TestCase): ) assert self.catched_msg - self.assertEqual(self.catched_msg["subject"], subject) - self.assertEqual(self.catched_msg["body"], body) + assert self.catched_msg["subject"] == subject + assert self.catched_msg["body"] == body msg = self.catched_msg["msg"] - self.assertEqual(msg["subject"], subject) - self.assertEqual(msg.get_payload(decode=True).decode("utf-8"), body) - self.assertEqual(msg.get_charset(), Charset("utf-8")) - self.assertEqual(msg.get("Content-Type"), 'text/plain; charset="utf-8"') + assert msg["subject"] == subject + assert msg.get_payload(decode=True).decode("utf-8") == body + assert msg.get_charset() == Charset("utf-8") + assert msg.get("Content-Type") == 'text/plain; charset="utf-8"' def test_send_attach_utf8(self): subject = "sübjèçt" @@ -134,22 +136,22 @@ class MailSenderTest(unittest.TestCase): ) assert self.catched_msg - self.assertEqual(self.catched_msg["subject"], subject) - self.assertEqual(self.catched_msg["body"], body) + assert self.catched_msg["subject"] == subject + assert self.catched_msg["body"] == body msg = self.catched_msg["msg"] - self.assertEqual(msg["subject"], subject) - self.assertEqual(msg.get_charset(), Charset("utf-8")) - self.assertEqual(msg.get("Content-Type"), 'multipart/mixed; charset="utf-8"') + assert msg["subject"] == subject + assert msg.get_charset() == Charset("utf-8") + assert msg.get("Content-Type") == 'multipart/mixed; charset="utf-8"' payload = msg.get_payload() assert isinstance(payload, list) - self.assertEqual(len(payload), 2) + assert len(payload) == 2 text, attach = payload - self.assertEqual(text.get_payload(decode=True).decode("utf-8"), body) - self.assertEqual(text.get_charset(), Charset("utf-8")) - self.assertEqual(attach.get_payload(decode=True).decode("utf-8"), body) + assert text.get_payload(decode=True).decode("utf-8") == body + assert text.get_charset() == Charset("utf-8") + assert attach.get_payload(decode=True).decode("utf-8") == body def test_create_sender_factory_with_host(self): mailsender = MailSender(debug=False, smtphost="smtp.testhost.com") @@ -159,11 +161,4 @@ class MailSenderTest(unittest.TestCase): ) context = factory.buildProtocol("test@scrapy.org").context - if twisted_version >= Version("twisted", 21, 2, 0): - self.assertIsInstance(context, ClientTLSOptions) - else: - self.assertIsInstance(context, ClientContextFactory) - - -if __name__ == "__main__": - unittest.main() + assert isinstance(context, ClientTLSOptions) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index a42c7b3d1..f3d619d3c 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -1,8 +1,16 @@ -from twisted.trial import unittest +from __future__ import annotations -from scrapy.exceptions import NotConfigured +from typing import TYPE_CHECKING + +import pytest + +from scrapy import Spider +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.middleware import MiddlewareManager -from scrapy.settings import Settings +from scrapy.utils.test import get_crawler + +if TYPE_CHECKING: + from scrapy.crawler import Crawler class M1: @@ -12,7 +20,7 @@ class M1: def close_spider(self, spider): pass - def process(self, response, request, spider): + def process(self, response, request): pass @@ -23,11 +31,9 @@ class M2: def close_spider(self, spider): pass - pass - class M3: - def process(self, response, request, spider): + def process(self, response, request): pass @@ -42,48 +48,62 @@ class MOff: raise NotConfigured("foo") -class TestMiddlewareManager(MiddlewareManager): +class MyMiddlewareManager(MiddlewareManager): + component_name = "my" + @classmethod def _get_mwlist_from_settings(cls, settings): return [M1, MOff, M3] def _add_middleware(self, mw): - super()._add_middleware(mw) + if hasattr(mw, "open_spider"): + self.methods["open_spider"].append(mw.open_spider) + if hasattr(mw, "close_spider"): + self.methods["close_spider"].appendleft(mw.close_spider) if hasattr(mw, "process"): self.methods["process"].append(mw.process) -class MiddlewareManagerTest(unittest.TestCase): - def test_init(self): - m1, m2, m3 = M1(), M2(), M3() - mwman = TestMiddlewareManager(m1, m2, m3) - self.assertEqual( - list(mwman.methods["open_spider"]), [m1.open_spider, m2.open_spider] - ) - self.assertEqual( - list(mwman.methods["close_spider"]), [m2.close_spider, m1.close_spider] - ) - self.assertEqual(list(mwman.methods["process"]), [m1.process, m3.process]) +@pytest.fixture +def crawler() -> Crawler: + return get_crawler(Spider) - def test_methods(self): - mwman = TestMiddlewareManager(M1(), M2(), M3()) - self.assertEqual( - [x.__self__.__class__ for x in mwman.methods["open_spider"]], [M1, M2] - ) - self.assertEqual( - [x.__self__.__class__ for x in mwman.methods["close_spider"]], [M2, M1] - ) - self.assertEqual( - [x.__self__.__class__ for x in mwman.methods["process"]], [M1, M3] - ) - def test_enabled(self): - m1, m2, m3 = M1(), M2(), M3() - mwman = MiddlewareManager(m1, m2, m3) - self.assertEqual(mwman.middlewares, (m1, m2, m3)) +def test_init(crawler: Crawler) -> None: + m1, m2, m3 = M1(), M2(), M3() + mwman = MyMiddlewareManager(m1, m2, m3, crawler=crawler) + assert list(mwman.methods["open_spider"]) == [m1.open_spider, m2.open_spider] + assert list(mwman.methods["close_spider"]) == [m2.close_spider, m1.close_spider] + assert list(mwman.methods["process"]) == [m1.process, m3.process] + assert mwman.crawler == crawler - def test_enabled_from_settings(self): - settings = Settings() - mwman = TestMiddlewareManager.from_settings(settings) - classes = [x.__class__ for x in mwman.middlewares] - self.assertEqual(classes, [M1, M3]) + +def test_methods(crawler: Crawler) -> None: + mwman = MyMiddlewareManager(M1(), M2(), M3(), crawler=crawler) + assert [x.__self__.__class__ for x in mwman.methods["open_spider"]] == [M1, M2] # type: ignore[union-attr] + assert [x.__self__.__class__ for x in mwman.methods["close_spider"]] == [M2, M1] # type: ignore[union-attr] + assert [x.__self__.__class__ for x in mwman.methods["process"]] == [M1, M3] # type: ignore[union-attr] + + +def test_enabled(crawler: Crawler) -> None: + m1, m2, m3 = M1(), M2(), M3() + mwman = MyMiddlewareManager(m1, m2, m3, crawler=crawler) + assert mwman.middlewares == (m1, m2, m3) + + +def test_enabled_from_settings(crawler: Crawler) -> None: + crawler = get_crawler() + mwman = MyMiddlewareManager.from_crawler(crawler) + classes = [x.__class__ for x in mwman.middlewares] + assert classes == [M1, M3] + assert mwman.crawler == crawler + + +def test_no_crawler() -> None: + m1, m2, m3 = M1(), M2(), M3() + with pytest.warns( + ScrapyDeprecationWarning, match="was called without the crawler argument" + ): + mwman = MyMiddlewareManager(m1, m2, m3) + assert mwman.middlewares == (m1, m2, m3) + assert mwman.crawler is None diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index c41ab483f..0376d27c4 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,17 +1,23 @@ +from __future__ import annotations + import shutil from pathlib import Path -from typing import Optional, Set +from tempfile import mkdtemp +from typing import TYPE_CHECKING, Any +import pytest from testfixtures import LogCapture -from twisted.internet import defer -from twisted.trial.unittest import TestCase from w3lib.url import add_or_replace_parameter -from scrapy import signals -from scrapy.crawler import CrawlerRunner +from scrapy import Spider, signals from scrapy.utils.misc import load_object -from tests.mockserver import MockServer +from scrapy.utils.test import get_crawler +from tests.mockserver.http import MockServer from tests.spiders import SimpleSpider +from tests.utils.decorators import inline_callbacks_test + +if TYPE_CHECKING: + from scrapy.crawler import Crawler class MediaDownloadSpider(SimpleSpider): @@ -51,150 +57,145 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider): ) -class FileDownloadCrawlTestCase(TestCase): +class TestFileDownloadCrawl: pipeline_class = "scrapy.pipelines.files.FilesPipeline" store_setting_key = "FILES_STORE" media_key = "files" media_urls_key = "file_urls" - expected_checksums: Optional[Set[str]] = { + expected_checksums: set[str] | None = { "5547178b89448faf0015a13f904c936e", "c2281c83670e31d8aaab7cb642b824db", "ed3f6538dc15d4d9179dae57319edc5f", } - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) + + def setup_method(self): # prepare a directory for storing files - self.tmpmediastore = Path(self.mktemp()) - self.tmpmediastore.mkdir() + self.tmpmediastore = Path(mkdtemp()) self.settings = { - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", "ITEM_PIPELINES": {self.pipeline_class: 1}, self.store_setting_key: str(self.tmpmediastore), } - self.runner = CrawlerRunner(self.settings) self.items = [] - def tearDown(self): + def teardown_method(self): shutil.rmtree(self.tmpmediastore) self.items = [] - self.mockserver.__exit__(None, None, None) def _on_item_scraped(self, item): self.items.append(item) - def _create_crawler(self, spider_class, runner=None, **kwargs): - if runner is None: - runner = self.runner - crawler = runner.create_crawler(spider_class, **kwargs) + def _create_crawler( + self, spider_class: type[Spider], settings: dict[str, Any] | None = None + ) -> Crawler: + if settings is None: + settings = self.settings + crawler = get_crawler(spider_class, settings) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) return crawler def _assert_files_downloaded(self, items, logs): - self.assertEqual(len(items), 1) - self.assertIn(self.media_key, items[0]) + assert len(items) == 1 + assert self.media_key in items[0] # check that logs show the expected number of successful file downloads file_dl_success = "File (downloaded): Downloaded file from" - self.assertEqual(logs.count(file_dl_success), 3) + assert logs.count(file_dl_success) == 3 # check that the images/files status is `downloaded` for item in items: for i in item[self.media_key]: - self.assertEqual(i["status"], "downloaded") + assert i["status"] == "downloaded" # check that the images/files checksums are what we know they should be if self.expected_checksums is not None: - checksums = set( - i["checksum"] for item in items for i in item[self.media_key] - ) - self.assertEqual(checksums, self.expected_checksums) + checksums = {i["checksum"] for item in items for i in item[self.media_key]} + assert checksums == self.expected_checksums # check that the image files where actually written to the media store for item in items: for i in item[self.media_key]: - self.assertTrue((self.tmpmediastore / i["path"]).exists()) + assert (self.tmpmediastore / i["path"]).exists() def _assert_files_download_failure(self, crawler, items, code, logs): # check that the item does NOT have the "images/files" field populated - self.assertEqual(len(items), 1) - self.assertIn(self.media_key, items[0]) - self.assertFalse(items[0][self.media_key]) + assert len(items) == 1 + assert self.media_key in items[0] + assert not items[0][self.media_key] # check that there was 1 successful fetch and 3 other responses with non-200 code - self.assertEqual( - crawler.stats.get_value("downloader/request_method_count/GET"), 4 - ) - self.assertEqual(crawler.stats.get_value("downloader/response_count"), 4) - self.assertEqual( - crawler.stats.get_value("downloader/response_status_count/200"), 1 - ) - self.assertEqual( - crawler.stats.get_value(f"downloader/response_status_count/{code}"), 3 - ) + assert crawler.stats.get_value("downloader/request_method_count/GET") == 4 + assert crawler.stats.get_value("downloader/response_count") == 4 + assert crawler.stats.get_value("downloader/response_status_count/200") == 1 + assert crawler.stats.get_value(f"downloader/response_status_count/{code}") == 3 # check that logs do show the failure on the file downloads file_dl_failure = f"File (code: {code}): Error downloading file from" - self.assertEqual(logs.count(file_dl_failure), 3) + assert logs.count(file_dl_failure) == 3 # check that no files were written to the media store - self.assertEqual([x for x in self.tmpmediastore.iterdir()], []) + assert not list(self.tmpmediastore.iterdir()) - @defer.inlineCallbacks + @inline_callbacks_test def test_download_media(self): crawler = self._create_crawler(MediaDownloadSpider) with LogCapture() as log: yield crawler.crawl( - self.mockserver.url("/files/images/"), + self.mockserver.url("/static/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, ) self._assert_files_downloaded(self.items, str(log)) - @defer.inlineCallbacks + @inline_callbacks_test def test_download_media_wrong_urls(self): crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) with LogCapture() as log: yield crawler.crawl( - self.mockserver.url("/files/images/"), + self.mockserver.url("/static/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, ) self._assert_files_download_failure(crawler, self.items, 404, str(log)) - @defer.inlineCallbacks + @inline_callbacks_test def test_download_media_redirected_default_failure(self): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: yield crawler.crawl( - self.mockserver.url("/files/images/"), + self.mockserver.url("/static/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, mockserver=self.mockserver, ) self._assert_files_download_failure(crawler, self.items, 302, str(log)) - @defer.inlineCallbacks + @inline_callbacks_test def test_download_media_redirected_allowed(self): - settings = dict(self.settings) - settings.update({"MEDIA_ALLOW_REDIRECTS": True}) - runner = CrawlerRunner(settings) - crawler = self._create_crawler(RedirectedMediaDownloadSpider, runner=runner) + settings = { + **self.settings, + "MEDIA_ALLOW_REDIRECTS": True, + } + crawler = self._create_crawler(RedirectedMediaDownloadSpider, settings) with LogCapture() as log: yield crawler.crawl( - self.mockserver.url("/files/images/"), + self.mockserver.url("/static/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, mockserver=self.mockserver, ) self._assert_files_downloaded(self.items, str(log)) - self.assertEqual( - crawler.stats.get_value("downloader/response_status_count/302"), 3 - ) + assert crawler.stats.get_value("downloader/response_status_count/302") == 3 - @defer.inlineCallbacks + @inline_callbacks_test def test_download_media_file_path_error(self): cls = load_object(self.pipeline_class) @@ -206,32 +207,31 @@ class FileDownloadCrawlTestCase(TestCase): **self.settings, "ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1}, } - runner = CrawlerRunner(settings) - crawler = self._create_crawler(MediaDownloadSpider, runner=runner) + crawler = self._create_crawler(MediaDownloadSpider, settings) with LogCapture() as log: yield crawler.crawl( - self.mockserver.url("/files/images/"), + self.mockserver.url("/static/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, mockserver=self.mockserver, ) - self.assertIn("ZeroDivisionError", str(log)) + assert "ZeroDivisionError" in str(log) -skip_pillow: Optional[str] +pillow_available: bool try: - from PIL import Image # noqa: imported just to check for the import error + from PIL import Image # noqa: F401 except ImportError: - skip_pillow = ( - "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" - ) + pillow_available = False else: - skip_pillow = None + pillow_available = True -class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase): - skip = skip_pillow - +@pytest.mark.skipif( + not pillow_available, + reason="Missing Python Imaging Library, install https://pypi.org/pypi/Pillow", +) +class TestImageDownloadCrawl(TestFileDownloadCrawl): pipeline_class = "scrapy.pipelines.images.ImagesPipeline" store_setting_key = "IMAGES_STORE" media_key = "images" diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e7000e314..c6a41fa6e 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -2,20 +2,26 @@ import dataclasses import os import random import time +import warnings +from abc import ABC, abstractmethod from datetime import datetime +from ftplib import FTP from io import BytesIO from pathlib import Path +from posixpath import split from shutil import rmtree from tempfile import mkdtemp -from typing import Dict, List +from typing import Any from unittest import mock +from unittest.mock import MagicMock from urllib.parse import urlparse import attr +import pytest from itemadapter import ItemAdapter -from twisted.internet import defer -from twisted.trial import unittest +from twisted.internet.defer import Deferred +from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.item import Field, Item from scrapy.pipelines.files import ( @@ -26,105 +32,167 @@ from scrapy.pipelines.files import ( S3FilesStore, ) from scrapy.settings import Settings -from scrapy.utils.test import ( - assert_gcs_environ, - get_crawler, - get_ftp_content_and_delete, - get_gcs_content_and_delete, - skip_if_no_boto, -) -from tests.mockserver import MockFTPServer +from scrapy.utils.asyncio import call_later +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.test import get_crawler +from tests.mockserver.ftp import MockFTPServer +from tests.utils.decorators import coroutine_test, inline_callbacks_test from .test_pipeline_media import _mocked_download_func -class FilesPipelineTestCase(unittest.TestCase): - def setUp(self): +def get_gcs_content_and_delete( + bucket: Any, path: str +) -> tuple[bytes, list[dict[str, str]], Any]: + from google.cloud import storage # noqa: PLC0415 + + client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) + bucket = client.get_bucket(bucket) + blob = bucket.get_blob(path) + content = blob.download_as_string() + acl = list(blob.acl) # loads acl before it will be deleted + bucket.delete_blob(path) + return content, acl, blob + + +def get_ftp_content_and_delete( + path: str, + host: str, + port: int, + username: str, + password: str, + use_active_mode: bool = False, +) -> bytes: + with FTP() as ftp: + ftp.connect(host, port) + ftp.login(username, password) + if use_active_mode: + ftp.set_pasv(False) + ftp_data: list[bytes] = [] + + def buffer_data(data: bytes) -> None: + ftp_data.append(data) + + ftp.retrbinary(f"RETR {path}", buffer_data) + dirname, filename = split(path) + ftp.cwd(dirname) + ftp.delete(filename) + return b"".join(ftp_data) + + +class DeferredFSFilesStore(FSFilesStore): + """A simple store with persist_file() returning a deferred.""" + + def persist_file(self, path, buf, info, meta=None, headers=None): + deferred = Deferred() + # short-hand super() doesn't work in nested functions + parent_persist_file = super().persist_file + + def cb(): + parent_persist_file(path, buf, info, meta=meta, headers=headers) + deferred.callback(None) + + call_later(0.5, cb) + return deferred + + +class TestFilesPipeline: + def setup_method(self): self.tempdir = mkdtemp() settings_dict = {"FILES_STORE": self.tempdir} - crawler = get_crawler(spidercls=None, settings_dict=settings_dict) + crawler = get_crawler(DefaultSpider, settings_dict=settings_dict) + crawler.spider = crawler._create_spider() + crawler.engine = MagicMock(download_async=_mocked_download_func) self.pipeline = FilesPipeline.from_crawler(crawler) - self.pipeline.download_func = _mocked_download_func - self.pipeline.open_spider(None) + self.pipeline.open_spider() - def tearDown(self): + def teardown_method(self): rmtree(self.tempdir) + def test_file_path_query_parameters(self): + file_path = self.pipeline.file_path + + req1 = Request("http://foo.bar/baz.txt?fizz") + assert file_path(req1) == "full/a2b4913a62f65445aeae2bac08cd8c3b41d7195e.txt" + + req2 = Request("http://foo.bar/get_img.foo?file=photo.jpg") + assert file_path(req2) == "full/7fc9461c9fd836515bea6983373097203a7d748e.jpg" + def test_file_path(self): file_path = self.pipeline.file_path - self.assertEqual( - file_path(Request("https://dev.mydeco.com/mydeco.pdf")), - "full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf", + assert ( + file_path(Request("https://dev.mydeco.com/mydeco.pdf")) + == "full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf" ) - self.assertEqual( + assert ( file_path( Request( "http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt" ) - ), - "full/4ce274dd83db0368bafd7e406f382ae088e39219.txt", + ) + == "full/4ce274dd83db0368bafd7e406f382ae088e39219.txt" ) - self.assertEqual( + assert ( file_path( Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc") - ), - "full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc", + ) + == "full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc" ) - self.assertEqual( + assert ( file_path( Request( - "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg" + "http://www.dfsonline.co.uk/get_prod_image?img=status_0907_mdm.jpg" ) - ), - "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg", + ) + == "full/c67f916ff9d542e822dedf38f9fcb146d1faba78.jpg" ) - self.assertEqual( - file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), - "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2", + assert ( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")) + == "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2" ) - self.assertEqual( - file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), - "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1", + assert ( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532")) + == "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1" ) - self.assertEqual( + assert ( file_path( Request("http://www.dorma.co.uk/images/product_details/2532"), response=Response("http://www.dorma.co.uk/images/product_details/2532"), info=object(), - ), - "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1", + ) + == "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1" ) - self.assertEqual( + assert ( file_path( Request( - "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha" + "http://www.dfsonline.co.uk/get_prod_image?img=status_0907_mdm.jpg.bohaha" ) - ), - "full/76c00cef2ef669ae65052661f68d451162829507", + ) + == "full/e75f2fa260521b56f6b6a867447b8002d00b5841" ) - self.assertEqual( + assert ( file_path( Request( "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\ //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y" ) - ), - "full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png", + ) + == "full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png" ) def test_fs_store(self): assert isinstance(self.pipeline.store, FSFilesStore) - self.assertEqual(self.pipeline.store.basedir, self.tempdir) + assert self.pipeline.store.basedir == self.tempdir path = "some/image/key.jpg" fullpath = Path(self.tempdir, "some", "image", "key.jpg") - self.assertEqual(self.pipeline.store._get_filesystem_path(path), fullpath) + assert self.pipeline.store._get_filesystem_path(path) == fullpath - @defer.inlineCallbacks - def test_file_not_expired(self): + @coroutine_test + async def test_file_not_expired(self): item_url = "http://example.com/file.pdf" item = _create_item_with_files(item_url) - patchers = [ + with ( mock.patch.object(FilesPipeline, "inc_stats", return_value=True), mock.patch.object( FSFilesStore, @@ -136,22 +204,16 @@ class FilesPipelineTestCase(unittest.TestCase): "get_media_requests", return_value=[_prepare_request_object(item_url)], ), - ] - for p in patchers: - p.start() + ): + result = await self.pipeline.process_item(item) + assert result["files"][0]["checksum"] == "abc" + assert result["files"][0]["status"] == "uptodate" - result = yield self.pipeline.process_item(item, None) - self.assertEqual(result["files"][0]["checksum"], "abc") - self.assertEqual(result["files"][0]["status"], "uptodate") - - for p in patchers: - p.stop() - - @defer.inlineCallbacks - def test_file_expired(self): + @coroutine_test + async def test_file_expired(self): item_url = "http://example.com/file2.pdf" item = _create_item_with_files(item_url) - patchers = [ + with ( mock.patch.object( FSFilesStore, "stat_file", @@ -167,22 +229,16 @@ class FilesPipelineTestCase(unittest.TestCase): return_value=[_prepare_request_object(item_url)], ), mock.patch.object(FilesPipeline, "inc_stats", return_value=True), - ] - for p in patchers: - p.start() + ): + result = await self.pipeline.process_item(item) + assert result["files"][0]["checksum"] != "abc" + assert result["files"][0]["status"] == "downloaded" - result = yield self.pipeline.process_item(item, None) - self.assertNotEqual(result["files"][0]["checksum"], "abc") - self.assertEqual(result["files"][0]["status"], "downloaded") - - for p in patchers: - p.stop() - - @defer.inlineCallbacks - def test_file_cached(self): + @coroutine_test + async def test_file_cached(self): item_url = "http://example.com/file3.pdf" item = _create_item_with_files(item_url) - patchers = [ + with ( mock.patch.object(FilesPipeline, "inc_stats", return_value=True), mock.patch.object( FSFilesStore, @@ -198,16 +254,33 @@ class FilesPipelineTestCase(unittest.TestCase): "get_media_requests", return_value=[_prepare_request_object(item_url, flags=["cached"])], ), - ] - for p in patchers: - p.start() + ): + result = await self.pipeline.process_item(item) + assert result["files"][0]["checksum"] != "abc" + assert result["files"][0]["status"] == "cached" - result = yield self.pipeline.process_item(item, None) - self.assertNotEqual(result["files"][0]["checksum"], "abc") - self.assertEqual(result["files"][0]["status"], "cached") + @coroutine_test + async def test_async_store(self) -> None: + """Test that async persist_file() works and is awaited.""" - for p in patchers: - p.stop() + self.pipeline.store = DeferredFSFilesStore(self.tempdir) + item_url = "http://example.com/file.pdf" + item = _create_item_with_files(item_url) + with ( + mock.patch.object(FilesPipeline, "inc_stats", return_value=True), + mock.patch.object( + FilesPipeline, + "get_media_requests", + return_value=[_prepare_request_object(item_url)], + ), + ): + result = await self.pipeline.process_item(item) + assert result["files"][0]["status"] == "downloaded" + assert result["files"][0]["checksum"] + # check that the file was written by persist_file() + path = Path(self.tempdir) / result["files"][0]["path"] + assert path.exists() + assert path.read_bytes() == b"data" def test_file_path_from_item(self): """ @@ -216,59 +289,79 @@ class FilesPipelineTestCase(unittest.TestCase): class CustomFilesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None, item=None): - return f'full/{item.get("path")}' + return f"full/{item.get('path')}" - file_path = CustomFilesPipeline.from_settings( - Settings({"FILES_STORE": self.tempdir}) + file_path = CustomFilesPipeline.from_crawler( + get_crawler(None, {"FILES_STORE": self.tempdir}) ).file_path - item = dict(path="path-to-store-file") + item = {"path": "path-to-store-file"} request = Request("http://example.com") - self.assertEqual(file_path(request, item=item), "full/path-to-store-file") + assert file_path(request, item=item) == "full/path-to-store-file" + + @pytest.mark.parametrize( + "bad_type", + [ + "http://example.com/file.pdf", + ("http://example.com/file.pdf",), + {"url": "http://example.com/file.pdf"}, + 123, + None, + ], + ) + def test_rejects_non_list_file_urls(self, tmp_path, bad_type): + pipeline = FilesPipeline.from_crawler( + get_crawler(None, {"FILES_STORE": str(tmp_path)}) + ) + item = ItemWithFiles() + item["file_urls"] = bad_type + + with pytest.raises(TypeError, match="file_urls must be a list of URLs"): + list(pipeline.get_media_requests(item, None)) -class FilesPipelineTestCaseFieldsMixin: - def setUp(self): - self.tempdir = mkdtemp() +class TestFilesPipelineFieldsMixin(ABC): + @property + @abstractmethod + def item_class(self) -> Any: + raise NotImplementedError - def tearDown(self): - rmtree(self.tempdir) - - def test_item_fields_default(self): + def test_item_fields_default(self, tmp_path): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", file_urls=[url]) - pipeline = FilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir})) + pipeline = FilesPipeline.from_crawler( + get_crawler(None, {"FILES_STORE": tmp_path}) + ) requests = list(pipeline.get_media_requests(item, None)) - self.assertEqual(requests[0].url, url) + assert requests[0].url == url results = [(True, {"url": url})] item = pipeline.item_completed(results, item, None) files = ItemAdapter(item).get("files") - self.assertEqual(files, [results[0][1]]) - self.assertIsInstance(item, self.item_class) + assert files == [results[0][1]] + assert isinstance(item, self.item_class) - def test_item_fields_override_settings(self): + def test_item_fields_override_settings(self, tmp_path): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", custom_file_urls=[url]) - pipeline = FilesPipeline.from_settings( - Settings( + pipeline = FilesPipeline.from_crawler( + get_crawler( + None, { - "FILES_STORE": self.tempdir, + "FILES_STORE": tmp_path, "FILES_URLS_FIELD": "custom_file_urls", "FILES_RESULT_FIELD": "custom_files", - } + }, ) ) requests = list(pipeline.get_media_requests(item, None)) - self.assertEqual(requests[0].url, url) + assert requests[0].url == url results = [(True, {"url": url})] item = pipeline.item_completed(results, item, None) custom_files = ItemAdapter(item).get("custom_files") - self.assertEqual(custom_files, [results[0][1]]) - self.assertIsInstance(item, self.item_class) + assert custom_files == [results[0][1]] + assert isinstance(item, self.item_class) -class FilesPipelineTestCaseFieldsDict( - FilesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestFilesPipelineFieldsDict(TestFilesPipelineFieldsMixin): item_class = dict @@ -282,9 +375,7 @@ class FilesPipelineTestItem(Item): custom_files = Field() -class FilesPipelineTestCaseFieldsItem( - FilesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestFilesPipelineFieldsItem(TestFilesPipelineFieldsMixin): item_class = FilesPipelineTestItem @@ -292,16 +383,14 @@ class FilesPipelineTestCaseFieldsItem( class FilesPipelineTestDataClass: name: str # default fields - file_urls: list = dataclasses.field(default_factory=list) - files: list = dataclasses.field(default_factory=list) + file_urls: list[str] = dataclasses.field(default_factory=list) + files: list[dict[str, str]] = dataclasses.field(default_factory=list) # overridden fields - custom_file_urls: list = dataclasses.field(default_factory=list) - custom_files: list = dataclasses.field(default_factory=list) + custom_file_urls: list[str] = dataclasses.field(default_factory=list) + custom_files: list[dict[str, str]] = dataclasses.field(default_factory=list) -class FilesPipelineTestCaseFieldsDataClass( - FilesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestFilesPipelineFieldsDataClass(TestFilesPipelineFieldsMixin): item_class = FilesPipelineTestDataClass @@ -309,20 +398,18 @@ class FilesPipelineTestCaseFieldsDataClass( class FilesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - file_urls: List[str] = attr.ib(default=lambda: []) - files: List[Dict[str, str]] = attr.ib(default=lambda: []) + file_urls: list[str] = attr.ib(default=list) + files: list[dict[str, str]] = attr.ib(default=list) # overridden fields - custom_file_urls: List[str] = attr.ib(default=lambda: []) - custom_files: List[Dict[str, str]] = attr.ib(default=lambda: []) + custom_file_urls: list[str] = attr.ib(default=list) + custom_files: list[dict[str, str]] = attr.ib(default=list) -class FilesPipelineTestCaseFieldsAttrsItem( - FilesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestFilesPipelineFieldsAttrsItem(TestFilesPipelineFieldsMixin): item_class = FilesPipelineTestAttrsItem -class FilesPipelineTestCaseCustomSettings(unittest.TestCase): +class TestFilesPipelineCustomSettings: default_cls_settings = { "EXPIRES": 90, "FILES_URLS_FIELD": "file_urls", @@ -334,13 +421,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): ("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field"), } - def setUp(self): - self.tempdir = mkdtemp() - - def tearDown(self): - rmtree(self.tempdir) - - def _generate_fake_settings(self, prefix=None): + def _generate_fake_settings(self, tmp_path, prefix=None): def random_string(): return "".join([chr(random.randint(97, 123)) for _ in range(10)]) @@ -348,7 +429,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): "FILES_EXPIRES": random.randint(100, 1000), "FILES_URLS_FIELD": random_string(), "FILES_RESULT_FIELD": random_string(), - "FILES_STORE": self.tempdir, + "FILES_STORE": tmp_path, } if not prefix: return settings @@ -366,47 +447,49 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): return UserDefinedFilePipeline - def test_different_settings_for_different_instances(self): + def test_different_settings_for_different_instances(self, tmp_path): """ If there are different instances with different settings they should keep different settings. """ - custom_settings = self._generate_fake_settings() - another_pipeline = FilesPipeline.from_settings(Settings(custom_settings)) - one_pipeline = FilesPipeline(self.tempdir) + custom_settings = self._generate_fake_settings(tmp_path) + another_pipeline = FilesPipeline.from_crawler( + get_crawler(None, custom_settings) + ) + one_pipeline = FilesPipeline(tmp_path, crawler=get_crawler(None)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: default_value = self.default_cls_settings[pipe_attr] - self.assertEqual(getattr(one_pipeline, pipe_attr), default_value) + assert getattr(one_pipeline, pipe_attr) == default_value custom_value = custom_settings[settings_attr] - self.assertNotEqual(default_value, custom_value) - self.assertEqual(getattr(another_pipeline, pipe_ins_attr), custom_value) + assert default_value != custom_value + assert getattr(another_pipeline, pipe_ins_attr) == custom_value - def test_subclass_attributes_preserved_if_no_settings(self): + def test_subclass_attributes_preserved_if_no_settings(self, tmp_path): """ If subclasses override class attributes and there are no special settings those values should be kept. """ pipe_cls = self._generate_fake_pipeline() - pipe = pipe_cls.from_settings(Settings({"FILES_STORE": self.tempdir})) - for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: + pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": tmp_path})) + for pipe_attr, _, pipe_ins_attr in self.file_cls_attr_settings_map: custom_value = getattr(pipe, pipe_ins_attr) - self.assertNotEqual(custom_value, self.default_cls_settings[pipe_attr]) - self.assertEqual(getattr(pipe, pipe_ins_attr), getattr(pipe, pipe_attr)) + assert custom_value != self.default_cls_settings[pipe_attr] + assert getattr(pipe, pipe_ins_attr) == getattr(pipe, pipe_attr) - def test_subclass_attrs_preserved_custom_settings(self): + def test_subclass_attrs_preserved_custom_settings(self, tmp_path): """ If file settings are defined but they are not defined for subclass settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline() - settings = self._generate_fake_settings() - pipeline = pipeline_cls.from_settings(Settings(settings)) + settings = self._generate_fake_settings(tmp_path) + pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: value = getattr(pipeline, pipe_ins_attr) setting_value = settings.get(settings_attr) - self.assertNotEqual(value, self.default_cls_settings[pipe_attr]) - self.assertEqual(value, setting_value) + assert value != self.default_cls_settings[pipe_attr] + assert value == setting_value - def test_no_custom_settings_for_subclasses(self): + def test_no_custom_settings_for_subclasses(self, tmp_path): """ If there are no settings for subclass and no subclass attributes, pipeline should use attributes of base class. @@ -415,15 +498,15 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): class UserDefinedFilesPipeline(FilesPipeline): pass - user_pipeline = UserDefinedFilesPipeline.from_settings( - Settings({"FILES_STORE": self.tempdir}) + user_pipeline = UserDefinedFilesPipeline.from_crawler( + get_crawler(None, {"FILES_STORE": tmp_path}) ) - for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: + for pipe_attr, _, pipe_ins_attr in self.file_cls_attr_settings_map: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = self.default_cls_settings.get(pipe_attr.upper()) - self.assertEqual(getattr(user_pipeline, pipe_ins_attr), custom_value) + assert getattr(user_pipeline, pipe_ins_attr) == custom_value - def test_custom_settings_for_subclasses(self): + def test_custom_settings_for_subclasses(self, tmp_path): """ If there are custom settings for subclass and NO class attributes, pipeline should use custom settings. @@ -433,86 +516,87 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): pass prefix = UserDefinedFilesPipeline.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) - user_pipeline = UserDefinedFilesPipeline.from_settings(Settings(settings)) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) + user_pipeline = UserDefinedFilesPipeline.from_crawler( + get_crawler(None, settings) + ) for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = settings.get(prefix + "_" + settings_attr) - self.assertNotEqual(custom_value, self.default_cls_settings[pipe_attr]) - self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value) + assert custom_value != self.default_cls_settings[pipe_attr] + assert getattr(user_pipeline, pipe_inst_attr) == custom_value - def test_custom_settings_and_class_attrs_for_subclasses(self): + def test_custom_settings_and_class_attrs_for_subclasses(self, tmp_path): """ If there are custom settings for subclass AND class attributes setting keys are preferred and override attributes. """ pipeline_cls = self._generate_fake_pipeline() prefix = pipeline_cls.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) - user_pipeline = pipeline_cls.from_settings(Settings(settings)) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) + user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for ( pipe_cls_attr, settings_attr, pipe_inst_attr, ) in self.file_cls_attr_settings_map: custom_value = settings.get(prefix + "_" + settings_attr) - self.assertNotEqual(custom_value, self.default_cls_settings[pipe_cls_attr]) - self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value) + assert custom_value != self.default_cls_settings[pipe_cls_attr] + assert getattr(user_pipeline, pipe_inst_attr) == custom_value - def test_cls_attrs_with_DEFAULT_prefix(self): + def test_cls_attrs_with_DEFAULT_prefix(self, tmp_path): class UserDefinedFilesPipeline(FilesPipeline): DEFAULT_FILES_RESULT_FIELD = "this" DEFAULT_FILES_URLS_FIELD = "that" - pipeline = UserDefinedFilesPipeline.from_settings( - Settings({"FILES_STORE": self.tempdir}) + pipeline = UserDefinedFilesPipeline.from_crawler( + get_crawler(None, {"FILES_STORE": tmp_path}) ) - self.assertEqual( - pipeline.files_result_field, - UserDefinedFilesPipeline.DEFAULT_FILES_RESULT_FIELD, + assert ( + pipeline.files_result_field + == UserDefinedFilesPipeline.DEFAULT_FILES_RESULT_FIELD ) - self.assertEqual( - pipeline.files_urls_field, UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD + assert ( + pipeline.files_urls_field + == UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD ) - def test_user_defined_subclass_default_key_names(self): + def test_user_defined_subclass_default_key_names(self, tmp_path): """Test situation when user defines subclass of FilesPipeline, but uses attribute names for default pipeline (without prefixing them with pipeline class name). """ - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) class UserPipe(FilesPipeline): pass - pipeline_cls = UserPipe.from_settings(Settings(settings)) + pipeline_cls = UserPipe.from_crawler(get_crawler(None, settings)) - for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: + for _, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: expected_value = settings.get(settings_attr) - self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), expected_value) + assert getattr(pipeline_cls, pipe_inst_attr) == expected_value - def test_file_pipeline_using_pathlike_objects(self): + def test_file_pipeline_using_pathlike_objects(self, tmp_path): class CustomFilesPipelineWithPathLikeDir(FilesPipeline): def file_path(self, request, response=None, info=None, *, item=None): return Path("subdir") / Path(request.url).name - pipeline = CustomFilesPipelineWithPathLikeDir.from_settings( - Settings({"FILES_STORE": Path("./Temp")}) + pipeline = CustomFilesPipelineWithPathLikeDir.from_crawler( + get_crawler(None, {"FILES_STORE": tmp_path}) ) request = Request("http://example.com/image01.jpg") - self.assertEqual(pipeline.file_path(request), Path("subdir/image01.jpg")) + assert pipeline.file_path(request) == Path("subdir/image01.jpg") - def test_files_store_constructor_with_pathlike_object(self): - path = Path("./FileDir") - fs_store = FSFilesStore(path) - self.assertEqual(fs_store.basedir, str(path)) + def test_files_store_constructor_with_pathlike_object(self, tmp_path): + fs_store = FSFilesStore(tmp_path) + assert fs_store.basedir == str(tmp_path) -class TestS3FilesStore(unittest.TestCase): - @defer.inlineCallbacks +@pytest.mark.requires_botocore +class TestS3FilesStore: + @inline_callbacks_test def test_persist(self): - skip_if_no_boto() - bucket = "mybucket" key = "export.csv" uri = f"s3://{bucket}/{key}" @@ -522,7 +606,7 @@ class TestS3FilesStore(unittest.TestCase): content_type = "image/png" store = S3FilesStore(uri) - from botocore.stub import Stubber + from botocore.stub import Stubber # noqa: PLC0415 with Stubber(store.s3_client) as stub: stub.add_response( @@ -548,18 +632,11 @@ class TestS3FilesStore(unittest.TestCase): ) stub.assert_no_pending_responses() - self.assertEqual( - buffer.method_calls, - [ - mock.call.seek(0), - # The call to read does not happen with Stubber - ], - ) + # The call to read does not happen with Stubber + assert buffer.method_calls == [mock.call.seek(0)] - @defer.inlineCallbacks + @inline_callbacks_test def test_stat(self): - skip_if_no_boto() - bucket = "mybucket" key = "export.csv" uri = f"s3://{bucket}/{key}" @@ -567,7 +644,7 @@ class TestS3FilesStore(unittest.TestCase): last_modified = datetime(2019, 12, 1) store = S3FilesStore(uri) - from botocore.stub import Stubber + from botocore.stub import Stubber # noqa: PLC0415 with Stubber(store.s3_client) as stub: stub.add_response( @@ -583,24 +660,23 @@ class TestS3FilesStore(unittest.TestCase): ) file_stats = yield store.stat_file("", info=None) - self.assertEqual( - file_stats, - { - "checksum": checksum, - "last_modified": last_modified.timestamp(), - }, - ) + assert file_stats == { + "checksum": checksum, + "last_modified": last_modified.timestamp(), + } stub.assert_no_pending_responses() -class TestGCSFilesStore(unittest.TestCase): - @defer.inlineCallbacks +@pytest.mark.skipif( + "GCS_PROJECT_ID" not in os.environ, reason="GCS_PROJECT_ID not found" +) +class TestGCSFilesStore: + @inline_callbacks_test def test_persist(self): - assert_gcs_environ() uri = os.environ.get("GCS_TEST_FILE_URI") if not uri: - raise unittest.SkipTest("No GCS URI available for testing") + pytest.skip("No GCS URI available for testing") data = b"TestGCSFilesStore: \xe2\x98\x83" buf = BytesIO(data) meta = {"foo": "bar"} @@ -610,59 +686,63 @@ class TestGCSFilesStore(unittest.TestCase): expected_policy = {"role": "READER", "entity": "allAuthenticatedUsers"} yield store.persist_file(path, buf, info=None, meta=meta, headers=None) s = yield store.stat_file(path, info=None) - self.assertIn("last_modified", s) - self.assertIn("checksum", s) - self.assertEqual(s["checksum"], "cdcda85605e46d0af6110752770dce3c") + assert "last_modified" in s + assert "checksum" in s + assert s["checksum"] == "cdcda85605e46d0af6110752770dce3c" u = urlparse(uri) content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:] + path) - self.assertEqual(content, data) - self.assertEqual(blob.metadata, {"foo": "bar"}) - self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL) - self.assertEqual(blob.content_type, "application/octet-stream") - self.assertIn(expected_policy, acl) + assert content == data + assert blob.metadata == {"foo": "bar"} + assert blob.cache_control == GCSFilesStore.CACHE_CONTROL + assert blob.content_type == "application/octet-stream" + assert expected_policy in acl - @defer.inlineCallbacks + @inline_callbacks_test def test_blob_path_consistency(self): """Test to make sure that paths used to store files is the same as the one used to get already uploaded files. """ - assert_gcs_environ() try: - import google.cloud.storage # noqa + import google.cloud.storage # noqa: F401,PLC0415 except ModuleNotFoundError: - raise unittest.SkipTest("google-cloud-storage is not installed") - else: - with mock.patch("google.cloud.storage") as _: - with mock.patch("scrapy.pipelines.files.time") as _: - uri = "gs://my_bucket/my_prefix/" - store = GCSFilesStore(uri) - store.bucket = mock.Mock() - path = "full/my_data.txt" - yield store.persist_file( - path, mock.Mock(), info=None, meta=None, headers=None - ) - yield store.stat_file(path, info=None) - expected_blob_path = store.prefix + path - store.bucket.blob.assert_called_with(expected_blob_path) - store.bucket.get_blob.assert_called_with(expected_blob_path) + pytest.skip("google-cloud-storage is not installed") + with ( + mock.patch("google.cloud.storage"), + mock.patch("scrapy.pipelines.files.time"), + ): + uri = "gs://my_bucket/my_prefix/" + store = GCSFilesStore(uri) + store.bucket = mock.Mock() + path = "full/my_data.txt" + yield store.persist_file( + path, mock.Mock(), info=None, meta=None, headers=None + ) + yield store.stat_file(path, info=None) + expected_blob_path = store.prefix + path + store.bucket.blob.assert_called_with(expected_blob_path) + store.bucket.get_blob.assert_called_with(expected_blob_path) -class TestFTPFileStore(unittest.TestCase): - @defer.inlineCallbacks +class TestFTPFileStore: + @inline_callbacks_test def test_persist(self): data = b"TestFTPFilesStore: \xe2\x98\x83" buf = BytesIO(data) meta = {"foo": "bar"} path = "full/filename" with MockFTPServer() as ftp_server: + # normally set via FilesPipeline.from_crawler() + FTPFilesStore.FTP_USERNAME = "anonymous" + FTPFilesStore.FTP_PASSWORD = "guest" + store = FTPFilesStore(ftp_server.url("/")) empty_dict = yield store.stat_file(path, info=None) - self.assertEqual(empty_dict, {}) + assert empty_dict == {} yield store.persist_file(path, buf, info=None, meta=meta, headers=None) stat = yield store.stat_file(path, info=None) - self.assertIn("last_modified", stat) - self.assertIn("checksum", stat) - self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6") + assert "last_modified" in stat + assert "checksum" in stat + assert stat["checksum"] == "d113d66b2ec7258724a268bd88eef6b6" path = f"{store.basedir}/{path}" content = get_ftp_content_and_delete( path, @@ -672,7 +752,7 @@ class TestFTPFileStore(unittest.TestCase): store.password, store.USE_ACTIVE_MODE, ) - self.assertEqual(data, content) + assert data == content class ItemWithFiles(Item): @@ -680,14 +760,66 @@ class ItemWithFiles(Item): files = Field() -def _create_item_with_files(*files): +def _create_item_with_files(*files: str) -> ItemWithFiles: item = ItemWithFiles() item["file_urls"] = files return item -def _prepare_request_object(item_url, flags=None): +def _prepare_request_object(item_url: str, flags: list[str] | None = None) -> Request: return Request( item_url, meta={"response": Response(item_url, status=200, body=b"data", flags=flags)}, ) + + +# this is separate from the one in test_pipeline_media.py to specifically test FilesPipeline subclasses +class TestBuildFromCrawler: + def setup_method(self): + self.tempdir = mkdtemp() + self.crawler = get_crawler(None, {"FILES_STORE": self.tempdir}) + + def teardown_method(self): + rmtree(self.tempdir) + + def test_simple(self): + class Pipeline(FilesPipeline): + pass + + with warnings.catch_warnings(record=True) as w: + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + assert len(w) == 0 + assert pipe.store + + def test_has_from_crawler_and_init(self): + class Pipeline(FilesPipeline): + _from_crawler_called = False + + @classmethod + def from_crawler(cls, crawler): + settings = crawler.settings + store_uri = settings["FILES_STORE"] + o = cls(store_uri, crawler=crawler) + o._from_crawler_called = True + return o + + with warnings.catch_warnings(record=True) as w: + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + assert len(w) == 0 + assert pipe.store + assert pipe._from_crawler_called + + +@pytest.mark.parametrize("store", [None, ""]) +def test_files_pipeline_raises_notconfigured_when_files_store_invalid(store): + settings = Settings() + settings.clear() + settings.set("FILES_STORE", store, priority="cmdline") + crawler = get_crawler(settings_dict=settings) + + with pytest.raises(NotConfigured): + FilesPipeline.from_crawler(crawler) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 2e2e06b89..38662348f 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -1,121 +1,116 @@ +from __future__ import annotations + import dataclasses -import hashlib import io import random -import warnings +from abc import ABC, abstractmethod from shutil import rmtree from tempfile import mkdtemp -from typing import Dict, List, Optional -from unittest.mock import patch +from typing import Any import attr +import pytest from itemadapter import ItemAdapter -from twisted.trial import unittest -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.item import Field, Item -from scrapy.pipelines.images import ImageException, ImagesPipeline, NoimagesDrop -from scrapy.settings import Settings -from scrapy.utils.python import to_bytes +from scrapy.pipelines.images import ImageException, ImagesPipeline +from scrapy.utils.test import get_crawler -skip_pillow: Optional[str] try: from PIL import Image except ImportError: - skip_pillow = ( - "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" + pytest.skip( + "Missing Python Imaging Library, install https://pypi.org/pypi/Pillow", + allow_module_level=True, ) else: encoders = {"jpeg_encoder", "jpeg_decoder"} if not encoders.issubset(set(Image.core.__dict__)): # type: ignore[attr-defined] - skip_pillow = "Missing JPEG encoders" - else: - skip_pillow = None + pytest.skip("Missing JPEG encoders", allow_module_level=True) -class ImagesPipelineTestCase(unittest.TestCase): - skip = skip_pillow - - def setUp(self): +class TestImagesPipeline: + def setup_method(self): self.tempdir = mkdtemp() - self.pipeline = ImagesPipeline(self.tempdir) + crawler = get_crawler() + self.pipeline = ImagesPipeline(self.tempdir, crawler=crawler) - def tearDown(self): + def teardown_method(self): rmtree(self.tempdir) def test_file_path(self): file_path = self.pipeline.file_path - self.assertEqual( - file_path(Request("https://dev.mydeco.com/mydeco.gif")), - "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg", + assert ( + file_path(Request("https://dev.mydeco.com/mydeco.gif")) + == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" ) - self.assertEqual( + assert ( file_path( Request( "http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg" ) - ), - "full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg", + ) + == "full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg" ) - self.assertEqual( + assert ( file_path( Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif") - ), - "full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg", + ) + == "full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg" ) - self.assertEqual( + assert ( file_path( Request( "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg" ) - ), - "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg", + ) + == "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg" ) - self.assertEqual( - file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), - "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg", + assert ( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")) + == "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg" ) - self.assertEqual( - file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), - "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg", + assert ( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532")) + == "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg" ) - self.assertEqual( + assert ( file_path( Request("http://www.dorma.co.uk/images/product_details/2532"), response=Response("http://www.dorma.co.uk/images/product_details/2532"), info=object(), - ), - "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg", + ) + == "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg" ) def test_thumbnail_name(self): thumb_path = self.pipeline.thumb_path name = "50" - self.assertEqual( - thumb_path(Request("file:///tmp/foo.jpg"), name), - "thumbs/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg", + assert ( + thumb_path(Request("file:///tmp/foo.jpg"), name) + == "thumbs/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg" ) - self.assertEqual( - thumb_path(Request("file://foo.png"), name), - "thumbs/50/e55b765eba0ec7348e50a1df496040449071b96a.jpg", + assert ( + thumb_path(Request("file://foo.png"), name) + == "thumbs/50/e55b765eba0ec7348e50a1df496040449071b96a.jpg" ) - self.assertEqual( - thumb_path(Request("file:///tmp/foo"), name), - "thumbs/50/0329ad83ebb8e93ea7c7906d46e9ed55f7349a50.jpg", + assert ( + thumb_path(Request("file:///tmp/foo"), name) + == "thumbs/50/0329ad83ebb8e93ea7c7906d46e9ed55f7349a50.jpg" ) - self.assertEqual( - thumb_path(Request("file:///tmp/some.name/foo"), name), - "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg", + assert ( + thumb_path(Request("file:///tmp/some.name/foo"), name) + == "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg" ) - self.assertEqual( + assert ( thumb_path( Request("file:///tmp/some.name/foo"), name, response=Response("file:///tmp/some.name/foo"), info=object(), - ), - "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg", + ) + == "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg" ) def test_thumbnail_name_from_item(self): @@ -129,13 +124,13 @@ class ImagesPipelineTestCase(unittest.TestCase): ): return f"thumb/{thumb_id}/{item.get('path')}" - thumb_path = CustomImagesPipeline.from_settings( - Settings({"IMAGES_STORE": self.tempdir}) + thumb_path = CustomImagesPipeline.from_crawler( + get_crawler(None, {"IMAGES_STORE": self.tempdir}) ).thumb_path - item = dict(path="path-to-store-file") + item = {"path": "path-to-store-file"} request = Request("http://example.com") - self.assertEqual( - thumb_path(request, "small", item=item), "thumb/small/path-to-store-file" + assert ( + thumb_path(request, "small", item=item) == "thumb/small/path-to-store-file" ) def test_get_images_exception(self): @@ -151,20 +146,20 @@ class ImagesPipelineTestCase(unittest.TestCase): resp3 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf3.getvalue()) req = Request(url="https://dev.mydeco.com/mydeco.gif") - with self.assertRaises(ImageException): + with pytest.raises(ImageException): next(self.pipeline.get_images(response=resp1, request=req, info=object())) - with self.assertRaises(ImageException): + with pytest.raises(ImageException): next(self.pipeline.get_images(response=resp2, request=req, info=object())) - with self.assertRaises(ImageException): + with pytest.raises(ImageException): next(self.pipeline.get_images(response=resp3, request=req, info=object())) - def test_get_images_new(self): + def test_get_images(self): self.pipeline.min_width = 0 self.pipeline.min_height = 0 self.pipeline.thumbs = {"small": (20, 20)} orig_im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0)) - orig_thumb, orig_thumb_buf = _create_image("JPEG", "RGB", (20, 20), (0, 0, 0)) + _, orig_thumb_buf = _create_image("JPEG", "RGB", (20, 20), (0, 0, 0)) resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue()) req = Request(url="https://dev.mydeco.com/mydeco.gif") @@ -173,199 +168,131 @@ class ImagesPipelineTestCase(unittest.TestCase): ) path, new_im, new_buf = next(get_images_gen) - self.assertEqual(path, "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg") - self.assertEqual(orig_im, new_im) - self.assertEqual(buf.getvalue(), new_buf.getvalue()) + assert path == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" + assert orig_im.copy() == new_im + assert buf.getvalue() == new_buf.getvalue() - thumb_path, thumb_img, thumb_buf = next(get_images_gen) - self.assertEqual( - thumb_path, "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" - ) - self.assertEqual(thumb_img, thumb_img) - self.assertEqual(orig_thumb_buf.getvalue(), thumb_buf.getvalue()) + thumb_path, _, thumb_buf = next(get_images_gen) + assert thumb_path == "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" + assert orig_thumb_buf.getvalue() == thumb_buf.getvalue() + + def test_get_transposed_images(self): + orig_im = Image.new("RGB", (2, 2), (0, 0, 0)) + orig_im.putpixel((1, 1), (255, 0, 0)) + exif = orig_im.getexif() + exif[274] = 3 + buf = io.BytesIO() + orig_im.save(buf, "PNG", exif=exif) + buf.seek(0) - def test_get_images_old(self): - self.pipeline.thumbs = {"small": (20, 20)} - orig_im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0)) resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue()) req = Request(url="https://dev.mydeco.com/mydeco.gif") - def overridden_convert_image(image, size=None): - im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0)) - return im, buf + get_images_gen = self.pipeline.get_images( + response=resp, request=req, info=object() + ) - with patch.object(self.pipeline, "convert_image", overridden_convert_image): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - get_images_gen = self.pipeline.get_images( - response=resp, request=req, info=object() - ) - path, new_im, new_buf = next(get_images_gen) - self.assertEqual( - path, "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" - ) - self.assertEqual(orig_im.mode, new_im.mode) - self.assertEqual(orig_im.getcolors(), new_im.getcolors()) - self.assertEqual(buf.getvalue(), new_buf.getvalue()) + path, new_im, _ = next(get_images_gen) + assert path == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" + assert new_im.getpixel((0, 0)) == (255, 0, 0) - thumb_path, thumb_img, thumb_buf = next(get_images_gen) - self.assertEqual( - thumb_path, - "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg", - ) - self.assertEqual(orig_im.mode, thumb_img.mode) - self.assertEqual(orig_im.getcolors(), thumb_img.getcolors()) - self.assertEqual(buf.getvalue(), thumb_buf.getvalue()) - - expected_warning_msg = ( - ".convert_image() method overridden in a deprecated way, " - "overridden method does not accept response_body argument." - ) - self.assertEqual( - len( - [ - warning - for warning in w - if expected_warning_msg in str(warning.message) - ] - ), - 1, - ) - - def test_convert_image_old(self): - # tests for old API - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - SIZE = (100, 100) - # straight forward case: RGB and JPEG - COLOUR = (0, 127, 255) - im, _ = _create_image("JPEG", "RGB", SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) - - # check that thumbnail keep image ratio - thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) - self.assertEqual(thumbnail.mode, "RGB") - self.assertEqual(thumbnail.size, (10, 10)) - - # transparency case: RGBA and PNG - COLOUR = (0, 127, 255, 50) - im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - # transparency case with palette: P and PNG - COLOUR = (0, 127, 255, 50) - im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR) - im = im.convert("P") - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - # ensure that we received deprecation warnings - expected_warning_msg = ".convert_image() method called in a deprecated way" - self.assertTrue( - len( - [ - warning - for warning in w - if expected_warning_msg in str(warning.message) - ] - ) - == 4 - ) - - def test_convert_image_new(self): - # tests for new API + def test_convert_image(self): SIZE = (100, 100) # straight forward case: RGB and JPEG COLOUR = (0, 127, 255) im, buf = _create_image("JPEG", "RGB", SIZE, COLOUR) converted, converted_buf = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + assert converted.mode == "RGB" + assert converted.getcolors() == [(10000, COLOUR)] # check that we don't convert JPEGs again - self.assertEqual(converted_buf, buf) + assert converted_buf == buf # check that thumbnail keep image ratio thumbnail, _ = self.pipeline.convert_image( converted, size=(10, 25), response_body=converted_buf ) - self.assertEqual(thumbnail.mode, "RGB") - self.assertEqual(thumbnail.size, (10, 10)) + assert thumbnail.mode == "RGB" + assert thumbnail.size == (10, 10) # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR) converted, _ = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + assert converted.mode == "RGB" + assert converted.getcolors() == [(10000, (205, 230, 255))] # transparency case with palette: P and PNG COLOUR = (0, 127, 255, 50) im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR) im = im.convert("P") converted, _ = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + assert converted.mode == "RGB" + assert converted.getcolors() == [(10000, (205, 230, 255))] + + @pytest.mark.parametrize( + "bad_type", + [ + "http://example.com/file.jpg", + ("http://example.com/file.jpg",), + {"url": "http://example.com/file.jpg"}, + 123, + None, + ], + ) + def test_rejects_non_list_image_urls(self, tmp_path, bad_type): + pipeline = ImagesPipeline.from_crawler( + get_crawler(None, {"IMAGES_STORE": str(tmp_path)}) + ) + item = ImagesPipelineTestItem() + item["image_urls"] = bad_type + + with pytest.raises(TypeError, match="image_urls must be a list of URLs"): + list(pipeline.get_media_requests(item, None)) -class DeprecatedImagesPipeline(ImagesPipeline): - def file_key(self, url): - return self.image_key(url) - - def image_key(self, url): - image_guid = hashlib.sha1(to_bytes(url)).hexdigest() - return f"empty/{image_guid}.jpg" - - def thumb_key(self, url, thumb_id): - thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest() - return f"thumbsup/{thumb_id}/{thumb_guid}.jpg" - - -class ImagesPipelineTestCaseFieldsMixin: - skip = skip_pillow +class TestImagesPipelineFieldsMixin(ABC): + @property + @abstractmethod + def item_class(self) -> Any: + raise NotImplementedError def test_item_fields_default(self): url = "http://www.example.com/images/1.jpg" item = self.item_class(name="item1", image_urls=[url]) - pipeline = ImagesPipeline.from_settings( - Settings({"IMAGES_STORE": "s3://example/images/"}) + pipeline = ImagesPipeline.from_crawler( + get_crawler(None, {"IMAGES_STORE": "s3://example/images/"}) ) requests = list(pipeline.get_media_requests(item, None)) - self.assertEqual(requests[0].url, url) + assert requests[0].url == url results = [(True, {"url": url})] item = pipeline.item_completed(results, item, None) images = ItemAdapter(item).get("images") - self.assertEqual(images, [results[0][1]]) - self.assertIsInstance(item, self.item_class) + assert images == [results[0][1]] + assert isinstance(item, self.item_class) def test_item_fields_override_settings(self): url = "http://www.example.com/images/1.jpg" item = self.item_class(name="item1", custom_image_urls=[url]) - pipeline = ImagesPipeline.from_settings( - Settings( + pipeline = ImagesPipeline.from_crawler( + get_crawler( + None, { "IMAGES_STORE": "s3://example/images/", "IMAGES_URLS_FIELD": "custom_image_urls", "IMAGES_RESULT_FIELD": "custom_images", - } + }, ) ) requests = list(pipeline.get_media_requests(item, None)) - self.assertEqual(requests[0].url, url) + assert requests[0].url == url results = [(True, {"url": url})] item = pipeline.item_completed(results, item, None) custom_images = ItemAdapter(item).get("custom_images") - self.assertEqual(custom_images, [results[0][1]]) - self.assertIsInstance(item, self.item_class) + assert custom_images == [results[0][1]] + assert isinstance(item, self.item_class) -class ImagesPipelineTestCaseFieldsDict( - ImagesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestImagesPipelineFieldsDict(TestImagesPipelineFieldsMixin): item_class = dict @@ -379,9 +306,7 @@ class ImagesPipelineTestItem(Item): custom_images = Field() -class ImagesPipelineTestCaseFieldsItem( - ImagesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestImagesPipelineFieldsItem(TestImagesPipelineFieldsMixin): item_class = ImagesPipelineTestItem @@ -389,16 +314,14 @@ class ImagesPipelineTestCaseFieldsItem( class ImagesPipelineTestDataClass: name: str # default fields - image_urls: list = dataclasses.field(default_factory=list) - images: list = dataclasses.field(default_factory=list) + image_urls: list[str] = dataclasses.field(default_factory=list) + images: list[dict[str, str]] = dataclasses.field(default_factory=list) # overridden fields - custom_image_urls: list = dataclasses.field(default_factory=list) - custom_images: list = dataclasses.field(default_factory=list) + custom_image_urls: list[str] = dataclasses.field(default_factory=list) + custom_images: list[dict[str, str]] = dataclasses.field(default_factory=list) -class ImagesPipelineTestCaseFieldsDataClass( - ImagesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestImagesPipelineFieldsDataClass(TestImagesPipelineFieldsMixin): item_class = ImagesPipelineTestDataClass @@ -406,22 +329,18 @@ class ImagesPipelineTestCaseFieldsDataClass( class ImagesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - image_urls: List[str] = attr.ib(default=lambda: []) - images: List[Dict[str, str]] = attr.ib(default=lambda: []) + image_urls: list[str] = attr.ib(default=list) + images: list[dict[str, str]] = attr.ib(default=list) # overridden fields - custom_image_urls: List[str] = attr.ib(default=lambda: []) - custom_images: List[Dict[str, str]] = attr.ib(default=lambda: []) + custom_image_urls: list[str] = attr.ib(default=list) + custom_images: list[dict[str, str]] = attr.ib(default=list) -class ImagesPipelineTestCaseFieldsAttrsItem( - ImagesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestImagesPipelineFieldsAttrsItem(TestImagesPipelineFieldsMixin): item_class = ImagesPipelineTestAttrsItem -class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): - skip = skip_pillow - +class TestImagesPipelineCustomSettings: img_cls_attribute_names = [ # Pipeline attribute names with corresponding setting names. ("EXPIRES", "IMAGES_EXPIRES"), @@ -433,22 +352,16 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): ] # This should match what is defined in ImagesPipeline. - default_pipeline_settings = dict( - MIN_WIDTH=0, - MIN_HEIGHT=0, - EXPIRES=90, - THUMBS={}, - IMAGES_URLS_FIELD="image_urls", - IMAGES_RESULT_FIELD="images", - ) + default_pipeline_settings = { + "MIN_WIDTH": 0, + "MIN_HEIGHT": 0, + "EXPIRES": 90, + "THUMBS": {}, + "IMAGES_URLS_FIELD": "image_urls", + "IMAGES_RESULT_FIELD": "images", + } - def setUp(self): - self.tempdir = mkdtemp() - - def tearDown(self): - rmtree(self.tempdir) - - def _generate_fake_settings(self, prefix=None): + def _generate_fake_settings(self, tmp_path, prefix=None): """ :param prefix: string for setting keys :return: dictionary of image pipeline settings @@ -459,7 +372,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): settings = { "IMAGES_EXPIRES": random.randint(100, 1000), - "IMAGES_STORE": self.tempdir, + "IMAGES_STORE": tmp_path, "IMAGES_RESULT_FIELD": random_string(), "IMAGES_URLS_FIELD": random_string(), "IMAGES_MIN_WIDTH": random.randint(1, 1000), @@ -496,54 +409,55 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): return UserDefinedImagePipeline - def test_different_settings_for_different_instances(self): + def test_different_settings_for_different_instances(self, tmp_path): """ If there are two instances of ImagesPipeline class with different settings, they should have different settings. """ - custom_settings = self._generate_fake_settings() - default_settings = Settings() - default_sts_pipe = ImagesPipeline(self.tempdir, settings=default_settings) - user_sts_pipe = ImagesPipeline.from_settings(Settings(custom_settings)) + custom_settings = self._generate_fake_settings(tmp_path) + default_sts_pipe = ImagesPipeline(tmp_path, crawler=get_crawler(None)) + user_sts_pipe = ImagesPipeline.from_crawler(get_crawler(None, custom_settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: expected_default_value = self.default_pipeline_settings.get(pipe_attr) custom_value = custom_settings.get(settings_attr) - self.assertNotEqual(expected_default_value, custom_value) - self.assertEqual( - getattr(default_sts_pipe, pipe_attr.lower()), expected_default_value + assert expected_default_value != custom_value + assert ( + getattr(default_sts_pipe, pipe_attr.lower()) == expected_default_value ) - self.assertEqual(getattr(user_sts_pipe, pipe_attr.lower()), custom_value) + assert getattr(user_sts_pipe, pipe_attr.lower()) == custom_value - def test_subclass_attrs_preserved_default_settings(self): + def test_subclass_attrs_preserved_default_settings(self, tmp_path): """ If image settings are not defined at all subclass of ImagePipeline takes values from class attributes. """ pipeline_cls = self._generate_fake_pipeline_subclass() - pipeline = pipeline_cls.from_settings(Settings({"IMAGES_STORE": self.tempdir})) - for pipe_attr, settings_attr in self.img_cls_attribute_names: + pipeline = pipeline_cls.from_crawler( + get_crawler(None, {"IMAGES_STORE": tmp_path}) + ) + for pipe_attr, _ in self.img_cls_attribute_names: # Instance attribute (lowercase) must be equal to class attribute (uppercase). attr_value = getattr(pipeline, pipe_attr.lower()) - self.assertNotEqual(attr_value, self.default_pipeline_settings[pipe_attr]) - self.assertEqual(attr_value, getattr(pipeline, pipe_attr)) + assert attr_value != self.default_pipeline_settings[pipe_attr] + assert attr_value == getattr(pipeline, pipe_attr) - def test_subclass_attrs_preserved_custom_settings(self): + def test_subclass_attrs_preserved_custom_settings(self, tmp_path): """ If image settings are defined but they are not defined for subclass default values taken from settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline_subclass() - settings = self._generate_fake_settings() - pipeline = pipeline_cls.from_settings(Settings(settings)) + settings = self._generate_fake_settings(tmp_path) + pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Instance attribute (lowercase) must be equal to # value defined in settings. value = getattr(pipeline, pipe_attr.lower()) - self.assertNotEqual(value, self.default_pipeline_settings[pipe_attr]) + assert value != self.default_pipeline_settings[pipe_attr] setings_value = settings.get(settings_attr) - self.assertEqual(value, setings_value) + assert value == setings_value - def test_no_custom_settings_for_subclasses(self): + def test_no_custom_settings_for_subclasses(self, tmp_path): """ If there are no settings for subclass and no subclass attributes, pipeline should use attributes of base class. @@ -552,15 +466,15 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): class UserDefinedImagePipeline(ImagesPipeline): pass - user_pipeline = UserDefinedImagePipeline.from_settings( - Settings({"IMAGES_STORE": self.tempdir}) + user_pipeline = UserDefinedImagePipeline.from_crawler( + get_crawler(None, {"IMAGES_STORE": tmp_path}) ) - for pipe_attr, settings_attr in self.img_cls_attribute_names: + for pipe_attr, _ in self.img_cls_attribute_names: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = self.default_pipeline_settings.get(pipe_attr.upper()) - self.assertEqual(getattr(user_pipeline, pipe_attr.lower()), custom_value) + assert getattr(user_pipeline, pipe_attr.lower()) == custom_value - def test_custom_settings_for_subclasses(self): + def test_custom_settings_for_subclasses(self, tmp_path): """ If there are custom settings for subclass and NO class attributes, pipeline should use custom settings. @@ -570,81 +484,66 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): pass prefix = UserDefinedImagePipeline.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) - user_pipeline = UserDefinedImagePipeline.from_settings(Settings(settings)) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) + user_pipeline = UserDefinedImagePipeline.from_crawler( + get_crawler(None, settings) + ) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = settings.get(prefix + "_" + settings_attr) - self.assertNotEqual(custom_value, self.default_pipeline_settings[pipe_attr]) - self.assertEqual(getattr(user_pipeline, pipe_attr.lower()), custom_value) + assert custom_value != self.default_pipeline_settings[pipe_attr] + assert getattr(user_pipeline, pipe_attr.lower()) == custom_value - def test_custom_settings_and_class_attrs_for_subclasses(self): + def test_custom_settings_and_class_attrs_for_subclasses(self, tmp_path): """ If there are custom settings for subclass AND class attributes setting keys are preferred and override attributes. """ pipeline_cls = self._generate_fake_pipeline_subclass() prefix = pipeline_cls.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) - user_pipeline = pipeline_cls.from_settings(Settings(settings)) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) + user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: custom_value = settings.get(prefix + "_" + settings_attr) - self.assertNotEqual(custom_value, self.default_pipeline_settings[pipe_attr]) - self.assertEqual(getattr(user_pipeline, pipe_attr.lower()), custom_value) + assert custom_value != self.default_pipeline_settings[pipe_attr] + assert getattr(user_pipeline, pipe_attr.lower()) == custom_value - def test_cls_attrs_with_DEFAULT_prefix(self): + def test_cls_attrs_with_DEFAULT_prefix(self, tmp_path): class UserDefinedImagePipeline(ImagesPipeline): DEFAULT_IMAGES_URLS_FIELD = "something" DEFAULT_IMAGES_RESULT_FIELD = "something_else" - pipeline = UserDefinedImagePipeline.from_settings( - Settings({"IMAGES_STORE": self.tempdir}) + pipeline = UserDefinedImagePipeline.from_crawler( + get_crawler(None, {"IMAGES_STORE": tmp_path}) ) - self.assertEqual( - pipeline.images_result_field, - UserDefinedImagePipeline.DEFAULT_IMAGES_RESULT_FIELD, + assert ( + pipeline.images_result_field + == UserDefinedImagePipeline.DEFAULT_IMAGES_RESULT_FIELD ) - self.assertEqual( - pipeline.images_urls_field, - UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD, + assert ( + pipeline.images_urls_field + == UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD ) - def test_user_defined_subclass_default_key_names(self): + def test_user_defined_subclass_default_key_names(self, tmp_path): """Test situation when user defines subclass of ImagePipeline, but uses attribute names for default pipeline (without prefixing them with pipeline class name). """ - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) class UserPipe(ImagesPipeline): pass - pipeline_cls = UserPipe.from_settings(Settings(settings)) + pipeline_cls = UserPipe.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: expected_value = settings.get(settings_attr) - self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), expected_value) + assert getattr(pipeline_cls, pipe_attr.lower()) == expected_value -class NoimagesDropTestCase(unittest.TestCase): - def test_deprecation_warning(self): - arg = str() - with warnings.catch_warnings(record=True) as w: - NoimagesDrop(arg) - self.assertEqual(len(w), 1) - self.assertEqual(w[0].category, ScrapyDeprecationWarning) - with warnings.catch_warnings(record=True) as w: - - class SubclassedNoimagesDrop(NoimagesDrop): - pass - - SubclassedNoimagesDrop(arg) - self.assertEqual(len(w), 1) - self.assertEqual(w[0].category, ScrapyDeprecationWarning) - - -def _create_image(format, *a, **kw): +def _create_image(format_, *a, **kw): buf = io.BytesIO() - Image.new(*a, **kw).save(buf, format) + Image.new(*a, **kw).save(buf, format_) buf.seek(0) return Image.open(buf), buf diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 820484565..44df0bdd4 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,107 +1,71 @@ -import io -from typing import Optional +from __future__ import annotations +import warnings +from unittest.mock import MagicMock + +import pytest from testfixtures import LogCapture -from twisted.internet import reactor -from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure -from twisted.trial import unittest from scrapy import signals +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import FileException -from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline -from scrapy.settings import Settings -from scrapy.spiders import Spider +from scrapy.utils.defer import _defer_sleep_async from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all +from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler - -try: - from PIL import Image # noqa: imported just to check for the import error -except ImportError: - skip_pillow: Optional[ - str - ] = "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" -else: - skip_pillow = None +from tests.utils.decorators import coroutine_test -def _mocked_download_func(request, info): +async def _mocked_download_func(request): assert request.callback is NO_CALLBACK response = request.meta.get("response") - return response() if callable(response) else response + if callable(response): + response = await response() + if isinstance(response, Exception): + raise response + return response -class BaseMediaPipelineTestCase(unittest.TestCase): - pipeline_class = MediaPipeline +class UserDefinedPipeline(MediaPipeline): + def media_to_download(self, request, info, *, item=None): + pass + + def get_media_requests(self, item, info): + pass + + def media_downloaded(self, response, request, info, *, item=None): + return {} + + def media_failed(self, failure, request, info): + failure.raiseException() + + def file_path(self, request, response=None, info=None, *, item=None): + return "" + + +class TestBaseMediaPipeline: + pipeline_class = UserDefinedPipeline settings = None - def setUp(self): - spider_cls = Spider - self.spider = spider_cls("media.com") - crawler = get_crawler(spider_cls, self.settings) + def setup_method(self): + crawler = get_crawler(DefaultSpider, self.settings) + crawler.spider = crawler._create_spider() + crawler.engine = MagicMock(download_async=_mocked_download_func) self.pipe = self.pipeline_class.from_crawler(crawler) - self.pipe.download_func = _mocked_download_func - self.pipe.open_spider(self.spider) + self.pipe.open_spider() self.info = self.pipe.spiderinfo self.fingerprint = crawler.request_fingerprinter.fingerprint - def tearDown(self): + def teardown_method(self): for name, signal in vars(signals).items(): if not name.startswith("_"): disconnect_all(signal) - def test_default_media_to_download(self): - request = Request("http://url") - assert self.pipe.media_to_download(request, self.info) is None - - def test_default_get_media_requests(self): - item = dict(name="name") - assert self.pipe.get_media_requests(item, self.info) is None - - def test_default_media_downloaded(self): - request = Request("http://url") - response = Response("http://url", body=b"") - assert self.pipe.media_downloaded(response, request, self.info) is response - - def test_default_media_failed(self): - request = Request("http://url") - fail = Failure(Exception()) - assert self.pipe.media_failed(fail, request, self.info) is fail - - def test_default_item_completed(self): - item = dict(name="name") - assert self.pipe.item_completed([], item, self.info) is item - - # Check that failures are logged by default - fail = Failure(Exception()) - results = [(True, 1), (False, fail)] - - with LogCapture() as log: - new_item = self.pipe.item_completed(results, item, self.info) - - assert new_item is item - assert len(log.records) == 1 - record = log.records[0] - assert record.levelname == "ERROR" - self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail)) - - # disable failure logging and check again - self.pipe.LOG_FAILED_RESULTS = False - with LogCapture() as log: - new_item = self.pipe.item_completed(results, item, self.info) - assert new_item is item - assert len(log.records) == 0 - - @inlineCallbacks - def test_default_process_item(self): - item = dict(name="name") - new_item = yield self.pipe.process_item(item, self.spider) - assert new_item is item - def test_modify_media_request(self): request = Request("http://url") self.pipe._modify_media_request(request) @@ -155,9 +119,9 @@ class BaseMediaPipelineTestCase(unittest.TestCase): failure = Failure(file_exc) # The Failure should encapsulate a FileException ... - self.assertEqual(failure.value, file_exc) + assert failure.value == file_exc # ... and it should have the StopIteration exception set as its context - self.assertEqual(failure.value.__context__, def_gen_return_exc) + assert failure.value.__context__ == def_gen_return_exc # Let's calculate the request fingerprint and fake some runtime data... fp = self.fingerprint(request) @@ -168,17 +132,47 @@ class BaseMediaPipelineTestCase(unittest.TestCase): # When calling the method that caches the Request's result ... self.pipe._cache_result_and_execute_waiters(failure, fp, info) # ... it should store the Twisted Failure ... - self.assertEqual(info.downloaded[fp], failure) + assert info.downloaded[fp] == failure # ... encapsulating the original FileException ... - self.assertEqual(info.downloaded[fp].value, file_exc) + assert info.downloaded[fp].value == file_exc # ... but it should not store the StopIteration exception on its context context = getattr(info.downloaded[fp].value, "__context__", None) - self.assertIsNone(context) + assert context is None + + def test_default_item_completed(self): + item = {"name": "name"} + assert self.pipe.item_completed([], item, self.info) is item + + # Check that failures are logged by default + fail = Failure(Exception()) + results = [(True, 1), (False, fail)] + + with LogCapture() as log: + new_item = self.pipe.item_completed(results, item, self.info) + + assert new_item is item + assert len(log.records) == 1 + record = log.records[0] + assert record.levelname == "ERROR" + assert record.exc_info == failure_to_exc_info(fail) + + # disable failure logging and check again + self.pipe.LOG_FAILED_RESULTS = False + with LogCapture() as log: + new_item = self.pipe.item_completed(results, item, self.info) + assert new_item is item + assert len(log.records) == 0 + + @coroutine_test + async def test_default_process_item(self): + item = {"name": "name"} + new_item = await self.pipe.process_item(item) + assert new_item is item -class MockedMediaPipeline(MediaPipeline): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) +class MockedMediaPipeline(UserDefinedPipeline): + def __init__(self, *args, crawler=None, **kwargs): + super().__init__(*args, crawler=crawler, **kwargs) self._mockcalled = [] def download(self, request, info): @@ -210,229 +204,195 @@ class MockedMediaPipeline(MediaPipeline): return item -class MediaPipelineTestCase(BaseMediaPipelineTestCase): - pipeline_class = MockedMediaPipeline +class AsyncMediaDownloadedPipeline(MockedMediaPipeline): + async def media_downloaded(self, response, request, info, *, item=None): + return super().media_downloaded(response, request, info) - def _callback(self, result): - self.pipe._mockcalled.append("request_callback") - return result + +class TestMediaPipeline(TestBaseMediaPipeline): + pipeline_class = MockedMediaPipeline def _errback(self, result): self.pipe._mockcalled.append("request_errback") return result - @inlineCallbacks - def test_result_succeed(self): + @coroutine_test + async def test_result_succeed(self): rsp = Response("http://url1") req = Request( "http://url1", - meta=dict(response=rsp), - callback=self._callback, + meta={"response": rsp}, errback=self._errback, ) - item = dict(requests=req) - new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, rsp)]) - self.assertEqual( - self.pipe._mockcalled, - [ - "get_media_requests", - "media_to_download", - "media_downloaded", - "request_callback", - "item_completed", - ], - ) + item = {"requests": req} + new_item = await self.pipe.process_item(item) + assert new_item["results"] == [(True, {})] + assert self.pipe._mockcalled == [ + "get_media_requests", + "media_to_download", + "media_downloaded", + "item_completed", + ] - @inlineCallbacks - def test_result_failure(self): + @coroutine_test + async def test_result_failure(self): self.pipe.LOG_FAILED_RESULTS = False - fail = Failure(Exception()) + exc = Exception("foo") req = Request( "http://url1", - meta=dict(response=fail), - callback=self._callback, + meta={"response": exc}, errback=self._errback, ) - item = dict(requests=req) - new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(False, fail)]) - self.assertEqual( - self.pipe._mockcalled, - [ - "get_media_requests", - "media_to_download", - "media_failed", - "request_errback", - "item_completed", - ], - ) + item = {"requests": req} + new_item = await self.pipe.process_item(item) + assert len(new_item["results"]) == 1 + assert new_item["results"][0][0] is False + assert isinstance(new_item["results"][0][1], Failure) + assert new_item["results"][0][1].value == exc + assert self.pipe._mockcalled == [ + "get_media_requests", + "media_to_download", + "media_failed", + "request_errback", + "item_completed", + ] - @inlineCallbacks - def test_mix_of_success_and_failure(self): + @coroutine_test + async def test_mix_of_success_and_failure(self): self.pipe.LOG_FAILED_RESULTS = False rsp1 = Response("http://url1") - req1 = Request("http://url1", meta=dict(response=rsp1)) - fail = Failure(Exception()) - req2 = Request("http://url2", meta=dict(response=fail)) - item = dict(requests=[req1, req2]) - new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, rsp1), (False, fail)]) + req1 = Request("http://url1", meta={"response": rsp1}) + exc = Exception("foo") + req2 = Request("http://url2", meta={"response": exc}) + item = {"requests": [req1, req2]} + new_item = await self.pipe.process_item(item) + assert len(new_item["results"]) == 2 + assert new_item["results"][0] == (True, {}) + assert new_item["results"][1][0] is False + assert isinstance(new_item["results"][1][1], Failure) + assert new_item["results"][1][1].value == exc m = self.pipe._mockcalled # only once - self.assertEqual(m[0], "get_media_requests") # first hook called - self.assertEqual(m.count("get_media_requests"), 1) - self.assertEqual(m.count("item_completed"), 1) - self.assertEqual(m[-1], "item_completed") # last hook called + assert m[0] == "get_media_requests" # first hook called + assert m.count("get_media_requests") == 1 + assert m.count("item_completed") == 1 + assert m[-1] == "item_completed" # last hook called # twice, one per request - self.assertEqual(m.count("media_to_download"), 2) + assert m.count("media_to_download") == 2 # one to handle success and other for failure - self.assertEqual(m.count("media_downloaded"), 1) - self.assertEqual(m.count("media_failed"), 1) + assert m.count("media_downloaded") == 1 + assert m.count("media_failed") == 1 - @inlineCallbacks - def test_get_media_requests(self): + @coroutine_test + async def test_get_media_requests(self): # returns single Request (without callback) req = Request("http://url") - item = dict(requests=req) # pass a single item - new_item = yield self.pipe.process_item(item, self.spider) + item = {"requests": req} # pass a single item + new_item = await self.pipe.process_item(item) assert new_item is item - self.assertIn(self.fingerprint(req), self.info.downloaded) + assert self.fingerprint(req) in self.info.downloaded # returns iterable of Requests req1 = Request("http://url1") req2 = Request("http://url2") - item = dict(requests=iter([req1, req2])) - new_item = yield self.pipe.process_item(item, self.spider) + item = {"requests": iter([req1, req2])} + new_item = await self.pipe.process_item(item) assert new_item is item assert self.fingerprint(req1) in self.info.downloaded assert self.fingerprint(req2) in self.info.downloaded - @inlineCallbacks - def test_results_are_cached_across_multiple_items(self): + @coroutine_test + async def test_results_are_cached_across_multiple_items(self): rsp1 = Response("http://url1") - req1 = Request("http://url1", meta=dict(response=rsp1)) - item = dict(requests=req1) - new_item = yield self.pipe.process_item(item, self.spider) - self.assertTrue(new_item is item) - self.assertEqual(new_item["results"], [(True, rsp1)]) + req1 = Request("http://url1", meta={"response": rsp1}) + item = {"requests": req1} + new_item = await self.pipe.process_item(item) + assert new_item is item + assert new_item["results"] == [(True, {})] # rsp2 is ignored, rsp1 must be in results because request fingerprints are the same req2 = Request( - req1.url, meta=dict(response=Response("http://donot.download.me")) + req1.url, meta={"response": Response("http://donot.download.me")} ) - item = dict(requests=req2) - new_item = yield self.pipe.process_item(item, self.spider) - self.assertTrue(new_item is item) - self.assertEqual(self.fingerprint(req1), self.fingerprint(req2)) - self.assertEqual(new_item["results"], [(True, rsp1)]) + item = {"requests": req2} + new_item = await self.pipe.process_item(item) + assert new_item is item + assert self.fingerprint(req1) == self.fingerprint(req2) + assert new_item["results"] == [(True, {})] - @inlineCallbacks - def test_results_are_cached_for_requests_of_single_item(self): + @coroutine_test + async def test_results_are_cached_for_requests_of_single_item(self): rsp1 = Response("http://url1") - req1 = Request("http://url1", meta=dict(response=rsp1)) + req1 = Request("http://url1", meta={"response": rsp1}) req2 = Request( - req1.url, meta=dict(response=Response("http://donot.download.me")) + req1.url, meta={"response": Response("http://donot.download.me")} ) - item = dict(requests=[req1, req2]) - new_item = yield self.pipe.process_item(item, self.spider) - self.assertTrue(new_item is item) - self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) + item = {"requests": [req1, req2]} + new_item = await self.pipe.process_item(item) + assert new_item is item + assert new_item["results"] == [(True, {}), (True, {})] - @inlineCallbacks - def test_wait_if_request_is_downloading(self): + @coroutine_test + async def test_wait_if_request_is_downloading(self): def _check_downloading(response): fp = self.fingerprint(req1) - self.assertTrue(fp in self.info.downloading) - self.assertTrue(fp in self.info.waiting) - self.assertTrue(fp not in self.info.downloaded) - self.assertEqual(len(self.info.waiting[fp]), 2) + assert fp in self.info.downloading + assert fp in self.info.waiting + assert fp not in self.info.downloaded + assert len(self.info.waiting[fp]) == 2 return response rsp1 = Response("http://url") - def rsp1_func(): - dfd = Deferred().addCallback(_check_downloading) - reactor.callLater(0.1, dfd.callback, rsp1) - return dfd + async def rsp1_func(): + await _defer_sleep_async() + _check_downloading(rsp1) - def rsp2_func(): - self.fail("it must cache rsp1 result and must not try to redownload") + async def rsp2_func(): + pytest.fail("it must cache rsp1 result and must not try to redownload") - req1 = Request("http://url", meta=dict(response=rsp1_func)) - req2 = Request(req1.url, meta=dict(response=rsp2_func)) - item = dict(requests=[req1, req2]) - new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) + req1 = Request("http://url", meta={"response": rsp1_func}) + req2 = Request(req1.url, meta={"response": rsp2_func}) + item = {"requests": [req1, req2]} + new_item = await self.pipe.process_item(item) + assert new_item["results"] == [(True, {}), (True, {})] - @inlineCallbacks - def test_use_media_to_download_result(self): - req = Request("http://url", meta=dict(result="ITSME", response=self.fail)) - item = dict(requests=req) - new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, "ITSME")]) - self.assertEqual( - self.pipe._mockcalled, - ["get_media_requests", "media_to_download", "item_completed"], + @coroutine_test + async def test_use_media_to_download_result(self): + req = Request("http://url", meta={"result": "ITSME"}) + item = {"requests": req} + new_item = await self.pipe.process_item(item) + assert new_item["results"] == [(True, "ITSME")] + assert self.pipe._mockcalled == [ + "get_media_requests", + "media_to_download", + "item_completed", + ] + + def test_key_for_pipe(self): + assert ( + self.pipe._key_for_pipe("IMAGES", base_class_name="MediaPipeline") + == "MOCKEDMEDIAPIPELINE_IMAGES" ) -class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._mockcalled = [] +class TestAsyncMediaDownloaded(TestMediaPipeline): + pipeline_class = AsyncMediaDownloadedPipeline - def get_media_requests(self, item, info): - item_url = item["image_urls"][0] - output_img = io.BytesIO() - img = Image.new("RGB", (60, 30), color="red") - img.save(output_img, format="JPEG") - return Request( - item_url, - meta={ - "response": Response(item_url, status=200, body=output_img.getvalue()) - }, + def test_key_for_pipe(self): + assert ( + self.pipe._key_for_pipe("IMAGES", base_class_name="MediaPipeline") + == "ASYNCMEDIADOWNLOADEDPIPELINE_IMAGES" ) - def inc_stats(self, *args, **kwargs): - return True - def media_to_download(self, request, info): - self._mockcalled.append("media_to_download") - return super().media_to_download(request, info) - - def media_downloaded(self, response, request, info): - self._mockcalled.append("media_downloaded") - return super().media_downloaded(response, request, info) - - def file_downloaded(self, response, request, info): - self._mockcalled.append("file_downloaded") - return super().file_downloaded(response, request, info) - - def file_path(self, request, response=None, info=None): - self._mockcalled.append("file_path") - return super().file_path(request, response, info) - - def thumb_path(self, request, thumb_id, response=None, info=None): - self._mockcalled.append("thumb_path") - return super().thumb_path(request, thumb_id, response, info) - - def get_images(self, response, request, info): - self._mockcalled.append("get_images") - return super().get_images(response, request, info) - - def image_downloaded(self, response, request, info): - self._mockcalled.append("image_downloaded") - return super().image_downloaded(response, request, info) - - -class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): +class TestMediaPipelineAllowRedirectSettings: def _assert_request_no3xx(self, pipeline_class, settings): - pipe = pipeline_class(settings=Settings(settings)) + pipe = pipeline_class(crawler=get_crawler(None, settings)) request = Request("http://url") pipe._modify_media_request(request) - self.assertIn("handle_httpstatus_list", request.meta) + assert "handle_httpstatus_list" in request.meta for status, check in [ (200, True), # These are the status codes we want @@ -448,23 +408,115 @@ class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): (500, True), ]: if check: - self.assertIn(status, request.meta["handle_httpstatus_list"]) + assert status in request.meta["handle_httpstatus_list"] else: - self.assertNotIn(status, request.meta["handle_httpstatus_list"]) - - def test_standard_setting(self): - self._assert_request_no3xx(MediaPipeline, {"MEDIA_ALLOW_REDIRECTS": True}) + assert status not in request.meta["handle_httpstatus_list"] def test_subclass_standard_setting(self): - class UserDefinedPipeline(MediaPipeline): - pass - self._assert_request_no3xx(UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True}) def test_subclass_specific_setting(self): - class UserDefinedPipeline(MediaPipeline): - pass - self._assert_request_no3xx( UserDefinedPipeline, {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True} ) + + +class TestBuildFromCrawler: + def setup_method(self): + self.crawler = get_crawler(None, {"FILES_STORE": "/foo"}) + + def test_simple(self): + class Pipeline(UserDefinedPipeline): + pass + + with warnings.catch_warnings(record=True) as w: + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + assert len(w) == 0 + + def test_has_from_crawler_and_init(self): + class Pipeline(UserDefinedPipeline): + _from_crawler_called = False + + def __init__(self, store_uri, settings, *, crawler): + super().__init__(crawler=crawler) + self._init_called = True + + @classmethod + def from_crawler(cls, crawler): + settings = crawler.settings + store_uri = settings["FILES_STORE"] + o = cls(store_uri, settings=settings, crawler=crawler) + o._from_crawler_called = True + return o + + with warnings.catch_warnings(record=True) as w: + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + assert len(w) == 0 + assert pipe._from_crawler_called + assert pipe._init_called + + def test_has_from_crawler(self): + class Pipeline(UserDefinedPipeline): + _from_crawler_called = False + + @classmethod + def from_crawler(cls, crawler): + settings = crawler.settings + o = super().from_crawler(crawler) + o._from_crawler_called = True + o.store_uri = settings["FILES_STORE"] + return o + + with warnings.catch_warnings(record=True) as w: + pipe = Pipeline.from_crawler(self.crawler) + # this and the next assert will fail as MediaPipeline.from_crawler() wasn't called + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + assert len(w) == 0 + assert pipe._from_crawler_called + + +class MediaFailedFailurePipeline(MockedMediaPipeline): + def media_failed(self, failure, request, info): + self._mockcalled.append("media_failed") + return failure # deprecated + + +class TestMediaFailedFailure(TestBaseMediaPipeline): + """Test that media_failed() can return a failure instead of raising.""" + + pipeline_class = MediaFailedFailurePipeline + + def _errback(self, result): + self.pipe._mockcalled.append("request_errback") + return result + + @coroutine_test + async def test_result_failure(self): + self.pipe.LOG_FAILED_RESULTS = False + exc = Exception("foo") + req = Request( + "http://url1", + meta={"response": exc}, + errback=self._errback, + ) + item = {"requests": req} + with pytest.warns( + ScrapyDeprecationWarning, match="media_failed returned a Failure instance" + ): + new_item = await self.pipe.process_item(item) + assert len(new_item["results"]) == 1 + assert new_item["results"][0][0] is False + assert isinstance(new_item["results"][0][1], Failure) + assert new_item["results"][0][1].value == exc + assert self.pipe._mockcalled == [ + "get_media_requests", + "media_to_download", + "media_failed", + "request_errback", + "item_completed", + ] diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 5ab288c1a..e65b5bc32 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -1,17 +1,35 @@ import asyncio +from typing import Any -from pytest import mark -from twisted.internet import defer -from twisted.internet.defer import Deferred -from twisted.trial import unittest +import pytest +from twisted.internet.defer import Deferred, fail, succeed from scrapy import Request, Spider, signals +from scrapy.crawler import Crawler +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.pipelines import ItemPipelineManager +from scrapy.utils.asyncio import call_later +from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future +from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_from_asyncio_queue -from tests.mockserver import MockServer +from tests.mockserver.http import MockServer +from tests.utils.decorators import coroutine_test class SimplePipeline: + def process_item(self, item): + item["pipeline_passed"] = True + return item + + +class DeprecatedSpiderArgPipeline: + def open_spider(self, spider): + pass + + def close_spider(self, spider): + pass + def process_item(self, item, spider): item["pipeline_passed"] = True return item @@ -22,7 +40,13 @@ class DeferredPipeline: item["pipeline_passed"] = True return item - def process_item(self, item, spider): + def open_spider(self): + return succeed(None) + + def close_spider(self): + return succeed(None) + + def process_item(self, item): d = Deferred() d.addCallback(self.cb) d.callback(item) @@ -30,22 +54,19 @@ class DeferredPipeline: class AsyncDefPipeline: - async def process_item(self, item, spider): + async def process_item(self, item): d = Deferred() - from twisted.internet import reactor - - reactor.callLater(0, d.callback, None) + call_later(0, d.callback, None) await maybe_deferred_to_future(d) item["pipeline_passed"] = True return item class AsyncDefAsyncioPipeline: - async def process_item(self, item, spider): + async def process_item(self, item): d = Deferred() - from twisted.internet import reactor - - reactor.callLater(0, d.callback, None) + loop = asyncio.get_event_loop() + loop.call_later(0, d.callback, None) await deferred_to_future(d) await asyncio.sleep(0.2) item["pipeline_passed"] = await get_from_asyncio_queue(True) @@ -53,7 +74,7 @@ class AsyncDefAsyncioPipeline: class AsyncDefNotAsyncioPipeline: - async def process_item(self, item, spider): + async def process_item(self, item): d1 = Deferred() from twisted.internet import reactor @@ -66,66 +87,467 @@ class AsyncDefNotAsyncioPipeline: return item +class ProcessItemExceptionPipeline: + def process_item(self, item): + raise ValueError("process_item error") + + +class ProcessItemExceptionDeferredPipeline: + def process_item(self, item): + return fail(ValueError("process_item error")) + + +class ProcessItemExceptionAsyncPipeline: + async def process_item(self, item): + raise ValueError("process_item error") + + +class OpenSpiderExceptionPipeline: + def open_spider(self): + raise ValueError("open_spider error") + + +class OpenSpiderExceptionDeferredPipeline: + def open_spider(self): + return fail(ValueError("open_spider error")) + + +class OpenSpiderExceptionAsyncPipeline: + async def open_spider(self): + raise ValueError("open_spider error") + + class ItemSpider(Spider): name = "itemspider" - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): return {"field": 42} -class PipelineTestCase(unittest.TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() - - def tearDown(self): - self.mockserver.__exit__(None, None, None) - +class TestPipeline: def _on_item_scraped(self, item): - self.assertIsInstance(item, dict) - self.assertTrue(item.get("pipeline_passed")) + assert isinstance(item, dict) + assert item.get("pipeline_passed") self.items.append(item) - def _create_crawler(self, pipeline_class): + def _create_crawler(self, pipeline_class: type) -> Crawler: settings = { "ITEM_PIPELINES": {pipeline_class: 1}, } crawler = get_crawler(ItemSpider, settings) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) - self.items = [] + self.items: list[Any] = [] return crawler - @defer.inlineCallbacks - def test_simple_pipeline(self): - crawler = self._create_crawler(SimplePipeline) - yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 1) + @pytest.mark.parametrize( + "pipeline_class", + [ + SimplePipeline, + AsyncDefPipeline, + pytest.param(AsyncDefAsyncioPipeline, marks=pytest.mark.only_asyncio), + pytest.param( + AsyncDefNotAsyncioPipeline, marks=pytest.mark.only_not_asyncio + ), + ], + ) + @coroutine_test + async def test_pipeline(self, mockserver: MockServer, pipeline_class: type) -> None: + crawler = self._create_crawler(pipeline_class) + await crawler.crawl_async(mockserver=mockserver) + assert len(self.items) == 1 - @defer.inlineCallbacks - def test_deferred_pipeline(self): + @coroutine_test + async def test_pipeline_deferred(self, mockserver: MockServer) -> None: crawler = self._create_crawler(DeferredPipeline) - yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 1) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match="DeferredPipeline.open_spider returned a Deferred", + ), + pytest.warns( + ScrapyDeprecationWarning, + match="DeferredPipeline.close_spider returned a Deferred", + ), + pytest.warns( + ScrapyDeprecationWarning, + match="DeferredPipeline.process_item returned a Deferred", + ), + ): + await crawler.crawl_async(mockserver=mockserver) + assert len(self.items) == 1 - @defer.inlineCallbacks - def test_asyncdef_pipeline(self): - crawler = self._create_crawler(AsyncDefPipeline) - yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 1) + @coroutine_test + async def test_deprecated_spider_arg(self, mockserver: MockServer) -> None: + crawler = self._create_crawler(DeprecatedSpiderArgPipeline) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.process_item\(\) requires a spider argument", + ), + ): + await crawler.crawl_async(mockserver=mockserver) - @mark.only_asyncio() - @defer.inlineCallbacks - def test_asyncdef_asyncio_pipeline(self): - crawler = self._create_crawler(AsyncDefAsyncioPipeline) - yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 1) + assert len(self.items) == 1 - @mark.only_not_asyncio() - @defer.inlineCallbacks - def test_asyncdef_not_asyncio_pipeline(self): - crawler = self._create_crawler(AsyncDefNotAsyncioPipeline) - yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 1) + @pytest.mark.parametrize( + "pipeline_class", + [ + ProcessItemExceptionPipeline, + pytest.param( + ProcessItemExceptionDeferredPipeline, + marks=pytest.mark.filterwarnings( + "ignore::scrapy.exceptions.ScrapyDeprecationWarning" + ), + ), + ProcessItemExceptionAsyncPipeline, + ], + ) + @coroutine_test + async def test_process_item_exception( + self, + caplog: pytest.LogCaptureFixture, + mockserver: MockServer, + pipeline_class: type, + ) -> None: + crawler = self._create_crawler(pipeline_class) + await crawler.crawl_async(mockserver=mockserver) + assert "Error processing {'field': 42}" in caplog.text + assert "process_item error" in caplog.text + + @pytest.mark.parametrize( + "pipeline_class", + [ + OpenSpiderExceptionPipeline, + pytest.param( + OpenSpiderExceptionDeferredPipeline, + marks=pytest.mark.filterwarnings( + "ignore::scrapy.exceptions.ScrapyDeprecationWarning" + ), + ), + OpenSpiderExceptionAsyncPipeline, + ], + ) + @coroutine_test + async def test_open_spider_exception( + self, mockserver: MockServer, pipeline_class: type + ) -> None: + crawler = self._create_crawler(pipeline_class) + with pytest.raises(ValueError, match="open_spider error"): + await crawler.crawl_async(mockserver=mockserver) + + +class TestCustomPipelineManager: + @coroutine_test + async def test_deprecated_process_item_spider_arg(self) -> None: + class CustomPipelineManager(ItemPipelineManager): + def process_item(self, item: Any, spider: Spider) -> Deferred[Any]: # pylint: disable=useless-parent-delegation + return super().process_item(item, spider) + + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() + itemproc = CustomPipelineManager.from_crawler(crawler) + with pytest.warns( + ScrapyDeprecationWarning, + match=r"CustomPipelineManager.process_item\(\) is deprecated, use process_item_async\(\)", + ): + itemproc.process_item({}, crawler.spider) + + @coroutine_test + async def test_integration_recommended(self, mockserver: MockServer) -> None: + class CustomPipelineManager(ItemPipelineManager): + async def process_item_async(self, item): + return await super().process_item_async(item) + + items = [] + + def _on_item_scraped(item): + assert isinstance(item, dict) + assert item.get("pipeline_passed") + items.append(item) + + crawler = get_crawler( + ItemSpider, + { + "ITEM_PROCESSOR": CustomPipelineManager, + "ITEM_PIPELINES": {SimplePipeline: 1}, + }, + ) + crawler.spider = crawler._create_spider() + crawler.signals.connect(_on_item_scraped, signals.item_scraped) + await crawler.crawl_async(mockserver=mockserver) + + assert len(items) == 1 + + @coroutine_test + async def test_integration_no_async_subclass(self, mockserver: MockServer) -> None: + class CustomPipelineManager(ItemPipelineManager): + def open_spider(self, spider): + with pytest.warns( + ScrapyDeprecationWarning, + match=r"CustomPipelineManager.open_spider\(\) is deprecated, use open_spider_async\(\)", + ): + return super().open_spider(spider) + + def close_spider(self, spider): + with pytest.warns( + ScrapyDeprecationWarning, + match=r"CustomPipelineManager.close_spider\(\) is deprecated, use close_spider_async\(\)", + ): + return super().close_spider(spider) + + def process_item(self, item, spider): + with pytest.warns( + ScrapyDeprecationWarning, + match=r"CustomPipelineManager.process_item\(\) is deprecated, use process_item_async\(\)", + ): + return super().process_item(item, spider) + + items = [] + + def _on_item_scraped(item): + assert isinstance(item, dict) + assert item.get("pipeline_passed") + items.append(item) + + crawler = get_crawler( + ItemSpider, + { + "ITEM_PROCESSOR": CustomPipelineManager, + "ITEM_PIPELINES": {SimplePipeline: 1}, + }, + ) + crawler.spider = crawler._create_spider() + crawler.signals.connect(_on_item_scraped, signals.item_scraped) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"CustomPipelineManager overrides open_spider\(\) but doesn't override open_spider_async\(\)", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"CustomPipelineManager overrides close_spider\(\) but doesn't override close_spider_async\(\)", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"CustomPipelineManager overrides process_item\(\) but doesn't override process_item_async\(\)", + ), + ): + await crawler.crawl_async(mockserver=mockserver) + + assert len(items) == 1 + + @coroutine_test + async def test_integration_no_async_not_subclass( + self, mockserver: MockServer + ) -> None: + class CustomPipelineManager: + def __init__(self, crawler): + self.pipelines = [ + p() + for p in build_component_list( + crawler.settings.getwithbase("ITEM_PIPELINES") + ) + ] + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def open_spider(self, spider): + return succeed(None) + + def close_spider(self, spider): + return succeed(None) + + def process_item(self, item, spider): + for pipeline in self.pipelines: + item = pipeline.process_item(item) + return succeed(item) + + items = [] + + def _on_item_scraped(item): + assert isinstance(item, dict) + assert item.get("pipeline_passed") + items.append(item) + + crawler = get_crawler( + ItemSpider, + { + "ITEM_PROCESSOR": CustomPipelineManager, + "ITEM_PIPELINES": {SimplePipeline: 1}, + }, + ) + crawler.spider = crawler._create_spider() + crawler.signals.connect(_on_item_scraped, signals.item_scraped) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"CustomPipelineManager doesn't define a open_spider_async\(\) method", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"CustomPipelineManager doesn't define a close_spider_async\(\) method", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"CustomPipelineManager doesn't define a process_item_async\(\) method", + ), + ): + await crawler.crawl_async(mockserver=mockserver) + + assert len(items) == 1 + + +class TestMiddlewareManagerSpider: + """Tests for the deprecated spider arg handling in MiddlewareManager. + + Here because MiddlewareManager doesn't have methods that could take a spider arg.""" + + @pytest.fixture + def crawler(self) -> Crawler: + return get_crawler(Spider) + + @coroutine_test + async def test_deprecated_spider_arg_no_crawler_spider( + self, crawler: Crawler + ) -> None: + """Crawler is provided, but doesn't have a spider, the methods raise an exception. + The instance passed to a deprecated method is ignored.""" + mwman = ItemPipelineManager(crawler=crawler) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.process_item\(\) requires a spider argument", + ), + ): + mwman._add_middleware(DeprecatedSpiderArgPipeline()) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"ItemPipelineManager.open_spider\(\) is deprecated, use open_spider_async\(\) instead", + ), + pytest.raises( + ValueError, + match=r"ItemPipelineManager needs to access self\.crawler\.spider but it is None", + ), + ): + await maybe_deferred_to_future(mwman.open_spider(DefaultSpider())) + with pytest.raises( + ValueError, + match=r"ItemPipelineManager needs to access self\.crawler\.spider but it is None", + ): + await mwman.open_spider_async() + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"ItemPipelineManager.close_spider\(\) is deprecated, use close_spider_async\(\) instead", + ), + pytest.raises( + ValueError, + match=r"ItemPipelineManager needs to access self\.crawler\.spider but it is None", + ), + ): + await maybe_deferred_to_future(mwman.close_spider(DefaultSpider())) + with pytest.raises( + ValueError, + match=r"ItemPipelineManager needs to access self\.crawler\.spider but it is None", + ): + await mwman.close_spider_async() + + @coroutine_test + async def test_deprecated_spider_arg_with_crawler(self, crawler: Crawler) -> None: + """Crawler is provided and has a spider, works. The instance passed to a deprecated method + is ignored, even if mismatched.""" + mwman = ItemPipelineManager(crawler=crawler) + crawler.spider = crawler._create_spider("foo") + with pytest.warns( + ScrapyDeprecationWarning, + match=r"ItemPipelineManager.open_spider\(\) is deprecated, use open_spider_async\(\) instead", + ): + await maybe_deferred_to_future(mwman.open_spider(DefaultSpider())) + with pytest.warns( + ScrapyDeprecationWarning, + match=r"ItemPipelineManager.close_spider\(\) is deprecated, use close_spider_async\(\) instead", + ): + await maybe_deferred_to_future(mwman.close_spider(DefaultSpider())) + + @coroutine_test + async def test_deprecated_spider_arg_without_crawler(self) -> None: + """The first instance passed to a deprecated method is used. Mismatched ones raise an error.""" + with pytest.warns( + ScrapyDeprecationWarning, + match="was called without the crawler argument", + ): + mwman = ItemPipelineManager() + spider = DefaultSpider() + with pytest.warns( + ScrapyDeprecationWarning, + match=r"ItemPipelineManager.open_spider\(\) is deprecated, use open_spider_async\(\) instead", + ): + await maybe_deferred_to_future(mwman.open_spider(spider)) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"ItemPipelineManager.close_spider\(\) is deprecated, use close_spider_async\(\) instead", + ), + pytest.raises( + RuntimeError, match="Different instances of Spider were passed" + ), + ): + await maybe_deferred_to_future(mwman.close_spider(DefaultSpider())) + with pytest.warns( + ScrapyDeprecationWarning, + match=r"ItemPipelineManager.close_spider\(\) is deprecated, use close_spider_async\(\) instead", + ): + await maybe_deferred_to_future(mwman.close_spider(spider)) + + @coroutine_test + async def test_no_spider_arg_without_crawler(self) -> None: + """If no crawler and no spider arg, raise an error.""" + with pytest.warns( + ScrapyDeprecationWarning, + match="was called without the crawler argument", + ): + mwman = ItemPipelineManager() + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.process_item\(\) requires a spider argument", + ), + ): + mwman._add_middleware(DeprecatedSpiderArgPipeline()) + with ( + pytest.raises( + ValueError, + match="has no known Spider instance", + ), + ): + await mwman.open_spider_async() diff --git a/tests/test_poet.py b/tests/test_poet.py new file mode 100644 index 000000000..9601c75a1 --- /dev/null +++ b/tests/test_poet.py @@ -0,0 +1,20 @@ +"""Tests that make sure parts needed for the scrapy-poet stack work.""" + +from typing import get_type_hints + +from scrapy import Spider +from scrapy.spiders import CrawlSpider, CSVFeedSpider, SitemapSpider, XMLFeedSpider + + +def test_callbacks(): + """Making sure annotations on all non-abstract callbacks can be resolved.""" + + for cb in [ + Spider._parse, + CrawlSpider._parse, + CrawlSpider._callback, + XMLFeedSpider._parse, + CSVFeedSpider._parse, + SitemapSpider._parse_sitemap, + ]: + get_type_hints(cb) diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 1584014b8..7be9241b9 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -1,18 +1,21 @@ import tempfile -import unittest +from unittest.mock import Mock +import pytest import queuelib +from scrapy.core.downloader import Downloader from scrapy.http.request import Request from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue from scrapy.spiders import Spider from scrapy.squeues import FifoMemoryQueue +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.test import get_crawler -from tests.test_scheduler import MockDownloader, MockEngine +from tests.test_scheduler import MockDownloader -class PriorityQueueTest(unittest.TestCase): - def setUp(self): +class TestPriorityQueue: + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") @@ -21,137 +24,238 @@ class PriorityQueueTest(unittest.TestCase): queue = ScrapyPriorityQueue.from_crawler( self.crawler, FifoMemoryQueue, temp_dir ) - self.assertIsNone(queue.pop()) - self.assertEqual(len(queue), 0) + assert queue.pop() is None + assert len(queue) == 0 req1 = Request("https://example.org/1", priority=1) queue.push(req1) - self.assertEqual(len(queue), 1) + assert len(queue) == 1 dequeued = queue.pop() - self.assertEqual(len(queue), 0) - self.assertEqual(dequeued.url, req1.url) - self.assertEqual(dequeued.priority, req1.priority) - self.assertEqual(queue.close(), []) + assert len(queue) == 0 + assert dequeued.url == req1.url + assert dequeued.priority == req1.priority + assert not queue.close() def test_no_peek_raises(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") + pytest.skip("queuelib.queue.FifoMemoryQueue.peek is defined") temp_dir = tempfile.mkdtemp() queue = ScrapyPriorityQueue.from_crawler( self.crawler, FifoMemoryQueue, temp_dir ) queue.push(Request("https://example.org")) - with self.assertRaises( + with pytest.raises( NotImplementedError, - msg="The underlying queue class does not implement 'peek'", + match="The underlying queue class does not implement 'peek'", ): queue.peek() queue.close() def test_peek(self): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined") + pytest.skip("queuelib.queue.FifoMemoryQueue.peek is undefined") temp_dir = tempfile.mkdtemp() queue = ScrapyPriorityQueue.from_crawler( self.crawler, FifoMemoryQueue, temp_dir ) - self.assertEqual(len(queue), 0) - self.assertIsNone(queue.peek()) + assert len(queue) == 0 + assert queue.peek() is None req1 = Request("https://example.org/1") req2 = Request("https://example.org/2") req3 = Request("https://example.org/3") queue.push(req1) queue.push(req2) queue.push(req3) - self.assertEqual(len(queue), 3) - self.assertEqual(queue.peek().url, req1.url) - self.assertEqual(queue.pop().url, req1.url) - self.assertEqual(len(queue), 2) - self.assertEqual(queue.peek().url, req2.url) - self.assertEqual(queue.pop().url, req2.url) - self.assertEqual(len(queue), 1) - self.assertEqual(queue.peek().url, req3.url) - self.assertEqual(queue.pop().url, req3.url) - self.assertEqual(queue.close(), []) + assert len(queue) == 3 + assert queue.peek().url == req1.url + assert queue.pop().url == req1.url + assert len(queue) == 2 + assert queue.peek().url == req2.url + assert queue.pop().url == req2.url + assert len(queue) == 1 + assert queue.peek().url == req3.url + assert queue.pop().url == req3.url + assert not queue.close() def test_queue_push_pop_priorities(self): temp_dir = tempfile.mkdtemp() queue = ScrapyPriorityQueue.from_crawler( self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3] ) - self.assertIsNone(queue.pop()) - self.assertEqual(len(queue), 0) + assert queue.pop() is None + assert len(queue) == 0 req1 = Request("https://example.org/1", priority=1) req2 = Request("https://example.org/2", priority=2) req3 = Request("https://example.org/3", priority=3) queue.push(req1) queue.push(req2) queue.push(req3) - self.assertEqual(len(queue), 3) + assert len(queue) == 3 dequeued = queue.pop() - self.assertEqual(len(queue), 2) - self.assertEqual(dequeued.url, req3.url) - self.assertEqual(dequeued.priority, req3.priority) - self.assertEqual(queue.close(), [-1, -2]) + assert len(queue) == 2 + assert dequeued.url == req3.url + assert dequeued.priority == req3.priority + assert set(queue.close()) == {-1, -2} -class DownloaderAwarePriorityQueueTest(unittest.TestCase): - def setUp(self): +class TestDownloaderAwarePriorityQueue: + def setup_method(self): crawler = get_crawler(Spider) - crawler.engine = MockEngine(downloader=MockDownloader()) + crawler.engine = Mock(downloader=MockDownloader()) self.queue = DownloaderAwarePriorityQueue.from_crawler( crawler=crawler, downstream_queue_cls=FifoMemoryQueue, key="foo/bar", ) - def tearDown(self): + def teardown_method(self): self.queue.close() def test_push_pop(self): - self.assertEqual(len(self.queue), 0) - self.assertIsNone(self.queue.pop()) + assert len(self.queue) == 0 + assert self.queue.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") self.queue.push(req1) self.queue.push(req2) self.queue.push(req3) - self.assertEqual(len(self.queue), 3) - self.assertEqual(self.queue.pop().url, req1.url) - self.assertEqual(len(self.queue), 2) - self.assertEqual(self.queue.pop().url, req2.url) - self.assertEqual(len(self.queue), 1) - self.assertEqual(self.queue.pop().url, req3.url) - self.assertEqual(len(self.queue), 0) - self.assertIsNone(self.queue.pop()) + assert len(self.queue) == 3 + assert self.queue.pop().url == req1.url + assert len(self.queue) == 2 + assert self.queue.pop().url == req2.url + assert len(self.queue) == 1 + assert self.queue.pop().url == req3.url + assert len(self.queue) == 0 + assert self.queue.pop() is None def test_no_peek_raises(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") + pytest.skip("queuelib.queue.FifoMemoryQueue.peek is defined") self.queue.push(Request("https://example.org")) - with self.assertRaises( + with pytest.raises( NotImplementedError, - msg="The underlying queue class does not implement 'peek'", + match="The underlying queue class does not implement 'peek'", ): self.queue.peek() def test_peek(self): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined") - self.assertEqual(len(self.queue), 0) + pytest.skip("queuelib.queue.FifoMemoryQueue.peek is undefined") + assert len(self.queue) == 0 req1 = Request("https://example.org/1") req2 = Request("https://example.org/2") req3 = Request("https://example.org/3") self.queue.push(req1) self.queue.push(req2) self.queue.push(req3) - self.assertEqual(len(self.queue), 3) - self.assertEqual(self.queue.peek().url, req1.url) - self.assertEqual(self.queue.pop().url, req1.url) - self.assertEqual(len(self.queue), 2) - self.assertEqual(self.queue.peek().url, req2.url) - self.assertEqual(self.queue.pop().url, req2.url) - self.assertEqual(len(self.queue), 1) - self.assertEqual(self.queue.peek().url, req3.url) - self.assertEqual(self.queue.pop().url, req3.url) - self.assertIsNone(self.queue.peek()) + assert len(self.queue) == 3 + assert self.queue.peek().url == req1.url + assert self.queue.pop().url == req1.url + assert len(self.queue) == 2 + assert self.queue.peek().url == req2.url + assert self.queue.pop().url == req2.url + assert len(self.queue) == 1 + assert self.queue.peek().url == req3.url + assert self.queue.pop().url == req3.url + assert self.queue.peek() is None + + def test_tie_breaking_rotates_slots(self): + # No active downloads are tracked in the downloader, so every slot has + # the same score and tie-breaking must not starve a slot. + req_a1 = Request("https://example.org/a1") + req_a1.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_b1 = Request("https://example.org/b1") + req_b1.meta[Downloader.DOWNLOAD_SLOT] = "slot-b" + req_a2 = Request("https://example.org/a2") + req_a2.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_b2 = Request("https://example.org/b2") + req_b2.meta[Downloader.DOWNLOAD_SLOT] = "slot-b" + + for request in (req_a1, req_b1, req_a2, req_b2): + self.queue.push(request) + + slots = [ + self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + ] + + assert slots == ["slot-a", "slot-b", "slot-a", "slot-b"] + + def test_tie_breaking_keeps_rotation_after_selected_slot_is_deleted(self): + # If the selected slot becomes empty, rotation should continue from + # that slot marker to avoid restarting from the smallest slot. + req_a1 = Request("https://example.org/a1") + req_a1.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_a2 = Request("https://example.org/a2") + req_a2.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_b1 = Request("https://example.org/b1") + req_b1.meta[Downloader.DOWNLOAD_SLOT] = "slot-b" + req_c1 = Request("https://example.org/c1") + req_c1.meta[Downloader.DOWNLOAD_SLOT] = "slot-c" + + for request in (req_a1, req_a2, req_b1, req_c1): + self.queue.push(request) + + slots = [ + self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + ] + + assert slots == ["slot-a", "slot-b", "slot-c", "slot-a"] + + +@pytest.mark.parametrize( + ("input_", "output"), + [ + # By default, start requests are FIFO, other requests are LIFO. + ([{}, {}], [2, 1]), + ([{"start": True}, {"start": True}], [1, 2]), + # Priority matters. + ([{"priority": 1}, {"start": True}], [1, 2]), + ([{}, {"start": True, "priority": 1}], [2, 1]), + # For the same priority, start requests pop last. + ([{}, {"start": True}], [1, 2]), + ([{"start": True}, {}], [2, 1]), + ], +) +def test_pop_order(input_, output): + def make_url(index): + return f"https://toscrape.com/{index}" + + def make_request(index, data): + meta = {} + if data.get("start", False): + meta["is_start_request"] = True + return Request( + url=make_url(index), + priority=data.get("priority", 0), + meta=meta, + ) + + input_requests = [ + make_request(index, data) for index, data in enumerate(input_, start=1) + ] + expected_output_urls = [make_url(index) for index in output] + + crawler = get_crawler(Spider) + settings = crawler.settings + queue = build_from_crawler( + ScrapyPriorityQueue, + crawler, + downstream_queue_cls=load_object(settings["SCHEDULER_MEMORY_QUEUE"]), + key="", + start_queue_cls=load_object(settings["SCHEDULER_START_MEMORY_QUEUE"]), + ) + + for request in input_requests: + queue.push(request) + + actual_output_urls = [] + while request := queue.pop(): + actual_output_urls.append(request.url) + + assert actual_output_urls == expected_output_urls diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py deleted file mode 100644 index 46d42e9f6..000000000 --- a/tests/test_proxy_connect.py +++ /dev/null @@ -1,119 +0,0 @@ -import json -import os -import re -import sys -from pathlib import Path -from subprocess import PIPE, Popen -from urllib.parse import urlsplit, urlunsplit - -from testfixtures import LogCapture -from twisted.internet import defer -from twisted.trial.unittest import TestCase - -from scrapy.http import Request -from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer -from tests.spiders import SimpleSpider, SingleRequestSpider - - -class MitmProxy: - auth_user = "scrapy" - auth_pass = "scrapy" - - def start(self): - script = """ -import sys -from mitmproxy.tools.main import mitmdump -sys.argv[0] = "mitmdump" -sys.exit(mitmdump()) - """ - cert_path = Path(__file__).parent.resolve() / "keys" / "mitmproxy-ca.pem" - self.proc = Popen( - [ - sys.executable, - "-u", - "-c", - script, - "--listen-host", - "127.0.0.1", - "--listen-port", - "0", - "--proxyauth", - f"{self.auth_user}:{self.auth_pass}", - "--certs", - str(cert_path), - "--ssl-insecure", - ], - stdout=PIPE, - ) - line = self.proc.stdout.readline().decode("utf-8") - host_port = re.search(r"listening at (?:http://)?([^:]+:\d+)", line).group(1) - address = f"http://{self.auth_user}:{self.auth_pass}@{host_port}" - return address - - def stop(self): - self.proc.kill() - self.proc.communicate() - - -def _wrong_credentials(proxy_url): - bad_auth_proxy = list(urlsplit(proxy_url)) - bad_auth_proxy[1] = bad_auth_proxy[1].replace("scrapy:scrapy@", "wrong:wronger@") - return urlunsplit(bad_auth_proxy) - - -class ProxyConnectTestCase(TestCase): - def setUp(self): - try: - import mitmproxy # noqa: F401 - except ImportError: - self.skipTest("mitmproxy is not installed") - - self.mockserver = MockServer() - self.mockserver.__enter__() - self._oldenv = os.environ.copy() - - self._proxy = MitmProxy() - proxy_url = self._proxy.start() - os.environ["https_proxy"] = proxy_url - os.environ["http_proxy"] = proxy_url - - def tearDown(self): - self.mockserver.__exit__(None, None, None) - self._proxy.stop() - os.environ = self._oldenv - - @defer.inlineCallbacks - def test_https_connect_tunnel(self): - crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, log) - - @defer.inlineCallbacks - def test_https_tunnel_auth_error(self): - os.environ["https_proxy"] = _wrong_credentials(os.environ["https_proxy"]) - crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - # The proxy returns a 407 error code but it does not reach the client; - # he just sees a TunnelError. - self._assert_got_tunnel_error(log) - - @defer.inlineCallbacks - def test_https_tunnel_without_leak_proxy_authorization_header(self): - request = Request(self.mockserver.url("/echo", is_secure=True)) - crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl(seed=request) - self._assert_got_response_code(200, log) - echo = json.loads(crawler.spider.meta["responses"][0].text) - self.assertTrue("Proxy-Authorization" not in echo["headers"]) - - def _assert_got_response_code(self, code, log): - print(log) - self.assertEqual(str(log).count(f"Crawled ({code})"), 1) - - def _assert_got_tunnel_error(self, log): - print(log) - self.assertIn("TunnelError", str(log)) diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index d65d74206..38d56e9bd 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -1,29 +1,28 @@ from testfixtures import LogCapture -from twisted.internet import defer -from twisted.trial.unittest import TestCase from scrapy import Request, signals from scrapy.http.response import Response from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer +from tests.mockserver.http import MockServer from tests.spiders import SingleRequestSpider +from tests.utils.decorators import inline_callbacks_test OVERRIDDEN_URL = "https://example.org" class ProcessResponseMiddleware: - def process_response(self, request, response, spider): + def process_response(self, request, response): return response.replace(request=Request(OVERRIDDEN_URL)) class RaiseExceptionRequestMiddleware: - def process_request(self, request, spider): + def process_request(self, request): 1 / 0 return request class CatchExceptionOverrideRequestMiddleware: - def process_exception(self, request, exception, spider): + def process_exception(self, request, exception): return Response( url="http://localhost/", body=b"Caught " + exception.__class__.__name__.encode("utf-8"), @@ -32,7 +31,7 @@ class CatchExceptionOverrideRequestMiddleware: class CatchExceptionDoNotOverrideRequestMiddleware: - def process_exception(self, request, exception, spider): + def process_exception(self, request, exception): return Response( url="http://localhost/", body=b"Caught " + exception.__class__.__name__.encode("utf-8"), @@ -47,32 +46,41 @@ class AlternativeCallbacksSpider(SingleRequestSpider): class AlternativeCallbacksMiddleware: - def process_response(self, request, response, spider): + def __init__(self, crawler): + self.crawler = crawler + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def process_response(self, request, response): new_request = request.replace( url=OVERRIDDEN_URL, - callback=spider.alt_callback, + callback=self.crawler.spider.alt_callback, cb_kwargs={"foo": "bar"}, ) return response.replace(request=new_request) -class CrawlTestCase(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() +class TestCrawl: + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) - @defer.inlineCallbacks + @inline_callbacks_test def test_response_200(self): url = self.mockserver.url("/status?n=200") crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] - self.assertEqual(response.request.url, url) + assert response.request.url == url - @defer.inlineCallbacks + @inline_callbacks_test def test_response_error(self): for status in ("404", "500"): url = self.mockserver.url(f"/status?n={status}") @@ -80,10 +88,10 @@ class CrawlTestCase(TestCase): yield crawler.crawl(seed=url, mockserver=self.mockserver) failure = crawler.spider.meta["failure"] response = failure.value.response - self.assertEqual(failure.request.url, url) - self.assertEqual(response.request.url, url) + assert failure.request.url == url + assert response.request.url == url - @defer.inlineCallbacks + @inline_callbacks_test def test_downloader_middleware_raise_exception(self): url = self.mockserver.url("/status?n=200") crawler = get_crawler( @@ -96,10 +104,10 @@ class CrawlTestCase(TestCase): ) yield crawler.crawl(seed=url, mockserver=self.mockserver) failure = crawler.spider.meta["failure"] - self.assertEqual(failure.request.url, url) - self.assertIsInstance(failure.value, ZeroDivisionError) + assert failure.request.url == url + assert isinstance(failure.value, ZeroDivisionError) - @defer.inlineCallbacks + @inline_callbacks_test def test_downloader_middleware_override_request_in_process_response(self): """ Downloader middleware which returns a response with an specific 'request' attribute. @@ -129,10 +137,10 @@ class CrawlTestCase(TestCase): yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] - self.assertEqual(response.request.url, OVERRIDDEN_URL) + assert response.request.url == OVERRIDDEN_URL - self.assertEqual(signal_params["response"].url, url) - self.assertEqual(signal_params["request"].url, OVERRIDDEN_URL) + assert signal_params["response"].url == url + assert signal_params["request"].url == OVERRIDDEN_URL log.check_present( ( @@ -142,7 +150,7 @@ class CrawlTestCase(TestCase): ), ) - @defer.inlineCallbacks + @inline_callbacks_test def test_downloader_middleware_override_in_process_exception(self): """ An exception is raised but caught by the next middleware, which @@ -162,10 +170,10 @@ class CrawlTestCase(TestCase): ) yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] - self.assertEqual(response.body, b"Caught ZeroDivisionError") - self.assertEqual(response.request.url, OVERRIDDEN_URL) + assert response.body == b"Caught ZeroDivisionError" + assert response.request.url == OVERRIDDEN_URL - @defer.inlineCallbacks + @inline_callbacks_test def test_downloader_middleware_do_not_override_in_process_exception(self): """ An exception is raised but caught by the next middleware, which @@ -185,10 +193,10 @@ class CrawlTestCase(TestCase): ) yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] - self.assertEqual(response.body, b"Caught ZeroDivisionError") - self.assertEqual(response.request.url, url) + assert response.body == b"Caught ZeroDivisionError" + assert response.request.url == url - @defer.inlineCallbacks + @inline_callbacks_test def test_downloader_middleware_alternative_callback(self): """ Downloader middleware which returns a response with a diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 7299972f6..0d96e1d88 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -1,13 +1,10 @@ -from typing import List - from testfixtures import LogCapture -from twisted.internet import defer -from twisted.trial.unittest import TestCase from scrapy.http import Request from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer +from tests.mockserver.http import MockServer from tests.spiders import MockServerSpider +from tests.utils.decorators import inline_callbacks_test class InjectArgumentsDownloaderMiddleware: @@ -15,12 +12,11 @@ class InjectArgumentsDownloaderMiddleware: Make sure downloader middlewares are able to update the keyword arguments """ - def process_request(self, request, spider): + def process_request(self, request): if request.callback.__name__ == "parse_downloader_mw": request.cb_kwargs["from_process_request"] = True - return None - def process_response(self, request, response, spider): + def process_response(self, request, response): if request.callback.__name__ == "parse_downloader_mw": request.cb_kwargs["from_process_response"] = True return response @@ -31,20 +27,19 @@ class InjectArgumentsSpiderMiddleware: Make sure spider middlewares are able to update the keyword arguments """ - def process_start_requests(self, start_requests, spider): - for request in start_requests: + async def process_start(self, start): + async for request in start: if request.callback.__name__ == "parse_spider_mw": - request.cb_kwargs["from_process_start_requests"] = True + request.cb_kwargs["from_process_start"] = True yield request - def process_spider_input(self, response, spider): + def process_spider_input(self, response): request = response.request if request.callback.__name__ == "parse_spider_mw": request.cb_kwargs["from_process_spider_input"] = True - return None - def process_spider_output(self, response, result, spider): - for element in result: + async def process_spider_output(self, response, result): + async for element in result: if ( isinstance(element, Request) and element.callback.__name__ == "parse_spider_mw_2" @@ -64,9 +59,9 @@ class KeywordArgumentsSpider(MockServerSpider): }, } - checks: List[bool] = [] + checks: list[bool] = [] - def start_requests(self): + async def start(self): data = {"key": "value", "number": 123, "callback": "some_callback"} yield Request(self.mockserver.url("/first"), self.parse_first, cb_kwargs=data) yield Request( @@ -142,11 +137,9 @@ class KeywordArgumentsSpider(MockServerSpider): self.checks.append(bool(from_process_response)) self.crawler.stats.inc_value("boolean_checks", 2) - def parse_spider_mw( - self, response, from_process_spider_input, from_process_start_requests - ): + def parse_spider_mw(self, response, from_process_spider_input, from_process_start): self.checks.append(bool(from_process_spider_input)) - self.checks.append(bool(from_process_start_requests)) + self.checks.append(bool(from_process_start)) self.crawler.stats.inc_value("boolean_checks", 2) return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2) @@ -155,42 +148,34 @@ class KeywordArgumentsSpider(MockServerSpider): self.crawler.stats.inc_value("boolean_checks", 1) -class CallbackKeywordArgumentsTestCase(TestCase): - maxDiff = None +class TestCallbackKeywordArguments: + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) - def tearDown(self): - self.mockserver.__exit__(None, None, None) - - @defer.inlineCallbacks + @inline_callbacks_test def test_callback_kwargs(self): crawler = get_crawler(KeywordArgumentsSpider) with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertTrue(all(crawler.spider.checks)) - self.assertEqual( - len(crawler.spider.checks), crawler.stats.get_value("boolean_checks") - ) + assert all(crawler.spider.checks) + assert len(crawler.spider.checks) == crawler.stats.get_value("boolean_checks") # check exceptions for argument mismatch exceptions = {} for line in log.records: for key in ("takes_less", "takes_more"): if key in line.getMessage(): exceptions[key] = line - self.assertEqual(exceptions["takes_less"].exc_info[0], TypeError) - self.assertTrue( - str(exceptions["takes_less"].exc_info[1]).endswith( - "parse_takes_less() got an unexpected keyword argument 'number'" - ), - msg="Exception message: " + str(exceptions["takes_less"].exc_info[1]), - ) - self.assertEqual(exceptions["takes_more"].exc_info[0], TypeError) - self.assertTrue( - str(exceptions["takes_more"].exc_info[1]).endswith( - "parse_takes_more() missing 1 required positional argument: 'other'" - ), - msg="Exception message: " + str(exceptions["takes_more"].exc_info[1]), - ) + assert exceptions["takes_less"].exc_info[0] is TypeError + assert str(exceptions["takes_less"].exc_info[1]).endswith( + "parse_takes_less() got an unexpected keyword argument 'number'" + ), "Exception message: " + str(exceptions["takes_less"].exc_info[1]) + assert exceptions["takes_more"].exc_info[0] is TypeError + assert str(exceptions["takes_more"].exc_info[1]).endswith( + "parse_takes_more() missing 1 required positional argument: 'other'" + ), "Exception message: " + str(exceptions["takes_more"].exc_info[1]) diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 7312eb036..78ff18b15 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -1,7 +1,7 @@ -import unittest +import pytest from scrapy import Request, Spider -from scrapy.http import FormRequest, JsonRequest +from scrapy.http import JsonRequest from scrapy.utils.request import request_from_dict @@ -9,9 +9,9 @@ class CustomRequest(Request): pass -class RequestSerializationTest(unittest.TestCase): - def setUp(self): - self.spider = TestSpider() +class TestRequestSerialization: + def setup_method(self): + self.spider = MethodsSpider() def test_basic(self): r = Request("http://www.example.com") @@ -48,31 +48,29 @@ class RequestSerializationTest(unittest.TestCase): self._assert_same_request(request, request2) def _assert_same_request(self, r1, r2): - self.assertEqual(r1.__class__, r2.__class__) - self.assertEqual(r1.url, r2.url) - self.assertEqual(r1.callback, r2.callback) - self.assertEqual(r1.errback, r2.errback) - self.assertEqual(r1.method, r2.method) - self.assertEqual(r1.body, r2.body) - self.assertEqual(r1.headers, r2.headers) - self.assertEqual(r1.cookies, r2.cookies) - self.assertEqual(r1.meta, r2.meta) - self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) - self.assertEqual(r1.encoding, r2.encoding) - self.assertEqual(r1._encoding, r2._encoding) - self.assertEqual(r1.priority, r2.priority) - self.assertEqual(r1.dont_filter, r2.dont_filter) - self.assertEqual(r1.flags, r2.flags) + assert r1.__class__ == r2.__class__ + assert r1.url == r2.url + assert r1.callback == r2.callback + assert r1.errback == r2.errback + assert r1.method == r2.method + assert r1.body == r2.body + assert r1.headers == r2.headers + assert r1.cookies == r2.cookies + assert r1.meta == r2.meta + assert r1.cb_kwargs == r2.cb_kwargs + assert r1.encoding == r2.encoding + assert r1._encoding == r2._encoding + assert r1.priority == r2.priority + assert r1.dont_filter == r2.dont_filter + assert r1.flags == r2.flags if isinstance(r1, JsonRequest): - self.assertEqual(r1.dumps_kwargs, r2.dumps_kwargs) + assert r1.dumps_kwargs == r2.dumps_kwargs def test_request_class(self): - r1 = FormRequest("http://www.example.com") + r1 = CustomRequest("http://www.example.com") self._assert_serializes_ok(r1, spider=self.spider) - r2 = CustomRequest("http://www.example.com") + r2 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4}) self._assert_serializes_ok(r2, spider=self.spider) - r3 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4}) - self._assert_serializes_ok(r3, spider=self.spider) def test_callback_serialization(self): r = Request( @@ -90,24 +88,24 @@ class RequestSerializationTest(unittest.TestCase): ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) - self.assertEqual(request_dict["callback"], "parse_item_reference") - self.assertEqual(request_dict["errback"], "handle_error_reference") + assert request_dict["callback"] == "parse_item_reference" + assert request_dict["errback"] == "handle_error_reference" def test_private_reference_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._TestSpider__parse_item_reference, - errback=self.spider._TestSpider__handle_error_reference, + callback=self.spider._MethodsSpider__parse_item_reference, + errback=self.spider._MethodsSpider__handle_error_reference, ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) - self.assertEqual(request_dict["callback"], "_TestSpider__parse_item_reference") - self.assertEqual(request_dict["errback"], "_TestSpider__handle_error_reference") + assert request_dict["callback"] == "_MethodsSpider__parse_item_reference" + assert request_dict["errback"] == "_MethodsSpider__handle_error_reference" def test_private_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._TestSpider__parse_item_private, + callback=self.spider._MethodsSpider__parse_item_private, errback=self.spider.handle_error, ) self._assert_serializes_ok(r, spider=self.spider) @@ -115,7 +113,7 @@ class RequestSerializationTest(unittest.TestCase): def test_mixin_private_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._TestSpiderMixin__mixin_callback, + callback=self.spider._SpiderMixin__mixin_callback, errback=self.spider.handle_error, ) self._assert_serializes_ok(r, spider=self.spider) @@ -130,11 +128,15 @@ class RequestSerializationTest(unittest.TestCase): def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) - self.assertRaises(ValueError, r.to_dict, spider=self.spider) + with pytest.raises( + ValueError, match="is not an instance method in: bool: # check if robotexclusionrulesparser is installed try: - from robotexclusionrulesparser import RobotExclusionRulesParser # noqa: F401 - except ImportError: - return False - return True - - -def protego_available(): - # check if protego parser is installed - try: - from protego import Protego # noqa: F401 + from robotexclusionrulesparser import ( # noqa: PLC0415 + RobotExclusionRulesParser, # noqa: F401 + ) except ImportError: return False return True @@ -34,71 +27,56 @@ class BaseRobotParserTest: def test_allowed(self): robotstxt_robotstxt_body = ( - "User-agent: * \n" - "Disallow: /disallowed \n" - "Allow: /allowed \n" - "Crawl-delay: 10".encode("utf-8") + b"User-agent: * \nDisallow: /disallowed \nAllow: /allowed \nCrawl-delay: 10" ) rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertTrue(rp.allowed("https://www.site.local/allowed", "*")) - self.assertFalse(rp.allowed("https://www.site.local/disallowed", "*")) + assert rp.allowed("https://www.site.local/allowed", "*") + assert not rp.allowed("https://www.site.local/disallowed", "*") def test_allowed_wildcards(self): - robotstxt_robotstxt_body = """User-agent: first + robotstxt_robotstxt_body = b"""User-agent: first Disallow: /disallowed/*/end$ User-agent: second Allow: /*allowed Disallow: / - """.encode( - "utf-8" - ) + """ rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertTrue(rp.allowed("https://www.site.local/disallowed", "first")) - self.assertFalse( - rp.allowed("https://www.site.local/disallowed/xyz/end", "first") - ) - self.assertFalse( - rp.allowed("https://www.site.local/disallowed/abc/end", "first") - ) - self.assertTrue( - rp.allowed("https://www.site.local/disallowed/xyz/endinglater", "first") - ) + assert rp.allowed("https://www.site.local/disallowed", "first") + assert not rp.allowed("https://www.site.local/disallowed/xyz/end", "first") + assert not rp.allowed("https://www.site.local/disallowed/abc/end", "first") + assert rp.allowed("https://www.site.local/disallowed/xyz/endinglater", "first") - self.assertTrue(rp.allowed("https://www.site.local/allowed", "second")) - self.assertTrue(rp.allowed("https://www.site.local/is_still_allowed", "second")) - self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second")) + assert rp.allowed("https://www.site.local/allowed", "second") + assert rp.allowed("https://www.site.local/is_still_allowed", "second") + assert rp.allowed("https://www.site.local/is_allowed_too", "second") def test_length_based_precedence(self): - robotstxt_robotstxt_body = ( - "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8") - ) + robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertTrue(rp.allowed("https://www.site.local/page", "*")) + assert rp.allowed("https://www.site.local/page", "*") def test_order_based_precedence(self): - robotstxt_robotstxt_body = ( - "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8") - ) + robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertFalse(rp.allowed("https://www.site.local/page", "*")) + assert not rp.allowed("https://www.site.local/page", "*") def test_empty_response(self): """empty response should equal 'allow all'""" rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b"") - self.assertTrue(rp.allowed("https://site.local/", "*")) - self.assertTrue(rp.allowed("https://site.local/", "chrome")) - self.assertTrue(rp.allowed("https://site.local/index.html", "*")) - self.assertTrue(rp.allowed("https://site.local/disallowed", "*")) + assert rp.allowed("https://site.local/", "*") + assert rp.allowed("https://site.local/", "chrome") + assert rp.allowed("https://site.local/index.html", "*") + assert rp.allowed("https://site.local/disallowed", "*") def test_garbage_response(self): """garbage response should be discarded, equal 'allow all'""" @@ -106,10 +84,10 @@ class BaseRobotParserTest: rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertTrue(rp.allowed("https://site.local/", "*")) - self.assertTrue(rp.allowed("https://site.local/", "chrome")) - self.assertTrue(rp.allowed("https://site.local/index.html", "*")) - self.assertTrue(rp.allowed("https://site.local/disallowed", "*")) + assert rp.allowed("https://site.local/", "*") + assert rp.allowed("https://site.local/", "chrome") + assert rp.allowed("https://site.local/index.html", "*") + assert rp.allowed("https://site.local/disallowed", "*") def test_unicode_url_and_useragent(self): robotstxt_robotstxt_body = """ @@ -121,81 +99,86 @@ class BaseRobotParserTest: Disallow: /wiki/Käyttäjä: User-Agent: UnicödeBöt - Disallow: /some/randome/page.html""".encode( - "utf-8" - ) + Disallow: /some/randome/page.html""".encode() rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertTrue(rp.allowed("https://site.local/", "*")) - self.assertFalse(rp.allowed("https://site.local/admin/", "*")) - self.assertFalse(rp.allowed("https://site.local/static/", "*")) - self.assertTrue(rp.allowed("https://site.local/admin/", "UnicödeBöt")) - self.assertFalse( - rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*") - ) - self.assertFalse(rp.allowed("https://site.local/wiki/Käyttäjä:", "*")) - self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*")) - self.assertFalse( - rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt") + assert rp.allowed("https://site.local/", "*") + assert not rp.allowed("https://site.local/admin/", "*") + assert not rp.allowed("https://site.local/static/", "*") + assert rp.allowed("https://site.local/admin/", "UnicödeBöt") + assert not rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*") + assert not rp.allowed("https://site.local/wiki/Käyttäjä:", "*") + assert rp.allowed("https://site.local/some/randome/page.html", "*") + assert not rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt") + + +class TestDecodeRobotsTxt: + def test_native_string_conversion(self): + robotstxt_body = b"User-agent: *\nDisallow: /\n" + decoded_content = decode_robotstxt( + robotstxt_body, spider=None, to_native_str_type=True ) + assert decoded_content == "User-agent: *\nDisallow: /\n" + + def test_decode_utf8(self): + robotstxt_body = b"User-agent: *\nDisallow: /\n" + decoded_content = decode_robotstxt(robotstxt_body, spider=None) + assert decoded_content == "User-agent: *\nDisallow: /\n" + + def test_decode_non_utf8(self): + robotstxt_body = b"User-agent: *\n\xffDisallow: /\n" + decoded_content = decode_robotstxt(robotstxt_body, spider=None) + assert decoded_content == "User-agent: *\nDisallow: /\n" + + # UTF-8 BOM at the beginning of the file ignored + def test_decode_utf8_bom(self): + robotstxt_body = b"\xef\xbb\xbfUser-agent: *\nDisallow: /\n" + decoded_content = decode_robotstxt(robotstxt_body, spider=None) + assert decoded_content == "User-agent: *\nDisallow: /\n" -class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): - def setUp(self): - from scrapy.robotstxt import PythonRobotParser +class TestPythonRobotParser(BaseRobotParserTest): + # https://github.com/python/cpython/pull/149374 improves it + IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5) + def setup_method(self): super()._setUp(PythonRobotParser) + @pytest.mark.skipif( + not IMPROVED_ROBOTFILEPARSER, + reason="RobotFileParser from this Python version does not support length based directives precedence.", + ) def test_length_based_precedence(self): - raise unittest.SkipTest( - "RobotFileParser does not support length based directives precedence." - ) - - def test_allowed_wildcards(self): - raise unittest.SkipTest("RobotFileParser does not support wildcards.") - - -class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase): - if not reppy_available(): - skip = "Reppy parser is not installed" - - def setUp(self): - from scrapy.robotstxt import ReppyRobotParser - - super()._setUp(ReppyRobotParser) + super().test_length_based_precedence() + @pytest.mark.skipif( + IMPROVED_ROBOTFILEPARSER, + reason="RobotFileParser from this Python version does not support order based directives precedence.", + ) def test_order_based_precedence(self): - raise unittest.SkipTest( - "Reppy does not support order based directives precedence." - ) + super().test_order_based_precedence() + + @pytest.mark.skipif( + not IMPROVED_ROBOTFILEPARSER, + reason="RobotFileParser from this Python version does not support wildcards.", + ) + def test_allowed_wildcards(self): + super().test_allowed_wildcards() -class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase): - if not rerp_available(): - skip = "Rerp parser is not installed" - - def setUp(self): - from scrapy.robotstxt import RerpRobotParser - +@pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed") +class TestRerpRobotParser(BaseRobotParserTest): + def setup_method(self): super()._setUp(RerpRobotParser) def test_length_based_precedence(self): - raise unittest.SkipTest( - "Rerp does not support length based directives precedence." - ) + pytest.skip("Rerp does not support length based directives precedence.") -class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase): - if not protego_available(): - skip = "Protego parser is not installed" - - def setUp(self): - from scrapy.robotstxt import ProtegoRobotParser - +class TestProtegoRobotParser(BaseRobotParserTest): + def setup_method(self): super()._setUp(ProtegoRobotParser) def test_order_based_precedence(self): - raise unittest.SkipTest( - "Protego does not support order based directives precedence." - ) + pytest.skip("Protego does not support order based directives precedence.") diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index f8465a5ff..eb88baf01 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,84 +1,119 @@ -import collections -import shutil -import tempfile -import unittest -from typing import Optional +from __future__ import annotations -from twisted.internet import defer -from twisted.trial.unittest import TestCase +import warnings +from abc import ABC, abstractmethod +from collections import deque +from contextlib import AbstractAsyncContextManager, asynccontextmanager +from typing import TYPE_CHECKING, Any, NamedTuple, cast +from unittest.mock import Mock + +import pytest from scrapy.core.downloader import Downloader -from scrapy.core.scheduler import Scheduler +from scrapy.core.scheduler import BaseScheduler, Scheduler from scrapy.crawler import Crawler from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.defer import ensure_awaitable from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer +from tests.mockserver.http import MockServer +from tests.utils.decorators import coroutine_test, inline_callbacks_test -MockEngine = collections.namedtuple("MockEngine", ["downloader"]) -MockSlot = collections.namedtuple("MockSlot", ["active"]) +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + from pathlib import Path + + +class MemoryScheduler(BaseScheduler): + paused = False + + def __init__(self, *args: Any, **kwargs: Any): + super().__init__(*args, **kwargs) + self.queue: deque[Request] = deque( + Request(value) if isinstance(value, str) else value + for value in getattr(self, "queue", []) + ) + + def enqueue_request(self, request: Request) -> bool: + self.queue.append(request) + return True + + def has_pending_requests(self) -> bool: + return self.paused or bool(self.queue) + + def next_request(self) -> Request | None: + if self.paused: + return None + try: + return self.queue.pop() + except IndexError: + return None + + def pause(self) -> None: + self.paused = True + + def unpause(self) -> None: + self.paused = False + + +class MockSlot(NamedTuple): + active: list[Any] class MockDownloader: - def __init__(self): - self.slots = {} + def __init__(self) -> None: + self.slots: dict[str, MockSlot] = {} - def _get_slot_key(self, request, spider): + def get_slot_key(self, request: Request) -> str: if Downloader.DOWNLOAD_SLOT in request.meta: - return request.meta[Downloader.DOWNLOAD_SLOT] + return cast("str", request.meta[Downloader.DOWNLOAD_SLOT]) return urlparse_cached(request).hostname or "" - def increment(self, slot_key): + def increment(self, slot_key: str) -> None: slot = self.slots.setdefault(slot_key, MockSlot(active=[])) slot.active.append(1) - def decrement(self, slot_key): - slot = self.slots.get(slot_key) + def decrement(self, slot_key: str) -> None: + slot = self.slots[slot_key] slot.active.pop() - def close(self): + def close(self) -> None: pass class MockCrawler(Crawler): - def __init__(self, priority_queue_cls, jobdir): - settings = dict( - SCHEDULER_DEBUG=False, - SCHEDULER_DISK_QUEUE="scrapy.squeues.PickleLifoDiskQueue", - SCHEDULER_MEMORY_QUEUE="scrapy.squeues.LifoMemoryQueue", - SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, - JOBDIR=jobdir, - DUPEFILTER_CLASS="scrapy.dupefilters.BaseDupeFilter", - REQUEST_FINGERPRINTER_IMPLEMENTATION="2.7", - ) + def __init__(self, priority_queue_cls: str, jobdir: Path | None): + settings = { + "SCHEDULER_DEBUG": False, + "SCHEDULER_DISK_QUEUE": "scrapy.squeues.PickleLifoDiskQueue", + "SCHEDULER_MEMORY_QUEUE": "scrapy.squeues.LifoMemoryQueue", + "SCHEDULER_PRIORITY_QUEUE": priority_queue_cls, + "JOBDIR": str(jobdir) if jobdir is not None else None, + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", + } super().__init__(Spider, settings) - self.engine = MockEngine(downloader=MockDownloader()) + self.engine = Mock(downloader=MockDownloader()) self.stats = load_object(self.settings["STATS_CLASS"])(self) -class SchedulerHandler: - priority_queue_cls: Optional[str] = None - jobdir = None - - def create_scheduler(self): - self.mock_crawler = MockCrawler(self.priority_queue_cls, self.jobdir) - self.scheduler = Scheduler.from_crawler(self.mock_crawler) - self.spider = Spider(name="spider") - self.scheduler.open(self.spider) - - def close_scheduler(self): - self.scheduler.close("finished") - self.mock_crawler.stop() - self.mock_crawler.engine.downloader.close() - - def setUp(self): - self.create_scheduler() - - def tearDown(self): - self.close_scheduler() +@asynccontextmanager +async def create_scheduler( + priority_queue_cls: str, jobdir: Path | None +) -> AsyncGenerator[Scheduler]: + mock_crawler = MockCrawler(priority_queue_cls, jobdir) + scheduler = Scheduler.from_crawler(mock_crawler) + spider = Spider(name="spider") + await ensure_awaitable(scheduler.open(spider)) + try: + yield scheduler + finally: + await ensure_awaitable(scheduler.close("finished")) + await mock_crawler.stop_async() + assert mock_crawler.engine + mock_crawler.engine.downloader.close() _PRIORITIES = [ @@ -93,98 +128,117 @@ _PRIORITIES = [ _URLS = {"http://foo.com/a", "http://foo.com/b", "http://foo.com/c"} -class BaseSchedulerInMemoryTester(SchedulerHandler): - def test_length(self): - self.assertFalse(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), 0) +class TestSchedulerBase(ABC): + @property + @abstractmethod + def priority_queue_cls(self) -> str: + raise NotImplementedError - for url in _URLS: - self.scheduler.enqueue_request(Request(url)) + @pytest.fixture + def jobdir(self) -> Path | None: + return None - self.assertTrue(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), len(_URLS)) + def create_scheduler( + self, jobdir: Path | None + ) -> AbstractAsyncContextManager[Scheduler]: + return create_scheduler(self.priority_queue_cls, jobdir) - def test_dequeue(self): - for url in _URLS: - self.scheduler.enqueue_request(Request(url)) + # TODO: unify test methods using "reopen" like in DownloaderAwareSchedulerTestMixin + + +class TestSchedulerInMemoryBase(TestSchedulerBase): + @coroutine_test + async def test_length(self, jobdir: Path | None) -> None: + async with self.create_scheduler(jobdir) as scheduler: + assert not scheduler.has_pending_requests() + assert len(scheduler) == 0 + + for url in _URLS: + scheduler.enqueue_request(Request(url)) + + assert scheduler.has_pending_requests() + assert len(scheduler) == len(_URLS) + + @coroutine_test + async def test_dequeue(self, jobdir: Path | None) -> None: + async with self.create_scheduler(jobdir) as scheduler: + for url in _URLS: + scheduler.enqueue_request(Request(url)) + + urls = set() + while scheduler.has_pending_requests(): + request = scheduler.next_request() + assert request is not None + urls.add(request.url) + + assert urls == _URLS + + @coroutine_test + async def test_dequeue_priorities(self, jobdir: Path | None) -> None: + async with self.create_scheduler(jobdir) as scheduler: + for url, priority in _PRIORITIES: + scheduler.enqueue_request(Request(url, priority=priority)) + + priorities = [] + while scheduler.has_pending_requests(): + request = scheduler.next_request() + assert request is not None + priorities.append(request.priority) + + assert priorities == sorted([x[1] for x in _PRIORITIES], key=lambda x: -x) + + +class TestSchedulerOnDiskBase(TestSchedulerBase): + @pytest.fixture + def jobdir(self, tmp_path: Path) -> Path | None: + return tmp_path + + @coroutine_test + async def test_length(self, jobdir: Path | None) -> None: + async with self.create_scheduler(jobdir) as scheduler: + assert not scheduler.has_pending_requests() + assert len(scheduler) == 0 + for url in _URLS: + scheduler.enqueue_request(Request(url)) + + async with self.create_scheduler(jobdir) as scheduler: + assert scheduler.has_pending_requests() + assert len(scheduler) == len(_URLS) + + @coroutine_test + async def test_dequeue(self, jobdir: Path | None) -> None: + async with self.create_scheduler(jobdir) as scheduler: + for url in _URLS: + scheduler.enqueue_request(Request(url)) urls = set() - while self.scheduler.has_pending_requests(): - urls.add(self.scheduler.next_request().url) + async with self.create_scheduler(jobdir) as scheduler: + while scheduler.has_pending_requests(): + request = scheduler.next_request() + assert request is not None + urls.add(request.url) + assert urls == _URLS - self.assertEqual(urls, _URLS) - - def test_dequeue_priorities(self): - for url, priority in _PRIORITIES: - self.scheduler.enqueue_request(Request(url, priority=priority)) + @coroutine_test + async def test_dequeue_priorities(self, jobdir: Path | None) -> None: + async with self.create_scheduler(jobdir) as scheduler: + for url, priority in _PRIORITIES: + scheduler.enqueue_request(Request(url, priority=priority)) priorities = [] - while self.scheduler.has_pending_requests(): - priorities.append(self.scheduler.next_request().priority) - - self.assertEqual( - priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x) - ) + async with self.create_scheduler(jobdir) as scheduler: + while scheduler.has_pending_requests(): + request = scheduler.next_request() + assert request is not None + priorities.append(request.priority) + assert priorities == sorted([x[1] for x in _PRIORITIES], key=lambda x: -x) -class BaseSchedulerOnDiskTester(SchedulerHandler): - def setUp(self): - self.jobdir = tempfile.mkdtemp() - self.create_scheduler() - - def tearDown(self): - self.close_scheduler() - - shutil.rmtree(self.jobdir) - self.jobdir = None - - def test_length(self): - self.assertFalse(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), 0) - - for url in _URLS: - self.scheduler.enqueue_request(Request(url)) - - self.close_scheduler() - self.create_scheduler() - - self.assertTrue(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), len(_URLS)) - - def test_dequeue(self): - for url in _URLS: - self.scheduler.enqueue_request(Request(url)) - - self.close_scheduler() - self.create_scheduler() - - urls = set() - while self.scheduler.has_pending_requests(): - urls.add(self.scheduler.next_request().url) - - self.assertEqual(urls, _URLS) - - def test_dequeue_priorities(self): - for url, priority in _PRIORITIES: - self.scheduler.enqueue_request(Request(url, priority=priority)) - - self.close_scheduler() - self.create_scheduler() - - priorities = [] - while self.scheduler.has_pending_requests(): - priorities.append(self.scheduler.next_request().priority) - - self.assertEqual( - priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x) - ) - - -class TestSchedulerInMemory(BaseSchedulerInMemoryTester, unittest.TestCase): +class TestSchedulerInMemory(TestSchedulerInMemoryBase): priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue" -class TestSchedulerOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): +class TestSchedulerOnDisk(TestSchedulerOnDiskBase): priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue" @@ -198,37 +252,26 @@ _URLS_WITH_SLOTS = [ ] -class TestMigration(unittest.TestCase): - def setUp(self): - self.tmpdir = tempfile.mkdtemp() +class TestMigration: + @coroutine_test + async def test_migration(self, tmp_path: Path) -> None: + async with create_scheduler( + "scrapy.pqueues.ScrapyPriorityQueue", tmp_path + ) as prev_scheduler: + for url in _URLS: + prev_scheduler.enqueue_request(Request(url)) - def tearDown(self): - shutil.rmtree(self.tmpdir) - - def _migration(self, tmp_dir): - prev_scheduler_handler = SchedulerHandler() - prev_scheduler_handler.priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue" - prev_scheduler_handler.jobdir = tmp_dir - - prev_scheduler_handler.create_scheduler() - for url in _URLS: - prev_scheduler_handler.scheduler.enqueue_request(Request(url)) - prev_scheduler_handler.close_scheduler() - - next_scheduler_handler = SchedulerHandler() - next_scheduler_handler.priority_queue_cls = ( - "scrapy.pqueues.DownloaderAwarePriorityQueue" - ) - next_scheduler_handler.jobdir = tmp_dir - - next_scheduler_handler.create_scheduler() - - def test_migration(self): - with self.assertRaises(ValueError): - self._migration(self.tmpdir) + with pytest.raises( + ValueError, + match="DownloaderAwarePriorityQueue accepts ``slot_startprios`` as a dict", + ): + async with create_scheduler( + "scrapy.pqueues.DownloaderAwarePriorityQueue", tmp_path + ): + pass -def _is_scheduling_fair(enqueued_slots, dequeued_slots): +def _is_scheduling_fair(enqueued_slots: list[str], dequeued_slots: list[str]) -> bool: """ We enqueued same number of requests for every slot. Assert correct order, e.g. @@ -253,50 +296,59 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots): return True -class DownloaderAwareSchedulerTestMixin: - priority_queue_cls: Optional[str] = "scrapy.pqueues.DownloaderAwarePriorityQueue" +class DownloaderAwareSchedulerTestMixin(TestSchedulerBase): reopen = False + priority_queue_cls = "scrapy.pqueues.DownloaderAwarePriorityQueue" - def test_logic(self): - for url, slot in _URLS_WITH_SLOTS: - request = Request(url) - request.meta[Downloader.DOWNLOAD_SLOT] = slot - self.scheduler.enqueue_request(request) + @coroutine_test + async def test_logic(self, jobdir: Path | None) -> None: + def _setup(scheduler: Scheduler) -> None: + for url, slot in _URLS_WITH_SLOTS: + request = Request(url) + request.meta[Downloader.DOWNLOAD_SLOT] = slot + scheduler.enqueue_request(request) + + def _assert(scheduler: Scheduler) -> None: + dequeued_slots: list[str] = [] + requests: list[Request] = [] + assert scheduler.crawler + assert scheduler.crawler.engine + downloader = scheduler.crawler.engine.downloader + assert isinstance(downloader, MockDownloader) + while scheduler.has_pending_requests(): + request = scheduler.next_request() + assert request is not None + slot = downloader.get_slot_key(request) + dequeued_slots.append(slot) + downloader.increment(slot) + requests.append(request) + + for request in requests: + slot = downloader.get_slot_key(request) + downloader.decrement(slot) + + assert _is_scheduling_fair([s for u, s in _URLS_WITH_SLOTS], dequeued_slots) + assert sum(len(s.active) for s in downloader.slots.values()) == 0 if self.reopen: - self.close_scheduler() - self.create_scheduler() - - dequeued_slots = [] - requests = [] - downloader = self.mock_crawler.engine.downloader - while self.scheduler.has_pending_requests(): - request = self.scheduler.next_request() - # pylint: disable=protected-access - slot = downloader._get_slot_key(request, None) - dequeued_slots.append(slot) - downloader.increment(slot) - requests.append(request) - - for request in requests: - # pylint: disable=protected-access - slot = downloader._get_slot_key(request, None) - downloader.decrement(slot) - - self.assertTrue( - _is_scheduling_fair(list(s for u, s in _URLS_WITH_SLOTS), dequeued_slots) - ) - self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0) + async with self.create_scheduler(jobdir) as scheduler: + _setup(scheduler) + async with self.create_scheduler(jobdir) as scheduler: + _assert(scheduler) + else: + async with self.create_scheduler(jobdir) as scheduler: + _setup(scheduler) + _assert(scheduler) class TestSchedulerWithDownloaderAwareInMemory( - DownloaderAwareSchedulerTestMixin, BaseSchedulerInMemoryTester, unittest.TestCase + DownloaderAwareSchedulerTestMixin, TestSchedulerInMemoryBase ): pass class TestSchedulerWithDownloaderAwareOnDisk( - DownloaderAwareSchedulerTestMixin, BaseSchedulerOnDiskTester, unittest.TestCase + DownloaderAwareSchedulerTestMixin, TestSchedulerOnDiskBase ): reopen = True @@ -310,8 +362,8 @@ class StartUrlsSpider(Spider): pass -class TestIntegrationWithDownloaderAwareInMemory(TestCase): - def setUp(self): +class TestIntegrationWithDownloaderAwareInMemory: + def setup_method(self): self.crawler = get_crawler( spidercls=StartUrlsSpider, settings_dict={ @@ -320,33 +372,32 @@ class TestIntegrationWithDownloaderAwareInMemory(TestCase): }, ) - @defer.inlineCallbacks - def tearDown(self): - yield self.crawler.stop() - - @defer.inlineCallbacks + @inline_callbacks_test def test_integration_downloader_aware_priority_queue(self): with MockServer() as mockserver: url = mockserver.url("/status?n=200", is_secure=False) start_urls = [url] * 6 yield self.crawler.crawl(start_urls) - self.assertEqual( - self.crawler.stats.get_value("downloader/response_count"), - len(start_urls), + assert self.crawler.stats.get_value("downloader/response_count") == len( + start_urls ) -class TestIncompatibility(unittest.TestCase): +class TestIncompatibility: def _incompatible(self): - settings = dict( - SCHEDULER_PRIORITY_QUEUE="scrapy.pqueues.DownloaderAwarePriorityQueue", - CONCURRENT_REQUESTS_PER_IP=1, - ) + settings = { + "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.DownloaderAwarePriorityQueue", + "CONCURRENT_REQUESTS_PER_IP": 1, + } crawler = get_crawler(Spider, settings) scheduler = Scheduler.from_crawler(crawler) spider = Spider(name="spider") scheduler.open(spider) def test_incompatibility(self): - with self.assertRaises(ValueError): - self._incompatible() + with warnings.catch_warnings(): + warnings.filterwarnings("ignore") + with pytest.raises( + ValueError, match="does not support CONCURRENT_REQUESTS_PER_IP" + ): + self._incompatible() diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 76ca777a8..db023e1f8 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -1,17 +1,19 @@ -from typing import Dict, Optional -from unittest import TestCase -from urllib.parse import urljoin, urlparse +from __future__ import annotations +from urllib.parse import urljoin + +import pytest from testfixtures import LogCapture from twisted.internet import defer -from twisted.trial.unittest import TestCase as TwistedTestCase from scrapy.core.scheduler import BaseScheduler from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.request import fingerprint from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer +from tests.mockserver.http import MockServer +from tests.utils.decorators import inline_callbacks_test PATHS = ["/a", "/b", "/c"] URLS = [urljoin("https://example.org", p) for p in PATHS] @@ -19,7 +21,7 @@ URLS = [urljoin("https://example.org", p) for p in PATHS] class MinimalScheduler: def __init__(self) -> None: - self.requests: Dict[bytes, Request] = {} + self.requests: dict[bytes, Request] = {} def has_pending_requests(self) -> bool: return bool(self.requests) @@ -31,134 +33,133 @@ class MinimalScheduler: return True return False - def next_request(self) -> Optional[Request]: + def next_request(self) -> Request | None: if self.has_pending_requests(): - fp, request = self.requests.popitem() + _, request = self.requests.popitem() return request return None class SimpleScheduler(MinimalScheduler): - def open(self, spider: Spider) -> defer.Deferred: + def open(self, spider: Spider) -> defer.Deferred[str]: return defer.succeed("open") - def close(self, reason: str) -> defer.Deferred: + def close(self, reason: str) -> defer.Deferred[str]: return defer.succeed("close") def __len__(self) -> int: return len(self.requests) -class TestSpider(Spider): - name = "test" +class PathsSpider(Spider): + name = "paths" def __init__(self, mockserver, *args, **kwargs): super().__init__(*args, **kwargs) self.start_urls = map(mockserver.url, PATHS) def parse(self, response): - return {"path": urlparse(response.url).path} + return {"path": urlparse_cached(response).path} class InterfaceCheckMixin: def test_scheduler_class(self): - self.assertTrue(isinstance(self.scheduler, BaseScheduler)) - self.assertTrue(issubclass(self.scheduler.__class__, BaseScheduler)) + assert isinstance(self.scheduler, BaseScheduler) + assert issubclass(self.scheduler.__class__, BaseScheduler) -class BaseSchedulerTest(TestCase, InterfaceCheckMixin): - def setUp(self): +class TestBaseScheduler(InterfaceCheckMixin): + def setup_method(self): self.scheduler = BaseScheduler() def test_methods(self): - self.assertIsNone(self.scheduler.open(Spider("foo"))) - self.assertIsNone(self.scheduler.close("finished")) - self.assertRaises(NotImplementedError, self.scheduler.has_pending_requests) - self.assertRaises( - NotImplementedError, - self.scheduler.enqueue_request, - Request("https://example.org"), - ) - self.assertRaises(NotImplementedError, self.scheduler.next_request) + assert self.scheduler.open(Spider("foo")) is None + assert self.scheduler.close("finished") is None + with pytest.raises(NotImplementedError): + self.scheduler.has_pending_requests() + with pytest.raises(NotImplementedError): + self.scheduler.enqueue_request(Request("https://example.org")) + with pytest.raises(NotImplementedError): + self.scheduler.next_request() -class MinimalSchedulerTest(TestCase, InterfaceCheckMixin): - def setUp(self): +class TestMinimalScheduler(InterfaceCheckMixin): + def setup_method(self): self.scheduler = MinimalScheduler() def test_open_close(self): - with self.assertRaises(AttributeError): + with pytest.raises(AttributeError): self.scheduler.open(Spider("foo")) - with self.assertRaises(AttributeError): + with pytest.raises(AttributeError): self.scheduler.close("finished") def test_len(self): - with self.assertRaises(AttributeError): + with pytest.raises(AttributeError): self.scheduler.__len__() - with self.assertRaises(TypeError): + with pytest.raises(TypeError): len(self.scheduler) def test_enqueue_dequeue(self): - self.assertFalse(self.scheduler.has_pending_requests()) + assert not self.scheduler.has_pending_requests() for url in URLS: - self.assertTrue(self.scheduler.enqueue_request(Request(url))) - self.assertFalse(self.scheduler.enqueue_request(Request(url))) - self.assertTrue(self.scheduler.has_pending_requests) + assert self.scheduler.enqueue_request(Request(url)) + assert not self.scheduler.enqueue_request(Request(url)) + assert self.scheduler.has_pending_requests dequeued = [] while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() dequeued.append(request.url) - self.assertEqual(set(dequeued), set(URLS)) - self.assertFalse(self.scheduler.has_pending_requests()) + assert set(dequeued) == set(URLS) + assert not self.scheduler.has_pending_requests() -class SimpleSchedulerTest(TwistedTestCase, InterfaceCheckMixin): - def setUp(self): +class TestSimpleScheduler(InterfaceCheckMixin): + def setup_method(self): self.scheduler = SimpleScheduler() - @defer.inlineCallbacks + @inline_callbacks_test def test_enqueue_dequeue(self): open_result = yield self.scheduler.open(Spider("foo")) - self.assertEqual(open_result, "open") - self.assertFalse(self.scheduler.has_pending_requests()) + assert open_result == "open" + assert not self.scheduler.has_pending_requests() for url in URLS: - self.assertTrue(self.scheduler.enqueue_request(Request(url))) - self.assertFalse(self.scheduler.enqueue_request(Request(url))) + assert self.scheduler.enqueue_request(Request(url)) + assert not self.scheduler.enqueue_request(Request(url)) - self.assertTrue(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), len(URLS)) + assert self.scheduler.has_pending_requests() + assert len(self.scheduler) == len(URLS) dequeued = [] while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() dequeued.append(request.url) - self.assertEqual(set(dequeued), set(URLS)) + assert set(dequeued) == set(URLS) - self.assertFalse(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), 0) + assert not self.scheduler.has_pending_requests() + assert len(self.scheduler) == 0 close_result = yield self.scheduler.close("") - self.assertEqual(close_result, "close") + assert close_result == "close" -class MinimalSchedulerCrawlTest(TwistedTestCase): +class TestMinimalSchedulerCrawl: scheduler_cls = MinimalScheduler - @defer.inlineCallbacks + @inline_callbacks_test def test_crawl(self): with MockServer() as mockserver: settings = { "SCHEDULER": self.scheduler_cls, } with LogCapture() as log: - crawler = get_crawler(TestSpider, settings) + crawler = get_crawler(PathsSpider, settings) yield crawler.crawl(mockserver) for path in PATHS: - self.assertIn(f"{{'path': '{path}'}}", str(log)) - self.assertIn(f"'item_scraped_count': {len(PATHS)}", str(log)) + assert f"{{'path': '{path}'}}" in str(log) + assert f"'item_scraped_count': {len(PATHS)}" in str(log) -class SimpleSchedulerCrawlTest(MinimalSchedulerCrawlTest): +class TestSimpleSchedulerCrawl(TestMinimalSchedulerCrawl): scheduler_cls = SimpleScheduler diff --git a/tests/test_scrapy__getattr__.py b/tests/test_scrapy__getattr__.py new file mode 100644 index 000000000..4c365859b --- /dev/null +++ b/tests/test_scrapy__getattr__.py @@ -0,0 +1,15 @@ +import warnings + + +def test_deprecated_concurrent_requests_per_ip_attribute(): + with warnings.catch_warnings(record=True) as warns: + from scrapy.settings.default_settings import ( # noqa: PLC0415 + CONCURRENT_REQUESTS_PER_IP, + ) + + assert CONCURRENT_REQUESTS_PER_IP is not None + assert isinstance(CONCURRENT_REQUESTS_PER_IP, int) + assert ( + "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead." + in warns[0].message.args + ) diff --git a/tests/test_selector.py b/tests/test_selector.py index 85527bba9..98595ac93 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -3,7 +3,6 @@ import weakref import parsel import pytest from packaging import version -from twisted.trial import unittest from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.selector import Selector @@ -12,7 +11,7 @@ PARSEL_VERSION = version.parse(getattr(parsel, "__version__", "0.0")) PARSEL_18_PLUS = PARSEL_VERSION >= version.parse("1.8.0") -class SelectorTestCase(unittest.TestCase): +class TestSelector: def test_simple_selection(self): """Simple selector tests""" body = b"

" @@ -20,57 +19,46 @@ class SelectorTestCase(unittest.TestCase): sel = Selector(response) xl = sel.xpath("//input") - self.assertEqual(2, len(xl)) + assert len(xl) == 2 for x in xl: assert isinstance(x, Selector) - self.assertEqual( - sel.xpath("//input").getall(), [x.get() for x in sel.xpath("//input")] - ) - self.assertEqual( - [x.get() for x in sel.xpath("//input[@name='a']/@name")], ["a"] - ) - self.assertEqual( - [ - x.get() - for x in sel.xpath( - "number(concat(//input[@name='a']/@value, //input[@name='b']/@value))" - ) - ], - ["12.0"], - ) - self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(), ["xpathrules"]) - self.assertEqual( - [ - x.get() - for x in sel.xpath( - "concat(//input[@name='a']/@value, //input[@name='b']/@value)" - ) - ], - ["12"], - ) + assert sel.xpath("//input").getall() == [x.get() for x in sel.xpath("//input")] + assert [x.get() for x in sel.xpath("//input[@name='a']/@name")] == ["a"] + assert [ + x.get() + for x in sel.xpath( + "number(concat(//input[@name='a']/@value, //input[@name='b']/@value))" + ) + ] == ["12.0"] + assert sel.xpath("concat('xpath', 'rules')").getall() == ["xpathrules"] + assert [ + x.get() + for x in sel.xpath( + "concat(//input[@name='a']/@value, //input[@name='b']/@value)" + ) + ] == ["12"] def test_root_base_url(self): body = b'' url = "http://example.com" response = TextResponse(url=url, body=body, encoding="utf-8") sel = Selector(response) - self.assertEqual(url, sel.root.base) + assert url == sel.root.base def test_flavor_detection(self): text = b'

Hello

' sel = Selector(XmlResponse("http://example.com", body=text, encoding="utf-8")) - self.assertEqual(sel.type, "xml") - self.assertEqual( - sel.xpath("//div").getall(), - ['

Hello

'], - ) + assert sel.type == "xml" + assert sel.xpath("//div").getall() == [ + '

Hello

' + ] sel = Selector(HtmlResponse("http://example.com", body=text, encoding="utf-8")) - self.assertEqual(sel.type, "html") - self.assertEqual( - sel.xpath("//div").getall(), ['

Hello

'] - ) + assert sel.type == "html" + assert sel.xpath("//div").getall() == [ + '

Hello

' + ] def test_http_header_encoding_precedence(self): # '\xa3' = pound symbol in unicode @@ -80,10 +68,10 @@ class SelectorTestCase(unittest.TestCase): meta = ( '' ) - head = "" + meta + "" + head = f"{meta}" body_content = '\xa3' - body = "" + body_content + "" - html = "" + head + body + "" + body = f"{body_content}" + html = f"{head}{body}" encoding = "utf-8" html_utf8 = html.encode(encoding) @@ -92,7 +80,7 @@ class SelectorTestCase(unittest.TestCase): url="http://example.com", headers=headers, body=html_utf8 ) x = Selector(response) - self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ["\xa3"]) + assert x.xpath("//span[@id='blank']/text()").getall() == ["\xa3"] def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence @@ -107,19 +95,17 @@ class SelectorTestCase(unittest.TestCase): """Check that classes are using slots and are weak-referenceable""" x = Selector(text="") weakref.ref(x) - assert not hasattr( - x, "__dict__" - ), f"{x.__class__.__name__} does not use __slots__" + assert not hasattr(x, "__dict__"), ( + f"{x.__class__.__name__} does not use __slots__" + ) def test_selector_bad_args(self): - with self.assertRaisesRegex(ValueError, "received both response and text"): + with pytest.raises(ValueError, match="received both response and text"): Selector(TextResponse(url="http://example.com", body=b""), text="") -class JMESPathTestCase(unittest.TestCase): - @pytest.mark.skipif( - not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" - ) +@pytest.mark.skipif(not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath") +class TestJMESPath: def test_json_has_html(self) -> None: """Sometimes the information is returned in a json wrapper""" @@ -149,20 +135,14 @@ class JMESPathTestCase(unittest.TestCase): } """ resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") - self.assertEqual( - resp.jmespath("html").get(), - "
def
", + assert ( + resp.jmespath("html").get() + == "
def
" ) - self.assertEqual( - resp.jmespath("html").xpath("//div/a/text()").getall(), - ["a", "b", "d"], - ) - self.assertEqual(resp.jmespath("html").css("div > b").getall(), ["f"]) - self.assertEqual(resp.jmespath("content").jmespath("name.age").get(), "18") + assert resp.jmespath("html").xpath("//div/a/text()").getall() == ["a", "b", "d"] + assert resp.jmespath("html").css("div > b").getall() == ["f"] + assert resp.jmespath("content").jmespath("name.age").get() == "18" - @pytest.mark.skipif( - not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" - ) def test_html_has_json(self) -> None: body = """
@@ -194,19 +174,20 @@ class JMESPathTestCase(unittest.TestCase):
""" resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") - self.assertEqual( - resp.xpath("//div/content/text()").jmespath("user[*].name").getall(), - ["A", "B", "C", "D"], - ) - self.assertEqual( - resp.xpath("//div/content").jmespath("user[*].name").getall(), - ["A", "B", "C", "D"], - ) - self.assertEqual(resp.xpath("//div/content").jmespath("total").get(), "4") + assert resp.xpath("//div/content/text()").jmespath("user[*].name").getall() == [ + "A", + "B", + "C", + "D", + ] + assert resp.xpath("//div/content").jmespath("user[*].name").getall() == [ + "A", + "B", + "C", + "D", + ] + assert resp.xpath("//div/content").jmespath("total").get() == "4" - @pytest.mark.skipif( - not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" - ) def test_jmestpath_with_re(self) -> None: body = """
@@ -238,38 +219,35 @@ class JMESPathTestCase(unittest.TestCase):
""" resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") - self.assertEqual( - resp.xpath("//div/content/text()").jmespath("user[*].name").re(r"(\w+)"), - ["A", "B", "C", "D"], - ) - self.assertEqual( - resp.xpath("//div/content").jmespath("user[*].name").re(r"(\w+)"), - ["A", "B", "C", "D"], + assert resp.xpath("//div/content/text()").jmespath("user[*].name").re( + r"(\w+)" + ) == ["A", "B", "C", "D"] + assert resp.xpath("//div/content").jmespath("user[*].name").re(r"(\w+)") == [ + "A", + "B", + "C", + "D", + ] + + assert resp.xpath("//div/content").jmespath("unavailable").re(r"(\d+)") == [] + + assert ( + resp.xpath("//div/content").jmespath("unavailable").re_first(r"(\d+)") + is None ) - self.assertEqual( - resp.xpath("//div/content").jmespath("unavailable").re(r"(\d+)"), [] - ) + assert resp.xpath("//div/content").jmespath("user[*].age.to_string(@)").re( + r"(\d+)" + ) == ["18", "32", "22", "25"] - self.assertEqual( - resp.xpath("//div/content").jmespath("unavailable").re_first(r"(\d+)"), - None, - ) - self.assertEqual( - resp.xpath("//div/content") - .jmespath("user[*].age.to_string(@)") - .re(r"(\d+)"), - ["18", "32", "22", "25"], - ) - - @pytest.mark.skipif(PARSEL_18_PLUS, reason="parsel >= 1.8 supports jmespath") - def test_jmespath_not_available(my_json_page) -> None: - body = """ - { - "website": {"name": "Example"} - } - """ - resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") - with pytest.raises(AttributeError): - resp.jmespath("website.name").get() +@pytest.mark.skipif(PARSEL_18_PLUS, reason="parsel >= 1.8 supports jmespath") +def test_jmespath_not_available() -> None: + body = """ + { + "website": {"name": "Example"} + } + """ + resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") + with pytest.raises(AttributeError): + resp.jmespath("website.name").get() diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 3fde5e8c5..1d4c3d487 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,8 +1,13 @@ -import unittest +# pylint: disable=unsubscriptable-object,unsupported-membership-test,use-implicit-booleaness-not-comparison +# (too many false positives) + +import logging +import warnings from unittest import mock import pytest +from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.settings import ( SETTINGS_PRIORITIES, BaseSettings, @@ -10,35 +15,44 @@ from scrapy.settings import ( SettingsAttribute, get_settings_priority, ) +from scrapy.settings import default_settings as scrapy_default_settings +from scrapy.utils.misc import build_from_crawler, load_object +from scrapy.utils.test import get_crawler from . import default_settings +NON_COMPONENT_PRIORITY_DICT_BASE_SETTING_NAMES = { + "DOWNLOAD_HANDLERS_BASE", + "FEED_EXPORTERS_BASE", + "FEED_STORAGES_BASE", +} -class SettingsGlobalFuncsTest(unittest.TestCase): + +class TestSettingsGlobalFuncs: def test_get_settings_priority(self): for prio_str, prio_num in SETTINGS_PRIORITIES.items(): - self.assertEqual(get_settings_priority(prio_str), prio_num) - self.assertEqual(get_settings_priority(99), 99) + assert get_settings_priority(prio_str) == prio_num + assert get_settings_priority(99) == 99 -class SettingsAttributeTest(unittest.TestCase): - def setUp(self): +class TestSettingsAttribute: + def setup_method(self): self.attribute = SettingsAttribute("value", 10) def test_set_greater_priority(self): self.attribute.set("value2", 20) - self.assertEqual(self.attribute.value, "value2") - self.assertEqual(self.attribute.priority, 20) + assert self.attribute.value == "value2" + assert self.attribute.priority == 20 def test_set_equal_priority(self): self.attribute.set("value2", 10) - self.assertEqual(self.attribute.value, "value2") - self.assertEqual(self.attribute.priority, 10) + assert self.attribute.value == "value2" + assert self.attribute.priority == 10 def test_set_less_priority(self): self.attribute.set("value2", 0) - self.assertEqual(self.attribute.value, "value") - self.assertEqual(self.attribute.priority, 10) + assert self.attribute.value == "value" + assert self.attribute.priority == 10 def test_overwrite_basesettings(self): original_dict = {"one": 10, "two": 20} @@ -47,73 +61,72 @@ class SettingsAttributeTest(unittest.TestCase): new_dict = {"three": 11, "four": 21} attribute.set(new_dict, 10) - self.assertIsInstance(attribute.value, BaseSettings) - self.assertCountEqual(attribute.value, new_dict) - self.assertCountEqual(original_settings, original_dict) + assert isinstance(attribute.value, BaseSettings) + assert set(attribute.value) == set(new_dict) + assert set(original_settings) == set(original_dict) new_settings = BaseSettings({"five": 12}, 0) attribute.set(new_settings, 0) # Insufficient priority - self.assertCountEqual(attribute.value, new_dict) + assert set(attribute.value) == set(new_dict) attribute.set(new_settings, 10) - self.assertCountEqual(attribute.value, new_settings) + assert set(attribute.value) == set(new_settings) def test_repr(self): - self.assertEqual( - repr(self.attribute), "" - ) + assert repr(self.attribute) == "" -class BaseSettingsTest(unittest.TestCase): - def setUp(self): +class TestBaseSettings: + def setup_method(self): self.settings = BaseSettings() def test_setdefault_not_existing_value(self): settings = BaseSettings() value = settings.setdefault("TEST_OPTION", "value") - self.assertEqual(settings["TEST_OPTION"], "value") - self.assertEqual(value, "value") - self.assertIsNotNone(value) + assert settings["TEST_OPTION"] == "value" + assert value == "value" + assert value is not None def test_setdefault_existing_value(self): settings = BaseSettings({"TEST_OPTION": "value"}) value = settings.setdefault("TEST_OPTION", None) - self.assertEqual(settings["TEST_OPTION"], "value") - self.assertEqual(value, "value") + assert settings["TEST_OPTION"] == "value" + assert value == "value" def test_set_new_attribute(self): self.settings.set("TEST_OPTION", "value", 0) - self.assertIn("TEST_OPTION", self.settings.attributes) + assert "TEST_OPTION" in self.settings.attributes attr = self.settings.attributes["TEST_OPTION"] - self.assertIsInstance(attr, SettingsAttribute) - self.assertEqual(attr.value, "value") - self.assertEqual(attr.priority, 0) + assert isinstance(attr, SettingsAttribute) + assert attr.value == "value" + assert attr.priority == 0 def test_set_settingsattribute(self): myattr = SettingsAttribute(0, 30) # Note priority 30 self.settings.set("TEST_ATTR", myattr, 10) - self.assertEqual(self.settings.get("TEST_ATTR"), 0) - self.assertEqual(self.settings.getpriority("TEST_ATTR"), 30) + assert self.settings.get("TEST_ATTR") == 0 + assert self.settings.getpriority("TEST_ATTR") == 30 def test_set_instance_identity_on_update(self): attr = SettingsAttribute("value", 0) self.settings.attributes = {"TEST_OPTION": attr} self.settings.set("TEST_OPTION", "othervalue", 10) - self.assertIn("TEST_OPTION", self.settings.attributes) - self.assertIs(attr, self.settings.attributes["TEST_OPTION"]) + assert "TEST_OPTION" in self.settings.attributes + assert attr is self.settings.attributes["TEST_OPTION"] def test_set_calls_settings_attributes_methods_on_update(self): attr = SettingsAttribute("value", 10) - with mock.patch.object(attr, "__setattr__") as mock_setattr, mock.patch.object( - attr, "set" - ) as mock_set: + with ( + mock.patch.object(attr, "__setattr__") as mock_setattr, + mock.patch.object(attr, "set") as mock_set, + ): self.settings.attributes = {"TEST_OPTION": attr} for priority in (0, 10, 20): self.settings.set("TEST_OPTION", "othervalue", priority) mock_set.assert_called_once_with("othervalue", priority) - self.assertFalse(mock_setattr.called) + assert not mock_setattr.called mock_set.reset_mock() mock_setattr.reset_mock() @@ -121,19 +134,19 @@ class BaseSettingsTest(unittest.TestCase): settings = BaseSettings() settings.set("key", "a", "default") settings["key"] = "b" - self.assertEqual(settings["key"], "b") - self.assertEqual(settings.getpriority("key"), 20) + assert settings["key"] == "b" + assert settings.getpriority("key") == 20 settings["key"] = "c" - self.assertEqual(settings["key"], "c") + assert settings["key"] == "c" settings["key2"] = "x" - self.assertIn("key2", settings) - self.assertEqual(settings["key2"], "x") - self.assertEqual(settings.getpriority("key2"), 20) + assert "key2" in settings + assert settings["key2"] == "x" + assert settings.getpriority("key2") == 20 def test_setdict_alias(self): with mock.patch.object(self.settings, "set") as mock_set: self.settings.setdict({"TEST_1": "value1", "TEST_2": "value2"}, 10) - self.assertEqual(mock_set.call_count, 2) + assert mock_set.call_count == 2 calls = [ mock.call("TEST_1", "value1", 10), mock.call("TEST_2", "value2", 10), @@ -148,10 +161,10 @@ class BaseSettingsTest(unittest.TestCase): self.settings.attributes = {} self.settings.setmodule(ModuleMock(), 10) - self.assertIn("UPPERCASE_VAR", self.settings.attributes) - self.assertNotIn("MIXEDcase_VAR", self.settings.attributes) - self.assertNotIn("lowercase_var", self.settings.attributes) - self.assertEqual(len(self.settings.attributes), 1) + assert "UPPERCASE_VAR" in self.settings.attributes + assert "MIXEDcase_VAR" not in self.settings.attributes + assert "lowercase_var" not in self.settings.attributes + assert len(self.settings.attributes) == 1 def test_setmodule_alias(self): with mock.patch.object(self.settings, "set") as mock_set: @@ -167,13 +180,13 @@ class BaseSettingsTest(unittest.TestCase): self.settings.attributes = {} self.settings.setmodule("tests.test_settings.default_settings", 10) - self.assertCountEqual(self.settings.attributes.keys(), ctrl_attributes.keys()) + assert set(self.settings.attributes) == set(ctrl_attributes) - for key in ctrl_attributes.keys(): + for key in ctrl_attributes: attr = self.settings.attributes[key] ctrl_attr = ctrl_attributes[key] - self.assertEqual(attr.value, ctrl_attr.value) - self.assertEqual(attr.priority, ctrl_attr.priority) + assert attr.value == ctrl_attr.value + assert attr.priority == ctrl_attr.priority def test_update(self): settings = BaseSettings({"key_lowprio": 0}, priority=0) @@ -185,21 +198,21 @@ class BaseSettingsTest(unittest.TestCase): custom_dict = {"key_lowprio": 2, "key_highprio": 12, "newkey_two": None} settings.update(custom_dict, priority=20) - self.assertEqual(settings["key_lowprio"], 2) - self.assertEqual(settings.getpriority("key_lowprio"), 20) - self.assertEqual(settings["key_highprio"], 10) - self.assertIn("newkey_two", settings) - self.assertEqual(settings.getpriority("newkey_two"), 20) + assert settings["key_lowprio"] == 2 + assert settings.getpriority("key_lowprio") == 20 + assert settings["key_highprio"] == 10 + assert "newkey_two" in settings + assert settings.getpriority("newkey_two") == 20 settings.update(custom_settings) - self.assertEqual(settings["key_lowprio"], 1) - self.assertEqual(settings.getpriority("key_lowprio"), 30) - self.assertEqual(settings["key_highprio"], 10) - self.assertIn("newkey_one", settings) - self.assertEqual(settings.getpriority("newkey_one"), 50) + assert settings["key_lowprio"] == 1 + assert settings.getpriority("key_lowprio") == 30 + assert settings["key_highprio"] == 10 + assert "newkey_one" in settings + assert settings.getpriority("newkey_one") == 50 settings.update({"key_lowprio": 3}, priority=20) - self.assertEqual(settings["key_lowprio"], 1) + assert settings["key_lowprio"] == 1 @pytest.mark.xfail( raises=TypeError, reason="BaseSettings.update doesn't support kwargs input" @@ -219,24 +232,24 @@ class BaseSettingsTest(unittest.TestCase): def test_update_jsonstring(self): settings = BaseSettings({"number": 0, "dict": BaseSettings({"key": "val"})}) settings.update('{"number": 1, "newnumber": 2}') - self.assertEqual(settings["number"], 1) - self.assertEqual(settings["newnumber"], 2) + assert settings["number"] == 1 + assert settings["newnumber"] == 2 settings.set("dict", '{"key": "newval", "newkey": "newval2"}') - self.assertEqual(settings["dict"]["key"], "newval") - self.assertEqual(settings["dict"]["newkey"], "newval2") + assert settings["dict"]["key"] == "newval" + assert settings["dict"]["newkey"] == "newval2" def test_delete(self): settings = BaseSettings({"key": None}) settings.set("key_highprio", None, priority=50) settings.delete("key") settings.delete("key_highprio") - self.assertNotIn("key", settings) - self.assertIn("key_highprio", settings) + assert "key" not in settings + assert "key_highprio" in settings del settings["key_highprio"] - self.assertNotIn("key_highprio", settings) - with self.assertRaises(KeyError): + assert "key_highprio" not in settings + with pytest.raises(KeyError): settings.delete("notkey") - with self.assertRaises(KeyError): + with pytest.raises(KeyError): del settings["notkey"] def test_get(self): @@ -259,6 +272,7 @@ class BaseSettingsTest(unittest.TestCase): "TEST_FLOAT2": "123.45", "TEST_LIST1": ["one", "two"], "TEST_LIST2": "one,two", + "TEST_LIST3": "", "TEST_STR": "value", "TEST_DICT1": {"key1": "val1", "ke2": 3}, "TEST_DICT2": '{"key1": "val1", "ke2": 3}', @@ -269,47 +283,58 @@ class BaseSettingsTest(unittest.TestCase): for key, value in test_configuration.items() } - self.assertTrue(settings.getbool("TEST_ENABLED1")) - self.assertTrue(settings.getbool("TEST_ENABLED2")) - self.assertTrue(settings.getbool("TEST_ENABLED3")) - self.assertTrue(settings.getbool("TEST_ENABLED4")) - self.assertTrue(settings.getbool("TEST_ENABLED5")) - self.assertFalse(settings.getbool("TEST_ENABLEDx")) - self.assertTrue(settings.getbool("TEST_ENABLEDx", True)) - self.assertFalse(settings.getbool("TEST_DISABLED1")) - self.assertFalse(settings.getbool("TEST_DISABLED2")) - self.assertFalse(settings.getbool("TEST_DISABLED3")) - self.assertFalse(settings.getbool("TEST_DISABLED4")) - self.assertFalse(settings.getbool("TEST_DISABLED5")) - self.assertEqual(settings.getint("TEST_INT1"), 123) - self.assertEqual(settings.getint("TEST_INT2"), 123) - self.assertEqual(settings.getint("TEST_INTx"), 0) - self.assertEqual(settings.getint("TEST_INTx", 45), 45) - self.assertEqual(settings.getfloat("TEST_FLOAT1"), 123.45) - self.assertEqual(settings.getfloat("TEST_FLOAT2"), 123.45) - self.assertEqual(settings.getfloat("TEST_FLOATx"), 0.0) - self.assertEqual(settings.getfloat("TEST_FLOATx", 55.0), 55.0) - self.assertEqual(settings.getlist("TEST_LIST1"), ["one", "two"]) - self.assertEqual(settings.getlist("TEST_LIST2"), ["one", "two"]) - self.assertEqual(settings.getlist("TEST_LISTx"), []) - self.assertEqual(settings.getlist("TEST_LISTx", ["default"]), ["default"]) - self.assertEqual(settings["TEST_STR"], "value") - self.assertEqual(settings.get("TEST_STR"), "value") - self.assertEqual(settings["TEST_STRx"], None) - self.assertEqual(settings.get("TEST_STRx"), None) - self.assertEqual(settings.get("TEST_STRx", "default"), "default") - self.assertEqual(settings.getdict("TEST_DICT1"), {"key1": "val1", "ke2": 3}) - self.assertEqual(settings.getdict("TEST_DICT2"), {"key1": "val1", "ke2": 3}) - self.assertEqual(settings.getdict("TEST_DICT3"), {}) - self.assertEqual(settings.getdict("TEST_DICT3", {"key1": 5}), {"key1": 5}) - self.assertRaises(ValueError, settings.getdict, "TEST_LIST1") - self.assertRaises(ValueError, settings.getbool, "TEST_ENABLED_WRONG") - self.assertRaises(ValueError, settings.getbool, "TEST_DISABLED_WRONG") + assert settings.getbool("TEST_ENABLED1") + assert settings.getbool("TEST_ENABLED2") + assert settings.getbool("TEST_ENABLED3") + assert settings.getbool("TEST_ENABLED4") + assert settings.getbool("TEST_ENABLED5") + assert not settings.getbool("TEST_ENABLEDx") + assert settings.getbool("TEST_ENABLEDx", True) + assert not settings.getbool("TEST_DISABLED1") + assert not settings.getbool("TEST_DISABLED2") + assert not settings.getbool("TEST_DISABLED3") + assert not settings.getbool("TEST_DISABLED4") + assert not settings.getbool("TEST_DISABLED5") + assert settings.getint("TEST_INT1") == 123 + assert settings.getint("TEST_INT2") == 123 + assert settings.getint("TEST_INTx") == 0 + assert settings.getint("TEST_INTx", 45) == 45 + assert settings.getfloat("TEST_FLOAT1") == 123.45 + assert settings.getfloat("TEST_FLOAT2") == 123.45 + assert settings.getfloat("TEST_FLOATx") == 0.0 + assert settings.getfloat("TEST_FLOATx", 55.0) == 55.0 + assert settings.getlist("TEST_LIST1") == ["one", "two"] + assert settings.getlist("TEST_LIST2") == ["one", "two"] + assert settings.getlist("TEST_LIST3") == [] + assert settings.getlist("TEST_LISTx") == [] + assert settings.getlist("TEST_LISTx", ["default"]) == ["default"] + assert settings["TEST_STR"] == "value" + assert settings.get("TEST_STR") == "value" + assert settings["TEST_STRx"] is None + assert settings.get("TEST_STRx") is None + assert settings.get("TEST_STRx", "default") == "default" + assert settings.getdict("TEST_DICT1") == {"key1": "val1", "ke2": 3} + assert settings.getdict("TEST_DICT2") == {"key1": "val1", "ke2": 3} + assert settings.getdict("TEST_DICT3") == {} + assert settings.getdict("TEST_DICT3", {"key1": 5}) == {"key1": 5} + with pytest.raises( + ValueError, + match=r"dictionary update sequence element #0 has length 3; 2 is required|sequence of pairs expected", + ): + settings.getdict("TEST_LIST1") + with pytest.raises( + ValueError, match="Supported values for boolean settings are" + ): + settings.getbool("TEST_ENABLED_WRONG") + with pytest.raises( + ValueError, match="Supported values for boolean settings are" + ): + settings.getbool("TEST_DISABLED_WRONG") def test_getpriority(self): settings = BaseSettings({"key": "value"}, priority=99) - self.assertEqual(settings.getpriority("key"), 99) - self.assertEqual(settings.getpriority("nonexistentkey"), None) + assert settings.getpriority("key") == 99 + assert settings.getpriority("nonexistentkey") is None def test_getwithbase(self): s = BaseSettings( @@ -320,16 +345,16 @@ class BaseSettingsTest(unittest.TestCase): } ) s["TEST"].set(2, 200, "cmdline") - self.assertCountEqual(s.getwithbase("TEST"), {1: 1, 2: 200, 3: 30}) - self.assertCountEqual(s.getwithbase("HASNOBASE"), s["HASNOBASE"]) - self.assertEqual(s.getwithbase("NONEXISTENT"), {}) + assert set(s.getwithbase("TEST")) == {1, 2, 3} + assert set(s.getwithbase("HASNOBASE")) == set(s["HASNOBASE"]) + assert s.getwithbase("NONEXISTENT") == {} def test_maxpriority(self): # Empty settings should return 'default' - self.assertEqual(self.settings.maxpriority(), 0) + assert self.settings.maxpriority() == 0 self.settings.set("A", 0, 10) self.settings.set("B", 0, 30) - self.assertEqual(self.settings.maxpriority(), 30) + assert self.settings.maxpriority() == 30 def test_copy(self): values = { @@ -343,17 +368,15 @@ class BaseSettingsTest(unittest.TestCase): self.settings.setdict(values) copy = self.settings.copy() self.settings.set("TEST_BOOL", False) - self.assertTrue(copy.get("TEST_BOOL")) + assert copy.get("TEST_BOOL") test_list = self.settings.get("TEST_LIST") test_list.append("three") - self.assertListEqual(copy.get("TEST_LIST"), ["one", "two"]) + assert copy.get("TEST_LIST") == ["one", "two"] test_list_of_lists = self.settings.get("TEST_LIST_OF_LISTS") test_list_of_lists[0].append("first_three") - self.assertListEqual( - copy.get("TEST_LIST_OF_LISTS")[0], ["first_one", "first_two"] - ) + assert copy.get("TEST_LIST_OF_LISTS")[0] == ["first_one", "first_two"] def test_copy_to_dict(self): s = BaseSettings( @@ -366,85 +389,220 @@ class BaseSettingsTest(unittest.TestCase): "HASNOBASE": BaseSettings({3: 3000}, "default"), } ) - self.assertDictEqual( - s.copy_to_dict(), - { - "HASNOBASE": {3: 3000}, - "TEST": {1: 10, 3: 30}, - "TEST_BASE": {1: 1, 2: 2}, - "TEST_LIST": [1, 2], - "TEST_BOOLEAN": False, - "TEST_STRING": "a string", - }, - ) + assert s.copy_to_dict() == { + "HASNOBASE": {3: 3000}, + "TEST": {1: 10, 3: 30}, + "TEST_BASE": {1: 1, 2: 2}, + "TEST_LIST": [1, 2], + "TEST_BOOLEAN": False, + "TEST_STRING": "a string", + } def test_freeze(self): self.settings.freeze() - with self.assertRaises(TypeError) as cm: + with pytest.raises( + TypeError, match="Trying to modify an immutable Settings object" + ): self.settings.set("TEST_BOOL", False) - self.assertEqual( - str(cm.exception), "Trying to modify an immutable Settings object" - ) def test_frozencopy(self): frozencopy = self.settings.frozencopy() - self.assertTrue(frozencopy.frozen) - self.assertIsNot(frozencopy, self.settings) + assert frozencopy.frozen + assert frozencopy is not self.settings + + def test_getwithbase_for_dotted_keys(self): + settings = BaseSettings( + { + "FEED_EXPORTERS_BASE": BaseSettings({"json": "foo"}), + "FEED_EXPORTERS": BaseSettings({"csv.gz": "bar"}), + } + ) + value = settings.getwithbase("FEED_EXPORTERS") + assert isinstance(value, BaseSettings) + assert dict(value) == { + "json": "foo", + "csv.gz": "bar", + } + + @pytest.mark.parametrize( + ("key", "exception"), + [ + pytest.param(1, TypeError, id="type-error"), + pytest.param("foo", ValueError, id="value-error"), + pytest.param("csv.gz", NameError, id="name-error"), + ], + ) + def test_get_component_priority_dict_with_base_handles_load_object_exceptions( + self, key, exception + ): + with pytest.raises(exception): + load_object(key) + + settings = BaseSettings( + { + "FOO": BaseSettings({key: 1}), + } + ) + value = settings.get_component_priority_dict_with_base("FOO") + + assert isinstance(value, BaseSettings) + assert dict(value) == {key: 1} + + def test_get_component_priority_dict_with_base_override_none_by_type(self): + settings = BaseSettings() + setting_names = set() + for k, v in scrapy_default_settings.__dict__.items(): + if ( + not k.endswith("_BASE") + or k in NON_COMPONENT_PRIORITY_DICT_BASE_SETTING_NAMES + ): + continue + settings[k] = v + setting_name = k[: -len("_BASE")] + setting_names.add(setting_name) + settings[setting_name] = { + load_object(import_path): None for import_path in v + } + for setting_name in setting_names: + value = settings.get_component_priority_dict_with_base(setting_name) + assert not dict(value) + + def test_get_component_priority_dict_with_base_override_value_by_type(self): + settings = BaseSettings() + setting_names = set() + value = 0 + for k, v in scrapy_default_settings.__dict__.items(): + if ( + not k.endswith("_BASE") + or k in NON_COMPONENT_PRIORITY_DICT_BASE_SETTING_NAMES + ): + continue + settings[k] = v + setting_name = k[: -len("_BASE")] + setting_names.add(setting_name) + settings[setting_name] = { + load_object(import_path): value for import_path in v + } + for setting_name in setting_names: + assert ( + settings.get_component_priority_dict_with_base(setting_name) + == settings[setting_name] + ) + + def test_getwithbase_for_non_component_priority_dicts(self): + settings = BaseSettings() + for base_name in NON_COMPONENT_PRIORITY_DICT_BASE_SETTING_NAMES: + base_value = getattr(scrapy_default_settings, base_name) + settings[base_name] = BaseSettings(base_value) + assert len(base_value) >= 2 + keys = list(base_value) + values = list(base_value.values()) + override = {keys[0]: values[1]} + expected = dict(base_value) + expected[keys[0]] = values[1] + name = base_name[: -len("_BASE")] + settings[name] = BaseSettings(override) + value = settings.getwithbase(name) + assert isinstance(value, BaseSettings) + assert dict(value) == expected + + def test_get_component_priority_dict_with_base_warns_on_duplicate_import_paths( + self, caplog + ): + settings = BaseSettings() + settings["FOO"] = BaseSettings( + { + "scrapy.Request": 1, + "scrapy.http.Request": 2, + } + ) + with caplog.at_level(logging.WARNING): + value = settings.get_component_priority_dict_with_base("FOO") + assert isinstance(value, BaseSettings) + assert dict(value) == {"scrapy.http.Request": 2} + assert caplog.records, "Expected a warning to be logged" + msg = caplog.records[0].message + assert "scrapy.http.request.Request" in msg + + def test_get_component_priority_dict_with_base_warns_on_duplicate_mixed_type_and_path( + self, caplog + ): + settings = BaseSettings() + settings["FOO"] = BaseSettings( + {Component1: 1, "tests.test_settings.Component1": 2} + ) + with caplog.at_level(logging.WARNING): + value = settings.get_component_priority_dict_with_base("FOO") + assert isinstance(value, BaseSettings) + assert dict(value) == {"tests.test_settings.Component1": 2} + assert caplog.records, "Expected a warning to be logged" + msg = caplog.records[0].message + assert "tests.test_settings.Component1" in msg + + def test_getwithbase_invalid_setting_name(self): + settings = BaseSettings() + with pytest.raises( + ValueError, match="Base setting key must be a string, got 123" + ): + settings.getwithbase(123) + + def test_get_component_priority_dict_with_base_invalid_setting_name(self): + settings = BaseSettings() + with pytest.raises( + ValueError, match="Base setting key must be a string, got 123" + ): + settings.get_component_priority_dict_with_base(123) -class SettingsTest(unittest.TestCase): - def setUp(self): +class TestSettings: + def setup_method(self): self.settings = Settings() @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {"default": 10}) @mock.patch("scrapy.settings.default_settings", default_settings) def test_initial_defaults(self): settings = Settings() - self.assertEqual(len(settings.attributes), 2) - self.assertIn("TEST_DEFAULT", settings.attributes) + assert len(settings.attributes) == 2 + assert "TEST_DEFAULT" in settings.attributes attr = settings.attributes["TEST_DEFAULT"] - self.assertIsInstance(attr, SettingsAttribute) - self.assertEqual(attr.value, "defvalue") - self.assertEqual(attr.priority, 10) + assert isinstance(attr, SettingsAttribute) + assert attr.value == "defvalue" + assert attr.priority == 10 @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {}) @mock.patch("scrapy.settings.default_settings", {}) def test_initial_values(self): settings = Settings({"TEST_OPTION": "value"}, 10) - self.assertEqual(len(settings.attributes), 1) - self.assertIn("TEST_OPTION", settings.attributes) + assert len(settings.attributes) == 1 + assert "TEST_OPTION" in settings.attributes attr = settings.attributes["TEST_OPTION"] - self.assertIsInstance(attr, SettingsAttribute) - self.assertEqual(attr.value, "value") - self.assertEqual(attr.priority, 10) + assert isinstance(attr, SettingsAttribute) + assert attr.value == "value" + assert attr.priority == 10 @mock.patch("scrapy.settings.default_settings", default_settings) def test_autopromote_dicts(self): settings = Settings() mydict = settings.get("TEST_DICT") - self.assertIsInstance(mydict, BaseSettings) - self.assertIn("key", mydict) - self.assertEqual(mydict["key"], "val") - self.assertEqual(mydict.getpriority("key"), 0) + assert isinstance(mydict, BaseSettings) + assert "key" in mydict + assert mydict["key"] == "val" + assert mydict.getpriority("key") == 0 @mock.patch("scrapy.settings.default_settings", default_settings) def test_getdict_autodegrade_basesettings(self): settings = Settings() mydict = settings.getdict("TEST_DICT") - self.assertIsInstance(mydict, dict) - self.assertEqual(len(mydict), 1) - self.assertIn("key", mydict) - self.assertEqual(mydict["key"], "val") + assert isinstance(mydict, dict) + assert len(mydict) == 1 + assert "key" in mydict + assert mydict["key"] == "val" def test_passing_objects_as_values(self): - from scrapy.core.downloader.handlers.file import FileDownloadHandler - from scrapy.utils.misc import build_from_crawler - from scrapy.utils.test import get_crawler - class TestPipeline: - def process_item(self, i, s): + def process_item(self, i): return i settings = Settings( @@ -458,45 +616,638 @@ class SettingsTest(unittest.TestCase): } ) - self.assertIn("ITEM_PIPELINES", settings.attributes) + assert "ITEM_PIPELINES" in settings.attributes mypipeline, priority = settings.getdict("ITEM_PIPELINES").popitem() - self.assertEqual(priority, 800) - self.assertEqual(mypipeline, TestPipeline) - self.assertIsInstance(mypipeline(), TestPipeline) - self.assertEqual(mypipeline().process_item("item", None), "item") + assert priority == 800 + assert mypipeline == TestPipeline + assert isinstance(mypipeline(), TestPipeline) + assert mypipeline().process_item("item") == "item" myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp") - self.assertEqual(myhandler, FileDownloadHandler) + assert myhandler == FileDownloadHandler myhandler_instance = build_from_crawler(myhandler, get_crawler()) - self.assertIsInstance(myhandler_instance, FileDownloadHandler) - self.assertTrue(hasattr(myhandler_instance, "download_request")) + assert isinstance(myhandler_instance, FileDownloadHandler) + assert hasattr(myhandler_instance, "download_request") def test_pop_item_with_default_value(self): settings = Settings() - with self.assertRaises(KeyError): + with pytest.raises(KeyError): settings.pop("DUMMY_CONFIG") dummy_config_value = settings.pop("DUMMY_CONFIG", "dummy_value") - self.assertEqual(dummy_config_value, "dummy_value") + assert dummy_config_value == "dummy_value" def test_pop_item_with_immutable_settings(self): settings = Settings( {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"} ) - self.assertEqual(settings.pop("DUMMY_CONFIG"), "dummy_value") + assert settings.pop("DUMMY_CONFIG") == "dummy_value" settings.freeze() - with self.assertRaises(TypeError) as error: + with pytest.raises( + TypeError, match="Trying to modify an immutable Settings object" + ): settings.pop("OTHER_DUMMY_CONFIG") - self.assertEqual( - str(error.exception), "Trying to modify an immutable Settings object" - ) + +@pytest.mark.parametrize( + ("before", "name", "item", "after"), + [ + ({}, "FOO", "BAR", {"FOO": ["BAR"]}), + ({"FOO": []}, "FOO", "BAR", {"FOO": ["BAR"]}), + ({"FOO": ["BAR"]}, "FOO", "BAZ", {"FOO": ["BAR", "BAZ"]}), + ({"FOO": ["BAR"]}, "FOO", "BAR", {"FOO": ["BAR"]}), + ({"FOO": ""}, "FOO", "BAR", {"FOO": ["BAR"]}), + ({"FOO": "BAR"}, "FOO", "BAR", {"FOO": "BAR"}), + ({"FOO": "BAR"}, "FOO", "BAZ", {"FOO": ["BAR", "BAZ"]}), + ({"FOO": "BAR,BAZ"}, "FOO", "BAZ", {"FOO": "BAR,BAZ"}), + ({"FOO": "BAR,BAZ"}, "FOO", "QUX", {"FOO": ["BAR", "BAZ", "QUX"]}), + ], +) +def test_add_to_list(before, name, item, after): + settings = BaseSettings(before, priority=0) + settings.add_to_list(name, item) + expected_priority = settings.getpriority(name) or 0 + expected_settings = BaseSettings(after, priority=expected_priority) + assert settings == expected_settings, ( + f"{settings[name]=} != {expected_settings[name]=}" + ) + assert settings.getpriority(name) == expected_settings.getpriority(name) -if __name__ == "__main__": - unittest.main() +@pytest.mark.parametrize( + ("before", "name", "item", "after"), + [ + ({}, "FOO", "BAR", ValueError), + ({"FOO": ["BAR"]}, "FOO", "BAR", {"FOO": []}), + ({"FOO": ["BAR"]}, "FOO", "BAZ", ValueError), + ({"FOO": ["BAR", "BAZ"]}, "FOO", "BAR", {"FOO": ["BAZ"]}), + ({"FOO": ""}, "FOO", "BAR", ValueError), + ({"FOO": "[]"}, "FOO", "BAR", ValueError), + ({"FOO": "BAR"}, "FOO", "BAR", {"FOO": []}), + ({"FOO": "BAR"}, "FOO", "BAZ", ValueError), + ({"FOO": "BAR,BAZ"}, "FOO", "BAR", {"FOO": ["BAZ"]}), + ], +) +def test_remove_from_list(before, name, item, after): + settings = BaseSettings(before, priority=0) + + if isinstance(after, type) and issubclass(after, Exception): + with pytest.raises(after): + settings.remove_from_list(name, item) + return + + settings.remove_from_list(name, item) + expected_priority = settings.getpriority(name) or 0 + expected_settings = BaseSettings(after, priority=expected_priority) + assert settings == expected_settings, ( + f"{settings[name]=} != {expected_settings[name]=}" + ) + assert settings.getpriority(name) == expected_settings.getpriority(name) + + +def test_deprecated_concurrent_requests_per_ip_setting(): + with warnings.catch_warnings(record=True) as warns: + settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1}) + settings.get("CONCURRENT_REQUESTS_PER_IP") + + assert ( + str(warns[0].message) + == "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead." + ) + + +class Component1: + pass + + +Component1Alias = Component1 + + +class Component1Subclass(Component1): + pass + + +Component1SubclassAlias = Component1Subclass + + +class Component2: + pass + + +class Component3: + pass + + +class Component4: + pass + + +@pytest.mark.parametrize( + ("before", "name", "old_cls", "new_cls", "priority", "after"), + [ + ({}, "FOO", Component1, Component2, None, KeyError), + ( + {"FOO": {Component1: 1}}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": {Component1: 1}}, + "FOO", + Component1, + Component2, + 2, + {"FOO": {Component2: 2}}, + ), + ( + {"FOO": {"tests.test_settings.Component1": 1}}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": {Component1Alias: 1}}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": {Component1Alias: 1}}, + "FOO", + Component1, + Component2, + 2, + {"FOO": {Component2: 2}}, + ), + ( + {"FOO": {"tests.test_settings.Component1Alias": 1}}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": {"tests.test_settings.Component1Alias": 1}}, + "FOO", + Component1, + Component2, + 2, + {"FOO": {Component2: 2}}, + ), + ( + { + "FOO": { + "tests.test_settings.Component1": 1, + "tests.test_settings.Component1Alias": 2, + } + }, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 2}}, + ), + ( + { + "FOO": { + "tests.test_settings.Component1": 1, + "tests.test_settings.Component1Alias": 2, + } + }, + "FOO", + Component1, + Component2, + 3, + {"FOO": {Component2: 3}}, + ), + ( + {"FOO": '{"tests.test_settings.Component1": 1}'}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": '{"tests.test_settings.Component1": 1}'}, + "FOO", + Component1, + Component2, + 2, + {"FOO": {Component2: 2}}, + ), + ( + {"FOO": '{"tests.test_settings.Component1Alias": 1}'}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": '{"tests.test_settings.Component1Alias": 1}'}, + "FOO", + Component1, + Component2, + 2, + {"FOO": {Component2: 2}}, + ), + ( + { + "FOO": '{"tests.test_settings.Component1": 1, "tests.test_settings.Component1Alias": 2}' + }, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 2}}, + ), + ( + { + "FOO": '{"tests.test_settings.Component1": 1, "tests.test_settings.Component1Alias": 2}' + }, + "FOO", + Component1, + Component2, + 3, + {"FOO": {Component2: 3}}, + ), + # If old_cls has None as value, raise KeyError. + ( + {"FOO": {Component1: None}}, + "FOO", + Component1, + Component2, + None, + KeyError, + ), + ( + {"FOO": '{"tests.test_settings.Component1": null}'}, + "FOO", + Component1, + Component2, + None, + KeyError, + ), + ( + {"FOO": {Component1: None, "tests.test_settings.Component1": None}}, + "FOO", + Component1, + Component2, + None, + KeyError, + ), + ( + {"FOO": {Component1: 1, "tests.test_settings.Component1": None}}, + "FOO", + Component1, + Component2, + None, + KeyError, + ), + ( + {"FOO": {Component1: None, "tests.test_settings.Component1": 1}}, + "FOO", + Component1, + Component2, + None, + KeyError, + ), + # Unrelated components are kept as is, as expected. + ( + { + "FOO": { + Component1: 1, + "tests.test_settings.Component2": 2, + Component3: 3, + } + }, + "FOO", + Component3, + Component4, + None, + { + "FOO": { + Component1: 1, + "tests.test_settings.Component2": 2, + Component4: 3, + } + }, + ), + ], +) +def test_replace_in_component_priority_dict( + before, name, old_cls, new_cls, priority, after +): + settings = BaseSettings(before, priority=0) + + if isinstance(after, type) and issubclass(after, Exception): + with pytest.raises(after): + settings.replace_in_component_priority_dict( + name, old_cls, new_cls, priority + ) + return + + expected_priority = settings.getpriority(name) or 0 + settings.replace_in_component_priority_dict(name, old_cls, new_cls, priority) + expected_settings = BaseSettings(after, priority=expected_priority) + assert settings == expected_settings + assert settings.getpriority(name) == expected_settings.getpriority(name) + + +@pytest.mark.parametrize( + ("before", "name", "cls", "priority", "after"), + [ + # Set + ({}, "FOO", Component1, None, {"FOO": {Component1: None}}), + ({}, "FOO", Component1, 0, {"FOO": {Component1: 0}}), + ({}, "FOO", Component1, 1, {"FOO": {Component1: 1}}), + # Add + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + None, + {"FOO": {Component1: 0, Component2: None}}, + ), + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + 0, + {"FOO": {Component1: 0, Component2: 0}}, + ), + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + 1, + {"FOO": {Component1: 0, Component2: 1}}, + ), + # Replace + ( + { + "FOO": { + Component1: None, + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + Component1Subclass: None, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1, + } + }, + "FOO", + Component1, + None, + { + "FOO": { + Component1: None, + Component1Subclass: None, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1, + } + }, + ), + ( + { + "FOO": { + Component1: 0, + "tests.test_settings.Component1": 1, + "tests.test_settings.Component1Alias": None, + Component1Subclass: 0, + "tests.test_settings.Component1Subclass": 1, + "tests.test_settings.Component1SubclassAlias": None, + } + }, + "FOO", + Component1, + 0, + { + "FOO": { + Component1: 0, + Component1Subclass: 0, + "tests.test_settings.Component1Subclass": 1, + "tests.test_settings.Component1SubclassAlias": None, + } + }, + ), + ( + { + "FOO": { + Component1: 1, + "tests.test_settings.Component1": None, + "tests.test_settings.Component1Alias": 0, + Component1Subclass: 1, + "tests.test_settings.Component1Subclass": None, + "tests.test_settings.Component1SubclassAlias": 0, + } + }, + "FOO", + Component1, + 1, + { + "FOO": { + Component1: 1, + Component1Subclass: 1, + "tests.test_settings.Component1Subclass": None, + "tests.test_settings.Component1SubclassAlias": 0, + } + }, + ), + # String-based setting values + ( + {"FOO": '{"tests.test_settings.Component1": 0}'}, + "FOO", + Component2, + None, + {"FOO": {"tests.test_settings.Component1": 0, Component2: None}}, + ), + ( + { + "FOO": """{ + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1 + }""" + }, + "FOO", + Component1, + None, + { + "FOO": { + Component1: None, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1, + } + }, + ), + ], +) +def test_set_in_component_priority_dict(before, name, cls, priority, after): + settings = BaseSettings(before, priority=0) + expected_priority = settings.getpriority(name) or 0 + settings.set_in_component_priority_dict(name, cls, priority) + expected_settings = BaseSettings(after, priority=expected_priority) + assert settings == expected_settings + assert settings.getpriority(name) == expected_settings.getpriority(name), ( + f"{settings.getpriority(name)=} != {expected_settings.getpriority(name)=}" + ) + + +@pytest.mark.parametrize( + ("before", "name", "cls", "priority", "after"), + [ + # Set + ({}, "FOO", Component1, None, {"FOO": {Component1: None}}), + ({}, "FOO", Component1, 0, {"FOO": {Component1: 0}}), + ({}, "FOO", Component1, 1, {"FOO": {Component1: 1}}), + # Add + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + None, + {"FOO": {Component1: 0, Component2: None}}, + ), + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + 0, + {"FOO": {Component1: 0, Component2: 0}}, + ), + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + 1, + {"FOO": {Component1: 0, Component2: 1}}, + ), + # Keep + ( + { + "FOO": { + Component1: None, + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + Component1Subclass: None, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1, + } + }, + "FOO", + Component1, + None, + { + "FOO": { + Component1: None, + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + Component1Subclass: None, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1, + } + }, + ), + ( + { + "FOO": { + Component1: 0, + "tests.test_settings.Component1": 1, + "tests.test_settings.Component1Alias": None, + Component1Subclass: 0, + "tests.test_settings.Component1Subclass": 1, + "tests.test_settings.Component1SubclassAlias": None, + } + }, + "FOO", + Component1, + 0, + { + "FOO": { + Component1: 0, + "tests.test_settings.Component1": 1, + "tests.test_settings.Component1Alias": None, + Component1Subclass: 0, + "tests.test_settings.Component1Subclass": 1, + "tests.test_settings.Component1SubclassAlias": None, + } + }, + ), + ( + { + "FOO": { + Component1: 1, + "tests.test_settings.Component1": None, + "tests.test_settings.Component1Alias": 0, + Component1Subclass: 1, + "tests.test_settings.Component1Subclass": None, + "tests.test_settings.Component1SubclassAlias": 0, + } + }, + "FOO", + Component1, + 1, + { + "FOO": { + Component1: 1, + "tests.test_settings.Component1": None, + "tests.test_settings.Component1Alias": 0, + Component1Subclass: 1, + "tests.test_settings.Component1Subclass": None, + "tests.test_settings.Component1SubclassAlias": 0, + } + }, + ), + # String-based setting values + ( + {"FOO": '{"tests.test_settings.Component1": 0}'}, + "FOO", + Component2, + None, + {"FOO": {"tests.test_settings.Component1": 0, Component2: None}}, + ), + ( + { + "FOO": """{ + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1 + }""" + }, + "FOO", + Component1, + None, + { + "FOO": """{ + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1 + }""" + }, + ), + ], +) +def test_setdefault_in_component_priority_dict(before, name, cls, priority, after): + settings = BaseSettings(before, priority=0) + expected_priority = settings.getpriority(name) or 0 + settings.setdefault_in_component_priority_dict(name, cls, priority) + expected_settings = BaseSettings(after, priority=expected_priority) + assert settings == expected_settings + assert settings.getpriority(name) == expected_settings.getpriority(name) diff --git a/tests/test_signals.py b/tests/test_signals.py index 0df104600..f5f13b992 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -1,16 +1,15 @@ -from pytest import mark -from twisted.internet import defer -from twisted.trial import unittest +import pytest from scrapy import Request, Spider, signals from scrapy.utils.test import get_crawler, get_from_asyncio_queue -from tests.mockserver import MockServer +from tests.mockserver.http import MockServer +from tests.utils.decorators import coroutine_test, inline_callbacks_test class ItemSpider(Spider): name = "itemspider" - def start_requests(self): + async def start(self): for index in range(10): yield Request( self.mockserver.url(f"/status?n=200&id={index}"), meta={"index": index} @@ -20,25 +19,43 @@ class ItemSpider(Spider): return {"index": response.meta["index"]} -class AsyncSignalTestCase(unittest.TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() - self.items = [] +class TestMain: + @coroutine_test + async def test_scheduler_empty(self): + crawler = get_crawler() + calls = [] - def tearDown(self): - self.mockserver.__exit__(None, None, None) + def track_call(): + calls.append(object()) + + crawler.signals.connect(track_call, signals.scheduler_empty) + await crawler.crawl_async() + assert len(calls) >= 1 + + +class TestMockServer: + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) + + def setup_method(self): + self.items = [] async def _on_item_scraped(self, item): item = await get_from_asyncio_queue(item) self.items.append(item) - @mark.only_asyncio() - @defer.inlineCallbacks + @pytest.mark.only_asyncio + @inline_callbacks_test def test_simple_pipeline(self): crawler = get_crawler(ItemSpider) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 10) + assert len(self.items) == 10 for index in range(10): - self.assertIn({"index": index}, self.items) + assert {"index": index} in self.items diff --git a/tests/test_spider.py b/tests/test_spider.py index 9ce40f921..23efed77a 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,70 +1,49 @@ -import gzip -import inspect -import warnings -from io import BytesIO +from __future__ import annotations + from typing import Any from unittest import mock +import pytest from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest -from w3lib.url import safe_url_string from scrapy import signals from scrapy.crawler import Crawler -from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse -from scrapy.linkextractors import LinkExtractor +from scrapy.http import Response, TextResponse, XmlResponse from scrapy.settings import Settings -from scrapy.spiders import ( - CrawlSpider, - CSVFeedSpider, - Rule, - SitemapSpider, - Spider, - XMLFeedSpider, -) -from scrapy.spiders.init import InitSpider -from scrapy.utils.test import get_crawler +from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider +from scrapy.utils.test import get_crawler, get_reactor_settings from tests import get_testdata +from tests.utils.decorators import inline_callbacks_test -class SpiderTest(unittest.TestCase): +class TestSpider: spider_class = Spider - def setUp(self): - warnings.simplefilter("always") - - def tearDown(self): - warnings.resetwarnings() - def test_base_spider(self): spider = self.spider_class("example.com") - self.assertEqual(spider.name, "example.com") - self.assertEqual(spider.start_urls, []) - - def test_start_requests(self): - spider = self.spider_class("example.com") - start_requests = spider.start_requests() - self.assertTrue(inspect.isgenerator(start_requests)) - self.assertEqual(list(start_requests), []) + assert spider.name == "example.com" + assert spider.start_urls == [] # pylint: disable=use-implicit-booleaness-not-comparison def test_spider_args(self): """``__init__`` method arguments are assigned to spider attributes""" spider = self.spider_class("example.com", foo="bar") - self.assertEqual(spider.foo, "bar") + assert spider.foo == "bar" def test_spider_without_name(self): """``__init__`` method arguments are assigned to spider attributes""" - self.assertRaises(ValueError, self.spider_class) - self.assertRaises(ValueError, self.spider_class, somearg="foo") + msg = "must have a name" + with pytest.raises(ValueError, match=msg): + self.spider_class() + with pytest.raises(ValueError, match=msg): + self.spider_class(somearg="foo") def test_from_crawler_crawler_and_settings_population(self): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertTrue(hasattr(spider, "crawler")) - self.assertIs(spider.crawler, crawler) - self.assertTrue(hasattr(spider, "settings")) - self.assertIs(spider.settings, crawler.settings) + assert hasattr(spider, "crawler") + assert spider.crawler is crawler + assert hasattr(spider, "settings") + assert spider.settings is crawler.settings def test_from_crawler_init_call(self): with mock.patch.object( @@ -86,7 +65,7 @@ class SpiderTest(unittest.TestCase): crawler.signals.send_catch_log( signal=signals.spider_closed, spider=spider, reason=None ) - self.assertTrue(spider.closed_called) + assert spider.closed_called def test_update_settings(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} @@ -95,14 +74,18 @@ class SpiderTest(unittest.TestCase): settings = Settings(project_settings, priority="project") self.spider_class.update_settings(settings) - self.assertEqual(settings.get("TEST1"), "spider") - self.assertEqual(settings.get("TEST2"), "spider") - self.assertEqual(settings.get("TEST3"), "project") + assert settings.get("TEST1") == "spider" + assert settings.get("TEST2") == "spider" + assert settings.get("TEST3") == "project" - @inlineCallbacks + @inline_callbacks_test def test_settings_in_from_crawler(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = {"TEST1": "project", "TEST3": "project"} + project_settings = { + "TEST1": "project", + "TEST3": "project", + **get_reactor_settings(), + } class TestSpider(self.spider_class): name = "test" @@ -115,11 +98,11 @@ class SpiderTest(unittest.TestCase): return spider crawler = Crawler(TestSpider, project_settings) - self.assertEqual(crawler.settings.get("TEST1"), "spider") - self.assertEqual(crawler.settings.get("TEST2"), "spider") - self.assertEqual(crawler.settings.get("TEST3"), "project") + assert crawler.settings.get("TEST1") == "spider" + assert crawler.settings.get("TEST2") == "spider" + assert crawler.settings.get("TEST3") == "project" yield crawler.crawl() - self.assertEqual(crawler.settings.get("TEST1"), "spider_instance") + assert crawler.settings.get("TEST1") == "spider_instance" def test_logger(self): spider = self.spider_class("example.com") @@ -128,8 +111,8 @@ class SpiderTest(unittest.TestCase): lc.check(("example.com", "INFO", "test log msg")) record = lc.records[0] - self.assertIn("spider", record.__dict__) - self.assertIs(record.spider, spider) + assert "spider" in record.__dict__ + assert record.spider is spider def test_log(self): spider = self.spider_class("example.com") @@ -138,21 +121,17 @@ class SpiderTest(unittest.TestCase): mock_logger.log.assert_called_once_with("INFO", "test log msg") -class InitSpiderTest(SpiderTest): - spider_class = InitSpider - - -class XMLFeedSpiderTest(SpiderTest): +class TestXMLFeedSpider(TestSpider): spider_class = XMLFeedSpider def test_register_namespace(self): body = b""" - http://www.example.com/Special-Offers.html2009-08-16 + http://www.example.com/Special-Offers.html2009-08-16 - http://www.example.com/2009-08-16 + http://www.example.com/2009-08-16 """ response = XmlResponse(url="http://example.com/sitemap.xml", body=body) @@ -174,28 +153,24 @@ class XMLFeedSpiderTest(SpiderTest): for iterator in ("iternodes", "xml"): spider = _XMLSpider("example", iterator=iterator) output = list(spider._parse(response)) - self.assertEqual(len(output), 2, iterator) - self.assertEqual( - output, - [ - { - "loc": ["http://www.example.com/Special-Offers.html"], - "updated": ["2009-08-16"], - "custom": ["fuu"], - "other": ["bar"], - }, - { - "loc": [], - "updated": ["2009-08-16"], - "other": ["foo"], - "custom": [], - }, - ], - iterator, - ) + assert len(output) == 2, iterator + assert output == [ + { + "loc": ["http://www.example.com/Special-Offers.html"], + "updated": ["2009-08-16"], + "custom": ["fuu"], + "other": ["bar"], + }, + { + "loc": [], + "updated": ["2009-08-16"], + "other": ["foo"], + "custom": [], + }, + ], iterator -class CSVFeedSpiderTest(SpiderTest): +class TestCSVFeedSpider(TestSpider): spider_class = CSVFeedSpider def test_parse_rows(self): @@ -216,489 +191,7 @@ class CSVFeedSpiderTest(SpiderTest): assert len(rows) == 4 -class CrawlSpiderTest(SpiderTest): - test_body = b"""Page title<title> - <body> - <p><a href="item/12.html">Item 12</a></p> - <div class='links'> - <p><a href="/about.html">About us</a></p> - </div> - <div> - <p><a href="/nofollow.html">This shouldn't be followed</a></p> - </div> - </body></html>""" - spider_class = CrawlSpider - - def test_rule_without_link_extractor(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = (Rule(),) - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) - - def test_process_links(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) - - def dummy_process_links(self, links): - return links - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) - - def test_process_links_filter(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - import re - - name = "test" - allowed_domains = ["example.org"] - rules = (Rule(LinkExtractor(), process_links="filter_process_links"),) - _test_regex = re.compile("nofollow") - - def filter_process_links(self, links): - return [link for link in links if not self._test_regex.search(link.url)] - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 2) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - ], - ) - - def test_process_links_generator(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) - - def dummy_process_links(self, links): - yield from links - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) - - def test_process_request(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - def process_request_change_domain(request, response): - return request.replace(url=request.url.replace(".org", ".com")) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = ( - Rule(LinkExtractor(), process_request=process_request_change_domain), - ) - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual( - [r.url for r in output], - [ - "http://example.com/somepage/item/12.html", - "http://example.com/about.html", - "http://example.com/nofollow.html", - ], - ) - - def test_process_request_with_response(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - def process_request_meta_response_class(request, response): - request.meta["response_class"] = response.__class__.__name__ - return request - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = ( - Rule( - LinkExtractor(), process_request=process_request_meta_response_class - ), - ) - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) - self.assertEqual( - [r.meta["response_class"] for r in output], - ["HtmlResponse", "HtmlResponse", "HtmlResponse"], - ) - - def test_process_request_instance_method(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = (Rule(LinkExtractor(), process_request="process_request_upper"),) - - def process_request_upper(self, request, response): - return request.replace(url=request.url.upper()) - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual( - [r.url for r in output], - [ - safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"), - safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"), - safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"), - ], - ) - - def test_process_request_instance_method_with_response(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = ( - Rule( - LinkExtractor(), - process_request="process_request_meta_response_class", - ), - ) - - def process_request_meta_response_class(self, request, response): - request.meta["response_class"] = response.__class__.__name__ - return request - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) - self.assertEqual( - [r.meta["response_class"] for r in output], - ["HtmlResponse", "HtmlResponse", "HtmlResponse"], - ) - - def test_follow_links_attribute_population(self): - crawler = get_crawler() - spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertTrue(hasattr(spider, "_follow_links")) - self.assertTrue(spider._follow_links) - - settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False} - crawler = get_crawler(settings_dict=settings_dict) - spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertTrue(hasattr(spider, "_follow_links")) - self.assertFalse(spider._follow_links) - - def test_start_url(self): - spider = self.spider_class("example.com") - spider.start_url = "https://www.example.com" - - with self.assertRaisesRegex(AttributeError, r"^Crawling could not start.*$"): - list(spider.start_requests()) - - -class SitemapSpiderTest(SpiderTest): - spider_class = SitemapSpider - - BODY = b"SITEMAP" - f = BytesIO() - g = gzip.GzipFile(fileobj=f, mode="w+b") - g.write(BODY) - g.close() - GZBODY = f.getvalue() - - def assertSitemapBody(self, response, body): - spider = self.spider_class("example.com") - self.assertEqual(spider._get_sitemap_body(response), body) - - def test_get_sitemap_body(self): - r = XmlResponse(url="http://www.example.com/", body=self.BODY) - self.assertSitemapBody(r, self.BODY) - - r = HtmlResponse(url="http://www.example.com/", body=self.BODY) - self.assertSitemapBody(r, None) - - r = Response(url="http://www.example.com/favicon.ico", body=self.BODY) - self.assertSitemapBody(r, None) - - def test_get_sitemap_body_gzip_headers(self): - r = Response( - url="http://www.example.com/sitemap", - body=self.GZBODY, - headers={"content-type": "application/gzip"}, - ) - self.assertSitemapBody(r, self.BODY) - - def test_get_sitemap_body_xml_url(self): - r = TextResponse(url="http://www.example.com/sitemap.xml", body=self.BODY) - self.assertSitemapBody(r, self.BODY) - - def test_get_sitemap_body_xml_url_compressed(self): - r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY) - self.assertSitemapBody(r, self.BODY) - - # .xml.gz but body decoded by HttpCompression middleware already - r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.BODY) - self.assertSitemapBody(r, self.BODY) - - def test_get_sitemap_urls_from_robotstxt(self): - robots = b"""# Sitemap files -Sitemap: http://example.com/sitemap.xml -Sitemap: http://example.com/sitemap-product-index.xml -Sitemap: HTTP://example.com/sitemap-uppercase.xml -Sitemap: /sitemap-relative-url.xml -""" - - r = TextResponse(url="http://www.example.com/robots.txt", body=robots) - spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://example.com/sitemap.xml", - "http://example.com/sitemap-product-index.xml", - "http://example.com/sitemap-uppercase.xml", - "http://www.example.com/sitemap-relative-url.xml", - ], - ) - - def test_alternate_url_locs(self): - sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> - <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" - xmlns:xhtml="http://www.w3.org/1999/xhtml"> - <url> - <loc>http://www.example.com/english/</loc> - <xhtml:link rel="alternate" hreflang="de" - href="http://www.example.com/deutsch/"/> - <xhtml:link rel="alternate" hreflang="de-ch" - href="http://www.example.com/schweiz-deutsch/"/> - <xhtml:link rel="alternate" hreflang="it" - href="http://www.example.com/italiano/"/> - <xhtml:link rel="alternate" hreflang="it"/><!-- wrong tag without href --> - </url> - </urlset>""" - r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) - spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/english/"], - ) - - spider.sitemap_alternate_links = True - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://www.example.com/english/", - "http://www.example.com/deutsch/", - "http://www.example.com/schweiz-deutsch/", - "http://www.example.com/italiano/", - ], - ) - - def test_sitemap_filter(self): - sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> - <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" - xmlns:xhtml="http://www.w3.org/1999/xhtml"> - <url> - <loc>http://www.example.com/english/</loc> - <lastmod>2010-01-01</lastmod> - </url> - <url> - <loc>http://www.example.com/portuguese/</loc> - <lastmod>2005-01-01</lastmod> - </url> - </urlset>""" - - class FilteredSitemapSpider(self.spider_class): - def sitemap_filter(self, entries): - from datetime import datetime - - for entry in entries: - date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d") - if date_time.year > 2008: - yield entry - - r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) - spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/english/", "http://www.example.com/portuguese/"], - ) - - spider = FilteredSitemapSpider("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/english/"], - ) - - def test_sitemap_filter_with_alternate_links(self): - sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> - <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" - xmlns:xhtml="http://www.w3.org/1999/xhtml"> - <url> - <loc>http://www.example.com/english/article_1/</loc> - <lastmod>2010-01-01</lastmod> - <xhtml:link rel="alternate" hreflang="de" - href="http://www.example.com/deutsch/article_1/"/> - </url> - <url> - <loc>http://www.example.com/english/article_2/</loc> - <lastmod>2015-01-01</lastmod> - </url> - </urlset>""" - - class FilteredSitemapSpider(self.spider_class): - def sitemap_filter(self, entries): - for entry in entries: - alternate_links = entry.get("alternate", tuple()) - for link in alternate_links: - if "/deutsch/" in link: - entry["loc"] = link - yield entry - - r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) - spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://www.example.com/english/article_1/", - "http://www.example.com/english/article_2/", - ], - ) - - spider = FilteredSitemapSpider("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/deutsch/article_1/"], - ) - - def test_sitemapindex_filter(self): - sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> - <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> - <sitemap> - <loc>http://www.example.com/sitemap1.xml</loc> - <lastmod>2004-01-01T20:00:00+00:00</lastmod> - </sitemap> - <sitemap> - <loc>http://www.example.com/sitemap2.xml</loc> - <lastmod>2005-01-01</lastmod> - </sitemap> - </sitemapindex>""" - - class FilteredSitemapSpider(self.spider_class): - def sitemap_filter(self, entries): - from datetime import datetime - - for entry in entries: - date_time = datetime.strptime( - entry["lastmod"].split("T")[0], "%Y-%m-%d" - ) - if date_time.year > 2004: - yield entry - - r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) - spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://www.example.com/sitemap1.xml", - "http://www.example.com/sitemap2.xml", - ], - ) - - spider = FilteredSitemapSpider("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/sitemap2.xml"], - ) - - -class DeprecationTest(unittest.TestCase): - def test_crawl_spider(self): - assert issubclass(CrawlSpider, Spider) - assert isinstance(CrawlSpider(name="foo"), Spider) - - -class NoParseMethodSpiderTest(unittest.TestCase): +class TestNoParseMethodSpider: spider_class = Spider def test_undefined_parse_method(self): @@ -707,5 +200,5 @@ class NoParseMethodSpiderTest(unittest.TestCase): resp = TextResponse(url="http://www.example.com/random_url", body=text) exc_msg = "Spider.parse callback is not defined" - with self.assertRaisesRegex(NotImplementedError, exc_msg): + with pytest.raises(NotImplementedError, match=exc_msg): spider.parse(resp) diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py new file mode 100644 index 000000000..05d77912f --- /dev/null +++ b/tests/test_spider_crawl.py @@ -0,0 +1,307 @@ +from __future__ import annotations + +import re +import warnings +from logging import ERROR + +from testfixtures import LogCapture +from w3lib.url import safe_url_string + +from scrapy.http import HtmlResponse, Request, TextResponse +from scrapy.linkextractors import LinkExtractor +from scrapy.spiders import CrawlSpider, Rule, Spider +from scrapy.utils.test import get_crawler +from tests.test_spider import TestSpider +from tests.utils.decorators import inline_callbacks_test + + +class TestCrawlSpider(TestSpider): + test_body = b"""<html><head><title>Page title + +

Item 12

+
+

About us

+
+ + """ + spider_class = CrawlSpider + + def test_rule_without_link_extractor(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(),) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + + def test_process_links(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) + + def dummy_process_links(self, links): + return links + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + + def test_process_links_filter(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(LinkExtractor(), process_links="filter_process_links"),) + _test_regex = re.compile("nofollow") + + def filter_process_links(self, links): + return [link for link in links if not self._test_regex.search(link.url)] + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 2 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + ] + + def test_process_links_generator(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) + + def dummy_process_links(self, links): + yield from links + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + + def test_process_request(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + def process_request_change_domain(request, response): + return request.replace(url=request.url.replace(".org", ".com")) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = ( + Rule(LinkExtractor(), process_request=process_request_change_domain), + ) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.com/somepage/item/12.html", + "http://example.com/about.html", + "http://example.com/nofollow.html", + ] + + def test_process_request_with_response(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + def process_request_meta_response_class(request, response): + request.meta["response_class"] = response.__class__.__name__ + return request + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = ( + Rule( + LinkExtractor(), process_request=process_request_meta_response_class + ), + ) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + assert [r.meta["response_class"] for r in output] == [ + "HtmlResponse", + "HtmlResponse", + "HtmlResponse", + ] + + def test_process_request_instance_method(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(LinkExtractor(), process_request="process_request_upper"),) + + def process_request_upper(self, request, response): + return request.replace(url=request.url.upper()) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"), + safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"), + safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"), + ] + + def test_process_request_instance_method_with_response(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = ( + Rule( + LinkExtractor(), + process_request="process_request_meta_response_class", + ), + ) + + def process_request_meta_response_class(self, request, response): + request.meta["response_class"] = response.__class__.__name__ + return request + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + assert [r.meta["response_class"] for r in output] == [ + "HtmlResponse", + "HtmlResponse", + "HtmlResponse", + ] + + def test_follow_links_attribute_population(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + assert hasattr(spider, "_follow_links") + assert spider._follow_links + + settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False} + crawler = get_crawler(settings_dict=settings_dict) + spider = self.spider_class.from_crawler(crawler, "example.com") + assert hasattr(spider, "_follow_links") + assert not spider._follow_links + + @inline_callbacks_test + def test_start_url(self): + class TestSpider(self.spider_class): + name = "test" + start_url = "https://www.example.com" + + crawler = get_crawler(TestSpider) + with LogCapture("scrapy.core.engine", propagate=False, level=ERROR) as log: + yield crawler.crawl() + assert "Error while reading start items and requests" in str(log) + assert "did you miss an 's'?" in str(log) + + def test_parse_response_use(self): + class _CrawlSpider(CrawlSpider): + name = "test" + start_urls = "https://www.example.com" + _follow_links = False + + with warnings.catch_warnings(record=True) as w: + spider = _CrawlSpider() + assert len(w) == 0 + spider._parse_response( + TextResponse(spider.start_urls, body=b""), None, None + ) + assert len(w) == 1 + + def test_parse_response_override(self): + class _CrawlSpider(CrawlSpider): + def _parse_response(self, response, callback, cb_kwargs, follow=True): + pass + + name = "test" + start_urls = "https://www.example.com" + _follow_links = False + + with warnings.catch_warnings(record=True) as w: + assert len(w) == 0 + spider = _CrawlSpider() + assert len(w) == 1 + spider._parse_response( + TextResponse(spider.start_urls, body=b""), None, None + ) + assert len(w) == 1 + + def test_parse_with_rules(self): + class _CrawlSpider(CrawlSpider): + name = "test" + start_urls = "https://www.example.com" + + with warnings.catch_warnings(record=True) as w: + spider = _CrawlSpider() + spider.parse_with_rules( + TextResponse(spider.start_urls, body=b""), None, None + ) + assert len(w) == 0 + + +class TestDeprecation: + def test_crawl_spider(self): + assert issubclass(CrawlSpider, Spider) + assert isinstance(CrawlSpider(name="foo"), Spider) diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py new file mode 100644 index 000000000..57c209615 --- /dev/null +++ b/tests/test_spider_sitemap.py @@ -0,0 +1,440 @@ +from __future__ import annotations + +import gzip +import re +import warnings +from datetime import datetime +from io import BytesIO +from logging import WARNING +from pathlib import Path + +import pytest +from testfixtures import LogCapture + +from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse +from scrapy.spiders import SitemapSpider +from scrapy.utils.test import get_crawler +from tests import tests_datadir +from tests.test_spider import TestSpider +from tests.utils.decorators import coroutine_test + + +class TestSitemapSpider(TestSpider): + spider_class = SitemapSpider + + BODY = b"SITEMAP" + f = BytesIO() + g = gzip.GzipFile(fileobj=f, mode="w+b") + g.write(BODY) + g.close() + GZBODY = f.getvalue() + + def assertSitemapBody(self, response: Response, body: bytes | None) -> None: + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + assert spider._get_sitemap_body(response) == body + + def test_get_sitemap_body(self): + r = XmlResponse(url="http://www.example.com/", body=self.BODY) + self.assertSitemapBody(r, self.BODY) + + r = HtmlResponse(url="http://www.example.com/", body=self.BODY) + self.assertSitemapBody(r, None) + + r = Response(url="http://www.example.com/favicon.ico", body=self.BODY) + self.assertSitemapBody(r, None) + + r = XmlResponse(url="http://www.example.com/", body=b"") + self.assertSitemapBody(r, b"") + + def test_get_sitemap_body_gzip_headers(self): + r = Response( + url="http://www.example.com/sitemap", + body=self.GZBODY, + headers={"content-type": "application/gzip"}, + request=Request("http://www.example.com/sitemap"), + ) + self.assertSitemapBody(r, self.BODY) + + def test_get_sitemap_body_xml_url(self): + r = TextResponse(url="http://www.example.com/sitemap.xml", body=self.BODY) + self.assertSitemapBody(r, self.BODY) + + def test_get_sitemap_body_xml_url_compressed(self): + r = Response( + url="http://www.example.com/sitemap.xml.gz", + body=self.GZBODY, + request=Request("http://www.example.com/sitemap"), + ) + self.assertSitemapBody(r, self.BODY) + + # .xml.gz but body decoded by HttpCompression middleware already + r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.BODY) + self.assertSitemapBody(r, self.BODY) + + def test_get_sitemap_urls_from_robotstxt(self): + robots = b"""# Sitemap files +Sitemap: http://example.com/sitemap.xml +Sitemap: http://example.com/sitemap-product-index.xml +Sitemap: HTTP://example.com/sitemap-uppercase.xml +Sitemap: /sitemap-relative-url.xml +""" + + r = TextResponse(url="http://www.example.com/robots.txt", body=robots) + spider = self.spider_class("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://example.com/sitemap.xml", + "http://example.com/sitemap-product-index.xml", + "http://example.com/sitemap-uppercase.xml", + "http://www.example.com/sitemap-relative-url.xml", + ] + + def test_get_sitemap_urls_from_robotstxt_skips_invalid_utf8_urls(self): + robots = ( + b"User-agent: *\n" + b"Sitemap: http://example.com/\xff.xml\n" + b"Sitemap: http://example.com/ok.xml\n" + ) + + r = TextResponse(url="http://www.example.com/robots.txt", body=robots) + spider = self.spider_class("example.com") + + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://example.com/ok.xml", + ] + + def test_alternate_url_locs(self): + sitemap = b""" + + + http://www.example.com/english/ + + + + + + """ + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/" + ] + + spider.sitemap_alternate_links = True + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/", + "http://www.example.com/deutsch/", + "http://www.example.com/schweiz-deutsch/", + "http://www.example.com/italiano/", + ] + + def test_sitemap_filter(self): + sitemap = b""" + + + http://www.example.com/english/ + 2010-01-01 + + + http://www.example.com/portuguese/ + 2005-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + for entry in entries: + date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d") + if date_time.year > 2008: + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/", + "http://www.example.com/portuguese/", + ] + + spider = FilteredSitemapSpider("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/" + ] + + def test_sitemap_filter_with_alternate_links(self): + sitemap = b""" + + + http://www.example.com/english/article_1/ + 2010-01-01 + + + + http://www.example.com/english/article_2/ + 2015-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + for entry in entries: + alternate_links = entry.get("alternate", ()) + for link in alternate_links: + if "/deutsch/" in link: + entry["loc"] = link + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/article_1/", + "http://www.example.com/english/article_2/", + ] + + spider = FilteredSitemapSpider("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/deutsch/article_1/" + ] + + def test_sitemapindex_filter(self): + sitemap = b""" + + + http://www.example.com/sitemap1.xml + 2004-01-01T20:00:00+00:00 + + + http://www.example.com/sitemap2.xml + 2005-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + for entry in entries: + date_time = datetime.strptime( + entry["lastmod"].split("T")[0], "%Y-%m-%d" + ) + if date_time.year > 2004: + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/sitemap1.xml", + "http://www.example.com/sitemap2.xml", + ] + + spider = FilteredSitemapSpider("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/sitemap2.xml" + ] + + @pytest.mark.parametrize( + ("rule", "result"), + [(r"english", ["http://www.example.com/english/"]), (r"nonexistent", [])], + ) + def test_sitemap_filter_with_rule(self, rule: str, result: list[str]): + sitemap = b""" + + http://www.example.com/english/ + http://www.example.com/portuguese/ + """ + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + + class _RuleSpider(self.spider_class): # type: ignore[name-defined,misc] + sitemap_rules = [(rule, "parse")] + + spider = _RuleSpider("example.com") + urls = [req.url for req in spider._parse_sitemap(r)] + assert urls == result + + def test_parse_sitemap_empty_body(self): + r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"") + spider = self.spider_class("example.com") + + with LogCapture() as lc: + results = list(spider._parse_sitemap(r)) + + assert not results + + lc.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + "Ignoring invalid sitemap: <200 http://www.example.com/sitemap.xml>", + ) + ) + + def test_parse_sitemap_not_sitemap(self): + body = b""" + + sometext + sometext2 + """ + r = XmlResponse(url="http://www.example.com/random.xml", body=body) + spider = self.spider_class("example.com") + + results = list(spider._parse_sitemap(r)) + + assert not results + + @pytest.mark.parametrize( + ("follow", "result"), + [ + (r"1.xml", ["http://www.example.com/sitemap1.xml"]), + (re.compile(r"sitemap\d"), ["http://www.example.com/sitemap1.xml"]), + (r"nonexistent", []), + ], + ) + def test_sitemap_follow(self, follow, result): + sitemap = b""" + + + http://www.example.com/sitemap1.xml + + """ + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + + class _FollowSpider(self.spider_class): + sitemap_follow = [follow] + + spider = _FollowSpider("example.com") + urls = [req.url for req in spider._parse_sitemap(r)] + assert urls == result + + def test_compression_bomb_setting(self): + settings = {"DOWNLOAD_MAXSIZE": 10_000_000} + crawler = get_crawler(settings_dict=settings) + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + assert spider._get_sitemap_body(response) is None + + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") + def test_compression_bomb_spider_attr(self): + class DownloadMaxSizeSpider(self.spider_class): + download_maxsize = 10_000_000 + + crawler = get_crawler() + spider = DownloadMaxSizeSpider.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + assert spider._get_sitemap_body(response) is None + + def test_compression_bomb_request_meta(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_maxsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) + assert spider._get_sitemap_body(response) is None + + def test_download_warnsize_setting(self): + settings = {"DOWNLOAD_WARNSIZE": 10_000_000} + crawler = get_crawler(settings_dict=settings) + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") + def test_download_warnsize_spider_attr(self): + class DownloadWarnSizeSpider(self.spider_class): + download_warnsize = 10_000_000 + + crawler = get_crawler() + spider = DownloadWarnSizeSpider.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_warnsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + + def test_download_warnsize_request_meta(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_warnsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + + @coroutine_test + async def test_sitemap_urls(self): + class TestSpider(self.spider_class): + name = "test" + sitemap_urls = ["https://toscrape.com/sitemap.xml"] + + crawler = get_crawler(TestSpider) + spider = TestSpider.from_crawler(crawler) + with warnings.catch_warnings(): + warnings.simplefilter("error") + requests = [request async for request in spider.start()] + + assert len(requests) == 1 + request = requests[0] + assert request.url == "https://toscrape.com/sitemap.xml" + assert request.dont_filter is False + assert request.callback == spider._parse_sitemap diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py new file mode 100644 index 000000000..aef2093ac --- /dev/null +++ b/tests/test_spider_start.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import warnings +from asyncio import sleep +from typing import Any + +import pytest + +from scrapy import Spider, signals +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.test import get_crawler + +from .utils import twisted_sleep +from .utils.decorators import coroutine_test + +SLEEP_SECONDS = 0.1 + +ITEM_A = {"id": "a"} +ITEM_B = {"id": "b"} + + +class TestMain: + async def _test_spider( + self, spider: type[Spider], expected_items: list[Any] | None = None + ) -> None: + actual_items = [] + expected_items = [] if expected_items is None else expected_items + + def track_item(item, response, spider): + actual_items.append(item) + + crawler = get_crawler(spider) + crawler.signals.connect(track_item, signals.item_scraped) + await crawler.crawl_async() + assert crawler.stats + assert crawler.stats.get_value("finish_reason") == "finished" + assert actual_items == expected_items + + @coroutine_test + async def test_start_urls(self): + class TestSpider(Spider): + name = "test" + start_urls = ["data:,"] + + async def parse(self, response): + yield ITEM_A + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @coroutine_test + async def test_start(self): + class TestSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_A + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @coroutine_test + async def test_start_subclass(self): + class BaseSpider(Spider): + async def start(self): + yield ITEM_A + + class TestSpider(BaseSpider): + name = "test" + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + async def _test_start(self, start_, expected_items=None): + class TestSpider(Spider): + name = "test" + start = start_ + + await self._test_spider(TestSpider, expected_items) + + @pytest.mark.only_asyncio + @coroutine_test + async def test_asyncio_delayed(self): + async def start(spider): + await sleep(SLEEP_SECONDS) + yield ITEM_A + + await self._test_start(start, [ITEM_A]) + + @pytest.mark.requires_reactor # needs a reactor for twisted_sleep() + @coroutine_test + async def test_twisted_delayed(self): + async def start(spider): + await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS)) + yield ITEM_A + + await self._test_start(start, [ITEM_A]) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 04025d30d..245507c0b 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,10 +1,11 @@ +import contextlib import shutil import sys -import tempfile import warnings from pathlib import Path +from unittest import mock -from twisted.trial import unittest +import pytest from zope.interface.verify import verifyObject # ugly hack to avoid cyclic imports of scrapy.spiders when running this test @@ -14,109 +15,126 @@ from scrapy.crawler import CrawlerRunner from scrapy.http import Request from scrapy.interfaces import ISpiderLoader from scrapy.settings import Settings -from scrapy.spiderloader import SpiderLoader +from scrapy.spiderloader import DummySpiderLoader, SpiderLoader, get_spider_loader module_dir = Path(__file__).resolve().parent def _copytree(source: Path, target: Path): - try: + with contextlib.suppress(shutil.Error): shutil.copytree(source, target) - except shutil.Error: - pass -class SpiderLoaderTest(unittest.TestCase): - def setUp(self): - orig_spiders_dir = module_dir / "test_spiders" - self.tmpdir = Path(tempfile.mkdtemp()) - self.spiders_dir = self.tmpdir / "test_spiders_xxx" - _copytree(orig_spiders_dir, self.spiders_dir) - sys.path.append(str(self.tmpdir)) - settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) - self.spider_loader = SpiderLoader.from_settings(settings) +@pytest.fixture +def spider_loader_env(tmp_path): + orig_spiders_dir = module_dir / "test_spiders" + spiders_dir = tmp_path / "test_spiders_xxx" + _copytree(orig_spiders_dir, spiders_dir) + sys.path.append(str(tmp_path)) + settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) - def tearDown(self): - del self.spider_loader - del sys.modules["test_spiders_xxx"] - sys.path.remove(str(self.tmpdir)) + yield settings, spiders_dir - def test_interface(self): - verifyObject(ISpiderLoader, self.spider_loader) + sys.modules.pop("test_spiders_xxx", None) + sys.path.remove(str(tmp_path)) - def test_list(self): - self.assertEqual( - set(self.spider_loader.list()), {"spider1", "spider2", "spider3", "spider4"} - ) - def test_load(self): - spider1 = self.spider_loader.load("spider1") - self.assertEqual(spider1.__name__, "Spider1") +@pytest.fixture +def spider_loader(spider_loader_env): + settings, _ = spider_loader_env + return SpiderLoader.from_settings(settings) - def test_find_by_request(self): - self.assertEqual( - self.spider_loader.find_by_request(Request("http://scrapy1.org/test")), - ["spider1"], - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://scrapy2.org/test")), - ["spider2"], - ) - self.assertEqual( - set(self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))), - {"spider1", "spider2"}, - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://scrapy999.org/test")), [] - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://spider3.com")), [] - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://spider3.com/onlythis")), - ["spider3"], - ) + +class TestSpiderLoader: + def test_interface(self, spider_loader): + verifyObject(ISpiderLoader, spider_loader) + + def test_list(self, spider_loader): + assert set(spider_loader.list()) == { + "spider1", + "spider2", + "spider3", + "spider4", + } + + def test_load(self, spider_loader): + spider1 = spider_loader.load("spider1") + assert spider1.__name__ == "Spider1" + + def test_find_by_request(self, spider_loader): + assert spider_loader.find_by_request(Request("http://scrapy1.org/test")) == [ + "spider1" + ] + assert spider_loader.find_by_request(Request("http://scrapy2.org/test")) == [ + "spider2" + ] + assert set( + spider_loader.find_by_request(Request("http://scrapy3.org/test")) + ) == {"spider1", "spider2"} + assert spider_loader.find_by_request(Request("http://scrapy999.org/test")) == [] + assert spider_loader.find_by_request(Request("http://spider3.com")) == [] + assert spider_loader.find_by_request( + Request("http://spider3.com/onlythis") + ) == ["spider3"] def test_load_spider_module(self): module = "tests.test_spiderloader.test_spiders.spider1" settings = Settings({"SPIDER_MODULES": [module]}) - self.spider_loader = SpiderLoader.from_settings(settings) - assert len(self.spider_loader._spiders) == 1 + spider_loader = SpiderLoader.from_settings(settings) + assert len(spider_loader._spiders) == 1 def test_load_spider_module_multiple(self): prefix = "tests.test_spiderloader.test_spiders." module = ",".join(prefix + s for s in ("spider1", "spider2")) settings = Settings({"SPIDER_MODULES": module}) - self.spider_loader = SpiderLoader.from_settings(settings) - assert len(self.spider_loader._spiders) == 2 + spider_loader = SpiderLoader.from_settings(settings) + assert len(spider_loader._spiders) == 2 def test_load_base_spider(self): module = "tests.test_spiderloader.test_spiders.spider0" settings = Settings({"SPIDER_MODULES": [module]}) - self.spider_loader = SpiderLoader.from_settings(settings) - assert len(self.spider_loader._spiders) == 0 + spider_loader = SpiderLoader.from_settings(settings) + assert len(spider_loader._spiders) == 0 + + def test_load_spider_module_from_addons(self): + module = "tests.test_spiderloader.spiders_from_addons.spider0" + + class SpiderModuleAddon: + @classmethod + def update_pre_crawler_settings(cls, settings): + settings.set( + "SPIDER_MODULES", + [module], + "project", + ) + + runner = CrawlerRunner({"ADDONS": {SpiderModuleAddon: 1}}) + + crawler = runner.create_crawler("spider_from_addon") + assert issubclass(crawler.spidercls, scrapy.Spider) + assert crawler.spidercls.name == "spider_from_addon" + assert len(crawler.settings["SPIDER_MODULES"]) == 1 def test_crawler_runner_loading(self): module = "tests.test_spiderloader.test_spiders.spider1" runner = CrawlerRunner( { "SPIDER_MODULES": [module], - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } ) - self.assertRaisesRegex( - KeyError, "Spider not found", runner.create_crawler, "spider2" - ) + with pytest.raises(KeyError, match="Spider not found"): + runner.create_crawler("spider2") crawler = runner.create_crawler("spider1") - self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) - self.assertEqual(crawler.spidercls.name, "spider1") + assert issubclass(crawler.spidercls, scrapy.Spider) + assert crawler.spidercls.name == "spider1" def test_bad_spider_modules_exception(self): module = "tests.test_spiderloader.test_spiders.doesnotexist" settings = Settings({"SPIDER_MODULES": [module]}) - self.assertRaises(ImportError, SpiderLoader.from_settings, settings) + with pytest.raises(ImportError): + SpiderLoader.from_settings(settings) def test_bad_spider_modules_warning(self): with warnings.catch_warnings(record=True) as w: @@ -130,75 +148,107 @@ class SpiderLoaderTest(unittest.TestCase): # at least until all six versions we can import (including botocore.vendored.six) # are updated to 1.16.0+ w.pop(0) - self.assertIn("Could not load spiders from module", str(w[0].message)) + assert "Could not load spiders from module" in str(w[0].message) spiders = spider_loader.list() - self.assertEqual(spiders, []) + assert not spiders + + def test_syntax_error_exception(self): + module = "tests.test_spiderloader.test_spiders.spider1" + with mock.patch.object(SpiderLoader, "_load_spiders") as m: + m.side_effect = SyntaxError + settings = Settings({"SPIDER_MODULES": [module]}) + with pytest.raises(SyntaxError): + SpiderLoader.from_settings(settings) + + def test_syntax_error_warning(self): + with ( + warnings.catch_warnings(record=True) as w, + mock.patch.object(SpiderLoader, "_load_spiders") as m, + ): + m.side_effect = SyntaxError + module = "tests.test_spiderloader.test_spiders.spider1" + settings = Settings( + {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} + ) + spider_loader = SpiderLoader.from_settings(settings) + if str(w[0].message).startswith("_SixMetaPathImporter"): + # needed on 3.10 because of https://github.com/benjaminp/six/issues/349, + # at least until all six versions we can import (including botocore.vendored.six) + # are updated to 1.16.0+ + w.pop(0) + assert "Could not load spiders from module" in str(w[0].message) + + spiders = spider_loader.list() + assert not spiders -class DuplicateSpiderNameLoaderTest(unittest.TestCase): - def setUp(self): - orig_spiders_dir = module_dir / "test_spiders" - self.tmpdir = Path(self.mktemp()) - self.tmpdir.mkdir() - self.spiders_dir = self.tmpdir / "test_spiders_xxx" - _copytree(orig_spiders_dir, self.spiders_dir) - sys.path.append(str(self.tmpdir)) - self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) +class TestDuplicateSpiderNameLoader: + def test_dupename_warning(self, spider_loader_env): + settings, spiders_dir = spider_loader_env - def tearDown(self): - del sys.modules["test_spiders_xxx"] - sys.path.remove(str(self.tmpdir)) - - def test_dupename_warning(self): # copy 1 spider module so as to have duplicate spider name - shutil.copyfile( - self.tmpdir / "test_spiders_xxx" / "spider3.py", - self.tmpdir / "test_spiders_xxx" / "spider3dupe.py", - ) + shutil.copyfile(spiders_dir / "spider3.py", spiders_dir / "spider3dupe.py") with warnings.catch_warnings(record=True) as w: - spider_loader = SpiderLoader.from_settings(self.settings) + spider_loader = SpiderLoader.from_settings(settings) - self.assertEqual(len(w), 1) + assert len(w) == 1 msg = str(w[0].message) - self.assertIn("several spiders with the same name", msg) - self.assertIn("'spider3'", msg) - self.assertTrue(msg.count("'spider3'") == 2) + assert "several spiders with the same name" in msg + assert "'spider3'" in msg + assert msg.count("'spider3'") == 2 - self.assertNotIn("'spider1'", msg) - self.assertNotIn("'spider2'", msg) - self.assertNotIn("'spider4'", msg) + assert "'spider1'" not in msg + assert "'spider2'" not in msg + assert "'spider4'" not in msg spiders = set(spider_loader.list()) - self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"}) + assert spiders == {"spider1", "spider2", "spider3", "spider4"} - def test_multiple_dupename_warning(self): + def test_multiple_dupename_warning(self, spider_loader_env): + settings, spiders_dir = spider_loader_env # copy 2 spider modules so as to have duplicate spider name # This should issue 2 warning, 1 for each duplicate spider name - shutil.copyfile( - self.tmpdir / "test_spiders_xxx" / "spider1.py", - self.tmpdir / "test_spiders_xxx" / "spider1dupe.py", - ) - shutil.copyfile( - self.tmpdir / "test_spiders_xxx" / "spider2.py", - self.tmpdir / "test_spiders_xxx" / "spider2dupe.py", - ) + shutil.copyfile(spiders_dir / "spider1.py", spiders_dir / "spider1dupe.py") + shutil.copyfile(spiders_dir / "spider2.py", spiders_dir / "spider2dupe.py") with warnings.catch_warnings(record=True) as w: - spider_loader = SpiderLoader.from_settings(self.settings) + spider_loader = SpiderLoader.from_settings(settings) - self.assertEqual(len(w), 1) + assert len(w) == 1 msg = str(w[0].message) - self.assertIn("several spiders with the same name", msg) - self.assertIn("'spider1'", msg) - self.assertTrue(msg.count("'spider1'") == 2) + assert "several spiders with the same name" in msg + assert "'spider1'" in msg + assert msg.count("'spider1'") == 2 - self.assertIn("'spider2'", msg) - self.assertTrue(msg.count("'spider2'") == 2) + assert "'spider2'" in msg + assert msg.count("'spider2'") == 2 - self.assertNotIn("'spider3'", msg) - self.assertNotIn("'spider4'", msg) + assert "'spider3'" not in msg + assert "'spider4'" not in msg spiders = set(spider_loader.list()) - self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"}) + assert spiders == {"spider1", "spider2", "spider3", "spider4"} + + +class CustomSpiderLoader(SpiderLoader): + pass + + +def test_custom_spider_loader(): + settings = Settings( + { + "SPIDER_LOADER_CLASS": CustomSpiderLoader, + } + ) + spider_loader = get_spider_loader(settings) + assert isinstance(spider_loader, CustomSpiderLoader) + + +def test_dummy_spider_loader(spider_loader_env): + settings, _ = spider_loader_env + spider_loader = DummySpiderLoader.from_settings(settings) + assert not spider_loader.list() + with pytest.raises(KeyError): + spider_loader.load("spider1") diff --git a/tests/test_spiderloader/spiders_from_addons/__init__.py b/tests/test_spiderloader/spiders_from_addons/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_spiderloader/spiders_from_addons/spider0.py b/tests/test_spiderloader/spiders_from_addons/spider0.py new file mode 100644 index 000000000..45c3f64a7 --- /dev/null +++ b/tests/test_spiderloader/spiders_from_addons/spider0.py @@ -0,0 +1,6 @@ +from scrapy.spiders import Spider + + +class SpiderFromAddon(Spider): + name = "spider_from_addon" + allowed_domains = ["scrapy1.org", "scrapy3.org"] diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 38ca8d950..a0d296553 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,546 +1,516 @@ -import collections.abc -from typing import Optional +from __future__ import annotations + +from collections.abc import AsyncIterator, Iterable +from inspect import isasyncgen +from typing import TYPE_CHECKING, Any, cast from unittest import mock -from testfixtures import LogCapture +import pytest from twisted.internet import defer -from twisted.python.failure import Failure -from twisted.trial.unittest import TestCase from scrapy.core.spidermw import SpiderMiddlewareManager -from scrapy.exceptions import _InvalidOutput +from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen -from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future +from scrapy.utils.asyncio import call_later +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from twisted.python.failure import Failure + + from scrapy.crawler import Crawler -class SpiderMiddlewareTestCase(TestCase): - def setUp(self): +class TestSpiderMiddleware: + def setup_method(self) -> None: self.request = Request("http://example.com/index.html") self.response = Response(self.request.url, request=self.request) self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES_BASE": {}}) - self.spider = self.crawler._create_spider("foo") + self.crawler.spider = self.crawler._create_spider("foo") self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - def _scrape_response(self): - """Execute spider mw manager's scrape_response method and return the result. + async def _scrape_response(self) -> Any: + """Execute spider mw manager's scrape_response_async method and return the result. Raise exception in case of failure. """ - scrape_func = mock.MagicMock() - dfd = self.mwman.scrape_response( - scrape_func, self.response, self.request, self.spider + + async def scrape_func( + response: Response | Failure, request: Request + ) -> Iterable[Any]: + return mock.MagicMock() + + return await self.mwman.scrape_response_async( + scrape_func, self.response, self.request ) - # catch deferred result and return the value - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - ret = results[0] - return ret -class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase): +class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_input method""" - def test_invalid_process_spider_input(self): + @coroutine_test + async def test_invalid_process_spider_input(self): class InvalidProcessSpiderInputMiddleware: - def process_spider_input(self, response, spider): + def process_spider_input(self, response): return 1 self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware()) - result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._scrape_response() -class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase): - """Invalid return value for process_spider_output method""" - - def test_invalid_process_spider_output(self): - class InvalidProcessSpiderOutputMiddleware: - def process_spider_output(self, response, result, spider): - return 1 - - self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware()) - result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, _InvalidOutput) - - -class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase): +class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_exception method""" - def test_invalid_process_spider_exception(self): + @coroutine_test + async def test_invalid_process_spider_exception(self): class InvalidProcessSpiderOutputExceptionMiddleware: - def process_spider_exception(self, response, exception, spider): + def process_spider_exception(self, response, exception): return 1 class RaiseExceptionProcessSpiderOutputMiddleware: - def process_spider_output(self, response, result, spider): - raise Exception() + async def process_spider_output(self, response, result): + raise RuntimeError + yield # pylint: disable=unreachable self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) - result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, _InvalidOutput) + it = await self._scrape_response() + with pytest.raises(_InvalidOutput): + await collect_asyncgen(it) -class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): +class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): """Re raise the exception by returning None""" - def test_process_spider_exception_return_none(self): + @coroutine_test + async def test_process_spider_exception_return_none(self): class ProcessSpiderExceptionReturnNoneMiddleware: - def process_spider_exception(self, response, exception, spider): + def process_spider_exception(self, response, exception): return None class RaiseExceptionProcessSpiderOutputMiddleware: - def process_spider_output(self, response, result, spider): + async def process_spider_output(self, response, result): 1 / 0 + yield self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) - result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, ZeroDivisionError) + it = await self._scrape_response() + with pytest.raises(ZeroDivisionError): + await collect_asyncgen(it) -class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): +class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): """Helpers for testing sync, async and mixed middlewares. - Should work for process_spider_output and, when it's supported, process_start_requests. + Should work for process_spider_output and, when it's supported, process_start. """ - ITEM_TYPE: type + ITEM_TYPE: type | tuple[type, ...] RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects @staticmethod - def _construct_mw_setting(*mw_classes, start_index: Optional[int] = None): + def _construct_mw_setting( + *mw_classes: type[Any], start_index: int | None = None + ) -> dict[type[Any], int]: if start_index is None: start_index = 10 return {i: c for c, i in enumerate(mw_classes, start=start_index)} - def _scrape_func(self, *args, **kwargs): + def _callback(self) -> Any: yield {"foo": 1} yield {"foo": 2} yield {"foo": 3} - @defer.inlineCallbacks - def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): + async def _scrape_func( + self, response: Response | Failure, request: Request + ) -> Iterable[Any] | AsyncIterator[Any]: + return cast("Iterable[Any] | AsyncIterator[Any]", self._callback()) + + async def _get_middleware_result( + self, *mw_classes: type[Any], start_index: int | None = None + ) -> Any: setting = self._construct_mw_setting(*mw_classes, start_index=start_index) self.crawler = get_crawler( Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} ) - self.spider = self.crawler._create_spider("foo") + self.crawler.spider = self.crawler._create_spider("foo") self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - result = yield self.mwman.scrape_response( - self._scrape_func, self.response, self.request, self.spider + return await self.mwman.scrape_response_async( + self._scrape_func, self.response, self.request ) - return result - @defer.inlineCallbacks - def _test_simple_base( - self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None - ): - with LogCapture() as log: - result = yield self._get_middleware_result( - *mw_classes, start_index=start_index - ) - self.assertIsInstance(result, collections.abc.Iterable) - result_list = list(result) - self.assertEqual(len(result_list), self.RESULT_COUNT) - self.assertIsInstance(result_list[0], self.ITEM_TYPE) - self.assertEqual("downgraded to a non-async" in str(log), downgrade) - - @defer.inlineCallbacks - def _test_asyncgen_base( - self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None - ): - with LogCapture() as log: - result = yield self._get_middleware_result( - *mw_classes, start_index=start_index - ) - self.assertIsInstance(result, collections.abc.AsyncIterator) - result_list = yield deferred_from_coro(collect_asyncgen(result)) - self.assertEqual(len(result_list), self.RESULT_COUNT) - self.assertIsInstance(result_list[0], self.ITEM_TYPE) - self.assertEqual("downgraded to a non-async" in str(log), downgrade) + async def _test_asyncgen_base( + self, + *mw_classes: type[Any], + start_index: int | None = None, + ) -> None: + result = await self._get_middleware_result(*mw_classes, start_index=start_index) + assert isinstance(result, AsyncIterator) + result_list = await collect_asyncgen(result) + assert len(result_list) == self.RESULT_COUNT + assert isinstance(result_list[0], self.ITEM_TYPE) -class ProcessSpiderOutputSimpleMiddleware: - def process_spider_output(self, response, result, spider): +class ProcessSpiderOutputSyncMiddleware: + def process_spider_output(self, response, result): yield from result class ProcessSpiderOutputAsyncGenMiddleware: - async def process_spider_output(self, response, result, spider): + async def process_spider_output(self, response, result): async for r in result: yield r class ProcessSpiderOutputUniversalMiddleware: - def process_spider_output(self, response, result, spider): + def process_spider_output(self, response, result): yield from result - async def process_spider_output_async(self, response, result, spider): + async def process_spider_output_async(self, response, result): async for r in result: yield r class ProcessSpiderExceptionSimpleIterableMiddleware: - def process_spider_exception(self, response, exception, spider): + def process_spider_exception(self, response, exception): yield {"foo": 1} yield {"foo": 2} yield {"foo": 3} -class ProcessSpiderExceptionAsyncIterableMiddleware: - async def process_spider_exception(self, response, exception, spider): +class ProcessSpiderExceptionAsyncIteratorMiddleware: + async def process_spider_exception(self, response, exception): yield {"foo": 1} d = defer.Deferred() - from twisted.internet import reactor - - reactor.callLater(0, d.callback, None) + call_later(0, d.callback, None) await maybe_deferred_to_future(d) yield {"foo": 2} yield {"foo": 3} -class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware): """process_spider_output tests for simple callbacks""" ITEM_TYPE = dict - MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware + MW_SYNC = ProcessSpiderOutputSyncMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware - def test_simple(self): - """Simple mw""" - return self._test_simple_base(self.MW_SIMPLE) + @coroutine_test + async def test_sync(self): + """Unsupported sync mw""" + with pytest.raises( + TypeError, match=r"doesn't support asynchronous spider output" + ): + await self._get_middleware_result(self.MW_SYNC) - def test_asyncgen(self): - """Asyncgen mw; upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN) + @coroutine_test + async def test_asyncgen(self): + """Asyncgen mw""" + await self._test_asyncgen_base(self.MW_ASYNCGEN) - def test_simple_asyncgen(self): - """Simple mw -> asyncgen mw; upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE) - - def test_asyncgen_simple(self): - """Asyncgen mw -> simple mw; upgrade then downgrade""" - return self._test_simple_base(self.MW_SIMPLE, self.MW_ASYNCGEN, downgrade=True) - - def test_universal(self): + @coroutine_test + async def test_universal(self): """Universal mw""" - return self._test_simple_base(self.MW_UNIVERSAL) + await self._test_asyncgen_base(self.MW_UNIVERSAL) - def test_universal_simple(self): - """Universal mw -> simple mw""" - return self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL) + @coroutine_test + async def test_universal_asyncgen(self): + """Universal mw -> asyncgen mw""" + await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL) - def test_simple_universal(self): - """Simple mw -> universal mw""" - return self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE) - - def test_universal_asyncgen(self): - """Universal mw -> asyncgen mw; upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL) - - def test_asyncgen_universal(self): - """Asyncgen mw -> universal mw; upgrade""" - return self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN) + @coroutine_test + async def test_asyncgen_universal(self): + """Asyncgen mw -> universal mw""" + await self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN) -class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple): +class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple): """process_spider_output tests for async generator callbacks""" - async def _scrape_func(self, *args, **kwargs): - for item in super()._scrape_func(): + async def _callback(self) -> Any: + for item in super()._callback(): yield item - def test_simple(self): - """Simple mw; downgrade""" - return self._test_simple_base(self.MW_SIMPLE, downgrade=True) - - def test_simple_asyncgen(self): - """Simple mw -> asyncgen mw; downgrade then upgrade""" - return self._test_asyncgen_base( - self.MW_ASYNCGEN, self.MW_SIMPLE, downgrade=True - ) - - def test_universal(self): - """Universal mw""" - return self._test_asyncgen_base(self.MW_UNIVERSAL) - - def test_universal_simple(self): - """Universal mw -> simple mw; downgrade""" - return self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL, downgrade=True) - - def test_simple_universal(self): - """Simple mw -> universal mw; downgrade""" - return self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE, downgrade=True) - class ProcessSpiderOutputNonIterableMiddleware: - def process_spider_output(self, response, result, spider): + def process_spider_output(self, response, result): return class ProcessSpiderOutputCoroutineMiddleware: - async def process_spider_output(self, response, result, spider): - results = [] - for r in result: - results.append(r) - return results + async def process_spider_output(self, response, result): + return result -class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): - @defer.inlineCallbacks - def test_non_iterable(self): - with self.assertRaisesRegex( - _InvalidOutput, - ( - r"\.process_spider_output must return an iterable, got " - ), - ): - yield self._get_middleware_result( - ProcessSpiderOutputNonIterableMiddleware, - ) - - @defer.inlineCallbacks - def test_coroutine(self): - with self.assertRaisesRegex( - _InvalidOutput, - r"\.process_spider_output must be an asynchronous generator", - ): - yield self._get_middleware_result( - ProcessSpiderOutputCoroutineMiddleware, - ) +class ProcessStartSimpleMiddleware: + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request -class ProcessStartRequestsSimpleMiddleware: - def process_start_requests(self, start_requests, spider): - yield from start_requests +class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware): + """process_start tests for simple start""" + ITEM_TYPE = (Request, dict) + MW_SIMPLE = ProcessStartSimpleMiddleware -class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): - """process_start_requests tests for simple start_requests""" + async def _get_processed_start( + self, *mw_classes: type[Any] + ) -> AsyncIterator[Any] | None: + class TestSpider(Spider): + name = "test" - ITEM_TYPE = Request - MW_SIMPLE = ProcessStartRequestsSimpleMiddleware + async def start(self): + for i in range(2): + yield Request(f"https://example.com/{i}", dont_filter=True) + yield {"name": "test item"} - def _start_requests(self): - for i in range(3): - yield Request(f"https://example.com/{i}", dont_filter=True) - - @defer.inlineCallbacks - def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): - setting = self._construct_mw_setting(*mw_classes, start_index=start_index) + setting = self._construct_mw_setting(*mw_classes) self.crawler = get_crawler( - Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} + TestSpider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} ) - self.spider = self.crawler._create_spider("foo") + self.crawler.spider = self.crawler._create_spider() self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - start_requests = iter(self._start_requests()) - results = yield self.mwman.process_start_requests(start_requests, self.spider) - return results + return await self.mwman.process_start() - def test_simple(self): + @coroutine_test + async def test_simple(self): """Simple mw""" - return self._test_simple_base(self.MW_SIMPLE) + start = await self._get_processed_start(self.MW_SIMPLE) + assert isasyncgen(start) + start_list = await collect_asyncgen(start) + assert len(start_list) == self.RESULT_COUNT + assert isinstance(start_list[0], self.ITEM_TYPE) class UniversalMiddlewareNoSync: - async def process_spider_output_async(self, response, result, spider): + async def process_spider_output_async(self, response, result): yield class UniversalMiddlewareBothSync: - def process_spider_output(self, response, result, spider): + def process_spider_output(self, response, result): yield - def process_spider_output_async(self, response, result, spider): + def process_spider_output_async(self, response, result): yield class UniversalMiddlewareBothAsync: - async def process_spider_output(self, response, result, spider): + async def process_spider_output(self, response, result): yield - async def process_spider_output_async(self, response, result, spider): + async def process_spider_output_async(self, response, result): yield -class UniversalMiddlewareManagerTest(TestCase): - def setUp(self): - self.mwman = SpiderMiddlewareManager() +class TestUniversalMiddlewareManager: + @pytest.fixture + def crawler(self) -> Crawler: + return get_crawler(Spider) - def test_simple_mw(self): - mw = ProcessSpiderOutputSimpleMiddleware - self.mwman._add_middleware(mw) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], mw.process_spider_output + @pytest.fixture + def mwman(self, crawler: Crawler) -> SpiderMiddlewareManager: + return SpiderMiddlewareManager.from_crawler(crawler) + + def test_simple_mw(self, mwman: SpiderMiddlewareManager) -> None: + mw = ProcessSpiderOutputSyncMiddleware() + with pytest.raises( + TypeError, match=r"doesn't support asynchronous spider output" + ): + mwman._add_middleware(mw) + + def test_async_mw(self, mwman: SpiderMiddlewareManager) -> None: + mw = ProcessSpiderOutputAsyncGenMiddleware() + mwman._add_middleware(mw) + assert ( + mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable ) - def test_async_mw(self): - mw = ProcessSpiderOutputAsyncGenMiddleware - self.mwman._add_middleware(mw) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], mw.process_spider_output + def test_universal_mw(self, mwman: SpiderMiddlewareManager) -> None: + mw = ProcessSpiderOutputUniversalMiddleware() + mwman._add_middleware(mw) + assert ( + mwman.methods["process_spider_output"][0] == mw.process_spider_output_async # pylint: disable=comparison-with-callable ) - def test_universal_mw(self): - mw = ProcessSpiderOutputUniversalMiddleware - self.mwman._add_middleware(mw) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], - (mw.process_spider_output, mw.process_spider_output_async), + def test_universal_mw_no_sync( + self, mwman: SpiderMiddlewareManager, caplog: pytest.LogCaptureFixture + ) -> None: + mwman._add_middleware(UniversalMiddlewareNoSync()) + assert ( + "UniversalMiddlewareNoSync has process_spider_output_async()" + " without process_spider_output" in caplog.text ) + assert mwman.methods["process_spider_output"][0] is None - def test_universal_mw_no_sync(self): - with LogCapture() as log: - self.mwman._add_middleware(UniversalMiddlewareNoSync) - self.assertIn( - "UniversalMiddlewareNoSync has process_spider_output_async" - " without process_spider_output", - str(log), - ) - self.assertEqual(self.mwman.methods["process_spider_output"][0], None) - - def test_universal_mw_both_sync(self): - mw = UniversalMiddlewareBothSync - with LogCapture() as log: - self.mwman._add_middleware(mw) - self.assertIn( + def test_universal_mw_both_sync( + self, mwman: SpiderMiddlewareManager, caplog: pytest.LogCaptureFixture + ) -> None: + mw = UniversalMiddlewareBothSync() + mwman._add_middleware(mw) + assert ( "UniversalMiddlewareBothSync.process_spider_output_async " - "is not an async generator function", - str(log), + "is not an async generator function" in caplog.text ) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], mw.process_spider_output + assert ( + mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable ) - def test_universal_mw_both_async(self): - with LogCapture() as log: - self.mwman._add_middleware(UniversalMiddlewareBothAsync) - self.assertIn( + def test_universal_mw_both_async( + self, mwman: SpiderMiddlewareManager, caplog: pytest.LogCaptureFixture + ) -> None: + mwman._add_middleware(UniversalMiddlewareBothAsync()) + assert ( "UniversalMiddlewareBothAsync.process_spider_output " - "is an async generator function while process_spider_output_async exists", - str(log), + "is an async generator function while process_spider_output_async() exists" + in caplog.text ) - self.assertEqual(self.mwman.methods["process_spider_output"][0], None) + assert mwman.methods["process_spider_output"][0] is None -class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): +class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware): ITEM_TYPE = dict - MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware - @defer.inlineCallbacks - def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): + async def _get_middleware_result( + self, *mw_classes: type[Any], start_index: int | None = None + ) -> Any: setting = self._construct_mw_setting(*mw_classes, start_index=start_index) self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES": setting}) - self.spider = self.crawler._create_spider("foo") + self.crawler.spider = self.crawler._create_spider("foo") self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - result = yield self.mwman.scrape_response( - self._scrape_func, self.response, self.request, self.spider + return await self.mwman.scrape_response_async( + self._scrape_func, self.response, self.request ) - return result - def test_just_builtin(self): - return self._test_simple_base() + @coroutine_test + async def test_just_builtin(self): + await self._test_asyncgen_base() - def test_builtin_simple(self): - return self._test_simple_base(self.MW_SIMPLE, start_index=1000) - - def test_builtin_async(self): + @coroutine_test + async def test_builtin_async(self): """Upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) + await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) - def test_builtin_universal(self): - return self._test_simple_base(self.MW_UNIVERSAL, start_index=1000) + @coroutine_test + async def test_builtin_universal(self): + await self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000) - def test_simple_builtin(self): - return self._test_simple_base(self.MW_SIMPLE) - - def test_async_builtin(self): + @coroutine_test + async def test_async_builtin(self): """Upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN) + await self._test_asyncgen_base(self.MW_ASYNCGEN) - def test_universal_builtin(self): - return self._test_simple_base(self.MW_UNIVERSAL) + @coroutine_test + async def test_universal_builtin(self): + await self._test_asyncgen_base(self.MW_UNIVERSAL) -class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest): - async def _scrape_func(self, *args, **kwargs): - for item in super()._scrape_func(): +class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple): + async def _callback(self) -> Any: + for item in super()._callback(): yield item - def test_just_builtin(self): - return self._test_asyncgen_base() - def test_builtin_simple(self): - """Downgrade""" - return self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000) - - def test_builtin_async(self): - return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) - - def test_builtin_universal(self): - return self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000) - - def test_simple_builtin(self): - """Downgrade""" - return self._test_simple_base(self.MW_SIMPLE, downgrade=True) - - def test_async_builtin(self): - return self._test_asyncgen_base(self.MW_ASYNCGEN) - - def test_universal_builtin(self): - return self._test_asyncgen_base(self.MW_UNIVERSAL) - - -class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): ITEM_TYPE = dict - MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware - MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIterableMiddleware + MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIteratorMiddleware - def _scrape_func(self, *args, **kwargs): + def _callback(self) -> Any: 1 / 0 - @defer.inlineCallbacks - def _test_asyncgen_nodowngrade(self, *mw_classes): - with self.assertRaisesRegex( - _InvalidOutput, "Async iterable returned from .+ cannot be downgraded" + async def _test_asyncgen_nodowngrade(self, *mw_classes: type[Any]) -> None: + with pytest.raises( + _InvalidOutput, + match=r"Async iterable returned from .+ cannot be downgraded", ): - yield self._get_middleware_result(*mw_classes) + await self._get_middleware_result(*mw_classes) - def test_exc_simple(self): + @coroutine_test + async def test_exc_simple(self): """Simple exc mw""" - return self._test_simple_base(self.MW_EXC_SIMPLE) + await self._test_asyncgen_base(self.MW_EXC_SIMPLE) - def test_exc_async(self): + @coroutine_test + async def test_exc_async(self): """Async exc mw""" - return self._test_asyncgen_base(self.MW_EXC_ASYNCGEN) + await self._test_asyncgen_base(self.MW_EXC_ASYNCGEN) - def test_exc_simple_simple(self): - """Simple exc mw -> simple output mw""" - return self._test_simple_base(self.MW_SIMPLE, self.MW_EXC_SIMPLE) - - def test_exc_async_async(self): + @coroutine_test + async def test_exc_async_async(self): """Async exc mw -> async output mw""" - return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_ASYNCGEN) + await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_ASYNCGEN) - def test_exc_simple_async(self): + @coroutine_test + async def test_exc_simple_async(self): """Simple exc mw -> async output mw; upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE) + await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE) - def test_exc_async_simple(self): - """Async exc mw -> simple output mw; cannot work as downgrading is not supported""" - return self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN) + +class TestDeprecatedSpiderArg(TestSpiderMiddleware): + @coroutine_test + async def test_deprecated_mw_spider_arg(self): + class DeprecatedSpiderArgMiddleware1: + def process_spider_input(self, response, spider): + return None + + async def process_spider_output(self, response, result, spider): + 1 / 0 + yield + + class DeprecatedSpiderArgMiddleware2: + def process_spider_exception(self, response, exception, spider): + return [] + + with pytest.warns( + ScrapyDeprecationWarning, + match=r"process_spider_exception\(\) requires a spider argument", + ): + self.mwman._add_middleware(DeprecatedSpiderArgMiddleware2()) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"process_spider_input\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"process_spider_output\(\) requires a spider argument", + ), + ): + self.mwman._add_middleware(DeprecatedSpiderArgMiddleware1()) + it = await self._scrape_response() + await collect_asyncgen(it) + + @coroutine_test + async def test_deprecated_mwman_spider_arg(self): + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a spider argument to SpiderMiddlewareManager.process_start\(\)" + r" is deprecated and the passed value is ignored", + ): + await self.mwman.process_start(DefaultSpider()) + + @coroutine_test + async def test_deprecated_mwman_spider_arg_no_crawler(self): + with pytest.warns( + ScrapyDeprecationWarning, + match=r"MiddlewareManager.__init__\(\) was called without the crawler argument", + ): + mwman = SpiderMiddlewareManager() + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a spider argument to SpiderMiddlewareManager.process_start\(\)" + r" is deprecated, SpiderMiddlewareManager should be instantiated with a Crawler", + ): + await mwman.process_start(DefaultSpider()) diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py new file mode 100644 index 000000000..70326f3f1 --- /dev/null +++ b/tests/test_spidermiddleware_base.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import pytest + +from scrapy import Request, Spider +from scrapy.http import Response +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen +from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from scrapy.crawler import Crawler + + +@pytest.fixture +def crawler() -> Crawler: + return get_crawler(Spider) + + +@coroutine_test +async def test_trivial(crawler: Crawler) -> None: + class TrivialSpiderMiddleware(BaseSpiderMiddleware): + pass + + mw = TrivialSpiderMiddleware.from_crawler(crawler) + assert hasattr(mw, "crawler") + assert mw.crawler is crawler + test_req = Request("data:,") + spider_output = [test_req, {"foo": "bar"}] + for processed in [ + list(mw.process_spider_output(Response("data:,"), spider_output)), + await collect_asyncgen(mw.process_start(as_async_generator(spider_output))), + ]: + assert processed == [test_req, {"foo": "bar"}] + + +@coroutine_test +async def test_processed_request(crawler: Crawler) -> None: + class ProcessReqSpiderMiddleware(BaseSpiderMiddleware): + def get_processed_request( + self, request: Request, response: Response | None + ) -> Request | None: + if request.url == "data:2,": + return None + if request.url == "data:3,": + return Request("data:30,") + return request + + mw = ProcessReqSpiderMiddleware.from_crawler(crawler) + test_req1 = Request("data:1,") + test_req2 = Request("data:2,") + test_req3 = Request("data:3,") + spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3] + for processed in [ + list(mw.process_spider_output(Response("data:,"), spider_output)), + await collect_asyncgen(mw.process_start(as_async_generator(spider_output))), + ]: + assert len(processed) == 3 + assert isinstance(processed[0], Request) + assert processed[0].url == "data:1," + assert processed[1] == {"foo": "bar"} + assert isinstance(processed[2], Request) + assert processed[2].url == "data:30," + + +@coroutine_test +async def test_processed_item(crawler: Crawler) -> None: + class ProcessItemSpiderMiddleware(BaseSpiderMiddleware): + def get_processed_item(self, item: Any, response: Response | None) -> Any: + if item["foo"] == 2: + return None + if item["foo"] == 3: + item["foo"] = 30 + return item + + mw = ProcessItemSpiderMiddleware.from_crawler(crawler) + test_req = Request("data:,") + spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}] + for processed in [ + list(mw.process_spider_output(Response("data:,"), spider_output)), + await collect_asyncgen(mw.process_start(as_async_generator(spider_output))), + ]: + assert processed == [{"foo": 1}, test_req, {"foo": 30}] + + +@coroutine_test +async def test_processed_both(crawler: Crawler) -> None: + class ProcessBothSpiderMiddleware(BaseSpiderMiddleware): + def get_processed_request( + self, request: Request, response: Response | None + ) -> Request | None: + if request.url == "data:2,": + return None + if request.url == "data:3,": + return Request("data:30,") + return request + + def get_processed_item(self, item: Any, response: Response | None) -> Any: + if item["foo"] == 2: + return None + if item["foo"] == 3: + item["foo"] = 30 + return item + + mw = ProcessBothSpiderMiddleware.from_crawler(crawler) + test_req1 = Request("data:1,") + test_req2 = Request("data:2,") + test_req3 = Request("data:3,") + spider_output = [ + test_req1, + {"foo": 1}, + {"foo": 2}, + test_req2, + {"foo": 3}, + test_req3, + ] + for processed in [ + list(mw.process_spider_output(Response("data:,"), spider_output)), + await collect_asyncgen(mw.process_start(as_async_generator(spider_output))), + ]: + assert len(processed) == 4 + assert isinstance(processed[0], Request) + assert processed[0].url == "data:1," + assert processed[1] == {"foo": 1} + assert processed[2] == {"foo": 30} + assert isinstance(processed[3], Request) + assert processed[3].url == "data:30," diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index e359d9cfc..00e547305 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -1,41 +1,57 @@ -from unittest import TestCase +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest from scrapy.http import Request, Response from scrapy.spidermiddlewares.depth import DepthMiddleware from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector from scrapy.utils.test import get_crawler +if TYPE_CHECKING: + from collections.abc import Generator -class TestDepthMiddleware(TestCase): - def setUp(self): - crawler = get_crawler(Spider) - self.spider = crawler._create_spider("scrapytest.org") + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector - self.stats = StatsCollector(crawler) - self.stats.open_spider(self.spider) - self.mw = DepthMiddleware(1, self.stats, True) +@pytest.fixture +def crawler() -> Crawler: + return get_crawler(Spider, {"DEPTH_LIMIT": 1, "DEPTH_STATS_VERBOSE": True}) - def test_process_spider_output(self): - req = Request("http://scrapytest.org") - resp = Response("http://scrapytest.org") - resp.request = req - result = [Request("http://scrapytest.org")] - out = list(self.mw.process_spider_output(resp, result, self.spider)) - self.assertEqual(out, result) +@pytest.fixture +def stats(crawler: Crawler) -> Generator[StatsCollector]: + assert crawler.stats is not None + crawler.stats.open_spider() - rdc = self.stats.get_value("request_depth_count/1", spider=self.spider) - self.assertEqual(rdc, 1) + yield crawler.stats - req.meta["depth"] = 1 + crawler.stats.close_spider() - out2 = list(self.mw.process_spider_output(resp, result, self.spider)) - self.assertEqual(out2, []) - rdm = self.stats.get_value("request_depth_max", spider=self.spider) - self.assertEqual(rdm, 1) +@pytest.fixture +def mw(crawler: Crawler) -> DepthMiddleware: + return DepthMiddleware.from_crawler(crawler) - def tearDown(self): - self.stats.close_spider(self.spider, "") + +def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> None: + req = Request("http://scrapytest.org") + resp = Response("http://scrapytest.org") + resp.request = req + result = [Request("http://scrapytest.org")] + + out = list(mw.process_spider_output(resp, result)) + assert out == result + + rdc = stats.get_value("request_depth_count/1") + assert rdc == 1 + + req.meta["depth"] = 1 + + out2 = list(mw.process_spider_output(resp, result)) + assert not out2 + + rdm = stats.get_value("request_depth_max") + assert rdm == 1 diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 044455415..6a054ded5 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,23 +1,24 @@ -import logging -from typing import Set -from unittest import TestCase +from __future__ import annotations -from testfixtures import LogCapture -from twisted.internet import defer -from twisted.trial.unittest import TestCase as TrialTestCase +import logging +from typing import TYPE_CHECKING + +import pytest from scrapy.http import Request, Response -from scrapy.settings import Settings from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware -from scrapy.spiders import Spider +from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer from tests.spiders import MockServerSpider +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from tests.mockserver.http import MockServer class _HttpErrorSpider(MockServerSpider): name = "httperror" - bypass_status_codes: Set[int] = set() + bypass_status_codes: set[int] = set() def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -31,7 +32,7 @@ class _HttpErrorSpider(MockServerSpider): self.skipped = set() self.parsed = set() - def start_requests(self): + async def start(self): for url in self.start_urls: yield Request(url, self.parse, errback=self.on_error) @@ -50,111 +51,132 @@ class _HttpErrorSpider(MockServerSpider): return failure -def _responses(request, status_codes): - responses = [] - for code in status_codes: - response = Response(request.url, status=code) - response.request = request - responses.append(response) - return responses +req = Request("http://scrapytest.org") -class TestHttpErrorMiddleware(TestCase): - def setUp(self): - crawler = get_crawler(Spider) - self.spider = Spider.from_crawler(crawler, name="foo") - self.mw = HttpErrorMiddleware(Settings({})) - self.req = Request("http://scrapytest.org") - self.res200, self.res404 = _responses(self.req, [200, 404]) +def _response(request: Request, status_code: int) -> Response: + return Response(request.url, status=status_code, request=request) - def test_process_spider_input(self): - self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises( - HttpError, self.mw.process_spider_input, self.res404, self.spider - ) - def test_process_spider_exception(self): - self.assertEqual( - [], - self.mw.process_spider_exception( - self.res404, HttpError(self.res404), self.spider - ), - ) - self.assertIsNone( - self.mw.process_spider_exception(self.res404, Exception(), self.spider) - ) +@pytest.fixture +def res200() -> Response: + return _response(req, 200) - def test_handle_httpstatus_list(self): - res = self.res404.copy() - res.request = Request( + +@pytest.fixture +def res402() -> Response: + return _response(req, 402) + + +@pytest.fixture +def res404() -> Response: + return _response(req, 404) + + +class TestHttpErrorMiddleware: + @pytest.fixture + def mw(self) -> HttpErrorMiddleware: + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() + return HttpErrorMiddleware.from_crawler(crawler) + + def test_process_spider_input( + self, mw: HttpErrorMiddleware, res200: Response, res404: Response + ) -> None: + mw.process_spider_input(res200) + with pytest.raises(HttpError): + mw.process_spider_input(res404) + + def test_process_spider_exception( + self, mw: HttpErrorMiddleware, res404: Response + ) -> None: + assert mw.process_spider_exception(res404, HttpError(res404)) == () + assert mw.process_spider_exception(res404, Exception()) is None + + def test_handle_httpstatus_list( + self, mw: HttpErrorMiddleware, res404: Response + ) -> None: + request = Request( "http://scrapytest.org", meta={"handle_httpstatus_list": [404]} ) - self.assertIsNone(self.mw.process_spider_input(res, self.spider)) + res = _response(request, 404) + mw.process_spider_input(res) - self.spider.handle_httpstatus_list = [404] - self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + assert mw.crawler.spider + mw.crawler.spider.handle_httpstatus_list = [404] # type: ignore[attr-defined] + mw.process_spider_input(res404) -class TestHttpErrorMiddlewareSettings(TestCase): +class TestHttpErrorMiddlewareSettings: """Similar test, but with settings""" - def setUp(self): - self.spider = Spider("foo") - self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)})) - self.req = Request("http://scrapytest.org") - self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) + @pytest.fixture + def mw(self) -> HttpErrorMiddleware: + crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOWED_CODES": (402,)}) + crawler.spider = crawler._create_spider() + return HttpErrorMiddleware.from_crawler(crawler) - def test_process_spider_input(self): - self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises( - HttpError, self.mw.process_spider_input, self.res404, self.spider - ) - self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider)) + def test_process_spider_input( + self, + mw: HttpErrorMiddleware, + res200: Response, + res402: Response, + res404: Response, + ) -> None: + mw.process_spider_input(res200) + with pytest.raises(HttpError): + mw.process_spider_input(res404) + mw.process_spider_input(res402) - def test_meta_overrides_settings(self): + def test_meta_overrides_settings(self, mw: HttpErrorMiddleware) -> None: request = Request( "http://scrapytest.org", meta={"handle_httpstatus_list": [404]} ) - res404 = self.res404.copy() - res404.request = request - res402 = self.res402.copy() - res402.request = request + res404 = _response(request, 404) + res402 = _response(request, 402) - self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) - self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) + mw.process_spider_input(res404) + with pytest.raises(HttpError): + mw.process_spider_input(res402) - def test_spider_override_settings(self): - self.spider.handle_httpstatus_list = [404] - self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) - self.assertRaises( - HttpError, self.mw.process_spider_input, self.res402, self.spider - ) + def test_spider_override_settings( + self, mw: HttpErrorMiddleware, res402: Response, res404: Response + ) -> None: + assert mw.crawler.spider + mw.crawler.spider.handle_httpstatus_list = [404] # type: ignore[attr-defined] + mw.process_spider_input(res404) + with pytest.raises(HttpError): + mw.process_spider_input(res402) -class TestHttpErrorMiddlewareHandleAll(TestCase): - def setUp(self): - self.spider = Spider("foo") - self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True})) - self.req = Request("http://scrapytest.org") - self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) +class TestHttpErrorMiddlewareHandleAll: + @pytest.fixture + def mw(self) -> HttpErrorMiddleware: + crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOW_ALL": True}) + crawler.spider = crawler._create_spider() + return HttpErrorMiddleware.from_crawler(crawler) - def test_process_spider_input(self): - self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) - self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + def test_process_spider_input( + self, + mw: HttpErrorMiddleware, + res200: Response, + res404: Response, + ) -> None: + mw.process_spider_input(res200) + mw.process_spider_input(res404) - def test_meta_overrides_settings(self): + def test_meta_overrides_settings(self, mw: HttpErrorMiddleware) -> None: request = Request( "http://scrapytest.org", meta={"handle_httpstatus_list": [404]} ) - res404 = self.res404.copy() - res404.request = request - res402 = self.res402.copy() - res402.request = request + res404 = _response(request, 404) + res402 = _response(request, 402) - self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) - self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) + mw.process_spider_input(res404) + with pytest.raises(HttpError): + mw.process_spider_input(res402) - def test_httperror_allow_all_false(self): + def test_httperror_allow_all_false(self) -> None: crawler = get_crawler(_HttpErrorSpider) mw = HttpErrorMiddleware.from_crawler(crawler) request_httpstatus_false = Request( @@ -163,73 +185,75 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): request_httpstatus_true = Request( "http://scrapytest.org", meta={"handle_httpstatus_all": True} ) - res404 = self.res404.copy() - res404.request = request_httpstatus_false - res402 = self.res402.copy() - res402.request = request_httpstatus_true + res404 = _response(request_httpstatus_false, 404) + res402 = _response(request_httpstatus_true, 402) - self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider) - self.assertIsNone(mw.process_spider_input(res402, self.spider)) + with pytest.raises(HttpError): + mw.process_spider_input(res404) + mw.process_spider_input(res402) -class TestHttpErrorMiddlewareIntegrational(TrialTestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() - - def tearDown(self): - self.mockserver.__exit__(None, None, None) - - @defer.inlineCallbacks - def test_middleware_works(self): +class TestHttpErrorMiddlewareIntegrational: + @coroutine_test + async def test_middleware_works(self, mockserver: MockServer) -> None: crawler = get_crawler(_HttpErrorSpider) - yield crawler.crawl(mockserver=self.mockserver) - assert not crawler.spider.skipped, crawler.spider.skipped - self.assertEqual(crawler.spider.parsed, {"200"}) - self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, _HttpErrorSpider) + assert not crawler.spider.skipped + assert crawler.spider.parsed == {"200"} + assert crawler.spider.failed == {"404", "402", "500"} + assert crawler.stats get_value = crawler.stats.get_value - self.assertEqual(get_value("httperror/response_ignored_count"), 3) - self.assertEqual(get_value("httperror/response_ignored_status_count/404"), 1) - self.assertEqual(get_value("httperror/response_ignored_status_count/402"), 1) - self.assertEqual(get_value("httperror/response_ignored_status_count/500"), 1) + assert get_value("httperror/response_ignored_count") == 3 + assert get_value("httperror/response_ignored_status_count/404") == 1 + assert get_value("httperror/response_ignored_status_count/402") == 1 + assert get_value("httperror/response_ignored_status_count/500") == 1 - @defer.inlineCallbacks - def test_logging(self): + @coroutine_test + async def test_logging( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(_HttpErrorSpider) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402}) - self.assertEqual(crawler.spider.parsed, {"200", "402"}) - self.assertEqual(crawler.spider.skipped, {"402"}) - self.assertEqual(crawler.spider.failed, {"404", "500"}) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver, bypass_status_codes={402}) + assert isinstance(crawler.spider, _HttpErrorSpider) + assert crawler.spider.parsed == {"200", "402"} + assert crawler.spider.skipped == {"402"} + assert crawler.spider.failed == {"404", "500"} - self.assertIn("Ignoring response <404", str(log)) - self.assertIn("Ignoring response <500", str(log)) - self.assertNotIn("Ignoring response <200", str(log)) - self.assertNotIn("Ignoring response <402", str(log)) + assert "Ignoring response <404" in caplog.text + assert "Ignoring response <500" in caplog.text + assert "Ignoring response <200" not in caplog.text + assert "Ignoring response <402" not in caplog.text - @defer.inlineCallbacks - def test_logging_level(self): + @coroutine_test + async def test_logging_level( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: # HttpError logs ignored responses with level INFO crawler = get_crawler(_HttpErrorSpider) - with LogCapture(level=logging.INFO) as log: - yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(crawler.spider.parsed, {"200"}) - self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, _HttpErrorSpider) + assert crawler.spider.parsed == {"200"} + assert crawler.spider.failed == {"404", "402", "500"} - self.assertIn("Ignoring response <402", str(log)) - self.assertIn("Ignoring response <404", str(log)) - self.assertIn("Ignoring response <500", str(log)) - self.assertNotIn("Ignoring response <200", str(log)) + assert "Ignoring response <402" in caplog.text + assert "Ignoring response <404" in caplog.text + assert "Ignoring response <500" in caplog.text + assert "Ignoring response <200" not in caplog.text # with level WARNING, we shouldn't capture anything from HttpError + caplog.clear() crawler = get_crawler(_HttpErrorSpider) - with LogCapture(level=logging.WARNING) as log: - yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(crawler.spider.parsed, {"200"}) - self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) + with caplog.at_level(logging.WARNING): + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, _HttpErrorSpider) + assert crawler.spider.parsed == {"200"} + assert crawler.spider.failed == {"404", "402", "500"} - self.assertNotIn("Ignoring response <402", str(log)) - self.assertNotIn("Ignoring response <404", str(log)) - self.assertNotIn("Ignoring response <500", str(log)) - self.assertNotIn("Ignoring response <200", str(log)) + assert "Ignoring response <402" not in caplog.text + assert "Ignoring response <404" not in caplog.text + assert "Ignoring response <500" not in caplog.text + assert "Ignoring response <200" not in caplog.text diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py deleted file mode 100644 index ea45b7698..000000000 --- a/tests/test_spidermiddleware_offsite.py +++ /dev/null @@ -1,102 +0,0 @@ -import warnings -from unittest import TestCase -from urllib.parse import urlparse - -from scrapy.http import Request, Response -from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, PortWarning, URLWarning -from scrapy.spiders import Spider -from scrapy.utils.test import get_crawler - - -class TestOffsiteMiddleware(TestCase): - def setUp(self): - crawler = get_crawler(Spider) - self.spider = crawler._create_spider(**self._get_spiderargs()) - self.mw = OffsiteMiddleware.from_crawler(crawler) - self.mw.spider_opened(self.spider) - - def _get_spiderargs(self): - return dict( - name="foo", - allowed_domains=["scrapytest.org", "scrapy.org", "scrapy.test.org"], - ) - - def test_process_spider_output(self): - res = Response("http://scrapytest.org") - - onsite_reqs = [ - Request("http://scrapytest.org/1"), - Request("http://scrapy.org/1"), - Request("http://sub.scrapy.org/1"), - Request("http://offsite.tld/letmepass", dont_filter=True), - Request("http://scrapy.test.org/"), - Request("http://scrapy.test.org:8000/"), - ] - offsite_reqs = [ - Request("http://scrapy2.org"), - Request("http://offsite.tld/"), - Request("http://offsite.tld/scrapytest.org"), - Request("http://offsite.tld/rogue.scrapytest.org"), - Request("http://rogue.scrapytest.org.haha.com"), - Request("http://roguescrapytest.org"), - Request("http://test.org/"), - Request("http://notscrapy.test.org/"), - ] - reqs = onsite_reqs + offsite_reqs - - out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEqual(out, onsite_reqs) - - -class TestOffsiteMiddleware2(TestOffsiteMiddleware): - def _get_spiderargs(self): - return dict(name="foo", allowed_domains=None) - - def test_process_spider_output(self): - res = Response("http://scrapytest.org") - reqs = [Request("http://a.com/b.html"), Request("http://b.com/1")] - out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEqual(out, reqs) - - -class TestOffsiteMiddleware3(TestOffsiteMiddleware2): - def _get_spiderargs(self): - return dict(name="foo") - - -class TestOffsiteMiddleware4(TestOffsiteMiddleware3): - def _get_spiderargs(self): - bad_hostname = urlparse("http:////scrapytest.org").hostname - return dict(name="foo", allowed_domains=["scrapytest.org", None, bad_hostname]) - - def test_process_spider_output(self): - res = Response("http://scrapytest.org") - reqs = [Request("http://scrapytest.org/1")] - out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEqual(out, reqs) - - -class TestOffsiteMiddleware5(TestOffsiteMiddleware4): - def test_get_host_regex(self): - self.spider.allowed_domains = [ - "http://scrapytest.org", - "scrapy.org", - "scrapy.test.org", - ] - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - self.mw.get_host_regex(self.spider) - assert issubclass(w[-1].category, URLWarning) - - -class TestOffsiteMiddleware6(TestOffsiteMiddleware4): - def test_get_host_regex(self): - self.spider.allowed_domains = [ - "scrapytest.org:8000", - "scrapy.org", - "scrapy.test.org", - ] - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - self.mw.get_host_regex(self.spider) - assert issubclass(w[-1].category, PortWarning) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index fad5dcaac..afdfaa322 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -1,25 +1,32 @@ from testfixtures import LogCapture -from twisted.internet import defer -from twisted.trial.unittest import TestCase from scrapy import Request, Spider from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer +from tests.mockserver.http import MockServer +from tests.utils.decorators import coroutine_test -class LogExceptionMiddleware: - def process_spider_exception(self, response, exception, spider): - spider.logger.info( +class _BaseSpiderMiddleware: + def __init__(self, crawler): + self.crawler = crawler + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + +class LogExceptionMiddleware(_BaseSpiderMiddleware): + def process_spider_exception(self, response, exception): + self.crawler.spider.logger.info( "Middleware: %s exception caught", exception.__class__.__name__ ) - return None # ================================================================================ # (0) recover from an exception on a spider callback -class RecoveryMiddleware: - def process_spider_exception(self, response, exception, spider): - spider.logger.info( +class RecoveryMiddleware(_BaseSpiderMiddleware): + def process_spider_exception(self, response, exception): + self.crawler.spider.logger.info( "Middleware: %s exception caught", exception.__class__.__name__ ) return [ @@ -37,14 +44,14 @@ class RecoverySpider(Spider): }, } - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): yield {"test": 1} self.logger.info("DONT_FAIL: %s", response.meta.get("dont_fail")) if not response.meta.get("dont_fail"): - raise TabError() + raise TabError class RecoveryAsyncGenSpider(RecoverySpider): @@ -57,10 +64,10 @@ class RecoveryAsyncGenSpider(RecoverySpider): # ================================================================================ # (1) exceptions from a spider middleware's process_spider_input method -class FailProcessSpiderInputMiddleware: - def process_spider_input(self, response, spider): - spider.logger.info("Middleware: will raise IndexError") - raise IndexError() +class FailProcessSpiderInputMiddleware(_BaseSpiderMiddleware): + def process_spider_input(self, response): + self.crawler.spider.logger.info("Middleware: will raise IndexError") + raise IndexError class ProcessSpiderInputSpiderWithoutErrback(Spider): @@ -74,7 +81,7 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): } } - def start_requests(self): + async def start(self): yield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse) def parse(self, response): @@ -84,7 +91,7 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback): name = "ProcessSpiderInputSpiderWithErrback" - def start_requests(self): + async def start(self): yield Request( self.mockserver.url("/status?n=200"), self.parse, errback=self.errback ) @@ -104,20 +111,20 @@ class GeneratorCallbackSpider(Spider): }, } - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): yield {"test": 1} yield {"test": 2} - raise ImportError() + raise ImportError class AsyncGeneratorCallbackSpider(GeneratorCallbackSpider): async def parse(self, response): yield {"test": 1} yield {"test": 2} - raise ImportError() + raise ImportError # ================================================================================ @@ -141,7 +148,7 @@ class NotGeneratorCallbackSpider(Spider): }, } - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -161,28 +168,31 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS # ================================================================================ # (4) exceptions from a middleware process_spider_output method (generator) -class _GeneratorDoNothingMiddleware: - def process_spider_output(self, response, result, spider): - for r in result: +class _GeneratorDoNothingMiddleware(_BaseSpiderMiddleware): + async def process_spider_output(self, response, result): + async for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") yield r - def process_spider_exception(self, response, exception, spider): + def process_spider_exception(self, response, exception): method = f"{self.__class__.__name__}.process_spider_exception" - spider.logger.info("%s: %s caught", method, exception.__class__.__name__) - return None + self.crawler.spider.logger.info( + "%s: %s caught", method, exception.__class__.__name__ + ) -class GeneratorFailMiddleware: - def process_spider_output(self, response, result, spider): - for r in result: +class GeneratorFailMiddleware(_BaseSpiderMiddleware): + async def process_spider_output(self, response, result): + async for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") yield r - raise LookupError() + raise LookupError - def process_spider_exception(self, response, exception, spider): + def process_spider_exception(self, response, exception): method = f"{self.__class__.__name__}.process_spider_exception" - spider.logger.info("%s: %s caught", method, exception.__class__.__name__) + self.crawler.spider.logger.info( + "%s: %s caught", method, exception.__class__.__name__ + ) yield {"processed": [method]} @@ -190,15 +200,17 @@ class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware): pass -class GeneratorRecoverMiddleware: - def process_spider_output(self, response, result, spider): - for r in result: +class GeneratorRecoverMiddleware(_BaseSpiderMiddleware): + async def process_spider_output(self, response, result): + async for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") yield r - def process_spider_exception(self, response, exception, spider): + def process_spider_exception(self, response, exception): method = f"{self.__class__.__name__}.process_spider_exception" - spider.logger.info("%s: %s caught", method, exception.__class__.__name__) + self.crawler.spider.logger.info( + "%s: %s caught", method, exception.__class__.__name__ + ) yield {"processed": [method]} @@ -217,7 +229,7 @@ class GeneratorOutputChainSpider(Spider): }, } - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -226,200 +238,125 @@ class GeneratorOutputChainSpider(Spider): # ================================================================================ -# (5) exceptions from a middleware process_spider_output method (not generator) +class TestSpiderMiddleware: + mockserver: MockServer - -class _NotGeneratorDoNothingMiddleware: - def process_spider_output(self, response, result, spider): - out = [] - for r in result: - r["processed"].append(f"{self.__class__.__name__}.process_spider_output") - out.append(r) - return out - - def process_spider_exception(self, response, exception, spider): - method = f"{self.__class__.__name__}.process_spider_exception" - spider.logger.info("%s: %s caught", method, exception.__class__.__name__) - return None - - -class NotGeneratorFailMiddleware: - def process_spider_output(self, response, result, spider): - out = [] - for r in result: - r["processed"].append(f"{self.__class__.__name__}.process_spider_output") - out.append(r) - raise ReferenceError() - return out - - def process_spider_exception(self, response, exception, spider): - method = f"{self.__class__.__name__}.process_spider_exception" - spider.logger.info("%s: %s caught", method, exception.__class__.__name__) - return [{"processed": [method]}] - - -class NotGeneratorDoNothingAfterFailureMiddleware(_NotGeneratorDoNothingMiddleware): - pass - - -class NotGeneratorRecoverMiddleware: - def process_spider_output(self, response, result, spider): - out = [] - for r in result: - r["processed"].append(f"{self.__class__.__name__}.process_spider_output") - out.append(r) - return out - - def process_spider_exception(self, response, exception, spider): - method = f"{self.__class__.__name__}.process_spider_exception" - spider.logger.info("%s: %s caught", method, exception.__class__.__name__) - return [{"processed": [method]}] - - -class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware): - pass - - -class NotGeneratorOutputChainSpider(Spider): - name = "NotGeneratorOutputChainSpider" - custom_settings = { - "SPIDER_MIDDLEWARES": { - NotGeneratorFailMiddleware: 10, - NotGeneratorDoNothingAfterFailureMiddleware: 8, - NotGeneratorRecoverMiddleware: 5, - NotGeneratorDoNothingAfterRecoveryMiddleware: 3, - }, - } - - def start_requests(self): - return [Request(self.mockserver.url("/status?n=200"))] - - def parse(self, response): - return [ - {"processed": ["parse-first-item"]}, - {"processed": ["parse-second-item"]}, - ] - - -# ================================================================================ -class TestSpiderMiddleware(TestCase): @classmethod - def setUpClass(cls): + def setup_class(cls): cls.mockserver = MockServer() cls.mockserver.__enter__() @classmethod - def tearDownClass(cls): + def teardown_class(cls): cls.mockserver.__exit__(None, None, None) - @defer.inlineCallbacks - def crawl_log(self, spider): + async def crawl_log(self, spider: type[Spider]) -> LogCapture: crawler = get_crawler(spider) with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=self.mockserver) return log - @defer.inlineCallbacks - def test_recovery(self): + @coroutine_test + async def test_recovery(self): """ (0) Recover from an exception in a spider callback. The final item count should be 3 (one yielded from the callback method before the exception is raised, one directly from the recovery middleware and one from the spider when processing the request that was enqueued from the recovery middleware) """ - log = yield self.crawl_log(RecoverySpider) - self.assertIn("Middleware: TabError exception caught", str(log)) - self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1) - self.assertIn("'item_scraped_count': 3", str(log)) + log = await self.crawl_log(RecoverySpider) + assert "Middleware: TabError exception caught" in str(log) + assert str(log).count("Middleware: TabError exception caught") == 1 + assert "'item_scraped_count': 3" in str(log) - @defer.inlineCallbacks - def test_recovery_asyncgen(self): + @coroutine_test + async def test_recovery_asyncgen(self): """ Same as test_recovery but with an async callback. """ - log = yield self.crawl_log(RecoveryAsyncGenSpider) - self.assertIn("Middleware: TabError exception caught", str(log)) - self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1) - self.assertIn("'item_scraped_count': 3", str(log)) + log = await self.crawl_log(RecoveryAsyncGenSpider) + assert "Middleware: TabError exception caught" in str(log) + assert str(log).count("Middleware: TabError exception caught") == 1 + assert "'item_scraped_count': 3" in str(log) - @defer.inlineCallbacks - def test_process_spider_input_without_errback(self): + @coroutine_test + async def test_process_spider_input_without_errback(self): """ (1.1) An exception from the process_spider_input chain should be caught by the process_spider_exception chain from the start if the Request has no errback """ - log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithoutErrback) - self.assertIn("Middleware: will raise IndexError", str(log1)) - self.assertIn("Middleware: IndexError exception caught", str(log1)) + log1 = await self.crawl_log(ProcessSpiderInputSpiderWithoutErrback) + assert "Middleware: will raise IndexError" in str(log1) + assert "Middleware: IndexError exception caught" in str(log1) - @defer.inlineCallbacks - def test_process_spider_input_with_errback(self): + @coroutine_test + async def test_process_spider_input_with_errback(self): """ (1.2) An exception from the process_spider_input chain should not be caught by the process_spider_exception chain if the Request has an errback """ - log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithErrback) - self.assertNotIn("Middleware: IndexError exception caught", str(log1)) - self.assertIn("Middleware: will raise IndexError", str(log1)) - self.assertIn("Got a Failure on the Request errback", str(log1)) - self.assertIn("{'from': 'errback'}", str(log1)) - self.assertNotIn("{'from': 'callback'}", str(log1)) - self.assertIn("'item_scraped_count': 1", str(log1)) + log1 = await self.crawl_log(ProcessSpiderInputSpiderWithErrback) + assert "Middleware: IndexError exception caught" not in str(log1) + assert "Middleware: will raise IndexError" in str(log1) + assert "Got a Failure on the Request errback" in str(log1) + assert "{'from': 'errback'}" in str(log1) + assert "{'from': 'callback'}" not in str(log1) + assert "'item_scraped_count': 1" in str(log1) - @defer.inlineCallbacks - def test_generator_callback(self): + @coroutine_test + async def test_generator_callback(self): """ (2) An exception from a spider callback (returning a generator) should be caught by the process_spider_exception chain. Items yielded before the exception is raised should be processed normally. """ - log2 = yield self.crawl_log(GeneratorCallbackSpider) - self.assertIn("Middleware: ImportError exception caught", str(log2)) - self.assertIn("'item_scraped_count': 2", str(log2)) + log2 = await self.crawl_log(GeneratorCallbackSpider) + assert "Middleware: ImportError exception caught" in str(log2) + assert "'item_scraped_count': 2" in str(log2) - @defer.inlineCallbacks - def test_async_generator_callback(self): + @coroutine_test + async def test_async_generator_callback(self): """ Same as test_generator_callback but with an async callback. """ - log2 = yield self.crawl_log(AsyncGeneratorCallbackSpider) - self.assertIn("Middleware: ImportError exception caught", str(log2)) - self.assertIn("'item_scraped_count': 2", str(log2)) + log2 = await self.crawl_log(AsyncGeneratorCallbackSpider) + assert "Middleware: ImportError exception caught" in str(log2) + assert "'item_scraped_count': 2" in str(log2) - @defer.inlineCallbacks - def test_generator_callback_right_after_callback(self): + @coroutine_test + async def test_generator_callback_right_after_callback(self): """ (2.1) Special case of (2): Exceptions should be caught even if the middleware is placed right after the spider """ - log21 = yield self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider) - self.assertIn("Middleware: ImportError exception caught", str(log21)) - self.assertIn("'item_scraped_count': 2", str(log21)) + log21 = await self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider) + assert "Middleware: ImportError exception caught" in str(log21) + assert "'item_scraped_count': 2" in str(log21) - @defer.inlineCallbacks - def test_not_a_generator_callback(self): + @coroutine_test + async def test_not_a_generator_callback(self): """ (3) An exception from a spider callback (returning a list) should be caught by the process_spider_exception chain. No items should be processed. """ - log3 = yield self.crawl_log(NotGeneratorCallbackSpider) - self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3)) - self.assertNotIn("item_scraped_count", str(log3)) + log3 = await self.crawl_log(NotGeneratorCallbackSpider) + assert "Middleware: ZeroDivisionError exception caught" in str(log3) + assert "item_scraped_count" not in str(log3) - @defer.inlineCallbacks - def test_not_a_generator_callback_right_after_callback(self): + @coroutine_test + async def test_not_a_generator_callback_right_after_callback(self): """ (3.1) Special case of (3): Exceptions should be caught even if the middleware is placed right after the spider """ - log31 = yield self.crawl_log( + log31 = await self.crawl_log( NotGeneratorCallbackSpiderMiddlewareRightAfterSpider ) - self.assertIn("Middleware: ZeroDivisionError exception caught", str(log31)) - self.assertNotIn("item_scraped_count", str(log31)) + assert "Middleware: ZeroDivisionError exception caught" in str(log31) + assert "item_scraped_count" not in str(log31) - @defer.inlineCallbacks - def test_generator_output_chain(self): + @coroutine_test + async def test_generator_output_chain(self): """ (4) An exception from a middleware's process_spider_output method should be sent to the process_spider_exception method from the next middleware in the chain. @@ -428,23 +365,23 @@ class TestSpiderMiddleware(TestCase): The final item count should be 2 (one from the spider callback and one from the process_spider_exception chain) """ - log4 = yield self.crawl_log(GeneratorOutputChainSpider) - self.assertIn("'item_scraped_count': 2", str(log4)) - self.assertIn( - "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", - str(log4), + log4 = await self.crawl_log(GeneratorOutputChainSpider) + assert "'item_scraped_count': 2" in str(log4) + assert ( + "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught" + in str(log4) ) - self.assertIn( - "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert ( + "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught" + in str(log4) ) - self.assertNotIn( - "GeneratorFailMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert ( + "GeneratorFailMiddleware.process_spider_exception: LookupError caught" + not in str(log4) ) - self.assertNotIn( - "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert ( + "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught" + not in str(log4) ) item_from_callback = { "processed": [ @@ -461,44 +398,6 @@ class TestSpiderMiddleware(TestCase): "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output", ] } - self.assertIn(str(item_from_callback), str(log4)) - self.assertIn(str(item_recovered), str(log4)) - self.assertNotIn("parse-second-item", str(log4)) - - @defer.inlineCallbacks - def test_not_a_generator_output_chain(self): - """ - (5) An exception from a middleware's process_spider_output method should be sent - to the process_spider_exception method from the next middleware in the chain. - The result of the recovery by the process_spider_exception method should be handled - by the process_spider_output method from the next middleware. - The final item count should be 1 (from the process_spider_exception chain, the items - from the spider callback are lost) - """ - log5 = yield self.crawl_log(NotGeneratorOutputChainSpider) - self.assertIn("'item_scraped_count': 1", str(log5)) - self.assertIn( - "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", - str(log5), - ) - self.assertIn( - "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", - str(log5), - ) - self.assertNotIn( - "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", - str(log5), - ) - self.assertNotIn( - "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", - str(log5), - ) - item_recovered = { - "processed": [ - "NotGeneratorRecoverMiddleware.process_spider_exception", - "NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output", - ] - } - self.assertIn(str(item_recovered), str(log5)) - self.assertNotIn("parse-first-item", str(log5)) - self.assertNotIn("parse-second-item", str(log5)) + assert str(item_from_callback) in str(log4) + assert str(item_recovered) in str(log4) + assert "parse-second-item" not in str(log4) diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py new file mode 100644 index 000000000..c907c6d73 --- /dev/null +++ b/tests/test_spidermiddleware_process_start.py @@ -0,0 +1,123 @@ +import warnings +from asyncio import sleep + +import pytest + +from scrapy import Spider, signals +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.test import get_crawler +from tests.test_spider_start import SLEEP_SECONDS + +from .utils import twisted_sleep +from .utils.decorators import coroutine_test + +ITEM_A = {"id": "a"} +ITEM_B = {"id": "b"} +ITEM_C = {"id": "c"} +ITEM_D = {"id": "d"} + + +class AsyncioSleepSpiderMiddleware: + async def process_start(self, start): + await sleep(SLEEP_SECONDS) + async for item_or_request in start: + yield item_or_request + + +class NoOpSpiderMiddleware: + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request + + +class TwistedSleepSpiderMiddleware: + async def process_start(self, start): + await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS)) + async for item_or_request in start: + yield item_or_request + + +# Spiders and spider middlewares for TestMain._test_wrap + + +class ModernWrapSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_B + + +class ModernWrapSpiderSubclass(ModernWrapSpider): + name = "test" + + +class ModernWrapSpiderMiddleware: + async def process_start(self, start): + yield ITEM_A + async for item_or_request in start: + yield item_or_request + yield ITEM_C + + +class TestMain: + async def _test(self, spider_middlewares, spider_cls, expected_items): + actual_items = [] + + def track_item(item, response, spider): + actual_items.append(item) + + settings = { + "SPIDER_MIDDLEWARES": {cls: n for n, cls in enumerate(spider_middlewares)}, + } + crawler = get_crawler(spider_cls, settings_dict=settings) + crawler.signals.connect(track_item, signals.item_scraped) + await crawler.crawl_async() + assert crawler.stats.get_value("finish_reason") == "finished" + assert actual_items == expected_items, f"{actual_items=} != {expected_items=}" + + async def _test_wrap(self, spider_middleware, spider_cls, expected_items=None): + expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C] + await self._test([spider_middleware], spider_cls, expected_items) + + async def _test_douple_wrap(self, smw1, smw2, spider_cls, expected_items=None): + expected_items = expected_items or [ITEM_A, ITEM_A, ITEM_B, ITEM_C, ITEM_C] + await self._test([smw1, smw2], spider_cls, expected_items) + + @coroutine_test + async def test_modern_mw_modern_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) + + async def _test_sleep(self, spider_middlewares): + class TestSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_A + + await self._test(spider_middlewares, TestSpider, [ITEM_A]) + + @pytest.mark.only_asyncio + @coroutine_test + async def test_asyncio_sleep_single(self): + await self._test_sleep([AsyncioSleepSpiderMiddleware]) + + @pytest.mark.only_asyncio + @coroutine_test + async def test_asyncio_sleep_multiple(self): + await self._test_sleep( + [NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware] + ) + + @pytest.mark.requires_reactor # needs a reactor for twisted_sleep() + @coroutine_test + async def test_twisted_sleep_single(self): + await self._test_sleep([TwistedSleepSpiderMiddleware]) + + @pytest.mark.requires_reactor # needs a reactor for twisted_sleep() + @coroutine_test + async def test_twisted_sleep_multiple(self): + await self._test_sleep( + [NoOpSpiderMiddleware, TwistedSleepSpiderMiddleware, NoOpSpiderMiddleware] + ) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index afffa87fb..7431ea6ac 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,9 +1,11 @@ +from __future__ import annotations + import warnings -from typing import Any, Dict, List, Optional, Tuple -from unittest import TestCase +from typing import Any from urllib.parse import urlparse -from scrapy.downloadermiddlewares.redirect import RedirectMiddleware +import pytest + from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spidermiddlewares.referer import ( @@ -28,34 +30,36 @@ from scrapy.spidermiddlewares.referer import ( StrictOriginWhenCrossOriginPolicy, UnsafeUrlPolicy, ) -from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test -class TestRefererMiddleware(TestCase): - req_meta: Dict[str, Any] = {} - resp_headers: Dict[str, str] = {} - settings: Dict[str, Any] = {} - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ +class TestRefererMiddleware: + req_meta: dict[str, Any] = {} + resp_headers: dict[str, str] = {} + settings: dict[str, Any] = {} + scenarii: list[tuple[str, str, bytes | None]] = [ ("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"), ] - def setUp(self): - self.spider = Spider("foo") + @pytest.fixture + def mw(self) -> RefererMiddleware: settings = Settings(self.settings) - self.mw = RefererMiddleware(settings) + return RefererMiddleware(settings) - def get_request(self, target): + def get_request(self, target: str) -> Request: return Request(target, meta=self.req_meta) - def get_response(self, origin): + def get_response(self, origin: str) -> Response: return Response(origin, headers=self.resp_headers) - def test(self): + def test(self, mw: RefererMiddleware) -> None: for origin, target, referrer in self.scenarii: response = self.get_response(origin) request = self.get_request(target) - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEqual(out[0].headers.get("Referer"), referrer) + out = list(mw.process_spider_output(response, [request])) + assert out[0].headers.get("Referer") == referrer class MixinDefault: @@ -65,7 +69,7 @@ class MixinDefault: with some additional filtering of s3:// """ - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ ("https://example.com/", "https://scrapy.org/", b"https://example.com/"), ("http://example.com/", "http://scrapy.org/", b"http://example.com/"), ("http://example.com/", "https://scrapy.org/", b"http://example.com/"), @@ -86,7 +90,7 @@ class MixinDefault: class MixinNoReferrer: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ ("https://example.com/page.html", "https://example.com/", None), ("http://www.example.com/", "https://scrapy.org/", None), ("http://www.example.com/", "http://scrapy.org/", None), @@ -96,7 +100,7 @@ class MixinNoReferrer: class MixinNoReferrerWhenDowngrade: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # TLS to TLS: send non-empty referrer ( "https://example.com/page.html", @@ -178,7 +182,7 @@ class MixinNoReferrerWhenDowngrade: class MixinSameOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -247,7 +251,7 @@ class MixinSameOrigin: class MixinOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) ( "https://example.com/page.html", @@ -271,7 +275,7 @@ class MixinOrigin: class MixinStrictOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades ( "https://example.com/page.html", @@ -299,7 +303,7 @@ class MixinStrictOrigin: class MixinOriginWhenCrossOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -406,7 +410,7 @@ class MixinOriginWhenCrossOrigin: class MixinStrictOriginWhenCrossOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -518,7 +522,7 @@ class MixinStrictOriginWhenCrossOrigin: class MixinUnsafeUrl: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # TLS to TLS: send referrer ( "https://example.com/sekrit.html", @@ -684,6 +688,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): return b"https://python.org/" if scheme == "http": return b"http://python.org/" + return None class TestSettingsCustomPolicy(TestRefererMiddleware): @@ -768,7 +773,7 @@ class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {"referrer_policy": POLICY_UNSAFE_URL} -class TestRequestMetaSettingFallback(TestCase): +class TestRequestMetaSettingFallback: params = [ ( # When an unknown policy is referenced in Request.meta @@ -839,14 +844,14 @@ class TestRequestMetaSettingFallback(TestCase): with warnings.catch_warnings(record=True) as w: policy = mw.policy(response, request) - self.assertIsInstance(policy, policy_class) + assert isinstance(policy, policy_class) if check_warning: - self.assertEqual(len(w), 1) - self.assertEqual(w[0].category, RuntimeWarning, w[0].message) + assert len(w) == 1 + assert w[0].category is RuntimeWarning, w[0].message -class TestSettingsPolicyByName(TestCase): +class TestSettingsPolicyByName: def test_valid_name(self): for s, p in [ (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), @@ -861,7 +866,7 @@ class TestSettingsPolicyByName(TestCase): ]: settings = Settings({"REFERRER_POLICY": s}) mw = RefererMiddleware(settings) - self.assertEqual(mw.default_policy, p) + assert mw.default_policy == p def test_valid_name_casevariants(self): for s, p in [ @@ -877,11 +882,52 @@ class TestSettingsPolicyByName(TestCase): ]: settings = Settings({"REFERRER_POLICY": s.upper()}) mw = RefererMiddleware(settings) - self.assertEqual(mw.default_policy, p) + assert mw.default_policy == p def test_invalid_name(self): settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"}) - with self.assertRaises(RuntimeError): + with pytest.raises(RuntimeError): + RefererMiddleware(settings) + + def test_multiple_policy_tokens(self): + # test parsing without space(s) after the comma + settings1 = Settings( + { + "REFERRER_POLICY": ( + f"some-custom-unknown-policy," + f"{POLICY_SAME_ORIGIN}," + f"{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}," + f"another-custom-unknown-policy" + ) + } + ) + mw1 = RefererMiddleware(settings1) + assert mw1.default_policy == StrictOriginWhenCrossOriginPolicy + + # test parsing with space(s) after the comma + settings2 = Settings( + { + "REFERRER_POLICY": ( + f"{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}," + f" another-custom-unknown-policy," + f" {POLICY_UNSAFE_URL}" + ) + } + ) + mw2 = RefererMiddleware(settings2) + assert mw2.default_policy == UnsafeUrlPolicy + + def test_multiple_policy_tokens_all_invalid(self): + settings = Settings( + { + "REFERRER_POLICY": ( + "some-custom-unknown-policy," + "another-custom-unknown-policy," + "yet-another-custom-unknown-policy" + ) + } + ) + with pytest.raises(RuntimeError): RefererMiddleware(settings) @@ -919,379 +965,161 @@ class TestPolicyHeaderPrecedence004( resp_headers = {"Referrer-Policy": ""} -class TestReferrerOnRedirect(TestRefererMiddleware): - settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"} - scenarii: List[ - Tuple[str, str, Tuple[Tuple[int, str], ...], Optional[bytes], Optional[bytes]] - ] = [ # type: ignore[assignment] - ( - "http://scrapytest.org/1", # parent - "http://scrapytest.org/2", # target - ( - # redirections: code, URL - (301, "http://scrapytest.org/3"), - (301, "http://scrapytest.org/4"), - ), - b"http://scrapytest.org/1", # expected initial referer - b"http://scrapytest.org/1", # expected referer for the redirection request - ), - ( - "https://scrapytest.org/1", - "https://scrapytest.org/2", - ( - # redirecting to non-secure URL - (301, "http://scrapytest.org/3"), - ), - b"https://scrapytest.org/1", - b"https://scrapytest.org/1", - ), - ( - "https://scrapytest.org/1", - "https://scrapytest.com/2", - ( - # redirecting to non-secure URL: different origin - (301, "http://scrapytest.com/3"), - ), - b"https://scrapytest.org/1", - b"https://scrapytest.org/1", - ), - ] +class TestPolicyMethodResponseParamRename: + def setup_method(self): + self.crawler = get_crawler() + self.mw = build_from_crawler(RefererMiddleware, self.crawler) + self.request = Request("http://www.example.com") + self.response = Response("http://www.example.com") - def setUp(self): - self.spider = Spider("foo") - settings = Settings(self.settings) - self.referrermw = RefererMiddleware(settings) - self.redirectmw = RedirectMiddleware(settings) + def test_pos_string(self): + with warnings.catch_warnings(record=True) as w: + self.mw.policy("http://old.com", self.request) + found = False + for warning in w: + if "Passing a response URL" in str(warning.message): + found = True + break + assert found - def test(self): - for ( - parent, - target, - redirections, - init_referrer, - final_referrer, - ) in self.scenarii: - response = self.get_response(parent) - request = self.get_request(target) + def test_pos_response(self): + with warnings.catch_warnings(record=True) as w: + self.mw.policy(self.response, self.request) + for warning in w: + assert "resp_or_url" not in str(warning.message) - out = list( - self.referrermw.process_spider_output(response, [request], self.spider) + def test_key_resp_or_url(self): + with warnings.catch_warnings(record=True) as w: + self.mw.policy(resp_or_url=self.response, request=self.request) + found = False + for warning in w: + if "Passing 'resp_or_url' is deprecated, use 'response' instead" in str( + warning.message + ): + found = True + break + assert found + + def test_key_response(self): + with warnings.catch_warnings(record=True) as w: + self.mw.policy(response=self.response, request=self.request) + for warning in w: + assert "resp_or_url" not in str(warning.message) + + def test_key_response_string(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + self.mw.policy(response="http://old.com", request=self.request) + found = False + for warning in w: + if "Passing a response URL" in str(warning.message): + found = True + break + assert found + + def test_both_resp_or_url_and_response(self): + with pytest.raises( + TypeError, match="Cannot pass both 'response' and 'resp_or_url'" + ): + self.mw.policy( + response=self.response, resp_or_url=self.response, request=self.request ) - self.assertEqual(out[0].headers.get("Referer"), init_referrer) - - for status, url in redirections: - response = Response( - request.url, headers={"Location": url}, status=status - ) - request = self.redirectmw.process_response( - request, response, self.spider - ) - self.referrermw.request_scheduled(request, self.spider) - - assert isinstance(request, Request) - self.assertEqual(request.headers.get("Referer"), final_referrer) -class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): - """ - No Referrer policy never sets the "Referer" header. - HTTP redirections should not change that. - """ +@coroutine_test +async def test_response_policy_only_supports_policy_names(): + crawler = get_crawler(settings_dict={"REFERRER_POLICY": "no-referrer"}) + mw = build_from_crawler(RefererMiddleware, crawler) - settings = {"REFERRER_POLICY": "no-referrer"} - scenarii = [ - ( - "http://scrapytest.org/1", # parent - "http://scrapytest.org/2", # target - ( - # redirections: code, URL - (301, "http://scrapytest.org/3"), - (301, "http://scrapytest.org/4"), - ), - None, # expected initial "Referer" - None, # expected "Referer" for the redirection request - ), - ( - "https://scrapytest.org/1", - "https://scrapytest.org/2", - ((301, "http://scrapytest.org/3"),), - None, - None, - ), - ( - "https://scrapytest.org/1", - "https://example.com/2", # different origin - ((301, "http://scrapytest.com/3"),), - None, - None, - ), + async def input_result(): + yield Request("https://example.com/") + + response = Response( + "https://example.com/", + headers={ + "Referrer-Policy": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy" + }, + ) + with pytest.warns( + RuntimeWarning, + match=r"Could not load referrer policy 'scrapy\.spidermiddlewares\.referer\.NoReferrerWhenDowngradePolicy' \(import paths from the response Referrer-Policy header are not allowed\)", + ): + output = [ + request + async for request in mw.process_spider_output_async( + response, input_result() + ) + ] + assert len(output) == 1 + assert b"Referer" not in output[0].headers + + response = Response( + "https://example.com/", + headers={"Referrer-Policy": "no-referrer-when-downgrade"}, + ) + output = [ + request + async for request in mw.process_spider_output_async(response, input_result()) ] + assert len(output) == 1 + assert output[0].headers == {b"Referer": [b"https://example.com/"]} -class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect): - """ - Same Origin policy sends the full URL as "Referer" if the target origin - is the same as the parent response (same protocol, same domain, same port). +@coroutine_test +async def test_referer_policies_setting(): + crawler = get_crawler( + settings_dict={ + "REFERRER_POLICY": "no-referrer", + "REFERRER_POLICIES": { + "no-referrer-when-downgrade": None, + "custom-policy": CustomPythonOrgPolicy, + "": CustomPythonOrgPolicy, + }, + } + ) + mw = build_from_crawler(RefererMiddleware, crawler) - HTTP redirections to a different domain or a lower secure level - should have the "Referer" removed. - """ + async def input_result(): + yield Request("https://example.com/") - settings = {"REFERRER_POLICY": "same-origin"} - scenarii = [ - ( - "http://scrapytest.org/101", # origin - "http://scrapytest.org/102", # target - ( - # redirections: code, URL - (301, "http://scrapytest.org/103"), - (301, "http://scrapytest.org/104"), - ), - b"http://scrapytest.org/101", # expected initial "Referer" - b"http://scrapytest.org/101", # expected referer for the redirection request - ), - ( - "https://scrapytest.org/201", - "https://scrapytest.org/202", - ( - # redirecting from secure to non-secure URL == different origin - (301, "http://scrapytest.org/203"), - ), - b"https://scrapytest.org/201", - None, - ), - ( - "https://scrapytest.org/301", - "https://scrapytest.org/302", - ( - # different domain == different origin - (301, "http://example.com/303"), - ), - b"https://scrapytest.org/301", - None, - ), + # "no-referrer-when-downgrade": None, + response = Response( + "https://example.com/", + headers={"Referrer-Policy": "no-referrer-when-downgrade"}, + ) + with pytest.warns( + RuntimeWarning, + match=r"Could not load referrer policy 'no-referrer-when-downgrade'", + ): + output = [ + request + async for request in mw.process_spider_output_async( + response, input_result() + ) + ] + assert len(output) == 1 + assert b"Referer" not in output[0].headers + + # "custom-policy": CustomPythonOrgPolicy, + response = Response( + "https://example.com/", + headers={"Referrer-Policy": "custom-policy"}, + ) + output = [ + request + async for request in mw.process_spider_output_async(response, input_result()) ] + assert len(output) == 1 + assert output[0].headers == {b"Referer": [b"https://python.org/"]} - -class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect): - """ - Strict Origin policy will always send the "origin" as referrer - (think of it as the parent URL without the path part), - unless the security level is lower and no "Referer" is sent. - - Redirections from secure to non-secure URLs should have the - "Referrer" header removed if necessary. - """ - - settings = {"REFERRER_POLICY": POLICY_STRICT_ORIGIN} - scenarii = [ - ( - "http://scrapytest.org/101", - "http://scrapytest.org/102", - ( - (301, "http://scrapytest.org/103"), - (301, "http://scrapytest.org/104"), - ), - b"http://scrapytest.org/", # send origin - b"http://scrapytest.org/", # redirects to same origin: send origin - ), - ( - "https://scrapytest.org/201", - "https://scrapytest.org/202", - ( - # redirecting to non-secure URL: no referrer - (301, "http://scrapytest.org/203"), - ), - b"https://scrapytest.org/", - None, - ), - ( - "https://scrapytest.org/301", - "https://scrapytest.org/302", - ( - # redirecting to non-secure URL (different domain): no referrer - (301, "http://example.com/303"), - ), - b"https://scrapytest.org/", - None, - ), - ( - "http://scrapy.org/401", - "http://example.com/402", - ((301, "http://scrapytest.org/403"),), - b"http://scrapy.org/", - b"http://scrapy.org/", - ), - ( - "https://scrapy.org/501", - "https://example.com/502", - ( - # HTTPS all along, so origin referrer is kept as-is - (301, "https://google.com/503"), - (301, "https://facebook.com/504"), - ), - b"https://scrapy.org/", - b"https://scrapy.org/", - ), - ( - "https://scrapytest.org/601", - "http://scrapytest.org/602", # TLS to non-TLS: no referrer - ( - ( - 301, - "https://scrapytest.org/603", - ), # TLS URL again: (still) no referrer - ), - None, - None, - ), - ] - - -class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect): - """ - Origin When Cross-Origin policy sends the full URL as "Referer", - unless the target's origin is different (different domain, different protocol) - in which case only the origin is sent. - - Redirections to a different origin should strip the "Referer" - to the parent origin. - """ - - settings = {"REFERRER_POLICY": POLICY_ORIGIN_WHEN_CROSS_ORIGIN} - scenarii = [ - ( - "http://scrapytest.org/101", # origin - "http://scrapytest.org/102", # target + redirection - ( - # redirections: code, URL - (301, "http://scrapytest.org/103"), - (301, "http://scrapytest.org/104"), - ), - b"http://scrapytest.org/101", # expected initial referer - b"http://scrapytest.org/101", # expected referer for the redirection request - ), - ( - "https://scrapytest.org/201", - "https://scrapytest.org/202", - ( - # redirecting to non-secure URL: send origin - (301, "http://scrapytest.org/203"), - ), - b"https://scrapytest.org/201", - b"https://scrapytest.org/", - ), - ( - "https://scrapytest.org/301", - "https://scrapytest.org/302", - ( - # redirecting to non-secure URL (different domain): send origin - (301, "http://example.com/303"), - ), - b"https://scrapytest.org/301", - b"https://scrapytest.org/", - ), - ( - "http://scrapy.org/401", - "http://example.com/402", - ((301, "http://scrapytest.org/403"),), - b"http://scrapy.org/", - b"http://scrapy.org/", - ), - ( - "https://scrapy.org/501", - "https://example.com/502", - ( - # all different domains: send origin - (301, "https://google.com/503"), - (301, "https://facebook.com/504"), - ), - b"https://scrapy.org/", - b"https://scrapy.org/", - ), - ( - "https://scrapytest.org/301", - "http://scrapytest.org/302", # TLS to non-TLS: send origin - ((301, "https://scrapytest.org/303"),), # TLS URL again: send origin (also) - b"https://scrapytest.org/", - b"https://scrapytest.org/", - ), - ] - - -class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect): - """ - Strict Origin When Cross-Origin policy sends the full URL as "Referer", - unless the target's origin is different (different domain, different protocol) - in which case only the origin is sent... - Unless there's also a downgrade in security and then the "Referer" header - is not sent. - - Redirections to a different origin should strip the "Referer" to the parent origin, - and from https:// to http:// will remove the "Referer" header. - """ - - settings = {"REFERRER_POLICY": POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} - scenarii = [ - ( - "http://scrapytest.org/101", # origin - "http://scrapytest.org/102", # target + redirection - ( - # redirections: code, URL - (301, "http://scrapytest.org/103"), - (301, "http://scrapytest.org/104"), - ), - b"http://scrapytest.org/101", # expected initial referer - b"http://scrapytest.org/101", # expected referer for the redirection request - ), - ( - "https://scrapytest.org/201", - "https://scrapytest.org/202", - ( - # redirecting to non-secure URL: do not send the "Referer" header - (301, "http://scrapytest.org/203"), - ), - b"https://scrapytest.org/201", - None, - ), - ( - "https://scrapytest.org/301", - "https://scrapytest.org/302", - ( - # redirecting to non-secure URL (different domain): send origin - (301, "http://example.com/303"), - ), - b"https://scrapytest.org/301", - None, - ), - ( - "http://scrapy.org/401", - "http://example.com/402", - ((301, "http://scrapytest.org/403"),), - b"http://scrapy.org/", - b"http://scrapy.org/", - ), - ( - "https://scrapy.org/501", - "https://example.com/502", - ( - # all different domains: send origin - (301, "https://google.com/503"), - (301, "https://facebook.com/504"), - ), - b"https://scrapy.org/", - b"https://scrapy.org/", - ), - ( - "https://scrapytest.org/601", - "http://scrapytest.org/602", # TLS to non-TLS: do not send "Referer" - ( - ( - 301, - "https://scrapytest.org/603", - ), # TLS URL again: (still) send nothing - ), - None, - None, - ), + # "": CustomPythonOrgPolicy, + response = Response( + "https://example.com/", + headers={"Referrer-Policy": ""}, + ) + output = [ + request + async for request in mw.process_spider_output_async(response, input_result()) ] + assert len(output) == 1 + assert output[0].headers == {b"Referer": [b"https://python.org/"]} diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py new file mode 100644 index 000000000..26397d37c --- /dev/null +++ b/tests/test_spidermiddleware_start.py @@ -0,0 +1,25 @@ +from scrapy.http import Request +from scrapy.spidermiddlewares.start import StartSpiderMiddleware +from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test + + +class TestMiddleware: + @coroutine_test + async def test_async(self): + crawler = get_crawler(Spider) + mw = build_from_crawler(StartSpiderMiddleware, crawler) + + async def start(): + yield Request("data:,1") + yield Request("data:,2", meta={"is_start_request": True}) + yield Request("data:,2", meta={"is_start_request": False}) + yield Request("data:,2", meta={"is_start_request": "foo"}) + + result = [ + request.meta["is_start_request"] + async for request in mw.process_start(start()) + ] + assert result == [True, True, False, "foo"] diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 9111e4c82..750ae3b07 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -1,44 +1,56 @@ -from unittest import TestCase +from __future__ import annotations -from testfixtures import LogCapture +from logging import INFO +from typing import TYPE_CHECKING + +import pytest from scrapy.http import Request, Response -from scrapy.settings import Settings from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +if TYPE_CHECKING: + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector -class TestUrlLengthMiddleware(TestCase): - def setUp(self): - self.maxlength = 25 - settings = Settings({"URLLENGTH_LIMIT": self.maxlength}) - crawler = get_crawler(Spider) - self.spider = crawler._create_spider("foo") - self.stats = crawler.stats - self.mw = UrlLengthMiddleware.from_settings(settings) +maxlength = 25 +response = Response("http://scrapytest.org") +short_url_req = Request("http://scrapytest.org/") +long_url_req = Request("http://scrapytest.org/this_is_a_long_url") +reqs: list[Request] = [short_url_req, long_url_req] - self.response = Response("http://scrapytest.org") - self.short_url_req = Request("http://scrapytest.org/") - self.long_url_req = Request("http://scrapytest.org/this_is_a_long_url") - self.reqs = [self.short_url_req, self.long_url_req] - def process_spider_output(self): - return list( - self.mw.process_spider_output(self.response, self.reqs, self.spider) - ) +@pytest.fixture +def crawler() -> Crawler: + return get_crawler(Spider, {"URLLENGTH_LIMIT": maxlength}) - def test_middleware_works(self): - self.assertEqual(self.process_spider_output(), [self.short_url_req]) - def test_logging(self): - with LogCapture() as log: - self.process_spider_output() +@pytest.fixture +def stats(crawler: Crawler) -> StatsCollector: + assert crawler.stats is not None + return crawler.stats - ric = self.stats.get_value( - "urllength/request_ignored_count", spider=self.spider - ) - self.assertEqual(ric, 1) - self.assertIn(f"Ignoring link (url length > {self.maxlength})", str(log)) +@pytest.fixture +def mw(crawler: Crawler) -> UrlLengthMiddleware: + return UrlLengthMiddleware.from_crawler(crawler) + + +def process_spider_output(mw: UrlLengthMiddleware) -> list[Request]: + return list(mw.process_spider_output(response, reqs)) + + +def test_middleware_works(mw: UrlLengthMiddleware) -> None: + assert process_spider_output(mw) == [short_url_req] + + +def test_logging( + stats: StatsCollector, mw: UrlLengthMiddleware, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(INFO): + process_spider_output(mw) + ric = stats.get_value("urllength/request_ignored_count") + assert ric == 1 + assert f"Ignoring link (url length > {maxlength})" in caplog.text diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index f97125b76..491fc88f7 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -1,8 +1,6 @@ -import shutil from datetime import datetime, timezone -from pathlib import Path -from twisted.trial import unittest +import pytest from scrapy.exceptions import NotConfigured from scrapy.extensions.spiderstate import SpiderState @@ -10,37 +8,36 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class SpiderStateTest(unittest.TestCase): - def test_store_load(self): - jobdir = self.mktemp() - Path(jobdir).mkdir() - try: - spider = Spider(name="default") - dt = datetime.now(tz=timezone.utc) +def test_store_load(tmp_path): + jobdir = str(tmp_path) - ss = SpiderState(jobdir) - ss.spider_opened(spider) - spider.state["one"] = 1 - spider.state["dt"] = dt - ss.spider_closed(spider) + spider = Spider(name="default") + dt = datetime.now(tz=timezone.utc) - spider2 = Spider(name="default") - ss2 = SpiderState(jobdir) - ss2.spider_opened(spider2) - self.assertEqual(spider.state, {"one": 1, "dt": dt}) - ss2.spider_closed(spider2) - finally: - shutil.rmtree(jobdir) + ss = SpiderState(jobdir) + ss.spider_opened(spider) + spider.state["one"] = 1 + spider.state["dt"] = dt + ss.spider_closed(spider) - def test_state_attribute(self): - # state attribute must be present if jobdir is not set, to provide a - # consistent interface - spider = Spider(name="default") - ss = SpiderState() - ss.spider_opened(spider) - self.assertEqual(spider.state, {}) - ss.spider_closed(spider) + spider2 = Spider(name="default") + ss2 = SpiderState(jobdir) + ss2.spider_opened(spider2) + assert spider.state == {"one": 1, "dt": dt} + ss2.spider_closed(spider2) - def test_not_configured(self): - crawler = get_crawler(Spider) - self.assertRaises(NotConfigured, SpiderState.from_crawler, crawler) + +def test_state_attribute(): + # state attribute must be present if jobdir is not set, to provide a + # consistent interface + spider = Spider(name="default") + ss = SpiderState() + ss.spider_opened(spider) + assert spider.state == {} + ss.spider_closed(spider) + + +def test_not_configured(): + crawler = get_crawler(Spider) + with pytest.raises(NotConfigured): + SpiderState.from_crawler(crawler) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 4ce7cc9a4..ddc12766c 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,6 +1,7 @@ import pickle import sys +import pytest from queuelib.tests import test_queue as t from scrapy.http import Request @@ -15,7 +16,7 @@ from scrapy.squeues import ( ) -class TestItem(Item): +class MyItem(Item): name = Field() @@ -23,17 +24,24 @@ def _test_procesor(x): return x + x -class TestLoader(ItemLoader): - default_item_class = TestItem +class MyLoader(ItemLoader): + default_item_class = MyItem name_out = staticmethod(_test_procesor) def nonserializable_object_test(self): q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + with pytest.raises( + ValueError, + match=r"unmarshallable object|Can't (get|pickle) local object|Can't pickle .*: it's not found as", + ): + q.push(lambda x: x) # Selectors should fail (lxml.html.HtmlElement objects can't be pickled) sel = Selector(text="

some text

") - self.assertRaises(ValueError, q.push, sel) + with pytest.raises( + ValueError, match=r"unmarshallable object|can't pickle Selector objects" + ): + q.push(sel) class FifoDiskQueueTestMixin: @@ -42,9 +50,9 @@ class FifoDiskQueueTestMixin: q.push("a") q.push(123) q.push({"a": "dict"}) - self.assertEqual(q.pop(), "a") - self.assertEqual(q.pop(), 123) - self.assertEqual(q.pop(), {"a": "dict"}) + assert q.pop() == "a" + assert q.pop() == 123 + assert q.pop() == {"a": "dict"} test_nonserializable_object = nonserializable_object_test @@ -80,20 +88,20 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): def test_serialize_item(self): q = self.queue() - i = TestItem(name="foo") + i = MyItem(name="foo") q.push(i) i2 = q.pop() - assert isinstance(i2, TestItem) - self.assertEqual(i, i2) + assert isinstance(i2, MyItem) + assert i == i2 def test_serialize_loader(self): q = self.queue() - loader = TestLoader() + loader = MyLoader() q.push(loader) loader2 = q.pop() - assert isinstance(loader2, TestLoader) - assert loader2.default_item_class is TestItem - self.assertEqual(loader2.name_out("x"), "xx") + assert isinstance(loader2, MyLoader) + assert loader2.default_item_class is MyItem + assert loader2.name_out("x") == "xx" def test_serialize_request_recursive(self): q = self.queue() @@ -102,23 +110,27 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): q.push(r) r2 = q.pop() assert isinstance(r2, Request) - self.assertEqual(r.url, r2.url) + assert r.url == r2.url assert r2.meta["request"] is r2 def test_non_pickable_object(self): q = self.queue() - try: + with pytest.raises( + ValueError, + match=r"Can't (get|pickle) local object|Can't pickle .*: it's not found as", + ) as exc_info: q.push(lambda x: x) - except ValueError as exc: - if hasattr(sys, "pypy_version_info"): - self.assertIsInstance(exc.__context__, pickle.PicklingError) - else: - self.assertIsInstance(exc.__context__, AttributeError) + if sys.version_info >= (3, 14) or hasattr(sys, "pypy_version_info"): + assert isinstance(exc_info.value.__context__, pickle.PicklingError) + else: + assert isinstance(exc_info.value.__context__, AttributeError) sel = Selector(text="

some text

") - try: + with pytest.raises( + ValueError, match="can't pickle Selector objects" + ) as exc_info: q.push(sel) - except ValueError as exc: - self.assertIsInstance(exc.__context__, TypeError) + assert isinstance(exc_info.value.__context__, TypeError) + q.close() class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): @@ -143,9 +155,9 @@ class LifoDiskQueueTestMixin: q.push("a") q.push(123) q.push({"a": "dict"}) - self.assertEqual(q.pop(), {"a": "dict"}) - self.assertEqual(q.pop(), 123) - self.assertEqual(q.pop(), "a") + assert q.pop() == {"a": "dict"} + assert q.pop() == 123 + assert q.pop() == "a" test_nonserializable_object = nonserializable_object_test @@ -161,20 +173,20 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): def test_serialize_item(self): q = self.queue() - i = TestItem(name="foo") + i = MyItem(name="foo") q.push(i) i2 = q.pop() - assert isinstance(i2, TestItem) - self.assertEqual(i, i2) + assert isinstance(i2, MyItem) + assert i == i2 def test_serialize_loader(self): q = self.queue() - loader = TestLoader() + loader = MyLoader() q.push(loader) loader2 = q.pop() - assert isinstance(loader2, TestLoader) - assert loader2.default_item_class is TestItem - self.assertEqual(loader2.name_out("x"), "xx") + assert isinstance(loader2, MyLoader) + assert loader2.default_item_class is MyItem + assert loader2.name_out("x") == "xx" def test_serialize_request_recursive(self): q = self.queue() @@ -183,5 +195,5 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): q.push(r) r2 = q.pop() assert isinstance(r2, Request) - self.assertEqual(r.url, r2.url) + assert r.url == r2.url assert r2.meta["request"] is r2 diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index b444c32b7..847a76ab6 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -1,7 +1,13 @@ -import shutil -import tempfile -import unittest +""" +Queues that handle requests +""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +import pytest import queuelib from scrapy.http import Request @@ -16,211 +22,146 @@ from scrapy.squeues import ( ) from scrapy.utils.test import get_crawler -""" -Queues that handle requests -""" +if TYPE_CHECKING: + from scrapy.crawler import Crawler -class BaseQueueTestCase(unittest.TestCase): - def setUp(self): - self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-") - self.qpath = self.tempfilename() - self.qdir = self.mkdtemp() - self.crawler = get_crawler(Spider) - - def tearDown(self): - shutil.rmtree(self.tmpdir) - - def tempfilename(self): - with tempfile.NamedTemporaryFile(dir=self.tmpdir) as nf: - return nf.name - - def mkdtemp(self): - return tempfile.mkdtemp(dir=self.tmpdir) +HAVE_PEEK = hasattr(queuelib.queue.FifoMemoryQueue, "peek") -class RequestQueueTestMixin: - def queue(self): - raise NotImplementedError() +@pytest.fixture +def crawler() -> Crawler: + return get_crawler(Spider) - def test_one_element_with_peek(self): - if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("The queuelib queues do not define peek") - q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + +class TestRequestQueueBase(ABC): + @property + @abstractmethod + def is_fifo(self) -> bool: + raise NotImplementedError + + @pytest.mark.parametrize("test_peek", [True, False]) + def test_one_element(self, q: queuelib.queue.BaseQueue, test_peek: bool): + if test_peek and not HAVE_PEEK: + pytest.skip("The queuelib queues do not define peek") + if not test_peek and HAVE_PEEK: + pytest.skip("The queuelib queues define peek") + assert len(q) == 0 + if test_peek: + assert q.peek() is None + assert q.pop() is None req = Request("http://www.example.com") q.push(req) - self.assertEqual(len(q), 1) - self.assertEqual(q.peek().url, req.url) - self.assertEqual(q.pop().url, req.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 1 + if test_peek: + result = q.peek() + assert result is not None + assert result.url == req.url + else: + with pytest.raises( + NotImplementedError, + match="The underlying queue class does not implement 'peek'", + ): + q.peek() + result = q.pop() + assert result is not None + assert result.url == req.url + assert len(q) == 0 + if test_peek: + assert q.peek() is None + assert q.pop() is None q.close() - def test_one_element_without_peek(self): - if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("The queuelib queues define peek") - q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) - req = Request("http://www.example.com") - q.push(req) - self.assertEqual(len(q), 1) - with self.assertRaises( - NotImplementedError, - msg="The underlying queue class does not implement 'peek'", - ): - q.peek() - self.assertEqual(q.pop().url, req.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) - q.close() - - -class FifoQueueMixin(RequestQueueTestMixin): - def test_fifo_with_peek(self): - if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("The queuelib queues do not define peek") - q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + @pytest.mark.parametrize("test_peek", [True, False]) + def test_order(self, q: queuelib.queue.BaseQueue, test_peek: bool): + if test_peek and not HAVE_PEEK: + pytest.skip("The queuelib queues do not define peek") + if not test_peek and HAVE_PEEK: + pytest.skip("The queuelib queues define peek") + assert len(q) == 0 + if test_peek: + assert q.peek() is None + assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") q.push(req1) q.push(req2) q.push(req3) - self.assertEqual(len(q), 3) - self.assertEqual(q.peek().url, req1.url) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.peek().url, req2.url) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.peek().url, req3.url) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) - q.close() - - def test_fifo_without_peek(self): - if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("The queuelib queues do not define peek") - q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) - req1 = Request("http://www.example.com/1") - req2 = Request("http://www.example.com/2") - req3 = Request("http://www.example.com/3") - q.push(req1) - q.push(req2) - q.push(req3) - with self.assertRaises( - NotImplementedError, - msg="The underlying queue class does not implement 'peek'", - ): - q.peek() - self.assertEqual(len(q), 3) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + if not test_peek: + with pytest.raises( + NotImplementedError, + match="The underlying queue class does not implement 'peek'", + ): + q.peek() + reqs = [req1, req2, req3] if self.is_fifo else [req3, req2, req1] + for i, req in enumerate(reqs): + assert len(q) == 3 - i + if test_peek: + result = q.peek() + assert result is not None + assert result.url == req.url + result = q.pop() + assert result is not None + assert result.url == req.url + assert len(q) == 0 + if test_peek: + assert q.peek() is None + assert q.pop() is None q.close() -class LifoQueueMixin(RequestQueueTestMixin): - def test_lifo_with_peek(self): - if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("The queuelib queues do not define peek") - q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) - req1 = Request("http://www.example.com/1") - req2 = Request("http://www.example.com/2") - req3 = Request("http://www.example.com/3") - q.push(req1) - q.push(req2) - q.push(req3) - self.assertEqual(len(q), 3) - self.assertEqual(q.peek().url, req3.url) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.peek().url, req2.url) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.peek().url, req1.url) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) - q.close() +class TestPickleFifoDiskQueueRequest(TestRequestQueueBase): + is_fifo = True - def test_lifo_without_peek(self): - if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("The queuelib queues do not define peek") - q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) - req1 = Request("http://www.example.com/1") - req2 = Request("http://www.example.com/2") - req3 = Request("http://www.example.com/3") - q.push(req1) - q.push(req2) - q.push(req3) - with self.assertRaises( - NotImplementedError, - msg="The underlying queue class does not implement 'peek'", - ): - q.peek() - self.assertEqual(len(q), 3) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) - q.close() + @pytest.fixture + def q(self, crawler, tmp_path): + return PickleFifoDiskQueue.from_crawler( + crawler=crawler, key=str(tmp_path / "pickle" / "fifo") + ) -class PickleFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): - def queue(self): - return PickleFifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/fifo") +class TestPickleLifoDiskQueueRequest(TestRequestQueueBase): + is_fifo = False + + @pytest.fixture + def q(self, crawler, tmp_path): + return PickleLifoDiskQueue.from_crawler( + crawler=crawler, key=str(tmp_path / "pickle" / "lifo") + ) -class PickleLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): - def queue(self): - return PickleLifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/lifo") +class TestMarshalFifoDiskQueueRequest(TestRequestQueueBase): + is_fifo = True - -class MarshalFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): - def queue(self): + @pytest.fixture + def q(self, crawler, tmp_path): return MarshalFifoDiskQueue.from_crawler( - crawler=self.crawler, key="marshal/fifo" + crawler=crawler, key=str(tmp_path / "marshal" / "fifo") ) -class MarshalLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): - def queue(self): +class TestMarshalLifoDiskQueueRequest(TestRequestQueueBase): + is_fifo = False + + @pytest.fixture + def q(self, crawler, tmp_path): return MarshalLifoDiskQueue.from_crawler( - crawler=self.crawler, key="marshal/lifo" + crawler=crawler, key=str(tmp_path / "marshal" / "lifo") ) -class FifoMemoryQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): - def queue(self): - return FifoMemoryQueue.from_crawler(crawler=self.crawler) +class TestFifoMemoryQueueRequest(TestRequestQueueBase): + is_fifo = True + + @pytest.fixture + def q(self, crawler): + return FifoMemoryQueue.from_crawler(crawler=crawler) -class LifoMemoryQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): - def queue(self): - return LifoMemoryQueue.from_crawler(crawler=self.crawler) +class TestLifoMemoryQueueRequest(TestRequestQueueBase): + is_fifo = False + + @pytest.fixture + def q(self, crawler): + return LifoMemoryQueue.from_crawler(crawler=crawler) diff --git a/tests/test_stats.py b/tests/test_stats.py index 3d4c7e88e..27af18bb1 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -1,96 +1,188 @@ -import unittest +from __future__ import annotations + from datetime import datetime +from typing import TYPE_CHECKING from unittest import mock +import pytest + +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.corestats import CoreStats from scrapy.spiders import Spider from scrapy.statscollectors import DummyStatsCollector, StatsCollector from scrapy.utils.test import get_crawler +from tests.spiders import SimpleSpider +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from scrapy.crawler import Crawler -class CoreStatsExtensionTest(unittest.TestCase): - def setUp(self): - self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider("foo") +@pytest.fixture +def crawler() -> Crawler: + return get_crawler(Spider) + +@pytest.fixture +def spider(crawler: Crawler) -> Spider: + return crawler._create_spider("foo") + + +class TestCoreStatsExtension: + @mock.patch("scrapy.extensions.corestats.monotonic", return_value=0) @mock.patch("scrapy.extensions.corestats.datetime") - def test_core_stats_default_stats_collector(self, mock_datetime): + def test_core_stats_default_stats_collector( + self, mock_datetime: mock.Mock, crawler: Crawler, spider: Spider + ) -> None: fixed_datetime = datetime(2019, 12, 1, 11, 38) mock_datetime.now = mock.Mock(return_value=fixed_datetime) - self.crawler.stats = StatsCollector(self.crawler) - ext = CoreStats.from_crawler(self.crawler) - ext.spider_opened(self.spider) - ext.item_scraped({}, self.spider) - ext.response_received(self.spider) - ext.item_dropped({}, self.spider, ZeroDivisionError()) - ext.spider_closed(self.spider, "finished") - self.assertEqual( - ext.stats._stats, - { - "start_time": fixed_datetime, - "finish_time": fixed_datetime, - "item_scraped_count": 1, - "response_received_count": 1, - "item_dropped_count": 1, - "item_dropped_reasons_count/ZeroDivisionError": 1, - "finish_reason": "finished", - "elapsed_time_seconds": 0.0, - }, - ) + crawler.stats = StatsCollector(crawler) + ext = CoreStats.from_crawler(crawler) + ext.spider_opened(spider) + ext.item_scraped({}, spider) + ext.response_received(spider) + ext.item_dropped({}, spider, ZeroDivisionError()) + ext.spider_closed(spider, "finished") + assert ext.stats._stats == { + "start_time": fixed_datetime, + "finish_time": fixed_datetime, + "item_scraped_count": 1, + "response_received_count": 1, + "item_dropped_count": 1, + "item_dropped_reasons_count/ZeroDivisionError": 1, + "finish_reason": "finished", + "elapsed_time_seconds": 0.0, + } - def test_core_stats_dummy_stats_collector(self): - self.crawler.stats = DummyStatsCollector(self.crawler) - ext = CoreStats.from_crawler(self.crawler) - ext.spider_opened(self.spider) - ext.item_scraped({}, self.spider) - ext.response_received(self.spider) - ext.item_dropped({}, self.spider, ZeroDivisionError()) - ext.spider_closed(self.spider, "finished") - self.assertEqual(ext.stats._stats, {}) + def test_core_stats_dummy_stats_collector( + self, crawler: Crawler, spider: Spider + ) -> None: + crawler.stats = DummyStatsCollector(crawler) + ext = CoreStats.from_crawler(crawler) + ext.spider_opened(spider) + ext.item_scraped({}, spider) + ext.response_received(spider) + ext.item_dropped({}, spider, ZeroDivisionError()) + ext.spider_closed(spider, "finished") + assert ext.stats._stats == {} -class StatsCollectorTest(unittest.TestCase): - def setUp(self): - self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider("foo") - - def test_collector(self): - stats = StatsCollector(self.crawler) - self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_value("anything"), None) - self.assertEqual(stats.get_value("anything", "default"), "default") +class TestStatsCollector: + def test_collector(self, crawler: Crawler) -> None: + stats = StatsCollector(crawler) + assert stats.get_stats() == {} + assert stats.get_value("anything") is None + assert stats.get_value("anything", "default") == "default" stats.set_value("test", "value") - self.assertEqual(stats.get_stats(), {"test": "value"}) + assert stats.get_stats() == {"test": "value"} stats.set_value("test2", 23) - self.assertEqual(stats.get_stats(), {"test": "value", "test2": 23}) - self.assertEqual(stats.get_value("test2"), 23) + assert stats.get_stats() == {"test": "value", "test2": 23} + assert stats.get_value("test2") == 23 stats.inc_value("test2") - self.assertEqual(stats.get_value("test2"), 24) + assert stats.get_value("test2") == 24 stats.inc_value("test2", 6) - self.assertEqual(stats.get_value("test2"), 30) + assert stats.get_value("test2") == 30 stats.max_value("test2", 6) - self.assertEqual(stats.get_value("test2"), 30) + assert stats.get_value("test2") == 30 stats.max_value("test2", 40) - self.assertEqual(stats.get_value("test2"), 40) + assert stats.get_value("test2") == 40 stats.max_value("test3", 1) - self.assertEqual(stats.get_value("test3"), 1) + assert stats.get_value("test3") == 1 stats.min_value("test2", 60) - self.assertEqual(stats.get_value("test2"), 40) + assert stats.get_value("test2") == 40 stats.min_value("test2", 35) - self.assertEqual(stats.get_value("test2"), 35) + assert stats.get_value("test2") == 35 stats.min_value("test4", 7) - self.assertEqual(stats.get_value("test4"), 7) + assert stats.get_value("test4") == 7 - def test_dummy_collector(self): - stats = DummyStatsCollector(self.crawler) - self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_value("anything"), None) - self.assertEqual(stats.get_value("anything", "default"), "default") + def test_dummy_collector(self, crawler: Crawler) -> None: + stats = DummyStatsCollector(crawler) + assert stats.get_stats() == {} + assert stats.get_value("anything") is None + assert stats.get_value("anything", "default") == "default" stats.set_value("test", "value") stats.inc_value("v1") stats.max_value("v2", 100) stats.min_value("v3", 100) - stats.open_spider("a") - stats.set_value("test", "value", spider=self.spider) - self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_stats("a"), {}) + stats.open_spider() + stats.set_value("test", "value") + assert stats.get_stats() == {} + + def test_deprecated_spider_arg(self, crawler: Crawler, spider: Spider) -> None: + stats = StatsCollector(crawler) + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a 'spider' argument to StatsCollector\.set_value\(\) is deprecated", + ): + stats.set_value("test", "value", spider=spider) + assert stats.get_stats() == {"test": "value"} + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a 'spider' argument to StatsCollector\.get_stats\(\) is deprecated", + ): + assert stats.get_stats(spider) == {"test": "value"} + + @coroutine_test + async def test_deprecated_spider_arg_custom_collector(self) -> None: + class CustomStatsCollector: + def __init__(self, crawler): + self._stats = {} + + def open_spider(self, spider): + pass + + def get_stats(self, spider=None): + return self._stats + + def inc_value(self, key, count=1, start=0, spider=None): + d = self._stats + d[key] = d.setdefault(key, start) + count + + def close_spider(self, spider, reason): + pass + + crawler = get_crawler(SimpleSpider, {"STATS_CLASS": CustomStatsCollector}) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"The open_spider\(\) method of .*CustomStatsCollector requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"The close_spider\(\) method of .*CustomStatsCollector requires a spider argument", + ), + ): + await crawler.crawl_async(url="data:,") + + @coroutine_test + async def test_deprecated_spider_arg_custom_collector_subclass(self) -> None: + class CustomStatsCollector(StatsCollector): + def open_spider(self, spider): # pylint: disable=signature-differs + super().open_spider(spider) + + def inc_value(self, key, count=1, start=0, spider=None): # pylint: disable=useless-parent-delegation + super().inc_value(key, count, start, spider) + + def close_spider(self, spider, reason): # pylint: disable=signature-differs + super().close_spider(spider, reason) + + crawler = get_crawler(SimpleSpider, {"STATS_CLASS": CustomStatsCollector}) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"The open_spider\(\) method of .*CustomStatsCollector requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a 'spider' argument to .*CustomStatsCollector\.open_spider\(\) is deprecated", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"The close_spider\(\) method of .*CustomStatsCollector requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a 'spider' argument to .*CustomStatsCollector\.close_spider\(\) is deprecated", + ), + ): + await crawler.crawl_async(url="data:,") diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py index d272101b8..b7497c608 100644 --- a/tests/test_toplevel.py +++ b/tests/test_toplevel.py @@ -1,33 +1,35 @@ -from unittest import TestCase - import scrapy -class ToplevelTestCase(TestCase): - def test_version(self): - self.assertIs(type(scrapy.__version__), str) +def test_version(): + assert isinstance(scrapy.__version__, str) - def test_version_info(self): - self.assertIs(type(scrapy.version_info), tuple) - def test_request_shortcut(self): - from scrapy.http import FormRequest, Request +def test_version_info(): + assert isinstance(scrapy.version_info, tuple) - self.assertIs(scrapy.Request, Request) - self.assertIs(scrapy.FormRequest, FormRequest) - def test_spider_shortcut(self): - from scrapy.spiders import Spider +def test_request_shortcut(): + from scrapy.http import FormRequest, Request # noqa: PLC0415 - self.assertIs(scrapy.Spider, Spider) + assert scrapy.Request is Request + assert scrapy.FormRequest is FormRequest - def test_selector_shortcut(self): - from scrapy.selector import Selector - self.assertIs(scrapy.Selector, Selector) +def test_spider_shortcut(): + from scrapy.spiders import Spider # noqa: PLC0415 - def test_item_shortcut(self): - from scrapy.item import Field, Item + assert scrapy.Spider is Spider - self.assertIs(scrapy.Item, Item) - self.assertIs(scrapy.Field, Field) + +def test_selector_shortcut(): + from scrapy.selector import Selector # noqa: PLC0415 + + assert scrapy.Selector is Selector + + +def test_item_shortcut(): + from scrapy.item import Field, Item # noqa: PLC0415 + + assert scrapy.Item is Item + assert scrapy.Field is Field diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py deleted file mode 100644 index c695968d7..000000000 --- a/tests/test_urlparse_monkeypatches.py +++ /dev/null @@ -1,11 +0,0 @@ -import unittest -from urllib.parse import urlparse - - -class UrlparseTestCase(unittest.TestCase): - def test_s3_url(self): - p = urlparse("s3://bucket/key/name?param=value") - self.assertEqual(p.scheme, "s3") - self.assertEqual(p.hostname, "bucket") - self.assertEqual(p.path, "/key/name") - self.assertEqual(p.query, "param=value") diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index 9ae66c57c..fc4e1c487 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -1,20 +1,16 @@ -from twisted.trial import unittest - from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen -from scrapy.utils.defer import deferred_f_from_coro_f +from tests.utils.decorators import coroutine_test -class AsyncgenUtilsTest(unittest.TestCase): - @deferred_f_from_coro_f +class TestAsyncgenUtils: + @coroutine_test async def test_as_async_generator(self): ag = as_async_generator(range(42)) - results = [] - async for i in ag: - results.append(i) - self.assertEqual(results, list(range(42))) + results = [i async for i in ag] + assert results == list(range(42)) - @deferred_f_from_coro_f + @coroutine_test async def test_collect_asyncgen(self): ag = as_async_generator(range(42)) results = await collect_asyncgen(ag) - self.assertEqual(results, list(range(42))) + assert results == list(range(42)) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 65e352053..a871a282e 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,37 +1,149 @@ +from __future__ import annotations + import asyncio -import warnings -from unittest import TestCase +import random +from typing import TYPE_CHECKING +from unittest import mock -from pytest import mark +import pytest +from twisted.internet.defer import Deferred -from scrapy.utils.defer import deferred_f_from_coro_f -from scrapy.utils.reactor import ( - install_reactor, - is_asyncio_reactor_installed, - set_asyncio_event_loop, +from scrapy.utils.asyncgen import as_async_generator +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + _parallel_asyncio, + is_asyncio_available, ) +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator -@mark.usefixtures("reactor_pytest") -class AsyncioTest(TestCase): - def test_is_asyncio_reactor_installed(self): +class TestAsyncio: + @coroutine_test + async def test_is_asyncio_available(self, reactor_pytest: str) -> None: # the result should depend only on the pytest --reactor argument - self.assertEqual( - is_asyncio_reactor_installed(), self.reactor_pytest == "asyncio" - ) + assert is_asyncio_available() == (reactor_pytest != "default") - def test_install_asyncio_reactor(self): - from twisted.internet import reactor as original_reactor - with warnings.catch_warnings(record=True) as w: - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") - self.assertEqual(len(w), 0) - from twisted.internet import reactor +@pytest.mark.only_asyncio +class TestParallelAsyncio: + """Test for scrapy.utils.asyncio.parallel_asyncio(), based on tests.test_utils_defer.TestParallelAsync.""" - assert original_reactor == reactor + CONCURRENT_ITEMS = 50 - @mark.only_asyncio() - @deferred_f_from_coro_f - async def test_set_asyncio_event_loop(self): - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") - assert set_asyncio_event_loop(None) is asyncio.get_running_loop() + @staticmethod + async def callable(o: int, results: list[int]) -> None: + if random.random() < 0.4: + # simulate async processing + await asyncio.sleep(random.random() / 8) + # simulate trivial sync processing + results.append(o) + + async def callable_wrapped( + self, + o: int, + results: list[int], + parallel_count: list[int], + max_parallel_count: list[int], + ) -> None: + parallel_count[0] += 1 + max_parallel_count[0] = max(max_parallel_count[0], parallel_count[0]) + await self.callable(o, results) + assert parallel_count[0] > 0, parallel_count[0] + parallel_count[0] -= 1 + + @staticmethod + def get_async_iterable(length: int) -> AsyncGenerator[int, None]: + # simulate a simple callback without delays between results + return as_async_generator(range(length)) + + @staticmethod + async def get_async_iterable_with_delays(length: int) -> AsyncGenerator[int, None]: + # simulate a callback with delays between some of the results + for i in range(length): + if random.random() < 0.1: + await asyncio.sleep(random.random() / 20) + yield i + + @coroutine_test + async def test_simple(self): + for length in [20, 50, 100]: + parallel_count = [0] + max_parallel_count = [0] + results = [] + ait = self.get_async_iterable(length) + await _parallel_asyncio( + ait, + self.CONCURRENT_ITEMS, + self.callable_wrapped, + results, + parallel_count, + max_parallel_count, + ) + assert list(range(length)) == sorted(results) + assert max_parallel_count[0] <= self.CONCURRENT_ITEMS + + @coroutine_test + async def test_delays(self): + for length in [20, 50, 100]: + parallel_count = [0] + max_parallel_count = [0] + results = [] + ait = self.get_async_iterable_with_delays(length) + await _parallel_asyncio( + ait, + self.CONCURRENT_ITEMS, + self.callable_wrapped, + results, + parallel_count, + max_parallel_count, + ) + assert list(range(length)) == sorted(results) + assert max_parallel_count[0] <= self.CONCURRENT_ITEMS + + +@pytest.mark.only_asyncio +class TestAsyncioLoopingCall: + @coroutine_test + async def test_looping_call(self): + func = mock.MagicMock() + looping_call = AsyncioLoopingCall(func) + looping_call.start(1, now=False) + assert looping_call.running + looping_call.stop() + assert not looping_call.running + assert not func.called + + @coroutine_test + async def test_looping_call_now(self): + func = mock.MagicMock() + looping_call = AsyncioLoopingCall(func) + looping_call.start(1) + looping_call.stop() + assert func.called + + @coroutine_test + async def test_looping_call_already_running(self): + looping_call = AsyncioLoopingCall(lambda: None) + looping_call.start(1) + with pytest.raises(RuntimeError): + looping_call.start(1) + looping_call.stop() + + @coroutine_test + async def test_looping_call_interval(self): + looping_call = AsyncioLoopingCall(lambda: None) + with pytest.raises(ValueError, match="Interval must be greater than 0"): + looping_call.start(0) + with pytest.raises(ValueError, match="Interval must be greater than 0"): + looping_call.start(-1) + assert not looping_call.running + + @coroutine_test + async def test_looping_call_bad_function(self): + looping_call = AsyncioLoopingCall(Deferred) + with pytest.raises(TypeError): + looping_call.start(0.1) + assert not looping_call.running diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index dc3f01d57..b6a9d8b06 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,9 +1,6 @@ -import unittest -import warnings - import pytest -from scrapy.exceptions import ScrapyDeprecationWarning, UsageError +from scrapy.exceptions import UsageError from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( arglist_to_dict, @@ -13,177 +10,99 @@ from scrapy.utils.conf import ( ) -class BuildComponentListTest(unittest.TestCase): +class TestBuildComponentList: def test_build_dict(self): d = {"one": 1, "two": None, "three": 8, "four": 4} - self.assertEqual( - build_component_list(d, convert=lambda x: x), ["one", "four", "three"] - ) - - def test_backward_compatible_build_dict(self): - base = {"one": 1, "two": 2, "three": 3, "five": 5, "six": None} - custom = {"two": None, "three": 8, "four": 4} - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - self.assertEqual( - build_component_list(base, custom, convert=lambda x: x), - ["one", "four", "five", "three"], - ) - - def test_return_list(self): - custom = ["a", "b", "c"] - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - self.assertEqual( - build_component_list(None, custom, convert=lambda x: x), custom - ) - - def test_map_dict(self): - custom = {"one": 1, "two": 2, "three": 3} - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - self.assertEqual( - build_component_list({}, custom, convert=lambda x: x.upper()), - ["ONE", "TWO", "THREE"], - ) - - def test_map_list(self): - custom = ["a", "b", "c"] - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - self.assertEqual( - build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"] - ) - - def test_duplicate_components_in_dict(self): - duplicate_dict = {"one": 1, "two": 2, "ONE": 4} - with self.assertRaises(ValueError): - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - build_component_list({}, duplicate_dict, convert=lambda x: x.lower()) - - def test_duplicate_components_in_list(self): - duplicate_list = ["a", "b", "a"] - with self.assertRaises(ValueError) as cm: - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - build_component_list(None, duplicate_list, convert=lambda x: x) - self.assertIn(str(duplicate_list), str(cm.exception)) + assert build_component_list(d, convert=lambda x: x) == ["one", "four", "three"] def test_duplicate_components_in_basesettings(self): # Higher priority takes precedence duplicate_bs = BaseSettings({"one": 1, "two": 2}, priority=0) duplicate_bs.set("ONE", 4, priority=10) - self.assertEqual( - build_component_list(duplicate_bs, convert=lambda x: x.lower()), - ["two", "one"], - ) + assert build_component_list(duplicate_bs, convert=lambda x: x.lower()) == [ + "two", + "one", + ] duplicate_bs.set("one", duplicate_bs["one"], priority=20) - self.assertEqual( - build_component_list(duplicate_bs, convert=lambda x: x.lower()), - ["one", "two"], - ) + assert build_component_list(duplicate_bs, convert=lambda x: x.lower()) == [ + "one", + "two", + ] # Same priority raises ValueError duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20) - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, match=r"Some paths in .* convert to the same object" + ): build_component_list(duplicate_bs, convert=lambda x: x.lower()) def test_valid_numbers(self): # work well with None and numeric values d = {"a": 10, "b": None, "c": 15, "d": 5.0} - self.assertEqual(build_component_list(d, convert=lambda x: x), ["d", "a", "c"]) + assert build_component_list(d, convert=lambda x: x) == ["d", "a", "c"] d = { "a": 33333333333333333333, "b": 11111111111111111111, "c": 22222222222222222222, } - self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"]) - # raise exception for invalid values - d = {"one": "5"} - with self.assertRaises(ValueError): - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - build_component_list({}, d, convert=lambda x: x) + assert build_component_list(d, convert=lambda x: x) == ["b", "c", "a"] -class UtilsConfTestCase(unittest.TestCase): - def test_arglist_to_dict(self): - self.assertEqual( - arglist_to_dict(["arg1=val1", "arg2=val2"]), - {"arg1": "val1", "arg2": "val2"}, - ) +def test_arglist_to_dict(): + assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == { + "arg1": "val1", + "arg2": "val2", + } -class FeedExportConfigTestCase(unittest.TestCase): +class TestFeedExportConfig: def test_feed_export_config_invalid_format(self): settings = Settings() - self.assertRaises( - UsageError, - feed_process_params_from_cli, - settings, - ["items.dat"], - "noformat", - ) + with pytest.raises(UsageError): + feed_process_params_from_cli(settings, ["items.dat"]) def test_feed_export_config_mismatch(self): settings = Settings() - self.assertRaises( - UsageError, - feed_process_params_from_cli, - settings, - ["items1.dat", "items2.dat"], - "noformat", - ) - - def test_feed_export_config_backward_compatible(self): - with warnings.catch_warnings(record=True) as cw: - settings = Settings() - self.assertEqual( - {"items.dat": {"format": "csv"}}, - feed_process_params_from_cli(settings, ["items.dat"], "csv"), - ) - self.assertEqual(cw[0].category, ScrapyDeprecationWarning) + with pytest.raises(UsageError): + feed_process_params_from_cli(settings, ["items1.dat", "items2.dat"]) def test_feed_export_config_explicit_formats(self): settings = Settings() - self.assertEqual( - { - "items_1.dat": {"format": "json"}, - "items_2.dat": {"format": "xml"}, - "items_3.dat": {"format": "csv"}, - }, - feed_process_params_from_cli( - settings, ["items_1.dat:json", "items_2.dat:xml", "items_3.dat:csv"] - ), + assert { + "items_1.dat": {"format": "json"}, + "items_2.dat": {"format": "xml"}, + "items_3.dat": {"format": "csv"}, + } == feed_process_params_from_cli( + settings, ["items_1.dat:json", "items_2.dat:xml", "items_3.dat:csv"] ) def test_feed_export_config_implicit_formats(self): settings = Settings() - self.assertEqual( - { - "items_1.json": {"format": "json"}, - "items_2.xml": {"format": "xml"}, - "items_3.csv": {"format": "csv"}, - }, - feed_process_params_from_cli( - settings, ["items_1.json", "items_2.xml", "items_3.csv"] - ), + assert { + "items_1.json": {"format": "json"}, + "items_2.xml": {"format": "xml"}, + "items_3.csv": {"format": "csv"}, + } == feed_process_params_from_cli( + settings, ["items_1.json", "items_2.xml", "items_3.csv"] ) def test_feed_export_config_stdout(self): settings = Settings() - self.assertEqual( - {"stdout:": {"format": "pickle"}}, - feed_process_params_from_cli(settings, ["-:pickle"]), + assert {"stdout:": {"format": "pickle"}} == feed_process_params_from_cli( + settings, ["-:pickle"] ) def test_feed_export_config_overwrite(self): settings = Settings() - self.assertEqual( - {"output.json": {"format": "json", "overwrite": True}}, - feed_process_params_from_cli(settings, [], None, ["output.json"]), + assert { + "output.json": {"format": "json", "overwrite": True} + } == feed_process_params_from_cli( + settings, [], overwrite_output=["output.json"] ) def test_output_and_overwrite_output(self): - with self.assertRaises(UsageError): + with pytest.raises(UsageError): feed_process_params_from_cli( - Settings(), - ["output1.json"], - None, - ["output2.json"], + Settings(), ["output1.json"], overwrite_output=["output2.json"] ) def test_feed_complete_default_values_from_settings_empty(self): @@ -199,18 +118,15 @@ class FeedExportConfigTestCase(unittest.TestCase): } ) new_feed = feed_complete_default_values_from_settings(feed, settings) - self.assertEqual( - new_feed, - { - "encoding": "custom encoding", - "fields": ["f1", "f2", "f3"], - "indent": 42, - "store_empty": True, - "uri_params": (1, 2, 3, 4), - "batch_item_count": 2, - "item_export_kwargs": {}, - }, - ) + assert new_feed == { + "encoding": "custom encoding", + "fields": ["f1", "f2", "f3"], + "indent": 42, + "store_empty": True, + "uri_params": (1, 2, 3, 4), + "batch_item_count": 2, + "item_export_kwargs": {}, + } def test_feed_complete_default_values_from_settings_non_empty(self): feed = { @@ -227,19 +143,12 @@ class FeedExportConfigTestCase(unittest.TestCase): } ) new_feed = feed_complete_default_values_from_settings(feed, settings) - self.assertEqual( - new_feed, - { - "encoding": "other encoding", - "fields": None, - "indent": 42, - "store_empty": True, - "uri_params": None, - "batch_item_count": 2, - "item_export_kwargs": {}, - }, - ) - - -if __name__ == "__main__": - unittest.main() + assert new_feed == { + "encoding": "other encoding", + "fields": None, + "indent": 42, + "store_empty": True, + "uri_params": None, + "batch_item_count": 2, + "item_export_kwargs": {}, + } diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index dabd6054d..dc1d96f66 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -1,4 +1,4 @@ -import unittest +import pytest from scrapy.utils.console import get_shell_embed_func @@ -18,27 +18,24 @@ except ImportError: ipy = False -class UtilsConsoleTestCase(unittest.TestCase): - def test_get_shell_embed_func(self): - shell = get_shell_embed_func(["invalid"]) - self.assertEqual(shell, None) +def test_get_shell_embed_func(): + shell = get_shell_embed_func(["invalid"]) + assert shell is None - shell = get_shell_embed_func(["invalid", "python"]) - self.assertTrue(callable(shell)) - self.assertEqual(shell.__name__, "_embed_standard_shell") - - @unittest.skipIf(not bpy, "bpython not available in testenv") - def test_get_shell_embed_func2(self): - shell = get_shell_embed_func(["bpython"]) - self.assertTrue(callable(shell)) - self.assertEqual(shell.__name__, "_embed_bpython_shell") - - @unittest.skipIf(not ipy, "IPython not available in testenv") - def test_get_shell_embed_func3(self): - # default shell should be 'ipython' - shell = get_shell_embed_func() - self.assertEqual(shell.__name__, "_embed_ipython_shell") + shell = get_shell_embed_func(["invalid", "python"]) + assert callable(shell) + assert shell.__name__ == "_embed_standard_shell" -if __name__ == "__main__": - unittest.main() +@pytest.mark.skipif(not bpy, reason="bpython not available in testenv") +def test_get_shell_embed_func_bpython(): + shell = get_shell_embed_func(["bpython"]) + assert callable(shell) + assert shell.__name__ == "_embed_bpython_shell" + + +@pytest.mark.skipif(not ipy, reason="IPython not available in testenv") +def test_get_shell_embed_func_ipython(): + # default shell should be 'ipython' + shell = get_shell_embed_func() + assert shell.__name__ == "_embed_ipython_shell" diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index 1816db29b..b1532ca77 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -1,22 +1,22 @@ -import unittest import warnings +from typing import Any +import pytest from w3lib.http import basic_auth_header from scrapy import Request from scrapy.utils.curl import curl_to_request_kwargs -class CurlToRequestKwargsTest(unittest.TestCase): - maxDiff = 5000 - - def _test_command(self, curl_command, expected_result): +class TestCurlToRequestKwargs: + @staticmethod + def _test_command(curl_command: str, expected_result: dict[str, Any]) -> None: result = curl_to_request_kwargs(curl_command) - self.assertEqual(result, expected_result) + assert result == expected_result try: Request(**result) except TypeError as e: - self.fail(f"Request kwargs are not correct {e}") + pytest.fail(f"Request kwargs are not correct {e}") def test_get(self): curl_command = "curl http://example.org/" @@ -49,8 +49,8 @@ class CurlToRequestKwargsTest(unittest.TestCase): "ml,application/xhtml+xml,application/xml;q=0.9,image/webp,image/a" "png,*/*;q=0.8' -H 'Referer: http://httpbin.org/' -H 'Cookie: _gau" "ges_unique_year=1; _gauges_unique=1; _gauges_unique_month=1; _gau" - "ges_unique_hour=1; _gauges_unique_day=1' -H 'Connection: keep-ali" - "ve' --compressed" + "ges_unique_hour=1' -H 'Connection: keep-alive' --compressed -b '_" + "gauges_unique_day=1'" ) expected_result = { "method": "GET", @@ -202,14 +202,14 @@ class CurlToRequestKwargsTest(unittest.TestCase): def test_get_silent(self): curl_command = 'curl --silent "www.example.com"' expected_result = {"method": "GET", "url": "http://www.example.com"} - self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) + assert curl_to_request_kwargs(curl_command) == expected_result def test_too_few_arguments_error(self): - self.assertRaisesRegex( + with pytest.raises( ValueError, - r"too few arguments|the following arguments are required:\s*url", - lambda: curl_to_request_kwargs("curl"), - ) + match=r"too few arguments|the following arguments are required:\s*url", + ): + curl_to_request_kwargs("curl") def test_ignore_unknown_options(self): # case 1: ignore_unknown_options=True: @@ -217,19 +217,14 @@ class CurlToRequestKwargsTest(unittest.TestCase): warnings.simplefilter("ignore") curl_command = "curl --bar --baz http://www.example.com" expected_result = {"method": "GET", "url": "http://www.example.com"} - self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) + assert curl_to_request_kwargs(curl_command) == expected_result # case 2: ignore_unknown_options=False (raise exception): - self.assertRaisesRegex( - ValueError, - "Unrecognized options:.*--bar.*--baz", - lambda: curl_to_request_kwargs( + with pytest.raises(ValueError, match=r"Unrecognized options:.*--bar.*--baz"): + curl_to_request_kwargs( "curl --bar --baz http://www.example.com", ignore_unknown_options=False - ), - ) + ) def test_must_start_with_curl_error(self): - self.assertRaises( - ValueError, - lambda: curl_to_request_kwargs("carl -X POST http://example.org"), - ) + with pytest.raises(ValueError, match="A curl command must start"): + curl_to_request_kwargs("carl -X POST http://example.org") diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 9e5f88f48..f573993a1 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,8 +1,10 @@ import copy -import unittest import warnings -from collections.abc import Mapping, MutableMapping -from typing import Iterator +from abc import ABC, abstractmethod +from collections.abc import Iterator, Mapping, MutableMapping +from typing import Any + +import pytest from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request @@ -15,21 +17,24 @@ from scrapy.utils.datatypes import ( ) from scrapy.utils.python import garbage_collect -__doctests__ = ["scrapy.utils.datatypes"] +class TestCaseInsensitiveDictBase(ABC): + @property + @abstractmethod + def dict_class(self) -> type[MutableMapping[str, Any]]: + raise NotImplementedError -class CaseInsensitiveDictMixin: def test_init_dict(self): seq = {"red": 1, "black": 3} d = self.dict_class(seq) - self.assertEqual(d["red"], 1) - self.assertEqual(d["black"], 3) + assert d["red"] == 1 + assert d["black"] == 3 def test_init_pair_sequence(self): seq = (("red", 1), ("black", 3)) d = self.dict_class(seq) - self.assertEqual(d["red"], 1) - self.assertEqual(d["black"], 3) + assert d["red"] == 1 + assert d["black"] == 3 def test_init_mapping(self): class MyMapping(Mapping): @@ -47,8 +52,8 @@ class CaseInsensitiveDictMixin: seq = MyMapping(red=1, black=3) d = self.dict_class(seq) - self.assertEqual(d["red"], 1) - self.assertEqual(d["black"], 3) + assert d["red"] == 1 + assert d["black"] == 3 def test_init_mutable_mapping(self): class MyMutableMapping(MutableMapping): @@ -72,61 +77,65 @@ class CaseInsensitiveDictMixin: seq = MyMutableMapping(red=1, black=3) d = self.dict_class(seq) - self.assertEqual(d["red"], 1) - self.assertEqual(d["black"], 3) + assert d["red"] == 1 + assert d["black"] == 3 def test_caseless(self): d = self.dict_class() d["key_Lower"] = 1 - self.assertEqual(d["KEy_loWer"], 1) - self.assertEqual(d.get("KEy_loWer"), 1) + assert d["KEy_loWer"] == 1 + assert d.get("KEy_loWer") == 1 d["KEY_LOWER"] = 3 - self.assertEqual(d["key_Lower"], 3) - self.assertEqual(d.get("key_Lower"), 3) + assert d["key_Lower"] == 3 + assert d.get("key_Lower") == 3 def test_delete(self): d = self.dict_class({"key_lower": 1}) del d["key_LOWER"] - self.assertRaises(KeyError, d.__getitem__, "key_LOWER") - self.assertRaises(KeyError, d.__getitem__, "key_lower") + with pytest.raises(KeyError): + d["key_LOWER"] + with pytest.raises(KeyError): + d["key_lower"] + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_getdefault(self): d = CaselessDict() - self.assertEqual(d.get("c", 5), 5) + assert d.get("c", 5) == 5 d["c"] = 10 - self.assertEqual(d.get("c", 5), 10) + assert d.get("c", 5) == 10 + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_setdefault(self): d = CaselessDict({"a": 1, "b": 2}) r = d.setdefault("A", 5) - self.assertEqual(r, 1) - self.assertEqual(d["A"], 1) + assert r == 1 + assert d["A"] == 1 r = d.setdefault("c", 5) - self.assertEqual(r, 5) - self.assertEqual(d["C"], 5) + assert r == 5 + assert d["C"] == 5 def test_fromkeys(self): keys = ("a", "b") d = self.dict_class.fromkeys(keys) - self.assertEqual(d["A"], None) - self.assertEqual(d["B"], None) + assert d["A"] is None + assert d["B"] is None d = self.dict_class.fromkeys(keys, 1) - self.assertEqual(d["A"], 1) - self.assertEqual(d["B"], 1) + assert d["A"] == 1 + assert d["B"] == 1 instance = self.dict_class() d = instance.fromkeys(keys) - self.assertEqual(d["A"], None) - self.assertEqual(d["B"], None) + assert d["A"] is None + assert d["B"] is None d = instance.fromkeys(keys, 1) - self.assertEqual(d["A"], 1) - self.assertEqual(d["B"], 1) + assert d["A"] == 1 + assert d["B"] == 1 def test_contains(self): d = self.dict_class() @@ -136,8 +145,9 @@ class CaseInsensitiveDictMixin: def test_pop(self): d = self.dict_class() d["a"] = 1 - self.assertEqual(d.pop("A"), 1) - self.assertRaises(KeyError, d.pop, "A") + assert d.pop("A") == 1 + with pytest.raises(KeyError): + d.pop("A") def test_normkey(self): class MyDict(self.dict_class): @@ -148,165 +158,166 @@ class CaseInsensitiveDictMixin: d = MyDict() d["key-one"] = 2 - self.assertEqual(list(d.keys()), ["Key-One"]) + assert list(d.keys()) == ["Key-One"] def test_normvalue(self): class MyDict(self.dict_class): def _normvalue(self, value): if value is not None: return value + 1 + return None normvalue = _normvalue # deprecated CaselessDict class d = MyDict({"key": 1}) - self.assertEqual(d["key"], 2) - self.assertEqual(d.get("key"), 2) + assert d["key"] == 2 + assert d.get("key") == 2 d = MyDict() d["key"] = 1 - self.assertEqual(d["key"], 2) - self.assertEqual(d.get("key"), 2) + assert d["key"] == 2 + assert d.get("key") == 2 d = MyDict() d.setdefault("key", 1) - self.assertEqual(d["key"], 2) - self.assertEqual(d.get("key"), 2) + assert d["key"] == 2 + assert d.get("key") == 2 d = MyDict() d.update({"key": 1}) - self.assertEqual(d["key"], 2) - self.assertEqual(d.get("key"), 2) + assert d["key"] == 2 + assert d.get("key") == 2 d = MyDict.fromkeys(("key",), 1) - self.assertEqual(d["key"], 2) - self.assertEqual(d.get("key"), 2) + assert d["key"] == 2 + assert d.get("key") == 2 def test_copy(self): h1 = self.dict_class({"header1": "value"}) h2 = copy.copy(h1) assert isinstance(h2, self.dict_class) - self.assertEqual(h1, h2) - self.assertEqual(h1.get("header1"), h2.get("header1")) - self.assertEqual(h1.get("header1"), h2.get("HEADER1")) + assert h1 == h2 + assert h1.get("header1") == h2.get("header1") + assert h1.get("header1") == h2.get("HEADER1") h3 = h1.copy() assert isinstance(h3, self.dict_class) - self.assertEqual(h1, h3) - self.assertEqual(h1.get("header1"), h3.get("header1")) - self.assertEqual(h1.get("header1"), h3.get("HEADER1")) + assert h1 == h3 + assert h1.get("header1") == h3.get("header1") + assert h1.get("header1") == h3.get("HEADER1") -class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase): - dict_class = CaseInsensitiveDict +class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase): + dict_class = CaseInsensitiveDict # type: ignore[assignment] def test_repr(self): d1 = self.dict_class({"foo": "bar"}) - self.assertEqual(repr(d1), "") + assert repr(d1) == "" d2 = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"}) - self.assertEqual( - repr(d2), "" - ) + assert repr(d2) == "" def test_iter(self): d = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"}) iterkeys = iter(d) - self.assertIsInstance(iterkeys, Iterator) - self.assertEqual(list(iterkeys), ["AsDf", "FoO"]) + assert isinstance(iterkeys, Iterator) + assert list(iterkeys) == ["AsDf", "FoO"] -class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase): +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class TestCaselessDict(TestCaseInsensitiveDictBase): dict_class = CaselessDict def test_deprecation_message(self): with warnings.catch_warnings(record=True) as caught: + warnings.filterwarnings("always", category=ScrapyDeprecationWarning) self.dict_class({"foo": "bar"}) - self.assertEqual(len(caught), 1) - self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning)) - self.assertEqual( - "scrapy.utils.datatypes.CaselessDict is deprecated," - " please use scrapy.utils.datatypes.CaseInsensitiveDict instead", - str(caught[0].message), + assert len(caught) == 1 + assert issubclass(caught[0].category, ScrapyDeprecationWarning) + assert ( + str(caught[0].message) + == "scrapy.utils.datatypes.CaselessDict is deprecated," + " please use scrapy.utils.datatypes.CaseInsensitiveDict instead" ) -class SequenceExcludeTest(unittest.TestCase): +class TestSequenceExclude: def test_list(self): seq = [1, 2, 3] d = SequenceExclude(seq) - self.assertIn(0, d) - self.assertIn(4, d) - self.assertNotIn(2, d) + assert 0 in d + assert 4 in d + assert 2 not in d def test_range(self): seq = range(10, 20) d = SequenceExclude(seq) - self.assertIn(5, d) - self.assertIn(20, d) - self.assertNotIn(15, d) + assert 5 in d + assert 20 in d + assert 15 not in d def test_range_step(self): seq = range(10, 20, 3) d = SequenceExclude(seq) are_not_in = [v for v in range(10, 20, 3) if v in d] - self.assertEqual([], are_not_in) + assert are_not_in == [] are_not_in = [v for v in range(10, 20) if v in d] - self.assertEqual([11, 12, 14, 15, 17, 18], are_not_in) + assert are_not_in == [11, 12, 14, 15, 17, 18] def test_string_seq(self): seq = "cde" d = SequenceExclude(seq) chars = "".join(v for v in "abcdefg" if v in d) - self.assertEqual("abfg", chars) + assert chars == "abfg" def test_stringset_seq(self): seq = set("cde") d = SequenceExclude(seq) chars = "".join(v for v in "abcdefg" if v in d) - self.assertEqual("abfg", chars) + assert chars == "abfg" def test_set(self): """Anything that is not in the supplied sequence will evaluate as 'in' the container.""" seq = {-3, "test", 1.1} d = SequenceExclude(seq) - self.assertIn(0, d) - self.assertIn("foo", d) - self.assertIn(3.14, d) - self.assertIn(set("bar"), d) + assert 0 in d + assert "foo" in d + assert 3.14 in d + assert set("bar") in d # supplied sequence is a set, so checking for list (non)inclusion fails - self.assertRaises(TypeError, (0, 1, 2) in d) - self.assertRaises(TypeError, d.__contains__, ["a", "b", "c"]) + with pytest.raises(TypeError): + ["a", "b", "c"] in d # noqa: B015 for v in [-3, "test", 1.1]: - self.assertNotIn(v, d) + assert v not in d -class LocalCacheTest(unittest.TestCase): +class TestLocalCache: def test_cache_with_limit(self): cache = LocalCache(limit=2) cache["a"] = 1 cache["b"] = 2 cache["c"] = 3 - self.assertEqual(len(cache), 2) - self.assertNotIn("a", cache) - self.assertIn("b", cache) - self.assertIn("c", cache) - self.assertEqual(cache["b"], 2) - self.assertEqual(cache["c"], 3) + assert len(cache) == 2 + assert "a" not in cache + assert "b" in cache + assert "c" in cache + assert cache["b"] == 2 + assert cache["c"] == 3 def test_cache_without_limit(self): maximum = 10**4 cache = LocalCache() for x in range(maximum): cache[str(x)] = x - self.assertEqual(len(cache), maximum) + assert len(cache) == maximum for x in range(maximum): - self.assertIn(str(x), cache) - self.assertEqual(cache[str(x)], x) + assert str(x) in cache + assert cache[str(x)] == x -class LocalWeakReferencedCacheTest(unittest.TestCase): +class TestLocalWeakReferencedCache: def test_cache_with_limit(self): cache = LocalWeakReferencedCache(limit=2) r1 = Request("https://example.org") @@ -315,19 +326,19 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): cache[r1] = 1 cache[r2] = 2 cache[r3] = 3 - self.assertEqual(len(cache), 2) - self.assertNotIn(r1, cache) - self.assertIn(r2, cache) - self.assertIn(r3, cache) - self.assertEqual(cache[r1], None) - self.assertEqual(cache[r2], 2) - self.assertEqual(cache[r3], 3) + assert len(cache) == 2 + assert r1 not in cache + assert r2 in cache + assert r3 in cache + assert cache[r1] is None + assert cache[r2] == 2 + assert cache[r3] == 3 del r2 # PyPy takes longer to collect dead references garbage_collect() - self.assertEqual(len(cache), 1) + assert len(cache) == 1 def test_cache_non_weak_referenceable_objects(self): cache = LocalWeakReferencedCache() @@ -337,36 +348,32 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): cache[k1] = 1 cache[k2] = 2 cache[k3] = 3 - self.assertNotIn(k1, cache) - self.assertNotIn(k2, cache) - self.assertNotIn(k3, cache) - self.assertEqual(len(cache), 0) + assert k1 not in cache + assert k2 not in cache + assert k3 not in cache + assert len(cache) == 0 def test_cache_without_limit(self): - max = 10**4 + maximum = 10**4 cache = LocalWeakReferencedCache() refs = [] - for x in range(max): + for x in range(maximum): refs.append(Request(f"https://example.org/{x}")) cache[refs[-1]] = x - self.assertEqual(len(cache), max) + assert len(cache) == maximum for i, r in enumerate(refs): - self.assertIn(r, cache) - self.assertEqual(cache[r], i) - del r # delete reference to the last object in the list + assert r in cache + assert cache[r] == i + del r # delete reference to the last object in the list # pylint: disable=undefined-loop-variable # delete half of the objects, make sure that is reflected in the cache - for _ in range(max // 2): + for _ in range(maximum // 2): refs.pop() # PyPy takes longer to collect dead references garbage_collect() - self.assertEqual(len(cache), max // 2) + assert len(cache) == maximum // 2 for i, r in enumerate(refs): - self.assertIn(r, cache) - self.assertEqual(cache[r], i) - - -if __name__ == "__main__": - unittest.main() + assert r in cache + assert cache[r] == i diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index a7d54b565..978c24f5a 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -1,121 +1,82 @@ -import random +from __future__ import annotations -from pytest import mark -from twisted.internet import defer, reactor -from twisted.python.failure import Failure -from twisted.trial import unittest +import asyncio +import random +from asyncio import Future +from typing import TYPE_CHECKING, Any + +import pytest +from twisted.internet.defer import Deferred, inlineCallbacks from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import ( aiter_errback, deferred_f_from_coro_f, + deferred_from_coro, + deferred_to_future, iter_errback, maybe_deferred_to_future, mustbe_deferred, parallel_async, - process_chain, - process_chain_both, - process_parallel, ) +from tests.utils.decorators import coroutine_test, inline_callbacks_test + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Awaitable, Callable, Generator -class MustbeDeferredTest(unittest.TestCase): - def test_success_function(self): - steps = [] +@pytest.mark.requires_reactor # mustbe_deferred() requires a reactor +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class TestMustbeDeferred: + @inline_callbacks_test + def test_success_function(self) -> Generator[Deferred[Any], Any, None]: + steps: list[int] = [] - def _append(v): + def _append(v: int) -> list[int]: steps.append(v) return steps + def _assert(v: list[int]) -> None: + assert v == [1, 2] # it is [1] with maybeDeferred + dfd = mustbe_deferred(_append, 1) - dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred + dfd.addCallback(_assert) steps.append(2) # add another value, that should be caught by assertEqual - return dfd + yield dfd - def test_unfired_deferred(self): - steps = [] + @inline_callbacks_test + def test_unfired_deferred(self) -> Generator[Deferred[Any], Any, None]: + steps: list[int] = [] + + def _append(v: int) -> Deferred[list[int]]: + from twisted.internet import reactor - def _append(v): steps.append(v) - dfd = defer.Deferred() + dfd: Deferred[list[int]] = Deferred() reactor.callLater(0, dfd.callback, steps) return dfd + def _assert(v: list[int]) -> None: + assert v == [1, 2] + dfd = mustbe_deferred(_append, 1) - dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred + dfd.addCallback(_assert) steps.append(2) # add another value, that should be caught by assertEqual - return dfd + yield dfd -def cb1(value, arg1, arg2): - return f"(cb1 {value} {arg1} {arg2})" - - -def cb2(value, arg1, arg2): - return defer.succeed(f"(cb2 {value} {arg1} {arg2})") - - -def cb3(value, arg1, arg2): - return f"(cb3 {value} {arg1} {arg2})" - - -def cb_fail(value, arg1, arg2): - return Failure(TypeError()) - - -def eb1(failure, arg1, arg2): - return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})" - - -class DeferUtilsTest(unittest.TestCase): - @defer.inlineCallbacks - def test_process_chain(self): - x = yield process_chain([cb1, cb2, cb3], "res", "v1", "v2") - self.assertEqual(x, "(cb3 (cb2 (cb1 res v1 v2) v1 v2) v1 v2)") - - gotexc = False - try: - yield process_chain([cb1, cb_fail, cb3], "res", "v1", "v2") - except TypeError: - gotexc = True - self.assertTrue(gotexc) - - @defer.inlineCallbacks - def test_process_chain_both(self): - x = yield process_chain_both( - [cb_fail, cb2, cb3], [None, eb1, None], "res", "v1", "v2" - ) - self.assertEqual(x, "(cb3 (eb1 TypeError v1 v2) v1 v2)") - - fail = Failure(ZeroDivisionError()) - x = yield process_chain_both( - [eb1, cb2, cb3], [eb1, None, None], fail, "v1", "v2" - ) - self.assertEqual(x, "(cb3 (cb2 (eb1 ZeroDivisionError v1 v2) v1 v2) v1 v2)") - - @defer.inlineCallbacks - def test_process_parallel(self): - x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2") - self.assertEqual(x, ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"]) - - def test_process_parallel_failure(self): - d = process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2") - self.failUnlessFailure(d, TypeError) - return d - - -class IterErrbackTest(unittest.TestCase): +class TestIterErrback: def test_iter_errback_good(self): - def itergood(): + def itergood() -> Generator[int, None, None]: yield from range(10) errors = [] out = list(iter_errback(itergood(), errors.append)) - self.assertEqual(out, list(range(10))) - self.assertFalse(errors) + assert out == list(range(10)) + assert not errors def test_iter_errback_bad(self): - def iterbad(): + def iterbad() -> Generator[int, None, None]: for x in range(10): if x == 5: 1 / 0 @@ -123,26 +84,26 @@ class IterErrbackTest(unittest.TestCase): errors = [] out = list(iter_errback(iterbad(), errors.append)) - self.assertEqual(out, [0, 1, 2, 3, 4]) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0].value, ZeroDivisionError) + assert out == [0, 1, 2, 3, 4] + assert len(errors) == 1 + assert isinstance(errors[0].value, ZeroDivisionError) -class AiterErrbackTest(unittest.TestCase): - @deferred_f_from_coro_f +class TestAiterErrback: + @coroutine_test async def test_aiter_errback_good(self): - async def itergood(): + async def itergood() -> AsyncGenerator[int, None]: for x in range(10): yield x errors = [] out = await collect_asyncgen(aiter_errback(itergood(), errors.append)) - self.assertEqual(out, list(range(10))) - self.assertFalse(errors) + assert out == list(range(10)) + assert not errors - @deferred_f_from_coro_f + @coroutine_test async def test_iter_errback_bad(self): - async def iterbad(): + async def iterbad() -> AsyncGenerator[int, None]: for x in range(10): if x == 5: 1 / 0 @@ -150,27 +111,24 @@ class AiterErrbackTest(unittest.TestCase): errors = [] out = await collect_asyncgen(aiter_errback(iterbad(), errors.append)) - self.assertEqual(out, [0, 1, 2, 3, 4]) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0].value, ZeroDivisionError) + assert out == [0, 1, 2, 3, 4] + assert len(errors) == 1 + assert isinstance(errors[0].value, ZeroDivisionError) -class AsyncDefTestsuiteTest(unittest.TestCase): - @deferred_f_from_coro_f - async def test_deferred_f_from_coro_f(self): +class TestAsyncDefTestsuite: + @coroutine_test + async def test_coroutine_test(self): pass - @deferred_f_from_coro_f - async def test_deferred_f_from_coro_f_generator(self): - yield - - @mark.xfail(reason="Checks that the test is actually executed", strict=True) - @deferred_f_from_coro_f - async def test_deferred_f_from_coro_f_xfail(self): - raise Exception("This is expected to be raised") + @pytest.mark.xfail(reason="Checks that the test is actually executed", strict=True) + @coroutine_test + async def test_coroutine_test_xfail(self): + raise RuntimeError("This is expected to be raised") -class AsyncCooperatorTest(unittest.TestCase): +@pytest.mark.requires_reactor # parallel_async() requires a reactor +class TestParallelAsync: """This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage. parallel_async is called with the results of a callback (so an iterable of items, requests and None, @@ -186,47 +144,257 @@ class AsyncCooperatorTest(unittest.TestCase): CONCURRENT_ITEMS = 50 @staticmethod - def callable(o, results): + def callable(o: int, results: list[int]) -> Deferred[None] | None: + from twisted.internet import reactor + if random.random() < 0.4: # simulate async processing - dfd = defer.Deferred() + dfd: Deferred[None] = Deferred() dfd.addCallback(lambda _: results.append(o)) delay = random.random() / 8 reactor.callLater(delay, dfd.callback, None) return dfd # simulate trivial sync processing results.append(o) + return None + + def callable_wrapped( + self, + o: int, + results: list[int], + parallel_count: list[int], + max_parallel_count: list[int], + ) -> Deferred[None] | None: + parallel_count[0] += 1 + max_parallel_count[0] = max(max_parallel_count[0], parallel_count[0]) + dfd = self.callable(o, results) + + def decrement(_: Any = None) -> None: + assert parallel_count[0] > 0, parallel_count[0] + parallel_count[0] -= 1 + + if dfd is not None: + dfd.addBoth(decrement) + else: + decrement() + return dfd @staticmethod - def get_async_iterable(length): + def get_async_iterable(length: int) -> AsyncGenerator[int, None]: # simulate a simple callback without delays between results return as_async_generator(range(length)) @staticmethod - async def get_async_iterable_with_delays(length): + async def get_async_iterable_with_delays(length: int) -> AsyncGenerator[int, None]: # simulate a callback with delays between some of the results + from twisted.internet import reactor + for i in range(length): if random.random() < 0.1: - dfd = defer.Deferred() + dfd: Deferred[None] = Deferred() delay = random.random() / 20 reactor.callLater(delay, dfd.callback, None) await maybe_deferred_to_future(dfd) yield i - @defer.inlineCallbacks + @inline_callbacks_test def test_simple(self): for length in [20, 50, 100]: + parallel_count = [0] + max_parallel_count = [0] results = [] ait = self.get_async_iterable(length) - dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results) + dl = parallel_async( + ait, + self.CONCURRENT_ITEMS, + self.callable_wrapped, + results, + parallel_count, + max_parallel_count, + ) yield dl - self.assertEqual(list(range(length)), sorted(results)) + assert list(range(length)) == sorted(results) + assert parallel_count[0] == 0 + assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0] - @defer.inlineCallbacks + @inline_callbacks_test def test_delays(self): for length in [20, 50, 100]: + parallel_count = [0] + max_parallel_count = [0] results = [] ait = self.get_async_iterable_with_delays(length) - dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results) + dl = parallel_async( + ait, + self.CONCURRENT_ITEMS, + self.callable_wrapped, + results, + parallel_count, + max_parallel_count, + ) yield dl - self.assertEqual(list(range(length)), sorted(results)) + assert list(range(length)) == sorted(results) + assert parallel_count[0] == 0 + assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0] + + +class TestDeferredFromCoro: + def test_deferred(self): + d = Deferred() + result = deferred_from_coro(d) + assert isinstance(result, Deferred) + assert result is d + + def test_object(self): + result = deferred_from_coro(42) + assert result == 42 + + @inline_callbacks_test + def test_coroutine(self): + async def coroutine() -> int: + return 42 + + result = deferred_from_coro(coroutine()) + assert isinstance(result, Deferred) + coro_result = yield result + assert coro_result == 42 + + @pytest.mark.only_asyncio + @inline_callbacks_test + def test_coroutine_asyncio(self): + async def coroutine() -> int: + await asyncio.sleep(0.01) + return 42 + + result = deferred_from_coro(coroutine()) + assert isinstance(result, Deferred) + coro_result = yield result + assert coro_result == 42 + + @pytest.mark.only_asyncio + @inline_callbacks_test + def test_future(self): + future = Future() + result = deferred_from_coro(future) + assert isinstance(result, Deferred) + future.set_result(42) + future_result = yield result + assert future_result == 42 + + +class TestDeferredFFromCoroF: + @inlineCallbacks + def _assert_result( + self, c_f: Callable[[], Awaitable[int]] + ) -> Generator[Deferred[Any], Any, None]: + d_f = deferred_f_from_coro_f(c_f) + d = d_f() + assert isinstance(d, Deferred) + result = yield d + assert result == 42 + + @inline_callbacks_test + def test_coroutine(self): + async def c_f() -> int: + return 42 + + yield self._assert_result(c_f) + + @pytest.mark.only_asyncio + @inline_callbacks_test + def test_coroutine_asyncio(self): + async def c_f() -> int: + await asyncio.sleep(0.01) + return 42 + + yield self._assert_result(c_f) + + @pytest.mark.only_asyncio + @inline_callbacks_test + def test_future(self): + def c_f() -> Future[int]: + f: Future[int] = Future() + f.set_result(42) + return f + + yield self._assert_result(c_f) + + +@pytest.mark.only_asyncio +class TestDeferredToFuture: + @coroutine_test + async def test_deferred(self): + d = Deferred() + result = deferred_to_future(d) + assert isinstance(result, Future) + d.callback(42) + future_result = await result + assert future_result == 42 + + @coroutine_test + async def test_wrapped_coroutine(self): + async def c_f() -> int: + return 42 + + d = deferred_from_coro(c_f()) + result = deferred_to_future(d) + assert isinstance(result, Future) + future_result = await result + assert future_result == 42 + + @coroutine_test + async def test_wrapped_coroutine_asyncio(self): + async def c_f() -> int: + await asyncio.sleep(0.01) + return 42 + + d = deferred_from_coro(c_f()) + result = deferred_to_future(d) + assert isinstance(result, Future) + future_result = await result + assert future_result == 42 + + +@pytest.mark.only_asyncio +class TestMaybeDeferredToFutureAsyncio: + @coroutine_test + async def test_deferred(self): + d = Deferred() + result = maybe_deferred_to_future(d) + assert isinstance(result, Future) + d.callback(42) + future_result = await result + assert future_result == 42 + + @coroutine_test + async def test_wrapped_coroutine(self): + async def c_f() -> int: + return 42 + + d = deferred_from_coro(c_f()) + result = maybe_deferred_to_future(d) + assert isinstance(result, Future) + future_result = await result + assert future_result == 42 + + @coroutine_test + async def test_wrapped_coroutine_asyncio(self): + async def c_f() -> int: + await asyncio.sleep(0.01) + return 42 + + d = deferred_from_coro(c_f()) + result = maybe_deferred_to_future(d) + assert isinstance(result, Future) + future_result = await result + assert future_result == 42 + + +@pytest.mark.only_not_asyncio +class TestMaybeDeferredToFutureNotAsyncio: + @coroutine_test + async def test_deferred(self): + d = Deferred() + result = maybe_deferred_to_future(d) + assert isinstance(result, Deferred) + assert result is d diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index eedb6f6af..c5425d99d 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,7 +1,9 @@ import inspect -import unittest import warnings from unittest import mock +from warnings import WarningMessage + +import pytest from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.deprecate import create_deprecated_class, update_classpath @@ -19,8 +21,10 @@ class NewName(SomeBaseClass): pass -class WarnWhenSubclassedTest(unittest.TestCase): - def _mywarnings(self, w, category=MyWarning): +class TestWarnWhenSubclassed: + def _mywarnings( + self, w: list[WarningMessage], category: type[Warning] = MyWarning + ) -> list[WarningMessage]: return [x for x in w if x.category is MyWarning] def test_no_warning_on_definition(self): @@ -28,7 +32,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): create_deprecated_class("Deprecated", NewName) w = self._mywarnings(w) - self.assertEqual(w, []) + assert w == [] def test_subclassing_warning_message(self): Deprecated = create_deprecated_class( @@ -41,15 +45,14 @@ class WarnWhenSubclassedTest(unittest.TestCase): pass w = self._mywarnings(w) - self.assertEqual(len(w), 1) - self.assertEqual( - str(w[0].message), - "tests.test_utils_deprecate.UserClass inherits from " + assert len(w) == 1 + assert ( + str(w[0].message) == "tests.test_utils_deprecate.UserClass inherits from " "deprecated class tests.test_utils_deprecate.Deprecated, " "please inherit from tests.test_utils_deprecate.NewName." - " (warning only on first subclass, there may be others)", + " (warning only on first subclass, there may be others)" ) - self.assertEqual(w[0].lineno, inspect.getsourcelines(UserClass)[1]) + assert w[0].lineno == inspect.getsourcelines(UserClass)[1] def test_custom_class_paths(self): Deprecated = create_deprecated_class( @@ -68,11 +71,11 @@ class WarnWhenSubclassedTest(unittest.TestCase): _ = Deprecated() w = self._mywarnings(w) - self.assertEqual(len(w), 2) - self.assertIn("foo.NewClass", str(w[0].message)) - self.assertIn("bar.OldClass", str(w[0].message)) - self.assertIn("foo.NewClass", str(w[1].message)) - self.assertIn("bar.OldClass", str(w[1].message)) + assert len(w) == 2 + assert "foo.NewClass" in str(w[0].message) + assert "bar.OldClass" in str(w[0].message) + assert "foo.NewClass" in str(w[1].message) + assert "bar.OldClass" in str(w[1].message) def test_subclassing_warns_only_on_direct_children(self): Deprecated = create_deprecated_class( @@ -88,8 +91,8 @@ class WarnWhenSubclassedTest(unittest.TestCase): pass w = self._mywarnings(w) - self.assertEqual(len(w), 1) - self.assertIn("UserClass", str(w[0].message)) + assert len(w) == 1 + assert "UserClass" in str(w[0].message) def test_subclassing_warns_once_by_default(self): Deprecated = create_deprecated_class( @@ -108,8 +111,8 @@ class WarnWhenSubclassedTest(unittest.TestCase): pass w = self._mywarnings(w) - self.assertEqual(len(w), 1) - self.assertIn("UserClass", str(w[0].message)) + assert len(w) == 1 + assert "UserClass" in str(w[0].message) def test_warning_on_instance(self): Deprecated = create_deprecated_class( @@ -128,13 +131,12 @@ class WarnWhenSubclassedTest(unittest.TestCase): _ = UserClass() # subclass instances don't warn w = self._mywarnings(w) - self.assertEqual(len(w), 1) - self.assertEqual( - str(w[0].message), - "tests.test_utils_deprecate.Deprecated is deprecated, " - "instantiate tests.test_utils_deprecate.NewName instead.", + assert len(w) == 1 + assert ( + str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, " + "instantiate tests.test_utils_deprecate.NewName instead." ) - self.assertEqual(w[0].lineno, lineno) + assert w[0].lineno == lineno def test_warning_auto_message(self): with warnings.catch_warnings(record=True) as w: @@ -144,8 +146,8 @@ class WarnWhenSubclassedTest(unittest.TestCase): pass msg = str(w[0].message) - self.assertIn("tests.test_utils_deprecate.NewName", msg) - self.assertIn("tests.test_utils_deprecate.Deprecated", msg) + assert "tests.test_utils_deprecate.NewName" in msg + assert "tests.test_utils_deprecate.Deprecated" in msg def test_issubclass(self): with warnings.catch_warnings(): @@ -167,21 +169,17 @@ class WarnWhenSubclassedTest(unittest.TestCase): class UnrelatedClass: pass - class OldStyleClass: - pass - assert issubclass(UpdatedUserClass1, NewName) assert issubclass(UpdatedUserClass1a, NewName) assert issubclass(UpdatedUserClass1, DeprecatedName) assert issubclass(UpdatedUserClass1a, DeprecatedName) assert issubclass(OutdatedUserClass1, DeprecatedName) assert not issubclass(UnrelatedClass, DeprecatedName) - assert not issubclass(OldStyleClass, DeprecatedName) - assert not issubclass(OldStyleClass, DeprecatedName) assert not issubclass(OutdatedUserClass1, OutdatedUserClass1a) assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1) - self.assertRaises(TypeError, issubclass, object(), DeprecatedName) + with pytest.raises(TypeError): + issubclass(object(), DeprecatedName) def test_isinstance(self): with warnings.catch_warnings(): @@ -203,9 +201,6 @@ class WarnWhenSubclassedTest(unittest.TestCase): class UnrelatedClass: pass - class OldStyleClass: - pass - assert isinstance(UpdatedUserClass2(), NewName) assert isinstance(UpdatedUserClass2a(), NewName) assert isinstance(UpdatedUserClass2(), DeprecatedName) @@ -215,14 +210,13 @@ class WarnWhenSubclassedTest(unittest.TestCase): assert not isinstance(OutdatedUserClass2a(), OutdatedUserClass2) assert not isinstance(OutdatedUserClass2(), OutdatedUserClass2a) assert not isinstance(UnrelatedClass(), DeprecatedName) - assert not isinstance(OldStyleClass(), DeprecatedName) def test_clsdict(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", ScrapyDeprecationWarning) Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"}) - self.assertEqual(Deprecated.foo, "bar") + assert Deprecated.foo == "bar" def test_deprecate_a_class_with_custom_metaclass(self): Meta1 = type("Meta1", (type,), {}) @@ -243,7 +237,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): ) w = self._mywarnings(w) - self.assertEqual(len(w), 0, str(map(str, w))) + assert len(w) == 0, [str(warning) for warning in w] with warnings.catch_warnings(record=True) as w: AlsoDeprecated() @@ -252,21 +246,23 @@ class WarnWhenSubclassedTest(unittest.TestCase): pass w = self._mywarnings(w) - self.assertEqual(len(w), 2) - self.assertIn("AlsoDeprecated", str(w[0].message)) - self.assertIn("foo.Bar", str(w[0].message)) - self.assertIn("AlsoDeprecated", str(w[1].message)) - self.assertIn("foo.Bar", str(w[1].message)) + assert len(w) == 2 + assert "AlsoDeprecated" in str(w[0].message) + assert "foo.Bar" in str(w[0].message) + assert "AlsoDeprecated" in str(w[1].message) + assert "foo.Bar" in str(w[1].message) def test_inspect_stack(self): - with mock.patch("inspect.stack", side_effect=IndexError): - with warnings.catch_warnings(record=True) as w: - DeprecatedName = create_deprecated_class("DeprecatedName", NewName) + with ( + mock.patch("inspect.stack", side_effect=IndexError), + warnings.catch_warnings(record=True) as w, + ): + DeprecatedName = create_deprecated_class("DeprecatedName", NewName) - class SubClass(DeprecatedName): - pass + class SubClass(DeprecatedName): + pass - self.assertIn("Error detecting parent module", str(w[0].message)) + assert "Error detecting parent module" in str(w[0].message) @mock.patch( @@ -276,27 +272,27 @@ class WarnWhenSubclassedTest(unittest.TestCase): ("scrapy.contrib.", "scrapy.extensions."), ], ) -class UpdateClassPathTest(unittest.TestCase): +class TestUpdateClassPath: def test_old_path_gets_fixed(self): with warnings.catch_warnings(record=True) as w: output = update_classpath("scrapy.contrib.debug.Debug") - self.assertEqual(output, "scrapy.extensions.debug.Debug") - self.assertEqual(len(w), 1) - self.assertIn("scrapy.contrib.debug.Debug", str(w[0].message)) - self.assertIn("scrapy.extensions.debug.Debug", str(w[0].message)) + assert output == "scrapy.extensions.debug.Debug" + assert len(w) == 1 + assert "scrapy.contrib.debug.Debug" in str(w[0].message) + assert "scrapy.extensions.debug.Debug" in str(w[0].message) def test_sorted_replacement(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", ScrapyDeprecationWarning) output = update_classpath("scrapy.contrib.pipeline.Pipeline") - self.assertEqual(output, "scrapy.pipelines.Pipeline") + assert output == "scrapy.pipelines.Pipeline" def test_unmatched_path_stays_the_same(self): with warnings.catch_warnings(record=True) as w: output = update_classpath("scrapy.unmatched.Path") - self.assertEqual(output, "scrapy.unmatched.Path") - self.assertEqual(len(w), 0) + assert output == "scrapy.unmatched.Path" + assert len(w) == 0 def test_returns_nonstring(self): for notastring in [None, True, [1, 2, 3], object()]: - self.assertEqual(update_classpath(notastring), notastring) + assert update_classpath(notastring) == notastring diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index d1bf64828..8b02116b8 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -1,90 +1,93 @@ +import builtins from io import StringIO -from unittest import TestCase, mock +from unittest import mock from scrapy.utils.display import pformat, pprint - -class TestDisplay(TestCase): - object = {"a": 1} - colorized_strings = { +value = {"a": 1} +colorized_strings = { + ( ( - ( - "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'" - "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}" - ) - + suffix + "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'" + "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}" ) - for suffix in ( - # https://github.com/pygments/pygments/issues/2313 - "\n", # pygments ≤ 2.13 - "\x1b[37m\x1b[39;49;00m\n", # pygments ≥ 2.14 - ) - } - plain_string = "{'a': 1}" + + suffix + ) + for suffix in ( + # https://github.com/pygments/pygments/issues/2313 + "\n", # pygments ≤ 2.13 + "\x1b[37m\x1b[39;49;00m\n", # pygments ≥ 2.14 + ) +} +plain_string = "{'a': 1}" - @mock.patch("sys.platform", "linux") - @mock.patch("sys.stdout.isatty") - def test_pformat(self, isatty): - isatty.return_value = True - self.assertIn(pformat(self.object), self.colorized_strings) - @mock.patch("sys.stdout.isatty") - def test_pformat_dont_colorize(self, isatty): - isatty.return_value = True - self.assertEqual(pformat(self.object, colorize=False), self.plain_string) +@mock.patch("sys.platform", "linux") +@mock.patch("sys.stdout.isatty") +def test_pformat(isatty): + isatty.return_value = True + assert pformat(value) in colorized_strings - def test_pformat_not_tty(self): - self.assertEqual(pformat(self.object), self.plain_string) - @mock.patch("sys.platform", "win32") - @mock.patch("platform.version") - @mock.patch("sys.stdout.isatty") - def test_pformat_old_windows(self, isatty, version): - isatty.return_value = True - version.return_value = "10.0.14392" - self.assertIn(pformat(self.object), self.colorized_strings) +@mock.patch("sys.stdout.isatty") +def test_pformat_dont_colorize(isatty): + isatty.return_value = True + assert pformat(value, colorize=False) == plain_string - @mock.patch("sys.platform", "win32") - @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") - @mock.patch("platform.version") - @mock.patch("sys.stdout.isatty") - def test_pformat_windows_no_terminal_processing( - self, isatty, version, terminal_processing - ): - isatty.return_value = True - version.return_value = "10.0.14393" - terminal_processing.return_value = False - self.assertEqual(pformat(self.object), self.plain_string) - @mock.patch("sys.platform", "win32") - @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") - @mock.patch("platform.version") - @mock.patch("sys.stdout.isatty") - def test_pformat_windows(self, isatty, version, terminal_processing): - isatty.return_value = True - version.return_value = "10.0.14393" - terminal_processing.return_value = True - self.assertIn(pformat(self.object), self.colorized_strings) +def test_pformat_not_tty(): + assert pformat(value) == plain_string - @mock.patch("sys.platform", "linux") - @mock.patch("sys.stdout.isatty") - def test_pformat_no_pygments(self, isatty): - isatty.return_value = True - import builtins +@mock.patch("sys.platform", "win32") +@mock.patch("platform.version") +@mock.patch("sys.stdout.isatty") +def test_pformat_old_windows(isatty, version): + isatty.return_value = True + version.return_value = "10.0.14392" + assert pformat(value) in colorized_strings - real_import = builtins.__import__ - def mock_import(name, globals, locals, fromlist, level): - if "pygments" in name: - raise ImportError - return real_import(name, globals, locals, fromlist, level) +@mock.patch("sys.platform", "win32") +@mock.patch("scrapy.utils.display._enable_windows_terminal_processing") +@mock.patch("platform.version") +@mock.patch("sys.stdout.isatty") +def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing): + isatty.return_value = True + version.return_value = "10.0.14393" + terminal_processing.return_value = False + assert pformat(value) == plain_string - builtins.__import__ = mock_import - self.assertEqual(pformat(self.object), self.plain_string) - builtins.__import__ = real_import - def test_pprint(self): - with mock.patch("sys.stdout", new=StringIO()) as mock_out: - pprint(self.object) - self.assertEqual(mock_out.getvalue(), "{'a': 1}\n") +@mock.patch("sys.platform", "win32") +@mock.patch("scrapy.utils.display._enable_windows_terminal_processing") +@mock.patch("platform.version") +@mock.patch("sys.stdout.isatty") +def test_pformat_windows(isatty, version, terminal_processing): + isatty.return_value = True + version.return_value = "10.0.14393" + terminal_processing.return_value = True + assert pformat(value) in colorized_strings + + +@mock.patch("sys.platform", "linux") +@mock.patch("sys.stdout.isatty") +def test_pformat_no_pygments(isatty): + isatty.return_value = True + + real_import = builtins.__import__ + + def mock_import(name, globals_, locals_, fromlist, level): + if "pygments" in name: + raise ImportError + return real_import(name, globals_, locals_, fromlist, level) + + builtins.__import__ = mock_import + assert pformat(value) == plain_string + builtins.__import__ = real_import + + +def test_pprint(): + with mock.patch("sys.stdout", new=StringIO()) as mock_out: + pprint(value) + assert mock_out.getvalue() == "{'a': 1}\n" diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 7b7a25db8..06fdf9cba 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -1,6 +1,7 @@ -import unittest +from gzip import BadGzipFile from pathlib import Path +import pytest from w3lib.encoding import html_to_unicode from scrapy.http import Response @@ -10,48 +11,51 @@ from tests import tests_datadir SAMPLEDIR = Path(tests_datadir, "compressed") -class GunzipTest(unittest.TestCase): - def test_gunzip_basic(self): - r1 = Response( - "http://www.example.com", - body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(), - ) - self.assertTrue(gzip_magic_number(r1)) +def test_gunzip_basic(): + r1 = Response( + "http://www.example.com", + body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(), + ) + assert gzip_magic_number(r1) - r2 = Response("http://www.example.com", body=gunzip(r1.body)) - self.assertFalse(gzip_magic_number(r2)) - self.assertEqual(len(r2.body), 9950) + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + assert not gzip_magic_number(r2) + assert len(r2.body) == 9950 - def test_gunzip_truncated(self): - text = gunzip((SAMPLEDIR / "truncated-crc-error.gz").read_bytes()) - assert text.endswith(b"") - self.assertFalse(gzip_magic_number(r2)) +def test_gunzip_no_gzip_file_raises(): + with pytest.raises(BadGzipFile): + gunzip((SAMPLEDIR / "feed-sample1.xml").read_bytes()) - def test_is_gzipped_empty(self): - r1 = Response("http://www.example.com") - self.assertFalse(gzip_magic_number(r1)) - def test_gunzip_illegal_eof(self): - text = html_to_unicode( - "charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes()) - )[1] - expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text( - encoding="utf-8" - ) - self.assertEqual(len(text), len(expected_text)) - self.assertEqual(text, expected_text) +def test_gunzip_truncated_short(): + r1 = Response( + "http://www.example.com", + body=(SAMPLEDIR / "truncated-crc-error-short.gz").read_bytes(), + ) + assert gzip_magic_number(r1) + + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + assert r2.body.endswith(b"") + assert not gzip_magic_number(r2) + + +def test_is_gzipped_empty(): + r1 = Response("http://www.example.com") + assert not gzip_magic_number(r1) + + +def test_gunzip_illegal_eof(): + text = html_to_unicode( + "charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes()) + )[1] + expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text( + encoding="utf-8" + ) + assert len(text) == len(expected_text) + assert text == expected_text diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index b824972d5..9bd86f7fb 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -1,26 +1,20 @@ -import unittest from urllib.parse import urlparse from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached -class HttpobjUtilsTest(unittest.TestCase): - def test_urlparse_cached(self): - url = "http://www.example.com/index.html" - request1 = Request(url) - request2 = Request(url) - req1a = urlparse_cached(request1) - req1b = urlparse_cached(request1) - req2 = urlparse_cached(request2) - urlp = urlparse(url) +def test_urlparse_cached(): + url = "http://www.example.com/index.html" + request1 = Request(url) + request2 = Request(url) + req1a = urlparse_cached(request1) + req1b = urlparse_cached(request1) + req2 = urlparse_cached(request2) + urlp = urlparse(url) - assert req1a == req2 - assert req1a == urlp - assert req1a is req1b - assert req1a is not req2 - assert req1a is not req2 - - -if __name__ == "__main__": - unittest.main() + assert req1a == req2 + assert req1a == urlp + assert req1a is req1b + assert req1a is not req2 + assert req1a is not req2 diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 4a0c34d82..1276c1392 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,13 +1,27 @@ -from pytest import mark -from twisted.trial import unittest +from __future__ import annotations +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any + +import pytest + +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse, XmlResponse from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml from tests import get_testdata +if TYPE_CHECKING: + from collections.abc import Iterator -class XmliterTestCase(unittest.TestCase): - xmliter = staticmethod(xmliter) + from scrapy import Selector + + +class TestXmliterBase(ABC): + @abstractmethod + def xmliter( + self, obj: Response | str | bytes, nodename: str, *args: Any + ) -> Iterator[Selector]: + raise NotImplementedError def test_xmliter(self): body = b""" @@ -26,19 +40,19 @@ class XmliterTestCase(unittest.TestCase): """ response = XmlResponse(url="http://example.com", body=body) - attrs = [] - for x in self.xmliter(response, "product"): - attrs.append( - ( - x.attrib["id"], - x.xpath("name/text()").getall(), - x.xpath("./type/text()").getall(), - ) + attrs = [ + ( + x.attrib["id"], + x.xpath("name/text()").getall(), + x.xpath("./type/text()").getall(), ) + for x in self.xmliter(response, "product") + ] - self.assertEqual( - attrs, [("001", ["Name 1"], ["Type 1"]), ("002", ["Name 2"], ["Type 2"])] - ) + assert attrs == [ + ("001", ["Name 1"], ["Type 1"]), + ("002", ["Name 2"], ["Type 2"]), + ] def test_xmliter_unusual_node(self): body = b""" @@ -51,7 +65,7 @@ class XmliterTestCase(unittest.TestCase): nodenames = [ e.xpath("name()").getall() for e in self.xmliter(response, "matchme...") ] - self.assertEqual(nodenames, [["matchme..."]]) + assert nodenames == [["matchme..."]] def test_xmliter_unicode(self): # example taken from https://github.com/scrapy/scrapy/issues/1665 @@ -97,20 +111,20 @@ class XmliterTestCase(unittest.TestCase): # Unicode body needs encoding information XmlResponse(url="http://example.com", body=body, encoding="utf-8"), ): - attrs = [] - for x in self.xmliter(r, "þingflokkur"): - attrs.append( - ( - x.attrib["id"], - x.xpath("./skammstafanir/stuttskammstöfun/text()").getall(), - x.xpath("./tímabil/fyrstaþing/text()").getall(), - ) + attrs = [ + ( + x.attrib["id"], + x.xpath("./skammstafanir/stuttskammstöfun/text()").getall(), + x.xpath("./tímabil/fyrstaþing/text()").getall(), ) + for x in self.xmliter(r, "þingflokkur") + ] - self.assertEqual( - attrs, - [("26", ["-"], ["80"]), ("21", ["Ab"], ["76"]), ("27", ["A"], ["27"])], - ) + assert attrs == [ + ("26", ["-"], ["80"]), + ("21", ["Ab"], ["76"]), + ("27", ["A"], ["27"]), + ] def test_xmliter_text(self): body = ( @@ -118,10 +132,10 @@ class XmliterTestCase(unittest.TestCase): "onetwo" ) - self.assertEqual( - [x.xpath("text()").getall() for x in self.xmliter(body, "product")], - [["one"], ["two"]], - ) + assert [x.xpath("text()").getall() for x in self.xmliter(body, "product")] == [ + ["one"], + ["two"], + ] def test_xmliter_namespaces(self): body = b""" @@ -146,21 +160,19 @@ class XmliterTestCase(unittest.TestCase): my_iter = self.xmliter(response, "item") node = next(my_iter) node.register_namespace("g", "http://base.google.com/ns/1.0") - self.assertEqual(node.xpath("title/text()").getall(), ["Item 1"]) - self.assertEqual(node.xpath("description/text()").getall(), ["This is item 1"]) - self.assertEqual( - node.xpath("link/text()").getall(), - ["http://www.mydummycompany.com/items/1"], - ) - self.assertEqual( - node.xpath("g:image_link/text()").getall(), - ["http://www.mydummycompany.com/images/item1.jpg"], - ) - self.assertEqual(node.xpath("g:id/text()").getall(), ["ITEM_1"]) - self.assertEqual(node.xpath("g:price/text()").getall(), ["400"]) - self.assertEqual(node.xpath("image_link/text()").getall(), []) - self.assertEqual(node.xpath("id/text()").getall(), []) - self.assertEqual(node.xpath("price/text()").getall(), []) + assert node.xpath("title/text()").getall() == ["Item 1"] + assert node.xpath("description/text()").getall() == ["This is item 1"] + assert node.xpath("link/text()").getall() == [ + "http://www.mydummycompany.com/items/1" + ] + assert node.xpath("g:image_link/text()").getall() == [ + "http://www.mydummycompany.com/images/item1.jpg" + ] + assert node.xpath("g:id/text()").getall() == ["ITEM_1"] + assert node.xpath("g:price/text()").getall() == ["400"] + assert node.xpath("image_link/text()").getall() == [] + assert node.xpath("id/text()").getall() == [] + assert node.xpath("price/text()").getall() == [] def test_xmliter_namespaced_nodename(self): body = b""" @@ -185,10 +197,9 @@ class XmliterTestCase(unittest.TestCase): my_iter = self.xmliter(response, "g:image_link") node = next(my_iter) node.register_namespace("g", "http://base.google.com/ns/1.0") - self.assertEqual( - node.xpath("text()").extract(), - ["http://www.mydummycompany.com/images/item1.jpg"], - ) + assert node.xpath("text()").extract() == [ + "http://www.mydummycompany.com/images/item1.jpg" + ] def test_xmliter_namespaced_nodename_missing(self): body = b""" @@ -211,7 +222,7 @@ class XmliterTestCase(unittest.TestCase): """ response = XmlResponse(url="http://mydummycompany.com", body=body) my_iter = self.xmliter(response, "g:link_image") - with self.assertRaises(StopIteration): + with pytest.raises(StopIteration): next(my_iter) def test_xmliter_exception(self): @@ -220,15 +231,16 @@ class XmliterTestCase(unittest.TestCase): "onetwo" ) - iter = self.xmliter(body, "product") - next(iter) - next(iter) - - self.assertRaises(StopIteration, next, iter) + my_iter = self.xmliter(body, "product") + next(my_iter) + next(my_iter) + with pytest.raises(StopIteration): + next(my_iter) def test_xmliter_objtype_exception(self): i = self.xmliter(42, "product") - self.assertRaises(TypeError, next, i) + with pytest.raises(TypeError): + next(i) def test_xmliter_encoding(self): body = ( @@ -238,18 +250,38 @@ class XmliterTestCase(unittest.TestCase): b"\n\n" ) response = XmlResponse("http://www.example.com", body=body) - self.assertEqual( - next(self.xmliter(response, "item")).get(), - "Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6", + assert ( + next(self.xmliter(response, "item")).get() + == "Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6" ) -class LxmlXmliterTestCase(XmliterTestCase): - xmliter = staticmethod(xmliter_lxml) +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class TestXmliter(TestXmliterBase): + def xmliter( + self, obj: Response | str | bytes, nodename: str, *args: Any + ) -> Iterator[Selector]: + return xmliter(obj, nodename) - @mark.xfail(reason="known bug of the current implementation") - def test_xmliter_namespaced_nodename(self): - super().test_xmliter_namespaced_nodename() + def test_deprecation(self): + body = b""" + + + + + """ + with pytest.warns( + ScrapyDeprecationWarning, + match="xmliter", + ): + next(self.xmliter(body, "product")) + + +class TestLxmlXmliter(TestXmliterBase): + def xmliter( + self, obj: Response | str | bytes, nodename: str, *args: Any + ) -> Iterator[Selector]: + return xmliter_lxml(obj, nodename, *args) def test_xmliter_iterate_namespace(self): body = b""" @@ -272,21 +304,19 @@ class LxmlXmliterTestCase(XmliterTestCase): response = XmlResponse(url="http://mydummycompany.com", body=body) no_namespace_iter = self.xmliter(response, "image_link") - self.assertEqual(len(list(no_namespace_iter)), 0) + assert len(list(no_namespace_iter)) == 0 namespace_iter = self.xmliter( response, "image_link", "http://base.google.com/ns/1.0" ) node = next(namespace_iter) - self.assertEqual( - node.xpath("text()").getall(), - ["http://www.mydummycompany.com/images/item1.jpg"], - ) + assert node.xpath("text()").getall() == [ + "http://www.mydummycompany.com/images/item1.jpg" + ] node = next(namespace_iter) - self.assertEqual( - node.xpath("text()").getall(), - ["http://www.mydummycompany.com/images/item2.jpg"], - ) + assert node.xpath("text()").getall() == [ + "http://www.mydummycompany.com/images/item2.jpg" + ] def test_xmliter_namespaces_prefix(self): body = b""" @@ -311,58 +341,53 @@ class LxmlXmliterTestCase(XmliterTestCase): my_iter = self.xmliter(response, "table", "http://www.w3.org/TR/html4/", "h") node = next(my_iter) - self.assertEqual(len(node.xpath("h:tr/h:td").getall()), 2) - self.assertEqual(node.xpath("h:tr/h:td[1]/text()").getall(), ["Apples"]) - self.assertEqual(node.xpath("h:tr/h:td[2]/text()").getall(), ["Bananas"]) + assert len(node.xpath("h:tr/h:td").getall()) == 2 + assert node.xpath("h:tr/h:td[1]/text()").getall() == ["Apples"] + assert node.xpath("h:tr/h:td[2]/text()").getall() == ["Bananas"] my_iter = self.xmliter( response, "table", "http://www.w3schools.com/furniture", "f" ) node = next(my_iter) - self.assertEqual(node.xpath("f:name/text()").getall(), ["African Coffee Table"]) + assert node.xpath("f:name/text()").getall() == ["African Coffee Table"] def test_xmliter_objtype_exception(self): i = self.xmliter(42, "product") - self.assertRaises(TypeError, next, i) + with pytest.raises(TypeError): + next(i) -class UtilsCsvTestCase(unittest.TestCase): +class TestUtilsCsv: def test_csviter_defaults(self): body = get_testdata("feeds", "feed-sample3.csv") response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) - result = [row for row in csv] - self.assertEqual( - result, - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + result = list(csv) + assert result == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assertTrue(all(isinstance(k, str) for k in result_row.keys())) - self.assertTrue(all(isinstance(v, str) for v in result_row.values())) + assert all(isinstance(k, str) for k in result_row) + assert all(isinstance(v, str) for v in result_row.values()) def test_csviter_delimiter(self): body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t") response = TextResponse(url="http://example.com/", body=body) csv = csviter(response, delimiter="\t") - self.assertEqual( - [row for row in csv], - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] def test_csviter_quotechar(self): body1 = get_testdata("feeds", "feed-sample6.csv") @@ -371,62 +396,50 @@ class UtilsCsvTestCase(unittest.TestCase): response1 = TextResponse(url="http://example.com/", body=body1) csv1 = csviter(response1, quotechar="'") - self.assertEqual( - [row for row in csv1], - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv1) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] response2 = TextResponse(url="http://example.com/", body=body2) csv2 = csviter(response2, delimiter="|", quotechar="'") - self.assertEqual( - [row for row in csv2], - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv2) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] def test_csviter_wrong_quotechar(self): body = get_testdata("feeds", "feed-sample6.csv") response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) - self.assertEqual( - [row for row in csv], - [ - {"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"}, - { - "'id'": "2", - "'name'": "'unicode'", - "'value'": "'\xfan\xedc\xf3d\xe9\u203d'", - }, - {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"}, - {"'id'": "4", "'name'": "'empty'", "'value'": ""}, - ], - ) + assert list(csv) == [ + {"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"}, + { + "'id'": "2", + "'name'": "'unicode'", + "'value'": "'\xfan\xedc\xf3d\xe9\u203d'", + }, + {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"}, + {"'id'": "4", "'name'": "'empty'", "'value'": ""}, + ] def test_csviter_delimiter_binary_response_assume_utf8_encoding(self): body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t") response = Response(url="http://example.com/", body=body) csv = csviter(response, delimiter="\t") - self.assertEqual( - [row for row in csv], - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] def test_csviter_headers(self): sample = get_testdata("feeds", "feed-sample3.csv").splitlines() @@ -435,15 +448,12 @@ class UtilsCsvTestCase(unittest.TestCase): response = TextResponse(url="http://example.com/", body=body) csv = csviter(response, headers=[h.decode("utf-8") for h in headers]) - self.assertEqual( - [row for row in csv], - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] def test_csviter_falserow(self): body = get_testdata("feeds", "feed-sample3.csv") @@ -452,27 +462,24 @@ class UtilsCsvTestCase(unittest.TestCase): response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) - self.assertEqual( - [row for row in csv], - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] def test_csviter_exception(self): body = get_testdata("feeds", "feed-sample3.csv") response = TextResponse(url="http://example.com/", body=body) - iter = csviter(response) - next(iter) - next(iter) - next(iter) - next(iter) - - self.assertRaises(StopIteration, next, iter) + my_iter = csviter(response) + next(my_iter) + next(my_iter) + next(my_iter) + next(my_iter) + with pytest.raises(StopIteration): + next(my_iter) def test_csviter_encoding(self): body1 = get_testdata("feeds", "feed-sample4.csv") @@ -482,48 +489,49 @@ class UtilsCsvTestCase(unittest.TestCase): url="http://example.com/", body=body1, encoding="latin1" ) csv = csviter(response) - self.assertEqual( - list(csv), - [ - {"id": "1", "name": "latin1", "value": "test"}, - {"id": "2", "name": "something", "value": "\xf1\xe1\xe9\xf3"}, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "latin1", "value": "test"}, + {"id": "2", "name": "something", "value": "\xf1\xe1\xe9\xf3"}, + ] response = TextResponse(url="http://example.com/", body=body2, encoding="cp852") csv = csviter(response) - self.assertEqual( - list(csv), - [ - {"id": "1", "name": "cp852", "value": "test"}, - { - "id": "2", - "name": "something", - "value": "\u255a\u2569\u2569\u2569\u2550\u2550\u2557", - }, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "cp852", "value": "test"}, + { + "id": "2", + "name": "something", + "value": "\u255a\u2569\u2569\u2569\u2550\u2550\u2557", + }, + ] -class TestHelper(unittest.TestCase): +class TestBodyOrStr: bbody = b"utf8-body" ubody = bbody.decode("utf8") - txtresponse = TextResponse(url="http://example.org/", body=bbody, encoding="utf-8") - response = Response(url="http://example.org/", body=bbody) - def test_body_or_str(self): - for obj in (self.bbody, self.ubody, self.txtresponse, self.response): - r1 = _body_or_str(obj) - self._assert_type_and_value(r1, self.ubody, obj) - r2 = _body_or_str(obj, unicode=True) - self._assert_type_and_value(r2, self.ubody, obj) - r3 = _body_or_str(obj, unicode=False) - self._assert_type_and_value(r3, self.bbody, obj) - self.assertTrue(type(r1) is type(r2)) - self.assertTrue(type(r1) is not type(r3)) + @pytest.mark.parametrize( + "obj", + [ + bbody, + ubody, + TextResponse(url="http://example.org/", body=bbody, encoding="utf-8"), + Response(url="http://example.org/", body=bbody), + ], + ) + def test_body_or_str(self, obj: Response | str | bytes) -> None: + r1 = _body_or_str(obj) + self._assert_type_and_value(r1, self.ubody, obj) + r2 = _body_or_str(obj, unicode=True) + self._assert_type_and_value(r2, self.ubody, obj) + r3 = _body_or_str(obj, unicode=False) + self._assert_type_and_value(r3, self.bbody, obj) + assert type(r1) is type(r2) + assert type(r1) is not type(r3) # type: ignore[comparison-overlap] - def _assert_type_and_value(self, a, b, obj): - self.assertTrue( - type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}" - ) - self.assertEqual(a, b) + @staticmethod + def _assert_type_and_value( + a: str | bytes, b: str | bytes, obj: Response | str | bytes + ) -> None: + assert type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}" + assert a == b diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index eae744df5..8e5020022 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,21 +1,33 @@ -import logging -import sys -import unittest +from __future__ import annotations +import json +import logging +import re +import sys +from io import StringIO +from typing import TYPE_CHECKING, Any + +import pytest from testfixtures import LogCapture from twisted.python.failure import Failure -from scrapy.extensions import telnet from scrapy.utils.log import ( LogCounterHandler, + SpiderLoggerAdapter, StreamLogger, TopLevelFormatter, failure_to_exc_info, ) from scrapy.utils.test import get_crawler +from tests.spiders import LogSpider + +if TYPE_CHECKING: + from collections.abc import Generator, Mapping, MutableMapping + + from scrapy.crawler import Crawler -class FailureToExcInfoTest(unittest.TestCase): +class TestFailureToExcInfo: def test_failure(self): try: 0 / 0 @@ -23,86 +35,279 @@ class FailureToExcInfoTest(unittest.TestCase): exc_info = sys.exc_info() failure = Failure() - self.assertTupleEqual(exc_info, failure_to_exc_info(failure)) + assert exc_info == failure_to_exc_info(failure) def test_non_failure(self): - self.assertIsNone(failure_to_exc_info("test")) + assert failure_to_exc_info("test") is None -class TopLevelFormatterTest(unittest.TestCase): - def setUp(self): - self.handler = LogCapture() - self.handler.addFilter(TopLevelFormatter(["test"])) - - def test_top_level_logger(self): +class TestTopLevelFormatter: + def test_top_level_logger(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.handler.addFilter(TopLevelFormatter(["test"])) logger = logging.getLogger("test") - with self.handler as log: - logger.warning("test log msg") - log.check(("test", "WARNING", "test log msg")) + logger.warning("test log msg") + assert ("test", logging.WARNING, "test log msg") in caplog.record_tuples - def test_children_logger(self): + def test_children_logger(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.handler.addFilter(TopLevelFormatter(["test"])) logger = logging.getLogger("test.test1") - with self.handler as log: - logger.warning("test log msg") - log.check(("test", "WARNING", "test log msg")) + logger.warning("test log msg") + assert ("test", logging.WARNING, "test log msg") in caplog.record_tuples - def test_overlapping_name_logger(self): + def test_overlapping_name_logger(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.handler.addFilter(TopLevelFormatter(["test"])) logger = logging.getLogger("test2") - with self.handler as log: - logger.warning("test log msg") - log.check(("test2", "WARNING", "test log msg")) + logger.warning("test log msg") + assert ("test2", logging.WARNING, "test log msg") in caplog.record_tuples - def test_different_name_logger(self): + def test_different_name_logger(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.handler.addFilter(TopLevelFormatter(["test"])) logger = logging.getLogger("different") - with self.handler as log: - logger.warning("test log msg") - log.check(("different", "WARNING", "test log msg")) + logger.warning("test log msg") + assert ("different", logging.WARNING, "test log msg") in caplog.record_tuples -class LogCounterHandlerTest(unittest.TestCase): - def setUp(self): +class TestLogCounterHandler: + @pytest.fixture + def crawler(self) -> Crawler: settings = {"LOG_LEVEL": "WARNING"} - if not telnet.TWISTED_CONCH_AVAILABLE: - # disable it to avoid the extra warning - settings["TELNETCONSOLE_ENABLED"] = False - self.logger = logging.getLogger("test") - self.logger.setLevel(logging.NOTSET) - self.logger.propagate = False - self.crawler = get_crawler(settings_dict=settings) - self.handler = LogCounterHandler(self.crawler) - self.logger.addHandler(self.handler) + return get_crawler(settings_dict=settings) - def tearDown(self): - self.logger.propagate = True - self.logger.removeHandler(self.handler) + @pytest.fixture + def logger(self, crawler: Crawler) -> Generator[logging.Logger]: + logger = logging.getLogger("test") + logger.setLevel(logging.NOTSET) + logger.propagate = False + handler = LogCounterHandler(crawler) + logger.addHandler(handler) - def test_init(self): - self.assertIsNone(self.crawler.stats.get_value("log_count/DEBUG")) - self.assertIsNone(self.crawler.stats.get_value("log_count/INFO")) - self.assertIsNone(self.crawler.stats.get_value("log_count/WARNING")) - self.assertIsNone(self.crawler.stats.get_value("log_count/ERROR")) - self.assertIsNone(self.crawler.stats.get_value("log_count/CRITICAL")) + yield logger - def test_accepted_level(self): - self.logger.error("test log msg") - self.assertEqual(self.crawler.stats.get_value("log_count/ERROR"), 1) + logger.propagate = True + logger.removeHandler(handler) - def test_filtered_out_level(self): - self.logger.debug("test log msg") - self.assertIsNone(self.crawler.stats.get_value("log_count/INFO")) + def test_init(self, crawler: Crawler, logger: logging.Logger) -> None: + assert crawler.stats + assert crawler.stats.get_value("log_count/DEBUG") is None + assert crawler.stats.get_value("log_count/INFO") is None + assert crawler.stats.get_value("log_count/WARNING") is None + assert crawler.stats.get_value("log_count/ERROR") is None + assert crawler.stats.get_value("log_count/CRITICAL") is None + + def test_accepted_level(self, crawler: Crawler, logger: logging.Logger) -> None: + logger.error("test log msg") + assert crawler.stats + assert crawler.stats.get_value("log_count/ERROR") == 1 + + def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None: + logger.debug("test log msg") + assert crawler.stats + assert crawler.stats.get_value("log_count/INFO") is None -class StreamLoggerTest(unittest.TestCase): - def setUp(self): - self.stdout = sys.stdout +class TestStreamLogger: + def test_redirect(self): logger = logging.getLogger("test") logger.setLevel(logging.WARNING) + old_stdout = sys.stdout sys.stdout = StreamLogger(logger, logging.ERROR) - def tearDown(self): - sys.stdout = self.stdout - - def test_redirect(self): with LogCapture() as log: print("test log msg") log.check(("test", "ERROR", "test log msg")) + + sys.stdout = old_stdout + + +@pytest.mark.parametrize( + ("base_extra", "log_extra", "expected_extra"), + [ + ( + {"spider": "test"}, + {"extra": {"log_extra": "info"}}, + {"extra": {"log_extra": "info", "spider": "test"}}, + ), + ( + {"spider": "test"}, + {"extra": None}, + {"extra": {"spider": "test"}}, + ), + ( + {"spider": "test"}, + {"extra": {"spider": "test2"}}, + {"extra": {"spider": "test"}}, + ), + ], +) +def test_spider_logger_adapter_process( + base_extra: Mapping[str, Any], + log_extra: MutableMapping[str, Any], + expected_extra: dict[str, Any], +) -> None: + logger = logging.getLogger("test") + spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra) + + log_message = "test_log_message" + result_message, result_kwargs = spider_logger_adapter.process( + log_message, log_extra + ) + + assert result_message == log_message + assert result_kwargs == expected_extra + + +class TestLogging: + @pytest.fixture + def log_stream(self) -> StringIO: + return StringIO() + + @pytest.fixture + def spider(self) -> LogSpider: + return LogSpider() + + @pytest.fixture(autouse=True) + def logger(self, log_stream: StringIO) -> Generator[logging.Logger]: + handler = logging.StreamHandler(log_stream) + logger = logging.getLogger("log_spider") + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + + yield logger + + logger.removeHandler(handler) + + def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None: + log_message = "Foo message" + spider.log_debug(log_message) + log_contents = log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None: + log_message = "Bar message" + spider.log_info(log_message) + log_contents = log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None: + log_message = "Baz message" + spider.log_warning(log_message) + log_contents = log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None: + log_message = "Foo bar message" + spider.log_error(log_message) + log_contents = log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None: + log_message = "Foo bar baz message" + spider.log_critical(log_message) + log_contents = log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + +class TestLoggingWithExtra: + regex_pattern = re.compile(r"^]+>$") + + @pytest.fixture + def log_stream(self) -> StringIO: + return StringIO() + + @pytest.fixture + def spider(self) -> LogSpider: + return LogSpider() + + @pytest.fixture(autouse=True) + def logger(self, log_stream: StringIO) -> Generator[logging.Logger]: + handler = logging.StreamHandler(log_stream) + formatter = logging.Formatter( + '{"levelname": "%(levelname)s", "message": "%(message)s", "spider": "%(spider)s", "important_info": "%(important_info)s"}' + ) + handler.setFormatter(formatter) + logger = logging.getLogger("log_spider") + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + + yield logger + + logger.removeHandler(handler) + + def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None: + log_message = "Foo message" + extra = {"important_info": "foo"} + spider.log_debug(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) + + assert log_contents["levelname"] == "DEBUG" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None: + log_message = "Bar message" + extra = {"important_info": "bar"} + spider.log_info(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) + + assert log_contents["levelname"] == "INFO" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None: + log_message = "Baz message" + extra = {"important_info": "baz"} + spider.log_warning(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) + + assert log_contents["levelname"] == "WARNING" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None: + log_message = "Foo bar message" + extra = {"important_info": "foo bar"} + spider.log_error(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) + + assert log_contents["levelname"] == "ERROR" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None: + log_message = "Foo bar baz message" + extra = {"important_info": "foo bar baz"} + spider.log_critical(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) + + assert log_contents["levelname"] == "CRITICAL" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_overwrite_spider_extra( + self, log_stream: StringIO, spider: LogSpider + ) -> None: + log_message = "Foo message" + extra = {"important_info": "foo", "spider": "shouldn't change"} + spider.log_error(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) + + assert log_contents["levelname"] == "ERROR" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index ee3314d8e..a995e38e6 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -1,79 +1,93 @@ import os import sys -import unittest from pathlib import Path from unittest import mock +import pytest + +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, build_from_crawler, - build_from_settings, - create_instance, load_object, rel_has_nofollow, set_environ, walk_modules, + walk_modules_iter, ) -__doctests__ = ["scrapy.utils.misc"] - -class UtilsMiscTestCase(unittest.TestCase): +class TestUtilsMisc: def test_load_object_class(self): obj = load_object(Field) - self.assertIs(obj, Field) + assert obj is Field obj = load_object("scrapy.item.Field") - self.assertIs(obj, Field) + assert obj is Field def test_load_object_function(self): obj = load_object(load_object) - self.assertIs(obj, load_object) + assert obj is load_object obj = load_object("scrapy.utils.misc.load_object") - self.assertIs(obj, load_object) + assert obj is load_object def test_load_object_exceptions(self): - self.assertRaises(ImportError, load_object, "nomodule999.mod.function") - self.assertRaises(NameError, load_object, "scrapy.utils.misc.load_object999") - self.assertRaises(TypeError, load_object, {}) + with pytest.raises(ImportError): + load_object("nomodule999.mod.function") + with pytest.raises(NameError): + load_object("scrapy.utils.misc.load_object999") + with pytest.raises(TypeError): + load_object({}) def test_walk_modules(self): - mods = walk_modules("tests.test_utils_misc.test_walk_modules") + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules") expected = [ "tests.test_utils_misc.test_walk_modules", "tests.test_utils_misc.test_walk_modules.mod", "tests.test_utils_misc.test_walk_modules.mod.mod0", "tests.test_utils_misc.test_walk_modules.mod1", ] - self.assertEqual({m.__name__ for m in mods}, set(expected)) + assert {m.__name__ for m in mods} == set(expected) - mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod") + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod") expected = [ "tests.test_utils_misc.test_walk_modules.mod", "tests.test_utils_misc.test_walk_modules.mod.mod0", ] - self.assertEqual({m.__name__ for m in mods}, set(expected)) + assert {m.__name__ for m in mods} == set(expected) - mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod1") + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1") expected = [ "tests.test_utils_misc.test_walk_modules.mod1", ] - self.assertEqual({m.__name__ for m in mods}, set(expected)) + assert {m.__name__ for m in mods} == set(expected) - self.assertRaises(ImportError, walk_modules, "nomodule999") + with pytest.raises(ImportError): + for _ in walk_modules_iter("nomodule999"): + pass + with ( + pytest.raises(ImportError), + pytest.warns( + ScrapyDeprecationWarning, + match="The scrapy.utils.misc.walk_modules function is deprecated and will be " + "removed in a future version of Scrapy. " + "Use scrapy.utils.misc.walk_modules_iter instead.", + ), + ): + walk_modules("nomodule999") def test_walk_modules_egg(self): egg = str(Path(__file__).parent / "test.egg") sys.path.append(egg) try: - mods = walk_modules("testegg") + mods = walk_modules_iter("testegg") expected = [ "testegg.spiders", "testegg.spiders.a", "testegg.spiders.b", "testegg", ] - self.assertEqual({m.__name__ for m in mods}, set(expected)) + assert {m.__name__ for m in mods} == set(expected) finally: sys.path.remove(egg) @@ -87,146 +101,45 @@ class UtilsMiscTestCase(unittest.TestCase): assert hasattr(arg_to_iter([1, 2, 3]), "__iter__") assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__") - self.assertEqual(list(arg_to_iter(None)), []) - self.assertEqual(list(arg_to_iter("lala")), ["lala"]) - self.assertEqual(list(arg_to_iter(100)), [100]) - self.assertEqual(list(arg_to_iter(c for c in "abc")), ["a", "b", "c"]) - self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3]) - self.assertEqual(list(arg_to_iter({"a": 1})), [{"a": 1}]) - self.assertEqual( - list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")] - ) - - def test_create_instance(self): - settings = mock.MagicMock() - crawler = mock.MagicMock(spec_set=["settings"]) - args = (True, 100.0) - kwargs = {"key": "val"} - - def _test_with_settings(mock, settings): - create_instance(mock, settings, None, *args, **kwargs) - if hasattr(mock, "from_crawler"): - self.assertEqual(mock.from_crawler.call_count, 0) - if hasattr(mock, "from_settings"): - mock.from_settings.assert_called_once_with(settings, *args, **kwargs) - self.assertEqual(mock.call_count, 0) - else: - mock.assert_called_once_with(*args, **kwargs) - - def _test_with_crawler(mock, settings, crawler): - create_instance(mock, settings, crawler, *args, **kwargs) - if hasattr(mock, "from_crawler"): - mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) - if hasattr(mock, "from_settings"): - self.assertEqual(mock.from_settings.call_count, 0) - self.assertEqual(mock.call_count, 0) - elif hasattr(mock, "from_settings"): - mock.from_settings.assert_called_once_with(settings, *args, **kwargs) - self.assertEqual(mock.call_count, 0) - else: - mock.assert_called_once_with(*args, **kwargs) - - # Check usage of correct constructor using four mocks: - # 1. with no alternative constructors - # 2. with from_settings() constructor - # 3. with from_crawler() constructor - # 4. with from_settings() and from_crawler() constructor - spec_sets = ( - ["__qualname__"], - ["__qualname__", "from_settings"], - ["__qualname__", "from_crawler"], - ["__qualname__", "from_settings", "from_crawler"], - ) - for specs in spec_sets: - m = mock.MagicMock(spec_set=specs) - _test_with_settings(m, settings) - m.reset_mock() - _test_with_crawler(m, settings, crawler) - - # Check adoption of crawler settings - m = mock.MagicMock(spec_set=["__qualname__", "from_settings"]) - create_instance(m, None, crawler, *args, **kwargs) - m.from_settings.assert_called_once_with(crawler.settings, *args, **kwargs) - - with self.assertRaises(ValueError): - create_instance(m, None, None) - - m.from_settings.return_value = None - with self.assertRaises(TypeError): - create_instance(m, settings, None) + assert not list(arg_to_iter(None)) + assert list(arg_to_iter("lala")) == ["lala"] + assert list(arg_to_iter(100)) == [100] + assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"] + assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3] + assert list(arg_to_iter({"a": 1})) == [{"a": 1}] + assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")] def test_build_from_crawler(self): - settings = mock.MagicMock() crawler = mock.MagicMock(spec_set=["settings"]) args = (True, 100.0) kwargs = {"key": "val"} - def _test_with_crawler(mock, settings, crawler): + def _test_with_crawler(mock, crawler): build_from_crawler(mock, crawler, *args, **kwargs) if hasattr(mock, "from_crawler"): mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) - if hasattr(mock, "from_settings"): - self.assertEqual(mock.from_settings.call_count, 0) - self.assertEqual(mock.call_count, 0) - elif hasattr(mock, "from_settings"): - mock.from_settings.assert_called_once_with(settings, *args, **kwargs) - self.assertEqual(mock.call_count, 0) + assert mock.call_count == 0 else: mock.assert_called_once_with(*args, **kwargs) - # Check usage of correct constructor using three mocks: + # Check usage of correct constructor using 2 mocks: # 1. with no alternative constructors # 2. with from_crawler() constructor - # 3. with from_settings() and from_crawler() constructor spec_sets = ( ["__qualname__"], ["__qualname__", "from_crawler"], - ["__qualname__", "from_settings", "from_crawler"], ) for specs in spec_sets: m = mock.MagicMock(spec_set=specs) - _test_with_crawler(m, settings, crawler) + _test_with_crawler(m, crawler) m.reset_mock() # Check adoption of crawler m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"]) m.from_crawler.return_value = None - with self.assertRaises(TypeError): + with pytest.raises(TypeError): build_from_crawler(m, crawler, *args, **kwargs) - def test_build_from_settings(self): - settings = mock.MagicMock() - args = (True, 100.0) - kwargs = {"key": "val"} - - def _test_with_settings(mock, settings): - build_from_settings(mock, settings, *args, **kwargs) - if hasattr(mock, "from_settings"): - mock.from_settings.assert_called_once_with(settings, *args, **kwargs) - self.assertEqual(mock.call_count, 0) - else: - mock.assert_called_once_with(*args, **kwargs) - - # Check usage of correct constructor using three mocks: - # 1. with no alternative constructors - # 2. with from_settings() constructor - # 3. with from_settings() and from_crawler() constructor - spec_sets = ( - ["__qualname__"], - ["__qualname__", "from_settings"], - ["__qualname__", "from_settings", "from_crawler"], - ) - for specs in spec_sets: - m = mock.MagicMock(spec_set=specs) - _test_with_settings(m, settings) - m.reset_mock() - - # Check adoption of crawler settings - m = mock.MagicMock(spec_set=["__qualname__", "from_settings"]) - m.from_settings.return_value = None - with self.assertRaises(TypeError): - build_from_settings(m, settings, *args, **kwargs) - def test_set_environ(self): assert os.environ.get("some_test_environ") is None with set_environ(some_test_environ="test_value"): @@ -247,7 +160,3 @@ class UtilsMiscTestCase(unittest.TestCase): assert rel_has_nofollow("nofollowfoo") is False assert rel_has_nofollow("foonofollow") is False assert rel_has_nofollow("ugc, , nofollow") is True - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 484757035..3783416b9 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -1,8 +1,9 @@ -import unittest import warnings from functools import partial from unittest import mock +import pytest + from scrapy.utils.misc import ( is_generator_with_return_value, warn_on_generator_with_return_value, @@ -10,7 +11,7 @@ from scrapy.utils.misc import ( def _indentation_error(*args, **kwargs): - raise IndentationError() + raise IndentationError def top_level_return_something(): @@ -32,7 +33,6 @@ def top_level_return_none(): https://example.org """ yield url - return def generator_that_returns_stuff(): @@ -41,8 +41,25 @@ def generator_that_returns_stuff(): return 3 -class UtilsMiscPy3TestCase(unittest.TestCase): - def test_generators_return_something(self): +class TestUtilsMisc: + @pytest.fixture + def mock_spider(self): + class MockSettings: + def __init__(self, settings_dict=None): + self.settings_dict = settings_dict or { + "WARN_ON_GENERATOR_RETURN_VALUE": True + } + + def getbool(self, name, default=False): + return self.settings_dict.get(name, default) + + class MockSpider: + def __init__(self): + self.settings = MockSettings() + + return MockSpider() + + def test_generators_return_something(self, mock_spider): def f1(): yield 1 return 2 @@ -77,37 +94,35 @@ https://example.org assert is_generator_with_return_value(i1) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, top_level_return_something) - self.assertEqual(len(w), 1) - self.assertIn( - 'The "NoneType.top_level_return_something" method is a generator', - str(w[0].message), + warn_on_generator_with_return_value(mock_spider, top_level_return_something) + assert len(w) == 1 + assert ( + 'The "MockSpider.top_level_return_something" method is a generator' + in str(w[0].message) ) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, f1) - self.assertEqual(len(w), 1) - self.assertIn('The "NoneType.f1" method is a generator', str(w[0].message)) + warn_on_generator_with_return_value(mock_spider, f1) + assert len(w) == 1 + assert 'The "MockSpider.f1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, g1) - self.assertEqual(len(w), 1) - self.assertIn('The "NoneType.g1" method is a generator', str(w[0].message)) + warn_on_generator_with_return_value(mock_spider, g1) + assert len(w) == 1 + assert 'The "MockSpider.g1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, h1) - self.assertEqual(len(w), 1) - self.assertIn('The "NoneType.h1" method is a generator', str(w[0].message)) + warn_on_generator_with_return_value(mock_spider, h1) + assert len(w) == 1 + assert 'The "MockSpider.h1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, i1) - self.assertEqual(len(w), 1) - self.assertIn('The "NoneType.i1" method is a generator', str(w[0].message)) + warn_on_generator_with_return_value(mock_spider, i1) + assert len(w) == 1 + assert 'The "MockSpider.i1" method is a generator' in str(w[0].message) - def test_generators_return_none(self): + def test_generators_return_none(self, mock_spider): def f2(): yield 1 - return None def g2(): yield 1 - return def h2(): yield 1 @@ -132,7 +147,6 @@ https://example.org https://example.org """ yield url - return def l2(): return @@ -147,31 +161,31 @@ https://example.org assert not is_generator_with_return_value(l2) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, top_level_return_none) - self.assertEqual(len(w), 0) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, f2) - self.assertEqual(len(w), 0) + warn_on_generator_with_return_value(mock_spider, f2) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, g2) - self.assertEqual(len(w), 0) + warn_on_generator_with_return_value(mock_spider, g2) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, h2) - self.assertEqual(len(w), 0) + warn_on_generator_with_return_value(mock_spider, h2) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, i2) - self.assertEqual(len(w), 0) + warn_on_generator_with_return_value(mock_spider, i2) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, j2) - self.assertEqual(len(w), 0) + warn_on_generator_with_return_value(mock_spider, j2) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, k2) - self.assertEqual(len(w), 0) + warn_on_generator_with_return_value(mock_spider, k2) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, l2) - self.assertEqual(len(w), 0) + warn_on_generator_with_return_value(mock_spider, l2) + assert len(w) == 0 - def test_generators_return_none_with_decorator(self): + def test_generators_return_none_with_decorator(self, mock_spider): # noqa: PLR0915 def decorator(func): def inner_func(): func() @@ -181,12 +195,10 @@ https://example.org @decorator def f3(): yield 1 - return None @decorator def g3(): yield 1 - return @decorator def h3(): @@ -215,7 +227,6 @@ https://example.org https://example.org """ yield url - return @decorator def l3(): @@ -231,38 +242,38 @@ https://example.org assert not is_generator_with_return_value(l3) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, top_level_return_none) - self.assertEqual(len(w), 0) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, f3) - self.assertEqual(len(w), 0) + warn_on_generator_with_return_value(mock_spider, f3) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, g3) - self.assertEqual(len(w), 0) + warn_on_generator_with_return_value(mock_spider, g3) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, h3) - self.assertEqual(len(w), 0) + warn_on_generator_with_return_value(mock_spider, h3) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, i3) - self.assertEqual(len(w), 0) + warn_on_generator_with_return_value(mock_spider, i3) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, j3) - self.assertEqual(len(w), 0) + warn_on_generator_with_return_value(mock_spider, j3) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, k3) - self.assertEqual(len(w), 0) + warn_on_generator_with_return_value(mock_spider, k3) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, l3) - self.assertEqual(len(w), 0) + warn_on_generator_with_return_value(mock_spider, l3) + assert len(w) == 0 @mock.patch( "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error ) - def test_indentation_error(self): + def test_indentation_error(self, mock_spider): with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, top_level_return_none) - self.assertEqual(len(w), 1) - self.assertIn("Unable to determine", str(w[0].message)) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) + assert len(w) == 1 + assert "Unable to determine" in str(w[0].message) def test_partial(self): def cb(arg1, arg2): @@ -270,3 +281,32 @@ https://example.org partial_cb = partial(cb, arg1=42) assert not is_generator_with_return_value(partial_cb) + + def test_warn_on_generator_with_return_value_settings_disabled(self): + class MockSettings: + def __init__(self, settings_dict=None): + self.settings_dict = settings_dict or {} + + def getbool(self, name, default=False): + return self.settings_dict.get(name, default) + + class MockSpider: + def __init__(self): + self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False}) + + spider = MockSpider() + + def gen_with_return(): + yield 1 + return "value" + + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(spider, gen_with_return) + assert len(w) == 0 + + spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True + + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(spider, gen_with_return) + assert len(w) == 1 + assert "is a generator" in str(w[0].message) diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 90bd350a5..20a3d940c 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -1,18 +1,17 @@ -import contextlib import os -import shutil -import tempfile -import unittest import warnings from pathlib import Path +import pytest + +from scrapy.utils.misc import set_environ from scrapy.utils.project import data_path, get_project_settings -@contextlib.contextmanager -def inside_a_project(): - prev_dir = os.getcwd() - project_dir = tempfile.mkdtemp() +@pytest.fixture +def proj_path(tmp_path): + prev_dir = Path.cwd() + project_dir = tmp_path try: os.chdir(project_dir) @@ -21,38 +20,22 @@ def inside_a_project(): yield project_dir finally: os.chdir(prev_dir) - shutil.rmtree(project_dir) -class ProjectUtilsTest(unittest.TestCase): - def test_data_path_outside_project(self): - self.assertEqual(str(Path(".scrapy", "somepath")), data_path("somepath")) - abspath = str(Path(os.path.sep, "absolute", "path")) - self.assertEqual(abspath, data_path(abspath)) - - def test_data_path_inside_project(self): - with inside_a_project() as proj_path: - expected = Path(proj_path, ".scrapy", "somepath") - self.assertEqual(expected.resolve(), Path(data_path("somepath")).resolve()) - abspath = str(Path(os.path.sep, "absolute", "path").resolve()) - self.assertEqual(abspath, data_path(abspath)) +def test_data_path_outside_project(): + assert str(Path(".scrapy", "somepath")) == data_path("somepath") + abspath = str(Path(os.path.sep, "absolute", "path")) + assert abspath == data_path(abspath) -@contextlib.contextmanager -def set_env(**update): - modified = set(update.keys()) & set(os.environ.keys()) - update_after = {k: os.environ[k] for k in modified} - remove_after = frozenset(k for k in update if k not in os.environ) - try: - os.environ.update(update) - yield - finally: - os.environ.update(update_after) - for k in remove_after: - os.environ.pop(k) +def test_data_path_inside_project(proj_path: Path) -> None: + expected = proj_path / ".scrapy" / "somepath" + assert expected.resolve() == Path(data_path("somepath")).resolve() + abspath = str(Path(os.path.sep, "absolute", "path").resolve()) + assert abspath == data_path(abspath) -class GetProjectSettingsTestCase(unittest.TestCase): +class TestGetProjectSettings: def test_valid_envvar(self): value = "tests.test_cmdline.settings" envvars = { @@ -60,7 +43,7 @@ class GetProjectSettingsTestCase(unittest.TestCase): } with warnings.catch_warnings(): warnings.simplefilter("error") - with set_env(**envvars): + with set_environ(**envvars): settings = get_project_settings() assert settings.get("SETTINGS_MODULE") == value @@ -69,7 +52,7 @@ class GetProjectSettingsTestCase(unittest.TestCase): envvars = { "SCRAPY_FOO": "bar", } - with set_env(**envvars): + with set_environ(**envvars): settings = get_project_settings() assert settings.get("SCRAPY_FOO") is None @@ -80,7 +63,7 @@ class GetProjectSettingsTestCase(unittest.TestCase): "SCRAPY_FOO": "bar", "SCRAPY_SETTINGS_MODULE": value, } - with set_env(**envvars): + with set_environ(**envvars): settings = get_project_settings() assert settings.get("SETTINGS_MODULE") == value assert settings.get("SCRAPY_FOO") is None diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 80d2e8da1..e8fe45749 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,38 +1,35 @@ +from __future__ import annotations + import functools import operator import platform +import sys +from typing import TYPE_CHECKING, TypeVar -from twisted.trial import unittest +import pytest from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen -from scrapy.utils.defer import aiter_errback, deferred_f_from_coro_f +from scrapy.utils.defer import aiter_errback from scrapy.utils.python import ( MutableAsyncChain, - MutableChain, binary_is_text, - equal_attributes, get_func_args, memoizemethod_noargs, to_bytes, to_unicode, without_none_values, ) +from tests.utils.decorators import coroutine_test -__doctests__ = ["scrapy.utils.python"] +if TYPE_CHECKING: + from collections.abc import Iterable, Mapping -class MutableChainTest(unittest.TestCase): - def test_mutablechain(self): - m = MutableChain(range(2), [2, 3], (4, 5)) - m.extend(range(6, 7)) - m.extend([7, 8]) - m.extend([9, 10], (11, 12)) - self.assertEqual(next(m), 0) - self.assertEqual(m.__next__(), 1) - self.assertEqual(list(m), list(range(2, 13))) +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") -class MutableAsyncChainTest(unittest.TestCase): +class TestMutableAsyncChain: @staticmethod async def g1(): for i in range(3): @@ -56,200 +53,156 @@ class MutableAsyncChainTest(unittest.TestCase): for i in range(5, 7): yield i - @staticmethod - async def collect_asyncgen_exc(asyncgen): - results = [] - async for x in asyncgen: - results.append(x) - return results - - @deferred_f_from_coro_f + @coroutine_test async def test_mutableasyncchain(self): m = MutableAsyncChain(self.g1(), as_async_generator(range(3, 7))) m.extend(self.g2()) m.extend(self.g3()) - self.assertEqual(await m.__anext__(), 0) + assert await m.__anext__() == 0 results = await collect_asyncgen(m) - self.assertEqual(results, list(range(1, 10))) + assert results == list(range(1, 10)) - @deferred_f_from_coro_f + @coroutine_test async def test_mutableasyncchain_exc(self): m = MutableAsyncChain(self.g1()) m.extend(self.g4()) m.extend(self.g3()) results = await collect_asyncgen(aiter_errback(m, lambda _: None)) - self.assertEqual(results, list(range(5))) + assert results == list(range(5)) -class ToUnicodeTest(unittest.TestCase): +class TestToUnicode: def test_converting_an_utf8_encoded_string_to_unicode(self): - self.assertEqual(to_unicode(b"lel\xc3\xb1e"), "lel\xf1e") + assert to_unicode(b"lel\xc3\xb1e") == "lel\xf1e" def test_converting_a_latin_1_encoded_string_to_unicode(self): - self.assertEqual(to_unicode(b"lel\xf1e", "latin-1"), "lel\xf1e") + assert to_unicode(b"lel\xf1e", "latin-1") == "lel\xf1e" def test_converting_a_unicode_to_unicode_should_return_the_same_object(self): - self.assertEqual(to_unicode("\xf1e\xf1e\xf1e"), "\xf1e\xf1e\xf1e") + assert to_unicode("\xf1e\xf1e\xf1e") == "\xf1e\xf1e\xf1e" - def test_converting_a_strange_object_should_raise_TypeError(self): - self.assertRaises(TypeError, to_unicode, 423) + def test_converting_a_strange_object_should_raise_type_error(self): + with pytest.raises(TypeError): + to_unicode(423) def test_errors_argument(self): - self.assertEqual(to_unicode(b"a\xedb", "utf-8", errors="replace"), "a\ufffdb") + assert to_unicode(b"a\xedb", "utf-8", errors="replace") == "a\ufffdb" -class ToBytesTest(unittest.TestCase): +class TestToBytes: def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self): - self.assertEqual(to_bytes("\xa3 49"), b"\xc2\xa3 49") + assert to_bytes("\xa3 49") == b"\xc2\xa3 49" def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self): - self.assertEqual(to_bytes("\xa3 49", "latin-1"), b"\xa3 49") + assert to_bytes("\xa3 49", "latin-1") == b"\xa3 49" def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self): - self.assertEqual(to_bytes(b"lel\xf1e"), b"lel\xf1e") + assert to_bytes(b"lel\xf1e") == b"lel\xf1e" - def test_converting_a_strange_object_should_raise_TypeError(self): - self.assertRaises(TypeError, to_bytes, unittest) + def test_converting_a_strange_object_should_raise_type_error(self): + with pytest.raises(TypeError): + to_bytes(pytest) def test_errors_argument(self): - self.assertEqual(to_bytes("a\ufffdb", "latin-1", errors="replace"), b"a?b") + assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b" -class MemoizedMethodTest(unittest.TestCase): - def test_memoizemethod_noargs(self): - class A: - @memoizemethod_noargs - def cached(self): - return object() +def test_memoizemethod_noargs(): + class A: + @memoizemethod_noargs + def cached(self): + return object() - def noncached(self): - return object() + def noncached(self): + return object() - a = A() - one = a.cached() - two = a.cached() - three = a.noncached() - assert one is two - assert one is not three + a = A() + one = a.cached() + two = a.cached() + three = a.noncached() + assert one is two + assert one is not three -class BinaryIsTextTest(unittest.TestCase): - def test_binaryistext(self): - assert binary_is_text(b"hello") - - def test_utf_16_strings_contain_null_bytes(self): - assert binary_is_text("hello".encode("utf-16")) - - def test_one_with_encoding(self): - assert binary_is_text(b"
Price \xa3
") - - def test_real_binary_bytes(self): - assert not binary_is_text(b"\x02\xa3") +@pytest.mark.parametrize( + ("value", "expected"), + [ + (b"hello", True), + ("hello".encode("utf-16"), True), + (b"
Price \xa3
", True), + (b"\x02\xa3", False), + ], +) +def test_binaryistext(value: bytes, expected: bool) -> None: + assert binary_is_text(value) is expected -class UtilsPythonTestCase(unittest.TestCase): - def test_equal_attributes(self): - class Obj: +def test_get_func_args(): + def f1(a, b, c): + pass + + def f2(a, b=None, c=None): + pass + + def f3(a, b=None, *, c=None): + pass + + class A: + def __init__(self, a, b, c): pass - a = Obj() - b = Obj() - # no attributes given return False - self.assertFalse(equal_attributes(a, b, [])) - # nonexistent attributes - self.assertFalse(equal_attributes(a, b, ["x", "y"])) - - a.x = 1 - b.x = 1 - # equal attribute - self.assertTrue(equal_attributes(a, b, ["x"])) - - b.y = 2 - # obj1 has no attribute y - self.assertFalse(equal_attributes(a, b, ["x", "y"])) - - a.y = 2 - # equal attributes - self.assertTrue(equal_attributes(a, b, ["x", "y"])) - - a.y = 1 - # differente attributes - self.assertFalse(equal_attributes(a, b, ["x", "y"])) - - # test callable - a.meta = {} - b.meta = {} - self.assertTrue(equal_attributes(a, b, ["meta"])) - - # compare ['meta']['a'] - a.meta["z"] = 1 - b.meta["z"] = 1 - - get_z = operator.itemgetter("z") - get_meta = operator.attrgetter("meta") - - def compare_z(obj): - return get_z(get_meta(obj)) - - self.assertTrue(equal_attributes(a, b, [compare_z, "x"])) - # fail z equality - a.meta["z"] = 2 - self.assertFalse(equal_attributes(a, b, [compare_z, "x"])) - - def test_get_func_args(self): - def f1(a, b, c): + def method(self, a, b, c): pass - def f2(a, b=None, c=None): + class Callable: + def __call__(self, a, b, c): pass - def f3(a, b=None, *, c=None): - pass + a = A(1, 2, 3) + cal = Callable() + partial_f1 = functools.partial(f1, None) + partial_f2 = functools.partial(f1, b=None) + partial_f3 = functools.partial(partial_f2, None) - class A: - def __init__(self, a, b, c): - pass + assert get_func_args(f1) == ["a", "b", "c"] + assert get_func_args(f2) == ["a", "b", "c"] + assert get_func_args(f3) == ["a", "b", "c"] + assert get_func_args(A) == ["a", "b", "c"] + assert get_func_args(a.method) == ["a", "b", "c"] + assert get_func_args(partial_f1) == ["b", "c"] + assert get_func_args(partial_f2) == ["a", "c"] + assert get_func_args(partial_f3) == ["c"] + assert get_func_args(cal) == ["a", "b", "c"] + assert get_func_args(object) == [] # pylint: disable=use-implicit-booleaness-not-comparison + assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"] + assert get_func_args(" ".join, stripself=True) == ["iterable"] - def method(self, a, b, c): - pass + if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy": + # the correct and correctly extracted signature + assert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"] + elif platform.python_implementation() == "CPython": + # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature + # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951) + assert get_func_args(operator.itemgetter(2), stripself=True) in [ + [], + ["args", "kwargs"], + ] - class Callable: - def __call__(self, a, b, c): - pass - a = A(1, 2, 3) - cal = Callable() - partial_f1 = functools.partial(f1, None) - partial_f2 = functools.partial(f1, b=None) - partial_f3 = functools.partial(partial_f2, None) - - self.assertEqual(get_func_args(f1), ["a", "b", "c"]) - self.assertEqual(get_func_args(f2), ["a", "b", "c"]) - self.assertEqual(get_func_args(f3), ["a", "b", "c"]) - self.assertEqual(get_func_args(A), ["a", "b", "c"]) - self.assertEqual(get_func_args(a.method), ["a", "b", "c"]) - self.assertEqual(get_func_args(partial_f1), ["b", "c"]) - self.assertEqual(get_func_args(partial_f2), ["a", "c"]) - self.assertEqual(get_func_args(partial_f3), ["c"]) - self.assertEqual(get_func_args(cal), ["a", "b", "c"]) - self.assertEqual(get_func_args(object), []) - self.assertEqual(get_func_args(str.split, stripself=True), ["sep", "maxsplit"]) - self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"]) - - if platform.python_implementation() == "CPython": - # doesn't work on CPython: https://bugs.python.org/issue42785 - self.assertEqual(get_func_args(operator.itemgetter(2)), []) - elif platform.python_implementation() == "PyPy": - self.assertEqual( - get_func_args(operator.itemgetter(2), stripself=True), ["obj"] - ) - - def test_without_none_values(self): - self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) - self.assertEqual(without_none_values((1, None, 3, 4)), (1, 3, 4)) - self.assertEqual( - without_none_values({"one": 1, "none": None, "three": 3, "four": 4}), +@pytest.mark.parametrize( + ("value", "expected"), + [ + ([1, None, 3, 4], [1, 3, 4]), + ((1, None, 3, 4), (1, 3, 4)), + ( + {"one": 1, "none": None, "three": 3, "four": 4}, {"one": 1, "three": 3, "four": 4}, - ) + ), + ], +) +def test_without_none_values( + value: Mapping[_KT, _VT] | Iterable[_KT], expected: dict[_KT, _VT] | Iterable[_KT] +) -> None: + assert without_none_values(value) == expected diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py new file mode 100644 index 000000000..7d39a478e --- /dev/null +++ b/tests/test_utils_reactor.py @@ -0,0 +1,25 @@ +import asyncio + +import pytest + +from scrapy.utils.reactor import ( + _asyncio_reactor_path, + install_reactor, + is_asyncio_reactor_installed, + set_asyncio_event_loop, +) +from tests.utils.decorators import coroutine_test + + +class TestAsyncio: + @pytest.mark.requires_reactor # needs a reactor + def test_is_asyncio_reactor_installed(self, reactor_pytest: str) -> None: + # the result should depend only on the pytest --reactor argument + assert is_asyncio_reactor_installed() == (reactor_pytest == "asyncio") + + @pytest.mark.requires_reactor # installs a reactor + @pytest.mark.only_asyncio + @coroutine_test + async def test_set_asyncio_event_loop(self): + install_reactor(_asyncio_reactor_path) + assert set_asyncio_event_loop(None) is asyncio.get_running_loop() diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 633077eec..55c46059e 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,67 +1,73 @@ +from __future__ import annotations + import json -import unittest -import warnings from hashlib import sha1 -from typing import Dict, Optional, Tuple, Union +from typing import Any from weakref import WeakKeyDictionary +import pytest + from scrapy.http import Request from scrapy.utils.python import to_bytes from scrapy.utils.request import ( _fingerprint_cache, fingerprint, - request_authenticate, request_httprepr, request_to_curl, ) from scrapy.utils.test import get_crawler -class UtilsRequestTest(unittest.TestCase): - def test_request_authenticate(self): - r = Request("http://www.example.com") - request_authenticate(r, "someuser", "somepass") - self.assertEqual(r.headers["Authorization"], b"Basic c29tZXVzZXI6c29tZXBhc3M=") - - def test_request_httprepr(self): - r1 = Request("http://www.example.com") - self.assertEqual( - request_httprepr(r1), b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n" - ) - - r1 = Request("http://www.example.com/some/page.html?arg=1") - self.assertEqual( - request_httprepr(r1), +@pytest.mark.parametrize( + ("r", "expected"), + [ + ( + Request("http://www.example.com"), + b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + ), + ( + Request("http://www.example.com/some/page.html?arg=1"), b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", - ) - - r1 = Request( - "http://www.example.com", - method="POST", - headers={"Content-type": b"text/html"}, - body=b"Some body", - ) - self.assertEqual( - request_httprepr(r1), + ), + ( + Request( + "http://www.example.com", + method="POST", + headers={"Content-type": b"text/html"}, + body=b"Some body", + ), b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body", - ) - - def test_request_httprepr_for_non_http_request(self): - # the representation is not important but it must not fail. - request_httprepr(Request("file:///tmp/foo.txt")) - request_httprepr(Request("ftp://localhost/tmp/foo.txt")) + ), + ], +) +def test_request_httprepr(r: Request, expected: bytes) -> None: + assert request_httprepr(r) == expected -class FingerprintTest(unittest.TestCase): - maxDiff = None +@pytest.mark.parametrize( + "r", + [ + Request("file:///tmp/foo.txt"), + Request("ftp://localhost/tmp/foo.txt"), + ], +) +def test_request_httprepr_for_non_http_request(r: Request) -> None: + # the representation is not important but it must not fail. + request_httprepr(r) - function: staticmethod = staticmethod(fingerprint) - cache: Union[ - "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]", - "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]", - ] = _fingerprint_cache - default_cache_key = (None, False) - known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = ( + +class TestFingerprint: + function: staticmethod[[Request], bytes] = staticmethod(fingerprint) + cache: ( + WeakKeyDictionary[ + Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes] + ] + | WeakKeyDictionary[ + Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], str] + ] + ) = _fingerprint_cache + default_cache_key = (None, False, False) + known_hashes: tuple[tuple[Request, bytes | str, dict[str, Any]], ...] = ( ( Request("http://example.org"), b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD", @@ -142,23 +148,23 @@ class FingerprintTest(unittest.TestCase): def test_query_string_key_order(self): r1 = Request("http://www.example.com/query?id=111&cat=222") r2 = Request("http://www.example.com/query?cat=222&id=111") - self.assertEqual(self.function(r1), self.function(r1)) - self.assertEqual(self.function(r1), self.function(r2)) + assert self.function(r1) == self.function(r1) + assert self.function(r1) == self.function(r2) def test_query_string_key_without_value(self): r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78132,199") r2 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199") - self.assertNotEqual(self.function(r1), self.function(r2)) + assert self.function(r1) != self.function(r2) def test_caching(self): r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199") - self.assertEqual(self.function(r1), self.cache[r1][self.default_cache_key]) + assert self.function(r1) == self.cache[r1][self.default_cache_key] def test_header(self): r1 = Request("http://www.example.com/members/offers.html") r2 = Request("http://www.example.com/members/offers.html") r2.headers["SESSIONID"] = b"somehash" - self.assertEqual(self.function(r1), self.function(r2)) + assert self.function(r1) == self.function(r2) def test_headers(self): r1 = Request("http://www.example.com/") @@ -168,36 +174,61 @@ class FingerprintTest(unittest.TestCase): r3.headers["Accept-Language"] = b"en" r3.headers["SESSIONID"] = b"somehash" - self.assertEqual(self.function(r1), self.function(r2), self.function(r3)) + assert self.function(r1) == self.function(r2) == self.function(r3) - self.assertEqual( - self.function(r1), self.function(r1, include_headers=["Accept-Language"]) + assert self.function(r1) == self.function( + r1, include_headers=["Accept-Language"] ) - self.assertNotEqual( - self.function(r1), self.function(r2, include_headers=["Accept-Language"]) + assert self.function(r1) != self.function( + r2, include_headers=["Accept-Language"] ) - self.assertEqual( - self.function(r3, include_headers=["accept-language", "sessionid"]), - self.function(r3, include_headers=["SESSIONID", "Accept-Language"]), - ) + assert self.function( + r3, include_headers=["accept-language", "sessionid"] + ) == self.function(r3, include_headers=["SESSIONID", "Accept-Language"]) def test_fragment(self): r1 = Request("http://www.example.com/test.html") r2 = Request("http://www.example.com/test.html#fragment") - self.assertEqual(self.function(r1), self.function(r2)) - self.assertEqual(self.function(r1), self.function(r1, keep_fragments=True)) - self.assertNotEqual(self.function(r2), self.function(r2, keep_fragments=True)) - self.assertNotEqual(self.function(r1), self.function(r2, keep_fragments=True)) + assert self.function(r1) == self.function(r2) + assert self.function(r1) == self.function(r1, keep_fragments=True) + assert self.function(r2) != self.function(r2, keep_fragments=True) + assert self.function(r1) != self.function(r2, keep_fragments=True) + + def test_verbatim_url(self): + # verbatim_url requests skip URL canonicalization + r1 = Request( + "http://www.example.com/query?a=1&b=2", meta={"verbatim_url": True} + ) + r2 = Request( + "http://www.example.com/query?b=2&a=1", meta={"verbatim_url": True} + ) + assert self.function(r1) != self.function(r2) + + # without verbatim_url, canonicalization makes query-param order irrelevant + r3 = Request("http://www.example.com/query?a=1&b=2") + r4 = Request("http://www.example.com/query?b=2&a=1") + assert self.function(r3) == self.function(r4) + + # with verbatim_url, the fragment is always kept in the fingerprint + r5 = Request( + "http://www.example.com/test#fragment", meta={"verbatim_url": True} + ) + r6 = Request("http://www.example.com/test", meta={"verbatim_url": True}) + assert self.function(r5) != self.function(r6) + + # keep_fragments parameter is ignored for verbatim_url requests + assert self.function(r5) == self.function(r5, keep_fragments=True) + assert self.function(r5) == self.function(r5, keep_fragments=False) def test_method_and_body(self): r1 = Request("http://www.example.com") r2 = Request("http://www.example.com", method="POST") r3 = Request("http://www.example.com", method="POST", body=b"request body") - self.assertNotEqual(self.function(r1), self.function(r2)) - self.assertNotEqual(self.function(r2), self.function(r3)) + assert self.function(r1) != self.function(r2) + assert self.function(r2) != self.function(r3) def test_request_replace(self): # cached fingerprint must be cleared on request copy @@ -205,7 +236,7 @@ class FingerprintTest(unittest.TestCase): fp1 = self.function(r1) r2 = r1.replace(url="http://www.example.com/other") fp2 = self.function(r2) - self.assertNotEqual(fp1, fp2) + assert fp1 != fp2 def test_part_separation(self): # An old implementation used to serialize request data in a way that @@ -214,7 +245,7 @@ class FingerprintTest(unittest.TestCase): fp1 = self.function(r1) r2 = Request("http://www.example.com/f", body=b"oo") fp2 = self.function(r2) - self.assertNotEqual(fp1, fp2) + assert fp1 != fp2 def test_hashes(self): """Test hardcoded hashes, to make sure future changes to not introduce @@ -223,62 +254,19 @@ class FingerprintTest(unittest.TestCase): self.function(request, **kwargs) for request, _, kwargs in self.known_hashes ] expected = [_fingerprint for _, _fingerprint, _ in self.known_hashes] - self.assertEqual(actual, expected) + assert actual == expected -REQUEST_OBJECTS_TO_TEST = ( - Request("http://www.example.com/"), - Request("http://www.example.com/query?id=111&cat=222"), - Request("http://www.example.com/query?cat=222&id=111"), - Request("http://www.example.com/hnnoticiaj1.aspx?78132,199"), - Request("http://www.example.com/hnnoticiaj1.aspx?78160,199"), - Request("http://www.example.com/members/offers.html"), - Request( - "http://www.example.com/members/offers.html", - headers={"SESSIONID": b"somehash"}, - ), - Request( - "http://www.example.com/", - headers={"Accept-Language": b"en"}, - ), - Request( - "http://www.example.com/", - headers={ - "Accept-Language": b"en", - "SESSIONID": b"somehash", - }, - ), - Request("http://www.example.com/test.html"), - Request("http://www.example.com/test.html#fragment"), - Request("http://www.example.com", method="POST"), - Request("http://www.example.com", method="POST", body=b"request body"), -) - - -class RequestFingerprinterTestCase(unittest.TestCase): - def test_default_implementation(self): +class TestRequestFingerprinter: + def test_fingerprint(self): crawler = get_crawler() request = Request("https://example.com") - self.assertEqual( - crawler.request_fingerprinter.fingerprint(request), - fingerprint(request), + assert crawler.request_fingerprinter.fingerprint(request) == fingerprint( + request ) - def test_deprecated_implementation(self): - settings = { - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", - } - with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(settings_dict=settings) - request = Request("https://example.com") - self.assertEqual( - crawler.request_fingerprinter.fingerprint(request), - fingerprint(request), - ) - self.assertTrue(logged_warnings) - -class CustomRequestFingerprinterTestCase(unittest.TestCase): +class TestCustomRequestFingerprinter: def test_include_headers(self): class RequestFingerprinter: def fingerprint(self, request): @@ -293,7 +281,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com", headers={"X-ID": "2"}) fp2 = crawler.request_fingerprinter.fingerprint(r2) - self.assertNotEqual(fp1, fp2) + assert fp1 != fp2 def test_dont_canonicalize(self): class RequestFingerprinter: @@ -315,7 +303,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com?a=2&a=1") fp2 = crawler.request_fingerprinter.fingerprint(r2) - self.assertNotEqual(fp1, fp2) + assert fp1 != fp2 def test_meta(self): class RequestFingerprinter: @@ -337,10 +325,10 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): fp3 = crawler.request_fingerprinter.fingerprint(r3) r4 = Request("http://www.example.com", meta={"fingerprint": "b"}) fp4 = crawler.request_fingerprinter.fingerprint(r4) - self.assertNotEqual(fp1, fp2) - self.assertNotEqual(fp1, fp4) - self.assertNotEqual(fp2, fp4) - self.assertEqual(fp2, fp3) + assert fp1 != fp2 + assert fp1 != fp4 + assert fp2 != fp4 + assert fp2 == fp3 def test_from_crawler(self): class RequestFingerprinter: @@ -362,62 +350,13 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) - self.assertEqual(fingerprint, settings["FINGERPRINT"]) - - def test_from_settings(self): - class RequestFingerprinter: - @classmethod - def from_settings(cls, settings): - return cls(settings) - - def __init__(self, settings): - self._fingerprint = settings["FINGERPRINT"] - - def fingerprint(self, request): - return self._fingerprint - - settings = { - "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, - "FINGERPRINT": b"fingerprint", - } - crawler = get_crawler(settings_dict=settings) - - request = Request("http://www.example.com") - fingerprint = crawler.request_fingerprinter.fingerprint(request) - self.assertEqual(fingerprint, settings["FINGERPRINT"]) - - def test_from_crawler_and_settings(self): - class RequestFingerprinter: - # This method is ignored due to the presence of from_crawler - @classmethod - def from_settings(cls, settings): - return cls(settings) - - @classmethod - def from_crawler(cls, crawler): - return cls(crawler) - - def __init__(self, crawler): - self._fingerprint = crawler.settings["FINGERPRINT"] - - def fingerprint(self, request): - return self._fingerprint - - settings = { - "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, - "FINGERPRINT": b"fingerprint", - } - crawler = get_crawler(settings_dict=settings) - - request = Request("http://www.example.com") - fingerprint = crawler.request_fingerprinter.fingerprint(request) - self.assertEqual(fingerprint, settings["FINGERPRINT"]) + assert fingerprint == settings["FINGERPRINT"] -class RequestToCurlTest(unittest.TestCase): +class TestRequestToCurl: def _test_request(self, request_object, expected_curl_command): curl_command = request_to_curl(request_object) - self.assertEqual(curl_command, expected_curl_command) + assert curl_command == expected_curl_command def test_get(self): request_object = Request("https://www.example.com") @@ -474,7 +413,3 @@ class RequestToCurlTest(unittest.TestCase): " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" ) self._test_request(request_object, expected_curl_command) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index f226d6c88..7a06d4689 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,6 +1,6 @@ -import unittest from itertools import chain from pathlib import Path +from time import process_time from urllib.parse import urlparse import pytest @@ -13,13 +13,20 @@ from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes from scrapy.utils.response import ( _get_encoding_or_mime_type_from_headers, + _remove_html_comments, + get_base_url, get_meta_refresh, get_response_class, open_in_browser, response_status_message, ) -__doctests__ = ["scrapy.utils.response"] + +def _read_browser_output(burl: str): + path = urlparse(burl).path + if not path or not Path(path).exists(): + path = burl.replace("file://", "") + return Path(path).read_bytes() # https://mimesniff.spec.whatwg.org/#interpreting-the-resource-metadata @@ -53,7 +60,7 @@ NON_BINARY_ASCII_BYTES = ( WHITESPACE_BYTES = ( b"\t", b"\n", - b"\x0C", + b"\x0c", b"\r", b" ", ) @@ -702,11 +709,11 @@ POST_XTRACTMIME_SCENARIOS = ( @pytest.mark.parametrize( - "kwargs,response_class", - ( + ("kwargs", "response_class"), + [ *PRE_XTRACTMIME_SCENARIOS, *POST_XTRACTMIME_SCENARIOS, - ), + ], ) def test_get_response_class_http(kwargs, response_class): kwargs = dict(kwargs) @@ -716,8 +723,8 @@ def test_get_response_class_http(kwargs, response_class): @pytest.mark.parametrize( - "headers,expected", - ( + ("headers", "expected"), + [ *( ( Headers({"Content-Encoding": content_encoding_header}), @@ -729,143 +736,308 @@ def test_get_response_class_http(kwargs, response_class): (["deflate, br"], b"br"), ) ), - ), + ], ) def test_get_encoding_or_mime_type_from_headers(headers, expected): assert _get_encoding_or_mime_type_from_headers(headers) == expected -class ResponseUtilsTest(unittest.TestCase): - dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response") +def test_open_in_browser(): + url = "http:///www.example.com/some/page.html" + body = ( + b" test page test body " + ) - def test_open_in_browser(self): - url = "http:///www.example.com/some/page.html" - body = b" test page test body " + def browser_open(burl: str) -> bool: + bbody = _read_browser_output(burl) + assert b'' in bbody + return True - def browser_open(burl): - path = urlparse(burl).path - if not path or not Path(path).exists(): - path = burl.replace("file://", "") - bbody = Path(path).read_bytes() - self.assertIn(b'', bbody) - return True + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=browser_open), "Browser not called" - response = HtmlResponse(url, body=body) - assert open_in_browser(response, _openfunc=browser_open), "Browser not called" + resp = Response(url, body=body) + with pytest.raises(TypeError): + open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg - resp = Response(url, body=body) - self.assertRaises(TypeError, open_in_browser, resp, debug=True) - def test_get_meta_refresh(self): - r1 = HtmlResponse( - "http://www.example.com", - body=b""" - - Dummy - blahablsdfsal& - """, - ) - r2 = HtmlResponse( - "http://www.example.com", - body=b""" - - Dummy - blahablsdfsal& - """, - ) - r3 = HtmlResponse( - "http://www.example.com", - body=b""" -