diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..623d48cfa --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: monthly + groups: + github-actions: + patterns: + - "*" + cooldown: + default-days: 7 diff --git a/.github/workflows/auto-close-llm-pr.yml b/.github/workflows/auto-close-llm-pr.yml index 160b39488..15120b0d9 100644 --- a/.github/workflows/auto-close-llm-pr.yml +++ b/.github/workflows/auto-close-llm-pr.yml @@ -1,5 +1,7 @@ name: Auto-close LLM PRs -on: +# The workflow only reads the pull request body through the API, it never +# checks out or runs pull request code, so pull_request_target is safe here. +on: # zizmor: ignore[dangerous-triggers] pull_request_target: types: [opened] permissions: @@ -11,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check PR body and close if LLM-written - uses: actions/github-script@v6 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ed2388a59..331fade61 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,4 +1,8 @@ name: Checks + +permissions: + contents: read + on: push: branches: @@ -13,6 +17,10 @@ concurrency: jobs: checks: runs-on: ubuntu-latest + env: + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: @@ -38,21 +46,31 @@ jobs: TOXENV: twinecheck steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + docs/requirements.txt + pyproject.toml + tox.ini + - name: Run check env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - uses: pre-commit/action@v3.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 000000000..c50576ec7 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,59 @@ +--- +name: codspeed + +on: + push: + branches: + - master + pull_request: + paths: + - scrapy/** + - tests/benchmarks/** + - .github/workflows/codspeed.yml + - tox.ini + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: {} + +jobs: + benchmark: + runs-on: ubuntu-latest + env: + # Make uv use the interpreter that actions/setup-python installed + # instead of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system + permissions: + contents: read + id-token: write # OIDC authentication with CodSpeed + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python 3.14 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.14' + + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + + - name: Install dependencies + # tox must stay on PATH for the CodSpeed action to invoke it. + run: | + uv tool install --with tox-uv tox + tox -n -e benchmark + - name: Run benchmarks + uses: CodSpeedHQ/action@0ca9cbbf4623b599a6c3ed4fc8a922942705d9f1 # v5.0.2 + with: + mode: simulation + run: tox -e benchmark diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7779bbb6b..697647131 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,4 +1,8 @@ name: Publish + +permissions: + contents: read + on: push: tags: @@ -9,8 +13,28 @@ concurrency: cancel-in-progress: true jobs: + build: + name: Build distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + - run: | + python -m pip install --upgrade build + python -m build + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: python-package-distributions + path: dist/ + publish: name: Upload release to PyPI + needs: + - build runs-on: ubuntu-latest environment: name: pypi @@ -18,12 +42,9 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - python-version: "3.14" - - run: | - python -m pip install --upgrade build - python -m build + name: python-package-distributions + path: dist/ - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 0409b3ef2..9a09aa67d 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,4 +1,8 @@ name: macOS + +permissions: + contents: read + on: push: branches: @@ -12,39 +16,62 @@ concurrency: jobs: tests: + name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }}) runs-on: macos-latest env: - PYTEST_ADDOPTS: -n auto + PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - env: - - TOXENV: py include: - - python-version: '3.14' - env: - TOXENV: no-reactor + - python-version: "3.10" + env: + TOXENV: py + - python-version: "3.14" + env: + TOXENV: py + coverage: true + - python-version: "3.14" + env: + TOXENV: no-reactor steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + + - name: Install mitmproxy + env: + # mitmproxy needs a newer Python than the oldest matrix entries, so let + # uv download one where no system interpreter is new enough. + UV_PYTHON_PREFERENCE: system + run: uv tool install mitmproxy + - name: Run tests env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox - name: Upload coverage report - uses: codecov/codecov-action@v5 + if: ${{ matrix.coverage }} + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - name: Upload test results if: ${{ !cancelled() }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: report_type: test_results diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 15f25d9b8..cd726a2fe 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,4 +1,8 @@ name: Ubuntu + +permissions: + contents: read + on: push: branches: @@ -12,9 +16,13 @@ concurrency: jobs: tests: + name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }}) runs-on: ubuntu-latest env: - PYTEST_ADDOPTS: -n auto + PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: @@ -34,27 +42,25 @@ jobs: - python-version: "3.14" env: TOXENV: py + coverage: true - python-version: "3.14" env: TOXENV: default-reactor + coverage: true - python-version: "3.14" env: TOXENV: no-reactor - # pinned due to https://github.com/pypy/pypy/issues/5388 - - python-version: pypy3.11-7.3.20 - env: - TOXENV: pypy3 + coverage: true # min deps - python-version: "3.10.19" env: TOXENV: min + coverage: true - python-version: "3.10.19" env: TOXENV: min-default-reactor - - python-version: "3.10.19" - env: - TOXENV: min-no-reactor + coverage: true # pinned due to https://github.com/pypy/pypy/issues/5388 - python-version: pypy3.11-7.3.20 env: @@ -62,16 +68,20 @@ jobs: - python-version: "3.10.19" env: TOXENV: min-extra-deps + coverage: true - python-version: "3.10.19" env: TOXENV: min-botocore + coverage: true - python-version: "3.14" env: TOXENV: extra-deps + coverage: true - python-version: "3.14" env: TOXENV: no-reactor-extra-deps + coverage: true # pinned due to https://github.com/pypy/pypy/issues/5388 - python-version: pypy3.11-7.3.20 env: @@ -79,12 +89,15 @@ jobs: - python-version: "3.14" env: TOXENV: botocore + coverage: true steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} @@ -94,20 +107,32 @@ jobs: sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + - name: Install mitmproxy - run: pipx install mitmproxy + env: + # mitmproxy needs a newer Python than the oldest matrix entries, so let + # uv download one where no system interpreter is new enough. + UV_PYTHON_PREFERENCE: system + # mitmproxy has no PyPy wheels, so run it on CPython regardless of the + # interpreter under test. + run: uv tool install --python cpython mitmproxy - name: Run tests env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox - name: Upload coverage report - uses: codecov/codecov-action@v5 + if: ${{ matrix.coverage }} + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - name: Upload test results if: ${{ !cancelled() }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: report_type: test_results diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index f413782bc..254e9395e 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,4 +1,8 @@ name: Windows + +permissions: + contents: read + on: push: branches: @@ -12,9 +16,13 @@ concurrency: jobs: tests: + name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }}) runs-on: windows-latest env: - PYTEST_ADDOPTS: -n auto + PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: @@ -22,21 +30,10 @@ jobs: - python-version: "3.10" env: TOXENV: py - - python-version: "3.11" - env: - TOXENV: py - - python-version: "3.12" - env: - TOXENV: py - - python-version: "3.13" - env: - TOXENV: py - python-version: "3.14" env: TOXENV: py - - python-version: "3.14" - env: - TOXENV: default-reactor + coverage: true - python-version: "3.14" env: TOXENV: no-reactor @@ -54,24 +51,39 @@ jobs: TOXENV: extra-deps steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + + - name: Install mitmproxy + env: + # mitmproxy needs a newer Python than the oldest matrix entries, so let + # uv download one where no system interpreter is new enough. + UV_PYTHON_PREFERENCE: system + run: uv tool install mitmproxy + - name: Run tests env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox - name: Upload coverage report - uses: codecov/codecov-action@v5 + if: ${{ matrix.coverage }} + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - name: Upload test results if: ${{ !cancelled() }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: report_type: test_results diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 311df7052..c2cb5056d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,11 @@ repos: hooks: - id: sphinx-lint - repo: https://github.com/scrapy/sphinx-scrapy - rev: 0.8.8 + rev: 0.8.10 hooks: - id: sphinx-scrapy +- repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.28.0 + hooks: + - id: zizmor + args: [--no-progress, --fix] diff --git a/README.rst b/README.rst index 6235cb20c..651294add 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ :alt: Scrapy :width: 480px -|version| |python_version| |ubuntu| |macos| |windows| |coverage| |conda| |deepwiki| +|version| |python_version| |tests| |coverage| |conda| |deepwiki| .. |version| image:: https://img.shields.io/pypi/v/Scrapy.svg :target: https://pypi.org/pypi/Scrapy @@ -15,17 +15,9 @@ :target: https://pypi.org/pypi/Scrapy :alt: Supported Python Versions -.. |ubuntu| image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu - :alt: Ubuntu - -.. |macos| image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS - :alt: macOS - -.. |windows| image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows - :alt: Windows +.. |tests| image:: https://img.shields.io/github/check-runs/scrapy/scrapy/master?label=tests + :target: https://github.com/scrapy/scrapy/actions?query=branch%3Amaster + :alt: Tests .. |coverage| image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg :target: https://codecov.io/github/scrapy/scrapy?branch=master diff --git a/conftest.py b/conftest.py index b8a9dc19e..5a535c168 100644 --- a/conftest.py +++ b/conftest.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from importlib.util import find_spec from pathlib import Path from typing import TYPE_CHECKING @@ -53,6 +54,9 @@ if not H2_ENABLED: if find_spec("httpx2") is None and find_spec("httpx") is None: collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py") +if find_spec("pytest_codspeed") is None: + collect_ignore.append("tests/benchmarks") + def pytest_addoption(parser, pluginmanager): if pluginmanager.hasplugin("twisted"): @@ -135,5 +139,6 @@ def pytest_runtest_setup(item): pytest.skip("mitmdump is not available") -# Generate localhost certificate files, needed by some tests -generate_keys() +# Generate localhost certificate files, needed by some tests (but only once if xdist is used) +if "PYTEST_XDIST_WORKER" not in os.environ: + generate_keys() diff --git a/docs/conf.py b/docs/conf.py index de722baac..1b41adaad 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -141,6 +141,8 @@ coverage_ignore_pyobjects = [ r"^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor", ] +# -- Options for the autodoc extension ---------------------------------------- +autodoc_member_order = "bysource" # -- Options for the InterSphinx extension ----------------------------------- # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration diff --git a/docs/faq.rst b/docs/faq.rst index 8f2013581..80658a5bf 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -97,7 +97,7 @@ handler documentation. How can I scrape an item with attributes in different pages? ------------------------------------------------------------ -See :ref:`topics-request-response-ref-request-callback-arguments`. +See :ref:`callback-data`. How can I simulate a user login in my spider? --------------------------------------------- @@ -136,7 +136,7 @@ middleware with a :ref:`custom downloader middleware ` that requires less memory. For example: - If your domain names are similar enough, use your own regular expression - instead joining the strings in :attr:`~scrapy.Spider.allowed_domains` into + instead of joining the strings in :attr:`~scrapy.Spider.allowed_domains` into a complex regular expression. - If you can meet the installation requirements, use pyre2_ instead of @@ -220,21 +220,15 @@ the :ref:`topics-signals-ref` to know which ones. What does the response status code 999 mean? -------------------------------------------- -999 is a custom response status code used by Yahoo sites to throttle requests. +999 is a custom response status code used by some sites to throttle requests. Try slowing down the crawling speed by using a download delay of ``2`` (or -higher) in your spider: +higher) for the affected domains, with the :setting:`DOWNLOAD_SLOTS` setting: .. code-block:: python - from scrapy.spiders import CrawlSpider - - - class MySpider(CrawlSpider): - name = "myspider" - - download_delay = 2 - - # [ ... rest of the spider code ... ] + DOWNLOAD_SLOTS = { + "example.com": {"delay": 2}, + } Or by setting a global download delay in your project with the :setting:`DOWNLOAD_DELAY` setting. @@ -332,8 +326,8 @@ section of the site (which varies each time). In that case, the credentials to log in would be settings, while the url of the section to scrape would be a spider argument. -I'm scraping a XML document and my XPath selector doesn't return any items --------------------------------------------------------------------------- +I'm scraping an XML document and my XPath selector doesn't return any items +--------------------------------------------------------------------------- You may need to remove namespaces. See :ref:`removing-namespaces`. diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index c4e04364b..eaf492c95 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -769,7 +769,7 @@ crawlers on top of it. Also, a common pattern is to build an item with data from more than one page, using a :ref:`trick to pass additional data to the callbacks -`. +`. Using spider arguments diff --git a/docs/news.rst b/docs/news.rst index 8f8477eaa..670843e0e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1417,7 +1417,8 @@ Deprecations - ``download_warnsize`` (use :setting:`DOWNLOAD_WARNSIZE`) - - ``max_concurrent_requests`` (use :setting:`CONCURRENT_REQUESTS`) + - ``max_concurrent_requests`` (use + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) - ``user_agent`` (use :setting:`USER_AGENT`) diff --git a/docs/requirements.in b/docs/requirements.in index a1f3a7468..3783dd1dc 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -5,4 +5,4 @@ sphinx sphinx-notfound-page sphinx-rtd-theme sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10 diff --git a/docs/requirements.txt b/docs/requirements.txt index a5cbad302..0f5969401 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0 # via # -r docs/requirements.in # sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@fe176adc1a8577601bc3fa39b590ebed71a7e9b8 # via -r docs/requirements.in sphinx-sitemap==2.9.0 # via sphinx-scrapy diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 63c217e93..afccb491d 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -267,6 +267,7 @@ Here are some examples of APIs and patterns that need a replacement: Scrapy provides unified helpers for some of these examples: +.. autofunction:: scrapy.utils.asyncio.sleep .. autofunction:: scrapy.utils.asyncio.call_later .. autofunction:: scrapy.utils.asyncio.create_looping_call .. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 4f28019da..33289545c 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -106,10 +106,9 @@ delay of its download slot: Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True}) Note, however, that AutoThrottle still determines the starting delay of every -download slot by setting the ``download_delay`` attribute on the running -spider. If you want AutoThrottle not to impact a download slot at all, in -addition to setting this meta key in all requests that use that download slot, -you might want to set a custom value for the ``delay`` attribute of that +download slot. If you want AutoThrottle not to impact a download slot at all, +in addition to setting this meta key in all requests that use that download +slot, you might want to set a custom value for the ``delay`` attribute of that download slot, e.g. using :setting:`DOWNLOAD_SLOTS`. Settings diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index ee2c3a3cd..343193627 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -114,8 +114,8 @@ some usage help and the available commands:: scrapy [options] [args] Available commands: - crawl Run a spider fetch Fetch a URL using the Scrapy downloader + runspider Run a spider from a Python file, no project required [...] The first line will print the currently active project if you're inside a @@ -263,7 +263,9 @@ crawl * Syntax: ``scrapy crawl `` * Requires project: *yes* -Start crawling using a spider. +Start crawling using the spider with the given :attr:`~scrapy.Spider.name`, +which must be one of those that :command:`list` reports. To run a spider from a +file instead, use :command:`runspider`. Supported options: @@ -571,8 +573,9 @@ runspider * Syntax: ``scrapy runspider `` * Requires project: *no* -Run a spider self-contained in a Python file, without having to create a -project. +Run the spider defined in the given Python file, without requiring a project. + +Supported options: the same as :command:`crawl`. Example usage:: @@ -665,6 +668,8 @@ Example: COMMANDS_MODULE = "mybot.commands" +.. note:: This is a :ref:`pre-crawler setting `. + .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html Register commands via setup.py entry points diff --git a/docs/topics/components.rst b/docs/topics/components.rst index c0df86922..354375577 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -9,37 +9,22 @@ A Scrapy component is any class whose objects are built using That includes the classes that you may assign to the following settings: -- :setting:`ADDONS` - -- :setting:`TWISTED_DNS_RESOLVER` - -- :setting:`DOWNLOAD_HANDLERS` - -- :setting:`DOWNLOADER_MIDDLEWARES` - -- :setting:`DUPEFILTER_CLASS` - -- :setting:`EXTENSIONS` - -- :setting:`FEED_EXPORTERS` - -- :setting:`FEED_STORAGES` - -- :setting:`ITEM_PIPELINES` - -- :setting:`SCHEDULER` - -- :setting:`SCHEDULER_DISK_QUEUE` - -- :setting:`SCHEDULER_MEMORY_QUEUE` - -- :setting:`SCHEDULER_PRIORITY_QUEUE` - -- :setting:`SCHEDULER_START_DISK_QUEUE` - -- :setting:`SCHEDULER_START_MEMORY_QUEUE` - -- :setting:`SPIDER_MIDDLEWARES` +- :setting:`ADDONS` +- :setting:`DOWNLOAD_HANDLERS` +- :setting:`DOWNLOADER_MIDDLEWARES` +- :setting:`DUPEFILTER_CLASS` +- :setting:`EXTENSIONS` +- :setting:`FEED_EXPORTERS` +- :setting:`FEED_STORAGES` +- :setting:`ITEM_PIPELINES` +- :setting:`SCHEDULER` +- :setting:`SCHEDULER_DISK_QUEUE` +- :setting:`SCHEDULER_MEMORY_QUEUE` +- :setting:`SCHEDULER_PRIORITY_QUEUE` +- :setting:`SCHEDULER_START_DISK_QUEUE` +- :setting:`SCHEDULER_START_MEMORY_QUEUE` +- :setting:`SPIDER_MIDDLEWARES` +- :setting:`TWISTED_DNS_RESOLVER` Third-party Scrapy components may also let you define additional Scrapy components, usually configurable through :ref:`settings `, to diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 61aef4bbb..df67bee02 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -30,43 +30,15 @@ You can use the following contracts: .. module:: scrapy.contracts.default -.. class:: UrlContract +.. autoclass:: UrlContract - This contract (``@url``) sets the sample URL used when checking other - contract conditions for this spider. This contract is mandatory. All - callbacks lacking this contract are ignored when running the checks:: +.. autoclass:: CallbackKeywordArgumentsContract - @url url +.. autoclass:: MetadataContract -.. class:: CallbackKeywordArgumentsContract +.. autoclass:: ReturnsContract - This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs ` - attribute for the sample request. It must be a valid JSON dictionary. - :: - - @cb_kwargs {"arg1": "value1", "arg2": "value2", ...} - -.. class:: MetadataContract - - This contract (``@meta``) sets the :attr:`meta ` - attribute for the sample request. It must be a valid JSON dictionary. - :: - - @meta {"arg1": "value1", "arg2": "value2", ...} - -.. class:: ReturnsContract - - This contract (``@returns``) sets lower and upper bounds for the items and - requests returned by the spider. The upper bound is optional:: - - @returns item(s)|request(s) [min [max]] - -.. class:: ScrapesContract - - This contract (``@scrapes``) checks that all the items returned by the - callback have the specified fields:: - - @scrapes field_1 field_2 ... +.. autoclass:: ScrapesContract Use the :command:`check` command to run the contract checks. @@ -89,30 +61,16 @@ override three methods: .. module:: scrapy.contracts -.. class:: Contract(method, *args) +.. autoclass:: Contract - :param method: callback function to which the contract is associated - :type method: collections.abc.Callable + .. automethod:: adjust_request_args - :param args: list of arguments passed into the docstring (whitespace - separated) - :type args: list - - .. method:: Contract.adjust_request_args(args) - - This receives a ``dict`` as an argument containing default arguments - for request object. :class:`~scrapy.Request` is used by default, - but this can be changed with the ``request_cls`` attribute. - If multiple contracts in chain have this attribute defined, the last one is used. - - Must return the same or a modified version of it. - - .. method:: Contract.pre_process(response) + .. method:: pre_process(response) This allows hooking in various checks on the response received from the sample request, before it's being passed to the callback. - .. method:: Contract.post_process(output) + .. method:: post_process(output) This allows processing the output of the callback. Iterators are converted to lists before being passed to this hook. diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 9dcd9d69c..b7ddb0a57 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -21,7 +21,9 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): .. versionadded:: 2.13 -- :class:`~scrapy.Request` callbacks. +- :class:`~scrapy.Request` :ref:`callbacks `, which may + also be defined as :term:`asynchronous generators `. - The :meth:`process_item` method of :ref:`item pipelines `. diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 34ab4f105..e0501c169 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -78,33 +78,15 @@ Writing your own download handler A download handler is a :ref:`component ` that defines the following API: -.. class:: SampleDownloadHandler - - .. attribute:: lazy - :type: bool - - If ``False``, the handler will be instantiated when Scrapy is - initialized. - - If ``True``, the handler will only be instantiated when the first - request handled by it needs to be downloaded. - - .. method:: download_request(request: Request) -> Response - :async: - - Download the given request and return a response. - - .. method:: close() -> None - :async: - - Clean up any resources used by the handler. +.. autoclass:: scrapy.core.downloader.handlers.DownloadHandlerProtocol + :members: An optional base class for custom handlers is provided: .. autoclass:: scrapy.core.downloader.handlers.base.BaseDownloadHandler :members: :undoc-members: - :member-order: bysource + :exclude-members: close, download_request, lazy .. _download-handlers-exceptions: diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 934eb19ac..fcfe7fd29 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -564,6 +564,10 @@ defines the methods described below. Return response if present in cache, or ``None`` otherwise. + If this method raises an exception, e.g. because the cache entry is + corrupted, the middleware logs a warning and handles the request as a + cache miss. + :param spider: the spider which generated the request :type spider: :class:`~scrapy.Spider` object diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index ecd154122..c43b7e20f 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -211,6 +211,16 @@ BaseItemExporter - ``None`` (all fields [2]_, default) + Fields are exported in declaration order, i.e. the order in which + they are defined in the :ref:`item class `. For + :class:`dict` items, which have no declared fields, the key order of + each item is used instead. + + .. versionchanged:: VERSION + Fields of non-\ :class:`dict` items used to be exported in the + order in which they had been populated, except in + :class:`CsvItemExporter`, which has always used declaration order. + - A list of fields: .. code-block:: python diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 5598ab983..78b38cc3f 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -136,18 +136,10 @@ Core Stats extension Enable the collection of core statistics, provided the stats collection is enabled (see :ref:`topics-stats`). -The following stats are collected: - -* ``start_time``: start date/time of the crawl (:class:`~datetime.datetime`). -* ``finish_time``: end date/time of the crawl (:class:`~datetime.datetime`). -* ``elapsed_time_seconds``: total crawl duration in seconds (:class:`float`). -* ``finish_reason``: the closing reason string (e.g. ``"finished"``, - ``"closespider_timeout"``). -* ``item_scraped_count``: total number of items that passed all pipelines. -* ``item_dropped_count``: total number of items dropped by a pipeline. -* ``item_dropped_reasons_count/``: per-exception drop count - (e.g. ``item_dropped_reasons_count/DropItem``). -* ``response_received_count``: total number of HTTP responses received. +The following stats are collected: :stat:`elapsed_time_seconds`, +:stat:`finish_reason`, :stat:`finish_time`, :stat:`item_dropped_count`, +:stat:`item_dropped_reasons_count/{exception}`, :stat:`item_scraped_count`, +:stat:`response_received_count`, :stat:`start_time`. Log Count extension ~~~~~~~~~~~~~~~~~~~ @@ -190,7 +182,7 @@ Monitors the memory used by the Scrapy process that runs the spider and: 1. sends a :signal:`memusage_warning_reached` signal when it exceeds :setting:`MEMUSAGE_WARNING_MB` -2. closes the spider with the `"memusage_exceeded"` reason when it exceeds +2. closes the spider with the ``"memusage_exceeded"`` reason when it exceeds :setting:`MEMUSAGE_LIMIT_MB` This extension is enabled by the :setting:`MEMUSAGE_ENABLED` setting and @@ -214,7 +206,8 @@ An extension for debugging memory usage. It collects information about: * objects left alive that shouldn't. For more info, see :ref:`topics-leaks-trackrefs` To enable this extension, turn on the :setting:`MEMDEBUG_ENABLED` setting. The -info will be stored in the stats. +info will be stored in the :stat:`memdebug/gc_garbage_count` and +:stat:`memdebug/live_refs/{cls}` stats. .. _topics-extensions-ref-spiderstate: diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 66768c97b..2f686fd0f 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -218,12 +218,13 @@ passed through the following settings: .. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html -You can also define a custom ACL, custom endpoint, and region name for exported -feeds using these settings: +You can also define a custom ACL, custom endpoint, region name and connection +pool size for exported feeds using these settings: - :setting:`FEED_STORAGE_S3_ACL` - :setting:`AWS_ENDPOINT_URL` - :setting:`AWS_REGION_NAME` +- :setting:`AWS_MAX_POOL_CONNECTIONS` The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index c9916110d..c3043204b 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -83,6 +83,14 @@ stopping it cleanly. Forced, sudden or otherwise unclean shutdown can lead to data corruption in the job directory, which may prevent the spider from resuming correctly. +Scrapy version changes +---------------------- + +The contents of a job directory are an implementation detail of the Scrapy +version that wrote them. A job must be resumed with the same Scrapy version +that paused it; after upgrading or downgrading Scrapy, start a new job with a +new job directory. + Cookies expiration ------------------ @@ -96,9 +104,13 @@ Request serialization --------------------- For persistence to work, :class:`~scrapy.Request` objects must be -serializable with :mod:`pickle`, except for the ``callback`` and ``errback`` -values passed to their ``__init__`` method, which must be methods of the -running :class:`~scrapy.Spider` class. +serializable with :mod:`pickle`, except for the :ref:`callback +` and :ref:`errback +` values passed to their ``__init__`` +method, which must be methods of the running :class:`~scrapy.Spider` class. + +Requests that cannot be serialized are kept in memory only: they are still +sent, but they are lost when the crawl is paused. If you wish to log the requests that couldn't be serialized, you can set the :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index d4577601d..0a8b146d2 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -187,30 +187,13 @@ scrapy.utils.trackref module Here are the functions available in the :mod:`~scrapy.utils.trackref` module. -.. class:: object_ref +.. autoclass:: object_ref - Inherit from this class if you want to track live - instances with the ``trackref`` module. +.. autofunction:: print_live_refs(ignore=NoneType) -.. function:: print_live_refs(ignore=NoneType) +.. autofunction:: get_oldest - Print a report of live references, grouped by class name. - - :param ignore: if given, all objects from the specified class (or tuple of - classes) will be ignored. - :type ignore: type or tuple - -.. function:: get_oldest(class_name) - - Return the oldest object alive with the given class name, or ``None`` if - none is found. Use :func:`print_live_refs` first to get a list of all - tracked live objects per class name. - -.. function:: iter_all(class_name) - - Return an iterator over all objects alive with the given class name. Use - :func:`print_live_refs` first to get a list of all tracked live objects - per class name. +.. autofunction:: iter_all .. skip: end diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 4ceb4732a..b16066d0c 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -268,6 +268,9 @@ For self-hosting you also might feel the need not to use SSL and not to verify S AWS_USE_SSL = False # or True (None by default) AWS_VERIFY = False # or True (None by default) +To reuse connections for as many files as you check or upload in parallel, set +:setting:`AWS_MAX_POOL_CONNECTIONS` accordingly. + .. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl .. _Minio: https://github.com/minio/minio .. _Zenko CloudServer: https://www.zenko.io/cloudserver/ diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 23738c98c..dfa1e21f6 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -533,8 +533,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites: * if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites directly * use a pool of rotating IPs. For example, the free `Tor project`_ or paid - services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a - super proxy that you can attach your own proxies to. + services like `ProxyMesh`_. * for HTTPS websites, if blocking appears related to TLS behavior, consider adjusting the :setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION` settings, since some websites may respond @@ -559,5 +558,4 @@ projects that detects common mistakes and anti-patterns. .. _ProxyMesh: https://proxymesh.com/ .. _Common Crawl: https://commoncrawl.org/ .. _testspiders: https://github.com/scrapinghub/testspiders -.. _scrapoxy: https://scrapoxy.io/ .. _Zyte API: https://docs.zyte.com/zyte-api/get-started.html diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 7468f1b64..75158440b 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -205,10 +205,11 @@ Request objects Request metadata can also be accessed through the :attr:`~scrapy.http.Response.meta` attribute of a response. - To pass data from one spider callback to another, consider using - :attr:`cb_kwargs` instead. However, request metadata may be the right - choice in certain scenarios, such as to maintain some debugging data - across all follow-up requests (e.g. the source URL). + To pass your own data from one spider callback to another, use + :attr:`cb_kwargs` instead, see :ref:`callback-data`. However, request + metadata may be the right choice in certain scenarios, such as to + maintain some debugging data across all follow-up requests (e.g. the + source URL). A common use of request metadata is to define request-specific parameters for Scrapy components (extensions, middlewares, etc.). For @@ -248,7 +249,7 @@ Request objects .. method:: Request.copy() Return a new Request which is a copy of this Request. See also: - :ref:`topics-request-response-ref-request-callback-arguments`. + :ref:`callback-data`. .. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs, cls]) @@ -256,10 +257,12 @@ Request objects given new values by whichever keyword arguments are specified. The :attr:`~scrapy.Request.cb_kwargs` and :attr:`~scrapy.Request.meta` attributes are shallow copied by default (unless new values are given as arguments). See also - :ref:`topics-request-response-ref-request-callback-arguments`. + :ref:`callback-data`. .. automethod:: from_curl + .. automethod:: to_curl + .. automethod:: to_dict @@ -342,159 +345,7 @@ Other functions related to requests .. autofunction:: scrapy.utils.request.request_from_dict - -.. _topics-request-response-ref-request-callback-arguments: - -Passing additional data to callback functions ---------------------------------------------- - -The callback of a request is a function that will be called when the response -of that request is downloaded. The callback function will be called with the -downloaded :class:`Response` object as its first argument. - -Example: - -.. code-block:: python - - def parse_page1(self, response): - return scrapy.Request( - "http://www.example.com/some_page.html", callback=self.parse_page2 - ) - - - def parse_page2(self, response): - # this would log http://www.example.com/some_page.html - self.logger.info("Visited %s", response.url) - -In some cases you may be interested in passing arguments to those callback -functions so you can receive the arguments later, in the second callback. -The following example shows how to achieve this by using the -:attr:`.Request.cb_kwargs` attribute: - -.. code-block:: python - - def parse(self, response): - request = scrapy.Request( - "http://www.example.com/index.html", - callback=self.parse_page2, - cb_kwargs=dict(main_url=response.url), - ) - request.cb_kwargs["foo"] = "bar" # add more arguments for the callback - yield request - - - def parse_page2(self, response, main_url, foo): - yield dict( - main_url=main_url, - other_url=response.url, - foo=foo, - ) - -.. caution:: :attr:`.Request.cb_kwargs` was introduced in version ``1.7``. - Prior to that, using :attr:`.Request.meta` was recommended for passing - information around callbacks. After ``1.7``, :attr:`.Request.cb_kwargs` - became the preferred way for handling user information, leaving :attr:`.Request.meta` - for communication with components like middlewares and extensions. - -.. _topics-request-response-ref-errbacks: - -Using errbacks to catch exceptions in request processing --------------------------------------------------------- - -The errback of a request is a function that will be called when an exception -is raise while processing it. - -It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can -be used to track connection establishment timeouts, DNS errors etc. - -Here's an example spider logging all errors and catching some specific -errors if needed: - -.. code-block:: python - - import scrapy - - from scrapy.spidermiddlewares.httperror import HttpError - from twisted.internet.error import DNSLookupError - from twisted.internet.error import TimeoutError, TCPTimedOutError - - - class ErrbackSpider(scrapy.Spider): - name = "errback_example" - start_urls = [ - "http://www.httpbin.org/", # HTTP 200 expected - "http://www.httpbin.org/status/404", # Not found error - "http://www.httpbin.org/status/500", # server issue - "http://www.httpbin.org:12345/", # non-responding host, timeout expected - "https://example.invalid/", # DNS error expected - ] - - async def start(self): - for u in self.start_urls: - yield scrapy.Request( - u, - callback=self.parse_httpbin, - errback=self.errback_httpbin, - dont_filter=True, - ) - - def parse_httpbin(self, response): - self.logger.info(f"Got successful response from {response.url}") - # do something useful here... - - def errback_httpbin(self, failure): - # log all failures - self.logger.error(repr(failure)) - - # in case you want to do something special for some errors, - # you may need the failure's type: - - if failure.check(HttpError): - # these exceptions come from HttpError spider middleware - # you can get the non-200 response - response = failure.value.response - self.logger.error("HttpError on %s", response.url) - - elif failure.check(DNSLookupError): - # this is the original request - request = failure.request - self.logger.error("DNSLookupError on %s", request.url) - - elif failure.check(TimeoutError, TCPTimedOutError): - request = failure.request - self.logger.error("TimeoutError on %s", request.url) - - -.. _errback-cb_kwargs: - -Accessing additional data in errback functions ----------------------------------------------- - -In case of a failure to process the request, you may be interested in -accessing arguments to the callback functions so you can process further -based on the arguments in the errback. The following example shows how to -achieve this by using ``Failure.request.cb_kwargs``: - -.. code-block:: python - - def parse(self, response): - request = scrapy.Request( - "http://www.example.com/index.html", - callback=self.parse_page2, - errback=self.errback_page2, - cb_kwargs=dict(main_url=response.url), - ) - yield request - - - def parse_page2(self, response, main_url): - pass - - - def errback_page2(self, failure): - yield dict( - main_url=failure.request.cb_kwargs["main_url"], - ) +.. autofunction:: scrapy.utils.httpobj.urlparse_cached .. _request-fingerprints: @@ -698,6 +549,319 @@ The following built-in Scrapy components have such restrictions: 45-character-long keys must be supported. +.. _callbacks: + +Callbacks +========= + +A callback is a function that Scrapy calls with the :class:`Response` of a +:class:`~scrapy.Request` once that request has been downloaded, so that you can +extract data from that response and generate additional requests to continue +the crawl: + +.. code-block:: python + + from scrapy import Request, Spider + + + class BookSpider(Spider): + name = "books" + + async def start(self): + yield Request("https://books.toscrape.com/", callback=self.parse_home) + + def parse_home(self, response): + for url in response.css("h3 a::attr(href)").getall(): + yield Request(response.urljoin(url), callback=self.parse_book) + + def parse_book(self, response): + yield {"title": response.css("h1::text").get()} + +Requests may also define an :ref:`errback `, which Scrapy calls +instead of the callback when an exception is raised while processing the +request or its response, e.g. a connection error or, by default, a non-2xx +response. + + +.. _callback-assignment: + +Assigning a callback to a request +--------------------------------- + +To assign a callback to a request, use the ``callback`` parameter of +:class:`~scrapy.Request`, which sets the :attr:`.Request.callback` attribute: + +.. code-block:: python + + from scrapy import Request + + + def parse_home(response): ... + + + request = Request("https://books.toscrape.com/", callback=parse_home) + +Requests with no callback, i.e. with :attr:`~scrapy.Request.callback` set to +``None``, are handled by the :meth:`~scrapy.Spider.parse` method of the spider: + +.. code-block:: python + + request = Request("https://books.toscrape.com/") # Handled by parse() + +If a request is never meant to reach a spider callback, e.g. because a +:ref:`component ` sends it and handles its response itself, +assign the special :func:`~scrapy.http.request.NO_CALLBACK` value to it +instead, so that :ref:`downloader middlewares ` +can tell such requests apart. + +While :attr:`~scrapy.Request.callback` only accepts callables, some spider +classes let you also define a callback by name: both :attr:`CrawlSpider.rules +` and :attr:`SitemapSpider.sitemap_rules +` accept the name of a spider +method as a string. + + +.. _writing-callbacks: + +Writing a callback +------------------ + +Any callable can be a callback, as long as it takes the response as its first +positional parameter, and any :ref:`additional callback data ` +as keyword parameters. Spider methods are the most common choice, but plain +functions, lambda expressions and other callable objects work as well. + +.. note:: If you enable :ref:`job persistence ` through the + :setting:`JOBDIR` setting, callbacks must be methods of the running spider. + Requests with any other callback cannot be serialized, so they are kept in + memory only and lost when you pause the crawl. See + :ref:`request-serialization`. + +A callback can be: + +- A regular function: + + .. code-block:: python + + def parse(self, response): + return {"url": response.url} + +- A generator function: + + .. code-block:: python + + def parse(self, response): + yield {"url": response.url} + +- A coroutine function, i.e. defined with ``async def``: + + .. code-block:: python + + async def parse(self, response): + return {"url": response.url} + +- An asynchronous generator function: + + .. code-block:: python + + async def parse(self, response): + yield {"url": response.url} + +The last two allow using ``await``, ``async for`` and ``async with`` in your +callback. See :ref:`topics-coroutines`. + + +.. _callback-output: + +Callback output +--------------- + +A callback may return or yield any of the following: + +- ``None``, which does nothing. + + Callbacks that produce no output at all, e.g. callbacks that only log + information about the response, are perfectly valid. ``None`` values within + an iterable of callback output are ignored as well. + +- A :class:`~scrapy.Request` object, which Scrapy schedules, downloads and + eventually sends to its own callback. + +- An :ref:`item object `, which Scrapy sends to the + :ref:`item pipelines `. + + Any object that is neither ``None`` nor a :class:`~scrapy.Request` object + is treated as an item. + +- An iterable of any of the values above, e.g. a list or, more commonly, a + generator. + + :term:`Asynchronous iterables `, e.g. an + :term:`asynchronous generator`, are also supported. + +.. note:: When a callback *returns* an object, Scrapy iterates that object if + it supports iteration, except for :class:`dict`, :class:`~scrapy.Item`, + :class:`str` and :class:`bytes` objects, which are always handled as single + items. + +.. note:: In a generator callback, a ``return`` statement with a value does not + produce any output, since such a value is not part of what the generator + yields. Scrapy logs a warning when it detects such a callback, see + :setting:`WARN_ON_GENERATOR_RETURN_VALUE`. + +Before Scrapy acts on the output of a callback, that output goes through the +:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method +of your :ref:`spider middlewares `, which may modify +it or drop part of it. + +If a callback raises an exception, the :attr:`~scrapy.Request.errback` of the +request is *not* called. The exception goes through the +:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception` +method of your spider middlewares instead and, unless one of them handles it, +Scrapy logs it and sends the :signal:`spider_error` signal. + + +.. _callback-data: +.. _topics-request-response-ref-request-callback-arguments: + +Passing additional data to callback functions +--------------------------------------------- + +In some cases you may be interested in passing data to a callback in addition +to the response, e.g. data extracted from the response that triggered the +request. The following example shows how to achieve this by using the +:attr:`.Request.cb_kwargs` attribute: + +.. code-block:: python + + from scrapy import Request + + + def parse(self, response): + request = Request( + "http://www.example.com/index.html", + callback=self.parse_page2, + cb_kwargs=dict(main_url=response.url), + ) + request.cb_kwargs["foo"] = "bar" # add more arguments for the callback + yield request + + + def parse_page2(self, response, main_url, foo): + yield dict( + main_url=main_url, + other_url=response.url, + foo=foo, + ) + +:attr:`.Request.cb_kwargs` is the recommended way to pass your own data to a +callback. Use :attr:`.Request.meta` only for data aimed at :ref:`components +`, such as middlewares and extensions. + +.. _errbacks: +.. _topics-request-response-ref-errbacks: + +Errbacks +======== + +The errback of a request is a function that will be called when an exception +is raise while processing it. + +It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can +be used to track connection establishment timeouts, DNS errors etc. + +Here's an example spider logging all errors and catching some specific +errors if needed: + +.. code-block:: python + + from scrapy import Request, Spider + from scrapy.spidermiddlewares.httperror import HttpError + from twisted.internet.error import DNSLookupError + from twisted.internet.error import TimeoutError, TCPTimedOutError + + + class ErrbackSpider(Spider): + name = "errback_example" + start_urls = [ + "http://www.httpbin.org/", # HTTP 200 expected + "http://www.httpbin.org/status/404", # Not found error + "http://www.httpbin.org/status/500", # server issue + "http://www.httpbin.org:12345/", # non-responding host, timeout expected + "https://example.invalid/", # DNS error expected + ] + + async def start(self): + for u in self.start_urls: + yield Request( + u, + callback=self.parse_httpbin, + errback=self.errback_httpbin, + dont_filter=True, + ) + + def parse_httpbin(self, response): + self.logger.info(f"Got successful response from {response.url}") + # do something useful here... + + def errback_httpbin(self, failure): + # log all failures + self.logger.error(repr(failure)) + + # in case you want to do something special for some errors, + # you may need the failure's type: + + if failure.check(HttpError): + # these exceptions come from HttpError spider middleware + # you can get the non-200 response + response = failure.value.response + self.logger.error("HttpError on %s", response.url) + + elif failure.check(DNSLookupError): + # this is the original request + request = failure.request + self.logger.error("DNSLookupError on %s", request.url) + + elif failure.check(TimeoutError, TCPTimedOutError): + request = failure.request + self.logger.error("TimeoutError on %s", request.url) + + +.. _errback-cb_kwargs: + +Accessing additional data in errback functions +---------------------------------------------- + +In case of a failure to process the request, you may be interested in +accessing arguments to the callback functions so you can process further +based on the arguments in the errback. The following example shows how to +achieve this by using ``Failure.request.cb_kwargs``: + +.. code-block:: python + + from scrapy import Request + + + def parse(self, response): + request = Request( + "http://www.example.com/index.html", + callback=self.parse_page2, + errback=self.errback_page2, + cb_kwargs=dict(main_url=response.url), + ) + yield request + + + def parse_page2(self, response, main_url): + pass + + + def errback_page2(self, failure): + yield dict( + main_url=failure.request.cb_kwargs["main_url"], + ) + + .. _topics-request-meta: Request.meta special keys diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 36e16cda6..6e750d76a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -305,10 +305,21 @@ These settings cannot be :ref:`set from a spider `. These settings are: -- :setting:`TWISTED_REACTOR_ENABLED` +- :setting:`ADDONS` +- :setting:`COMMANDS_MODULE` +- :setting:`FORCE_CRAWLER_PROCESS` - :setting:`SPIDER_LOADER_CLASS` and settings used by the corresponding spider loader class, e.g. :setting:`SPIDER_MODULES` and :setting:`SPIDER_LOADER_WARN_ONLY` for the default spider loader class. +- :setting:`TWISTED_REACTOR_ENABLED` + +:setting:`ADDONS` is a special case: it can be set from a spider, but the +``update_pre_crawler_settings()`` method of :ref:`add-ons ` +enabled that way is not called. + +:setting:`TWISTED_REACTOR` also acts as a pre-crawler setting when running a +:ref:`command that needs a CrawlerProcess `, +since its project-level value determines the crawler process class. .. _reactor-settings: @@ -409,6 +420,9 @@ Default: ``{}`` A dict containing paths to the add-ons enabled in your project and their priorities. For more information, see :ref:`topics-addons`. +.. note:: This is a :ref:`pre-crawler setting `, with a + caveat described in that section. + .. setting:: ASYNCIO_EVENT_LOOP ASYNCIO_EVENT_LOOP @@ -458,6 +472,26 @@ Default: ``None`` Endpoint URL used for S3-like storage, for example Minio or s3.scality. +.. setting:: AWS_MAX_POOL_CONNECTIONS + +AWS_MAX_POOL_CONNECTIONS +------------------------ + +.. versionadded:: VERSION + +Default: ``None`` + +Maximum number of connections that AWS clients, such as those of the +:ref:`S3 feed storage backend ` and of the +:ref:`S3 media pipeline storage backend `, keep in their +connection pool. + +If ``None``, the value of :setting:`REACTOR_THREADPOOL_MAXSIZE` is used. + +Values lower than the number of parallel AWS calls do not limit those calls, but +their connections are closed instead of reused, which hurts performance, and +``Connection pool is full, discarding connection`` warnings are logged. + .. setting:: AWS_REGION_NAME AWS_REGION_NAME @@ -541,7 +575,7 @@ CONCURRENT_REQUESTS Default: ``16`` The maximum number of concurrent (i.e. simultaneous) requests that will be -performed by the Scrapy downloader. +performed by the Scrapy downloader. Use ``0`` for no limit. .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN @@ -919,10 +953,6 @@ desired. .. _spider-download_delay-attribute: -.. note:: - - This delay can be set per spider using :attr:`download_delay` spider attribute. - It is possible to change this setting per domain by using :setting:`DOWNLOAD_SLOTS`. @@ -1382,6 +1412,8 @@ When :setting:`TWISTED_REACTOR_ENABLED` is set to ``False``, Set this to ``True`` if you want to set :setting:`TWISTED_REACTOR` to a non-default value in :ref:`per-spider settings `. +.. note:: This is a :ref:`pre-crawler setting `. + .. setting:: FTP_PASSIVE_MODE FTP_PASSIVE_MODE @@ -1829,7 +1861,8 @@ Default: ``False`` Setting to ``True`` will log debug information about the requests scheduler. This currently logs (only once) if the requests cannot be serialized to disk. -Stats counter (``scheduler/unserializable``) tracks the number of times this happens. +The :stat:`scheduler/unserializable` stat tracks the number of times this +happens. Example entry in logs:: diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 03996bee6..f7f9f5cca 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -504,6 +504,27 @@ headers_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.Spider` object +robots_parsed +~~~~~~~~~~~~~ + +.. signal:: robots_parsed +.. function:: robots_parsed(robotparser, request) + + .. versionadded:: VERSION + + Sent by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` after it + downloads and parses a :file:`robots.txt` file, for the host that *request* + targets. + + This signal supports :ref:`asynchronous handlers `. + + :param robotparser: the parser holding the parsed :file:`robots.txt` contents + :type robotparser: :class:`~scrapy.robotstxt.RobotParser` object + + :param request: the request that triggered the :file:`robots.txt` download + :type request: :class:`~scrapy.Request` object + Response signals ---------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 69ff08fa1..8fbf0c52d 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -4,43 +4,31 @@ Spiders ======= -Spiders are classes which define how a certain site (or a group of sites) will be -scraped, including how to perform the crawl (i.e. follow links) and how to -extract structured data from their pages (i.e. scraping items). In other words, -Spiders are the place where you define the custom behaviour for crawling and -parsing pages for a particular site (or, in some cases, a group of sites). +Spiders are classes that define how a site, or a group of sites, is scraped: +which requests to send, and how to parse their responses to extract data and to +send additional requests. -For spiders, the scraping cycle goes through something like this: +A crawl goes as follows: -1. You start by generating the initial requests to crawl the first URLs, and - specify a callback function to be called with the response downloaded from - those requests. +1. Scrapy iterates the :meth:`~scrapy.Spider.start` method of the spider to + get the initial requests. By default, that method yields a + :class:`~scrapy.Request` object for each URL in + :attr:`~scrapy.Spider.start_urls`, with :meth:`~scrapy.Spider.parse` as + :ref:`callback `. - The first requests to perform are obtained by iterating the - :meth:`~scrapy.Spider.start` method, which by default yields a - :class:`~scrapy.Request` object for each URL in the - :attr:`~scrapy.Spider.start_urls` spider attribute, with the - :attr:`~scrapy.Spider.parse` method set as :attr:`~scrapy.Request.callback` - function to handle each :class:`~scrapy.http.Response`. +2. Scrapy downloads each request and calls its callback with the resulting + :class:`~scrapy.http.Response`. -2. In the callback function, you parse the response (web page) and return - :ref:`item objects `, - :class:`~scrapy.Request` objects, or an iterable of these objects. - Those Requests will also contain a callback (maybe - the same) and will then be downloaded by Scrapy and then their - response handled by the specified callback. +3. Callbacks parse the response, typically using :ref:`topics-selectors`, and + return or yield :ref:`item objects ` with the extracted data + and :class:`~scrapy.Request` objects to continue the crawl, which go back + to step 2. See :ref:`callback-output`. -3. In callback functions, you parse the page contents, typically using - :ref:`topics-selectors` (but you can also use BeautifulSoup, lxml or whatever - mechanism you prefer) and generate items with the parsed data. +4. Items go through :ref:`item pipelines `, and are + usually stored through :ref:`topics-feed-exports`. -4. Finally, the items returned from the spider will be typically persisted to a - database (in some :ref:`Item Pipeline `) or written to - a file using :ref:`topics-feed-exports`. - -Even though this cycle applies (more or less) to any kind of spider, there are -different kinds of default spiders bundled into Scrapy for different purposes. -We will talk about those types here. +Scrapy includes different spider classes for different purposes, described +below. .. _topics-spiders-ref: @@ -191,22 +179,7 @@ scrapy.Spider .. automethod:: start - .. method:: parse(response) - - This is the default callback used by Scrapy to process downloaded - responses, when their requests don't specify a callback. - - The ``parse`` method is in charge of processing the response and returning - scraped data and/or more URLs to follow. Other Requests callbacks have - the same requirements as the :class:`~scrapy.Spider` class. - - This method, as well as any other Request callback, must return a - :class:`~scrapy.Request` object, an :ref:`item object `, an - iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects - `, or ``None``. - - :param response: the response to parse - :type response: :class:`~scrapy.http.Response` + .. automethod:: parse .. method:: closed(reason) diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 0cf4a72cc..c702cefe7 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -21,6 +21,8 @@ using the Stats Collector from. Another feature of the Stats Collector is that it's very efficient (when enabled) and extremely efficient (almost unnoticeable) when disabled. +See :ref:`topics-stats-reference` below for the stats that Scrapy sets. + .. _topics-stats-usecases: Common Stats Collector uses @@ -101,3 +103,642 @@ DummyStatsCollector ------------------- .. autoclass:: DummyStatsCollector + +.. _topics-stats-reference: + +Built-in stats reference +======================== + +Scrapy sets the following :ref:`stats `. Components other than +those built into Scrapy may set additional stats; see their documentation. + +Stat keys that contain a ``{placeholder}`` below stand for a family of stats, +one per actual value of the placeholder. + +.. note:: Most stats are set by a specific :ref:`component + `, and are only present if that component is enabled and + its code path is reached. A stat that is missing from + :meth:`~scrapy.statscollectors.StatsCollector.get_stats` output is + equivalent to a counter of 0. + +.. stat:: downloader/exception_count + +``downloader/exception_count`` + Number of exceptions raised while downloading requests. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/exception_type_count/{exception_type} + +``downloader/exception_type_count/{exception_type}`` + Number of exceptions raised while downloading requests, per exception type, + where ``{exception_type}`` is the import path of the exception class, e.g. + ``twisted.internet.error.DNSLookupError``. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/request_bytes + +``downloader/request_bytes`` + Total size, in bytes, of the requests sent, counting the request line, the + headers and the body. As with :stat:`downloader/request_count`, requests + served from the cache are also counted. + + It is an approximation, reconstructed from each :class:`~scrapy.Request` + object instead of measured on the wire, so it does not account for the + actual bytes that the :ref:`download handler + ` sends, e.g. transport-level overhead. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/request_count + +``downloader/request_count`` + Number of requests sent. + + Requests that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + serves from the cache are also counted, even though they are never sent, + because it handles requests after + :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/request_method_count/{method} + +``downloader/request_method_count/{method}`` + Number of requests sent, per HTTP method, e.g. ``GET`` or ``POST``. As with + :stat:`downloader/request_count`, requests served from the cache are also + counted. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/response_bytes + +``downloader/response_bytes`` + Total size, in bytes, of the responses received, counting the status line, + the headers and the body. It covers the same responses as + :stat:`downloader/response_count`. + + The body is counted as received, i.e. still compressed for responses that + used ``Content-Encoding``, because + :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` handles + responses before + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + decompresses them. See :stat:`httpcompression/response_bytes` for + decompressed sizes. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/response_count + +``downloader/response_count`` + Number of responses received. + + It counts responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + serves from the cache, even though they do not come from the network, and + responses that a downloader middleware consumes before they reach your + spider, e.g. redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware` + turns into new requests. Compare with :stat:`response_received_count`. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/response_status_count/{status_code} + +``downloader/response_status_count/{status_code}`` + Number of responses received, per HTTP status code, e.g. ``200`` or + ``404``. It covers the same responses as :stat:`downloader/response_count`. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: dupefilter/filtered + +``dupefilter/filtered`` + Number of requests dropped as duplicates. + + Set by :class:`~scrapy.dupefilters.RFPDupeFilter`. + +.. stat:: elapsed_time_seconds + +``elapsed_time_seconds`` + Time, as a :class:`float`, in seconds, between the :signal:`spider_opened` + and the :signal:`spider_closed` signals. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: feedexport/failed_count/{storage} + +``feedexport/failed_count/{storage}`` + Number of :ref:`feeds ` that could not be stored, per + :ref:`storage backend `, where ``{storage}`` + is the class name of the storage backend, e.g. ``FileFeedStorage``. + +.. stat:: feedexport/success_count/{storage} + +``feedexport/success_count/{storage}`` + Number of :ref:`feeds ` stored successfully, per + :ref:`storage backend `, where ``{storage}`` + is the class name of the storage backend, e.g. ``FileFeedStorage``. + +.. stat:: file_count + +``file_count`` + Number of files handled by the :ref:`media pipelines + `. + +.. stat:: file_status_count/{status} + +``file_status_count/{status}`` + Number of files handled by the :ref:`media pipelines + `, per status, where ``{status}`` is one of: + + - ``downloaded``: the file was downloaded. + + - ``cached``: the file came from the + :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + cache. + + - ``uptodate``: the file was already in the storage backend and had not + :ref:`expired `, so it was not downloaded again. + +.. stat:: finish_reason + +``finish_reason`` + String indicating why the crawl finished. It matches the *reason* argument + of the :signal:`spider_closed` signal. + + Scrapy uses the following reasons: + + - ``cancelled``: the spider was closed without a more specific reason, + e.g. because :exc:`~scrapy.exceptions.CloseSpider` was raised without + one. + + - ``closespider_errorcount``: see :setting:`CLOSESPIDER_ERRORCOUNT`. + + - ``closespider_itemcount``: see :setting:`CLOSESPIDER_ITEMCOUNT`. + + - ``closespider_pagecount``: see :setting:`CLOSESPIDER_PAGECOUNT`. + + - ``closespider_pagecount_no_item``: see + :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM`. + + - ``closespider_timeout``: see :setting:`CLOSESPIDER_TIMEOUT`. + + - ``closespider_timeout_no_item``: see + :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`. + + - ``finished``: the spider became idle with no pending requests, i.e. it + finished normally. + + - ``memusage_exceeded``: see :setting:`MEMUSAGE_LIMIT_MB`. + + - ``shutdown``: the crawl was interrupted, e.g. by a system signal such + as ``SIGINT`` (:kbd:`Ctrl-C`). + + Third-party components and your own code may use any other reason, e.g. by + raising :exc:`~scrapy.exceptions.CloseSpider` with it. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: finish_time + +``finish_time`` + Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when + the :signal:`spider_closed` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: httpcache/errorrecovery + +``httpcache/errorrecovery`` + Number of times that a stale cached response was used because downloading a + fresh response raised an exception. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/firsthand + +``httpcache/firsthand`` + Number of responses that were downloaded without a matching cache entry to + validate against, i.e. responses for requests counted in + :stat:`httpcache/miss`. + + It is lower than :stat:`httpcache/miss` when some of those requests yield + no response, either because they are dropped (see + :stat:`httpcache/ignore`) or because their download fails. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/hit + +``httpcache/hit`` + Number of requests served from the cache. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/ignore + +``httpcache/ignore`` + Number of requests dropped because they were not in the cache and + :setting:`HTTPCACHE_IGNORE_MISSING` is ``True``. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/invalidate + +``httpcache/invalidate`` + Number of times that a cached response failed validation and was replaced + with a freshly downloaded response. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/miss + +``httpcache/miss`` + Number of requests for which no cache entry could be read, either because + there was none or because reading it failed, in which case the request is + also counted in :stat:`httpcache/retrieve_error`. Those requests are + downloaded (see :stat:`httpcache/firsthand`), or dropped if + :setting:`HTTPCACHE_IGNORE_MISSING` is ``True`` (see + :stat:`httpcache/ignore`). + + Requests with a stale cache entry are not counted here; see + :stat:`httpcache/revalidate` and :stat:`httpcache/invalidate`. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/retrieve_error + +``httpcache/retrieve_error`` + Number of cache entries that could not be read, and hence were treated as + cache misses. Those requests are also counted in :stat:`httpcache/miss`. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/revalidate + +``httpcache/revalidate`` + Number of times that a cached response was successfully validated against + the target server, and hence used instead of the fresh response. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/store + +``httpcache/store`` + Number of responses stored in the cache. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/uncacheable + +``httpcache/uncacheable`` + Number of responses not stored in the cache because the + :setting:`HTTPCACHE_POLICY` did not allow it. + + Every response considered for caching is counted either here or in + :stat:`httpcache/store`, so ``httpcache/store + httpcache/uncacheable`` + equals ``httpcache/firsthand + httpcache/invalidate``. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcompression/response_bytes + +``httpcompression/response_bytes`` + Total size, in bytes, of decompressed response bodies, counting only the + body and only responses that were actually decompressed. Compare with + :stat:`downloader/response_bytes`. + + Set by + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`. + +.. stat:: httpcompression/response_count + +``httpcompression/response_count`` + Number of decompressed responses. + + Set by + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`. + +.. stat:: httperror/response_ignored_count + +``httperror/response_ignored_count`` + Number of responses dropped because of their HTTP status code. + + Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`. + +.. stat:: httperror/response_ignored_status_count/{status_code} + +``httperror/response_ignored_status_count/{status_code}`` + Number of responses dropped because of their HTTP status code, per HTTP + status code, e.g. ``404``. + + Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`. + +.. stat:: item_dropped_count + +``item_dropped_count`` + Number of items dropped by an :ref:`item pipeline + `, i.e. number of times that the + :signal:`item_dropped` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: item_dropped_reasons_count/{exception} + +``item_dropped_reasons_count/{exception}`` + Number of items dropped, per exception, where ``{exception}`` is the class + name of the exception that caused the item to be dropped. + + Only :exc:`~scrapy.exceptions.DropItem` and its subclasses drop items, and + each one is counted under its own class name, e.g. + ``item_dropped_reasons_count/DropItem`` for + :exc:`~scrapy.exceptions.DropItem` itself and + ``item_dropped_reasons_count/MyDropItem`` for a ``MyDropItem`` subclass of + it. Any other exception raised by an :ref:`item pipeline + ` triggers the :signal:`item_error` signal instead of + :signal:`item_dropped`, and is not counted here or in + :stat:`item_dropped_count`. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: item_scraped_count + +``item_scraped_count`` + Number of items that passed all :ref:`item pipelines + `, i.e. number of times that the + :signal:`item_scraped` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: items_per_minute + +``items_per_minute`` + Average number of items scraped per minute during the crawl. + + It is ``None`` if the crawl took less than a minute. + + Set by :class:`~scrapy.extensions.logstats.LogStats`. + +.. stat:: log_count/{level} + +``log_count/{level}`` + Number of log messages, per logging level name, e.g. ``INFO`` or + ``WARNING``. + + Only messages that the :setting:`LOG_LEVEL` setting allows are counted. + + Set by :class:`~scrapy.extensions.logcount.LogCount`. + +.. stat:: memdebug/gc_garbage_count + +``memdebug/gc_garbage_count`` + Number of objects in :data:`gc.garbage` when the spider is closed. + + Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires + :setting:`MEMDEBUG_ENABLED` to be ``True``. + +.. stat:: memdebug/live_refs/{cls} + +``memdebug/live_refs/{cls}`` + Number of live objects of class ``{cls}`` when the spider is closed, as + reported by :ref:`trackref `, e.g. + ``memdebug/live_refs/HtmlResponse``. + + Only set for classes with at least 1 live object. + + Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires + :setting:`MEMDEBUG_ENABLED` to be ``True``. + +.. stat:: memusage/limit_reached + +``memusage/limit_reached`` + ``1`` if memory usage exceeded :setting:`MEMUSAGE_LIMIT_MB`, which also + stops the crawl. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: memusage/max + +``memusage/max`` + Maximum peak memory usage, in bytes, observed during the crawl. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: memusage/startup + +``memusage/startup`` + Peak memory usage, in bytes, when the engine started. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: memusage/warning_reached + +``memusage/warning_reached`` + ``1`` if memory usage exceeded :setting:`MEMUSAGE_WARNING_MB`. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: offsite/domains + +``offsite/domains`` + Number of distinct domains for which at least 1 request was dropped for + being offsite. + + Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`. + +.. stat:: offsite/filtered + +``offsite/filtered`` + Number of requests dropped for being offsite. + + Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`. + +.. stat:: request_depth_count/{depth} + +``request_depth_count/{depth}`` + Number of requests scheduled at depth ``{depth}``, e.g. + ``request_depth_count/2``. + + Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`, which + requires :setting:`DEPTH_STATS_VERBOSE` to be ``True`` for this stat. + +.. stat:: request_depth_max + +``request_depth_max`` + Maximum depth reached. + + Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`. + +.. stat:: response_received_count + +``response_received_count`` + Number of responses received, i.e. number of times that the + :signal:`response_received` signal was sent. + + Unlike :stat:`downloader/response_count`, it does not count responses that + a downloader middleware consumes before they reach the engine, e.g. + redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware` + turns into new requests. Both count responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + serves from the cache. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: responses_per_minute + +``responses_per_minute`` + Average number of responses received per minute during the crawl. + + It is ``None`` if the crawl took less than a minute. + + Set by :class:`~scrapy.extensions.logstats.LogStats`. + +.. stat:: retry/count + +``retry/count`` + Number of requests retried. + + Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses. + +.. stat:: retry/max_reached + +``retry/max_reached`` + Number of requests that were not retried because they had already been + retried :setting:`RETRY_TIMES` times. + + Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses. + +.. stat:: retry/reason_count/{reason} + +``retry/reason_count/{reason}`` + Number of requests retried, per reason, e.g. + ``retry/reason_count/twisted.internet.error.TimeoutError`` or + ``retry/reason_count/504 Gateway Time-out``. + + Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses. + +.. note:: Code calling + :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` may pass a + custom *stats_base_key*, in which case ``retry`` is replaced with that key + in the 3 stats above. + +.. stat:: robotstxt/exception_count/{exception_type} + +``robotstxt/exception_count/{exception_type}`` + Number of exceptions raised while downloading ``robots.txt`` files, per + exception type, where ``{exception_type}`` is the string representation of + the exception class, e.g. ````. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/forbidden + +``robotstxt/forbidden`` + Number of requests dropped for being disallowed by ``robots.txt``. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/request_count + +``robotstxt/request_count`` + Number of ``robots.txt`` files requested, i.e. 1 per network location for + which at least 1 request was sent. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/response_count + +``robotstxt/response_count`` + Number of ``robots.txt`` responses received. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/response_status_count/{status_code} + +``robotstxt/response_status_count/{status_code}`` + Number of ``robots.txt`` responses received, per HTTP status code, e.g. + ``404``. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: scheduler/dequeued + +``scheduler/dequeued`` + Number of requests read from the :ref:`scheduler `. + +.. stat:: scheduler/dequeued/disk + +``scheduler/dequeued/disk`` + Number of requests read from the disk queue of the :ref:`scheduler + `. + +.. stat:: scheduler/dequeued/memory + +``scheduler/dequeued/memory`` + Number of requests read from the memory queue of the :ref:`scheduler + `. + +.. stat:: scheduler/enqueued + +``scheduler/enqueued`` + Number of requests stored into the :ref:`scheduler `. + +.. stat:: scheduler/enqueued/disk + +``scheduler/enqueued/disk`` + Number of requests stored into the disk queue of the :ref:`scheduler + `. + +.. stat:: scheduler/enqueued/memory + +``scheduler/enqueued/memory`` + Number of requests stored into the memory queue of the :ref:`scheduler + `. + +.. stat:: scheduler/unserializable + +``scheduler/unserializable`` + Number of requests that could not be stored into the disk queue of the + :ref:`scheduler ` because they could not be + :ref:`serialized `, and hence were stored into the + memory queue instead. + +.. stat:: spider_exceptions/count + +``spider_exceptions/count`` + Number of unhandled exceptions raised by spider callbacks. + + Set by the :ref:`scraper `. + +.. stat:: spider_exceptions/{exception} + +``spider_exceptions/{exception}`` + Number of unhandled exceptions raised by spider callbacks, per exception, + where ``{exception}`` is the class name of the exception, e.g. + ``spider_exceptions/ValueError``. + + Set by the :ref:`scraper `. + +.. stat:: start_time + +``start_time`` + Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when + the :signal:`spider_opened` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: urllength/request_ignored_count + +``urllength/request_ignored_count`` + Number of requests dropped for having a URL longer than + :setting:`URLLENGTH_LIMIT`. + + Set by :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`. diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 8e5001c1d..efb582254 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -16,23 +16,20 @@ class QPSSpider(Spider): name = "qps" benchurl = "http://localhost:8880/" - # Max concurrency is limited by global CONCURRENT_REQUESTS setting - max_concurrent_requests = 8 # Requests per second goal - qps = None # same as: 1 / download_delay - download_delay = None + qps = None # same as: 1 / DOWNLOAD_DELAY # time in seconds to delay server responses latency = None # number of slots to create slots = 1 - def __init__(self, *a, **kw): - super().__init__(*a, **kw) - if self.qps is not None: - self.qps = float(self.qps) - self.download_delay = 1 / self.qps - elif self.download_delay is not None: - self.download_delay = float(self.download_delay) + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + if spider.qps is not None: + spider.qps = float(spider.qps) + crawler.settings.set("DOWNLOAD_DELAY", 1 / spider.qps, priority="spider") + return spider async def start(self): url = self.benchurl diff --git a/pyproject.toml b/pyproject.toml index 4220b12ed..13267e427 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,8 +69,8 @@ brotli = [ gcs = ["google-cloud-storage>=1.29.0"] httpx = ["httpx2[http2,socks]>=2.0.0"] images = ["Pillow>=8.3.2"] -ipython = ["ipython>=7.1.0"] -ptpython = ["ptpython>=2.0.1"] +ipython = ["ipython>=8.15.0"] +ptpython = ["ptpython>=3.0.23"] robotparser = ["robotexclusionrulesparser>=1.6.2"] s3 = ["boto3>=1.20.0"] twisted-http2 = ["Twisted[http2]>=21.7.0"] @@ -118,25 +118,10 @@ allow_incomplete_defs = true # 59 errors # TODO [[tool.mypy.overrides]] module = [ - "tests.mockserver.*", "tests.spiders", "tests.test_closespider", "tests.test_cmdline", "tests.test_contracts", - "tests.test_core_downloader", - "tests.test_downloader_handler_twisted_ftp", - "tests.test_downloadermiddleware_cookies", - "tests.test_downloadermiddleware_httpauth", - "tests.test_downloadermiddleware_httpcache", - "tests.test_downloadermiddleware_httpcompression", - "tests.test_downloadermiddleware_httpproxy", - "tests.test_downloadermiddleware_offsite", - "tests.test_downloadermiddleware_redirect", - "tests.test_downloadermiddleware_redirect_base", - "tests.test_downloadermiddleware_redirect_metarefresh", - "tests.test_downloadermiddleware_retry", - "tests.test_downloadermiddleware_robotstxt", - "tests.test_downloadermiddleware_stats", "tests.test_downloaderslotssettings", "tests.test_dupefilters", "tests.test_engine_loop", @@ -147,18 +132,10 @@ module = [ "tests.test_feedexport_postprocess", "tests.test_feedexport_storages", "tests.test_feedexport_uri_params", - "tests.test_http2_client_protocol", - "tests.test_http_headers", - "tests.test_http_request", - "tests.test_http_request_form", - "tests.test_http_response", - "tests.test_http_response_text", "tests.test_item", - "tests.test_link", "tests.test_linkextractors", "tests.test_loader", "tests.test_logformatter", - "tests.test_logstats", "tests.test_mail", "tests.test_pipeline_crawl", "tests.test_pipeline_files", @@ -166,11 +143,6 @@ module = [ "tests.test_pipeline_media", "tests.test_pipelines", "tests.test_pqueues", - "tests.test_request_attribute_binding", - "tests.test_request_cb_kwargs", - "tests.test_request_dict", - "tests.test_request_left", - "tests.test_robotstxt_interface", "tests.test_scheduler_base", "tests.test_settings", "tests.test_spider", @@ -181,13 +153,7 @@ module = [ "tests.test_squeues", "tests.test_squeues_request", "tests.test_stats", - "tests.test_utils_datatypes", - "tests.test_utils_decorators", - "tests.test_utils_defer", - "tests.test_utils_deprecate", - "tests.test_utils_misc.test_return_with_argument_inside_generator", - "tests.test_utils_python", - "tests.test_utils_request", + "tests.utils.bases.spider", ] check_untyped_defs = false @@ -225,7 +191,6 @@ module = [ "pyftpdlib.*", "pytest_twisted", "robotexclusionrulesparser", - "testfixtures", "zope.interface.*", ] ignore_missing_imports = true diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 6c306afdb..e6d5ff96a 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -225,13 +225,11 @@ def _run_command(cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace) def _run_command_profiled( cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace ) -> None: - if opts.profile: - sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") + sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") loc = locals() p = cProfile.Profile() p.runctx("cmd.run(args, opts)", globals(), loc) - if opts.profile: - p.dump_stats(opts.profile) + p.dump_stats(opts.profile) if __name__ == "__main__": diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 866ba9f6b..4e086e057 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -16,7 +16,7 @@ class Command(BaseRunSpiderCommand): return "[options] " def short_desc(self) -> str: - return "Run a spider" + return "Run a spider of the current project, by name" def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) < 1: diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 4277232c3..52f9cd4b0 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -32,10 +32,7 @@ def sanitize_module_name(module_name: str) -> str: def extract_domain(url: str) -> str: """Extract domain name from URL string""" - o = urlparse(url) - if o.scheme == "" and o.netloc == "": - o = urlparse("//" + url.lstrip("/")) - return o.netloc + return urlparse(url).netloc def verify_url_scheme(url: str) -> str: diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 2ac65bf3f..51caed57f 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -41,7 +41,7 @@ class Command(BaseRunSpiderCommand): spider: Spider | None = None items: ClassVar[dict[int, list[Any]]] = {} requests: ClassVar[dict[int, list[Request]]] = {} - spidercls: type[Spider] | None + spidercls: type[Spider] | None = None first_response = None @@ -346,6 +346,8 @@ class Command(BaseRunSpiderCommand): self.first_response = response cb = self._get_callback(spider=spider, opts=opts, response=response) + assert response.request + response.request.callback = cb # parse items and requests depth: int = response.meta["_depth"] diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 0b9036457..9cdb393ab 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -38,7 +38,7 @@ class Command(BaseRunSpiderCommand): return "[options] " def short_desc(self) -> str: - return "Run a self-contained spider (without creating a project)" + return "Run a spider from a Python file, no project required" def long_desc(self) -> str: return "Run the spider defined in the given file" diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index ebbdf1b98..dbdbbd456 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -22,7 +22,16 @@ if TYPE_CHECKING: class Contract: - """Abstract class for contracts""" + """Base class for :ref:`custom contracts `. + + *method* is the callback function to which the contract is associated. + + *args* is the list of arguments passed into the docstring, separated by + whitespace. + + Subclasses may override :meth:`adjust_request_args`, and define a + ``pre_process`` method or a ``post_process`` method, or both. + """ request_cls: type[Request] | None = None name: str @@ -90,6 +99,13 @@ class Contract: return request def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: + """Receive a ``dict`` with the default arguments for the sample request + and return it, either unmodified or with changes. + + :class:`~scrapy.Request` is used by default, but this can be changed + with the ``request_cls`` attribute. If multiple contracts in the chain + define this attribute, the last one is used. + """ return args diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 9b42ca36f..e2e27165a 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -15,8 +15,15 @@ if TYPE_CHECKING: # contracts class UrlContract(Contract): - """Contract to set the url of the request (mandatory) - @url http://scrapy.org + """Sets (``@url``) the sample URL used when checking the other contract + conditions of a callback. + + This contract is mandatory: callbacks lacking it are ignored when running + the checks. + + .. code-block:: none + + @url url """ name = "url" @@ -27,10 +34,14 @@ class UrlContract(Contract): class CallbackKeywordArgumentsContract(Contract): - """Contract to set the keyword arguments for the request. - The value should be a JSON-encoded dictionary, e.g.: + """Sets (``@cb_kwargs``) the :attr:`cb_kwargs ` + attribute of the sample request. - @cb_kwargs {"arg1": "some value"} + Its value must be a valid JSON dictionary. + + .. code-block:: none + + @cb_kwargs {"arg1": "value1", "arg2": "value2", ...} """ name = "cb_kwargs" @@ -41,10 +52,14 @@ class CallbackKeywordArgumentsContract(Contract): class MetadataContract(Contract): - """Contract to set metadata arguments for the request. - The value should be JSON-encoded dictionary, e.g.: + """Sets (``@meta``) the :attr:`meta ` attribute of the + sample request. - @meta {"arg1": "some value"} + Its value must be a valid JSON dictionary. + + .. code-block:: none + + @meta {"arg1": "value1", "arg2": "value2", ...} """ name = "meta" @@ -55,16 +70,29 @@ class MetadataContract(Contract): class ReturnsContract(Contract): - """Contract to check the output of a callback + """Sets (``@returns``) lower and upper bounds for the items and requests + returned by a callback. - general form: - @returns request(s)/item(s) [min=1 [max]] + The upper bound is optional: - e.g.: - @returns request - @returns request 2 - @returns request 2 10 - @returns request 0 10 + .. code-block:: none + + @returns item(s)|request(s) [min [max]] + + For example: + + .. code-block:: none + + @returns request + @returns request 2 + @returns request 2 10 + @returns request 0 10 + + Set both bounds to the same value to require an exact number: + + .. code-block:: none + + @returns request 2 2 """ name = "returns" @@ -115,8 +143,12 @@ class ReturnsContract(Contract): class ScrapesContract(Contract): - """Contract to check presence of fields in scraped items - @scrapes page_name page_body + """Checks (``@scrapes``) that all items returned by a callback have the + specified fields. + + .. code-block:: none + + @scrapes field_1 field_2 ... """ name = "scrapes" diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 7c0ee0eec..f9ee62838 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -27,7 +27,6 @@ from scrapy.utils.defer import ( deferred_from_coro, maybe_deferred_to_future, ) -from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: @@ -80,22 +79,6 @@ class Slot: ) -def _get_concurrency_delay( - concurrency: int, spider: Spider, settings: BaseSettings -) -> tuple[int, float]: - delay: float = settings.getfloat("DOWNLOAD_DELAY") - if hasattr(spider, "download_delay"): - delay = spider.download_delay - - if hasattr(spider, "max_concurrent_requests"): # pragma: no cover - warn_on_deprecated_spider_attribute( - "max_concurrent_requests", "CONCURRENT_REQUESTS" - ) - concurrency = spider.max_concurrent_requests - - return concurrency, delay - - class Downloader: DOWNLOAD_SLOT = "download_slot" _SLOT_GC_INTERVAL: float = 60.0 # seconds @@ -112,6 +95,9 @@ class Downloader: "CONCURRENT_REQUESTS_PER_DOMAIN" ) self.ip_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS_PER_IP") + # Default delay of new slots. AutoThrottle overrides it to apply + # AUTOTHROTTLE_START_DELAY. + self._delay: float = self.settings.getfloat("DOWNLOAD_DELAY") self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") self.middleware: DownloaderMiddlewareManager = ( DownloaderMiddlewareManager.from_crawler(crawler) @@ -138,7 +124,8 @@ class Downloader: self.active.remove(request) def needs_backout(self) -> bool: - return len(self.active) >= self.total_concurrency + # A total concurrency of 0 means no limit. + return 0 < self.total_concurrency <= len(self.active) @_warn_spider_arg def _get_slot( @@ -146,16 +133,11 @@ class Downloader: ) -> tuple[str, Slot]: key = self.get_slot_key(request) if key not in self.slots: - assert self.crawler.spider slot_settings = self.per_slot_settings.get(key, {}) - conc = self.ip_concurrency or self.domain_concurrency - conc, delay = _get_concurrency_delay( - conc, self.crawler.spider, self.settings - ) - conc, delay = ( - slot_settings.get("concurrency", conc), - slot_settings.get("delay", delay), + conc = slot_settings.get( + "concurrency", self.ip_concurrency or self.domain_concurrency ) + delay = slot_settings.get("delay", self._delay) randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index fb27cdb8b..84dc6216b 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -39,11 +39,23 @@ logger = logging.getLogger(__name__) class DownloadHandlerProtocol(Protocol): + """Interface that :ref:`download handlers ` must + implement. + + Besides implementing this protocol, the contract of a download handler + includes **never** calling :meth:`crawler.engine.download_async() + `. + """ + lazy: bool + """Whether to delay instantiation of the handler; see :ref:`lazy + `.""" - async def download_request(self, request: Request) -> Response: ... + async def download_request(self, request: Request) -> Response: + """Download *request* and return a response.""" - async def close(self) -> None: ... + async def close(self) -> None: + """Clean up any resources used by the handler.""" class DownloadHandlers: diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py index d5a4e9fcd..8bbffb233 100644 --- a/scrapy/core/downloader/handlers/_httpx.py +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -92,10 +92,11 @@ class HttpxDownloadHandler(_Base): self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings) self._bind_host: str | None = self._get_bind_address_host() self._limits: httpx.Limits = httpx.Limits( - # hard limit on simultaneous connections - max_connections=self._pool_size_total, + # hard limit on simultaneous connections (None for no limit, which + # is what a CONCURRENT_REQUESTS of 0 means) + max_connections=self._pool_size_total or None, # total number of idle connections in the pool (extra ones are closed) - max_keepalive_connections=self._pool_size_total, + max_keepalive_connections=self._pool_size_total or None, ) self._default_client: httpx.AsyncClient = self._make_client() diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 07ff4a74e..29b3e3c0f 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -126,5 +126,4 @@ class FTPDownloadHandler(BaseDownloadHandler): headers = {"local filename": protocol.filename or b"", "size": protocol.size} body = protocol.filename or protocol.body.read() respcls = responsetypes.from_args(url=request.url, body=body) - # hints for Headers-related types may need to be fixed to not use AnyStr - return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type] + return respcls(url=request.url, status=200, body=body, headers=headers) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 1033e874f..efac5f71c 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -470,16 +470,17 @@ class ExecutionEngine: """ if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - try: - response_or_request = await maybe_deferred_to_future( - self._download(request) - ) - finally: - assert self._slot is not None - self._slot.remove_request(request) - if isinstance(response_or_request, Request): - return await self.download_async(response_or_request) - return response_or_request + while True: + try: + response_or_request = await maybe_deferred_to_future( + self._download(request) + ) + finally: + assert self._slot is not None + self._slot.remove_request(request) + if not isinstance(response_or_request, Request): + return response_or_request + request = response_or_request @inlineCallbacks def _download( diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 4d072c555..c6226bbca 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -315,7 +315,7 @@ class Stream: 0, self.metadata["remaining_content_length"] ) - # End the stream if no more data needs to be send + # End the stream if no more data needs to be sent if self.metadata["remaining_content_length"] == 0: self._protocol.conn.end_stream(self.stream_id) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 42c517222..a511b0c7b 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -366,8 +366,8 @@ class Scheduler(BaseScheduler): Unless the received request is filtered out by the Dupefilter, attempt to push it into the disk queue, falling back to pushing it into the memory queue. - Increment the appropriate stats, such as: ``scheduler/enqueued``, - ``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``. + Increment the appropriate stats, such as: :stat:`scheduler/enqueued`, + :stat:`scheduler/enqueued/disk`, :stat:`scheduler/enqueued/memory`. Return ``True`` if the request was stored successfully, ``False`` otherwise. """ @@ -390,8 +390,8 @@ class Scheduler(BaseScheduler): falling back to the disk queue if the memory queue is empty. Return ``None`` if there are no more enqueued requests. - Increment the appropriate stats, such as: ``scheduler/dequeued``, - ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``. + Increment the appropriate stats, such as: :stat:`scheduler/dequeued`, + :stat:`scheduler/dequeued/disk`, :stat:`scheduler/dequeued/memory`. """ request: Request | None = self.mqs.pop() assert self.stats is not None diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 223161135..5732ee7a1 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -100,6 +100,10 @@ class Crawler: return self.addons.load_settings(self.settings) + self._apply_deprecated_spider_attr("download_delay", "DOWNLOAD_DELAY") + self._apply_deprecated_spider_attr( + "max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN" + ) self._warn_on_deprecated_default_settings() self.stats = load_object(self.settings["STATS_CLASS"])(self) @@ -173,6 +177,30 @@ class Crawler: stacklevel=3, ) + def _apply_deprecated_spider_attr(self, attr: str, setting: str) -> None: + """Bridge a deprecated spider attribute onto *setting*, warning about + the deprecation (and about being ignored when *setting* is already set + at spider or higher priority).""" + spider = self.spider if self.spider is not None else self.spidercls + if not hasattr(spider, attr): + return + if (self.settings.getpriority(setting) or 0) >= SETTINGS_PRIORITIES["spider"]: + warnings.warn( + f"The {attr!r} spider attribute is deprecated. It is also being " + f"ignored because {setting} is already set at spider or higher " + f"priority. Remove the {attr!r} attribute from your spider.", + category=ScrapyDeprecationWarning, + stacklevel=3, + ) + return + warnings.warn( + f"The {attr!r} spider attribute is deprecated. Use the {setting} " + f"setting instead.", + category=ScrapyDeprecationWarning, + stacklevel=3, + ) + self.settings.set(setting, getattr(spider, attr), priority="spider") + def _apply_reactorless_default_settings(self) -> None: """Change some setting defaults when not using a Twisted reactor. diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index c6c811809..e7ca0ac0e 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from email.utils import formatdate from typing import TYPE_CHECKING @@ -28,6 +29,9 @@ if TYPE_CHECKING: from scrapy.statscollectors import StatsCollector +logger = logging.getLogger(__name__) + + class HttpCacheMiddleware: DOWNLOAD_EXCEPTIONS = ( ConnectionDone, @@ -77,9 +81,20 @@ class HttpCacheMiddleware: return None # Look for cached response and check if expired - cachedresponse: Response | None = self.storage.retrieve_response( - self.crawler.spider, request - ) + cachedresponse: Response | None + try: + cachedresponse = self.storage.retrieve_response( + self.crawler.spider, request + ) + except Exception: + self.stats.inc_value("httpcache/retrieve_error") + logger.warning( + f"Could not read the cache entry for {request}, treating it as a " + f"cache miss.", + exc_info=True, + extra={"spider": self.crawler.spider}, + ) + cachedresponse = None if cachedresponse is None: self.stats.inc_value("httpcache/miss") if self.ignore_missing: diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index e03dc040e..db85b62a1 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -61,7 +61,7 @@ class OffsiteMiddleware: ) self.stats.inc_value("offsite/domains") self.stats.inc_value("offsite/filtered") - raise IgnoreRequest + raise IgnoreRequest(f"Filtered offsite request to {domain!r}") def should_follow(self, request: Request, spider: Spider) -> bool: regex = self.host_regex diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index e1dcd90d5..f910d07c8 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -14,7 +14,7 @@ from typing import TYPE_CHECKING from scrapy.exceptions import NotConfigured from scrapy.utils.decorators import _warn_spider_arg -from scrapy.utils.misc import load_object +from scrapy.utils.misc import _load_objects from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message @@ -149,10 +149,7 @@ class RetryMiddleware: self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")} self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") self.give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"] - self.exceptions_to_retry = tuple( - load_object(x) if isinstance(x, str) else x - for x in settings.getlist("RETRY_EXCEPTIONS") - ) + self.exceptions_to_retry = _load_objects(settings.getlist("RETRY_EXCEPTIONS")) @classmethod def from_crawler(cls, crawler: Crawler) -> Self: diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7d0c17884..81a3a887f 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING from twisted.internet.defer import Deferred +from scrapy import signals from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK @@ -98,7 +99,7 @@ class RobotsTxtMiddleware: assert self.crawler.stats try: resp = await self.crawler.engine.download_async(robotsreq) - self._parse_robots(resp, netloc) + await self._parse_robots(resp, netloc, request) except Exception as e: if not isinstance(e, IgnoreRequest): logger.error( @@ -115,13 +116,20 @@ class RobotsTxtMiddleware: return await maybe_deferred_to_future(parser) return parser - def _parse_robots(self, response: Response, netloc: str) -> None: + async def _parse_robots( + self, response: Response, netloc: str, request: Request + ) -> None: assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/response_count") self.crawler.stats.inc_value( f"robotstxt/response_status_count/{response.status}" ) rp = self._parserimpl.from_crawler(self.crawler, response.body) + await self.crawler.signals.send_catch_log_async( + signal=signals.robots_parsed, + robotparser=rp, + request=request, + ) rp_dfd = self._parsers[netloc] assert isinstance(rp_dfd, Deferred) self._parsers[netloc] = rp diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 5a11df833..ea600d1a8 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -74,6 +74,17 @@ class BaseItemExporter(ABC): def finish_exporting(self) -> None: # noqa: B027 pass + @staticmethod + def _get_populated_field_names(adapter: ItemAdapter) -> Iterable[str]: + """Return the populated field names of *adapter*, in declaration order. + + Populated fields that are not declared, which some item types allow, + come last, in item order. + """ + populated = set(adapter.keys()) + declared = (name for name in adapter.field_names() if name in populated) + return dict.fromkeys([*declared, *adapter.keys()]) + def _get_serialized_fields( self, item: Any, default_value: Any = None, include_empty: bool | None = None ) -> Iterable[tuple[str, Any]]: @@ -86,7 +97,11 @@ class BaseItemExporter(ABC): include_empty = self.export_empty_fields if self.fields_to_export is None: - field_iter = item.field_names() if include_empty else item.keys() + field_iter = ( + item.field_names() + if include_empty + else self._get_populated_field_names(item) + ) elif isinstance(self.fields_to_export, Mapping): if include_empty: field_iter = self.fields_to_export.items() diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index a4362b182..9cb792e30 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -119,7 +119,7 @@ class CloseSpider: self.task = None if self.task_no_item: - if self.task_no_item.running: + if self.task_no_item.running: # pragma: no branch self.task_no_item.stop() self.task_no_item = None diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 678a29e2e..118462bc9 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -13,7 +13,7 @@ import re import sys import warnings from abc import ABC, abstractmethod -from collections.abc import Callable, Coroutine +from collections.abc import Callable from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile @@ -28,6 +28,7 @@ from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.asyncio import is_asyncio_available, run_in_thread +from scrapy.utils.boto import _get_max_pool_connections from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.defer import deferred_from_coro, ensure_awaitable from scrapy.utils.ftp import ftp_store_file @@ -213,11 +214,14 @@ class S3FeedStorage(BlockingFeedStorage): feed_options: dict[str, Any] | None = None, session_token: str | None = None, region_name: str | None = None, + max_pool_connections: int | None = None, ): try: import boto3.session # noqa: PLC0415 except ImportError: raise NotConfigured("missing boto3 library") from None + from botocore.config import Config # noqa: PLC0415 + u = urlparse(uri) assert u.hostname self.bucketname: str = u.hostname @@ -228,6 +232,7 @@ class S3FeedStorage(BlockingFeedStorage): self.acl: str | None = acl self.endpoint_url: str | None = endpoint_url self.region_name: str | None = region_name + self.max_pool_connections: int | None = max_pool_connections boto3_session = boto3.session.Session() self.s3_client = boto3_session.client( @@ -237,6 +242,11 @@ class S3FeedStorage(BlockingFeedStorage): aws_session_token=self.session_token, endpoint_url=self.endpoint_url, region_name=self.region_name, + config=( + Config(max_pool_connections=self.max_pool_connections) + if self.max_pool_connections is not None + else None + ), ) if feed_options and feed_options.get("overwrite", True) is False: @@ -262,6 +272,7 @@ class S3FeedStorage(BlockingFeedStorage): acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None, endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None, region_name=crawler.settings["AWS_REGION_NAME"] or None, + max_pool_connections=_get_max_pool_connections(crawler.settings), feed_options=feed_options, ) @@ -454,7 +465,7 @@ class FeedSlot: ) def finish_exporting(self) -> None: - if self._exporting: + if self._exporting: # pragma: no branch assert self.exporter self.exporter.finish_exporting() self._exporting = False @@ -475,7 +486,7 @@ class FeedExporter: self.feeds = {} self.slots: list[FeedSlot] = [] self.filters: dict[str, ItemFilter] = {} - self._pending_close_coros: list[Coroutine[Any, Any, None]] = [] + self._pending_close_tasks: list[asyncio.Task[None] | Deferred[None]] = [] if not self.settings["FEEDS"] and not self.settings["FEED_URI"]: raise NotConfigured @@ -539,23 +550,44 @@ class FeedExporter: ) async def close_spider(self, spider: Spider) -> None: - self._pending_close_coros.extend( - self._close_slot(slot, spider) for slot in self.slots - ) + for slot in self.slots: + self._schedule_slot_close(slot, spider) - if self._pending_close_coros: + if self._pending_close_tasks: # pragma: no branch if is_asyncio_available(): await asyncio.wait( - [asyncio.create_task(coro) for coro in self._pending_close_coros] + cast("list[asyncio.Task[None]]", list(self._pending_close_tasks)) ) else: await DeferredList( - deferred_from_coro(coro) for coro in self._pending_close_coros + cast("list[Deferred[None]]", list(self._pending_close_tasks)) ) # Send FEED_EXPORTER_CLOSED signal await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed) + def _schedule_slot_close( + self, slot: FeedSlot, spider: Spider + ) -> asyncio.Task[None] | Deferred[None]: + """Start closing the slot without waiting for it to finish, keeping + track of the pending work so that it can be awaited in + :meth:`close_spider` if it hasn't finished by then.""" + aw: asyncio.Task[None] | Deferred[None] + coro = self._close_slot(slot, spider) + if is_asyncio_available(): + aw = asyncio.create_task(coro) + self._pending_close_tasks.append(aw) + aw.add_done_callback(self._pending_close_tasks.remove) + else: + aw = deferred_from_coro(coro) + self._pending_close_tasks.append(aw) + aw.addBoth(self._untrack_pending_close_task, aw) + return aw + + def _untrack_pending_close_task(self, result: Any, aw: Deferred[None]) -> Any: + self._pending_close_tasks.remove(aw) + return result + @staticmethod def _get_file(slot_: FeedSlot) -> IO[bytes]: assert slot_.file @@ -652,7 +684,7 @@ class FeedExporter: uri_params = self._get_uri_params( spider, self.feeds[slot.uri_template]["uri_params"], slot ) - self._pending_close_coros.append(self._close_slot(slot, spider)) + self._schedule_slot_close(slot, spider) slots.append( self._start_new_batch( batch_id=slot.batch_id + 1, diff --git a/scrapy/extensions/logcount.py b/scrapy/extensions/logcount.py index e6d51a7d8..fcce64438 100644 --- a/scrapy/extensions/logcount.py +++ b/scrapy/extensions/logcount.py @@ -20,7 +20,7 @@ class LogCount: """Install a log handler that counts log messages by level. The handler installed is :class:`scrapy.utils.log.LogCounterHandler`. - The counts are stored in stats as ``log_count/``. + The counts are stored in the :stat:`log_count/{level}` stat. .. versionadded:: 2.14 """ diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 1444c8941..e0e289ce8 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -94,7 +94,7 @@ class MemoryUsage: def engine_stopped(self) -> None: for tsk in self.tasks: - if tsk.running: + if tsk.running: # pragma: no branch tsk.stop() def update(self) -> None: diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index cd35c8165..cbcc8b70e 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -38,7 +38,6 @@ class PeriodicLog: ): self.stats: StatsCollector = stats self.interval: float = interval - self.multiplier: float = 60.0 / self.interval self.task: AsyncioLoopingCall | LoopingCall | None = None self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) self.ext_stats_enabled: bool = bool(ext_stats) @@ -165,5 +164,5 @@ class PeriodicLog: def spider_closed(self, spider: Spider, reason: str) -> None: self.log() - if self.task and self.task.running: + if self.task and self.task.running: # pragma: no branch self.task.stop() diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 542ff1cdc..cde73f12e 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -43,18 +43,18 @@ class AutoThrottle: return cls(crawler) def _spider_opened(self, spider: Spider) -> None: - self.mindelay = self._min_delay(spider) - self.maxdelay = self._max_delay(spider) - spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined] + self.mindelay = self._min_delay() + self.maxdelay = self._max_delay() + assert self.crawler.engine + self.crawler.engine.downloader._delay = self._start_delay() - def _min_delay(self, spider: Spider) -> float: - s = self.crawler.settings - return getattr(spider, "download_delay", s.getfloat("DOWNLOAD_DELAY")) + def _min_delay(self) -> float: + return self.crawler.settings.getfloat("DOWNLOAD_DELAY") - def _max_delay(self, spider: Spider) -> float: + def _max_delay(self) -> float: return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY") - def _start_delay(self, spider: Spider) -> float: + def _start_delay(self) -> float: return max( self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") ) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 34d4ec6f2..b55ef6191 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, AnyStr, TypeAlias, cast +from typing import TYPE_CHECKING, Any, TypeAlias, cast from w3lib.http import headers_dict_to_raw @@ -25,14 +25,20 @@ class Headers(CaselessDict): def __init__( self, - seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + seq: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, encoding: str = "utf-8", ): self.encoding: str = encoding super().__init__(seq) def update( # type: ignore[override] - self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] + self, + seq: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]], ) -> None: seq = seq.items() if isinstance(seq, Mapping) else seq iseq: dict[bytes, list[bytes]] = {} @@ -40,7 +46,7 @@ class Headers(CaselessDict): iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v)) super().update(iseq) - def normkey(self, key: AnyStr) -> bytes: # type: ignore[override] + def normkey(self, key: str | bytes) -> bytes: """Normalize key to bytes""" return self._tobytes(key.title()) @@ -67,19 +73,19 @@ class Headers(CaselessDict): return str(x).encode(self.encoding) raise TypeError(f"Unsupported value type: {type(x)}") - def __getitem__(self, key: AnyStr) -> bytes | None: + def __getitem__(self, key: str | bytes) -> bytes | None: try: return cast("list[bytes]", super().__getitem__(key))[-1] except IndexError: return None - def get(self, key: AnyStr, def_val: Any = None) -> bytes | None: + def get(self, key: str | bytes, def_val: Any = None) -> bytes | None: try: return cast("list[bytes]", super().get(key, def_val))[-1] except IndexError: return None - def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]: + def getlist(self, key: str | bytes, def_val: Any = None) -> list[bytes]: try: return cast("list[bytes]", super().__getitem__(key)) except KeyError: @@ -87,15 +93,15 @@ class Headers(CaselessDict): return self.normvalue(def_val) return [] - def setlist(self, key: AnyStr, list_: Iterable[_RawValue]) -> None: + def setlist(self, key: str | bytes, list_: Iterable[_RawValue]) -> None: self[key] = list_ def setlistdefault( - self, key: AnyStr, default_list: Iterable[_RawValue] = () + self, key: str | bytes, default_list: Iterable[_RawValue] = () ) -> Any: return self.setdefault(key, default_list) - def appendlist(self, key: AnyStr, value: Iterable[_RawValue]) -> None: + def appendlist(self, key: str | bytes, value: Iterable[_RawValue]) -> None: lst = self.getlist(key) lst.extend(self.normvalue(value)) self[key] = lst diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 73c2e7dd4..7c53b6b48 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -11,7 +11,6 @@ import inspect from typing import ( TYPE_CHECKING, Any, - AnyStr, Concatenate, NoReturn, TypeAlias, @@ -51,7 +50,9 @@ class VerboseCookie(TypedDict): secure: NotRequired[bool] -CookiesT: TypeAlias = dict[str | bytes, str | bytes] | list[VerboseCookie] +CookiesT: TypeAlias = ( + dict[str | bytes, str | bytes | bool | float | int] | list[VerboseCookie] +) RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") @@ -125,7 +126,10 @@ class Request(object_ref): url: str, callback: CallbackT | None = None, method: str = "GET", - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes | str | None = None, cookies: CookiesT | None = None, meta: dict[str, Any] | None = None, @@ -167,7 +171,8 @@ class Request(object_ref): #: #: The callable must expect the response as its first parameter, and #: support any additional keyword arguments set through - #: :attr:`cb_kwargs`. + #: :attr:`cb_kwargs`. See :ref:`writing-callbacks` and + #: :ref:`callback-output`. #: #: In addition to an arbitrary callable, the following values are also #: supported: @@ -188,8 +193,7 @@ class Request(object_ref): #: raises exceptions for non-2xx responses by default, sending them #: to the :attr:`errback` instead. #: - #: .. seealso:: - #: :ref:`topics-request-response-ref-request-callback-arguments` + #: .. seealso:: :ref:`callbacks` self.callback: CallbackT | None = callback #: :class:`~collections.abc.Callable` to handle exceptions raised @@ -198,7 +202,7 @@ class Request(object_ref): #: The callable must expect a :exc:`~twisted.python.failure.Failure` as #: its first parameter. #: - #: .. seealso:: :ref:`topics-request-response-ref-errbacks` + #: .. seealso:: :ref:`errbacks` self.errback: Callable[[Failure], Any] | None = errback self._cookies: CookiesT | None = cookies or None @@ -310,7 +314,11 @@ class Request(object_ref): @headers.setter def headers( - self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None + self, + value: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None, ) -> None: if isinstance(value, Headers): self._headers = value @@ -381,6 +389,20 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) + def to_curl(self) -> str: + """Return a string with a `cURL `_ command equivalent + to this request. + + Inverse of :meth:`from_curl`. See also + :func:`scrapy.utils.request.request_to_curl`. + + .. versionadded:: VERSION + """ + # Imported here to avoid a circular import. + from scrapy.utils.request import request_to_curl # noqa: PLC0415 + + return request_to_curl(self) + def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]: """Return a dictionary containing the Request's data. diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index f1a8dbf3b..12745292b 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit from warnings import warn @@ -34,7 +34,7 @@ if TYPE_CHECKING: FormdataVType: TypeAlias = str | Iterable[str] FormdataKVType: TypeAlias = tuple[str, FormdataVType] -FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None +FormdataType: TypeAlias = Mapping[str, FormdataVType] | Iterable[FormdataKVType] | None class FormRequest(Request): @@ -100,7 +100,7 @@ class FormRequest(Request): super().__init__(*args, **kwargs) if formdata: - items = formdata.items() if isinstance(formdata, dict) else formdata + items = formdata.items() if isinstance(formdata, Mapping) else formdata form_query_str = _urlencode(items, self.encoding) if self.method == "POST": self.headers.setdefault( @@ -248,7 +248,7 @@ def _get_inputs( if clickable and clickable[0] not in formdata and clickable[0] is not None: values.append(clickable) - formdata_items = formdata.items() if isinstance(formdata, dict) else formdata + formdata_items = formdata.items() if isinstance(formdata, Mapping) else formdata values.extend((k, v) for k, v in formdata_items if v is not None) return values diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 09b1c8b32..f1db11488 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, overload +from typing import TYPE_CHECKING, Any, TypeVar, overload from urllib.parse import urljoin from scrapy.exceptions import NotSupported @@ -72,7 +72,10 @@ class Response(object_ref): self, url: str, status: int = 200, - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes = b"", flags: list[str] | None = None, request: Request | None = None, @@ -145,7 +148,11 @@ class Response(object_ref): @headers.setter def headers( - self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None + self, + value: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None, ) -> None: if isinstance(value, Headers): self._headers = value @@ -222,7 +229,10 @@ class Response(object_ref): url: str | Link, callback: CallbackT | None = None, method: str = "GET", - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes | str | None = None, cookies: CookiesT | None = None, meta: dict[str, Any] | None = None, @@ -272,7 +282,10 @@ class Response(object_ref): urls: Iterable[str | Link], callback: CallbackT | None = None, method: str = "GET", - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes | str | None = None, cookies: CookiesT | None = None, meta: dict[str, Any] | None = None, diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 6876e35e8..d01e23e47 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -9,7 +9,7 @@ from __future__ import annotations import json from contextlib import suppress -from typing import TYPE_CHECKING, Any, AnyStr, cast +from typing import TYPE_CHECKING, Any, cast from urllib.parse import urljoin import parsel @@ -170,7 +170,10 @@ class TextResponse(Response): url: str | Link | parsel.Selector, callback: CallbackT | None = None, method: str = "GET", - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes | str | None = None, cookies: CookiesT | None = None, meta: dict[str, Any] | None = None, @@ -223,7 +226,10 @@ class TextResponse(Response): urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None, callback: CallbackT | None = None, method: str = "GET", - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes | str | None = None, cookies: CookiesT | None = None, meta: dict[str, Any] | None = None, diff --git a/scrapy/mail.py b/scrapy/mail.py index 97123e63c..0691312a3 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -2,6 +2,8 @@ Mail sending helpers """ +# pragma: no file cover + from __future__ import annotations import logging diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 70dc36e35..55a3676e5 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -27,9 +27,17 @@ from twisted.internet.defer import Deferred, maybeDeferred from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline +from scrapy.pipelines.media import ( + FileException as FileException, # noqa: PLC0414 # re-exported for backward compatibility +) +from scrapy.pipelines.media import ( + FileInfo, + FileInfoOrError, + MediaPipeline, + _MediaRequestFiltered, +) from scrapy.utils.asyncio import run_in_thread -from scrapy.utils.boto import is_botocore_available +from scrapy.utils.boto import _get_max_pool_connections, is_botocore_available from scrapy.utils.datatypes import CaseInsensitiveDict from scrapy.utils.defer import deferred_from_coro, ensure_awaitable from scrapy.utils.ftp import ftp_store_file @@ -75,10 +83,6 @@ def _md5sum(file: IO[bytes]) -> str: return m.hexdigest() -class FileException(Exception): - """General media error exception""" - - class StatInfo(TypedDict, total=False): checksum: str last_modified: float @@ -160,6 +164,9 @@ class S3FilesStore: AWS_REGION_NAME = None AWS_USE_SSL = None AWS_VERIFY = None + # Overridden from settings.AWS_MAX_POOL_CONNECTIONS in + # FilesPipeline.from_crawler(); None means the botocore default + AWS_MAX_POOL_CONNECTIONS: int | None = None POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler() HEADERS: ClassVar[dict[str, str]] = { @@ -170,7 +177,13 @@ class S3FilesStore: if not is_botocore_available(): raise NotConfigured("missing botocore library") import botocore.session # noqa: PLC0415 + from botocore.config import Config # noqa: PLC0415 + config = ( + Config(max_pool_connections=self.AWS_MAX_POOL_CONNECTIONS) + if self.AWS_MAX_POOL_CONNECTIONS is not None + else None + ) session = botocore.session.get_session() self.s3_client = session.create_client( "s3", @@ -181,6 +194,7 @@ class S3FilesStore: region_name=self.AWS_REGION_NAME, use_ssl=self.AWS_USE_SSL, verify=self.AWS_VERIFY, + config=config, ) if not uri.startswith("s3://"): raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'") @@ -518,6 +532,7 @@ class FilesPipeline(MediaPipeline): s3store.AWS_REGION_NAME = settings["AWS_REGION_NAME"] s3store.AWS_USE_SSL = settings["AWS_USE_SSL"] s3store.AWS_VERIFY = settings["AWS_VERIFY"] + s3store.AWS_MAX_POOL_CONNECTIONS = _get_max_pool_connections(settings) s3store.POLICY = settings["FILES_STORE_S3_ACL"] gcs_store: type[GCSFilesStore] = cast( @@ -597,20 +612,20 @@ class FilesPipeline(MediaPipeline): def media_failed( self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo ) -> NoReturn: - if not isinstance(failure.value, IgnoreRequest): - referer = referer_str(request) - logger.warning( - "File (unknown-error): Error downloading %(medianame)s from " - "%(request)s referred in <%(referer)s>: %(exception)s", - { - "medianame": self.MEDIA_NAME, - "request": request, - "referer": referer, - "exception": failure.value, - }, + referer = referer_str(request) + if isinstance(failure.value, IgnoreRequest): + logger.debug( + f"File (filtered): Not downloading {self.MEDIA_NAME} from " + f"{request} referred in <{referer}>: {failure.value}", extra={"spider": info.spider}, ) + raise _MediaRequestFiltered(str(failure.value)) from failure.value + logger.warning( + f"File (unknown-error): Error downloading {self.MEDIA_NAME} from " + f"{request} referred in <{referer}>: {failure.value}", + extra={"spider": info.spider}, + ) raise FileException async def media_downloaded( diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 5b5d2dcb2..764f82a78 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -55,6 +55,25 @@ FileInfoOrError: TypeAlias = ( logger = logging.getLogger(__name__) +class FileException(Exception): + """General media error exception""" + + +class _MediaRequestFiltered(FileException): + """Raised internally by media pipelines when a media request is filtered + out (e.g. as an offsite request) instead of being downloaded. + + It is a subclass of :exc:`FileException` for backward compatibility, but + unlike an actual download error it is logged at the ``DEBUG`` level and + without a traceback, since filtering a request is expected behavior rather + than an error. + """ + + +def _media_request_filtered(failure: Failure) -> bool: + return isinstance(failure.value, _MediaRequestFiltered) + + class MediaPipeline(ABC): LOG_FAILED_RESULTS: bool = True @@ -193,7 +212,8 @@ class MediaPipeline(ABC): result = await self._check_media_to_download(request, info, item=item) except Exception: result = Failure() - logger.exception(result) + if not _media_request_filtered(result): + logger.exception(result) self._cache_result_and_execute_waiters(result, fp, info) return await maybe_deferred_to_future(wad) # it must return wad at last @@ -304,6 +324,8 @@ class MediaPipeline(ABC): for ok, value in results: if not ok: assert isinstance(value, Failure) + if _media_request_filtered(value): + continue logger.error( "%(class)s found errors processing %(item)s", {"class": self.__class__.__name__, "item": item}, diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 270a7fbf5..0e9775fd1 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -2,6 +2,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any +import attr from twisted.internet import defer from twisted.internet.base import ReactorBase, ThreadedResolver from twisted.internet.interfaces import ( @@ -70,6 +71,12 @@ class CachingThreadedResolver(ThreadedResolver): return result +def _address_with_port(address: IAddress, port: int) -> IAddress: + if getattr(address, "port", port) == port: + return address + return attr.evolve(address, port=port) + + @implementer(IHostResolution) class HostResolution: def __init__(self, name: str): @@ -97,7 +104,11 @@ class _CachingResolutionReceiver: def resolutionComplete(self) -> None: self.resolutionReceiver.resolutionComplete() if self.addresses: - dnscache[self.hostName] = self.addresses + # Name resolution does not depend on the port, so cache entries are + # kept port-agnostic and the requested port is set on cache hits. + dnscache[self.hostName] = [ + _address_with_port(address, 0) for address in self.addresses + ] @implementer(IHostnameResolver) @@ -142,7 +153,7 @@ class CachingHostnameResolver: transportSemantics, ) resolutionReceiver.resolutionBegan(HostResolution(hostName)) - for addr in addresses: - resolutionReceiver.addressResolved(addr) + for address in addresses: + resolutionReceiver.addressResolved(_address_with_port(address, portNumber)) resolutionReceiver.resolutionComplete() return resolutionReceiver diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 0c64ea5a5..b54011784 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -67,6 +67,15 @@ class RobotParser(metaclass=ABCMeta): :type user_agent: str or bytes """ + def crawl_delay(self, user_agent: str | bytes) -> float | None: + """Return the ``Crawl-delay`` directive for ``user_agent`` as a number + of seconds, or ``None`` if it is not set or the backend does not support + it. + + .. versionadded:: VERSION + """ + return None + class PythonRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -85,6 +94,10 @@ class PythonRobotParser(RobotParser): url = to_unicode(url) return self.rp.can_fetch(user_agent, url) + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) + class RerpRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -105,6 +118,10 @@ class RerpRobotParser(RobotParser): url = to_unicode(url) return cast("bool", self.rp.is_allowed(user_agent, url)) + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.get_crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) + class ProtegoRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -121,3 +138,7 @@ class ProtegoRobotParser(RobotParser): user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(url, user_agent) + + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 993f2436d..a44b36c8a 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -28,6 +28,7 @@ __all__ = [ "AUTOTHROTTLE_TARGET_CONCURRENCY", "AWS_ACCESS_KEY_ID", "AWS_ENDPOINT_URL", + "AWS_MAX_POOL_CONNECTIONS", "AWS_REGION_NAME", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", @@ -229,6 +230,7 @@ AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 AWS_ACCESS_KEY_ID = None AWS_SECRET_ACCESS_KEY = None AWS_ENDPOINT_URL = None +AWS_MAX_POOL_CONNECTIONS = None AWS_REGION_NAME = None AWS_SESSION_TOKEN = None AWS_USE_SSL = None diff --git a/scrapy/signals.py b/scrapy/signals.py index 972f4fd60..3afeb6eab 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -21,6 +21,7 @@ response_received = object() response_downloaded = object() headers_received = object() bytes_received = object() +robots_parsed = object() item_scraped = object() item_dropped = object() item_error = object() diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 02dfa2ac6..6244e3264 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -143,6 +143,22 @@ class Spider(object_ref): else: def parse(self, response: Response, **kwargs: Any) -> Any: + """Process *response*, i.e. extract data from it and generate new + requests. + + This is the default :ref:`callback `: Scrapy uses + it for the response to any request that does not define a + :attr:`~scrapy.Request.callback`, such as the requests that + :meth:`start` yields by default. + + Any :attr:`~scrapy.Request.cb_kwargs` of the request are passed as + keyword parameters. + + Spiders must define this method, unless every request that they + send defines a callback. + + See :ref:`callback-output` about the supported return values. + """ raise NotImplementedError( f"{self.__class__.__name__}.parse callback is not defined" ) diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 925f31ede..1e7ac9c34 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -9,7 +9,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any -from scrapy.exceptions import NotConfigured, NotSupported +from scrapy.exceptions import NotSupported from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.spiders import Spider @@ -76,11 +76,6 @@ class XMLFeedSpider(Spider): yield from self.process_results(response, ret) def _parse(self, response: Response, **kwargs: Any) -> Any: - if not hasattr(self, "parse_node"): - raise NotConfigured( - "You must define parse_node method in order to scrape this XML feed" - ) - response = self.adapt_response(response) nodes: Iterable[Selector] if self.iterator == "iternodes": @@ -158,9 +153,5 @@ class CSVFeedSpider(Spider): yield from self.process_results(response, ret) def _parse(self, response: Response, **kwargs: Any) -> Any: - if not hasattr(self, "parse_row"): - raise NotConfigured( - "You must define parse_row method in order to scrape this CSV feed" - ) response = self.adapt_response(response) return self.parse_rows(response) diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index 44604c0fe..7c7697f56 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -9,7 +9,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar from twisted.internet.defer import Deferred -from twisted.internet.task import LoopingCall +from twisted.internet.task import LoopingCall, deferLater from twisted.internet.threads import deferToThread from scrapy.utils.asyncgen import as_async_generator @@ -293,6 +293,24 @@ class CallLaterResult: self._delayed_call = None +async def sleep(seconds: float) -> None: + """Sleep for *seconds*. + + .. versionadded:: VERSION + + This uses either :func:`asyncio.sleep` or + :func:`~twisted.internet.task.deferLater`, depending on whether asyncio + support is available. + """ + if is_asyncio_available(): + await asyncio.sleep(seconds) + return + + from twisted.internet import reactor + + await deferLater(reactor, seconds) + + async def run_in_thread( func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs ) -> _T: diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 2a77ee2ac..76ee0e7ec 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,7 +1,22 @@ """Boto/botocore helpers""" +from __future__ import annotations + from importlib.util import find_spec +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from scrapy.settings import BaseSettings def is_botocore_available() -> bool: return find_spec("botocore") is not None + + +def _get_max_pool_connections(settings: BaseSettings) -> int: + """Return the maximum number of connections that AWS clients may keep in + their connection pool. + """ + return settings.getint("AWS_MAX_POOL_CONNECTIONS") or settings.getint( + "REACTOR_THREADPOOL_MAXSIZE" + ) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 31a4bb32f..23b4401e8 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,8 +1,9 @@ from __future__ import annotations +import asyncio import code from collections.abc import Callable -from functools import wraps +from functools import partial, wraps from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -16,16 +17,8 @@ def _embed_ipython_shell( namespace: dict[str, Any] | None = None, banner: str = "" ) -> EmbedFuncT: """Start an IPython Shell""" - try: - from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415 - from IPython.terminal.ipapp import load_default_config # noqa: PLC0415 - except ImportError: - from IPython.frontend.terminal.embed import ( # type: ignore[import-not-found,no-redef] # noqa: T100,PLC0415 - InteractiveShellEmbed, - ) - from IPython.frontend.terminal.ipapp import ( # type: ignore[import-not-found,no-redef] # noqa: PLC0415 - load_default_config, - ) + from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415 + from IPython.terminal.ipapp import load_default_config # noqa: PLC0415 @wraps(_embed_ipython_shell) def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None: @@ -38,6 +31,19 @@ def _embed_ipython_shell( shell = InteractiveShellEmbed.instance( banner1=banner, user_ns=namespace, config=config ) + # If an asyncio event loop is already running in this thread, e.g. when + # inspect_response() is called from a spider callback while using the + # asyncio reactor, prompt_toolkit cannot run its own event loop here, so + # ask it to run the prompt in a separate thread instead. pt_app is None + # when IPython falls back to its simple prompt, which needs no event loop. + # See https://github.com/scrapy/scrapy/issues/5447 + if (pt_app := getattr(shell, "pt_app", None)) is not None: + try: + asyncio.get_running_loop() + except RuntimeError: + pass + else: + pt_app.prompt = partial(pt_app.prompt, in_thread=True) shell() return wrapper diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index c020ff4b9..9a945c61c 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -11,12 +11,12 @@ import warnings import weakref from collections import OrderedDict from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, cast +from typing import TYPE_CHECKING, Any, TypeVar, cast from scrapy.exceptions import ScrapyDeprecationWarning if TYPE_CHECKING: - from collections.abc import Iterable, Sequence + from collections.abc import Container, Iterable # typing.Self requires Python 3.11 from typing_extensions import Self @@ -44,22 +44,25 @@ class CaselessDict(dict): # type: ignore[type-arg] def __init__( self, - seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + seq: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, ): super().__init__() if seq: self.update(seq) - def __getitem__(self, key: AnyStr) -> Any: + def __getitem__(self, key: str | bytes) -> Any: return dict.__getitem__(self, self.normkey(key)) - def __setitem__(self, key: AnyStr, value: Any) -> None: + def __setitem__(self, key: str | bytes, value: Any) -> None: dict.__setitem__(self, self.normkey(key), self.normvalue(value)) - def __delitem__(self, key: AnyStr) -> None: + def __delitem__(self, key: str | bytes) -> None: dict.__delitem__(self, self.normkey(key)) - def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] + def __contains__(self, key: str | bytes) -> bool: # type: ignore[override] return dict.__contains__(self, self.normkey(key)) has_key = __contains__ @@ -69,7 +72,7 @@ class CaselessDict(dict): # type: ignore[type-arg] copy = __copy__ - def normkey(self, key: AnyStr) -> AnyStr: + def normkey(self, key: str | bytes) -> str | bytes: """Method to normalize dictionary key access""" return key.lower() @@ -77,23 +80,28 @@ class CaselessDict(dict): # type: ignore[type-arg] """Method to normalize values prior to be set""" return value - def get(self, key: AnyStr, def_val: Any = None) -> Any: + def get(self, key: str | bytes, def_val: Any = None) -> Any: return dict.get(self, self.normkey(key), self.normvalue(def_val)) - def setdefault(self, key: AnyStr, def_val: Any = None) -> Any: + def setdefault(self, key: str | bytes, def_val: Any = None) -> Any: return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) # doesn't fully implement MutableMapping.update() - def update(self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]) -> None: # type: ignore[override] + def update( # type: ignore[override] + self, + seq: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]], + ) -> None: seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) super().update(iseq) @classmethod - def fromkeys(cls, keys: Iterable[AnyStr], value: Any = None) -> Self: # type: ignore[override] - return cls((k, value) for k in keys) # type: ignore[misc] + def fromkeys(cls, keys: Iterable[str | bytes], value: Any = None) -> Self: # type: ignore[override] + return cls((k, value) for k in keys) - def pop(self, key: AnyStr, *args: Any) -> Any: + def pop(self, key: str | bytes, *args: Any) -> Any: return dict.pop(self, self.normkey(key), *args) @@ -132,6 +140,22 @@ class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]): def __repr__(self) -> str: return f"<{self.__class__.__name__}: {super().__repr__()}>" + # UserDict.copy() shallow-copies the instance, which would share self._keys + # between the copy and the original. + def __copy__(self) -> Self: + new = self.__class__() + new.data = self.data.copy() + new._keys = self._keys.copy() + return new + + copy = __copy__ + + # UserDict.__ior__ updates self.data directly, which would leave self._keys + # out of date. + def __ior__(self, other: Any) -> Self: # type: ignore[override,misc] + self.update(other) + return self + def _normkey(self, key: str | bytes) -> str | bytes: return key @@ -189,8 +213,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]): class SequenceExclude: """Object to test if an item is NOT within some sequence.""" - def __init__(self, seq: Sequence[Any]): - self.seq: Sequence[Any] = seq + def __init__(self, seq: Container[Any]): + self.seq: Container[Any] = seq def __contains__(self, item: Any) -> bool: return item not in self.seq diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index a5bb6fa24..2924c81f9 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -10,6 +10,7 @@ from twisted.internet.defer import Deferred, maybeDeferred from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncio import run_in_thread from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.python import _signature if TYPE_CHECKING: from collections.abc import AsyncGenerator, Callable, Coroutine @@ -19,9 +20,19 @@ _T = TypeVar("_T") _P = ParamSpec("_P") +@overload +def deprecated(use_instead: Callable[_P, _T]) -> Callable[_P, _T]: ... + + +@overload def deprecated( - use_instead: Any = None, -) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: + use_instead: str | None = None, +) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: ... + + +def deprecated( + use_instead: Callable[_P, _T] | str | None = None, +) -> Callable[_P, _T] | Callable[[Callable[_P, _T]], Callable[_P, _T]]: """This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted when the function is used.""" @@ -38,8 +49,9 @@ def deprecated( return wrapped if callable(use_instead): - deco = deco(use_instead) + func = use_instead use_instead = None + return deco(func) return deco @@ -98,7 +110,7 @@ def _warn_spider_arg( ): """Decorator to warn if a ``spider`` argument is passed to a function.""" - sig = inspect.signature(func) + sig = _signature(func) def check_args(*args: _P.args, **kwargs: _P.kwargs) -> None: bound = sig.bind(*args, **kwargs) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index d0259b634..7c6235f29 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -27,7 +27,7 @@ from twisted.internet.task import Cooperator from twisted.python import failure from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.asyncio import is_asyncio_available, sleep from scrapy.utils.python import global_object_name if TYPE_CHECKING: @@ -90,14 +90,7 @@ async def _defer_sleep_async() -> None: """Delay by _DEFER_DELAY so reactor has a chance to go through readers and writers before attending pending delayed calls, so do not set delay to zero. """ - if is_asyncio_available(): - await asyncio.sleep(_DEFER_DELAY) - else: - from twisted.internet import reactor - - d: Deferred[None] = Deferred() - reactor.callLater(_DEFER_DELAY, d.callback, None) - await d + await sleep(_DEFER_DELAY) def defer_result(result: Any) -> Deferred[Any]: # pragma: no cover diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index 58b4539bf..5965e1f21 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -16,8 +16,15 @@ _urlparse_cache: WeakKeyDictionary[Request | Response, ParseResult] = ( def urlparse_cached(request_or_response: Request | Response) -> ParseResult: - """Return urlparse.urlparse caching the result, where the argument can be a - Request or Response object + """Return the result of parsing the URL of *request_or_response*, a + :class:`~scrapy.Request` or :class:`~scrapy.http.Response` object, with + :func:`urllib.parse.urlparse`. + + The result is cached, using a :class:`weakref.WeakKeyDictionary` keyed on + *request_or_response*, so that the URL of a given object is parsed only + once. Prefer this function over calling :func:`urllib.parse.urlparse` on + ``request_or_response.url`` directly when the same URL may be parsed more + than once. """ if request_or_response not in _urlparse_cache: _urlparse_cache[request_or_response] = urlparse(request_or_response.url) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 20e7cb381..57b526be6 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -90,6 +90,11 @@ def load_object(path: str | Callable[..., Any]) -> Any: return obj +def _load_objects(objects: Iterable[str | Callable[..., Any]]) -> tuple[Any, ...]: + """Resolve *objects* (objects or import paths) to a tuple of objects.""" + return tuple(load_object(obj) if isinstance(obj, str) else obj for obj in objects) + + def walk_modules_iter(path: str) -> Iterable[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 8a7517c1d..40fc05257 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -178,11 +178,30 @@ def binary_is_text(data: bytes) -> bool: return all(c not in _BINARYCHARS for c in data) +# PEP 649 (Python 3.14+) made annotation evaluation lazy, so inspect.signature() +# can raise NameError for names imported only under TYPE_CHECKING. We only need +# parameter names, kinds and defaults, so leave such annotations as ForwardRefs. +if sys.version_info >= (3, 14): + from annotationlib import Format + + def _signature(func: Callable[..., Any]) -> inspect.Signature: + return inspect.signature(func, annotation_format=Format.FORWARDREF) + +else: + + def _signature(func: Callable[..., Any]) -> inspect.Signature: + return inspect.signature(func) + + def get_func_args_dict( func: Callable[..., Any], stripself: bool = False ) -> Mapping[str, inspect.Parameter]: """Return the argument dict of a callable object. + Annotations are not evaluated, so on Python 3.14 and later the ``annotation`` + attribute of the returned parameters may be a ``ForwardRef`` instead of the + resolved type. + .. versionadded:: 2.14 """ if not callable(func): @@ -190,7 +209,7 @@ def get_func_args_dict( args: Mapping[str, inspect.Parameter] try: - sig = inspect.signature(func) + sig = _signature(func) except ValueError: return {} diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 22f9eadd0..0bbe68def 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -34,7 +34,8 @@ live_refs: defaultdict[type, WeakKeyDictionary[object, float]] = defaultdict( class object_ref: - """Inherit from this class to a keep a record of live instances""" + """Inherit from this class if you want to track live instances with the + ``trackref`` module.""" __slots__ = () @@ -60,12 +61,19 @@ def format_live_refs(ignore: Any = NoneType) -> str: def print_live_refs(*a: Any, **kw: Any) -> None: - """Print tracked objects""" + """Print a report of live references, grouped by class name. + + :param ignore: if given, all objects from the specified class (or tuple of + classes) will be ignored. + :type ignore: type or tuple + """ print(format_live_refs(*a, **kw)) def get_oldest(class_name: str) -> Any: - """Get the oldest object for a specific class name""" + """Return the oldest object alive with the given class name, or ``None`` if + none is found. Use :func:`print_live_refs` first to get a list of all + tracked live objects per class name.""" for cls, wdict in live_refs.items(): if cls.__name__ == class_name: if not wdict: @@ -75,7 +83,9 @@ def get_oldest(class_name: str) -> Any: def iter_all(class_name: str) -> Iterable[Any]: - """Iterate over all objects of the same class by its class name""" + """Return an iterator over all objects alive with the given class name. Use + :func:`print_live_refs` first to get a list of all tracked live objects per + class name.""" for cls, wdict in live_refs.items(): if cls.__name__ == class_name: return wdict.keys() diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.py new file mode 100644 index 000000000..7b5ca0cb9 --- /dev/null +++ b/tests/benchmarks/__init__.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from scrapy.utils.test import get_crawler + +if TYPE_CHECKING: + from scrapy import Spider + from scrapy.crawler import Crawler + + +def crawl(spidercls: type[Spider], settings: dict[str, Any], **kwargs: Any) -> Crawler: + """Run a crawl to completion and return its crawler. + + Unlike the rest of the test suite, benchmarks run without ``pytest-twisted`` + and drive the reactor themselves, since the code being measured must be + callable synchronously by ``pytest-codspeed``. + """ + from twisted.internet import reactor + + crawler = get_crawler(spidercls, settings) + result: list[Any] = [] + crawler.crawl(**kwargs).addBoth(result.append) + while not result: + reactor.iterate(0.001) + if isinstance(result[0], BaseException): + raise result[0] + return crawler diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py new file mode 100644 index 000000000..55356083d --- /dev/null +++ b/tests/benchmarks/conftest.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from scrapy.utils.reactor import install_reactor + +if TYPE_CHECKING: + from collections.abc import Generator + + +@pytest.fixture(scope="session", autouse=True) +def running_reactor() -> Generator[None]: + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + + from twisted.internet import reactor + + # Marks the reactor as running without blocking, so that crawls can be + # driven with reactor.iterate(), see tests.benchmarks.crawl(). + reactor.startRunning(installSignalHandlers=False) + + yield + + reactor.stop() + # Lets the shutdown event triggers run, e.g. to join the thread pool. + reactor.iterate(0) diff --git a/tests/benchmarks/test_crawl.py b/tests/benchmarks/test_crawl.py new file mode 100644 index 000000000..0fdfe742b --- /dev/null +++ b/tests/benchmarks/test_crawl.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any +from urllib.parse import urlencode + +import pytest + +from scrapy import Field, Item, Request, Spider +from scrapy.linkextractors import LinkExtractor +from tests.benchmarks import crawl + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found] + + from scrapy.http import Response + from tests.mockserver.http import MockServer + +pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed") + +PAGES = 100 +LINKS_PER_PAGE = 5 + + +class _Page(Item): + url = Field() + anchors = Field() + + +class _FollowSpider(Spider): + name = "benchmark" + url: str + link_extractor = LinkExtractor() + + async def start(self) -> AsyncIterator[Any]: + yield Request(self.url, dont_filter=True) + + def parse(self, response: Response) -> Any: + yield _Page( + url=response.url, + anchors=response.css("a::text").getall(), + ) + for link in self.link_extractor.extract_links(response): # type: ignore[arg-type] + yield Request(link.url) + + +class _Pipeline: + def process_item(self, item: Any) -> Any: + return item + + +def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> None: + """Per-request overhead of a crawl over HTTP. + + The pages are small on purpose, so that the cost of parsing them stays + negligible next to the cost of moving requests and responses through the + engine, the middlewares and the download handler. + """ + query = urlencode({"total": PAGES, "show": LINKS_PER_PAGE, "order": "desc"}) + url = mockserver.url(f"/follow?{query}") + settings = {"ITEM_PIPELINES": {_Pipeline: 100}, "LOG_ENABLED": False} + + def run() -> None: + crawler = crawl(_FollowSpider, settings, url=url) + assert crawler.stats + assert crawler.stats.get_value("item_scraped_count") == PAGES + 1 + + benchmark(run) diff --git a/tests/mockserver/dns.py b/tests/mockserver/dns.py index e19a2e61a..2af018c66 100644 --- a/tests/mockserver/dns.py +++ b/tests/mockserver/dns.py @@ -2,6 +2,7 @@ from __future__ import annotations import sys from subprocess import PIPE, Popen +from typing import TYPE_CHECKING from twisted.internet import defer from twisted.names import dns, error @@ -9,39 +10,63 @@ from twisted.names.server import DNSServerFactory from tests.utils import get_script_run_env +if TYPE_CHECKING: + from collections.abc import Sequence + from types import TracebackType + + from twisted.internet.defer import Deferred + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + +_Answers = tuple[list[dns.RRHeader], list[dns.RRHeader], list[dns.RRHeader]] + class MockDNSResolver: """ Implements twisted.internet.interfaces.IResolver partially """ - def _resolve(self, name): + def _resolve(self, name: bytes) -> _Answers: record = dns.Record_A(address=b"127.0.0.1") - answer = dns.RRHeader(name=name, payload=record) + # zope.interface has no type hints, so mypy cannot tell that Record_A + # provides the IEncodableRecord interface. + answer = dns.RRHeader(name=name, payload=record) # type: ignore[arg-type] return [answer], [], [] - def query(self, query, timeout=None): + def query( + self, query: dns.Query, timeout: Sequence[int] | None = None + ) -> Deferred[_Answers]: if query.type == dns.A: return defer.succeed(self._resolve(query.name.name)) return defer.fail(error.DomainError()) - def lookupAllRecords(self, name, timeout=None): + def lookupAllRecords( + self, name: bytes, timeout: Sequence[int] | None = None + ) -> Deferred[_Answers]: return defer.succeed(self._resolve(name)) class MockDNSServer: - def __enter__(self): + def __enter__(self) -> Self: self.proc = Popen( [sys.executable, "-u", "-m", "tests.mockserver.dns"], stdout=PIPE, env=get_script_run_env(), text=True, ) + assert self.proc.stdout is not None self.host = "127.0.0.1" self.port = int(self.proc.stdout.readline().strip().split(":")[1]) return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: self.proc.kill() self.proc.communicate() @@ -54,7 +79,7 @@ def main() -> None: protocol = dns.DNSDatagramProtocol(controller=factory) listener = reactor.listenUDP(0, protocol) - def print_listening(): + def print_listening() -> None: host = listener.getHost() print(f"{host.host}:{host.port}") diff --git a/tests/mockserver/ftp.py b/tests/mockserver/ftp.py index 22efc966b..1edd64dda 100644 --- a/tests/mockserver/ftp.py +++ b/tests/mockserver/ftp.py @@ -7,6 +7,7 @@ from pathlib import Path from shutil import rmtree from subprocess import PIPE, Popen from tempfile import mkdtemp +from typing import TYPE_CHECKING from pyftpdlib.authorizers import DummyAuthorizer from pyftpdlib.handlers import FTPHandler @@ -14,6 +15,12 @@ from pyftpdlib.servers import FTPServer from tests.utils import get_script_run_env +if TYPE_CHECKING: + from types import TracebackType + + # typing.Self requires Python 3.11 + from typing_extensions import Self + class MockFTPServer: """Creates an FTP server on a random port with a default passwordless user @@ -26,7 +33,7 @@ class MockFTPServer: self.port: int | None = None self.path: Path | None = None - def __enter__(self): + def __enter__(self) -> Self: self.path = Path(mkdtemp()) self.proc = Popen( [sys.executable, "-u", "-m", "tests.mockserver.ftp", "-d", str(self.path)], @@ -34,6 +41,7 @@ class MockFTPServer: env=get_script_run_env(), text=True, ) + assert self.proc.stderr is not None for line in self.proc.stderr: if "starting FTP server" in line and ( m := re.search(r"starting FTP server on ([^ :]+):(\d+),", line) @@ -48,12 +56,18 @@ class MockFTPServer: ) return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: rmtree(str(self.path)) + assert self.proc is not None self.proc.kill() self.proc.communicate() - def url(self, path): + def url(self, path: str) -> str: return f"ftp://{self.host}:{self.port}/{path}" diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py index 7ad873c02..c4fd4464e 100644 --- a/tests/mockserver/http.py +++ b/tests/mockserver/http.py @@ -1,8 +1,8 @@ from __future__ import annotations from pathlib import Path +from typing import TYPE_CHECKING -from twisted.web import resource from twisted.web.static import Data, File from twisted.web.util import Redirect @@ -11,6 +11,7 @@ from tests import tests_datadir from .http_base import BaseMockServer, main_factory from .http_resources import ( ArbitraryLengthPayloadResource, + BaseResource, BrokenChunkedResource, BrokenDownloadResource, ChunkedResource, @@ -35,62 +36,68 @@ from .http_resources import ( SetCookie, Status, UriResource, + put_child, ) +if TYPE_CHECKING: + from twisted.web.server import Request -class Root(resource.Resource): - def __init__(self): + +class Root(BaseResource): + def __init__(self) -> None: super().__init__() - self.putChild(b"status", Status()) - self.putChild(b"follow", Follow()) - self.putChild(b"delay", Delay()) - self.putChild(b"partial", Partial()) - self.putChild(b"drop", Drop()) - self.putChild(b"raw", Raw()) - self.putChild(b"echo", Echo()) - self.putChild(b"payload", PayloadResource()) - self.putChild(b"alpayload", ArbitraryLengthPayloadResource()) - self.putChild(b"static", File(str(Path(tests_datadir, "test_site/")))) - self.putChild(b"redirect-to", RedirectTo()) - self.putChild(b"text", Data(b"Works", "text/plain")) - self.putChild( + put_child(self, b"status", Status()) + put_child(self, b"follow", Follow()) + put_child(self, b"delay", Delay()) + put_child(self, b"partial", Partial()) + put_child(self, b"drop", Drop()) + put_child(self, b"raw", Raw()) + put_child(self, b"echo", Echo()) + put_child(self, b"payload", PayloadResource()) + put_child(self, b"alpayload", ArbitraryLengthPayloadResource()) + put_child(self, b"static", File(str(Path(tests_datadir, "test_site/")))) + put_child(self, b"redirect-to", RedirectTo()) + put_child(self, b"text", Data(b"Works", "text/plain")) + put_child( + self, b"html", Data( b"

Works

World

", "text/html", ), ) - self.putChild( + put_child( + self, b"enc-gb18030", Data(b"

gb18030 encoding

", "text/html; charset=gb18030"), ) - self.putChild(b"redirect", Redirect(b"/redirected")) - self.putChild( - b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected") + put_child(self, b"redirect", Redirect(b"/redirected")) + put_child( + self, b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected") ) - self.putChild(b"redirected", Data(b"Redirected here", "text/plain")) + put_child(self, b"redirected", Data(b"Redirected here", "text/plain")) numbers = [str(x).encode("utf8") for x in range(2**18)] - self.putChild(b"numbers", Data(b"".join(numbers), "text/plain")) - self.putChild(b"wait", ForeverTakingResource()) - self.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) - self.putChild(b"host", HostHeaderResource()) - self.putChild(b"client-ip", ClientIPResource()) - self.putChild(b"broken", BrokenDownloadResource()) - self.putChild(b"chunked", ChunkedResource()) - self.putChild(b"broken-chunked", BrokenChunkedResource()) - self.putChild(b"contentlength", ContentLengthHeaderResource()) - self.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) - self.putChild(b"largechunkedfile", LargeChunkedFileResource()) - self.putChild(b"compress", Compress()) - self.putChild(b"duplicate-header", DuplicateHeaderResource()) - self.putChild(b"response-headers", ResponseHeadersResource()) - self.putChild(b"set-cookie", SetCookie()) - self.putChild(b"uri", UriResource()) + put_child(self, b"numbers", Data(b"".join(numbers), "text/plain")) + put_child(self, b"wait", ForeverTakingResource()) + put_child(self, b"hang-after-headers", ForeverTakingResource(write=True)) + put_child(self, b"host", HostHeaderResource()) + put_child(self, b"client-ip", ClientIPResource()) + put_child(self, b"broken", BrokenDownloadResource()) + put_child(self, b"chunked", ChunkedResource()) + put_child(self, b"broken-chunked", BrokenChunkedResource()) + put_child(self, b"contentlength", ContentLengthHeaderResource()) + put_child(self, b"nocontenttype", EmptyContentTypeHeaderResource()) + put_child(self, b"largechunkedfile", LargeChunkedFileResource()) + put_child(self, b"compress", Compress()) + put_child(self, b"duplicate-header", DuplicateHeaderResource()) + put_child(self, b"response-headers", ResponseHeadersResource()) + put_child(self, b"set-cookie", SetCookie()) + put_child(self, b"uri", UriResource()) - def getChild(self, path, request): + def getChild(self, path: bytes, request: Request) -> Root: return self - def render(self, request): + def render(self, request: Request) -> bytes: return b"Scrapy mock HTTP server\n" diff --git a/tests/mockserver/http_base.py b/tests/mockserver/http_base.py index 7b38409ff..5bc1252d6 100644 --- a/tests/mockserver/http_base.py +++ b/tests/mockserver/http_base.py @@ -17,9 +17,13 @@ from .utils import ssl_context_factory if TYPE_CHECKING: from collections.abc import Callable + from types import TracebackType from twisted.web import resource + # typing.Self requires Python 3.11 + from typing_extensions import Self + class BaseMockServer(ABC): listen_http: bool = True @@ -39,13 +43,14 @@ class BaseMockServer(ABC): self.http_port: int | None = None self.https_port: int | None = None - def __enter__(self): + def __enter__(self) -> Self: self.proc = Popen( [sys.executable, "-u", "-m", self.module_name, *self.get_additional_args()], stdout=PIPE, env=get_script_run_env(), text=True, ) + assert self.proc.stdout is not None if self.listen_http: http_address = self.proc.stdout.readline().strip() http_parsed = urlparse(http_address) @@ -56,7 +61,12 @@ class BaseMockServer(ABC): self.https_port = https_parsed.port return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: if self.proc: self.proc.kill() self.proc.communicate() @@ -131,7 +141,7 @@ def main_factory( context_factory = ssl_context_factory(**context_factory_kw) https_port = reactor.listenSSL(0, factory, context_factory) - def print_listening(): + def print_listening() -> None: if listen_http: http_host = http_port.getHost() http_address = f"http://{http_host.host}:{http_host.port}" diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.py index 98ac6cf6a..cb028bc10 100644 --- a/tests/mockserver/http_resources.py +++ b/tests/mockserver/http_resources.py @@ -3,7 +3,7 @@ from __future__ import annotations import gzip import json import random -from typing import TYPE_CHECKING, ParamSpec, TypeVar +from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar from urllib.parse import urlencode from twisted.internet.task import deferLater @@ -14,17 +14,24 @@ from twisted.web.util import Redirect, redirectTo from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Sequence from twisted.internet.defer import Deferred - from twisted.web.http import Request + from twisted.python.failure import Failure + from twisted.web.http import Request as HTTPRequest + from twisted.web.server import Request _T = TypeVar("_T") _P = ParamSpec("_P") -def getarg(request, name, default=None, type_=None): +def getarg( + request: Request, + name: bytes, + default: Any = None, + type_: Callable[[bytes], Any] | None = None, +) -> Any: if name in request.args: value = request.args[name][0] if type_ is not None: @@ -33,73 +40,91 @@ def getarg(request, name, default=None, type_=None): return default -def close_connection(request): +def close_connection(request: Request) -> None: # We have to force a disconnection for HTTP/1.1 clients. Otherwise # client keeps the connection open waiting for more data. request.channel.loseConnection() request.finish() +def put_child(parent: resource.Resource, path: bytes, child: resource.Resource) -> None: + # zope.interface has no type hints, so mypy cannot tell that Resource + # instances provide the IResource interface that putChild() expects. + parent.putChild(path, child) # type: ignore[arg-type] + + +class BaseResource(resource.Resource): + """Base class for mockserver resources, with type hints.""" + + # Only needed to give subclasses a typed __init__ to call. + def __init__(self) -> None: # pylint: disable=useless-parent-delegation + super().__init__() # type: ignore[no-untyped-call] + + # most of the following resources are copied from twisted.web.test.test_webclient -class ForeverTakingResource(resource.Resource): +class ForeverTakingResource(BaseResource): """ L{ForeverTakingResource} is a resource which never finishes responding to requests. """ - def __init__(self, write=False): - resource.Resource.__init__(self) + def __init__(self, write: bool = False): + super().__init__() self._write = write - def render(self, request): + def render(self, request: Request) -> int: if self._write: request.write(b"some bytes") return server.NOT_DONE_YET -class HostHeaderResource(resource.Resource): +class HostHeaderResource(BaseResource): """ A testing resource which renders itself as the value of the host header from the request. """ - def render(self, request): - return request.requestHeaders.getRawHeaders(b"host")[0] + def render(self, request: Request) -> bytes: + headers = request.requestHeaders.getRawHeaders(b"host") + assert headers + return headers[0] -class ClientIPResource(resource.Resource): +class ClientIPResource(BaseResource): """ A testing resource which renders itself as the request client IP address. """ - def render(self, request): + def render(self, request: Request) -> bytes: client_address = request.getClientAddress() if client_address is None or client_address.host is None: return b"" return to_bytes(client_address.host) -class PayloadResource(resource.Resource): +class PayloadResource(BaseResource): """ A testing resource which renders itself as the contents of the request body as long as the request body is 100 bytes long, otherwise which renders itself as C{"ERROR"}. """ - def render(self, request): - data = request.content.read() - contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0] - if len(data) != 100 or int(contentLength) != 100: + def render(self, request: Request) -> bytes: + assert request.content + data: bytes = request.content.read() + content_length = request.requestHeaders.getRawHeaders(b"content-length") + assert content_length + if len(data) != 100 or int(content_length[0]) != 100: return b"ERROR" return data -class LeafResource(resource.Resource): +class LeafResource(BaseResource): isLeaf = True def deferRequest( self, - request: Request, + request: HTTPRequest, delay: float, f: Callable[_P, _T], *a: _P.args, @@ -107,7 +132,7 @@ class LeafResource(resource.Resource): ) -> Deferred[_T]: from twisted.internet import reactor - def _cancelrequest(_): + def _cancelrequest(_: Failure) -> None: # silence CancelledError d.addErrback(lambda _: None) d.cancel() @@ -118,12 +143,13 @@ class LeafResource(resource.Resource): class Follow(LeafResource): - def render(self, request): + def render(self, request: Request) -> int: total = getarg(request, b"total", 100, type_=int) show = getarg(request, b"show", 1, type_=int) order = getarg(request, b"order", b"desc") maxlatency = getarg(request, b"maxlatency", 0, type_=float) n = getarg(request, b"n", total, type_=int) + nlist: Sequence[int] if order == b"rand": nlist = [random.randint(1, total) for _ in range(show)] else: # order == "desc" @@ -133,7 +159,7 @@ class Follow(LeafResource): self.deferRequest(request, lag, self.renderRequest, request, nlist) return NOT_DONE_YET - def renderRequest(self, request, nlist): + def renderRequest(self, request: Request, nlist: Sequence[int]) -> None: s = """ """ args = request.args.copy() for nl in nlist: @@ -146,45 +172,47 @@ class Follow(LeafResource): class Delay(LeafResource): - def render_GET(self, request): + def render_GET(self, request: Request) -> int: n = getarg(request, b"n", 1, type_=float) b = getarg(request, b"b", 1, type_=int) if b: # send headers now and delay body - request.write("") + request.write(b"") self.deferRequest(request, n, self._delayedRender, request, n) return NOT_DONE_YET - def _delayedRender(self, request, n): + def _delayedRender(self, request: Request, n: float) -> None: request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n")) request.finish() class Status(LeafResource): - def render_GET(self, request): + def render_GET(self, request: Request) -> bytes: n = getarg(request, b"n", 200, type_=int) request.setResponseCode(n) return b"" class Raw(LeafResource): - def render_GET(self, request): + def render_GET(self, request: Request) -> int: request.startedWriting = 1 self.deferRequest(request, 0, self._delayedRender, request) return NOT_DONE_YET render_POST = render_GET - def _delayedRender(self, request): + def _delayedRender(self, request: Request) -> None: raw = getarg(request, b"raw", b"HTTP 1.1 200 OK\n") request.startedWriting = 1 request.write(raw) + assert request.channel.transport is not None request.channel.transport.loseConnection() request.finish() class Echo(LeafResource): - def render_GET(self, request): + def render_GET(self, request: Request) -> bytes: + assert request.content output = { "headers": { to_unicode(k): [to_unicode(v) for v in vs] @@ -198,27 +226,29 @@ class Echo(LeafResource): class RedirectTo(LeafResource): - def render(self, request): + def render(self, request: Request) -> bytes: goto = getarg(request, b"goto", b"/") # we force the body content, otherwise Twisted redirectTo() # returns HTML with int: request.setHeader(b"Content-Length", b"1024") self.deferRequest(request, 0, self._delayedRender, request) return NOT_DONE_YET - def _delayedRender(self, request): + def _delayedRender(self, request: Request) -> None: request.write(b"partial content\n") request.finish() class Drop(Partial): - def _delayedRender(self, request): + def _delayedRender(self, request: Request) -> None: abort = getarg(request, b"abort", 0, type_=int) request.write(b"this connection will be dropped\n") tr = request.channel.transport @@ -233,8 +263,10 @@ class Drop(Partial): class ArbitraryLengthPayloadResource(LeafResource): - def render(self, request): - return request.content.read() + def render(self, request: Request) -> bytes: + assert request.content + data: bytes = request.content.read() + return data class NoMetaRefreshRedirect(Redirect): @@ -245,21 +277,23 @@ class NoMetaRefreshRedirect(Redirect): ) -class ContentLengthHeaderResource(resource.Resource): +class ContentLengthHeaderResource(BaseResource): """ A testing resource which renders itself as the value of the Content-Length header from the request. """ - def render(self, request): - return request.requestHeaders.getRawHeaders(b"content-length")[0] + def render(self, request: Request) -> bytes: + headers = request.requestHeaders.getRawHeaders(b"content-length") + assert headers + return headers[0] -class ChunkedResource(resource.Resource): - def render(self, request): +class ChunkedResource(BaseResource): + def render(self, request: Request) -> int: from twisted.internet import reactor - def response(): + def response() -> None: request.write(b"chunked ") request.write(b"content\n") request.finish() @@ -268,11 +302,11 @@ class ChunkedResource(resource.Resource): return server.NOT_DONE_YET -class BrokenChunkedResource(resource.Resource): - def render(self, request): +class BrokenChunkedResource(BaseResource): + def render(self, request: Request) -> int: from twisted.internet import reactor - def response(): + def response() -> None: request.write(b"chunked ") request.write(b"content\n") # Disable terminating chunk on finish. @@ -283,11 +317,11 @@ class BrokenChunkedResource(resource.Resource): return server.NOT_DONE_YET -class BrokenDownloadResource(resource.Resource): - def render(self, request): +class BrokenDownloadResource(BaseResource): + def render(self, request: Request) -> int: from twisted.internet import reactor - def response(): + def response() -> None: request.setHeader(b"Content-Length", b"20") request.write(b"partial") close_connection(request) @@ -296,22 +330,24 @@ class BrokenDownloadResource(resource.Resource): return server.NOT_DONE_YET -class EmptyContentTypeHeaderResource(resource.Resource): +class EmptyContentTypeHeaderResource(BaseResource): """ A testing resource which renders itself as the value of request body without content-type header in response. """ - def render(self, request): + def render(self, request: Request) -> bytes: + assert request.content request.setHeader("content-type", "") - return request.content.read() + data: bytes = request.content.read() + return data -class LargeChunkedFileResource(resource.Resource): - def render(self, request): +class LargeChunkedFileResource(BaseResource): + def render(self, request: Request) -> int: from twisted.internet import reactor - def response(): + def response() -> None: for _ in range(1024): request.write(b"x" * 1024) request.finish() @@ -320,43 +356,45 @@ class LargeChunkedFileResource(resource.Resource): return server.NOT_DONE_YET -class DuplicateHeaderResource(resource.Resource): - def render(self, request): +class DuplicateHeaderResource(BaseResource): + def render(self, request: Request) -> bytes: request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"]) return b"" -class UriResource(resource.Resource): +class UriResource(BaseResource): """Return the full uri that was requested""" - def getChild(self, path, request): + def getChild(self, path: bytes, request: Request) -> resource.Resource: return self - def render(self, request): + def render(self, request: Request) -> bytes | int: # Note: this is an ugly hack for CONNECT request timeout test. # Returning some data here fail SSL/TLS handshake # ToDo: implement proper HTTPS proxy tests, not faking them. if request.method != b"CONNECT": return request.uri + assert request.transport is not None request.transport.write(b"HTTP/1.1 200 Connection established\r\n\r\n") return NOT_DONE_YET -class ResponseHeadersResource(resource.Resource): +class ResponseHeadersResource(BaseResource): """Return a response with headers set from the JSON request body""" - def render(self, request): + def render(self, request: Request) -> bytes: + assert request.content body = json.loads(request.content.read().decode()) for header_name, header_value in body.items(): request.responseHeaders.setRawHeaders(header_name, [header_value]) return json.dumps(body).encode("utf-8") -class Compress(resource.Resource): +class Compress(BaseResource): """Compress the data sent in the request url params and set Content-Encoding header""" - def render(self, request): - data = request.args.get(b"data")[0] + def render(self, request: Request) -> bytes: + data = request.args[b"data"][0] accept_encoding_header = request.getHeader(b"accept-encoding") @@ -370,10 +408,10 @@ class Compress(resource.Resource): return b"Did not receive a valid accept-encoding header" -class SetCookie(resource.Resource): +class SetCookie(BaseResource): """Return a response with a Set-Cookie header for each request url parameter""" - def render(self, request): + def render(self, request: Request) -> bytes: for cookie_name, cookie_values in request.args.items(): for cookie_value in cookie_values: cookie = (cookie_name.decode() + "=" + cookie_value.decode()).encode() diff --git a/tests/mockserver/simple_https.py b/tests/mockserver/simple_https.py index fdea666e1..2a6cb6dd8 100644 --- a/tests/mockserver/simple_https.py +++ b/tests/mockserver/simple_https.py @@ -2,18 +2,23 @@ from __future__ import annotations -from twisted.web import resource +from typing import TYPE_CHECKING + from twisted.web.static import Data from .http_base import BaseMockServer, main_factory +from .http_resources import BaseResource, put_child + +if TYPE_CHECKING: + from twisted.web.server import Request -class Root(resource.Resource): - def __init__(self): - resource.Resource.__init__(self) - self.putChild(b"file", Data(b"0123456789", "text/plain")) +class Root(BaseResource): + def __init__(self) -> None: + super().__init__() + put_child(self, b"file", Data(b"0123456789", "text/plain")) - def getChild(self, path, request): + def getChild(self, path: bytes, request: Request) -> Root: return self @@ -29,7 +34,7 @@ class SimpleMockServer(BaseMockServer): cipher_string: str | None = None, tls_min_version: str | None = None, tls_max_version: str | None = None, - ): + ) -> None: super().__init__() self.keyfile = keyfile self.certfile = certfile diff --git a/tests/spiders.py b/tests/spiders.py index da14fdbe3..7c7d3007c 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -38,6 +38,35 @@ class MockServerSpider(Spider): self.is_secure = is_secure +class RawResponseSpider(MockServerSpider): + """Base class for spiders that fetch a response built by the test itself. + + Subclasses return the body from :meth:`raw_body` and request + :attr:`raw_url`, which the mock server answers with that body verbatim + under :attr:`content_type`. This lets tests reach parsing code that only + a specific kind of response triggers while still going through a regular + crawl, instead of calling internal parsing methods directly. + """ + + name = "raw_response" + content_type = "text/plain" + + def raw_body(self) -> str: + raise NotImplementedError + + @property + def raw_url(self) -> str: + assert self.mockserver + raw = ( + "HTTP/1.1 200 OK\r\n" + f"Content-Type: {self.content_type}\r\n" + "Connection: close\r\n" + "\r\n" + f"{self.raw_body()}" + ) + return self.mockserver.url("/raw?" + urlencode({"raw": raw})) + + class MetaSpider(MockServerSpider): name = "meta" @@ -496,6 +525,23 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): self.logger.info("[errback] status %i", failure.value.response.status) +class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod): + name = "crawl_spider_without_errback" + + async def start(self): + test_body = b""" + + Page title + +

Item 200

+

Item 404

+ + + """ + url = self.mockserver.url("/alpayload") + yield Request(url, method="POST", body=test_body) + + class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParseMethod): name = "crawl_spider_with_process_request_cb_kwargs" rules = ( diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 794c6d400..240f44584 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -7,7 +7,7 @@ from unittest import TestCase from unittest.mock import MagicMock, Mock, PropertyMock, call, patch from scrapy.commands.check import Command, TextTestResult -from tests.utils.base_commands import TestProjectBase +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import proc if TYPE_CHECKING: diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py index 6293e973e..5306e3bf8 100644 --- a/tests/test_command_crawl.py +++ b/tests/test_command_crawl.py @@ -2,7 +2,7 @@ from __future__ import annotations from typing import TYPE_CHECKING -from tests.utils.base_commands import TestProjectBase +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import proc if TYPE_CHECKING: @@ -23,6 +23,18 @@ class TestCrawlCommand(TestProjectBase): _, _, stderr = self.crawl(code, proj_path, args=args) return stderr + def test_no_spider(self, proj_path: Path) -> None: + returncode, out, _ = proc("crawl", cwd=proj_path) + assert returncode == 2 + assert "Usage" in out + + def test_multiple_spiders(self, proj_path: Path) -> None: + returncode, _, err = proc("crawl", "myspider", "myspider2", cwd=proj_path) + assert returncode == 2 + assert ( + "running 'scrapy crawl' with more than one spider is not supported" in err + ) + def test_no_output(self, proj_path: Path) -> None: spider_code = """ import scrapy diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index d98dac968..c6a3afc91 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -2,13 +2,24 @@ from __future__ import annotations from typing import TYPE_CHECKING +import pytest + +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import proc if TYPE_CHECKING: + from pathlib import Path + from tests.mockserver.http import MockServer class TestFetchCommand: + @pytest.mark.parametrize("args", [(), ("not-a-url",), ("a:b", "c:d")]) + def test_bad_arguments(self, args: tuple[str, ...]) -> None: + returncode, out, _ = proc("fetch", *args) + assert returncode == 2 + assert "Usage" in out + def test_output(self, mockserver: MockServer) -> None: _, out, _ = proc("fetch", mockserver.url("/text")) assert out.strip() == "Works" @@ -36,3 +47,24 @@ class TestFetchCommand: "fetch", "-s", "TWISTED_REACTOR_ENABLED=False", mockserver.url("/text") ) assert out.strip() == "Works" + + +class TestFetchCommandWithSpider(TestProjectBase): + @pytest.fixture(autouse=True) + def create_files(self, proj_path: Path) -> None: + (proj_path / self.project_name / "spiders" / "myspider.py").write_text( + """ +import scrapy + +class MySpider(scrapy.Spider): + name = "myspider" + custom_settings = {"USER_AGENT": "myspider-user-agent"} +""", + encoding="utf-8", + ) + + def test_spider(self, proj_path: Path, mockserver: MockServer) -> None: + _, out, err = proc( + "fetch", "--spider", "myspider", mockserver.url("/echo"), cwd=proj_path + ) + assert "myspider-user-agent" in out, err diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py index f465d0b30..ddf25af4c 100644 --- a/tests/test_command_genspider.py +++ b/tests/test_command_genspider.py @@ -6,7 +6,7 @@ from pathlib import Path import pytest -from tests.utils.base_commands import TestProjectBase +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import call, proc, write_recording_editor @@ -64,6 +64,24 @@ class TestGenspiderCommand(TestProjectBase): assert call("genspider", "--dump=basic", cwd=proj_path) == 0 assert call("genspider", "-d", "basic", cwd=proj_path) == 0 + @pytest.mark.parametrize( + "args", + [("--dump=nonexistent",), ("-t", "nonexistent", "test_name", "test.com")], + ) + def test_unknown_template(self, args: tuple[str, ...], proj_path: Path) -> None: + returncode, out, err = proc("genspider", *args, cwd=proj_path) + assert returncode == 0, err + assert "Unable to find template: nonexistent" in out + assert not (proj_path / self.project_name / "spiders" / "test_name.py").exists() + + def test_name_not_starting_with_a_letter(self, proj_path: Path) -> None: + """The module name, unlike the spider name, is prefixed with a letter.""" + _, out, err = proc("genspider", "1st_spider", "test.com", cwd=proj_path) + assert "Created spider '1st_spider'" in out, err + spider = proj_path / self.project_name / "spiders" / "a1st_spider.py" + assert spider.exists() + assert find_in_file(spider, r'name\s*=\s*"1st_spider"') is not None + @pytest.mark.skipif( sys.platform == "win32", reason="requires a POSIX shell editor script" ) @@ -87,7 +105,8 @@ class TestGenspiderCommand(TestProjectBase): ) def test_same_name_as_project(self, proj_path: Path) -> None: - assert call("genspider", self.project_name, cwd=proj_path) == 2 + _, out, err = proc("genspider", self.project_name, "test.com", cwd=proj_path) + assert "Cannot create a spider with the same name as your project" in out, err assert not ( proj_path / self.project_name / "spiders" / f"{self.project_name}.py" ).exists() diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index a3581d764..e434055b7 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -3,12 +3,13 @@ from __future__ import annotations import argparse import re from typing import TYPE_CHECKING +from urllib.parse import urlparse import pytest from scrapy.commands import parse from scrapy.settings import Settings -from tests.utils.base_commands import TestProjectBase +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import call, proc if TYPE_CHECKING: @@ -126,6 +127,32 @@ class MySpider(scrapy.Spider): else: self.logger.debug('It Does Not Work :(') +class RetryRequestSpider(BaseSpider): + name = 'retry_request' + + def parse(self, response): + if response.meta.get('retried'): + yield {{'retried': True}} + return + response.meta['retried'] = True + yield response.request.replace(dont_filter=True) + +class CustomCallbackRetryRequestSpider(BaseSpider): + name = 'retry_request_custom_callback' + + def parse(self, response): + yield response.request.replace( + callback=self.parse_retry, + dont_filter=True, + ) + + def parse_retry(self, response): + if response.meta.get('retried'): + yield {{'retried_with_custom_callback': True}} + return + response.meta['retried'] = True + yield response.request.replace(dont_filter=True) + class MyGoodCrawlSpider(CrawlSpider): name = 'goodcrawl{self.spider_name}' @@ -381,6 +408,36 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "[{}, {'foo': 'bar'}]" in out + def test_retry_response_request( + self, proj_path: Path, mockserver: MockServer + ) -> None: + _, out, stderr = proc( + "parse", + "--spider", + "retry_request", + "-d", + "2", + mockserver.url("/html"), + cwd=proj_path, + ) + assert "RecursionError" not in stderr + assert "{'retried': True}" in out + + def test_retry_response_request_with_custom_callback( + self, proj_path: Path, mockserver: MockServer + ) -> None: + _, out, stderr = proc( + "parse", + "--spider", + "retry_request_custom_callback", + "-d", + "3", + mockserver.url("/html"), + cwd=proj_path, + ) + assert "RecursionError" not in stderr + assert "{'retried_with_custom_callback': True}" in out + def test_wrong_callback_passed( self, proj_path: Path, mockserver: MockServer ) -> None: @@ -496,6 +553,130 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} content = '[\n{},\n{"foo": "bar"}\n]' assert file_path.read_text(encoding="utf-8") == content + @pytest.mark.parametrize("args", [(), ("not-a-url",), ("a:b", "c:d")]) + def test_bad_arguments(self, args: tuple[str, ...], proj_path: Path) -> None: + returncode, out, _ = proc("parse", *args, cwd=proj_path) + assert returncode == 2 + assert "Usage" in out + + @pytest.mark.parametrize( + ("option", "message"), + [ + ("--meta", "Invalid -m/--meta value"), + ("-m", "Invalid -m/--meta value"), + ("--cbkwargs", "Invalid --cbkwargs value"), + ], + ) + def test_invalid_json( + self, option: str, message: str, proj_path: Path, mockserver: MockServer + ) -> None: + returncode, _, err = proc( + "parse", + "--spider", + self.spider_name, + option, + "{invalid", + mockserver.url("/html"), + cwd=proj_path, + ) + assert returncode == 2 + assert message in err + + def test_unknown_spider(self, proj_path: Path, mockserver: MockServer) -> None: + returncode, _, err = proc( + "parse", + "--spider", + "nonexistent", + mockserver.url("/html"), + cwd=proj_path, + ) + assert returncode == 0, err + assert "Unable to find spider: nonexistent" in err + + def test_spider_found_by_url(self, proj_path: Path, mockserver: MockServer) -> None: + """Without --spider, the spider is chosen based on the URL.""" + url = mockserver.url("/html") + # The spider name doubles as a domain of the spider, and it is matched + # against the netloc of the URL, hence the port. + (proj_path / self.project_name / "spiders" / "urlspider.py").write_text( + f""" +import scrapy + +class UrlSpider(scrapy.Spider): + name = "{urlparse(url).netloc}" + + def parse(self, response): + return [{{"found_by_url": True}}] +""", + encoding="utf-8", + ) + returncode, out, err = proc("parse", url, cwd=proj_path) + assert returncode == 0, err + assert "Unable to find spider for" not in err + assert "{'found_by_url': True}" in out + + def test_legacy_item_processor( + self, proj_path: Path, mockserver: MockServer + ) -> None: + """--pipelines supports an ITEM_PROCESSOR without process_item_async().""" + (proj_path / self.project_name / "legacy.py").write_text( + """ +import logging + +from twisted.internet.defer import succeed + + +class LegacyItemProcessor: + @classmethod + def from_crawler(cls, crawler): + return cls() + + def open_spider(self, spider): + return succeed(None) + + def close_spider(self, spider): + return succeed(None) + + def process_item(self, item, spider): + logging.info("Legacy item processor!") + return succeed(item) +""", + encoding="utf-8", + ) + _, _, stderr = proc( + "parse", + "--spider", + self.spider_name, + "--pipelines", + "-c", + "parse", + "-s", + f"ITEM_PROCESSOR={self.project_name}.legacy.LegacyItemProcessor", + mockserver.url("/html"), + cwd=proj_path, + ) + assert "INFO: Legacy item processor!" in stderr + + @pytest.mark.parametrize("verbose", [True, False]) + def test_no_items_no_links( + self, verbose: bool, proj_path: Path, mockserver: MockServer + ) -> None: + args = ["--verbose"] if verbose else [] + _, out, err = proc( + "parse", + "--spider", + self.spider_name, + "-c", + "parse", + "--noitems", + "--nolinks", + *args, + mockserver.url("/html"), + cwd=proj_path, + ) + assert "# Scraped Items" not in out, err + assert "# Requests" not in out + def test_parse_add_options(self): command = parse.Command() command.settings = Settings() diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index 2b410b5c6..11036eaeb 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -136,6 +136,12 @@ class MySpider(scrapy.Spider): log = self.get_log(tmp_path, "from scrapy.spiders import Spider\n") assert "No spider found in file" in log + @pytest.mark.parametrize("args", [(), ("a.py", "b.py")]) + def test_runspider_bad_arguments(self, args: tuple[str, ...]) -> None: + returncode, out, _ = proc("runspider", *args) + assert returncode == 2 + assert "Usage" in out + def test_runspider_file_not_found(self) -> None: _, _, log = proc("runspider", "some_non_existent_file") assert "File not found: some_non_existent_file" in log diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index f24200f53..29667a1ae 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -18,6 +18,7 @@ from scrapy.shell import Shell, inspect_response from scrapy.utils.reactor import _asyncio_reactor_path from scrapy.utils.test import get_crawler from tests import NON_EXISTING_RESOLVABLE, tests_datadir +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import proc from tests.utils.decorators import coroutine_test @@ -162,6 +163,33 @@ class TestShellCommand: assert ret == 0, out +class TestShellCommandWithSpider(TestProjectBase): + @pytest.fixture(autouse=True) + def create_files(self, proj_path: Path) -> None: + (proj_path / self.project_name / "spiders" / "myspider.py").write_text( + """ +import scrapy + +class MySpider(scrapy.Spider): + name = "myspider" +""", + encoding="utf-8", + ) + + def test_spider(self, proj_path: Path, mockserver: MockServer) -> None: + ret, out, err = proc( + "shell", + "--spider", + "myspider", + mockserver.url("/text"), + "-c", + "spider.name", + cwd=proj_path, + ) + assert ret == 0, err + assert out.strip() == "myspider" + + class TestInteractiveShell: def test_fetch(self, mockserver: MockServer) -> None: args = ( diff --git a/tests/test_commands.py b/tests/test_commands.py index 9e7d4d5a1..f20ecc153 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3,23 +3,27 @@ from __future__ import annotations import argparse import json import sys -from io import StringIO +from pathlib import Path from typing import TYPE_CHECKING -from unittest import mock import pytest import scrapy -from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg -from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view +from scrapy.cmdline import _pop_command_name, execute +from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings from scrapy.utils.reactor import _asyncio_reactor_path -from tests.utils.base_commands import TestProjectBase -from tests.utils.cmdline import call, proc, write_recording_editor +from tests.utils.bases.commands import TestProjectBase +from tests.utils.cmdline import ( + call, + proc, + write_recording_browser, + write_recording_editor, +) if TYPE_CHECKING: - from pathlib import Path + from tests.mockserver.http import MockServer class EmptyCommand(ScrapyCommand): @@ -109,6 +113,93 @@ class TestCommandSettings: ) +class TestGlobalOptions: + """Tests for the options that every command supports.""" + + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = "myspider" + + async def start(self): + self.logger.debug("It works!") + return + yield +""" + + @pytest.fixture + def spider_path(self, tmp_path: Path) -> Path: + path = tmp_path / "myspider.py" + path.write_text(self.spider_code, encoding="utf-8") + return path + + def test_invalid_set(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "-s", "FOO") + assert returncode == 2 + assert "Invalid -s value, use -s NAME=VALUE" in err + + def test_invalid_spider_argument(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "-a", "FOO") + assert returncode == 2 + assert "Invalid -a value, use -a NAME=VALUE" in err + + def test_logfile(self, tmp_path: Path, spider_path: Path) -> None: + logfile = tmp_path / "scrapy.log" + returncode, _, err = proc( + "runspider", str(spider_path), "--logfile", str(logfile) + ) + assert returncode == 0, err + assert "It works!" in logfile.read_text(encoding="utf-8") + assert "It works!" not in err + + def test_loglevel(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "--loglevel", "INFO") + assert returncode == 0, err + assert "It works!" not in err + assert "Spider closed (finished)" in err + + def test_nolog(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "--nolog") + assert returncode == 0, err + assert not err + + def test_pidfile(self, tmp_path: Path, spider_path: Path) -> None: + pidfile = tmp_path / "scrapy.pid" + returncode, _, err = proc( + "runspider", str(spider_path), "--pidfile", str(pidfile) + ) + assert returncode == 0, err + assert pidfile.read_text(encoding="utf-8").strip().isdigit() + + def test_pdb(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "--pdb") + assert returncode == 0, err + assert "It works!" in err + + +class TestSettingsCommand: + @pytest.mark.parametrize( + ("option", "setting", "expected"), + [ + ("--get", "BOT_NAME", "scrapybot"), + ("--getbool", "COOKIES_ENABLED", "True"), + ("--getint", "CONCURRENT_REQUESTS", "16"), + ("--getfloat", "DOWNLOAD_DELAY", "0.0"), + ("--getlist", "SPIDER_MODULES", "[]"), + ], + ) + def test_get(self, option: str, setting: str, expected: str) -> None: + returncode, out, err = proc("settings", option, setting) + assert returncode == 0, err + assert out.startswith(expected) + + def test_no_option(self) -> None: + returncode, out, err = proc("settings") + assert returncode == 0, err + assert not out + + class TestCommandCrawlerProcess(TestProjectBase): """Test that the command uses the expected kind of *CrawlerProcess and produces expected errors when needed.""" @@ -153,12 +244,6 @@ class MySpider(scrapy.Spider): self._append_settings(proj_mod_path, "LOG_LEVEL = 'DEBUG'\n") - @staticmethod - def _append_settings(proj_mod_path: Path, text: str) -> None: - """Add text to the end of the project settings.py.""" - with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f: - f.write(text) - @staticmethod def _replace_custom_settings( proj_mod_path: Path, spider_name: str, text: str @@ -347,23 +432,223 @@ class TestMiscCommands(TestProjectBase): subdir.mkdir(exist_ok=True) assert call("list", cwd=subdir) == 0 - def test_command_not_found(self) -> None: - na_msg = """ -The list command is not available from this location. -These commands are only available from within a project: check, crawl, edit, list, parse. -""" - not_found_msg = """ -Unknown command: abc -""" - params = [ - ("list", False, na_msg), - ("abc", False, not_found_msg), - ("abc", True, not_found_msg), - ] - for cmdname, inproject, message in params: - with mock.patch("sys.stdout", new=StringIO()) as out: - _print_unknown_command_msg(Settings(), cmdname, inproject) - assert out.getvalue().strip() == message.strip() + +class TestCommandListing(TestProjectBase): + """Tests for the command list that ``scrapy`` prints when called without a + command name.""" + + def test_outside_project(self) -> None: + returncode, out, err = proc() + assert returncode == 0, err + assert f"Scrapy {scrapy.__version__} - no active project" in out + assert "Available commands:" in out + assert "Create new project" in out + assert "More commands available when run from project directory" in out + assert 'Use "scrapy -h" to see more info about a command' in out + + def test_inside_project(self, proj_path: Path) -> None: + returncode, out, err = proc(cwd=proj_path) + assert returncode == 0, err + assert ( + f"Scrapy {scrapy.__version__} - active project: {self.project_name}" in out + ) + assert "List available spiders" in out + assert "More commands available when run from project directory" not in out + + +class TestUnknownCommand(TestProjectBase): + def test_outside_project(self) -> None: + returncode, out, err = proc("abc") + assert returncode == 2, err + assert f"Scrapy {scrapy.__version__} - no active project" in out + assert "Unknown command: abc" in out + assert 'Use "scrapy" to see available commands' in out + + def test_inside_project(self, proj_path: Path) -> None: + returncode, out, err = proc("abc", cwd=proj_path) + assert returncode == 2, err + assert ( + f"Scrapy {scrapy.__version__} - active project: {self.project_name}" in out + ) + assert "Unknown command: abc" in out + + def test_project_only_command_outside_project(self) -> None: + returncode, out, err = proc("list") + assert returncode == 2, err + assert "The list command is not available from this location." in out + assert ( + "These commands are only available from within a project: " + "check, crawl, edit, list, parse." in out + ) + + +class TestCommandsModule(TestProjectBase): + """Tests for commands defined in the module of the COMMANDS_MODULE setting.""" + + @pytest.fixture + def proj_path_with_commands(self, proj_path: Path) -> Path: + commands_path = proj_path / self.project_name / "commands" + commands_path.mkdir() + (commands_path / "__init__.py").touch() + (commands_path / "mycmd.py").write_text( + """ +from scrapy.commands import ScrapyCommand + + +class Command(ScrapyCommand): + requires_crawler_process = False + + def short_desc(self): + return "My custom command" + + def run(self, args, opts): + print("My custom command ran") +""", + encoding="utf-8", + ) + (commands_path / "helpcmd.py").write_text( + """ +from scrapy.commands import ScrapyCommand +from scrapy.exceptions import UsageError + + +class Command(ScrapyCommand): + requires_crawler_process = False + + def short_desc(self): + return "My command that asks for its help message" + + def run(self, args, opts): + raise UsageError +""", + encoding="utf-8", + ) + (commands_path / "silentcmd.py").write_text( + """ +from scrapy.commands import ScrapyCommand +from scrapy.exceptions import UsageError + + +class Command(ScrapyCommand): + requires_crawler_process = False + + def short_desc(self): + return "My command that fails silently" + + def run(self, args, opts): + raise UsageError(print_help=False) +""", + encoding="utf-8", + ) + self._append_settings( + proj_path / self.project_name, + f'\nCOMMANDS_MODULE = "{self.project_name}.commands"\n', + ) + return proj_path + + def test_listed(self, proj_path_with_commands: Path) -> None: + returncode, out, err = proc(cwd=proj_path_with_commands) + assert returncode == 0, err + assert "My custom command" in out + + def test_run(self, proj_path_with_commands: Path) -> None: + returncode, out, err = proc("mycmd", cwd=proj_path_with_commands) + assert returncode == 0, err + assert "My custom command ran" in out + + def test_usage_error(self, proj_path_with_commands: Path) -> None: + """A message-less UsageError makes the help message be printed.""" + returncode, out, err = proc("helpcmd", cwd=proj_path_with_commands) + assert returncode == 2, err + assert "scrapy helpcmd" in out + + def test_usage_error_without_help(self, proj_path_with_commands: Path) -> None: + """A message-less UsageError with print_help disabled prints nothing.""" + returncode, out, err = proc("silentcmd", cwd=proj_path_with_commands) + assert returncode == 2, err + assert not out + + +class TestEntryPointCommands: + """Tests for commands defined in the scrapy.commands entry point group.""" + + @staticmethod + def _write_dist(path: Path, entry_point: str) -> None: + """Write into *path* a package with a command and a function, and the + metadata of an installed distribution that declares *entry_point* in + the scrapy.commands entry point group. + + Since ``python -m scrapy.cmdline`` puts the current working directory + in the import path, running it with *path* as the working directory + makes Scrapy find that entry point. + """ + package_path = path / "mycmds" + package_path.mkdir() + (package_path / "__init__.py").touch() + (package_path / "mycmd.py").write_text( + """ +from scrapy.commands import ScrapyCommand + + +class Command(ScrapyCommand): + requires_crawler_process = False + + def short_desc(self): + return "My entry point command" + + def run(self, args, opts): + print("My entry point command ran") + + +def not_a_command(): + pass +""", + encoding="utf-8", + ) + dist_info_path = path / "mycmds-1.0.dist-info" + dist_info_path.mkdir() + (dist_info_path / "METADATA").write_text( + "Metadata-Version: 2.1\nName: mycmds\nVersion: 1.0\n", encoding="utf-8" + ) + (dist_info_path / "entry_points.txt").write_text( + f"[scrapy.commands]\n{entry_point}\n", encoding="utf-8" + ) + + def test_listed(self, tmp_path: Path) -> None: + self._write_dist(tmp_path, "mycmd = mycmds.mycmd:Command") + returncode, out, err = proc(cwd=tmp_path) + assert returncode == 0, err + assert "My entry point command" in out + + def test_run(self, tmp_path: Path) -> None: + self._write_dist(tmp_path, "mycmd = mycmds.mycmd:Command") + returncode, out, err = proc("mycmd", cwd=tmp_path) + assert returncode == 0, err + assert "My entry point command ran" in out + + def test_not_a_class(self, tmp_path: Path) -> None: + self._write_dist(tmp_path, "mycmd = mycmds.mycmd:not_a_command") + returncode, _, err = proc("version", cwd=tmp_path) + assert returncode == 1 + assert "ValueError: Invalid entry point mycmd" in err + + +class TestExecute: + """Tests for calls to scrapy.cmdline.execute() from Python code, which the + command line does not cover.""" + + def test_argv(self, capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit) as exc_info: + execute(["scrapy", "version"]) + assert exc_info.value.code == 0 + assert scrapy.__version__ in capsys.readouterr().out + + def test_settings(self, capsys: pytest.CaptureFixture[str]) -> None: + settings = Settings() + with pytest.raises(SystemExit) as exc_info: + execute(["scrapy", "settings", "--get", "BOT_NAME"], settings=settings) + assert exc_info.value.code == 0 + assert capsys.readouterr().out.strip() == "scrapybot" class TestBenchCommand: @@ -385,18 +670,31 @@ class TestBenchCommand: class TestViewCommand: - def test_methods(self) -> None: - command = view.Command() - command.settings = Settings() - parser = argparse.ArgumentParser( - prog="scrapy", - prefix_chars="-", - formatter_class=ScrapyHelpFormatter, - conflict_handler="resolve", + @pytest.mark.skipif( + sys.platform == "win32", reason="requires a POSIX shell browser script" + ) + def test_view( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mockserver: MockServer + ) -> None: + opened = tmp_path / "opened.txt" + browser = tmp_path / "fake-browser.sh" + write_recording_browser(browser, opened) + monkeypatch.setenv("BROWSER", str(browser)) + + returncode, _, err = proc("view", mockserver.url("/html"), cwd=tmp_path) + + assert returncode == 0, err + url = opened.read_text(encoding="utf-8") + assert url.startswith("file://") + body = Path(url.removeprefix("file://")).read_text(encoding="utf-8") + assert "

Works

" in body + + def test_non_text_response(self, mockserver: MockServer) -> None: + returncode, _, err = proc( + "view", mockserver.url("/static/files/images/scrapy.png") ) - command.add_options(parser) - assert command.short_desc() == "Open URL in browser, as seen by Scrapy" - assert "URL using the Scrapy downloader and show its" in command.long_desc() + assert returncode == 0, err + assert "Cannot view a non-text response." in err class TestEditCommand(TestProjectBase): @@ -423,6 +721,11 @@ class TestEditCommand(TestProjectBase): assert returncode == 1 assert "Spider not found: nonexistent" in err + def test_edit_no_spider(self, proj_path: Path) -> None: + returncode, out, _ = proc("edit", cwd=proj_path) + assert returncode == 2 + assert "Usage" in out + class TestHelpMessage(TestProjectBase): @pytest.mark.parametrize( diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index fdd5edc27..912c0450b 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -14,6 +14,7 @@ from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse +from scrapy import Request, Spider from scrapy.core.downloader import Downloader, Slot, tls from scrapy.core.downloader.contextfactory import ( _load_context_factory_from_settings, @@ -30,14 +31,17 @@ from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -from tests.mockserver.http_resources import PayloadResource +from tests.mockserver.http_resources import PayloadResource, put_child from tests.mockserver.utils import ssl_context_factory from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from twisted.internet.defer import Deferred + from twisted.internet.interfaces import IListeningPort from twisted.web.iweb import IBodyProducer + from scrapy.http import Response + class TestSlot: def test_repr(self): @@ -51,7 +55,7 @@ class TestContextFactoryBase: async def server_url(self, tmp_path): (tmp_path / "file").write_bytes(b"0123456789") r = static.File(str(tmp_path)) - r.putChild(b"payload", PayloadResource()) + put_child(r, b"payload", PayloadResource()) site = server.Site(r, timeout=None) port = self._listen(site) portno = port.getHost().port @@ -60,7 +64,7 @@ class TestContextFactoryBase: await port.stopListening() - def _listen(self, site): + def _listen(self, site: server.Site) -> IListeningPort: from twisted.internet import reactor return reactor.listenSSL( @@ -296,10 +300,30 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase): await self._assert_factory_works(server_url, client_context_factory) +@pytest.mark.parametrize( + ("concurrency", "active", "expected"), + [ + (2, 1, False), + (2, 2, True), + (0, 0, False), + (0, 2, False), + ], +) +def test_needs_backout(concurrency: int, active: int, expected: bool) -> None: + crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency}) + downloader = Downloader(crawler) + downloader.active = {Request(f"https://example.com/{i}") for i in range(active)} + assert downloader.needs_backout() is expected + downloader.close() + + @coroutine_test async def test_fetch_deprecated_spider_arg(): class CustomDownloader(Downloader): - def fetch(self, request, spider): # pylint: disable=signature-differs + # requiring the spider argument is what triggers the deprecation + def fetch( # type: ignore[override] # pylint: disable=signature-differs + self, request: Request, spider: Spider + ) -> Deferred[Response | Request]: return super().fetch(request, spider) crawler = get_crawler(DefaultSpider, {"DOWNLOADER": CustomDownloader}) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 9c479068a..d284805be 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -41,6 +41,7 @@ from tests.spiders import ( CrawlSpiderWithAsyncCallback, CrawlSpiderWithAsyncGeneratorCallback, CrawlSpiderWithErrback, + CrawlSpiderWithoutErrback, CrawlSpiderWithParseMethod, CrawlSpiderWithProcessRequestCallbackKeywordArguments, DelaySpider, @@ -505,6 +506,21 @@ class TestCrawlSpider: assert "[errback] status 500" in caplog.text assert "[errback] status 501" in caplog.text + @coroutine_test + async def test_crawlspider_without_errback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + crawler = get_crawler(CrawlSpiderWithoutErrback) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) + + # The failing request (404) is followed by a rule without an errback, + # so the failure is dropped silently and the crawl finishes normally. + assert "[parse] status 200 (foo: None)" in caplog.text + assert "[errback]" not in caplog.text + assert crawler.stats + assert crawler.stats.get_value("downloader/response_status_count/404") == 1 + @coroutine_test async def test_crawlspider_process_request_cb_kwargs( self, caplog: pytest.LogCaptureFixture, mockserver: MockServer diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ec335d815..d77097bd4 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -131,6 +131,39 @@ class TestCrawler: any(name in msg for msg in warned_messages) for name in setting_names ) + @pytest.mark.parametrize( + ("attr", "setting"), + [ + ("download_delay", "DOWNLOAD_DELAY"), + ("max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN"), + ], + ) + def test_deprecated_spider_attr(self, attr: str, setting: str) -> None: + crawler = get_raw_crawler(type("_Spider", (DefaultSpider,), {attr: 2})) + with pytest.warns( + ScrapyDeprecationWarning, + match=f"The {attr!r} spider attribute is deprecated. Use the {setting} ", + ): + crawler._apply_settings() + assert crawler.settings.getint(setting) == 2 + + @pytest.mark.parametrize( + ("attr", "setting"), + [ + ("download_delay", "DOWNLOAD_DELAY"), + ("max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN"), + ], + ) + def test_deprecated_spider_attr_ignored(self, attr: str, setting: str) -> None: + crawler = get_raw_crawler(type("_Spider", (DefaultSpider,), {attr: 2})) + crawler.settings.set(setting, 3, priority="spider") + with pytest.warns( + ScrapyDeprecationWarning, + match=f"The {attr!r} spider attribute is deprecated. It is also being ", + ): + crawler._apply_settings() + assert crawler.settings.getint(setting) == 3 + def test_crawler_accepts_dict(self) -> None: crawler = get_crawler(DefaultSpider, {"foo": "bar"}) assert crawler.settings["foo"] == "bar" diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index fe2f83161..733b6797d 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -14,7 +14,8 @@ from packaging.version import parse as parse_version from pexpect.popen_spawn import PopenSpawn from w3lib import __version__ as w3lib_version -from tests.utils import async_sleep, get_script_run_env +from scrapy.utils.asyncio import sleep +from tests.utils import get_script_run_env from tests.utils.decorators import coroutine_test if TYPE_CHECKING: @@ -244,7 +245,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): p.kill(sig) p.expect_exact("shutting down gracefully") # sending the second signal too fast often causes problems - await async_sleep(0.01) + await sleep(0.01) p.kill(sig) p.expect_exact("forcing unclean shutdown") p.wait() # type: ignore[no-untyped-call] diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index fd50248a7..976daacaf 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -15,7 +15,9 @@ from scrapy.core.downloader.handlers._httpx import ( HttpxDownloadHandler, ) from scrapy.exceptions import DownloadFailedError -from tests.test_downloader_handlers_http_base import ( +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler +from tests.utils.bases.download_handlers_http import ( TestHttpBase, TestHttpProxyBase, TestHttpsBase, @@ -161,3 +163,15 @@ class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase): @pytest.mark.requires_internet class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase): pass + + +@pytest.mark.parametrize(("concurrency", "expected"), [(16, 16), (0, None)]) +@coroutine_test +async def test_pool_limits(concurrency: int, expected: int | None) -> None: + crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency}) + handler = build_from_crawler(HttpxDownloadHandler, crawler) + try: + assert handler._limits.max_connections == expected + assert handler._limits.max_keepalive_connections == expected + finally: + await handler.close() diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py index 489b70e74..14de97b21 100644 --- a/tests/test_downloader_handler_twisted_ftp.py +++ b/tests/test_downloader_handler_twisted_ftp.py @@ -156,14 +156,17 @@ class TestFTP(TestFTPBase): for filename, content in self.test_files: (userdir / filename).write_bytes(content) - def _get_factory(self, root): + def _get_factory(self, root: Path) -> FTPFactory: from twisted.protocols.ftp import FTPFactory, FTPRealm realm = FTPRealm(anonymousRoot=str(root), userHome=str(root)) - p = portal.Portal(realm) + # zope.interface has no type hints, so mypy cannot tell that these + # objects provide the interfaces that Portal expects. + p = portal.Portal(realm) # type: ignore[arg-type] users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() - users_checker.addUser(self.username, self.password) - p.registerChecker(users_checker, credentials.IUsernamePassword) + # the FTP protocol authenticates with str credentials + users_checker.addUser(self.username, self.password) # type: ignore[arg-type] + p.registerChecker(users_checker, credentials.IUsernamePassword) # type: ignore[arg-type] return FTPFactory(portal=p) @deferred_f_from_coro_f @@ -192,12 +195,17 @@ class TestAnonymousFTP(TestFTPBase): for filename, content in self.test_files: (root / filename).write_bytes(content) - def _get_factory(self, tmp_path): + def _get_factory(self, tmp_path: Path) -> FTPFactory: from twisted.protocols.ftp import FTPFactory, FTPRealm realm = FTPRealm(anonymousRoot=str(tmp_path)) - p = portal.Portal(realm) - p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous) + # zope.interface has no type hints, so mypy cannot tell that these + # objects provide the interfaces that Portal expects. + p = portal.Portal(realm) # type: ignore[arg-type] + p.registerChecker( + checkers.AllowAnonymousAccess(), # type: ignore[arg-type] + credentials.IAnonymous, + ) return FTPFactory(portal=p, userAnonymous=self.username) diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py index 79750a136..32dfd7540 100644 --- a/tests/test_downloader_handler_twisted_http11.py +++ b/tests/test_downloader_handler_twisted_http11.py @@ -11,7 +11,7 @@ from scrapy import Spider from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from tests.test_downloader_handlers_http_base import ( +from tests.utils.bases.download_handlers_http import ( TestHttpBase, TestHttpProxyBase, TestHttpsBase, diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index bea97642e..449f2d635 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -2,18 +2,18 @@ from __future__ import annotations +import logging import sys from typing import TYPE_CHECKING, Any import pytest -from testfixtures import LogCapture from twisted.web.http import H2_ENABLED from scrapy import Spider from scrapy.crawler import Crawler from scrapy.exceptions import DownloadFailedError, NotConfigured from scrapy.http import Request -from tests.test_downloader_handlers_http_base import ( +from tests.utils.bases.download_handlers_http import ( TestHttpProxyBase, TestHttpsBase, TestHttpsCustomCiphersBase, @@ -130,24 +130,24 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase): assert response.text == custom_content_length @coroutine_test - async def test_custom_content_length_bad(self, mockserver: MockServer) -> None: + async def test_custom_content_length_bad( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: request = Request(mockserver.url("/contentlength", is_secure=self.is_secure)) actual_content_length = str(len(request.body)) bad_content_length = str(len(request.body) + 1) request.headers["Content-Length"] = bad_content_length async with self.get_dh() as download_handler: - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): response = await download_handler.download_request(request) assert response.text == actual_content_length - log.check_present( - ( - "scrapy.core.http2.stream", - "WARNING", - f"Ignoring bad Content-Length header " - f"{bad_content_length!r} of request {request}, sending " - f"{actual_content_length!r} instead", - ) - ) + assert ( + "scrapy.core.http2.stream", + logging.WARNING, + f"Ignoring bad Content-Length header " + f"{bad_content_length!r} of request {request}, sending " + f"{actual_content_length!r} instead", + ) in caplog.record_tuples @coroutine_test async def test_data_loss_handling(self, mockserver: MockServer) -> None: diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index b4419dc67..8d999d952 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -1,8 +1,8 @@ import logging from collections.abc import Iterable +from typing import Any import pytest -from testfixtures import LogCapture from scrapy.downloadermiddlewares.cookies import CookiesMiddleware from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware @@ -111,14 +111,15 @@ class TestCookiesMiddleware: CookiesMiddleware, ) - def test_setting_enabled_cookies_debug(self): + def test_setting_enabled_cookies_debug( + self, caplog: pytest.LogCaptureFixture + ) -> None: crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True}) mw = CookiesMiddleware.from_crawler(crawler) - with LogCapture( - "scrapy.downloadermiddlewares.cookies", - propagate=False, - level=logging.DEBUG, - ) as log: + caplog.clear() + with caplog.at_level( + logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies" + ): req = Request("http://scrapytest.org/") res = Response( "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"} @@ -127,43 +128,44 @@ class TestCookiesMiddleware: req2 = Request("http://scrapytest.org/sub1/") mw.process_request(req2) - log.check( - ( - "scrapy.downloadermiddlewares.cookies", - "DEBUG", - "Received cookies from: <200 http://scrapytest.org/>\n" - "Set-Cookie: C1=value1; path=/\n", - ), - ( - "scrapy.downloadermiddlewares.cookies", - "DEBUG", - "Sending cookies to: \n" - "Cookie: C1=value1\n", - ), - ) + assert caplog.record_tuples == [ + ( + "scrapy.downloadermiddlewares.cookies", + logging.DEBUG, + "Received cookies from: <200 http://scrapytest.org/>\n" + "Set-Cookie: C1=value1; path=/\n", + ), + ( + "scrapy.downloadermiddlewares.cookies", + logging.DEBUG, + "Sending cookies to: \n" + "Cookie: C1=value1\n", + ), + ] - def test_debug_no_cookies(self): + def test_debug_no_cookies(self, caplog: pytest.LogCaptureFixture) -> None: crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True}) mw = CookiesMiddleware.from_crawler(crawler) - with LogCapture( - "scrapy.downloadermiddlewares.cookies", - propagate=False, - level=logging.DEBUG, - ) as log: + caplog.clear() + with caplog.at_level( + logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies" + ): req = Request("http://scrapytest.org/") res = Response("http://scrapytest.org/") # no Set-Cookie header mw.process_response(req, res) mw.process_request(req) # no cookies to send either - log.check() # no log output since cl is empty in both cases + # no log output since cl is empty in both cases + assert caplog.record_tuples == [] - def test_setting_disabled_cookies_debug(self): + def test_setting_disabled_cookies_debug( + self, caplog: pytest.LogCaptureFixture + ) -> None: crawler = get_crawler(settings_dict={"COOKIES_DEBUG": False}) mw = CookiesMiddleware.from_crawler(crawler) - with LogCapture( - "scrapy.downloadermiddlewares.cookies", - propagate=False, - level=logging.DEBUG, - ) as log: + caplog.clear() + with caplog.at_level( + logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies" + ): req = Request("http://scrapytest.org/") res = Response( "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"} @@ -172,7 +174,7 @@ class TestCookiesMiddleware: req2 = Request("http://scrapytest.org/sub1/") mw.process_request(req2) - log.check() + assert caplog.record_tuples == [] def test_do_not_break_on_non_utf8_header(self): req = Request("http://scrapytest.org/") @@ -218,7 +220,7 @@ class TestCookiesMiddleware: def test_complex_cookies(self): # merge some cookies into jar - cookies = [ + cookies: list[VerboseCookie] = [ { "name": "C1", "value": "value1", @@ -420,44 +422,41 @@ class TestCookiesMiddleware: assert self.mw.process_request(req3) is None self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1") - def test_invalid_cookies(self): + def test_invalid_cookies(self, caplog: pytest.LogCaptureFixture) -> None: """ Invalid cookies are logged as warnings and discarded """ - with LogCapture( - "scrapy.utils.request", - propagate=False, - level=logging.INFO, - ) as lc: + caplog.clear() + with caplog.at_level(logging.INFO, logger="scrapy.utils.request"): cookies1 = [{"value": "bar"}, {"name": "key", "value": "value1"}] - req1 = Request("http://example.org/1", cookies=cookies1) + req1 = Request("http://example.org/1", cookies=cookies1) # type: ignore[arg-type] assert self.mw.process_request(req1) is None cookies2 = [{"name": "foo"}, {"name": "key", "value": "value2"}] - req2 = Request("http://example.org/2", cookies=cookies2) + req2 = Request("http://example.org/2", cookies=cookies2) # type: ignore[arg-type] assert self.mw.process_request(req2) is None cookies3 = [{"name": "foo", "value": None}, {"name": "key", "value": ""}] - req3 = Request("http://example.org/3", cookies=cookies3) + req3 = Request("http://example.org/3", cookies=cookies3) # type: ignore[arg-type] assert self.mw.process_request(req3) is None - lc.check( - ( - "scrapy.utils.request", - "WARNING", - "Invalid cookie found in request :" - " {'value': 'bar', 'secure': False} ('name' is missing)", - ), - ( - "scrapy.utils.request", - "WARNING", - "Invalid cookie found in request :" - " {'name': 'foo', 'secure': False} ('value' is missing)", - ), - ( - "scrapy.utils.request", - "WARNING", - "Invalid cookie found in request :" - " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)", - ), - ) + assert caplog.record_tuples == [ + ( + "scrapy.utils.request", + logging.WARNING, + "Invalid cookie found in request :" + " {'value': 'bar', 'secure': False} ('name' is missing)", + ), + ( + "scrapy.utils.request", + logging.WARNING, + "Invalid cookie found in request :" + " {'name': 'foo', 'secure': False} ('value' is missing)", + ), + ( + "scrapy.utils.request", + logging.WARNING, + "Invalid cookie found in request :" + " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)", + ), + ] self.assertCookieValEqual(req1.headers["Cookie"], "key=value1") self.assertCookieValEqual(req2.headers["Cookie"], "key=value2") self.assertCookieValEqual(req3.headers["Cookie"], "key=") @@ -485,13 +484,13 @@ class TestCookiesMiddleware: def _test_cookie_redirect( self, - source, - target, + source: str | dict[str, Any], + target: str | dict[str, Any], *, - cookies1, - cookies2, - ): - input_cookies = {"a": "b"} + cookies1: bool, + cookies2: bool, + ) -> None: + input_cookies: CookiesT = {"a": "b"} if not isinstance(source, dict): source = {"url": source} @@ -553,11 +552,11 @@ class TestCookiesMiddleware: def _test_cookie_header_redirect( self, - source, - target, + source: str | dict[str, Any], + target: str | dict[str, Any], *, - cookies2, - ): + cookies2: bool, + ) -> None: """Test the handling of a user-defined Cookie header when building a redirect follow-up request. @@ -625,14 +624,14 @@ class TestCookiesMiddleware: def _test_user_set_cookie_domain_followup( self, - url1, - url2, - domain, + url1: str, + url2: str, + domain: str, *, - cookies1, - cookies2, - ): - input_cookies = [ + cookies1: bool, + cookies2: bool, + ) -> None: + input_cookies: list[VerboseCookie] = [ { "name": "a", "value": "b", @@ -688,16 +687,16 @@ class TestCookiesMiddleware: def _test_server_set_cookie_domain_followup( self, - url1, - url2, - domain, + url1: str, + url2: str, + domain: str, *, - cookies, - ): + cookies: bool, + ) -> None: request1 = Request(url1) self.mw.process_request(request1) - input_cookies = [ + input_cookies: list[VerboseCookie] = [ { "name": "a", "value": "b", @@ -749,8 +748,14 @@ class TestCookiesMiddleware: ) def _test_cookie_redirect_scheme_change( - self, secure, from_scheme, to_scheme, cookies1, cookies2, cookies3 - ): + self, + secure: bool | object, + from_scheme: str, + to_scheme: str, + cookies1: bool, + cookies2: bool, + cookies3: bool, + ) -> None: """When a redirect causes the URL scheme to change from *from_scheme* to *to_scheme*, while domain and port remain the same, and given a cookie on the initial request with its secure attribute set to @@ -758,10 +763,11 @@ class TestCookiesMiddleware: initial request (*cookies1*), if it should be kept by the redirect middleware (*cookies2*), and if it should be present on the Cookie header in the redirected request (*cookie3*).""" - cookie_kwargs = {} + cookie: VerboseCookie = {"name": "a", "value": "b"} if secure is not UNSET: - cookie_kwargs["secure"] = secure - input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}] + assert isinstance(secure, bool) + cookie["secure"] = secure + input_cookies = [cookie] request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies) self.mw.process_request(request1) diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 827133d2e..dd5af3bc5 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,3 +1,5 @@ +from typing import Any + import pytest from w3lib.http import basic_auth_header @@ -10,8 +12,10 @@ from scrapy.utils.test import get_crawler _DOMAIN_NOT_SET = object() -def make_mw(user="", passwd="", domain=_DOMAIN_NOT_SET): - settings: dict = { +def make_mw( + user: str = "", passwd: str = "", domain: str | object = _DOMAIN_NOT_SET +) -> HttpAuthMiddleware: + settings: dict[str, Any] = { "HTTPAUTH_USER": user, "HTTPAUTH_PASS": passwd, } diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 9d5d6874e..dc8228470 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -1,10 +1,12 @@ from __future__ import annotations import email.utils +import logging import shutil import tempfile import time from contextlib import contextmanager +from pathlib import Path from typing import TYPE_CHECKING, Any from unittest import mock @@ -12,6 +14,7 @@ import pytest from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware from scrapy.exceptions import IgnoreRequest +from scrapy.extensions.httpcache import DummyPolicy from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider from scrapy.utils.test import get_crawler @@ -22,6 +25,14 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler +class AlwaysStalePolicy(DummyPolicy): + """:class:`~scrapy.extensions.httpcache.DummyPolicy` that always + revalidates cached responses.""" + + def is_cached_response_fresh(self, cachedresponse, request): + return False + + class TestBase: """Base class with common setup and helper methods.""" @@ -83,16 +94,22 @@ class TestBase: finally: mw.spider_closed(crawler.spider) - def assertEqualResponse(self, response1, response2): + def assertEqualResponse(self, response1: Response, response2: Response) -> None: assert response1.url == response2.url assert response1.status == response2.status assert response1.headers == response2.headers assert response1.body == response2.body -class StorageTestMixin: +class StorageTestMixin(TestBase): """Mixin containing storage-specific test methods.""" + def _corrupt_cache_entry( + self, storage: Any, spider: Spider, request: Request + ) -> None: + """Make the cache entry of *request* unreadable for *storage*.""" + raise NotImplementedError + def test_storage(self): with self._storage(HTTPCACHE_EXPIRATION_SECS=1) as (storage, crawler): request2 = self.request.copy() @@ -115,6 +132,42 @@ class StorageTestMixin: with mock.patch("scrapy.extensions.httpcache.time", return_value=future): assert storage.retrieve_response(crawler.spider, self.request) + def test_corrupted_cache_entry_is_a_miss(self, caplog): + with self._middleware() as mw: + spider = mw.crawler.spider + assert spider + assert mw.crawler.stats + mw.storage.store_response(spider, self.request, self.response) + self._corrupt_cache_entry(mw.storage, spider, self.request) + + caplog.clear() + with caplog.at_level(logging.WARNING): + assert mw.process_request(self.request) is None + + assert "treating it as a cache miss" in caplog.text + assert mw.crawler.stats.get_value("httpcache/retrieve_error") == 1 + assert mw.crawler.stats.get_value("httpcache/miss") == 1 + + # Storing the response again replaces the corrupted cache entry. + mw.storage.store_response(spider, self.request, self.response) + self.assertEqualResponse( + self.response, mw.storage.retrieve_response(spider, self.request) + ) + + def test_corrupted_cache_entry_ignore_missing(self): + with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw: + spider = mw.crawler.spider + assert spider + assert mw.crawler.stats + mw.storage.store_response(spider, self.request, self.response) + self._corrupt_cache_entry(mw.storage, spider, self.request) + + with pytest.raises(IgnoreRequest): + mw.process_request(self.request) + + assert mw.crawler.stats.get_value("httpcache/retrieve_error") == 1 + assert mw.crawler.stats.get_value("httpcache/ignore") == 1 + def test_storage_no_content_type_header(self): """Test that the response body is used to get the right response class even if there is no Content-Type header""" @@ -131,7 +184,7 @@ class StorageTestMixin: self.assertEqualResponse(response, cached_response) -class PolicyTestMixin: +class PolicyTestMixin(TestBase): """Mixin containing policy-specific test methods.""" def test_dont_cache(self): @@ -242,6 +295,22 @@ class DummyPolicyTestMixin(PolicyTestMixin): self.assertEqualResponse(self.response, response) assert "cached" in response.flags + def test_revalidation_keeps_cached_response(self): + # The dummy policy considers every cached response valid, so a policy + # that subclasses it to force revalidation always gets the cached + # response back, whatever the new response is. + with self._middleware(HTTPCACHE_POLICY=AlwaysStalePolicy) as mw: + assert mw.process_request(self.request) is None + mw.process_response(self.request, self.response) + + assert mw.process_request(self.request) is None + fresh_response = self.response.replace(body=b"new body") + response = mw.process_response(self.request, fresh_response) + assert isinstance(response, Response) + self.assertEqualResponse(self.response, response) + assert "cached" in response.flags + assert mw.stats.get_value("httpcache/revalidate") == 1 + class RFC2616PolicyTestMixin(PolicyTestMixin): """Mixin containing RFC2616 policy specific test methods.""" @@ -249,12 +318,12 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): @staticmethod def _process_requestresponse( mw: HttpCacheMiddleware, request: Request, response: Response | None - ) -> Response | Request: - result = None + ) -> Response: + result: Request | Response | None = None try: result = mw.process_request(request) if result: - assert isinstance(result, (Request, Response)) + assert isinstance(result, Response) return result assert response is not None result = mw.process_response(request, response) @@ -282,6 +351,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): res2 = self._process_requestresponse(mw, req0, res0) assert "cached" not in res2.flags res3 = mw.process_request(req0) + assert isinstance(res3, Response) assert "cached" in res3.flags self.assertEqualResponse(res2, res3) # request with no-cache directive must not return cached response @@ -513,6 +583,53 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): else: assert "cached" in res5.flags + def test_middleware_ignore_schemes(self): + # file responses are not cached by default + req = Request("file:///tmp/t.txt") + res = Response(req.url, headers={"Expires": self.tomorrow}) + with self._middleware() as mw: + assert mw.process_request(req) is None + mw.process_response(req, res) + + assert mw.storage.retrieve_response(mw.crawler.spider, req) is None + assert mw.process_request(req) is None + + def test_max_stale_with_value(self): + # A response that expired one day ago. + headers = {"Date": self.yesterday, "Expires": self.yesterday} + with self._middleware() as mw: + req0 = Request("http://example.com") + res0 = Response(req0.url, headers=headers) + self._process_requestresponse(mw, req0, res0) + + # max-stale greater than the staleness of the cached response + req1 = req0.replace(headers={"Cache-Control": "max-stale=172800"}) + res1 = mw.process_request(req1) + assert isinstance(res1, Response) + assert "cached" in res1.flags + + # max-stale lower than the staleness of the cached response + req2 = req0.replace(headers={"Cache-Control": "max-stale=60"}) + assert mw.process_request(req2) is None + + # a non-integer max-stale value is ignored + req3 = req0.replace(headers={"Cache-Control": "max-stale=soon"}) + assert mw.process_request(req3) is None + + def test_response_dated_in_the_future(self): + # A Date header ahead of the local clock must not make the cached + # response look aged. + headers = {"Date": self.tomorrow, "Cache-Control": "max-age=10"} + with self._middleware() as mw: + req0 = Request("http://example.com") + res0 = Response(req0.url, headers=headers) + res1 = self._process_requestresponse(mw, req0, res0) + assert "cached" not in res1.flags + + res2 = self._process_requestresponse(mw, req0, None) + self.assertEqualResponse(res1, res2) + assert "cached" in res2.flags + def test_process_exception(self): with self._middleware() as mw: res0 = Response(self.request.url, headers={"Expires": self.yesterday}) @@ -523,6 +640,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): assert mw.process_request(req0) is None res1 = mw.process_exception(req0, e("foo")) # Use cached response as recovery + assert isinstance(res1, Response) assert "cached" in res1.flags self.assertEqualResponse(res0, res1) # Do not use cached response for unhandled exceptions @@ -556,29 +674,39 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): # Concrete test classes that combine storage and policy mixins -class TestFilesystemStorageWithDummyPolicy( - TestBase, StorageTestMixin, DummyPolicyTestMixin -): +class FilesystemStorageTestMixin(StorageTestMixin): storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" + + def _corrupt_cache_entry(self, storage, spider, request) -> None: + rpath = Path(storage._get_request_path(spider, request)) + (rpath / "response_body").unlink() + + +class DbmStorageTestMixin(StorageTestMixin): + storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" + + def _corrupt_cache_entry(self, storage, spider, request) -> None: + key = storage._fingerprinter.fingerprint(request).hex() + storage.db[f"{key}_data"] = b"not a pickle" + + +class TestFilesystemStorageWithDummyPolicy( + FilesystemStorageTestMixin, DummyPolicyTestMixin +): policy_class = "scrapy.extensions.httpcache.DummyPolicy" class TestFilesystemStorageWithRFC2616Policy( - TestBase, StorageTestMixin, RFC2616PolicyTestMixin + FilesystemStorageTestMixin, RFC2616PolicyTestMixin ): - storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" policy_class = "scrapy.extensions.httpcache.RFC2616Policy" -class TestDbmStorageWithDummyPolicy(TestBase, StorageTestMixin, DummyPolicyTestMixin): - storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" +class TestDbmStorageWithDummyPolicy(DbmStorageTestMixin, DummyPolicyTestMixin): policy_class = "scrapy.extensions.httpcache.DummyPolicy" -class TestDbmStorageWithRFC2616Policy( - TestBase, StorageTestMixin, RFC2616PolicyTestMixin -): - storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" +class TestDbmStorageWithRFC2616Policy(DbmStorageTestMixin, RFC2616PolicyTestMixin): policy_class = "scrapy.extensions.httpcache.RFC2616Policy" @@ -599,3 +727,8 @@ class TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPol def _get_settings(self, **new_settings) -> dict[str, Any]: new_settings.setdefault("HTTPCACHE_GZIP", True) return super()._get_settings(**new_settings) + + def _corrupt_cache_entry(self, storage, spider, request) -> None: + # A spider killed while writing a gzip file leaves it truncated. + body_path = Path(storage._get_request_path(spider, request), "response_body") + body_path.write_bytes(body_path.read_bytes()[:-5]) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index ca18c8038..fa0707491 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -3,9 +3,9 @@ from importlib.util import find_spec from io import BytesIO from logging import WARNING from pathlib import Path +from typing import Any import pytest -from testfixtures import LogCapture from w3lib.encoding import resolve_encoding from scrapy.downloadermiddlewares.httpcompression import ( @@ -16,6 +16,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWar from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider +from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler from tests import tests_datadir @@ -73,9 +74,10 @@ class TestHttpCompression: def setup_method(self): self.crawler = get_crawler(Spider) self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) + assert self.crawler.stats self.crawler.stats.open_spider() - def _getresponse(self, coding): + def _getresponse(self, coding: str) -> Response: if coding not in FORMAT: raise ValueError @@ -97,7 +99,8 @@ class TestHttpCompression: ) return response - def assertStatsEqual(self, key, value): + def assertStatsEqual(self, key: str, value: Any) -> None: + assert self.crawler.stats assert self.crawler.stats.get_value(key) == value, str( self.crawler.stats.get_stats() ) @@ -146,6 +149,7 @@ class TestHttpCompression: def test_process_response_gzip(self): response = self._getresponse("gzip") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"gzip" @@ -160,6 +164,7 @@ class TestHttpCompression: _skip_if_no_br() response = self._getresponse("br") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"br" newresponse = self.mw.process_response(request, response) @@ -169,29 +174,29 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_bytes", 74837) - def test_process_response_br_unsupported(self): + def test_process_response_br_unsupported(self, caplog: pytest.LogCaptureFixture): if find_spec("brotli") is not None or find_spec("brotlicffi") is not None: pytest.skip("Requires not having brotli support") response = self._getresponse("br") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"br" - with LogCapture( - "scrapy.downloadermiddlewares.httpcompression", - propagate=False, - level=WARNING, - ) as log: + caplog.clear() + with caplog.at_level( + WARNING, logger="scrapy.downloadermiddlewares.httpcompression" + ): newresponse = self.mw.process_response(request, response) - log.check( + assert caplog.record_tuples == [ ( "scrapy.downloadermiddlewares.httpcompression", - "WARNING", + WARNING, ( "HttpCompressionMiddleware cannot decode the response for " "http://scrapytest.org/ from unsupported encoding(s) 'br'. " "You need to install brotli or brotlicffi >= 1.2.0 to decode 'br'." ), ), - ) + ] assert newresponse is not response assert newresponse.headers.getlist("Content-Encoding") == [b"br"] @@ -203,6 +208,7 @@ class TestHttpCompression: if not check_key.startswith("zstd-"): continue response = self._getresponse(check_key) + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"zstd" newresponse = self.mw.process_response(request, response) @@ -214,34 +220,35 @@ class TestHttpCompression: assert newresponse.body.startswith(b" None: response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = [b"gzip, foo, deflate"] + assert response.request request = response.request - with LogCapture( - "scrapy.downloadermiddlewares.httpcompression", - propagate=False, - level=WARNING, - ) as log: + caplog.clear() + with caplog.at_level( + WARNING, logger="scrapy.downloadermiddlewares.httpcompression" + ): newresponse = self.mw.process_response(request, response) - log.check( + assert caplog.record_tuples == [ ( "scrapy.downloadermiddlewares.httpcompression", - "WARNING", + WARNING, ( "HttpCompressionMiddleware cannot decode the response for" " http://scrapytest.org/ from unsupported encoding(s) 'gzip,foo'." ), ), - ) + ] assert newresponse is not response assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"] def test_multi_compression_multiple_header(self): response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = ["gzip", "deflate"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -326,6 +339,7 @@ class TestHttpCompression: def test_multi_compression_multiple_header_invalid_compression(self): response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = ["gzip", "foo", "deflate"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -334,6 +348,7 @@ class TestHttpCompression: def test_multi_compression_single_and_multiple_header(self): response = self._getresponse("gzip-deflate-gzip") response.headers["Content-Encoding"] = ["gzip", "deflate, gzip"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -343,6 +358,7 @@ class TestHttpCompression: def test_multi_compression_single_and_multiple_header_invalid_compression(self): response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = ["gzip", "foo,deflate"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -399,9 +415,7 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) def test_process_response_no_content_type_header(self): - headers = { - "Content-Encoding": "identity", - } + headers = {b"Content-Encoding": b"identity"} plainbody = ( b"Some page" b'' @@ -416,6 +430,7 @@ class TestHttpCompression: newresponse = self.mw.process_response(request, response) assert isinstance(newresponse, respcls) + assert isinstance(newresponse, HtmlResponse) assert newresponse.body == plainbody assert newresponse.encoding == resolve_encoding("gb2312") self.assertStatsEqual("httpcompression/response_count", 1) @@ -424,6 +439,7 @@ class TestHttpCompression: def test_process_response_gzipped_contenttype(self): response = self._getresponse("gzip") response.headers["Content-Type"] = "application/gzip" + assert response.request request = response.request newresponse = self.mw.process_response(request, response) @@ -436,6 +452,7 @@ class TestHttpCompression: def test_process_response_gzip_app_octetstream_contenttype(self): response = self._getresponse("gzip") response.headers["Content-Type"] = "application/octet-stream" + assert response.request request = response.request newresponse = self.mw.process_response(request, response) @@ -448,6 +465,7 @@ class TestHttpCompression: def test_process_response_gzip_binary_octetstream_contenttype(self): response = self._getresponse("x-gzip") response.headers["Content-Type"] = "binary/octet-stream" + assert response.request request = response.request newresponse = self.mw.process_response(request, response) @@ -506,6 +524,7 @@ class TestHttpCompression: def test_process_response_head_request_no_decode_required(self): response = self._getresponse("gzip") response.headers["Content-Type"] = "application/gzip" + assert response.request request = response.request request.method = "HEAD" response = response.replace(body=None) @@ -515,7 +534,7 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_count", None) self.assertStatsEqual("httpcompression/response_bytes", None) - def _test_compression_bomb_setting(self, compression_id): + def _test_compression_bomb_setting(self, compression_id: str) -> None: settings = {"DOWNLOAD_MAXSIZE": 1_000_000} crawler = get_crawler(Spider, settings_dict=settings) spider = crawler._create_spider("scrapytest.org") @@ -523,9 +542,12 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B + assert response.request with pytest.raises(IgnoreRequest) as exc_info: mw.process_response(response.request, response) - assert exc_info.value.__cause__.decompressed_size < 1_100_000 + cause = exc_info.value.__cause__ + assert isinstance(cause, _DecompressionMaxSizeExceeded) + assert cause.decompressed_size < 1_100_000 def test_compression_bomb_setting_br(self): _skip_if_no_br() @@ -551,6 +573,7 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse("bomb-gzip") # 11_511_612 B + assert response.request caplog.clear() with ( caplog.at_level( @@ -567,7 +590,7 @@ class TestHttpCompression: ) ] - def _test_compression_bomb_spider_attr(self, compression_id): + def _test_compression_bomb_spider_attr(self, compression_id: str) -> None: class DownloadMaxSizeSpider(Spider): download_maxsize = 1_000_000 @@ -577,9 +600,12 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") + assert response.request with pytest.raises(IgnoreRequest) as exc_info: mw.process_response(response.request, response) - assert exc_info.value.__cause__.decompressed_size < 1_100_000 + cause = exc_info.value.__cause__ + assert isinstance(cause, _DecompressionMaxSizeExceeded) + assert cause.decompressed_size < 1_100_000 @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_compression_bomb_spider_attr_br(self): @@ -601,7 +627,7 @@ class TestHttpCompression: self._test_compression_bomb_spider_attr("zstd") - def _test_compression_bomb_request_meta(self, compression_id): + def _test_compression_bomb_request_meta(self, compression_id: str) -> None: crawler = get_crawler(Spider) spider = crawler._create_spider("scrapytest.org") mw = HttpCompressionMiddleware.from_crawler(crawler) @@ -609,9 +635,12 @@ class TestHttpCompression: response = self._getresponse(f"bomb-{compression_id}") response.meta["download_maxsize"] = 1_000_000 + assert response.request with pytest.raises(IgnoreRequest) as exc_info: mw.process_response(response.request, response) - assert exc_info.value.__cause__.decompressed_size < 1_100_000 + cause = exc_info.value.__cause__ + assert isinstance(cause, _DecompressionMaxSizeExceeded) + assert cause.decompressed_size < 1_100_000 def test_compression_bomb_request_meta_br(self): _skip_if_no_br() @@ -629,7 +658,9 @@ class TestHttpCompression: self._test_compression_bomb_request_meta("zstd") - def _test_download_warnsize_setting(self, compression_id): + def _test_download_warnsize_setting( + self, caplog: pytest.LogCaptureFixture, compression_id: str + ) -> None: settings = {"DOWNLOAD_WARNSIZE": 10_000_000} crawler = get_crawler(Spider, settings_dict=settings) spider = crawler._create_spider("scrapytest.org") @@ -637,41 +668,51 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") - with LogCapture( - "scrapy.downloadermiddlewares.httpcompression", - propagate=False, - level=WARNING, - ) as log: + assert response.request + caplog.clear() + with caplog.at_level( + WARNING, logger="scrapy.downloadermiddlewares.httpcompression" + ): mw.process_response(response.request, response) - log.check( + assert caplog.record_tuples == [ ( "scrapy.downloadermiddlewares.httpcompression", - "WARNING", + WARNING, ( "<200 http://scrapytest.org/> body size after " "decompression (11511612 B) is larger than the download " "warning size (10000000 B)." ), ), - ) + ] - def test_download_warnsize_setting_br(self): + def test_download_warnsize_setting_br( + self, caplog: pytest.LogCaptureFixture + ) -> None: _skip_if_no_br() - self._test_download_warnsize_setting("br") + self._test_download_warnsize_setting(caplog, "br") - def test_download_warnsize_setting_deflate(self): - self._test_download_warnsize_setting("deflate") + def test_download_warnsize_setting_deflate( + self, caplog: pytest.LogCaptureFixture + ) -> None: + self._test_download_warnsize_setting(caplog, "deflate") - def test_download_warnsize_setting_gzip(self): - self._test_download_warnsize_setting("gzip") + def test_download_warnsize_setting_gzip( + self, caplog: pytest.LogCaptureFixture + ) -> None: + self._test_download_warnsize_setting(caplog, "gzip") - def test_download_warnsize_setting_zstd(self): + def test_download_warnsize_setting_zstd( + self, caplog: pytest.LogCaptureFixture + ) -> None: _skip_if_no_zstd() - self._test_download_warnsize_setting("zstd") + self._test_download_warnsize_setting(caplog, "zstd") - def _test_download_warnsize_spider_attr(self, compression_id): + def _test_download_warnsize_spider_attr( + self, caplog: pytest.LogCaptureFixture, compression_id: str + ) -> None: class DownloadWarnSizeSpider(Spider): download_warnsize = 10_000_000 @@ -681,45 +722,55 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") - with LogCapture( - "scrapy.downloadermiddlewares.httpcompression", - propagate=False, - level=WARNING, - ) as log: + assert response.request + caplog.clear() + with caplog.at_level( + WARNING, logger="scrapy.downloadermiddlewares.httpcompression" + ): mw.process_response(response.request, response) - log.check( + assert caplog.record_tuples == [ ( "scrapy.downloadermiddlewares.httpcompression", - "WARNING", + WARNING, ( "<200 http://scrapytest.org/> body size after " "decompression (11511612 B) is larger than the download " "warning size (10000000 B)." ), ), - ) + ] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_download_warnsize_spider_attr_br(self): + def test_download_warnsize_spider_attr_br( + self, caplog: pytest.LogCaptureFixture + ) -> None: _skip_if_no_br() - self._test_download_warnsize_spider_attr("br") + self._test_download_warnsize_spider_attr(caplog, "br") @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_download_warnsize_spider_attr_deflate(self): - self._test_download_warnsize_spider_attr("deflate") + def test_download_warnsize_spider_attr_deflate( + self, caplog: pytest.LogCaptureFixture + ) -> None: + self._test_download_warnsize_spider_attr(caplog, "deflate") @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_download_warnsize_spider_attr_gzip(self): - self._test_download_warnsize_spider_attr("gzip") + def test_download_warnsize_spider_attr_gzip( + self, caplog: pytest.LogCaptureFixture + ) -> None: + self._test_download_warnsize_spider_attr(caplog, "gzip") @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_download_warnsize_spider_attr_zstd(self): + def test_download_warnsize_spider_attr_zstd( + self, caplog: pytest.LogCaptureFixture + ) -> None: _skip_if_no_zstd() - self._test_download_warnsize_spider_attr("zstd") + self._test_download_warnsize_spider_attr(caplog, "zstd") - def _test_download_warnsize_request_meta(self, compression_id): + def _test_download_warnsize_request_meta( + self, caplog: pytest.LogCaptureFixture, compression_id: str + ) -> None: crawler = get_crawler(Spider) spider = crawler._create_spider("scrapytest.org") mw = HttpCompressionMiddleware.from_crawler(crawler) @@ -727,41 +778,49 @@ class TestHttpCompression: response = self._getresponse(f"bomb-{compression_id}") response.meta["download_warnsize"] = 10_000_000 - with LogCapture( - "scrapy.downloadermiddlewares.httpcompression", - propagate=False, - level=WARNING, - ) as log: + assert response.request + caplog.clear() + with caplog.at_level( + WARNING, logger="scrapy.downloadermiddlewares.httpcompression" + ): mw.process_response(response.request, response) - log.check( + assert caplog.record_tuples == [ ( "scrapy.downloadermiddlewares.httpcompression", - "WARNING", + WARNING, ( "<200 http://scrapytest.org/> body size after " "decompression (11511612 B) is larger than the download " "warning size (10000000 B)." ), ), - ) + ] - def test_download_warnsize_request_meta_br(self): + def test_download_warnsize_request_meta_br( + self, caplog: pytest.LogCaptureFixture + ) -> None: _skip_if_no_br() - self._test_download_warnsize_request_meta("br") + self._test_download_warnsize_request_meta(caplog, "br") - def test_download_warnsize_request_meta_deflate(self): - self._test_download_warnsize_request_meta("deflate") + def test_download_warnsize_request_meta_deflate( + self, caplog: pytest.LogCaptureFixture + ) -> None: + self._test_download_warnsize_request_meta(caplog, "deflate") - def test_download_warnsize_request_meta_gzip(self): - self._test_download_warnsize_request_meta("gzip") + def test_download_warnsize_request_meta_gzip( + self, caplog: pytest.LogCaptureFixture + ) -> None: + self._test_download_warnsize_request_meta(caplog, "gzip") - def test_download_warnsize_request_meta_zstd(self): + def test_download_warnsize_request_meta_zstd( + self, caplog: pytest.LogCaptureFixture + ) -> None: _skip_if_no_zstd() - self._test_download_warnsize_request_meta("zstd") + self._test_download_warnsize_request_meta(caplog, "zstd") - def _get_truncated_response(self, compression_id): + def _get_truncated_response(self, compression_id: str) -> Response: crawler = get_crawler(Spider) spider = crawler._create_spider("scrapytest.org") mw = HttpCompressionMiddleware.from_crawler(crawler) @@ -769,7 +828,10 @@ class TestHttpCompression: response = self._getresponse(compression_id) truncated_body = response.body[: len(response.body) // 2] response = response.replace(body=truncated_body) - return mw.process_response(response.request, response) + assert response.request + new_response = mw.process_response(response.request, response) + assert isinstance(new_response, Response) + return new_response def test_process_truncated_response_br(self): _skip_if_no_br() diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 7ed848764..54d4601a7 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -14,7 +14,8 @@ class TestHttpProxyMiddleware: self._oldenv = os.environ.copy() def teardown_method(self): - os.environ = self._oldenv + os.environ.clear() + os.environ.update(self._oldenv) def test_not_enabled(self): crawler = get_crawler(Spider, {"HTTPPROXY_ENABLED": False}) @@ -22,7 +23,8 @@ class TestHttpProxyMiddleware: HttpProxyMiddleware.from_crawler(crawler) def test_no_environment_proxies(self): - os.environ = {"dummy_proxy": "reset_env_and_do_not_raise"} + os.environ.clear() + os.environ["dummy_proxy"] = "reset_env_and_do_not_raise" mw = HttpProxyMiddleware() for url in ("http://e.com", "https://e.com", "file:///tmp/a"): diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index c0b8dc4dd..cb17c2553 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -1,3 +1,6 @@ +import re +from typing import Any + import pytest from scrapy import Request, Spider @@ -51,7 +54,7 @@ def test_process_request_dont_filter(value, filtered): crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) - kwargs = {} + kwargs: dict[str, Any] = {} if value is not UNSET: kwargs["dont_filter"] = value request = Request("https://b.example", **kwargs) @@ -80,7 +83,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) - kwargs = {"meta": {}} + kwargs: dict[str, Any] = {"meta": {}} if allow_offsite is not UNSET: kwargs["meta"]["allow_offsite"] = allow_offsite if dont_filter is not UNSET: @@ -103,7 +106,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): ) def test_process_request_no_allowed_domains(value): crawler = get_crawler(Spider) - kwargs = {} + kwargs: dict[str, Any] = {} if value is not UNSET: kwargs["allowed_domains"] = value crawler.spider = crawler._create_spider(name="a", **kwargs) @@ -150,7 +153,7 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): mw.spider_opened(crawler.spider) request = Request(url) if allowed: - assert mw.request_scheduled(request, crawler.spider) is None + mw.request_scheduled(request, crawler.spider) else: with pytest.raises(IgnoreRequest): mw.request_scheduled(request, crawler.spider) @@ -170,7 +173,7 @@ def test_request_scheduled_dont_filter(value, filtered): crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) - kwargs = {} + kwargs: dict[str, Any] = {} if value is not UNSET: kwargs["dont_filter"] = value request = Request("https://b.example", **kwargs) @@ -178,7 +181,7 @@ def test_request_scheduled_dont_filter(value, filtered): with pytest.raises(IgnoreRequest): mw.request_scheduled(request, crawler.spider) else: - assert mw.request_scheduled(request, crawler.spider) is None + mw.request_scheduled(request, crawler.spider) @pytest.mark.parametrize( @@ -191,14 +194,14 @@ def test_request_scheduled_dont_filter(value, filtered): ) def test_request_scheduled_no_allowed_domains(value): crawler = get_crawler(Spider) - kwargs = {} + kwargs: dict[str, Any] = {} if value is not UNSET: kwargs["allowed_domains"] = value crawler.spider = crawler._create_spider(name="a", **kwargs) mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) request = Request("https://example.com") - assert mw.request_scheduled(request, crawler.spider) is None + mw.request_scheduled(request, crawler.spider) def test_request_scheduled_invalid_domains(): @@ -208,7 +211,7 @@ def test_request_scheduled_invalid_domains(): mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) request = Request("https://a.example") - assert mw.request_scheduled(request, crawler.spider) is None + mw.request_scheduled(request, crawler.spider) for letter in ("b", "c"): request = Request(f"https://{letter}.example") with pytest.raises(IgnoreRequest): @@ -225,9 +228,22 @@ def test_repeated_offsite_domain(): with pytest.raises(IgnoreRequest): mw.process_request(req1) assert "other.org" in mw.domains_seen + assert crawler.stats assert crawler.stats.get_value("offsite/domains") == 1 assert crawler.stats.get_value("offsite/filtered") == 1 with pytest.raises(IgnoreRequest): mw.process_request(req2) assert crawler.stats.get_value("offsite/domains") == 1 # not incremented again assert crawler.stats.get_value("offsite/filtered") == 2 + + +def test_ignore_request_reason(): + crawler = get_crawler(Spider) + crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(crawler.spider) + request = Request("http://other.org/1") + with pytest.raises( + IgnoreRequest, match=re.escape("Filtered offsite request to 'other.org'") + ): + mw.process_request(request) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 28cf8ba34..de97aaadb 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -16,11 +16,11 @@ from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -from tests.test_downloadermiddleware_redirect_base import Base +from tests.utils.bases.redirect import TestRedirectBase from tests.utils.redirect import REDIRECT_SCHEME_CASES, SCHEME_PARAMS -class TestRedirectMiddleware(Base.Test): +class TestRedirectMiddleware(TestRedirectBase): mwcls = RedirectMiddleware reason = 302 @@ -309,7 +309,7 @@ class TestRedirectMiddleware(Base.Test): url = "http://www.example.com/301" url2 = "http://www.example.com/redirected" - def _test_passthrough(req): + def _test_passthrough(req: Request) -> None: rsp = Response(url, headers={"Location": url2}, status=301, request=req) r = self.mw.process_response(req, rsp) assert r is rsp @@ -404,15 +404,17 @@ def test_response_referrer_policy(policy, source_url, target_url, expected_refer status=301, headers={"Location": target_url, **extra_headers}, ) - source_request = redirect_mw.process_response(source_request, response_redirect) - assert isinstance(source_request, Request) + target_request = redirect_mw.process_response(source_request, response_redirect) + assert isinstance(target_request, Request) - assert source_request.headers.get("Referer") == expected_referrer + assert target_request.headers.get("Referer") == expected_referrer def test_no_warning_when_referer_middleware_present(caplog): crawler = get_crawler() - crawler.get_spider_middleware = MagicMock(return_value=MagicMock()) + crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign] + return_value=MagicMock() + ) mw = build_from_crawler(RedirectMiddleware, crawler) caplog.clear() with caplog.at_level(logging.WARNING): @@ -426,7 +428,9 @@ def test_no_warning_when_referer_middleware_present(caplog): def test_warning_redirect_middleware(caplog): crawler = get_crawler() - crawler.get_spider_middleware = MagicMock(return_value=None) + crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign] + return_value=None + ) mw = build_from_crawler(RedirectMiddleware, crawler) with caplog.at_level(logging.WARNING): mw._engine_started() @@ -449,7 +453,9 @@ def test_warning_subclass(caplog): pass crawler = get_crawler() - crawler.get_spider_middleware = MagicMock(return_value=None) + crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign] + return_value=None + ) mw = build_from_crawler(MyRedirectMiddleware, crawler) with caplog.at_level(logging.WARNING): mw._engine_started() diff --git a/tests/test_downloadermiddleware_redirect_base.py b/tests/test_downloadermiddleware_redirect_base.py deleted file mode 100644 index bd8bc796c..000000000 --- a/tests/test_downloadermiddleware_redirect_base.py +++ /dev/null @@ -1,984 +0,0 @@ -from __future__ import annotations - -import pytest - -from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware -from scrapy.exceptions import IgnoreRequest -from scrapy.http import Request, Response -from scrapy.utils.misc import set_environ -from scrapy.utils.test import get_crawler - - -class Base: - class Test: - def test_priority_adjust(self): - req = Request("http://a.example") - rsp = self.get_response(req, "http://a.example/redirected") - req2 = self.mw.process_response(req, rsp) - assert req2.priority > req.priority - - def test_dont_redirect(self): - url = "http://www.example.com/301" - url2 = "http://www.example.com/redirected" - req = Request(url, meta={"dont_redirect": True}) - rsp = self.get_response(req, url2) - - r = self.mw.process_response(req, rsp) - assert isinstance(r, Response) - assert r is rsp - - # Test that it redirects when dont_redirect is False - req = Request(url, meta={"dont_redirect": False}) - rsp = self.get_response(req, url2) - - r = self.mw.process_response(req, rsp) - assert isinstance(r, Request) - - def test_post(self): - url = "http://www.example.com/302" - url2 = "http://www.example.com/redirected2" - req = Request( - url, - method="POST", - body="test", - headers={"Content-Type": "text/plain", "Content-length": "4"}, - ) - rsp = self.get_response(req, url2) - - req2 = self.mw.process_response(req, rsp) - assert isinstance(req2, Request) - assert req2.url == url2 - assert req2.method == "GET" - assert "Content-Type" not in req2.headers, ( - "Content-Type header must not be present in redirected request" - ) - assert "Content-Length" not in req2.headers, ( - "Content-Length header must not be present in redirected request" - ) - assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" - - def test_max_redirect_times(self): - self.mw.max_redirect_times = 1 - req = Request("http://a.example/302") - rsp = self.get_response(req, "/redirected") - - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - assert "redirect_times" in req.meta - assert req.meta["redirect_times"] == 1 - with pytest.raises(IgnoreRequest): - self.mw.process_response(req, rsp) - - def test_ttl(self): - self.mw.max_redirect_times = 100 - req = Request("http://a.example/302", meta={"redirect_ttl": 1}) - rsp = self.get_response(req, "/a") - - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - with pytest.raises(IgnoreRequest): - self.mw.process_response(req, rsp) - - def test_redirect_urls(self): - req1 = Request("http://a.example/first") - rsp1 = self.get_response(req1, "/redirected") - req2 = self.mw.process_response(req1, rsp1) - rsp2 = self.get_response(req2, "/redirected2") - req3 = self.mw.process_response(req2, rsp2) - - assert req2.url == "http://a.example/redirected" - assert req2.meta["redirect_urls"] == ["http://a.example/first"] - assert req3.url == "http://a.example/redirected2" - assert req3.meta["redirect_urls"] == [ - "http://a.example/first", - "http://a.example/redirected", - ] - - def test_redirect_reasons(self): - req1 = Request("http://a.example/first") - rsp1 = self.get_response(req1, "/redirected1") - req2 = self.mw.process_response(req1, rsp1) - rsp2 = self.get_response(req2, "/redirected2") - req3 = self.mw.process_response(req2, rsp2) - assert req2.meta["redirect_reasons"] == [self.reason] - assert req3.meta["redirect_reasons"] == [self.reason, self.reason] - - def test_cross_origin_header_dropping(self): - safe_headers = {"A": "B"} - cookie_header = {"Cookie": "a=b"} - authorization_header = {"Authorization": "Bearer 123456"} - - original_request = Request( - "https://example.com", - headers={**safe_headers, **cookie_header, **authorization_header}, - ) - - # Redirects to the same origin (same scheme, same domain, same port) - # keep all headers. - internal_response = self.get_response( - original_request, "https://example.com/a" - ) - internal_redirect_request = self.mw.process_response( - original_request, internal_response - ) - assert isinstance(internal_redirect_request, Request) - assert original_request.headers == internal_redirect_request.headers - - # Redirects to the same origin (same scheme, same domain, same port) - # keep all headers also when the scheme is http. - http_request = Request( - "http://example.com", - headers={**safe_headers, **cookie_header, **authorization_header}, - ) - http_response = self.get_response(http_request, "http://example.com/a") - http_redirect_request = self.mw.process_response( - http_request, http_response - ) - assert isinstance(http_redirect_request, Request) - assert http_request.headers == http_redirect_request.headers - - # For default ports, whether the port is explicit or implicit does not - # affect the outcome, it is still the same origin. - to_explicit_port_response = self.get_response( - original_request, "https://example.com:443/a" - ) - to_explicit_port_redirect_request = self.mw.process_response( - original_request, to_explicit_port_response - ) - assert isinstance(to_explicit_port_redirect_request, Request) - assert original_request.headers == to_explicit_port_redirect_request.headers - - # For default ports, whether the port is explicit or implicit does not - # affect the outcome, it is still the same origin. - to_implicit_port_response = self.get_response( - original_request, "https://example.com/a" - ) - to_implicit_port_redirect_request = self.mw.process_response( - original_request, to_implicit_port_response - ) - assert isinstance(to_implicit_port_redirect_request, Request) - assert original_request.headers == to_implicit_port_redirect_request.headers - - # A port change drops the Authorization header because the origin - # changes, but keeps the Cookie header because the domain remains the - # same. - different_port_response = self.get_response( - original_request, "https://example.com:8080/a" - ) - different_port_redirect_request = self.mw.process_response( - original_request, different_port_response - ) - assert isinstance(different_port_redirect_request, Request) - assert { - **safe_headers, - **cookie_header, - } == different_port_redirect_request.headers.to_unicode_dict() - - # A domain change drops both the Authorization and the Cookie header. - external_response = self.get_response( - original_request, "https://example.org/a" - ) - external_redirect_request = self.mw.process_response( - original_request, external_response - ) - assert isinstance(external_redirect_request, Request) - assert safe_headers == external_redirect_request.headers.to_unicode_dict() - - # A scheme upgrade (http → https) drops the Authorization header - # because the origin changes, but keeps the Cookie header because the - # domain remains the same. - upgrade_response = self.get_response(http_request, "https://example.com/a") - upgrade_redirect_request = self.mw.process_response( - http_request, upgrade_response - ) - assert isinstance(upgrade_redirect_request, Request) - assert { - **safe_headers, - **cookie_header, - } == upgrade_redirect_request.headers.to_unicode_dict() - - # A scheme downgrade (https → http) drops the Authorization header - # because the origin changes, and the Cookie header because its value - # cannot indicate whether the cookies were secure (HTTPS-only) or not. - # - # Note: If the Cookie header is set by the cookie management - # middleware, as recommended in the docs, the dropping of Cookie on - # scheme downgrade is not an issue, because the cookie management - # middleware will add again the Cookie header to the new request if - # appropriate. - downgrade_response = self.get_response( - original_request, "http://example.com/a" - ) - downgrade_redirect_request = self.mw.process_response( - original_request, downgrade_response - ) - assert isinstance(downgrade_redirect_request, Request) - assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict() - - def test_meta_proxy_http_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("http://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_http_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("http://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_https_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("https://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_https_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("https://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_http_to_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("http://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_https_to_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("https://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_http_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_http_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_https_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_https_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_proxied_http_to_proxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request2.meta["_auth_proxy"] == "https://b.example" - assert request2.meta["proxy"] == "https://b.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_proxied_http_to_unproxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_unproxied_http_to_proxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request2.meta["_auth_proxy"] == "https://b.example" - assert request2.meta["proxy"] == "https://b.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - def test_system_proxy_unproxied_http_to_unproxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - def test_system_proxy_proxied_https_to_proxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request1.meta["_auth_proxy"] == "https://b.example" - assert request1.meta["proxy"] == "https://b.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request3.meta["_auth_proxy"] == "https://b.example" - assert request3.meta["proxy"] == "https://b.example" - - def test_system_proxy_proxied_https_to_unproxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request1.meta["_auth_proxy"] == "https://b.example" - assert request1.meta["proxy"] == "https://b.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request3.meta["_auth_proxy"] == "https://b.example" - assert request3.meta["proxy"] == "https://b.example" - - def test_system_proxy_unproxied_https_to_proxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - def test_system_proxy_unproxied_https_to_unproxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py index b5d39080c..83dc6825f 100644 --- a/tests/test_downloadermiddleware_redirect_metarefresh.py +++ b/tests/test_downloadermiddleware_redirect_metarefresh.py @@ -12,7 +12,7 @@ from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler -from tests.test_downloadermiddleware_redirect_base import Base +from tests.utils.bases.redirect import TestRedirectBase from tests.utils.redirect import ( HTTP_SCHEMES, NON_HTTP_SCHEMES, @@ -21,12 +21,12 @@ from tests.utils.redirect import ( ) -def meta_refresh_body(url, interval=5): +def meta_refresh_body(url: str, interval: int = 5) -> bytes: html = f"""""" return html.encode("utf-8") -class TestMetaRefreshMiddleware(Base.Test): +class TestMetaRefreshMiddleware(TestRedirectBase): mwcls = MetaRefreshMiddleware reason = "meta refresh" @@ -34,10 +34,14 @@ class TestMetaRefreshMiddleware(Base.Test): crawler = get_crawler(Spider) self.mw = self.mwcls.from_crawler(crawler) - def _body(self, interval=5, url="http://example.org/newpage"): + def _body( + self, interval: int = 5, url: str = "http://example.org/newpage" + ) -> bytes: return meta_refresh_body(url, interval) - def get_response(self, request, location): + def get_response( + self, request: Request, location: str, status: int = 302 + ) -> Response: return HtmlResponse(request.url, body=self._body(url=location)) def test_meta_refresh(self): @@ -75,7 +79,7 @@ class TestMetaRefreshMiddleware(Base.Test): assert "Content-Length" not in req2.headers, ( "Content-Length header must not be present in redirected request" ) - assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" + assert not req2.body, f"Redirected body must be empty, not {req2.body!r}" def test_ignore_tags_default(self): req = Request(url="http://example.org") @@ -142,7 +146,9 @@ def test_meta_refresh_schemes(url, location, target): def test_warning_meta_refresh_middleware(caplog): crawler = get_crawler() - crawler.get_spider_middleware = MagicMock(return_value=None) + crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign] + return_value=None + ) mw = build_from_crawler(MetaRefreshMiddleware, crawler) with caplog.at_level(logging.WARNING): mw._engine_started() diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 825efce2f..ab52590c7 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,7 +1,9 @@ +from __future__ import annotations + import logging +from typing import Any, cast import pytest -from testfixtures import LogCapture from twisted.internet.error import ConnectError, ConnectionDone, ConnectionLost from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request @@ -29,6 +31,7 @@ class TestRetry: req = Request("http://www.scrapytest.org/503") rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) assert req2.priority < req.priority def test_404(self): @@ -51,9 +54,9 @@ class TestRetry: rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) # first retry - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 1 + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.meta["retry_times"] == 1 def test_dont_retry_exc(self): req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True}) @@ -66,18 +69,19 @@ class TestRetry: rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) # first retry - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 1 + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.meta["retry_times"] == 1 # second retry - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 2 + req3 = self.mw.process_response(req2, rsp) + assert isinstance(req3, Request) + assert req3.meta["retry_times"] == 2 # discard it - assert self.mw.process_response(req, rsp) is rsp + assert self.mw.process_response(req3, rsp) is rsp + assert self.crawler.stats assert self.crawler.stats.get_value("retry/max_reached") == 1 assert ( self.crawler.stats.get_value("retry/reason_count/503 Service Unavailable") @@ -85,7 +89,7 @@ class TestRetry: ) assert self.crawler.stats.get_value("retry/count") == 2 - def test_give_up_log_level_setting(self): + def test_give_up_log_level_setting(self, caplog: pytest.LogCaptureFixture) -> None: crawler = get_crawler( DefaultSpider, settings_dict={"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"} ) @@ -94,29 +98,25 @@ class TestRetry: mw.max_retry_times = 0 req = Request("http://example.com/503") rsp = Response("http://example.com/503", body=b"", status=503) - with LogCapture() as log: + with caplog.at_level(logging.WARNING): assert mw.process_response(req, rsp) is rsp - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "WARNING", - f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.WARNING, + f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable", + ) in caplog.record_tuples - def test_give_up_log_level_meta(self): + def test_give_up_log_level_meta(self, caplog: pytest.LogCaptureFixture) -> None: self.mw.max_retry_times = 0 req = Request("http://example.com/503", meta={"give_up_log_level": "WARNING"}) rsp = Response("http://example.com/503", body=b"", status=503) - with LogCapture() as log: + with caplog.at_level(logging.WARNING): assert self.mw.process_response(req, rsp) is rsp - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "WARNING", - f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.WARNING, + f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable", + ) in caplog.record_tuples def test_twistederrors(self): exceptions = [ @@ -133,6 +133,7 @@ class TestRetry: self._test_retry_exception(req, exc("foo")) stats = self.crawler.stats + assert stats assert stats.get_value("retry/max_reached") == len(exceptions) assert stats.get_value("retry/count") == len(exceptions) * 2 assert ( @@ -151,29 +152,30 @@ class TestRetry: req = Request(f"http://www.scrapytest.org/{exc.__name__}") self._test_retry_exception(req, exc("foo"), mw) - def _test_retry_exception(self, req, exception, mw=None): + def _test_retry_exception( + self, req: Request, exception: Exception, mw: RetryMiddleware | None = None + ) -> None: if mw is None: mw = self.mw # first retry - req = mw.process_exception(req, exception) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 1 + req2 = mw.process_exception(req, exception) + assert isinstance(req2, Request) + assert req2.meta["retry_times"] == 1 # second retry - req = mw.process_exception(req, exception) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 2 + req3 = mw.process_exception(req2, exception) + assert isinstance(req3, Request) + assert req3.meta["retry_times"] == 2 # discard it - req = mw.process_exception(req, exception) - assert req is None + assert mw.process_exception(req3, exception) is None class TestMaxRetryTimes: invalid_url = "http://www.scrapytest.org/invalid_url" - def get_middleware(self, settings=None): + def get_middleware(self, settings: dict[str, Any] | None = None) -> RetryMiddleware: crawler = get_crawler(DefaultSpider, settings or {}) crawler.spider = crawler._create_spider() return RetryMiddleware.from_crawler(crawler) @@ -277,31 +279,30 @@ class TestMaxRetryTimes: def _test_retry( self, - req, - exception, - max_retry_times, - middleware=None, - ): - middleware = middleware or self.mw - + req: Request, + exception: Exception, + max_retry_times: int, + middleware: RetryMiddleware, + ) -> None: for _ in range(max_retry_times): - req = middleware.process_exception(req, exception) - assert isinstance(req, Request) + result = middleware.process_exception(req, exception) + assert isinstance(result, Request) + req = result # discard it - req = middleware.process_exception(req, exception) - assert req is None + assert middleware.process_exception(req, exception) is None class TestGetRetryRequest: - def get_spider(self, settings=None): + @staticmethod + def get_spider(settings: dict[str, Any] | None = None) -> Spider: crawler = get_crawler(Spider, settings or {}) return crawler._create_spider("foo") - def test_basic_usage(self): + def test_basic_usage(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): new_request = get_retry_request( request, spider=spider, @@ -313,44 +314,42 @@ class TestGetRetryRequest: assert new_request.meta["retry_times"] == expected_retry_times assert new_request.priority == -1 expected_reason = "unspecified" + assert spider.crawler.stats for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): assert spider.crawler.stats.get_value(stat) == 1 - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "DEBUG", - f"Retrying {request} (failed {expected_retry_times} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.DEBUG, + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) in caplog.record_tuples - def test_max_retries_reached(self): + def test_max_retries_reached(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() max_retry_times = 0 - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): new_request = get_retry_request( request, spider=spider, max_retry_times=max_retry_times, ) assert new_request is None + assert spider.crawler.stats assert spider.crawler.stats.get_value("retry/max_reached") == 1 failure_count = max_retry_times + 1 expected_reason = "unspecified" - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "ERROR", - f"Gave up retrying {request} (failed {failure_count} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.ERROR, + f"Gave up retrying {request} (failed {failure_count} times): " + f"{expected_reason}", + ) in caplog.record_tuples - def test_one_retry(self): + def test_one_retry(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): new_request = get_retry_request( request, spider=spider, @@ -363,28 +362,31 @@ class TestGetRetryRequest: assert new_request.meta["retry_times"] == expected_retry_times assert new_request.priority == -1 expected_reason = "unspecified" + assert spider.crawler.stats for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): assert spider.crawler.stats.get_value(stat) == 1 - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "DEBUG", - f"Retrying {request} (failed {expected_retry_times} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.DEBUG, + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) in caplog.record_tuples - def test_two_retries(self): + def test_two_retries(self, caplog: pytest.LogCaptureFixture) -> None: spider = self.get_spider() request = Request("https://example.com") new_request = request max_retry_times = 2 for index in range(max_retry_times): - with LogCapture() as log: - new_request = get_retry_request( - new_request, - spider=spider, - max_retry_times=max_retry_times, + caplog.clear() + with caplog.at_level(logging.DEBUG): + new_request = cast( + "Request", + get_retry_request( + new_request, + spider=spider, + max_retry_times=max_retry_times, + ), ) assert isinstance(new_request, Request) assert new_request != request @@ -393,41 +395,42 @@ class TestGetRetryRequest: assert new_request.meta["retry_times"] == expected_retry_times assert new_request.priority == -expected_retry_times expected_reason = "unspecified" + assert spider.crawler.stats for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): value = spider.crawler.stats.get_value(stat) assert value == expected_retry_times - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "DEBUG", - f"Retrying {request} (failed {expected_retry_times} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.DEBUG, + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) in caplog.record_tuples - with LogCapture() as log: - new_request = get_retry_request( - new_request, - spider=spider, - max_retry_times=max_retry_times, + caplog.clear() + with caplog.at_level(logging.DEBUG): + new_request = cast( + "Request", + get_retry_request( + new_request, + spider=spider, + max_retry_times=max_retry_times, + ), ) assert new_request is None assert spider.crawler.stats.get_value("retry/max_reached") == 1 failure_count = max_retry_times + 1 expected_reason = "unspecified" - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "ERROR", - f"Gave up retrying {request} (failed {failure_count} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.ERROR, + f"Gave up retrying {request} (failed {failure_count} times): " + f"{expected_reason}", + ) in caplog.record_tuples def test_no_spider(self): request = Request("https://example.com") with pytest.raises(TypeError): - get_retry_request(request) # pylint: disable=missing-kwoa + get_retry_request(request) # type: ignore[call-arg] # pylint: disable=missing-kwoa def test_max_retry_times_setting(self): max_retry_times = 0 @@ -470,6 +473,7 @@ class TestGetRetryRequest: request, spider=spider, ) + assert new_request assert new_request.priority == priority_adjust def test_priority_adjust_argument(self): @@ -481,240 +485,234 @@ class TestGetRetryRequest: spider=spider, priority_adjust=priority_adjust, ) + assert new_request assert new_request.priority == priority_adjust - def test_log_extra_retry_success(self): + def test_log_extra_retry_success(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() - with LogCapture(attributes=("spider",)) as log: + with caplog.at_level(logging.DEBUG): get_retry_request( request, spider=spider, ) - log.check_present(spider) + assert any(getattr(r, "spider", None) is spider for r in caplog.records) - def test_log_extra_retries_exceeded(self): + def test_log_extra_retries_exceeded(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() - with LogCapture(attributes=("spider",)) as log: + with caplog.at_level(logging.DEBUG): get_retry_request( request, spider=spider, max_retry_times=0, ) - log.check_present(spider) + assert any(getattr(r, "spider", None) is spider for r in caplog.records) - def test_reason_string(self): + def test_reason_string(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() expected_reason = "because" - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): get_retry_request( request, spider=spider, reason=expected_reason, ) expected_retry_times = 1 + assert spider.crawler.stats for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): assert spider.crawler.stats.get_value(stat) == 1 - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "DEBUG", - f"Retrying {request} (failed {expected_retry_times} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.DEBUG, + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) in caplog.record_tuples - def test_reason_builtin_exception(self): + def test_reason_builtin_exception(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() expected_reason = NotImplementedError() expected_reason_string = "builtins.NotImplementedError" - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): get_retry_request( request, spider=spider, reason=expected_reason, ) expected_retry_times = 1 + assert spider.crawler.stats stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) assert stat == 1 - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "DEBUG", - f"Retrying {request} (failed {expected_retry_times} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.DEBUG, + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) in caplog.record_tuples - def test_reason_builtin_exception_class(self): + def test_reason_builtin_exception_class( + self, caplog: pytest.LogCaptureFixture + ) -> None: request = Request("https://example.com") spider = self.get_spider() expected_reason = NotImplementedError expected_reason_string = "builtins.NotImplementedError" - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): get_retry_request( request, spider=spider, reason=expected_reason, ) expected_retry_times = 1 + assert spider.crawler.stats stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) assert stat == 1 - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "DEBUG", - f"Retrying {request} (failed {expected_retry_times} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.DEBUG, + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) in caplog.record_tuples - def test_reason_custom_exception(self): + def test_reason_custom_exception(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() expected_reason = IgnoreRequest() expected_reason_string = "scrapy.exceptions.IgnoreRequest" - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): get_retry_request( request, spider=spider, reason=expected_reason, ) expected_retry_times = 1 + assert spider.crawler.stats stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) assert stat == 1 - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "DEBUG", - f"Retrying {request} (failed {expected_retry_times} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.DEBUG, + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) in caplog.record_tuples - def test_reason_custom_exception_class(self): + def test_reason_custom_exception_class( + self, caplog: pytest.LogCaptureFixture + ) -> None: request = Request("https://example.com") spider = self.get_spider() expected_reason = IgnoreRequest expected_reason_string = "scrapy.exceptions.IgnoreRequest" - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): get_retry_request( request, spider=spider, reason=expected_reason, ) expected_retry_times = 1 + assert spider.crawler.stats stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) assert stat == 1 - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "DEBUG", - f"Retrying {request} (failed {expected_retry_times} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.DEBUG, + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) in caplog.record_tuples - def test_custom_logger(self): + def test_custom_logger(self, caplog: pytest.LogCaptureFixture) -> None: logger = logging.getLogger("custom-logger") request = Request("https://example.com") spider = self.get_spider() expected_reason = "because" - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): get_retry_request( request, spider=spider, reason=expected_reason, logger=logger, ) - log.check_present( - ( - "custom-logger", - "DEBUG", - f"Retrying {request} (failed 1 times): {expected_reason}", - ) - ) + assert ( + "custom-logger", + logging.DEBUG, + f"Retrying {request} (failed 1 times): {expected_reason}", + ) in caplog.record_tuples - def test_give_up_log_level_default(self): + def test_give_up_log_level_default(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() - with LogCapture() as log: + with caplog.at_level(logging.ERROR): get_retry_request( request, spider=spider, max_retry_times=0, ) - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "ERROR", - f"Gave up retrying {request} (failed 1 times): unspecified", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.ERROR, + f"Gave up retrying {request} (failed 1 times): unspecified", + ) in caplog.record_tuples - def test_give_up_log_level_argument_name(self): + def test_give_up_log_level_argument_name( + self, caplog: pytest.LogCaptureFixture + ) -> None: request = Request("https://example.com") spider = self.get_spider() - with LogCapture() as log: + with caplog.at_level(logging.WARNING): get_retry_request( request, spider=spider, max_retry_times=0, give_up_log_level="WARNING", ) - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "WARNING", - f"Gave up retrying {request} (failed 1 times): unspecified", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.WARNING, + f"Gave up retrying {request} (failed 1 times): unspecified", + ) in caplog.record_tuples - def test_give_up_log_level_argument_number(self): + def test_give_up_log_level_argument_number( + self, caplog: pytest.LogCaptureFixture + ) -> None: request = Request("https://example.com") spider = self.get_spider() - with LogCapture() as log: + with caplog.at_level(logging.WARNING): get_retry_request( request, spider=spider, max_retry_times=0, give_up_log_level=logging.WARNING, ) - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "WARNING", - f"Gave up retrying {request} (failed 1 times): unspecified", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.WARNING, + f"Gave up retrying {request} (failed 1 times): unspecified", + ) in caplog.record_tuples - def test_give_up_log_level_setting(self): + def test_give_up_log_level_setting(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider({"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"}) - with LogCapture() as log: + with caplog.at_level(logging.WARNING): get_retry_request( request, spider=spider, max_retry_times=0, ) - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "WARNING", - f"Gave up retrying {request} (failed 1 times): unspecified", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.WARNING, + f"Gave up retrying {request} (failed 1 times): unspecified", + ) in caplog.record_tuples def test_give_up_log_level_invalid(self): request = Request("https://example.com") @@ -738,6 +736,7 @@ class TestGetRetryRequest: reason=expected_reason, stats_base_key=stats_key, ) + assert spider.crawler.stats for stat in ( f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}", diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index f82041a62..793a2b5be 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,13 +1,13 @@ from __future__ import annotations import asyncio -from typing import TYPE_CHECKING from unittest import mock import pytest from twisted.internet.defer import Deferred, DeferredList from twisted.python import failure +from scrapy import signals from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware from scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse @@ -18,15 +18,13 @@ from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from tests.utils.decorators import coroutine_test from tests.utils.robotstxt import rerp_available -if TYPE_CHECKING: - from scrapy.crawler import Crawler - class TestRobotsTxtMiddleware: def setup_method(self) -> None: self.crawler: mock.MagicMock = mock.MagicMock() self.crawler.settings = Settings() self.crawler.engine.download_async = mock.AsyncMock() + self.crawler.signals.send_catch_log_async = mock.AsyncMock(return_value=[]) def teardown_method(self): del self.crawler @@ -37,7 +35,7 @@ class TestRobotsTxtMiddleware: with pytest.raises(NotConfigured): RobotsTxtMiddleware(self.crawler) - def _get_successful_crawler(self) -> Crawler: + def _get_successful_crawler(self) -> mock.MagicMock: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) ROBOTS = """ @@ -52,8 +50,8 @@ Disallow: /some/randome/page.html """.encode() response = TextResponse("http://site.local/robots.txt", body=ROBOTS) - async def return_response(request): - deferred = Deferred() + async def return_response(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.callback, response) return await maybe_deferred_to_future(deferred) @@ -74,6 +72,21 @@ Disallow: /some/randome/page.html Request("http://site.local/wiki/Käyttäjä:"), middleware ) + @coroutine_test + async def test_robotstxt_emits_robots_parsed_signal(self): + crawler = self._get_successful_crawler() + middleware = RobotsTxtMiddleware(crawler) + request = Request("http://site.local/allowed") + await self.assertNotIgnored(request, middleware) + calls = [ + kwargs + for _, kwargs in crawler.signals.send_catch_log_async.call_args_list + if kwargs.get("signal") is signals.robots_parsed + ] + assert len(calls) == 1 + assert calls[0]["request"] is request + assert calls[0]["robotparser"] is not None + @coroutine_test async def test_robotstxt_multiple_reqs(self) -> None: middleware = RobotsTxtMiddleware(self._get_successful_crawler()) @@ -113,15 +126,15 @@ Disallow: /some/randome/page.html Request("http://site.local/static/", meta=meta), middleware ) - def _get_garbage_crawler(self) -> Crawler: + def _get_garbage_crawler(self) -> mock.MagicMock: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) response = Response( "http://site.local/robots.txt", body=b"GIF89a\xd3\x00\xfe\x00\xa2" ) - async def return_response(request): - deferred = Deferred() + async def return_response(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.callback, response) return await maybe_deferred_to_future(deferred) @@ -137,13 +150,13 @@ Disallow: /some/randome/page.html await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware) await self.assertNotIgnored(Request("http://site.local/static/"), middleware) - def _get_emptybody_crawler(self) -> Crawler: + def _get_emptybody_crawler(self) -> mock.MagicMock: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) response = Response("http://site.local/robots.txt") - async def return_response(request): - deferred = Deferred() + async def return_response(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.callback, response) return await maybe_deferred_to_future(deferred) @@ -163,8 +176,8 @@ Disallow: /some/randome/page.html self.crawler.settings.set("ROBOTSTXT_OBEY", True) err = CannotResolveHostError("Robotstxt address not found") - async def return_failure(request): - deferred = Deferred() + async def return_failure(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.errback, failure.Failure(err)) return await maybe_deferred_to_future(deferred) @@ -191,8 +204,8 @@ Disallow: /some/randome/page.html async def test_ignore_robotstxt_request(self): self.crawler.settings.set("ROBOTSTXT_OBEY", True) - async def ignore_request(request): - deferred = Deferred() + async def ignore_request(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.errback, failure.Failure(IgnoreRequest())) return await maybe_deferred_to_future(deferred) @@ -219,7 +232,7 @@ Disallow: /some/randome/page.html @coroutine_test async def test_robotstxt_local_file(self): middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) - middleware.process_request_2 = mock.MagicMock() + middleware.process_request_2 = mock.MagicMock() # type: ignore[method-assign] await middleware.process_request(Request("data:text/plain,Hello World data")) assert not middleware.process_request_2.called diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 5609360a7..b8fda25b4 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from scrapy.downloadermiddlewares.stats import DownloaderStats, get_header_size @@ -12,8 +14,9 @@ class MyException(Exception): class TestDownloaderStats: - def setup_method(self): + def setup_method(self) -> None: self.crawler = get_crawler(Spider) + assert self.crawler.stats is not None self.mw = DownloaderStats(self.crawler.stats) self.crawler.stats.open_spider() @@ -21,20 +24,21 @@ class TestDownloaderStats: self.req = Request("http://scrapytest.org") self.res = Response("http://scrapytest.org", status=400) - def assertStatsEqual(self, key, value): + def assertStatsEqual(self, key: str, value: object) -> None: + assert self.crawler.stats is not None assert self.crawler.stats.get_value(key) == value, str( self.crawler.stats.get_stats() ) - def test_process_request(self): + def test_process_request(self) -> None: self.mw.process_request(self.req) self.assertStatsEqual("downloader/request_count", 1) - def test_process_response(self): + def test_process_response(self) -> None: self.mw.process_response(self.req, self.res) self.assertStatsEqual("downloader/response_count", 1) - def test_process_exception(self): + def test_process_exception(self) -> None: self.mw.process_exception(self.req, MyException()) self.assertStatsEqual("downloader/exception_count", 1) self.assertStatsEqual( @@ -42,14 +46,17 @@ class TestDownloaderStats: 1, ) - def test_from_crawler_not_configured(self): + def test_from_crawler_not_configured(self) -> None: crawler = get_crawler(Spider, {"DOWNLOADER_STATS": False}) with pytest.raises(NotConfigured): DownloaderStats.from_crawler(crawler) - def teardown_method(self): + def teardown_method(self) -> None: + assert self.crawler.stats is not None self.crawler.stats.close_spider() -def test_get_header_size_non_list_value(): - assert get_header_size({"Content-Type": "text/html"}) == 0 +def test_get_header_size_non_list_value() -> None: + # Deliberately passing a non-list/tuple header value to make sure + # get_header_size() degrades gracefully instead of raising. + assert get_header_size({"Content-Type": "text/html"}) == 0 # type: ignore[dict-item] diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 5d79691b2..479332eae 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -1,11 +1,14 @@ +from __future__ import annotations + import hashlib +import logging import shutil import sys import tempfile from pathlib import Path +from typing import TYPE_CHECKING, Any import pytest -from testfixtures import LogCapture from scrapy.core.scheduler import Scheduler from scrapy.dupefilters import BaseDupeFilter, RFPDupeFilter @@ -15,8 +18,16 @@ from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider +if TYPE_CHECKING: + from scrapy.crawler import Crawler -def _get_dupefilter(*, crawler=None, settings=None, open_=True): + +def _get_dupefilter( + *, + crawler: Crawler | None = None, + settings: dict[str, Any] | None = None, + open_: bool = True, +) -> BaseDupeFilter: if crawler is None: crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) @@ -151,108 +162,71 @@ class TestRFPDupeFilter: finally: shutil.rmtree(path) - def test_log(self): - with LogCapture() as log: - settings = { - "DUPEFILTER_DEBUG": False, - "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, - } - crawler = get_crawler(SimpleSpider, settings_dict=settings) - spider = SimpleSpider.from_crawler(crawler) - dupefilter = _get_dupefilter(crawler=crawler) + def test_log(self, caplog: pytest.LogCaptureFixture) -> None: + settings = { + "DUPEFILTER_DEBUG": False, + "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, + } + crawler = get_crawler(SimpleSpider, settings_dict=settings) + spider = SimpleSpider.from_crawler(crawler) + dupefilter = _get_dupefilter(crawler=crawler) - r1 = Request("http://scrapytest.org/index.html") - r2 = Request("http://scrapytest.org/index.html") + r1 = Request("http://scrapytest.org/index.html") + r2 = Request("http://scrapytest.org/index.html") + with caplog.at_level(logging.DEBUG): dupefilter.log(r1, spider) dupefilter.log(r2, spider) - assert crawler.stats.get_value("dupefilter/filtered") == 2 - log.check_present( - ( - "scrapy.dupefilters", - "DEBUG", - "Filtered duplicate request: - no more" - " duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)", - ) - ) + assert crawler.stats + assert crawler.stats.get_value("dupefilter/filtered") == 2 + assert ( + "scrapy.dupefilters", + logging.DEBUG, + "Filtered duplicate request: - no more" + " duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)", + ) in caplog.record_tuples - dupefilter.close("finished") + dupefilter.close("finished") - def test_log_debug(self): - with LogCapture() as log: - settings = { - "DUPEFILTER_DEBUG": True, - "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, - } - crawler = get_crawler(SimpleSpider, settings_dict=settings) - spider = SimpleSpider.from_crawler(crawler) - dupefilter = _get_dupefilter(crawler=crawler) + @pytest.mark.parametrize("df", [None, FromCrawlerRFPDupeFilter]) + def test_log_debug( + self, caplog: pytest.LogCaptureFixture, df: type[BaseDupeFilter] | None + ) -> None: + settings: dict[str, Any] = { + "DUPEFILTER_DEBUG": True, + } + if df: + settings["DUPEFILTER_CLASS"] = df + crawler = get_crawler(SimpleSpider, settings_dict=settings) + spider = SimpleSpider.from_crawler(crawler) + dupefilter = _get_dupefilter(crawler=crawler) - r1 = Request("http://scrapytest.org/index.html") - r2 = Request( - "http://scrapytest.org/index.html", - headers={"Referer": "http://scrapytest.org/INDEX.html"}, - ) + r1 = Request("http://scrapytest.org/index.html") + r2 = Request( + "http://scrapytest.org/index.html", + headers={"Referer": "http://scrapytest.org/INDEX.html"}, + ) + with caplog.at_level(logging.DEBUG): dupefilter.log(r1, spider) dupefilter.log(r2, spider) - assert crawler.stats.get_value("dupefilter/filtered") == 2 - log.check_present( - ( - "scrapy.dupefilters", - "DEBUG", - "Filtered duplicate request: (referer: None)", - ) - ) - log.check_present( - ( - "scrapy.dupefilters", - "DEBUG", - "Filtered duplicate request: " - " (referer: http://scrapytest.org/INDEX.html)", - ) - ) + assert crawler.stats + assert crawler.stats.get_value("dupefilter/filtered") == 2 + assert ( + "scrapy.dupefilters", + logging.DEBUG, + "Filtered duplicate request: (referer: None)", + ) in caplog.record_tuples + assert ( + "scrapy.dupefilters", + logging.DEBUG, + "Filtered duplicate request: " + " (referer: http://scrapytest.org/INDEX.html)", + ) in caplog.record_tuples - dupefilter.close("finished") - - def test_log_debug_default_dupefilter(self): - with LogCapture() as log: - settings = { - "DUPEFILTER_DEBUG": True, - } - crawler = get_crawler(SimpleSpider, settings_dict=settings) - spider = SimpleSpider.from_crawler(crawler) - dupefilter = _get_dupefilter(crawler=crawler) - - r1 = Request("http://scrapytest.org/index.html") - r2 = Request( - "http://scrapytest.org/index.html", - headers={"Referer": "http://scrapytest.org/INDEX.html"}, - ) - - dupefilter.log(r1, spider) - dupefilter.log(r2, spider) - - assert crawler.stats.get_value("dupefilter/filtered") == 2 - log.check_present( - ( - "scrapy.dupefilters", - "DEBUG", - "Filtered duplicate request: (referer: None)", - ) - ) - log.check_present( - ( - "scrapy.dupefilters", - "DEBUG", - "Filtered duplicate request: " - " (referer: http://scrapytest.org/INDEX.html)", - ) - ) - - dupefilter.close("finished") + dupefilter.close("finished") class TestBaseDupeFilter: diff --git a/tests/test_engine.py b/tests/test_engine.py index 3ab8f6d1b..8a3bceccb 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -1,115 +1,45 @@ from __future__ import annotations import asyncio -import re +import logging import subprocess import sys -from collections import defaultdict -from dataclasses import dataclass from typing import TYPE_CHECKING, Any from unittest.mock import Mock -from urllib.parse import urlparse -import attr import pytest -from itemadapter import ItemAdapter -from pydispatch import dispatcher -from testfixtures import LogCapture -from twisted.internet import defer from scrapy import signals from scrapy.core.engine import ExecutionEngine, _Slot from scrapy.core.scheduler import BaseScheduler from scrapy.exceptions import CloseSpider, IgnoreRequest -from scrapy.http import Headers, Request, Response -from scrapy.item import Field, Item -from scrapy.linkextractors import LinkExtractor +from scrapy.http import Request from scrapy.spiders import Spider -from scrapy.utils.defer import ( - _schedule_coro, - deferred_from_coro, - maybe_deferred_to_future, -) -from scrapy.utils.signal import disconnect_all +from scrapy.utils.defer import _schedule_coro, deferred_from_coro from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -from tests import get_testdata +from tests.utils.bases.engine import TestEngineBase from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.engine import ( + AttrsItemsSpider, + CrawlerRun, + DataClassItemsSpider, + DictItemsSpider, + MySpider, +) if TYPE_CHECKING: from collections.abc import AsyncIterator - from twisted.python.failure import Failure - from tests.mockserver.http import MockServer -class MyItem(Item): - name = Field() - url = Field() - price = Field() - - -@attr.s -class AttrsItem: - name = attr.ib(default="") - url = attr.ib(default="") - price = attr.ib(default=0) - - -@dataclass -class DataClassItem: - name: str = "" - url: str = "" - price: int = 0 - - -class MySpider(Spider): - name = "scrapytest.org" - - itemurl_re = re.compile(r"item\d+.html") - name_re = re.compile(r"

(.*?)

", re.MULTILINE) - price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE) - - item_cls: type = MyItem - - def parse(self, response): - xlink = LinkExtractor() - itemre = re.compile(self.itemurl_re) - for link in xlink.extract_links(response): - if itemre.search(link.url): - yield Request(url=link.url, callback=self.parse_item) - - def parse_item(self, response): - adapter = ItemAdapter(self.item_cls()) - m = self.name_re.search(response.text) - if m: - adapter["name"] = m.group(1) - adapter["url"] = response.url - m = self.price_re.search(response.text) - if m: - adapter["price"] = m.group(1) - return adapter.item - - class DupeFilterSpider(MySpider): async def start(self): for url in self.start_urls: yield Request(url) # no dont_filter=True -class DictItemsSpider(MySpider): - item_cls = dict - - -class AttrsItemsSpider(MySpider): - item_cls = AttrsItem - - -class DataClassItemsSpider(MySpider): - item_cls = DataClassItem - - class ItemZeroDivisionErrorSpider(MySpider): custom_settings = { "ITEM_PIPELINES": { @@ -130,253 +60,6 @@ class ChangeCloseReasonSpider(MySpider): raise CloseSpider(reason="custom_reason") -class CrawlerRun: - """A class to run the crawler and keep track of events occurred""" - - def __init__(self, spider_class: type[Spider]): - self.respplug: list[tuple[Response, Spider]] = [] - self.reqplug: list[tuple[Request, Spider]] = [] - self.reqdropped: list[tuple[Request, Spider]] = [] - self.reqreached: list[tuple[Request, Spider]] = [] - self.itemerror: list[tuple[Any, Response, Spider, Failure]] = [] - self.itemresp: list[tuple[Any, Response]] = [] - self.headers: dict[Request, Headers] = {} - self.bytes: defaultdict[Request, list[bytes]] = defaultdict(list) - self.signals_caught: dict[Any, dict[str, Any]] = {} - self.spider_class = spider_class - - async def run(self, mockserver: MockServer) -> None: - self.mockserver = mockserver - - start_urls = [ - self.geturl("/static/"), - self.geturl("/redirect"), - self.geturl("/redirect"), # duplicate - self.geturl("/numbers"), - ] - - for name, signal in vars(signals).items(): - if not name.startswith("_"): - dispatcher.connect(self.record_signal, signal) - - self.crawler = get_crawler(self.spider_class) - self.crawler.signals.connect(self.item_scraped, signals.item_scraped) - self.crawler.signals.connect(self.item_error, signals.item_error) - self.crawler.signals.connect(self.headers_received, signals.headers_received) - self.crawler.signals.connect(self.bytes_received, signals.bytes_received) - self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) - self.crawler.signals.connect(self.request_dropped, signals.request_dropped) - self.crawler.signals.connect( - self.request_reached, signals.request_reached_downloader - ) - self.crawler.signals.connect( - self.response_downloaded, signals.response_downloaded - ) - self.crawler.crawl(start_urls=start_urls) - - self.deferred: defer.Deferred[None] = defer.Deferred() - dispatcher.connect(self.stop, signals.engine_stopped) - await maybe_deferred_to_future(self.deferred) - - async def stop(self): - for name, signal in vars(signals).items(): - if not name.startswith("_"): - disconnect_all(signal) - self.deferred.callback(None) - await self.crawler.stop_async() - - def geturl(self, path: str) -> str: - return self.mockserver.url(path) - - def getpath(self, url: str) -> str: - u = urlparse(url) - return u.path - - def item_error( - self, item: Any, response: Response, spider: Spider, failure: Failure - ) -> None: - self.itemerror.append((item, response, spider, failure)) - - def item_scraped(self, item: Any, spider: Spider, response: Response) -> None: - self.itemresp.append((item, response)) - - def headers_received( - self, headers: Headers, body_length: int, request: Request, spider: Spider - ) -> None: - self.headers[request] = headers - - def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None: - self.bytes[request].append(data) - - def request_scheduled(self, request: Request, spider: Spider) -> None: - self.reqplug.append((request, spider)) - - def request_reached(self, request: Request, spider: Spider) -> None: - self.reqreached.append((request, spider)) - - def request_dropped(self, request: Request, spider: Spider) -> None: - self.reqdropped.append((request, spider)) - - def response_downloaded(self, response: Response, spider: Spider) -> None: - self.respplug.append((response, spider)) - - def record_signal(self, *args: Any, **kwargs: Any) -> None: - """Record a signal and its parameters""" - signalargs = kwargs.copy() - sig = signalargs.pop("signal") - signalargs.pop("sender", None) - self.signals_caught[sig] = signalargs - - -class TestEngineBase: - @staticmethod - def _assert_visited_urls(run: CrawlerRun) -> None: - must_be_visited = [ - "/static/", - "/redirect", - "/redirected", - "/static/item1.html", - "/static/item2.html", - "/static/item999.html", - ] - urls_visited = {rp[0].url for rp in run.respplug} - urls_expected = {run.geturl(p) for p in must_be_visited} - assert urls_expected <= urls_visited, ( - f"URLs not visited: {list(urls_expected - urls_visited)}" - ) - - @staticmethod - def _assert_scheduled_requests(run: CrawlerRun, count: int) -> None: - assert len(run.reqplug) == count - - paths_expected = [ - "/static/item999.html", - "/static/item2.html", - "/static/item1.html", - ] - - urls_requested = {rq[0].url for rq in run.reqplug} - urls_expected = {run.geturl(p) for p in paths_expected} - assert urls_expected <= urls_requested - scheduled_requests_count = len(run.reqplug) - dropped_requests_count = len(run.reqdropped) - responses_count = len(run.respplug) - assert scheduled_requests_count == dropped_requests_count + responses_count - assert len(run.reqreached) == responses_count - - @staticmethod - def _assert_dropped_requests(run: CrawlerRun) -> None: - assert len(run.reqdropped) == 1 - - @staticmethod - def _assert_downloaded_responses(run: CrawlerRun, count: int) -> None: - # response tests - assert len(run.respplug) == count - assert len(run.reqreached) == count - - for response, _ in run.respplug: - if run.getpath(response.url) == "/static/item999.html": - assert response.status == 404 - if run.getpath(response.url) == "/redirect": - assert response.status == 302 - - @staticmethod - def _assert_items_error(run: CrawlerRun) -> None: - assert len(run.itemerror) == 2 - for item, response, spider, failure in run.itemerror: - assert failure.value.__class__ is ZeroDivisionError - assert spider == run.crawler.spider - - assert item["url"] == response.url - if "item1.html" in item["url"]: - assert item["name"] == "Item 1 name" - assert item["price"] == "100" - if "item2.html" in item["url"]: - assert item["name"] == "Item 2 name" - assert item["price"] == "200" - - @staticmethod - def _assert_scraped_items(run: CrawlerRun) -> None: - assert len(run.itemresp) == 2 - for item_, response in run.itemresp: - item = ItemAdapter(item_) - assert item["url"] == response.url - if "item1.html" in item["url"]: - assert item["name"] == "Item 1 name" - assert item["price"] == "100" - if "item2.html" in item["url"]: - assert item["name"] == "Item 2 name" - assert item["price"] == "200" - - @staticmethod - def _assert_headers_received(run: CrawlerRun) -> None: - for headers in run.headers.values(): - assert b"Server" in headers - assert headers[b"Server"] - assert b"TwistedWeb" in headers[b"Server"] - assert b"Date" in headers - assert b"Content-Type" in headers - - @staticmethod - def _assert_bytes_received(run: CrawlerRun) -> None: - assert len(run.bytes) == 9 - for request, data in run.bytes.items(): - joined_data = b"".join(data) - if run.getpath(request.url) == "/static/": - assert joined_data == get_testdata("test_site", "index.html") - elif run.getpath(request.url) == "/static/item1.html": - assert joined_data == get_testdata("test_site", "item1.html") - elif run.getpath(request.url) == "/static/item2.html": - assert joined_data == get_testdata("test_site", "item2.html") - elif run.getpath(request.url) == "/redirected": - assert joined_data == b"Redirected here" - elif run.getpath(request.url) == "/redirect": - assert ( - joined_data == b"\n\n" - b" \n" - b' \n' - b" \n" - b' \n' - b' click here\n' - b" \n" - b"\n" - ) - elif run.getpath(request.url) == "/static/item999.html": - assert ( - joined_data == b"\n\n" - b" 404 - No Such Resource\n" - b" \n" - b"

No Such Resource

\n" - b"

File not found.

\n" - b" \n" - b"\n" - ) - elif run.getpath(request.url) == "/numbers": - # signal was fired multiple times - assert len(data) > 1 - # bytes were received in order - numbers = [str(x).encode("utf8") for x in range(2**18)] - assert joined_data == b"".join(numbers) - - @staticmethod - def _assert_signals_caught(run: CrawlerRun) -> None: - assert signals.engine_started in run.signals_caught - assert signals.engine_stopped in run.signals_caught - assert signals.spider_opened in run.signals_caught - assert signals.spider_idle in run.signals_caught - assert signals.spider_closed in run.signals_caught - assert signals.headers_received in run.signals_caught - - assert {"spider": run.crawler.spider} == run.signals_caught[ - signals.spider_opened - ] - assert {"spider": run.crawler.spider} == run.signals_caught[signals.spider_idle] - assert { - "spider": run.crawler.spider, - "reason": "finished", - } == run.signals_caught[signals.spider_closed] - - class TestEngine(TestEngineBase): @coroutine_test async def test_crawler(self, mockserver: MockServer) -> None: @@ -462,8 +145,10 @@ class TestEngine(TestEngineBase): await asyncio.gather(e.start_async(), e.start_async()) await e.stop_async() - @inline_callbacks_test - def test_start_request_processing_exception(self): + @coroutine_test + async def test_start_request_processing_exception( + self, caplog: pytest.LogCaptureFixture + ) -> None: class BadRequestFingerprinter: def fingerprint(self, request): raise ValueError # to make Scheduler.enqueue_request() fail @@ -477,10 +162,10 @@ class TestEngine(TestEngineBase): crawler = get_crawler( SimpleSpider, {"REQUEST_FINGERPRINTER_CLASS": BadRequestFingerprinter} ) - with LogCapture() as log: - yield crawler.crawl() - assert "Error while processing requests from start()" in str(log) - assert "Spider closed (shutdown)" in str(log) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async() + assert "Error while processing requests from start()" in caplog.text + assert "Spider closed (shutdown)" in caplog.text def test_short_timeout(self): args = ( diff --git a/tests/test_engine_download.py b/tests/test_engine_download.py index f15bfd5e2..09b998f6a 100644 --- a/tests/test_engine_download.py +++ b/tests/test_engine_download.py @@ -1,5 +1,6 @@ from __future__ import annotations +import sys from unittest.mock import Mock, call import pytest @@ -73,6 +74,25 @@ class TestEngineDownloadAsync: [call(original_request), call(redirect_request)] ) + @coroutine_test + async def test_download_async_many_redirects(self, engine): + """A long chain of requests being replaced by new ones is handled + iteratively, without hitting the recursion limit.""" + count = sys.getrecursionlimit() * 2 + requests = [Request(f"http://example.com/{i}") for i in range(count)] + final_response = Response("http://example.com/final", body=b"done") + engine.downloader.fetch.side_effect = [ + *(defer.succeed(request) for request in requests[1:]), + defer.succeed(final_response), + ] + engine.spider = Mock() + engine._slot.add_request = Mock() + engine._slot.remove_request = Mock() + + result = await self._download(engine, requests[0]) + assert result == final_response + assert engine.downloader.fetch.call_count == count + @coroutine_test async def test_download_async_no_spider(self, engine): """Test async download attempt when no spider is available.""" @@ -96,6 +116,18 @@ class TestEngineDownloadAsync: engine._slot.add_request.assert_called_once_with(request) engine._slot.remove_request.assert_called_once_with(request) + @coroutine_test + async def test_download_async_fetch_needs_spider(self, engine): + engine._downloader_fetch_needs_spider = True + request = Request("http://example.com") + response = Response("http://example.com", body=b"test body") + engine.spider = Mock() + engine.downloader.fetch.return_value = defer.succeed(response) + + result = await self._download(engine, request) + assert result == response + engine.downloader.fetch.assert_called_once_with(request, engine.spider) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestEngineDownload(TestEngineDownloadAsync): diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index c15c396d3..14ec3d184 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -6,10 +6,9 @@ from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals from scrapy.core.scheduler import BaseScheduler -from scrapy.utils.asyncio import call_later +from scrapy.utils.asyncio import call_later, sleep from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer -from tests.utils import async_sleep from tests.utils.decorators import coroutine_test if TYPE_CHECKING: @@ -65,23 +64,23 @@ class TestMain: async def start(self): yield Request("data:,a") - await async_sleep(seconds) + await sleep(seconds) self.crawler.engine._slot.scheduler.pause() self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b")) # During this time, the scheduler reports having requests but # returns None. - await async_sleep(seconds) + await sleep(seconds) self.crawler.engine._slot.scheduler.unpause() # The scheduler request is processed. - await async_sleep(seconds) + await sleep(seconds) yield Request("data:,c") - await async_sleep(seconds) + await sleep(seconds) self.crawler.engine._slot.scheduler.pause() self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d")) diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index 091485781..27f282af4 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -3,15 +3,15 @@ from __future__ import annotations from typing import TYPE_CHECKING from scrapy.exceptions import StopDownload -from tests.test_engine import ( +from tests.utils.bases.engine import TestEngineBase +from tests.utils.decorators import coroutine_test +from tests.utils.engine import ( AttrsItemsSpider, CrawlerRun, DataClassItemsSpider, DictItemsSpider, MySpider, - TestEngineBase, ) -from tests.utils.decorators import coroutine_test if TYPE_CHECKING: import pytest diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index 8ebc948e2..09fa14326 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -3,15 +3,15 @@ from __future__ import annotations from typing import TYPE_CHECKING from scrapy.exceptions import StopDownload -from tests.test_engine import ( +from tests.utils.bases.engine import TestEngineBase +from tests.utils.decorators import coroutine_test +from tests.utils.engine import ( AttrsItemsSpider, CrawlerRun, DataClassItemsSpider, DictItemsSpider, MySpider, - TestEngineBase, ) -from tests.utils.decorators import coroutine_test if TYPE_CHECKING: import pytest diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 3b997767e..b857728ba 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -118,6 +118,18 @@ class TestBaseItemExporter(ABC): ie = self._get_exporter(fields_to_export={"name": "名稱"}) assert list(ie._get_serialized_fields(self.i)) == [("名稱", "John\xa3")] + def test_field_order(self): + item = self.item_class(age="22", name="John\xa3") + ie = self._get_exporter() + assert [name for name, _ in ie._get_serialized_fields(item)] == ["name", "age"] + + def test_field_order_dict_item(self): + ie = self._get_exporter() + assert [name for name, _ in ie._get_serialized_fields({"age": "22"})] == ["age"] + assert [ + name for name, _ in ie._get_serialized_fields({"age": "22", "name": "John"}) + ] == ["age", "name"] + def test_field_custom_serializer(self): i = self.custom_field_item_class(name="John\xa3", age="22") a = ItemAdapter(i) diff --git a/tests/test_extension_memusage.py b/tests/test_extension_memusage.py index a474725d8..76e8ca5d6 100644 --- a/tests/test_extension_memusage.py +++ b/tests/test_extension_memusage.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging import sys +from typing import TYPE_CHECKING import pytest @@ -13,8 +14,12 @@ from scrapy.extensions.memusage import MemoryUsage from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.utils import OneShotLoop +from tests.utils.cmdline import proc from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + from tests.mockserver.http import MockServer + # MemoryUsage relies on the stdlib 'resource' module (not available on Windows) pytestmark = pytest.mark.skipif( sys.platform.startswith("win"), @@ -25,6 +30,14 @@ pytestmark = pytest.mark.skipif( MB = 1024 * 1024 +class TwoShotLoop(OneShotLoop): + """Like :class:`OneShotLoop`, but runs the check twice.""" + + def start(self, interval: float, now: bool = True) -> None: + super().start(interval, now=now) + self.func() + + class _LoopSpider(Spider): name = "loop-data-spider" @@ -50,6 +63,49 @@ def test_memusage_disabled() -> None: MemoryUsage.from_crawler(get_crawler(settings_dict=settings)) +def test_memusage_limit_stops_crawler_without_spider(mockserver: MockServer) -> None: + # The Scrapy shell starts the engine without opening a spider, so the + # whole crawler is stopped instead of a spider being closed. + _, out, err = proc( + "shell", + mockserver.url("/text"), + "-c", + "response.status", + "--set", + "MEMUSAGE_LIMIT_MB=1", + ) + assert "Memory usage exceeded 1MiB" in err + assert "200" in out + + +@coroutine_test +async def test_memusage_below_thresholds_logs_peak( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + settings = { + "MEMUSAGE_LIMIT_MB": 100, + "MEMUSAGE_WARNING_MB": 50, + "MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01, + "TELNETCONSOLE_ENABLED": False, + "LOG_LEVEL": "INFO", + } + + monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop) + monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda _: 25 * MB) + + crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings) + + with caplog.at_level(logging.INFO, logger="scrapy.extensions.memusage"): + await crawler.crawl_async(url="data:,", loops=1) + + assert crawler.stats + assert crawler.stats.get_value("memusage/limit_reached") is None + assert crawler.stats.get_value("memusage/warning_reached") is None + assert crawler.stats.get_value("memusage/max") == 25 * MB + assert crawler.stats.get_value("finish_reason") == "finished" + assert any("Peak memory usage is 25MiB" in r.getMessage() for r in caplog.records) + + @coroutine_test async def test_memusage_limit_closes_spider_with_reason_and_error_log( caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch @@ -92,8 +148,9 @@ async def test_memusage_warning_logs_but_allows_normal_finish( "LOG_LEVEL": "INFO", } - # Avoid background LoopingCall that can log after the test finishes. - monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop) + # Avoid background LoopingCall that can log after the test finishes; check + # twice, since the warning is only meant to be reported once. + monkeypatch.setattr(memusage_mod, "create_looping_call", TwoShotLoop) monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda self: 75 * MB) crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings) @@ -112,4 +169,7 @@ async def test_memusage_warning_logs_but_allows_normal_finish( assert crawler.stats assert crawler.stats.get_value("memusage/warning_reached") == 1 assert crawler.stats.get_value("finish_reason") == "finished" - assert any("memory usage reached" in r.getMessage().lower() for r in caplog.records) + warnings_logged = [ + r for r in caplog.records if "memory usage reached" in r.getMessage().lower() + ] + assert len(warnings_logged) == 1 diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index ffe7a0dc7..723fb6e17 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -1,8 +1,13 @@ from __future__ import annotations import datetime +import json +import logging from typing import TYPE_CHECKING, Any +import pytest + +from scrapy.exceptions import NotConfigured from scrapy.extensions.periodic_log import PeriodicLog from scrapy.utils.test import get_crawler @@ -86,6 +91,14 @@ class TestPeriodicLog: assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60}) assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60}) + def test_no_interval(self): + with pytest.raises(NotConfigured): + extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 0}) + + def test_nothing_enabled(self): + with pytest.raises(NotConfigured): + extension({"LOGSTATS_INTERVAL": 60}) + @coroutine_test async def test_log_delta(self): def emulate( @@ -212,3 +225,26 @@ class TestPeriodicLog: {"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}}, lambda k, v: "downloader/" in k and "bytes" not in k, ) + + @coroutine_test + async def test_log_timing(self, caplog: pytest.LogCaptureFixture) -> None: + settings = { + "EXTENSIONS": {"scrapy.extensions.periodic_log.PeriodicLog": 0}, + "PERIODIC_LOG_TIMING_ENABLED": True, + "LOGSTATS_INTERVAL": 30, + } + crawler = get_crawler(MetaSpider, settings) + with caplog.at_level(logging.INFO, logger="scrapy.extensions.periodic_log"): + await crawler.crawl_async() + + records = [ + r for r in caplog.records if r.name == "scrapy.extensions.periodic_log" + ] + assert records, "PeriodicLog logged nothing" + # Only the timing section is enabled, and it is logged on spider close. + data = json.loads(records[-1].getMessage()) + assert list(data) == ["time"] + assert data["time"]["log_interval"] == 30 + assert data["time"]["log_interval_real"] >= 0 + assert data["time"]["elapsed"] >= 0 + assert data["time"]["start_time"] <= data["time"]["utcnow"] diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 20c801558..fca0e3153 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -7,7 +7,8 @@ import pytest from twisted.conch.telnet import ITelnetProtocol from twisted.cred import credentials -from scrapy.extensions.telnet import TelnetConsole +from scrapy import Spider +from scrapy.extensions.telnet import TelnetConsole, update_telnet_vars from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler from tests.utils.decorators import coroutine_test @@ -16,16 +17,20 @@ if TYPE_CHECKING: from collections.abc import Generator from scrapy.crawler import Crawler + from scrapy.http import Response pytestmark = pytest.mark.requires_reactor # TelnetConsole requires a reactor -def _get_crawler(settings_dict: dict[str, Any] | None = None) -> Crawler: +def _get_crawler( + spidercls: type[Spider] | None = None, + settings_dict: dict[str, Any] | None = None, +) -> Crawler: settings = { "TELNETCONSOLE_ENABLED": True, **(settings_dict or {}), } - return get_crawler(settings_dict=settings) + return get_crawler(spidercls, settings_dict=settings) @contextmanager @@ -84,3 +89,47 @@ def test_invalid_reversed_portrange() -> None: console = TelnetConsole(_get_crawler(settings_dict=settings)) with pytest.raises(ValueError, match=r"invalid portrange: \[2, 1\]"): console.start_listening() + + +@coroutine_test +async def test_telnet_vars() -> None: + """Log into the console of a running crawl, which is when the telnet + variables are built.""" + received: list[dict[str, Any]] = [] + + def on_update_telnet_vars(telnet_vars: dict[str, Any]) -> None: + received.append(telnet_vars) + + class TelnetSpider(Spider): + name = "telnet" + start_urls = ["data:,"] + + async def parse(self, response: Response) -> None: + assert self.crawler.extensions + console = next( + ext + for ext in self.crawler.extensions.middlewares + if isinstance(ext, TelnetConsole) + ) + creds = credentials.UsernamePassword( + console.username.encode("utf8"), console.password.encode("utf8") + ) + portal = console.protocol().protocolArgs[0] + await maybe_deferred_to_future(portal.login(creds, None, ITelnetProtocol)) + + crawler = _get_crawler(TelnetSpider) + crawler.signals.connect(on_update_telnet_vars, signal=update_telnet_vars) + await crawler.crawl_async() + + assert len(received) == 1 + telnet_vars = received[0] + assert telnet_vars["crawler"] is crawler + assert telnet_vars["engine"] is crawler.engine + assert telnet_vars["spider"] is crawler.spider + assert telnet_vars["extensions"] is crawler.extensions + assert telnet_vars["stats"] is crawler.stats + assert telnet_vars["settings"] is crawler.settings + assert callable(telnet_vars["est"]) + assert callable(telnet_vars["p"]) + assert callable(telnet_vars["prefs"]) + assert "telnetconsole.html" in telnet_vars["help"] diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index 4874f284a..2c718d95f 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -3,7 +3,7 @@ from unittest.mock import Mock import pytest -from scrapy import Request, Spider +from scrapy import Request from scrapy.exceptions import NotConfigured from scrapy.extensions.throttle import AutoThrottle from scrapy.http.response import Response @@ -25,6 +25,13 @@ def get_crawler(settings=None, spidercls=None): return _get_crawler(settings_dict=settings, spidercls=spidercls) +def _mock_downloader(crawler): + """Give *crawler* a mock engine, whose downloader AutoThrottle reads.""" + crawler.engine = Mock() + crawler.engine.downloader.slots = {} + return crawler.engine.downloader + + @pytest.mark.parametrize( ("value", "expected"), [ @@ -60,29 +67,21 @@ def test_target_concurrency_invalid(value): @pytest.mark.parametrize( - ("spider", "setting", "expected"), + ("setting", "expected"), [ - (UNSET, UNSET, DOWNLOAD_DELAY), - (1.0, UNSET, 1.0), - (UNSET, 1.0, 1.0), - (1.0, 2.0, 1.0), - (3.0, 2.0, 3.0), + (UNSET, DOWNLOAD_DELAY), + (1.0, 1.0), ], ) -def test_mindelay_definition(spider, setting, expected): +def test_mindelay_definition(setting, expected): settings = {} if setting is not UNSET: settings["DOWNLOAD_DELAY"] = setting - class _TestSpider(Spider): - name = "test" - - if spider is not UNSET: - _TestSpider.download_delay = spider - - crawler = get_crawler(settings, _TestSpider) + crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - at._spider_opened(_TestSpider()) + _mock_downloader(crawler) + at._spider_opened(DefaultSpider()) assert at.mindelay == expected @@ -99,58 +98,43 @@ def test_maxdelay_definition(value, expected): settings["AUTOTHROTTLE_MAX_DELAY"] = value crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) + _mock_downloader(crawler) at._spider_opened(DefaultSpider()) assert at.maxdelay == expected @pytest.mark.parametrize( - ("min_spider", "min_setting", "start_setting", "expected"), + ("min_setting", "start_setting", "expected"), [ - (UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), - (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), - (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), - (UNSET, AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY), - (UNSET, AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), - (UNSET, UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0), - (UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 1.0), - ( - AUTOTHROTTLE_START_DELAY + 1.0, - AUTOTHROTTLE_START_DELAY + 2.0, - UNSET, - AUTOTHROTTLE_START_DELAY + 1.0, - ), + (UNSET, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), + (UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0), + (UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 1.0), ( AUTOTHROTTLE_START_DELAY + 2.0, - UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 2.0, ), ( AUTOTHROTTLE_START_DELAY + 1.0, - UNSET, AUTOTHROTTLE_START_DELAY + 2.0, AUTOTHROTTLE_START_DELAY + 2.0, ), ], ) -def test_startdelay_definition(min_spider, min_setting, start_setting, expected): +def test_startdelay_definition(min_setting, start_setting, expected): settings = {} if min_setting is not UNSET: settings["DOWNLOAD_DELAY"] = min_setting if start_setting is not UNSET: settings["AUTOTHROTTLE_START_DELAY"] = start_setting - class _TestSpider(Spider): - name = "test" - - if min_spider is not UNSET: - _TestSpider.download_delay = min_spider - - crawler = get_crawler(settings, _TestSpider) + crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - spider = _TestSpider() - at._spider_opened(spider) - assert spider.download_delay == expected + downloader = _mock_downloader(crawler) + at._spider_opened(DefaultSpider()) + assert downloader._delay == expected @pytest.mark.parametrize( @@ -174,15 +158,13 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) def test_skipped(meta, slot): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) request = Request("https://example.com", meta=meta) - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} if slot is not None: - crawler.engine.downloader.slots[slot] = object() + downloader.slots[slot] = object() at._adjust_delay = None # Raise exception if called. at._response_downloaded(None, request, spider) @@ -204,18 +186,16 @@ def test_adjustment(download_latency, target_concurrency, slot_delay, expected): settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) response = Response(request.url) - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} slot = Mock() slot.delay = slot_delay - crawler.engine.downloader.slots["foo"] = slot + downloader.slots["foo"] = slot at._response_downloaded(response, request, spider) @@ -240,18 +220,16 @@ def test_adjustment_limits(mindelay, maxdelay, expected): } crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) response = Response(request.url) - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} slot = Mock() slot.delay = slot_delay - crawler.engine.downloader.slots["foo"] = slot + downloader.slots["foo"] = slot at._response_downloaded(response, request, spider) @@ -272,18 +250,16 @@ def test_adjustment_bad_response( settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) response = Response(request.url, status=400) - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} slot = Mock() slot.delay = slot_delay - crawler.engine.downloader.slots["foo"] = slot + downloader.slots["foo"] = slot at._response_downloaded(response, request, spider) @@ -294,19 +270,17 @@ def test_debug(caplog): settings = {"AUTOTHROTTLE_DEBUG": True} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": 1.0, "download_slot": "foo"} request = Request("https://example.com", meta=meta) response = Response(request.url, body=b"foo") - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} slot = Mock() slot.delay = 2.0 slot.transferring = (None, None) - crawler.engine.downloader.slots["foo"] = slot + downloader.slots["foo"] = slot caplog.clear() with caplog.at_level(INFO): @@ -324,19 +298,17 @@ def test_debug(caplog): def test_debug_disabled(caplog): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": 1.0, "download_slot": "foo"} request = Request("https://example.com", meta=meta) response = Response(request.url, body=b"foo") - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} slot = Mock() slot.delay = 2.0 slot.transferring = (None, None) - crawler.engine.downloader.slots["foo"] = slot + downloader.slots["foo"] = slot caplog.clear() with caplog.at_level(INFO): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 3f659ee0b..40a763efd 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2,21 +2,17 @@ from __future__ import annotations import csv import json +import logging import marshal import pickle -import random -import shutil import tempfile -from abc import ABC, abstractmethod from logging import getLogger from pathlib import Path -from string import ascii_letters, digits from typing import IO, TYPE_CHECKING, Any from unittest import mock import lxml.etree import pytest -from testfixtures import LogCapture from w3lib.url import file_uri_to_path import scrapy @@ -28,12 +24,13 @@ from scrapy.extensions.feedexport import ( FeedExporter, FeedSlot, FileFeedStorage, + ItemFilter, apply_uri_params, ) from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler -from tests.mockserver.http import MockServer from tests.spiders import ItemSpider +from tests.utils.bases.feedexport import TestFeedExportBase from tests.utils.decorators import coroutine_test, inline_callbacks_test from tests.utils.feedexport import MyItem, MyItem2, path_to_url, printf_escape @@ -99,141 +96,6 @@ class LogOnStoreFileStorage: file.close() -class TestFeedExportBase(ABC): - mockserver: MockServer - - def _random_temp_filename(self, inter_dir="") -> Path: - chars = [random.choice(ascii_letters + digits) for _ in range(15)] - filename = "".join(chars) - return Path(self.temp_dir, inter_dir, filename) - - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - - def setup_method(self): - self.temp_dir = tempfile.mkdtemp() - - def teardown_method(self): - shutil.rmtree(self.temp_dir, ignore_errors=True) - - async def exported_data( - self, items: Iterable[Any], settings: dict[str, Any] - ) -> dict[str, Any]: - """ - Return exported data which a spider yielding ``items`` would return. - """ - - class TestSpider(scrapy.Spider): - name = "testspider" - - def parse(self, response): - yield from items - - return await self.run_and_export(TestSpider, settings) - - async def exported_no_data(self, settings: dict[str, Any]) -> dict[str, Any]: - """ - Return exported data which a spider yielding no ``items`` would return. - """ - - class TestSpider(scrapy.Spider): - name = "testspider" - - def parse(self, response): - pass - - return await self.run_and_export(TestSpider, settings) - - async def assertExported( - self, - items: Iterable[Any], - header: Iterable[str], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - await self.assertExportedCsv(items, header, rows, settings) - await self.assertExportedJsonLines(items, rows, settings) - await self.assertExportedXml(items, rows, settings) - await self.assertExportedPickle(items, rows, settings) - await self.assertExportedMarshal(items, rows, settings) - await self.assertExportedMultiple(items, rows, settings) - - async def assertExportedCsv( # noqa: B027 - self, - items: Iterable[Any], - header: Iterable[str], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - async def assertExportedJsonLines( # noqa: B027 - self, - items: Iterable[Any], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - async def assertExportedXml( # noqa: B027 - self, - items: Iterable[Any], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - async def assertExportedMultiple( # noqa: B027 - self, - items: Iterable[Any], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - async def assertExportedPickle( # noqa: B027 - self, - items: Iterable[Any], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - async def assertExportedMarshal( # noqa: B027 - self, - items: Iterable[Any], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - @abstractmethod - async def run_and_export( - self, spider_cls: type[Spider], settings: dict[str, Any] - ) -> dict[str, Any]: - pass - - def _load_until_eof( - self, data: bytes, load_func: Callable[[IO[bytes]], Any] - ) -> list[Any]: - result: list[Any] = [] - with tempfile.TemporaryFile() as temp: - temp.write(data) - temp.seek(0) - while True: - try: - result.append(load_func(temp)) - except EOFError: - break - return result - - class InstrumentedFeedSlot(FeedSlot): """Instrumented FeedSlot subclass for keeping track of calls to start_exporting and finish_exporting.""" @@ -702,7 +564,9 @@ class TestFeedExport(TestFeedExportBase): assert expctd == data[fmt] @coroutine_test - async def test_export_no_items_multiple_feeds(self): + async def test_export_no_items_multiple_feeds( + self, caplog: pytest.LogCaptureFixture + ) -> None: """Make sure that `storage.store` is not called.""" settings = { "FEEDS": { @@ -714,10 +578,10 @@ class TestFeedExport(TestFeedExportBase): "FEED_STORE_EMPTY": False, } - with LogCapture() as log: + with caplog.at_level(logging.INFO): await self.exported_no_data(settings) - assert str(log).count("Storage.store is called") == 0 + assert caplog.text.count("Storage.store is called") == 0 @coroutine_test async def test_export_multiple_item_classes(self): @@ -812,14 +676,14 @@ class TestFeedExport(TestFeedExportBase): formats = { "csv": b"foo,egg,baz\r\nbar1,spam1,\r\n", - "json": b'[\n{"hello": "world2", "foo": "bar2"}\n]', + "json": b'[\n{"foo": "bar2", "hello": "world2"}\n]', "jsonlines": ( - b'{"foo": "bar1", "egg": "spam1"}\n{"hello": "world2", "foo": "bar2"}\n' + b'{"foo": "bar1", "egg": "spam1"}\n{"foo": "bar2", "hello": "world2"}\n' ), "xml": ( b'\n\n' - b"bar1spam1\n" - b"world2bar2\nworld3" + b"bar1spam1\n" + b"bar2world2\nworld3" b"spam3\n" ), } @@ -877,8 +741,8 @@ class TestFeedExport(TestFeedExportBase): "json": b'[\n{"foo": "bar1", "egg": "spam1"}\n]', "xml": ( b'\n\n' - b"bar1spam1\n" - b"world2bar2\n" + b"bar1spam1\n" + b"bar2world2\n" ), "jsonlines": b'{"foo": "bar1", "egg": "spam1"}\n', } @@ -1219,7 +1083,9 @@ class TestFeedExport(TestFeedExportBase): assert data["csv"] == b"" @coroutine_test - async def test_multiple_feeds_success_logs_blocking_feed_storage(self): + async def test_multiple_feeds_success_logs_blocking_feed_storage( + self, caplog: pytest.LogCaptureFixture + ): settings = { "FEEDS": { self._random_temp_filename(): {"format": "json"}, @@ -1232,14 +1098,16 @@ class TestFeedExport(TestFeedExportBase): {"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}, ] - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): await self.exported_data(items, settings) for fmt in ["json", "xml", "csv"]: - assert f"Stored {fmt} feed (2 items)" in str(log) + assert f"Stored {fmt} feed (2 items)" in caplog.text @coroutine_test - async def test_multiple_feeds_failing_logs_blocking_feed_storage(self): + async def test_multiple_feeds_failing_logs_blocking_feed_storage( + self, caplog: pytest.LogCaptureFixture + ): settings = { "FEEDS": { self._random_temp_filename(): {"format": "json"}, @@ -1252,11 +1120,11 @@ class TestFeedExport(TestFeedExportBase): {"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}, ] - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): await self.exported_data(items, settings) for fmt in ["json", "xml", "csv"]: - assert f"Error storing {fmt} feed (2 items)" in str(log) + assert f"Error storing {fmt} feed (2 items)" in caplog.text @coroutine_test async def test_extend_kwargs(self): @@ -1422,6 +1290,13 @@ class TestFeedExporterSignals: assert self.feed_exporter_closed_received +class TestItemFilter: + def test_no_feed_options(self): + item_filter = ItemFilter(None) + assert item_filter.item_classes == () + assert item_filter.accepts(MyItem({"foo": "bar"})) + + class TestFeedExportInit: def test_unsupported_storage(self): settings = { @@ -1433,6 +1308,24 @@ class TestFeedExportInit: with pytest.raises(NotConfigured): FeedExporter.from_crawler(crawler) + def test_disabled_storage(self, caplog: pytest.LogCaptureFixture): + class DisabledFeedStorage: + def __init__(self, uri, *, feed_options=None): + raise NotConfigured("not today") + + settings = { + "FEED_STORAGES": {"disabled": DisabledFeedStorage}, + "FEEDS": { + "disabled://uri": {}, + }, + } + crawler = get_crawler(settings_dict=settings) + with caplog.at_level(logging.ERROR), pytest.raises(NotConfigured): + FeedExporter.from_crawler(crawler) + assert ( + "Disabled feed storage scheme: disabled. Reason: not today" in caplog.text + ) + def test_unsupported_format(self): settings = { "FEEDS": { diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py index 3b50cd492..4b0962c43 100644 --- a/tests/test_feedexport_batch.py +++ b/tests/test_feedexport_batch.py @@ -21,7 +21,7 @@ from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler from tests.spiders import ItemSpider -from tests.test_feedexport import TestFeedExportBase +from tests.utils.bases.feedexport import TestFeedExportBase from tests.utils.decorators import coroutine_test, inline_callbacks_test from tests.utils.feedexport import MyItem @@ -210,6 +210,47 @@ class TestBatchDeliveries(TestFeedExportBase): header = MyItem.fields.keys() await self.assertExported(items, header, rows, settings=settings) + @coroutine_test + async def test_batch_delivered_when_full(self): + """Full batches must be finalized and delivered as soon as they are + full, instead of when the spider closes.""" + dir_path = self._random_temp_filename() + batch1_path = Path(dir_path, "1.json") + mockserver_url = self.mockserver.url("/") + batch1_contents: list[bytes | None] = [] + + class TestSpider(scrapy.Spider): + name = "testspider" + start_urls = [mockserver_url] + + def parse(self, response): + yield {"foo": "bar1"} + yield {"foo": "bar2"} + yield scrapy.Request( + mockserver_url, callback=self.parse2, dont_filter=True + ) + + def parse2(self, response): + # the first batch was full after the second item, so it must + # have been delivered by now + batch1_contents.append( + batch1_path.read_bytes() if batch1_path.exists() else None + ) + yield {"foo": "bar3"} + + settings = { + "FEEDS": { + build_url(dir_path / "%(batch_id)d.json"): {"format": "json"}, + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 2, + } + crawler = get_crawler(TestSpider, settings) + await crawler.crawl_async() + + assert batch1_contents, "the second request was not processed" + assert batch1_contents[0] is not None, "batch 1 was not stored during the crawl" + assert json.loads(batch1_contents[0]) == [{"foo": "bar1"}, {"foo": "bar2"}] + def test_wrong_path(self): """If path is without %(batch_time)s and %(batch_id) an exception must be raised""" settings = { diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py index 83d976e94..36d8586ce 100644 --- a/tests/test_feedexport_postprocess.py +++ b/tests/test_feedexport_postprocess.py @@ -12,8 +12,9 @@ from typing import TYPE_CHECKING, Any import pytest +from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.test import get_crawler -from tests.test_feedexport import TestFeedExportBase +from tests.utils.bases.feedexport import TestFeedExportBase from tests.utils.decorators import coroutine_test from tests.utils.feedexport import path_to_url, printf_escape @@ -87,6 +88,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data_stream.seek(0) return data_stream.read() + def test_tell_reports_target_file_position(self): + """Exporters that wrap the file they get, e.g. through + :class:`io.TextIOWrapper`, need it to report a position.""" + file = BytesIO() + manager = PostProcessingManager([self.MyPlugin1], file, {}) + assert manager.tell() == 0 + manager.write(b"foo") + assert manager.tell() == file.tell() == 3 + @coroutine_test async def test_gzip_plugin(self): filename = self._named_tempfile("gzip_file") diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py index bdd0d7614..4d28872b7 100644 --- a/tests/test_feedexport_storages.py +++ b/tests/test_feedexport_storages.py @@ -1,7 +1,9 @@ from __future__ import annotations +import logging import os import string +import sys import tempfile from io import BytesIO from pathlib import Path @@ -10,10 +12,10 @@ from unittest import mock from urllib.parse import quote import pytest -from testfixtures import LogCapture from w3lib.url import path_to_file_uri import scrapy +from scrapy.exceptions import NotConfigured from scrapy.extensions.feedexport import ( BlockingFeedStorage, FileFeedStorage, @@ -166,6 +168,12 @@ class TestFTPFeedStorage: st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {}) assert st.password == string.punctuation + def test_uri_without_hostname(self): + with pytest.raises( + ValueError, match="Got a storage URI without a hostname: ftp:///some_path" + ): + FTPFeedStorage("ftp:///some_path") + class MyBlockingFeedStorage(BlockingFeedStorage): def _store_in_thread(self, file: IO[bytes]) -> None: @@ -205,6 +213,13 @@ class TestBlockingFeedStorage: b.open(spider=spider) +def test_s3_without_boto3(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "boto3", None) + monkeypatch.setitem(sys.modules, "boto3.session", None) + with pytest.raises(NotConfigured, match="missing boto3 library"): + S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") + + @pytest.mark.requires_boto3 class TestS3FeedStorage: def test_parse_credentials(self): @@ -381,6 +396,41 @@ class TestS3FeedStorage: assert storage.region_name == region_name assert storage.s3_client._client_config.region_name == region_name + def test_init_without_max_pool_connections(self) -> None: + storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") + assert storage.max_pool_connections is None + config: Any = storage.s3_client.meta.config + assert config.max_pool_connections == 10 + + def test_init_with_max_pool_connections(self) -> None: + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + max_pool_connections=30, + ) + assert storage.max_pool_connections == 30 + config: Any = storage.s3_client.meta.config + assert config.max_pool_connections == 30 + + @pytest.mark.parametrize( + ("settings", "expected"), + [ + ({}, 10), + ({"REACTOR_THREADPOOL_MAXSIZE": 20}, 20), + ({"AWS_MAX_POOL_CONNECTIONS": 30}, 30), + ({"AWS_MAX_POOL_CONNECTIONS": 30, "REACTOR_THREADPOOL_MAXSIZE": 20}, 30), + ], + ) + def test_from_crawler_max_pool_connections( + self, settings: dict[str, Any], expected: int + ) -> None: + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") + assert storage.max_pool_connections == expected + config: Any = storage.s3_client.meta.config + assert config.max_pool_connections == expected + @coroutine_test async def test_store_without_acl(self): storage = S3FeedStorage( @@ -415,23 +465,21 @@ class TestS3FeedStorage: acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] assert acl == "custom-acl" - def test_overwrite_default(self): - with LogCapture() as log: - S3FeedStorage( - "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" - ) - assert "S3 does not support appending to files" not in str(log) + def test_overwrite_default(self, caplog: pytest.LogCaptureFixture) -> None: + S3FeedStorage( + "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" + ) + assert "S3 does not support appending to files" not in caplog.text - def test_overwrite_false(self): - with LogCapture() as log: - S3FeedStorage( - "s3://mybucket/export.csv", - "access_key", - "secret_key", - "custom-acl", - feed_options={"overwrite": False}, - ) - assert "S3 does not support appending to files" in str(log) + def test_overwrite_false(self, caplog: pytest.LogCaptureFixture) -> None: + S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + "custom-acl", + feed_options={"overwrite": False}, + ) + assert "S3 does not support appending to files" in caplog.text class TestGCSFeedStorage: @@ -505,20 +553,20 @@ class TestGCSFeedStorage: blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) f.close.assert_called_once_with() - def test_overwrite_default(self): - with LogCapture() as log: + def test_overwrite_default(self, caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.DEBUG): GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl") - assert "GCS does not support appending to files" not in str(log) + assert "GCS does not support appending to files" not in caplog.text - def test_overwrite_false(self): - with LogCapture() as log: + def test_overwrite_false(self, caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.DEBUG): GCSFeedStorage( "gs://mybucket/export.csv", "myproject-123", "custom-acl", feed_options={"overwrite": False}, ) - assert "GCS does not support appending to files" in str(log) + assert "GCS does not support appending to files" in caplog.text class TestStdoutFeedStorage: @@ -530,17 +578,18 @@ class TestStdoutFeedStorage: storage.store(file) assert out.getvalue() == b"content" - def test_overwrite_default(self): - with LogCapture() as log: + def test_overwrite_default(self, caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.DEBUG): StdoutFeedStorage("stdout:") assert ( "Standard output (stdout) storage does not support overwriting" - not in str(log) + not in caplog.text ) - def test_overwrite_true(self): - with LogCapture() as log: + def test_overwrite_true(self, caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.DEBUG): StdoutFeedStorage("stdout:", feed_options={"overwrite": True}) - assert "Standard output (stdout) storage does not support overwriting" in str( - log + assert ( + "Standard output (stdout) storage does not support overwriting" + in caplog.text ) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 3c1347fd3..b8586d1ca 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -30,7 +30,7 @@ from scrapy.utils.defer import ( deferred_from_coro, maybe_deferred_to_future, ) -from tests.mockserver.http_resources import LeafResource, Status +from tests.mockserver.http_resources import LeafResource, Status, put_child from tests.mockserver.utils import ssl_context_factory if TYPE_CHECKING: @@ -199,18 +199,18 @@ class TestHttps2ClientProtocol: @pytest.fixture def site(self, tmp_path): r = File(str(tmp_path)) - r.putChild(b"get-data-html-small", GetDataHtmlSmall()) - r.putChild(b"get-data-html-large", GetDataHtmlLarge()) + put_child(r, b"get-data-html-small", GetDataHtmlSmall()) + put_child(r, b"get-data-html-large", GetDataHtmlLarge()) - r.putChild(b"post-data-json-small", PostDataJsonSmall()) - r.putChild(b"post-data-json-large", PostDataJsonLarge()) + put_child(r, b"post-data-json-small", PostDataJsonSmall()) + put_child(r, b"post-data-json-large", PostDataJsonLarge()) - r.putChild(b"dataloss", Dataloss()) - r.putChild(b"no-content-length-header", NoContentLengthHeader()) - r.putChild(b"status", Status()) - r.putChild(b"query-params", QueryParams()) - r.putChild(b"timeout", TimeoutResponse()) - r.putChild(b"request-headers", RequestHeaders()) + put_child(r, b"dataloss", Dataloss()) + put_child(r, b"no-content-length-header", NoContentLengthHeader()) + put_child(r, b"status", Status()) + put_child(r, b"query-params", QueryParams()) + put_child(r, b"timeout", TimeoutResponse()) + put_child(r, b"request-headers", RequestHeaders()) return Site(r, timeout=None) @async_yield_fixture # type: ignore[untyped-decorator] diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index aff3562e3..e7ed17615 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -6,9 +6,6 @@ from scrapy.http import Headers class TestHeaders: - def assertSortedEqual(self, first, second, msg=None): - assert sorted(first) == sorted(second), msg - def test_basics(self): h = Headers({"Content-Type": "text/html", "Content-Length": 1234}) assert h["Content-Type"] @@ -39,7 +36,7 @@ class TestHeaders: assert h["X-Forwarded-For"] == b"ip2" assert h.get("X-Forwarded-For") == b"ip2" assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"] - assert h.getlist("X-Forwarded-For") is not hlist + assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap] def test_multivalue_for_one_header(self): h = Headers((("a", "b"), ("a", "c"))) @@ -49,19 +46,19 @@ class TestHeaders: def test_encode_utf8(self): h = Headers({"key": "\xa3"}, encoding="utf-8") - key, val = dict(h).popitem() + key, val = dict(h.items()).popitem() assert isinstance(key, bytes), key assert isinstance(val[0], bytes), val[0] assert val[0] == b"\xc2\xa3" def test_encode_latin1(self): h = Headers({"key": "\xa3"}, encoding="latin1") - _, val = dict(h).popitem() + _, val = dict(h.items()).popitem() assert val[0] == b"\xa3" def test_encode_multiple(self): h = Headers({"key": ["\xa3"]}, encoding="utf-8") - _, val = dict(h).popitem() + _, val = dict(h.items()).popitem() assert val[0] == b"\xc2\xa3" def test_delete_and_contains(self): @@ -75,7 +72,7 @@ class TestHeaders: h = Headers() hlist = ["ip1", "ip2"] olist = h.setdefault("X-Forwarded-For", hlist) - assert h.getlist("X-Forwarded-For") is not hlist + assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap] assert h.getlist("X-Forwarded-For") is olist h = Headers() @@ -87,16 +84,16 @@ class TestHeaders: idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]} h = Headers(idict) - assert dict(h) == { + assert dict(h.items()) == { b"Content-Type": [b"text/html"], b"X-Forwarded-For": [b"ip1", b"ip2"], } - self.assertSortedEqual(h.keys(), [b"X-Forwarded-For", b"Content-Type"]) - self.assertSortedEqual( - h.items(), - [(b"X-Forwarded-For", [b"ip1", b"ip2"]), (b"Content-Type", [b"text/html"])], - ) - self.assertSortedEqual(h.values(), [b"ip2", b"text/html"]) + assert sorted(h.keys()) == [b"Content-Type", b"X-Forwarded-For"] + assert sorted(h.items()) == [ + (b"Content-Type", [b"text/html"]), + (b"X-Forwarded-For", [b"ip1", b"ip2"]), + ] + assert set(h.values()) == {b"ip2", b"text/html"} def test_update(self): h = Headers() @@ -162,4 +159,4 @@ class TestHeaders: with pytest.raises(TypeError, match="Unsupported value type"): Headers().setdefault("foo", object()) with pytest.raises(TypeError, match="Unsupported value type"): - Headers().setlist("foo", [object()]) + Headers().setlist("foo", [object()]) # type: ignore[list-item] diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 1941b826f..b9cec93a1 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,498 +1,24 @@ -import warnings import xmlrpc.client from typing import Any import pytest -from scrapy.http import Headers, Request, XmlRpcRequest -from scrapy.http.request import NO_CALLBACK +from scrapy import Request +from scrapy.http import XmlRpcRequest from scrapy.utils.python import to_bytes +from tests.utils.bases.http_request import TestRequestBase -class TestRequest: +class TestRequest(TestRequestBase): request_class = Request - default_method = "GET" - default_headers: dict[bytes, list[bytes]] = {} - default_meta: dict[str, Any] = {} - def test_init(self): - # Request requires url in the __init__ method - with pytest.raises(TypeError): - self.request_class() - # url argument must be basestring - with pytest.raises(TypeError): - self.request_class(123) - - # priority argument must be an integer - with pytest.raises(TypeError, match="Request priority not an integer"): - self.request_class("http://www.example.com", priority="1") - - r = self.request_class("http://www.example.com") - assert isinstance(r.url, str) - assert r.url == "http://www.example.com" - assert r.method == self.default_method - - assert isinstance(r.headers, Headers) - assert r.headers == self.default_headers - assert r.meta == self.default_meta - - meta = {"lala": "lolo"} - headers = {b"caca": b"coco"} - r = self.request_class( - "http://www.example.com", meta=meta, headers=headers, body="a body" - ) - - assert r.meta is not meta - assert r.meta == meta - assert r.headers is not headers - assert r.headers[b"caca"] == b"coco" - - def test_url_scheme(self): - # This test passes by not raising any (ValueError) exception - self.request_class("http://example.org") - self.request_class("https://example.org") - self.request_class("s3://example.org") - self.request_class("ftp://example.org") - self.request_class("about:config") - self.request_class("data:,Hello%2C%20World!") - - def test_url_no_scheme(self): - msg = "Missing scheme in request url:" - with pytest.raises(ValueError, match=msg): - self.request_class("foo") - with pytest.raises(ValueError, match=msg): - self.request_class("/foo/") - with pytest.raises(ValueError, match=msg): - self.request_class("/foo:bar") - - def test_headers(self): - # Different ways of setting headers attribute - url = "http://www.scrapy.org" - headers = {b"Accept": "gzip", b"Custom-Header": "nothing to tell you"} - r = self.request_class(url=url, headers=headers) - p = self.request_class(url=url, headers=r.headers) - - assert r.headers == p.headers - assert r.headers is not headers - assert p.headers is not r.headers - - # headers must not be unicode - h = Headers({"key1": "val1", "key2": "val2"}) - h["newkey"] = "newval" - for k, v in h.items(): - assert isinstance(k, bytes) - for s in v: - assert isinstance(s, bytes) - - def test_eq(self): - url = "http://www.scrapy.org" - r1 = self.request_class(url=url) - r2 = self.request_class(url=url) - assert r1 != r2 - - set_ = set() - set_.add(r1) - set_.add(r2) - assert len(set_) == 2 - - def test_url(self): - r = self.request_class(url="http://www.scrapy.org/path") - assert r.url == "http://www.scrapy.org/path" - - def test_url_quoting(self): - r = self.request_class(url="http://www.scrapy.org/blank%20space") - assert r.url == "http://www.scrapy.org/blank%20space" - r = self.request_class(url="http://www.scrapy.org/blank space") - assert r.url == "http://www.scrapy.org/blank%20space" - - def test_url_encoding(self): - r = self.request_class(url="http://www.scrapy.org/price/£") - assert r.url == "http://www.scrapy.org/price/%C2%A3" - - def test_url_encoding_other(self): - # encoding affects only query part of URI, not path - # path part should always be UTF-8 encoded before percent-escaping - r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8") - assert r.url == "http://www.scrapy.org/price/%C2%A3" - - r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1") - assert r.url == "http://www.scrapy.org/price/%C2%A3" - - def test_url_encoding_query(self): - r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ") - assert r1.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" - - # should be same as above - r2 = self.request_class( - url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8" - ) - assert r2.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" - - def test_url_encoding_query_latin1(self): - # encoding is used for encoding query-string before percent-escaping; - # path is still UTF-8 encoded before percent-escaping - r3 = self.request_class( - url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1" - ) - assert r3.url == "http://www.scrapy.org/price/%C2%B5?currency=%A3" - - def test_url_encoding_nonutf8_untouched(self): - # percent-escaping sequences that do not match valid UTF-8 sequences - # should be kept untouched (just upper-cased perhaps) - # - # See https://datatracker.ietf.org/doc/html/rfc3987#section-3.2 - # - # "Conversions from URIs to IRIs MUST NOT use any character encoding - # other than UTF-8 in steps 3 and 4, even if it might be possible to - # guess from the context that another character encoding than UTF-8 was - # used in the URI. For example, the URI - # "http://www.example.org/r%E9sum%E9.html" might with some guessing be - # interpreted to contain two e-acute characters encoded as iso-8859-1. - # It must not be converted to an IRI containing these e-acute - # characters. Otherwise, in the future the IRI will be mapped to - # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different - # URI from "http://www.example.org/r%E9sum%E9.html". - r1 = self.request_class(url="http://www.scrapy.org/price/%a3") - assert r1.url == "http://www.scrapy.org/price/%a3" - - r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") - assert r2.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" - - r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3") - assert r3.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" - - r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") - assert r4.url == "http://www.example.org/r%E9sum%E9.html" - - def test_url_verbatim(self): - r = self.request_class( - url="http://www.scrapy.org/price/£", - meta={"verbatim_url": True}, - ) - assert r.url == "http://www.scrapy.org/price/£" - - r = self.request_class( - url="http://www.scrapy.org/blank space", - meta={"verbatim_url": True}, - ) - assert r.url == "http://www.scrapy.org/blank space" - - def test_body(self): - r1 = self.request_class(url="http://www.example.com/") - assert r1.body == b"" - - r2 = self.request_class(url="http://www.example.com/", body=b"") - assert isinstance(r2.body, bytes) - assert r2.encoding == "utf-8" # default encoding - - r3 = self.request_class( - url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8" - ) - assert isinstance(r3.body, bytes) - assert r3.body == b"Price: \xc2\xa3100" - - r4 = self.request_class( - url="http://www.example.com/", body="Price: \xa3100", encoding="latin1" - ) - assert isinstance(r4.body, bytes) - assert r4.body == b"Price: \xa3100" - - def test_copy(self): - """Test Request copy""" - - def somecallback(): - pass - - r1 = self.request_class( - "http://www.example.com", - flags=["f1", "f2"], - callback=somecallback, - errback=somecallback, - ) - r1.meta["foo"] = "bar" - r1.cb_kwargs["key"] = "value" - r2 = r1.copy() - - # make sure callbaclks are copied - assert r1.callback is somecallback - assert r1.errback is somecallback - assert r2.callback is r1.callback - assert r2.errback is r1.errback - - # make sure flags list is shallow copied - assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" - assert r1.flags == r2.flags - - # make sure cb_kwargs dict is shallow copied - assert r1.cb_kwargs is not r2.cb_kwargs, ( - "cb_kwargs must be a shallow copy, not identical" - ) - assert r1.cb_kwargs == r2.cb_kwargs - - # make sure meta dict is shallow copied - assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" - assert r1.meta == r2.meta - - # make sure headers attribute is shallow copied - assert r1.headers is not r2.headers, ( - "headers must be a shallow copy, not identical" - ) - assert r1.headers == r2.headers - assert r1.encoding == r2.encoding - assert r1.dont_filter == r2.dont_filter - - # Request.body can be identical since it's an immutable object (str) - - def test_copy_inherited_classes(self): - """Test Request children copies preserve their class""" - - class CustomRequest(self.request_class): - pass - - r1 = CustomRequest("http://www.example.com") - r2 = r1.copy() - - assert isinstance(r2, CustomRequest) - - def test_replace(self): - """Test Request.replace() method""" - r1 = self.request_class("http://www.example.com", method="GET") - hdrs = Headers(r1.headers) - hdrs[b"key"] = b"value" - r2 = r1.replace(method="POST", body="New body", headers=hdrs) - assert r1.url == r2.url - assert (r1.method, r2.method) == ("GET", "POST") - assert (r1.body, r2.body) == (b"", b"New body") - assert (r1.headers, r2.headers) == (self.default_headers, hdrs) - - # Empty attributes (which may fail if not compared properly) - r3 = self.request_class( - "http://www.example.com", meta={"a": 1}, dont_filter=True - ) - r4 = r3.replace( - url="http://www.example.com/2", body=b"", meta={}, dont_filter=False - ) - assert r4.url == "http://www.example.com/2" - assert r4.body == b"" - assert r4.meta == {} - assert r4.dont_filter is False - - # the cls argument allows changing the resulting class - custom_request_cls = type("CustomRequest", (self.request_class,), {}) - r5 = r1.replace(cls=custom_request_cls) - assert isinstance(r5, custom_request_cls) - assert r5.url == r1.url - - def test_method_always_str(self): - r = self.request_class("http://www.example.com", method="POST") - assert isinstance(r.method, str) - - def test_immutable_attributes(self): - r = self.request_class("http://example.com") - with pytest.raises(AttributeError): - r.url = "http://example2.com" - with pytest.raises(AttributeError): - r.body = "xxx" - - def test_callback_and_errback(self): - def a_function(): - pass - - r1 = self.request_class("http://example.com") - assert r1.callback is None - assert r1.errback is None - - r2 = self.request_class("http://example.com", callback=a_function) - assert r2.callback is a_function - assert r2.errback is None - - r3 = self.request_class("http://example.com", errback=a_function) - assert r3.callback is None - assert r3.errback is a_function - - r4 = self.request_class( - url="http://example.com", - callback=a_function, - errback=a_function, - ) - assert r4.callback is a_function - assert r4.errback is a_function - - r5 = self.request_class( - url="http://example.com", - callback=NO_CALLBACK, - errback=NO_CALLBACK, - ) - assert r5.callback is NO_CALLBACK - assert r5.errback is NO_CALLBACK - - def test_callback_and_errback_type(self): - with pytest.raises(TypeError): - self.request_class("http://example.com", callback="a_function") - with pytest.raises(TypeError): - self.request_class("http://example.com", errback="a_function") - with pytest.raises(TypeError): - self.request_class( - url="http://example.com", - callback="a_function", - errback="a_function", - ) - - def test_setters(self): - request = self.request_class("http://example.com") - - request.flags = ["f1"] - assert request.flags == ["f1"] - - request.cookies = {"sid": "1"} - assert request.cookies == {"sid": "1"} - - headers = Headers({b"X-Test": b"1"}) - request.headers = headers - assert request._headers is headers - request.headers = {b"A": b"b"} - assert isinstance(request.headers, Headers) - assert request._headers[b"A"] == b"b" - - def test_setter_mutable_lazy_loading(self): - """Mutable attributes are set internally to None only until they are - read, then they always return the same falsy instance of the - corresponding mutable structure. - - Setting them to None causes the next read to return a different object. - """ - - request = self.request_class("http://example.com") - - assert request._flags is None - assert request.flags == [] - assert request.flags is request.flags - assert request._flags == [] - original_flags = request.flags - request.flags = None - assert request._flags is None - assert request.flags == [] - assert request.flags is not original_flags - - assert request._cookies is None - assert request.cookies == {} - assert request.cookies is request.cookies - assert request._cookies == {} - original_cookies = request.cookies - request.cookies = None - assert request._cookies is None - assert request.cookies == {} - assert request.cookies is not original_cookies - - if self.default_headers: - assert request._headers == self.default_headers - assert request._headers is not self.default_headers - assert request.headers == self.default_headers - else: - assert request._headers is None - assert request.headers == {} - assert request.headers is request.headers - assert isinstance(request.headers, Headers) - assert isinstance(request._headers, Headers) - original_headers = request.headers - request.headers = None - assert request._headers is None - assert request.headers == {} - assert request._headers == {} - assert request.headers is not original_headers - - def test_no_callback(self): - with pytest.raises(RuntimeError): - NO_CALLBACK() - - def test_from_curl(self): - # Note: more curated tests regarding curl conversion are in - # `test_utils_curl.py` - curl_command = ( - "curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique" - "_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique" - "_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H" - " 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q" - "=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use" - "r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM" - "L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S" - "afari/537.36' -H 'Content-Type: application /x-www-form-urlencode" - "d' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=" - "0.9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0" - "' -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: kee" - "p-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith" - "%40example.org&size=small&topping=cheese&topping=onion&delivery=1" - "2%3A15&comments=' --compressed" - ) - r = self.request_class.from_curl(curl_command) - assert r.method == "POST" - assert r.url == "http://httpbin.org/post" - assert ( - r.body == b"custname=John+Smith&custtel=500&custemail=jsmith%40" - b"example.org&size=small&topping=cheese&topping=onion" - b"&delivery=12%3A15&comments=" - ) - assert r.cookies == { - "_gauges_unique_year": "1", - "_gauges_unique": "1", - "_gauges_unique_month": "1", - "_gauges_unique_hour": "1", - "_gauges_unique_day": "1", - } - assert r.headers == { - b"Origin": [b"http://httpbin.org"], - b"Accept-Encoding": [b"gzip, deflate"], - b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"], - b"Upgrade-Insecure-Requests": [b"1"], - b"User-Agent": [ - b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537." - b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202" - b".75 Chrome/62.0.3202.75 Safari/537.36" - ], - b"Content-Type": [b"application /x-www-form-urlencoded"], - b"Accept": [ - b"text/html,application/xhtml+xml,application/xml;q=0." - b"9,image/webp,image/apng,*/*;q=0.8" - ], - b"Cache-Control": [b"max-age=0"], - b"Referer": [b"http://httpbin.org/forms/post"], - b"Connection": [b"keep-alive"], - } - - def test_from_curl_with_kwargs(self): - r = self.request_class.from_curl( - 'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"} - ) - assert r.method == "POST" - assert r.meta == {"key": "value"} - - def test_from_curl_ignore_unknown_options(self): - # By default: it works and ignores the unknown options: --foo and -z - with warnings.catch_warnings(): # avoid warning when executing tests - warnings.filterwarnings( - "ignore", category=UserWarning, message="Unrecognized options:" - ) - r = self.request_class.from_curl( - 'curl -X DELETE "http://example.org" --foo -z', - ) - assert r.method == "DELETE" - - # If `ignore_unknown_options` is set to `False` it raises an error with - # the unknown options: --foo and -z - with pytest.raises(ValueError, match="Unrecognized options:"): - self.request_class.from_curl( - 'curl -X PATCH "http://example.org" --foo -z', - ignore_unknown_options=False, - ) - - -class TestXmlRpcRequest(TestRequest): +class TestXmlRpcRequest(TestRequestBase): request_class = XmlRpcRequest default_method = "POST" default_headers = {b"Content-Type": [b"text/xml"]} - def _test_request(self, **kwargs): + def _test_request(self, **kwargs: Any) -> None: r = self.request_class("http://scrapytest.org/rpc2", **kwargs) assert r.headers[b"Content-Type"] == b"text/xml" assert r.body == to_bytes( diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py index af86b35c0..cb18a0b03 100644 --- a/tests/test_http_request_form.py +++ b/tests/test_http_request_form.py @@ -2,6 +2,7 @@ from __future__ import annotations import re import warnings +from typing import TYPE_CHECKING, Any from urllib.parse import parse_qs, unquote_to_bytes import pytest @@ -10,35 +11,42 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import FormRequest, HtmlResponse from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode -from tests.test_http_request import TestRequest +from tests.utils.bases.http_request import TestRequestBase + +if TYPE_CHECKING: + from scrapy import Request -def _buildresponse(body, **kwargs): +def _buildresponse(body: bytes | str, **kwargs: Any) -> HtmlResponse: kwargs.setdefault("body", body) kwargs.setdefault("url", "http://example.com") kwargs.setdefault("encoding", "utf-8") return HtmlResponse(**kwargs) -def _qs(req, encoding="utf-8", to_unicode=False): - qs = req.body if req.method == "POST" else req.url.partition("?")[2] - uqs = unquote_to_bytes(qs) - if to_unicode: - uqs = uqs.decode(encoding) - return parse_qs(uqs, True) +def _query_string(req: Request) -> bytes: + return req.body if req.method == "POST" else req.url.partition("?")[2].encode() + + +def _qs(req: Request) -> dict[bytes, list[bytes]]: + return parse_qs(unquote_to_bytes(_query_string(req)), True) + + +def _qs_unicode(req: Request, encoding: str = "utf-8") -> dict[str, list[str]]: + qs = unquote_to_bytes(_query_string(req)).decode(encoding) + return parse_qs(qs, True) + + +def _assert_query_equal(first: bytes, second: bytes) -> None: + assert sorted(to_unicode(first).split("&")) == sorted(to_unicode(second).split("&")) # FormRequest.from_response() is deprecated in favor of form2request, so the # many tests below that exercise it ignore the resulting deprecation warning. @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestFormRequest(TestRequest): +class TestFormRequest(TestRequestBase): request_class = FormRequest - def assertQueryEqual(self, first, second, msg=None): - first = to_unicode(first).split("&") - second = to_unicode(second).split("&") - assert sorted(first) == sorted(second), msg - def test_init_not_deprecated(self): # Building a request directly from form data is not deprecated. with warnings.catch_warnings(): @@ -75,20 +83,22 @@ class TestFormRequest(TestRequest): assert fs[b"b"] == [b"2"] assert fs.get(b"c") is None - data = {"a": "1", "b": "2"} + mapping = {"a": "1", "b": "2"} fs = _qs( - self.request_class("http://www.example.com/", method="GET", formdata=data) + self.request_class( + "http://www.example.com/", method="GET", formdata=mapping + ) ) assert fs[b"a"] == [b"1"] assert fs[b"b"] == [b"2"] def test_default_encoding_bytes(self): # using default encoding (utf-8) - data = {b"one": b"two", b"price": b"\xc2\xa3 100"} + data: dict[Any, Any] = {b"one": b"two", b"price": b"\xc2\xa3 100"} r2 = self.request_class("http://www.example.com", formdata=data) assert r2.method == "POST" assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two") + _assert_query_equal(r2.body, b"price=%C2%A3+100&one=two") assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_default_encoding_textual_data(self): @@ -97,26 +107,26 @@ class TestFormRequest(TestRequest): r2 = self.request_class("http://www.example.com", formdata=data) assert r2.method == "POST" assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") + _assert_query_equal(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_default_encoding_mixed_data(self): # using default encoding (utf-8) - data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} + data: dict[Any, Any] = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} r2 = self.request_class("http://www.example.com", formdata=data) assert r2.method == "POST" assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") + _assert_query_equal(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_custom_encoding_bytes(self): - data = {b"\xb5 one": b"two", b"price": b"\xa3 100"} + data: dict[Any, Any] = {b"\xb5 one": b"two", b"price": b"\xa3 100"} r2 = self.request_class( "http://www.example.com", formdata=data, encoding="latin1" ) assert r2.method == "POST" assert r2.encoding == "latin1" - self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two") + _assert_query_equal(r2.body, b"price=%A3+100&%B5+one=two") assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_custom_encoding_textual_data(self): @@ -131,7 +141,7 @@ class TestFormRequest(TestRequest): # using multiples values for a single key data = {"price": "\xa3 100", "colours": ["red", "blue", "green"]} r3 = self.request_class("http://www.example.com", formdata=data) - self.assertQueryEqual( + _assert_query_equal( r3.body, b"colours=red&colours=blue&colours=green&price=%C2%A3+100" ) @@ -173,7 +183,7 @@ class TestFormRequest(TestRequest): assert req.method == "POST" assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True) + fs = _qs_unicode(req) assert set(fs["test £"]) == {"val1", "val2"} assert set(fs["one"]) == {"two", "three"} assert fs["test2"] == ["xxx µ"] @@ -196,7 +206,7 @@ class TestFormRequest(TestRequest): assert req.method == "POST" assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True, encoding="latin1") + fs = _qs_unicode(req, encoding="latin1") assert set(fs["test £"]) == {"val1", "val2"} assert set(fs["one"]) == {"two", "three"} assert fs["test2"] == ["xxx µ"] @@ -218,7 +228,7 @@ class TestFormRequest(TestRequest): assert req.method == "POST" assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True) + fs = _qs_unicode(req) assert set(fs["test £"]) == {"val1", "val2"} assert set(fs["one"]) == {"two", "three"} assert fs["test2"] == ["xxx µ"] @@ -305,7 +315,10 @@ class TestFormRequest(TestRequest): """ ) - req = self.request_class.from_response(response, formdata={"two": None}) + req = self.request_class.from_response( + response, + formdata={"two": None}, # type: ignore[arg-type] + ) fs = _qs(req) assert fs[b"one"] == [b"1"] assert b"two" not in fs @@ -450,7 +463,7 @@ class TestFormRequest(TestRequest): req = self.request_class.from_response( response, clickdata={"name": "price in \u00a3"} ) - fs = _qs(req, to_unicode=True) + fs = _qs_unicode(req) assert fs["price in \u00a3"] def test_from_response_unicode_clickdata_latin1(self): @@ -466,7 +479,7 @@ class TestFormRequest(TestRequest): req = self.request_class.from_response( response, clickdata={"name": "price in \u00a5"} ) - fs = _qs(req, to_unicode=True, encoding="latin1") + fs = _qs_unicode(req, encoding="latin1") assert fs["price in \u00a5"] def test_from_response_multiple_forms_clickdata(self): @@ -737,7 +750,7 @@ class TestFormRequest(TestRequest): """ ) req = self.request_class.from_response(res) - fs = _qs(req, to_unicode=True) + fs = _qs_unicode(req) assert fs == {"i1": ["i1v2"], "i2": ["i2v1"], "i4": ["i4v2", "i4v3"]} def test_from_response_radio(self): @@ -1022,7 +1035,7 @@ class TestFormRequest(TestRequest): with pytest.raises( ValueError, match="formdata should be a dict or iterable of tuples" ): - FormRequest.from_response(response, formdata=123) + FormRequest.from_response(response, formdata=123) # type: ignore[arg-type] def test_form_response_with_custom_invalid_formdata_value_error(self): """Test that a ValueError is raised for fault-inducing iterable formdata input""" @@ -1037,7 +1050,7 @@ class TestFormRequest(TestRequest): with pytest.raises( ValueError, match="formdata should be a dict or iterable of tuples" ): - FormRequest.from_response(response, formdata=("a",)) + FormRequest.from_response(response, formdata=("a",)) # type: ignore[arg-type] def test_get_form_with_xpath_no_form_parent(self): """Test that _get_from raised a ValueError when an XPath selects an element diff --git a/tests/test_http_request_json.py b/tests/test_http_request_json.py index 65022afe1..1bfbbbdf5 100644 --- a/tests/test_http_request_json.py +++ b/tests/test_http_request_json.py @@ -8,10 +8,10 @@ import pytest from scrapy.http import JsonRequest from scrapy.utils.python import to_bytes -from tests.test_http_request import TestRequest +from tests.utils.bases.http_request import TestRequestBase -class TestJsonRequest(TestRequest): +class TestJsonRequest(TestRequestBase): request_class = JsonRequest default_method = "GET" default_headers = { diff --git a/tests/test_http_response.py b/tests/test_http_response.py index a8ea4920e..369c2495f 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,413 +1,8 @@ from __future__ import annotations -from typing import TYPE_CHECKING - -import pytest -from w3lib.encoding import resolve_encoding - -from scrapy.exceptions import NotSupported -from scrapy.http import Headers, Request, Response -from scrapy.link import Link -from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS -from tests import get_testdata - -if TYPE_CHECKING: - from collections.abc import Iterable +from scrapy.http import Response +from tests.utils.bases.http_response import TestResponseBase -class TestResponse: +class TestResponse(TestResponseBase): response_class = Response - - def test_init(self): - # Response requires url in the constructor - with pytest.raises(TypeError): - self.response_class() - assert isinstance( - self.response_class("http://example.com/"), self.response_class - ) - with pytest.raises(TypeError): - self.response_class(b"http://example.com") - with pytest.raises(TypeError): - self.response_class(url="http://example.com", body={}) - # body can be str or None - assert isinstance( - self.response_class("http://example.com/", body=b""), - self.response_class, - ) - assert isinstance( - self.response_class("http://example.com/", body=b"body"), - self.response_class, - ) - # test presence of all optional parameters - assert isinstance( - self.response_class( - "http://example.com/", body=b"", headers={}, status=200 - ), - self.response_class, - ) - - r = self.response_class("http://www.example.com") - assert isinstance(r.url, str) - assert r.url == "http://www.example.com" - assert r.status == 200 - - assert isinstance(r.headers, Headers) - assert not r.headers - - headers = {"foo": "bar"} - body = b"a body" - r = self.response_class("http://www.example.com", headers=headers, body=body) - - assert r.headers is not headers - assert r.headers[b"foo"] == b"bar" - - r = self.response_class("http://www.example.com", status=301) - assert r.status == 301 - r = self.response_class("http://www.example.com", status="301") - assert r.status == 301 - with pytest.raises(ValueError, match=r"invalid literal for int\(\)"): - self.response_class("http://example.com", status="lala200") - - def test_copy(self): - """Test Response copy""" - - r1 = self.response_class("http://www.example.com", body=b"Some body") - r1.flags.append("cached") - r2 = r1.copy() - - assert r1.status == r2.status - assert r1.body == r2.body - - # make sure flags list is shallow copied - assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" - assert r1.flags == r2.flags - - # make sure headers attribute is shallow copied - assert r1.headers is not r2.headers, ( - "headers must be a shallow copy, not identical" - ) - assert r1.headers == r2.headers - - def test_copy_meta(self): - req = Request("http://www.example.com") - req.meta["foo"] = "bar" - r1 = self.response_class( - "http://www.example.com", body=b"Some body", request=req - ) - assert r1.meta is req.meta - - def test_copy_cb_kwargs(self): - req = Request("http://www.example.com") - req.cb_kwargs["foo"] = "bar" - r1 = self.response_class( - "http://www.example.com", body=b"Some body", request=req - ) - assert r1.cb_kwargs is req.cb_kwargs - - def test_unavailable_meta(self): - r1 = self.response_class("http://www.example.com", body=b"Some body") - with pytest.raises(AttributeError, match=r"Response\.meta not available"): - r1.meta - - def test_unavailable_cb_kwargs(self): - r1 = self.response_class("http://www.example.com", body=b"Some body") - with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"): - r1.cb_kwargs - - def test_copy_inherited_classes(self): - """Test Response children copies preserve their class""" - - class CustomResponse(self.response_class): - pass - - r1 = CustomResponse("http://www.example.com") - r2 = r1.copy() - - assert isinstance(r2, CustomResponse) - - def test_replace(self): - """Test Response.replace() method""" - hdrs = Headers({"key": "value"}) - r1 = self.response_class("http://www.example.com") - r2 = r1.replace(status=301, body=b"New body", headers=hdrs) - assert r1.body == b"" - assert r1.url == r2.url - assert (r1.status, r2.status) == (200, 301) - assert (r1.body, r2.body) == (b"", b"New body") - assert (r1.headers, r2.headers) == ({}, hdrs) - - # Empty attributes (which may fail if not compared properly) - r3 = self.response_class("http://www.example.com", flags=["cached"]) - r4 = r3.replace(body=b"", flags=[]) - assert r4.body == b"" - assert not r4.flags - - def _assert_response_values(self, response, encoding, body): - if isinstance(body, str): - body_unicode = body - body_bytes = body.encode(encoding) - else: - body_unicode = body.decode(encoding) - body_bytes = body - - assert isinstance(response.body, bytes) - assert isinstance(response.text, str) - self._assert_response_encoding(response, encoding) - assert response.body == body_bytes - assert response.text == body_unicode - - def _assert_response_encoding(self, response, encoding): - assert response.encoding == resolve_encoding(encoding) - - def test_immutable_attributes(self): - r = self.response_class("http://example.com") - with pytest.raises(AttributeError): - r.url = "http://example2.com" - with pytest.raises(AttributeError): - r.body = "xxx" - - def test_setter_mutable_lazy_loading(self): - """Mutable attributes are set internally to None only until they are - read, then they always return the same falsy instance of the - corresponding mutable structure. - - Setting them to None causes the next read to return a different object. - """ - - response = self.response_class("http://example.com") - - response.request = Request("http://example.com") - - assert response._flags is None - assert response.flags == [] - assert response.flags is response.flags - assert response._flags == [] - original_flags = response.flags - response.flags = None - assert response._flags is None - assert response.flags == [] - assert response.flags is not original_flags - - assert response._headers is None - assert response.headers == {} - assert response.headers is response.headers - assert isinstance(response.headers, Headers) - assert isinstance(response._headers, Headers) - original_headers = response.headers - response.headers = None - assert response._headers is None - assert response.headers == {} - assert response._headers == {} - assert response.headers is not original_headers - - def test_setters(self): - response = self.response_class("http://example.com") - - response.flags = ["f1"] - assert response.flags == ["f1"] - - headers = Headers({b"X-Test": b"1"}) - response.headers = headers - assert response._headers is headers - response.headers = {b"A": b"b"} - assert isinstance(response.headers, Headers) - assert response._headers[b"A"] == b"b" - - def test_urljoin(self): - """Test urljoin shortcut (only for existence, since behavior equals urljoin)""" - joined = self.response_class("http://www.example.com").urljoin("/test") - absolute = "http://www.example.com/test" - assert joined == absolute - - def test_shortcut_attributes(self): - r = self.response_class("http://example.com", body=b"hello") - if self.response_class == Response: - msg = "Response content isn't text" - with pytest.raises(AttributeError, match=msg): - r.text - with pytest.raises(NotSupported, match=msg): - r.css("body") - with pytest.raises(NotSupported, match=msg): - r.xpath("//body") - with pytest.raises(NotSupported, match=msg): - r.jmespath("body") - else: - r.text - r.css("body") - r.xpath("//body") - - # Response.follow - - def test_follow_url_absolute(self): - self._assert_followed_url("http://foo.example.com", "http://foo.example.com") - - def test_follow_url_relative(self): - self._assert_followed_url("foo", "http://example.com/foo") - - def test_follow_link(self): - self._assert_followed_url( - Link("http://example.com/foo"), "http://example.com/foo" - ) - - def test_follow_None_url(self): - r = self.response_class("http://example.com") - with pytest.raises(ValueError, match="url can't be None"): - r.follow(None) - - def test_follow_None_encoding(self): - r = self.response_class("http://example.com") - with pytest.raises(ValueError, match="encoding can't be None"): - r.follow("foo", encoding=None) - - @pytest.mark.xfail( - not W3LIB_STRIPS_URLS, - reason="https://github.com/scrapy/w3lib/pull/207", - strict=True, - ) - def test_follow_whitespace_url(self): - self._assert_followed_url("foo ", "http://example.com/foo") - - @pytest.mark.xfail( - not W3LIB_STRIPS_URLS, - reason="https://github.com/scrapy/w3lib/pull/207", - strict=True, - ) - def test_follow_whitespace_link(self): - self._assert_followed_url( - Link("http://example.com/foo "), "http://example.com/foo" - ) - - def test_follow_flags(self): - res = self.response_class("http://example.com/") - fol = res.follow("http://example.com/", flags=["cached", "allowed"]) - assert fol.flags == ["cached", "allowed"] - - # Response.follow_all - - def test_follow_all_absolute(self): - url_list = [ - "http://example.org", - "http://www.example.org", - "http://example.com", - "http://www.example.com", - ] - self._assert_followed_all_urls(url_list, url_list) - - def test_follow_all_relative(self): - relative = ["foo", "bar", "foo/bar", "bar/foo"] - absolute = [ - "http://example.com/foo", - "http://example.com/bar", - "http://example.com/foo/bar", - "http://example.com/bar/foo", - ] - self._assert_followed_all_urls(relative, absolute) - - def test_follow_all_links(self): - absolute = [ - "http://example.com/foo", - "http://example.com/bar", - "http://example.com/foo/bar", - "http://example.com/bar/foo", - ] - links = map(Link, absolute) - self._assert_followed_all_urls(links, absolute) - - def test_follow_all_empty(self): - r = self.response_class("http://example.com") - assert not list(r.follow_all([])) - - def test_follow_all_invalid(self): - r = self.response_class("http://example.com") - if self.response_class == Response: - with pytest.raises(TypeError): - list(r.follow_all(urls=None)) - with pytest.raises(TypeError): - list(r.follow_all(urls=12345)) - with pytest.raises(ValueError, match="url can't be None"): - list(r.follow_all(urls=[None])) - else: - with pytest.raises( - ValueError, match="Please supply exactly one of the following arguments" - ): - list(r.follow_all(urls=None)) - with pytest.raises(TypeError): - list(r.follow_all(urls=12345)) - with pytest.raises(ValueError, match="url can't be None"): - list(r.follow_all(urls=[None])) - - @pytest.mark.xfail( - not W3LIB_STRIPS_URLS, - reason="https://github.com/scrapy/w3lib/pull/207", - strict=True, - ) - def test_follow_all_whitespace(self): - relative = ["foo ", "bar ", "foo/bar ", "bar/foo "] - absolute = [ - "http://example.com/foo", - "http://example.com/bar", - "http://example.com/foo/bar", - "http://example.com/bar/foo", - ] - self._assert_followed_all_urls(relative, absolute) - - @pytest.mark.xfail( - not W3LIB_STRIPS_URLS, - reason="https://github.com/scrapy/w3lib/pull/207", - strict=True, - ) - def test_follow_all_whitespace_links(self): - absolute = [ - "http://example.com/foo ", - "http://example.com/bar ", - "http://example.com/foo/bar ", - "http://example.com/bar/foo ", - ] - links = [Link(u) for u in absolute] - expected = [u.strip() for u in absolute] - self._assert_followed_all_urls(links, expected) - - def test_follow_all_flags(self): - re = self.response_class("http://www.example.com/") - urls = [ - "http://www.example.com/", - "http://www.example.com/2", - "http://www.example.com/foo", - ] - fol = re.follow_all(urls, flags=["cached", "allowed"]) - for req in fol: - assert req.flags == ["cached", "allowed"] - - def _assert_followed_url( - self, - follow_obj: str | Link, - target_url: str, - response: Response | None = None, - encoding: str | None = None, - ) -> None: - if response is None: - response = self._links_response() - req = response.follow(follow_obj) - assert req.url == target_url - if encoding is not None: - assert req.encoding == encoding - - def _assert_followed_all_urls( - self, - follow_obj: Iterable[str | Link], - target_urls: Iterable[str], - response: Response | None = None, - ) -> None: - if response is None: - response = self._links_response() - followed = response.follow_all(follow_obj) - for req, target in zip(followed, target_urls, strict=True): - assert req.url == target - - def _links_response(self) -> Response: - body = get_testdata("link_extractor", "linkextractor.html") - return self.response_class("http://example.com/index", body=body) - - def _links_response_no_href(self) -> Response: - body = get_testdata("link_extractor", "linkextractor_no_href.html") - return self.response_class("http://example.com/index", body=body) diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py index daf6e66e5..efa63e049 100644 --- a/tests/test_http_response_text.py +++ b/tests/test_http_response_text.py @@ -1,6 +1,7 @@ from __future__ import annotations import codecs +from typing import cast from unittest import mock import pytest @@ -8,12 +9,18 @@ import pytest from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.selector import Selector from scrapy.utils.python import to_unicode -from tests.test_http_response import TestResponse +from tests.utils.bases.http_response import TestResponseBase -class TestTextResponse(TestResponse): +class TestTextResponse(TestResponseBase): response_class = TextResponse + def _links_response(self) -> TextResponse: + return cast("TextResponse", super()._links_response()) + + def _links_response_no_href(self) -> TextResponse: + return cast("TextResponse", super()._links_response_no_href()) + def test_follow_None_encoding(self): # unlike the base Response, TextResponse.follow() falls back to the # response encoding when encoding is None instead of raising @@ -21,7 +28,7 @@ class TestTextResponse(TestResponse): req = r.follow("foo", encoding=None) assert req.encoding == "cp1252" - def test_replace(self): + def test_replace(self) -> None: super().test_replace() r1 = self.response_class( "http://www.example.com", body="hello", encoding="cp852" @@ -344,7 +351,7 @@ class TestTextResponse(TestResponse): def test_follow_selector_list(self): resp = self._links_response() with pytest.raises(ValueError, match="SelectorList"): - resp.follow(resp.css("a")) + resp.follow(resp.css("a")) # type: ignore[arg-type] def test_follow_selector_invalid(self): resp = self._links_response() @@ -616,7 +623,7 @@ class CustomResponse(TextResponse): class TestCustomResponse(TestTextResponse): response_class = CustomResponse - def test_copy(self): + def test_copy(self) -> None: super().test_copy() r1 = self.response_class( url="https://example.org", @@ -632,7 +639,7 @@ class TestCustomResponse(TestTextResponse): assert r1.lost == "lost" assert r2.lost is None - def test_replace(self): + def test_replace(self) -> None: super().test_replace() r1 = self.response_class( url="https://example.org", diff --git a/tests/test_link.py b/tests/test_link.py index 0eeffe12b..40c53edd3 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -1,18 +1,20 @@ +from __future__ import annotations + import pytest from scrapy.link import Link class TestLink: - def _assert_same_links(self, link1, link2): + def _assert_same_links(self, link1: Link, link2: Link) -> None: assert link1 == link2 assert hash(link1) == hash(link2) - def _assert_different_links(self, link1, link2): + def _assert_different_links(self, link1: Link, link2: Link) -> None: assert link1 != link2 assert hash(link1) != hash(link2) - def test_eq_and_hash(self): + def test_eq_and_hash(self) -> None: l1 = Link("http://www.example.com") l2 = Link("http://www.example.com/other") l3 = Link("http://www.example.com") @@ -45,17 +47,17 @@ class TestLink: self._assert_different_links(l7, l9) self._assert_different_links(l7, l10) - def test_repr(self): + def test_repr(self) -> None: l1 = Link( "http://www.example.com", text="test", fragment="something", nofollow=True ) l2 = eval(repr(l1)) self._assert_same_links(l1, l2) - def test_bytes_url(self): + def test_bytes_url(self) -> None: with pytest.raises(TypeError): - Link(b"http://www.example.com/\xc2\xa3") + Link(b"http://www.example.com/\xc2\xa3") # type: ignore[arg-type] - def test_eq_non_link(self): + def test_eq_non_link(self) -> None: url = "http://example.com" assert Link(url) != url diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 360aa613e..fc6ccf981 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -1,7 +1,9 @@ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING import pytest -from testfixtures import LogCapture from twisted.python.failure import Failure from scrapy.exceptions import DropItem @@ -10,9 +12,11 @@ from scrapy.item import Field, Item from scrapy.logformatter import LogFormatter from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from tests.mockserver.http import MockServer from tests.spiders import ItemSpider -from tests.utils.decorators import inline_callbacks_test +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from tests.mockserver.http import MockServer class CustomItem(Item): @@ -254,15 +258,6 @@ class DropSomeItemsPipeline: class TestShowOrSkipMessages: - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - def setup_method(self): self.base_settings = { "LOG_LEVEL": "DEBUG", @@ -271,22 +266,26 @@ class TestShowOrSkipMessages: }, } - @inline_callbacks_test - def test_show_messages(self): + @coroutine_test + async def test_show_messages( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(ItemSpider, self.base_settings) - with LogCapture() as lc: - yield crawler.crawl(mockserver=self.mockserver) - assert "Scraped from <200 http://127.0.0.1:" in str(lc) - assert "Crawled (200) None: settings = self.base_settings.copy() settings["LOG_FORMATTER"] = SkipMessagesLogFormatter crawler = get_crawler(ItemSpider, settings) - with LogCapture() as lc: - yield crawler.crawl(mockserver=self.mockserver) - assert "Scraped from <200 http://127.0.0.1:" not in str(lc) - assert "Crawled (200) None: self.crawler = get_crawler(SimpleSpider) self.spider = self.crawler._create_spider("spidey") + assert self.crawler.stats is not None self.stats = self.crawler.stats self.stats.set_value("response_received_count", 4802) self.stats.set_value("item_scraped_count", 3201) @coroutine_test - async def test_stats_calculations(self): + async def test_stats_calculations(self) -> None: logstats = LogStats.from_crawler(self.crawler) with pytest.raises(AttributeError): @@ -56,7 +59,7 @@ class TestLogStats: assert self.stats.get_value("responses_per_minute") == 172.9 assert self.stats.get_value("items_per_minute") == 116.4 - def test_stats_calculations_no_time(self): + def test_stats_calculations_no_time(self) -> None: """The stat values should be None since the start and finish time are not available. """ @@ -65,7 +68,7 @@ class TestLogStats: assert self.stats.get_value("responses_per_minute") is None assert self.stats.get_value("items_per_minute") is None - def test_stats_calculation_no_elapsed_time(self): + def test_stats_calculation_no_elapsed_time(self) -> None: """The stat values should be None since the elapsed time is 0.""" logstats = LogStats.from_crawler(self.crawler) self.stats.set_value("start_time", datetime.fromtimestamp(1655100172)) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 0376d27c4..0681371ef 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,12 +1,12 @@ from __future__ import annotations +import logging import shutil from pathlib import Path from tempfile import mkdtemp from typing import TYPE_CHECKING, Any import pytest -from testfixtures import LogCapture from w3lib.url import add_or_replace_parameter from scrapy import Spider, signals @@ -14,7 +14,7 @@ from scrapy.utils.misc import load_object from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import SimpleSpider -from tests.utils.decorators import inline_callbacks_test +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from scrapy.crawler import Crawler @@ -58,6 +58,8 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider): class TestFileDownloadCrawl: + mockserver: MockServer + pipeline_class = "scrapy.pipelines.files.FilesPipeline" store_setting_key = "FILES_STORE" media_key = "files" @@ -102,7 +104,7 @@ class TestFileDownloadCrawl: crawler.signals.connect(self._on_item_scraped, signals.item_scraped) return crawler - def _assert_files_downloaded(self, items, logs): + def _assert_files_downloaded(self, items: list[Any], logs: str) -> None: assert len(items) == 1 assert self.media_key in items[0] @@ -125,13 +127,16 @@ class TestFileDownloadCrawl: for i in item[self.media_key]: assert (self.tmpmediastore / i["path"]).exists() - def _assert_files_download_failure(self, crawler, items, code, logs): + def _assert_files_download_failure( + self, crawler: Crawler, items: list[Any], code: int, logs: str + ) -> None: # check that the item does NOT have the "images/files" field populated assert len(items) == 1 assert self.media_key in items[0] assert not items[0][self.media_key] # check that there was 1 successful fetch and 3 other responses with non-200 code + assert crawler.stats assert crawler.stats.get_value("downloader/request_method_count/GET") == 4 assert crawler.stats.get_value("downloader/response_count") == 4 assert crawler.stats.get_value("downloader/response_status_count/200") == 1 @@ -144,62 +149,71 @@ class TestFileDownloadCrawl: # check that no files were written to the media store assert not list(self.tmpmediastore.iterdir()) - @inline_callbacks_test - def test_download_media(self): + @coroutine_test + async def test_download_media(self, caplog: pytest.LogCaptureFixture) -> None: crawler = self._create_crawler(MediaDownloadSpider) - with LogCapture() as log: - yield crawler.crawl( + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( self.mockserver.url("/static/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, ) - self._assert_files_downloaded(self.items, str(log)) + self._assert_files_downloaded(self.items, caplog.text) - @inline_callbacks_test - def test_download_media_wrong_urls(self): + @coroutine_test + async def test_download_media_wrong_urls( + self, caplog: pytest.LogCaptureFixture + ) -> None: crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) - with LogCapture() as log: - yield crawler.crawl( + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( self.mockserver.url("/static/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, ) - self._assert_files_download_failure(crawler, self.items, 404, str(log)) + self._assert_files_download_failure(crawler, self.items, 404, caplog.text) - @inline_callbacks_test - def test_download_media_redirected_default_failure(self): + @coroutine_test + async def test_download_media_redirected_default_failure( + self, caplog: pytest.LogCaptureFixture + ): crawler = self._create_crawler(RedirectedMediaDownloadSpider) - with LogCapture() as log: - yield crawler.crawl( + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( self.mockserver.url("/static/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, mockserver=self.mockserver, ) - self._assert_files_download_failure(crawler, self.items, 302, str(log)) + self._assert_files_download_failure(crawler, self.items, 302, caplog.text) - @inline_callbacks_test - def test_download_media_redirected_allowed(self): + @coroutine_test + async def test_download_media_redirected_allowed( + self, caplog: pytest.LogCaptureFixture + ) -> None: settings = { **self.settings, "MEDIA_ALLOW_REDIRECTS": True, } crawler = self._create_crawler(RedirectedMediaDownloadSpider, settings) - with LogCapture() as log: - yield crawler.crawl( + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( self.mockserver.url("/static/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, mockserver=self.mockserver, ) - self._assert_files_downloaded(self.items, str(log)) + self._assert_files_downloaded(self.items, caplog.text) + assert crawler.stats assert crawler.stats.get_value("downloader/response_status_count/302") == 3 - @inline_callbacks_test - def test_download_media_file_path_error(self): + @coroutine_test + async def test_download_media_file_path_error( + self, caplog: pytest.LogCaptureFixture + ) -> None: cls = load_object(self.pipeline_class) - class ExceptionRaisingMediaPipeline(cls): + class ExceptionRaisingMediaPipeline(cls): # type: ignore[misc,valid-type] def file_path(self, request, response=None, info=None, *, item=None): return 1 / 0 @@ -208,14 +222,14 @@ class TestFileDownloadCrawl: "ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1}, } crawler = self._create_crawler(MediaDownloadSpider, settings) - with LogCapture() as log: - yield crawler.crawl( + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( self.mockserver.url("/static/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, mockserver=self.mockserver, ) - assert "ZeroDivisionError" in str(log) + assert "ZeroDivisionError" in caplog.text pillow_available: bool diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 0d9aac830..4e7fb118b 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -2,6 +2,7 @@ import base64 import dataclasses import logging import random +import re import time from abc import ABC, abstractmethod from datetime import datetime @@ -19,18 +20,20 @@ import attr import pytest from itemadapter import ItemAdapter from twisted.internet.defer import Deferred +from twisted.python.failure import Failure -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.item import Field, Item from scrapy.pipelines.files import ( + FileException, FilesPipeline, FSFilesStore, FTPFilesStore, GCSFilesStore, S3FilesStore, ) -from scrapy.pipelines.media import MediaPipeline +from scrapy.pipelines.media import _MediaRequestFiltered from scrapy.settings import Settings from scrapy.utils.asyncio import call_later from scrapy.utils.defer import maybe_deferred_to_future @@ -39,12 +42,8 @@ from scrapy.utils.test import get_crawler from tests.mockserver.ftp import MockFTPServer from tests.utils.decorators import coroutine_test, inline_callbacks_test -from .test_pipeline_media import _mocked_download_func from .utils.cloud import mock_google_cloud_storage - -# required by persist_file() and stat_file(), but as some stores don't use the argument -# we can pass this singleton to keep type hints correct -DUMMY_SPIDER_INFO = MediaPipeline.SpiderInfo(DefaultSpider()) +from .utils.media_pipelines import DUMMY_SPIDER_INFO, mocked_download_func def get_ftp_content_and_delete( @@ -91,16 +90,19 @@ class DeferredFSFilesStore(FSFilesStore): class TestFilesPipeline: def setup_method(self): self.tempdir = mkdtemp() - settings_dict = {"FILES_STORE": self.tempdir} - crawler = get_crawler(DefaultSpider, settings_dict=settings_dict) - crawler.spider = crawler._create_spider() - crawler.engine = MagicMock(download_async=_mocked_download_func) - self.pipeline = FilesPipeline.from_crawler(crawler) - self.pipeline.open_spider() + self.pipeline = self._create_pipeline(FilesPipeline) def teardown_method(self): rmtree(self.tempdir) + def _create_pipeline(self, pipeline_cls: type[FilesPipeline]) -> FilesPipeline: + crawler = get_crawler(DefaultSpider, {"FILES_STORE": self.tempdir}) + crawler.spider = crawler._create_spider() + crawler.engine = MagicMock(download_async=mocked_download_func) + pipeline = pipeline_cls.from_crawler(crawler) + pipeline.open_spider() + return pipeline + def test_file_path_query_parameters(self): file_path = self.pipeline.file_path @@ -251,6 +253,107 @@ class TestFilesPipeline: assert result["files"][0]["checksum"] != "abc" assert result["files"][0]["status"] == "cached" + @coroutine_test + async def test_file_stat_without_last_modified(self) -> None: + """A stat result without a last modification time forces a download.""" + item_url = "http://example.com/file4.pdf" + item = _create_item_with_files(item_url) + with ( + mock.patch.object(FilesPipeline, "inc_stats", return_value=True), + mock.patch.object( + FSFilesStore, "stat_file", return_value={"checksum": "abc"} + ), + mock.patch.object( + FilesPipeline, + "get_media_requests", + return_value=[_prepare_request_object(item_url)], + ), + ): + result = await self.pipeline.process_item(item) + assert result["files"][0]["checksum"] != "abc" + assert result["files"][0]["status"] == "downloaded" + + @coroutine_test + async def test_file_empty_content(self, caplog: pytest.LogCaptureFixture) -> None: + item_url = "http://example.com/empty.pdf" + item = _create_item_with_files(item_url) + request = Request( + item_url, meta={"response": Response(item_url, status=200, body=b"")} + ) + with ( + caplog.at_level(logging.WARNING), + mock.patch.object( + FilesPipeline, "get_media_requests", return_value=[request] + ), + ): + result = await self.pipeline.process_item(item) + assert result["files"] == [] + assert "File (empty-content): Empty file from" in caplog.text + + @coroutine_test + async def test_file_downloaded_file_exception( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """A FileException from file_downloaded() is logged as a warning and + kept as is.""" + + class FailingFilesPipeline(FilesPipeline): + def file_downloaded(self, response, request, info, *, item=None): + raise FileException("boom") + + item_url = "http://example.com/file5.pdf" + item = _create_item_with_files(item_url) + pipeline = self._create_pipeline(FailingFilesPipeline) + with ( + caplog.at_level(logging.WARNING), + mock.patch.object( + FilesPipeline, + "get_media_requests", + return_value=[_prepare_request_object(item_url)], + ), + ): + result = await pipeline.process_item(item) + assert result["files"] == [] + records = [ + r for r in caplog.records if "Error processing file" in r.getMessage() + ] + assert len(records) == 1 + assert records[0].levelname == "WARNING" + assert "boom" in records[0].getMessage() + + @coroutine_test + async def test_file_downloaded_unknown_error( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Any other exception from file_downloaded() is logged as an error and + reported as a FileException.""" + + class FailingFilesPipeline(FilesPipeline): + def file_downloaded(self, response, request, info, *, item=None): + raise RuntimeError("boom") + + item_url = "http://example.com/file6.pdf" + item = _create_item_with_files(item_url) + pipeline = self._create_pipeline(FailingFilesPipeline) + with ( + caplog.at_level(logging.WARNING), + mock.patch.object( + FilesPipeline, + "get_media_requests", + return_value=[_prepare_request_object(item_url)], + ), + ): + result = await pipeline.process_item(item) + assert result["files"] == [] + records = [ + r for r in caplog.records if "Error processing file" in r.getMessage() + ] + assert len(records) == 1 + assert records[0].levelname == "ERROR" + exc_info = records[0].exc_info + assert exc_info is not None + assert exc_info[0] is RuntimeError + @coroutine_test async def test_async_store(self) -> None: """Test that async persist_file() works and is awaited.""" @@ -290,6 +393,71 @@ class TestFilesPipeline: request = Request("http://example.com") assert file_path(request, item=item) == "full/path-to-store-file" + def test_media_failed_filtered_request( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """A filtered media request (IgnoreRequest) is reported as a + _MediaRequestFiltered exception and logged at the DEBUG level, instead + of as a download error with a traceback.""" + request = Request("http://example.com/file.pdf") + reason = "Filtered offsite request to 'example.com'" + failure = Failure(IgnoreRequest(reason)) + + with ( + caplog.at_level(logging.DEBUG), + pytest.raises(_MediaRequestFiltered, match=re.escape(reason)), + ): + self.pipeline.media_failed(failure, request, self.pipeline.spiderinfo) + + assert len(caplog.records) == 1 + record = caplog.records[0] + assert record.levelname == "DEBUG" + assert record.exc_info is None + assert reason in record.getMessage() + + def test_media_failed_download_error( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """A genuine download error is reported as a FileException and logged as + a warning.""" + request = Request("http://example.com/file.pdf") + failure = Failure(Exception("boom")) + + with caplog.at_level(logging.WARNING), pytest.raises(FileException): + self.pipeline.media_failed(failure, request, self.pipeline.spiderinfo) + + assert len(caplog.records) == 1 + assert caplog.records[0].levelname == "WARNING" + + @coroutine_test + async def test_process_item_filtered_request( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """A filtered (e.g. offsite) media request is processed as a failed + result without being logged as an error with a traceback.""" + item_url = "http://example.com/file.pdf" + item = _create_item_with_files(item_url) + request = Request( + item_url, + meta={ + "response": IgnoreRequest("Filtered offsite request to 'example.com'") + }, + ) + with ( + caplog.at_level(logging.DEBUG), + mock.patch.object( + FilesPipeline, "get_media_requests", return_value=[request] + ), + ): + result = await self.pipeline.process_item(item) + + assert result["files"] == [] + assert not any(r.levelname in ("WARNING", "ERROR") for r in caplog.records) + assert any( + "Filtered offsite request to 'example.com'" in r.getMessage() + for r in caplog.records + ) + @pytest.mark.parametrize( "bad_type", [ @@ -580,9 +748,24 @@ class TestFilesPipelineCustomSettings: request = Request("http://example.com/image01.jpg") assert pipeline.file_path(request) == Path("subdir/image01.jpg") - def test_files_store_constructor_with_pathlike_object(self, tmp_path): - fs_store = FSFilesStore(tmp_path) - assert fs_store.basedir == str(tmp_path) + +class TestFSFilesStore: + def test_constructor_with_pathlike_object(self, tmp_path: Path) -> None: + assert FSFilesStore(tmp_path).basedir == str(tmp_path) + + def test_constructor_with_uri(self, tmp_path: Path) -> None: + assert FSFilesStore(f"file://{tmp_path}").basedir == str(tmp_path) + + def test_stat_file(self, tmp_path: Path) -> None: + store = FSFilesStore(tmp_path) + store.persist_file("full/filename", BytesIO(b"data"), DUMMY_SPIDER_INFO) + stat = store.stat_file("full/filename", DUMMY_SPIDER_INFO) + assert stat["checksum"] == "8d777f385d3dfec8815d20f7496026dc" + assert stat["last_modified"] == pytest.approx(time.time(), abs=60) + + def test_stat_missing_file(self, tmp_path: Path) -> None: + store = FSFilesStore(tmp_path) + assert store.stat_file("full/filename", DUMMY_SPIDER_INFO) == {} @pytest.mark.requires_botocore @@ -627,6 +810,59 @@ class TestS3FilesStore: # The call to read does not happen with Stubber assert buffer.method_calls == [mock.call.seek(0)] + @inline_callbacks_test + def test_persist_without_headers(self): + """Without custom headers only the default ones are sent.""" + bucket = "mybucket" + key = "export.csv" + buffer = mock.MagicMock() + + store = S3FilesStore(f"s3://{bucket}/{key}") + from botocore.stub import Stubber # noqa: PLC0415 + + with Stubber(store.s3_client) as stub: + stub.add_response( + "put_object", + expected_params={ + "ACL": S3FilesStore.POLICY, + "Body": buffer, + "Bucket": bucket, + "CacheControl": S3FilesStore.HEADERS["Cache-Control"], + "Key": key, + "Metadata": {}, + }, + service_response={}, + ) + + yield store.persist_file("", buffer, info=DUMMY_SPIDER_INFO) + + stub.assert_no_pending_responses() + + def test_missing_botocore(self): + with ( + mock.patch( + "scrapy.pipelines.files.is_botocore_available", return_value=False + ), + pytest.raises(NotConfigured, match="missing botocore library"), + ): + S3FilesStore("s3://mybucket/key") + + def test_wrong_uri_scheme(self): + with pytest.raises( + ValueError, + match=re.escape( + "Incorrect URI scheme in ftp://mybucket/key, expected 's3'" + ), + ): + S3FilesStore("ftp://mybucket/key") + + def test_unsupported_header(self): + store = S3FilesStore("s3://mybucket/key") + with pytest.raises( + TypeError, match='Header "X-Custom" is not supported by botocore' + ): + store._headers_to_botocore_kwargs({"X-Custom": "value"}) + @inline_callbacks_test def test_stat(self): bucket = "mybucket" @@ -659,6 +895,33 @@ class TestS3FilesStore: stub.assert_no_pending_responses() + def test_default_max_pool_connections(self) -> None: + store = S3FilesStore("s3://mybucket/prefix/") + config: Any = store.s3_client.meta.config + assert config.max_pool_connections == 10 + + @pytest.mark.parametrize( + ("settings", "expected"), + [ + ({}, 10), + ({"REACTOR_THREADPOOL_MAXSIZE": 20}, 20), + ({"AWS_MAX_POOL_CONNECTIONS": 30}, 30), + ({"AWS_MAX_POOL_CONNECTIONS": 30, "REACTOR_THREADPOOL_MAXSIZE": 20}, 30), + ], + ) + def test_max_pool_connections( + self, monkeypatch: pytest.MonkeyPatch, settings: dict[str, Any], expected: int + ) -> None: + # restores the value that FilesPipeline.from_crawler() sets on the class + monkeypatch.setattr(S3FilesStore, "AWS_MAX_POOL_CONNECTIONS", None) + crawler = get_crawler( + settings_dict={"FILES_STORE": "s3://mybucket/prefix/", **settings} + ) + store = FilesPipeline.from_crawler(crawler).store + assert isinstance(store, S3FilesStore) + config: Any = store.s3_client.meta.config + assert config.max_pool_connections == expected + class TestGCSFilesStore: @staticmethod @@ -833,6 +1096,28 @@ class TestFTPFileStore: ) assert data == content + @inline_callbacks_test + def test_persist_active_mode(self, monkeypatch: pytest.MonkeyPatch): + data = b"active mode" + path = "full/filename" + monkeypatch.setattr(FTPFilesStore, "FTP_USERNAME", "anonymous") + monkeypatch.setattr(FTPFilesStore, "FTP_PASSWORD", "guest") + monkeypatch.setattr(FTPFilesStore, "USE_ACTIVE_MODE", True) + with MockFTPServer() as ftp_server: + store = FTPFilesStore(ftp_server.url("/")) + yield store.persist_file(path, BytesIO(data), info=DUMMY_SPIDER_INFO) + stat = yield store.stat_file(path, info=DUMMY_SPIDER_INFO) + assert stat["checksum"] == "ff1575649a39a27c13faa0d37c84bab3" + + def test_wrong_uri_scheme(self): + with pytest.raises( + ValueError, + match=re.escape( + "Incorrect URI scheme in http://example.com/, expected 'ftp'" + ), + ): + FTPFilesStore("http://example.com/") + class ItemWithFiles(Item): file_urls = Field() diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 1b73dd157..19e61579f 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -3,20 +3,26 @@ from __future__ import annotations import dataclasses import io import random +import sys from abc import ABC, abstractmethod +from pathlib import Path from shutil import rmtree from tempfile import mkdtemp +from types import SimpleNamespace from typing import Any import attr import pytest from itemadapter import ItemAdapter +from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.item import Field, Item -from scrapy.pipelines.files import GCSFilesStore, S3FilesStore +from scrapy.pipelines.files import GCSFilesStore, S3FilesStore, _md5sum from scrapy.pipelines.images import ImageException, ImagesPipeline from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test +from tests.utils.media_pipelines import DUMMY_SPIDER_INFO try: from PIL import Image @@ -40,6 +46,11 @@ class TestImagesPipeline: def teardown_method(self): rmtree(self.tempdir) + def test_missing_pillow(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "PIL", None) + with pytest.raises(NotConfigured, match="requires installing Pillow"): + ImagesPipeline(self.tempdir, crawler=get_crawler()) + def test_file_path(self): file_path = self.pipeline.file_path assert ( @@ -197,6 +208,25 @@ class TestImagesPipeline: assert path == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" assert new_im.getpixel((0, 0)) == (255, 0, 0) + @coroutine_test + async def test_image_downloaded(self) -> None: + """The image and its thumbnails are stored, and the checksum of the + full-size image is returned.""" + self.pipeline.thumbs = {"small": (20, 20)} + _, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0)) + url = "https://dev.mydeco.com/mydeco.gif" + response = Response(url=url, body=buf.getvalue()) + + checksum = await self.pipeline.image_downloaded( + response, Request(url=url), DUMMY_SPIDER_INFO + ) + + buf.seek(0) + assert checksum == _md5sum(buf) + name = "3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" + assert Path(self.tempdir, "full", name).read_bytes() == buf.getvalue() + assert Path(self.tempdir, "thumbs", "small", name).exists() + def test_convert_image(self): SIZE = (100, 100) # straight forward case: RGB and JPEG @@ -230,6 +260,24 @@ class TestImagesPipeline: assert converted.mode == "RGB" assert converted.getcolors() == [(10000, (205, 230, 255))] + def test_convert_image_legacy_resampling_filter( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Pillow older than 9.1.0 has Image.ANTIALIAS instead of + Image.Resampling.LANCZOS.""" + # Image.LANCZOS is the only spelling that exists in every supported + # Pillow version, but Pillow defines it dynamically, hence the ignore. + monkeypatch.setattr( + self.pipeline, + "_Image", + SimpleNamespace(ANTIALIAS=Image.LANCZOS), # type: ignore[attr-defined] + ) + im, buf = _create_image("JPEG", "RGB", (100, 100), (0, 127, 255)) + + thumbnail, _ = self.pipeline.convert_image(im, size=(10, 25), response_body=buf) + + assert thumbnail.size == (10, 10) + @pytest.mark.parametrize( "bad_type", [ @@ -581,7 +629,7 @@ class TestImagesPipelineCustomSettings: GCSFilesStore.POLICY = old_policy -def _create_image(format_, *a, **kw): +def _create_image(format_: str, *a: Any, **kw: Any) -> tuple[Image.Image, io.BytesIO]: buf = io.BytesIO() Image.new(*a, **kw).save(buf, format_) buf.seek(0) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 6862c34eb..ba1c18006 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,33 +1,23 @@ from __future__ import annotations +import logging from unittest.mock import MagicMock import pytest -from testfixtures import LogCapture from twisted.python.failure import Failure from scrapy import signals from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response -from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import FileException -from scrapy.pipelines.media import MediaPipeline +from scrapy.pipelines.media import MediaPipeline, _MediaRequestFiltered from scrapy.utils.defer import _defer_sleep_async from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.utils.decorators import coroutine_test - - -async def _mocked_download_func(request): - assert request.callback is NO_CALLBACK - response = request.meta.get("response") - if callable(response): - response = await response() - if isinstance(response, Exception): - raise response - return response +from tests.utils.media_pipelines import mocked_download_func class UserDefinedPipeline(MediaPipeline): @@ -54,7 +44,7 @@ class TestBaseMediaPipeline: def setup_method(self): crawler = get_crawler(DefaultSpider, self.settings) crawler.spider = crawler._create_spider() - crawler.engine = MagicMock(download_async=_mocked_download_func) + crawler.engine = MagicMock(download_async=mocked_download_func) self.pipe = self.pipeline_class.from_crawler(crawler) self.pipe.open_spider() self.info = self.pipe.spiderinfo @@ -138,7 +128,7 @@ class TestBaseMediaPipeline: context = getattr(info.downloaded[fp].value, "__context__", None) assert context is None - def test_default_item_completed(self): + def test_default_item_completed(self, caplog: pytest.LogCaptureFixture) -> None: item = {"name": "name"} assert self.pipe.item_completed([], item, self.info) is item @@ -146,21 +136,35 @@ class TestBaseMediaPipeline: fail = Failure(Exception()) results = [(True, 1), (False, fail)] - with LogCapture() as log: - new_item = self.pipe.item_completed(results, item, self.info) - + caplog.clear() + new_item = self.pipe.item_completed(results, item, self.info) assert new_item is item - assert len(log.records) == 1 - record = log.records[0] + assert len(caplog.records) == 1 + record = caplog.records[0] assert record.levelname == "ERROR" assert record.exc_info == failure_to_exc_info(fail) # disable failure logging and check again + caplog.clear() self.pipe.LOG_FAILED_RESULTS = False - with LogCapture() as log: - new_item = self.pipe.item_completed(results, item, self.info) + new_item = self.pipe.item_completed(results, item, self.info) assert new_item is item - assert len(log.records) == 0 + assert len(caplog.records) == 0 + + def test_item_completed_filtered_request_not_logged( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Filtered media requests (e.g. offsite ones) are not logged as errors + by item_completed(), as they are not download errors.""" + item = {"name": "name"} + fail = Failure(_MediaRequestFiltered("Filtered offsite request")) + results = [(True, 1), (False, fail)] + + with caplog.at_level(logging.DEBUG): + new_item = self.pipe.item_completed(results, item, self.info) + + assert new_item is item + assert len(caplog.records) == 0 @coroutine_test async def test_default_process_item(self): @@ -315,6 +319,42 @@ class TestMediaPipeline(TestBaseMediaPipeline): assert self.fingerprint(req1) == self.fingerprint(req2) assert new_item["results"] == [(True, {})] + @coroutine_test + async def test_failures_are_cached_across_multiple_items(self): + self.pipe.LOG_FAILED_RESULTS = False + exc = Exception("foo") + req1 = Request("http://url1", meta={"response": exc}) + new_item = await self.pipe.process_item({"requests": req1}) + assert new_item["results"][0][1].value is exc + + # rsp2 is ignored, the cached failure must be reused because request + # fingerprints are the same + req2 = Request( + req1.url, meta={"response": Response("http://donot.download.me")} + ) + new_item = await self.pipe.process_item({"requests": req2}) + assert new_item["results"][0][0] is False + assert new_item["results"][0][1].value is exc + assert self.pipe._mockcalled.count("media_to_download") == 1 + + @coroutine_test + async def test_cached_failure_calls_errback(self): + """The errback of a request is called for a cached failure as well.""" + self.pipe.LOG_FAILED_RESULTS = False + exc = Exception("foo") + await self.pipe.process_item( + {"requests": Request("http://url1", meta={"response": exc})} + ) + + def errback(failure): + self.pipe._mockcalled.append("request_errback") + return {"recovered": failure.value} + + req = Request("http://url1", errback=errback) + new_item = await self.pipe.process_item({"requests": req}) + assert new_item["results"] == [(True, {"recovered": exc})] + assert self.pipe._mockcalled.count("request_errback") == 1 + @coroutine_test async def test_results_are_cached_for_requests_of_single_item(self): rsp1 = Response("http://url1") @@ -468,6 +508,30 @@ class TestBuildFromCrawler: assert pipe._from_crawler_called +class MediaFailedNonePipeline(MockedMediaPipeline): + def media_failed(self, failure, request, info): + self._mockcalled.append("media_failed") + + +class TestMediaFailedNone(TestBaseMediaPipeline): + """Test what happens when media_failed() neither raises an exception nor + returns a failure.""" + + pipeline_class = MediaFailedNonePipeline + + @coroutine_test + async def test_result_none(self): + req = Request("http://url1", meta={"response": Exception("foo")}) + new_item = await self.pipe.process_item({"requests": req}) + assert new_item["results"] == [(True, None)] + assert self.pipe._mockcalled == [ + "get_media_requests", + "media_to_download", + "media_failed", + "item_completed", + ] + + class MediaFailedFailurePipeline(MockedMediaPipeline): def media_failed(self, failure, request, info): self._mockcalled.append("media_failed") diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 6c6a6584a..85fefd172 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -11,7 +11,7 @@ from scrapy.spiders import Spider from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueue from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.test import get_crawler -from tests.test_scheduler import MockDownloader +from tests.utils.downloader import MockDownloader class TestPriorityQueue: diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 38d56e9bd..a2a4e8fdb 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -1,11 +1,17 @@ -from testfixtures import LogCapture +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING from scrapy import Request, signals from scrapy.http.response import Response from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import SingleRequestSpider -from tests.utils.decorators import inline_callbacks_test +from tests.utils.decorators import coroutine_test, inline_callbacks_test + +if TYPE_CHECKING: + import pytest OVERRIDDEN_URL = "https://example.org" @@ -63,6 +69,8 @@ class AlternativeCallbacksMiddleware: class TestCrawl: + mockserver: MockServer + @classmethod def setup_class(cls): cls.mockserver = MockServer() @@ -77,6 +85,7 @@ class TestCrawl: url = self.mockserver.url("/status?n=200") crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) response = crawler.spider.meta["responses"][0] assert response.request.url == url @@ -86,6 +95,7 @@ class TestCrawl: url = self.mockserver.url(f"/status?n={status}") crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) failure = crawler.spider.meta["failure"] response = failure.value.response assert failure.request.url == url @@ -103,12 +113,15 @@ class TestCrawl: }, ) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) failure = crawler.spider.meta["failure"] assert failure.request.url == url assert isinstance(failure.value, ZeroDivisionError) - @inline_callbacks_test - def test_downloader_middleware_override_request_in_process_response(self): + @coroutine_test + async def test_downloader_middleware_override_request_in_process_response( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ Downloader middleware which returns a response with an specific 'request' attribute. @@ -133,22 +146,21 @@ class TestCrawl: ) crawler.signals.connect(signal_handler, signal=signals.response_received) - with LogCapture() as log: - yield crawler.crawl(seed=url, mockserver=self.mockserver) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) response = crawler.spider.meta["responses"][0] assert response.request.url == OVERRIDDEN_URL assert signal_params["response"].url == url assert signal_params["request"].url == OVERRIDDEN_URL - log.check_present( - ( - "scrapy.core.engine", - "DEBUG", - f"Crawled (200) (referer: None)", - ), - ) + assert ( + "scrapy.core.engine", + logging.DEBUG, + f"Crawled (200) (referer: None)", + ) in caplog.record_tuples @inline_callbacks_test def test_downloader_middleware_override_in_process_exception(self): @@ -169,6 +181,7 @@ class TestCrawl: }, ) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) response = crawler.spider.meta["responses"][0] assert response.body == b"Caught ZeroDivisionError" assert response.request.url == OVERRIDDEN_URL @@ -192,12 +205,15 @@ class TestCrawl: }, ) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) response = crawler.spider.meta["responses"][0] assert response.body == b"Caught ZeroDivisionError" assert response.request.url == url - @inline_callbacks_test - def test_downloader_middleware_alternative_callback(self): + @coroutine_test + async def test_downloader_middleware_alternative_callback( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ Downloader middleware which returns a response with a specific 'request' attribute, with an alternative callback @@ -211,14 +227,11 @@ class TestCrawl: }, ) - with LogCapture() as log: - url = self.mockserver.url("/status?n=200") - yield crawler.crawl(seed=url, mockserver=self.mockserver) - - log.check_present( - ( - "alternative_callbacks_spider", - "INFO", - "alt_callback was invoked with foo=bar", - ), - ) + url = self.mockserver.url("/status?n=200") + with caplog.at_level(logging.INFO): + await crawler.crawl_async(seed=url, mockserver=self.mockserver) + assert ( + "alternative_callbacks_spider", + logging.INFO, + "alt_callback was invoked with foo=bar", + ) in caplog.record_tuples diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index ee9d2ed51..6c26aa878 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -1,10 +1,17 @@ -from testfixtures import LogCapture +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING from scrapy.http import Request from scrapy.utils.test import get_crawler -from tests.mockserver.http import MockServer from tests.spiders import MockServerSpider -from tests.utils.decorators import inline_callbacks_test +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + import pytest + + from tests.mockserver.http import MockServer class InjectArgumentsDownloaderMiddleware: @@ -42,6 +49,7 @@ class InjectArgumentsSpiderMiddleware: async for element in result: if ( isinstance(element, Request) + and element.callback and element.callback.__name__ == "parse_spider_mw_2" ): element.cb_kwargs["from_process_spider_output"] = True @@ -61,7 +69,12 @@ class KeywordArgumentsSpider(MockServerSpider): checks: list[bool] = [] + def _inc_checks(self, count: int = 1) -> None: + assert self.crawler.stats + self.crawler.stats.inc_value("boolean_checks", count) + async def start(self): + assert self.mockserver data = {"key": "value", "number": 123, "callback": "some_callback"} yield Request(self.mockserver.url("/first"), self.parse_first, cb_kwargs=data) yield Request( @@ -82,9 +95,10 @@ class KeywordArgumentsSpider(MockServerSpider): yield Request(self.mockserver.url("/spider_mw"), self.parse_spider_mw) def parse_first(self, response, key, number): + assert self.mockserver self.checks.append(key == "value") self.checks.append(number == 123) - self.crawler.stats.inc_value("boolean_checks", 2) + self._inc_checks(2) yield response.follow( self.mockserver.url("/two"), self.parse_second, @@ -93,28 +107,28 @@ class KeywordArgumentsSpider(MockServerSpider): def parse_second(self, response, new_key): self.checks.append(new_key == "new_value") - self.crawler.stats.inc_value("boolean_checks") + self._inc_checks() def parse_general(self, response, **kwargs): if response.url.endswith("/general_with"): self.checks.append(kwargs["key"] == "value") self.checks.append(kwargs["number"] == 123) self.checks.append(kwargs["callback"] == "some_callback") - self.crawler.stats.inc_value("boolean_checks", 3) + self._inc_checks(3) elif response.url.endswith("/general_without"): self.checks.append(kwargs == {}) - self.crawler.stats.inc_value("boolean_checks") + self._inc_checks() def parse_no_kwargs(self, response): self.checks.append(response.url.endswith("/no_kwargs")) - self.crawler.stats.inc_value("boolean_checks") + self._inc_checks() def parse_default(self, response, key, number=None, default=99): self.checks.append(response.url.endswith("/default")) self.checks.append(key == "value") self.checks.append(number == 123) self.checks.append(default == 99) - self.crawler.stats.inc_value("boolean_checks", 4) + self._inc_checks(4) def parse_takes_less(self, response, key, callback): """ @@ -133,47 +147,47 @@ class KeywordArgumentsSpider(MockServerSpider): ): self.checks.append(bool(from_process_request)) self.checks.append(bool(from_process_response)) - self.crawler.stats.inc_value("boolean_checks", 2) + self._inc_checks(2) def parse_spider_mw(self, response, from_process_spider_input, from_process_start): + assert self.mockserver self.checks.append(bool(from_process_spider_input)) self.checks.append(bool(from_process_start)) - self.crawler.stats.inc_value("boolean_checks", 2) + self._inc_checks(2) return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2) def parse_spider_mw_2(self, response, from_process_spider_output): self.checks.append(bool(from_process_spider_output)) - self.crawler.stats.inc_value("boolean_checks", 1) + self._inc_checks() class TestCallbackKeywordArguments: - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - - @inline_callbacks_test - def test_callback_kwargs(self): + @coroutine_test + async def test_callback_kwargs( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(KeywordArgumentsSpider) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.ERROR): + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, KeywordArgumentsSpider) assert all(crawler.spider.checks) + assert crawler.stats assert len(crawler.spider.checks) == crawler.stats.get_value("boolean_checks") # check exceptions for argument mismatch exceptions = {} - for line in log.records: + for line in caplog.records: for key in ("takes_less", "takes_more"): if key in line.getMessage(): exceptions[key] = line - assert exceptions["takes_less"].exc_info[0] is TypeError - assert str(exceptions["takes_less"].exc_info[1]).endswith( + takes_less_exc_info = exceptions["takes_less"].exc_info + assert takes_less_exc_info is not None + assert takes_less_exc_info[0] is TypeError + assert str(takes_less_exc_info[1]).endswith( "parse_takes_less() got an unexpected keyword argument 'number'" - ), "Exception message: " + str(exceptions["takes_less"].exc_info[1]) - assert exceptions["takes_more"].exc_info[0] is TypeError - assert str(exceptions["takes_more"].exc_info[1]).endswith( + ) + takes_more_exc_info = exceptions["takes_more"].exc_info + assert takes_more_exc_info is not None + assert takes_more_exc_info[0] is TypeError + assert str(takes_more_exc_info[1]).endswith( "parse_takes_more() missing 1 required positional argument: 'other'" - ), "Exception message: " + str(exceptions["takes_more"].exc_info[1]) + ) diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 78ff18b15..c7596e45d 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -1,7 +1,10 @@ +from typing import Any + import pytest +from twisted.python.failure import Failure from scrapy import Request, Spider -from scrapy.http import JsonRequest +from scrapy.http import JsonRequest, Response from scrapy.utils.request import request_from_dict @@ -10,7 +13,7 @@ class CustomRequest(Request): class TestRequestSerialization: - def setup_method(self): + def setup_method(self) -> None: self.spider = MethodsSpider() def test_basic(self): @@ -42,12 +45,14 @@ class TestRequestSerialization: r = Request("http://www.example.com", body=b"\xc2\xa3") self._assert_serializes_ok(r) - def _assert_serializes_ok(self, request, spider=None): + def _assert_serializes_ok( + self, request: Request, spider: Spider | None = None + ) -> None: d = request.to_dict(spider=spider) request2 = request_from_dict(d, spider=spider) self._assert_same_request(request, request2) - def _assert_same_request(self, r1, r2): + def _assert_same_request(self, r1: Request, r2: Request) -> None: assert r1.__class__ == r2.__class__ assert r1.url == r2.url assert r1.callback == r2.callback @@ -64,6 +69,7 @@ class TestRequestSerialization: assert r1.dont_filter == r2.dont_filter assert r1.flags == r2.flags if isinstance(r1, JsonRequest): + assert isinstance(r2, JsonRequest) assert r1.dumps_kwargs == r2.dumps_kwargs def test_request_class(self): @@ -83,8 +89,8 @@ class TestRequestSerialization: def test_reference_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider.parse_item_reference, - errback=self.spider.handle_error_reference, + callback=self.spider.parse_item_reference, # type: ignore[arg-type,misc] + errback=self.spider.handle_error_reference, # type: ignore[arg-type,misc] ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) @@ -94,8 +100,8 @@ class TestRequestSerialization: def test_private_reference_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._MethodsSpider__parse_item_reference, - errback=self.spider._MethodsSpider__handle_error_reference, + callback=self.spider._MethodsSpider__parse_item_reference, # type: ignore[attr-defined] + errback=self.spider._MethodsSpider__handle_error_reference, # type: ignore[attr-defined] ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) @@ -105,7 +111,7 @@ class TestRequestSerialization: def test_private_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._MethodsSpider__parse_item_private, + callback=self.spider._MethodsSpider__parse_item_private, # type: ignore[attr-defined] errback=self.spider.handle_error, ) self._assert_serializes_ok(r, spider=self.spider) @@ -113,7 +119,7 @@ class TestRequestSerialization: def test_mixin_private_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._SpiderMixin__mixin_callback, + callback=self.spider._SpiderMixin__mixin_callback, # type: ignore[attr-defined] errback=self.spider.handle_error, ) self._assert_serializes_ok(r, spider=self.spider) @@ -127,7 +133,7 @@ class TestRequestSerialization: self._assert_serializes_ok(r, spider=self.spider) def test_unserializable_callback1(self): - r = Request("http://www.example.com", callback=lambda x: x) + r = Request("http://www.example.com", callback=lambda x: x) # type: ignore[misc] with pytest.raises( ValueError, match="is not an instance method in: None: pass spider = MySpider() r = Request("http://www.example.com", callback=spider.parse) - spider.parse = None + spider.parse = None # type: ignore[method-assign,assignment] with pytest.raises(ValueError, match="is not an instance method in: None: pass class SpiderDelegation: - def delegated_callback(self, response): + def delegated_callback(self, response: Response) -> None: pass -def parse_item(response): +def parse_item(response: Response) -> None: pass -def handle_error(failure): +def handle_error(failure: Failure) -> None: pass -def private_parse_item(response): +def private_parse_item(response: Response) -> None: pass -def private_handle_error(failure): +def private_handle_error(failure: Failure) -> None: pass @@ -197,15 +205,17 @@ class MethodsSpider(Spider, SpiderMixin): __parse_item_reference = private_parse_item __handle_error_reference = private_handle_error - def __init__(self, **kwargs): + def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) self.delegated_callback = SpiderDelegation().delegated_callback - def parse_item(self, response): + def parse_item(self, response: Response) -> None: pass - def handle_error(self, failure): + def handle_error(self, failure: Failure) -> None: pass - def __parse_item_private(self, response): # pylint: disable=unused-private-member + def __parse_item_private( # pylint: disable=unused-private-member + self, response: Response + ) -> None: pass diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 726e0573a..46a16ad1e 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -1,57 +1,62 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + from scrapy.signals import request_left_downloader from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from tests.mockserver.http import MockServer from tests.utils.decorators import inline_callbacks_test +if TYPE_CHECKING: + from scrapy import Request + from scrapy.crawler import Crawler + from tests.mockserver.http import MockServer + class SignalCatcherSpider(Spider): name = "signal_catcher" - def __init__(self, crawler, url, *args, **kwargs): + def __init__(self, crawler: Crawler, url: str, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) crawler.signals.connect(self.on_request_left, signal=request_left_downloader) self.caught_times = 0 self.start_urls = [url] @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler( + cls, crawler: Crawler, *args: Any, **kwargs: Any + ) -> SignalCatcherSpider: return cls(crawler, *args, **kwargs) - def on_request_left(self, request, spider): + def on_request_left(self, request: Request, spider: Spider) -> None: self.caught_times += 1 class TestCatching: - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - @inline_callbacks_test - def test_success(self): + def test_success(self, mockserver: MockServer): crawler = get_crawler(SignalCatcherSpider) - yield crawler.crawl(self.mockserver.url("/status?n=200")) + yield crawler.crawl(mockserver.url("/status?n=200")) + assert isinstance(crawler.spider, SignalCatcherSpider) assert crawler.spider.caught_times == 1 @inline_callbacks_test - def test_timeout(self): + def test_timeout(self, mockserver: MockServer): crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1}) - yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) + yield crawler.crawl(mockserver.url("/delay?n=0.2")) + assert isinstance(crawler.spider, SignalCatcherSpider) assert crawler.spider.caught_times == 1 @inline_callbacks_test - def test_disconnect(self): + def test_disconnect(self, mockserver: MockServer): crawler = get_crawler(SignalCatcherSpider) - yield crawler.crawl(self.mockserver.url("/drop")) + yield crawler.crawl(mockserver.url("/drop")) + assert isinstance(crawler.spider, SignalCatcherSpider) assert crawler.spider.caught_times == 1 @inline_callbacks_test def test_noconnect(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl("http://thereisdefinetelynosuchdomain.com") + assert isinstance(crawler.spider, SignalCatcherSpider) assert crawler.spider.caught_times == 1 diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 7cca45ed1..8e69f62a0 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -3,6 +3,7 @@ from __future__ import annotations from unittest.mock import Mock import pytest +from twisted.internet.address import IPv4Address, IPv6Address from scrapy.resolver import CachingHostnameResolver, CachingThreadedResolver, dnscache from scrapy.utils.defer import maybe_deferred_to_future @@ -55,11 +56,77 @@ def test_caching_hostname_resolver_no_addresses_not_cached(): assert "example.com" not in dnscache +def test_caching_hostname_resolver_cached_addresses_have_no_port(): + def fake_resolve(receiver, *_): + receiver.resolutionBegan(Mock()) + receiver.addressResolved(IPv4Address("TCP", "1.2.3.4", 80)) + receiver.addressResolved(IPv6Address("TCP", "::1", 80)) + receiver.resolutionComplete() + return receiver + + reactor = Mock() + reactor.nameResolver.resolveHostName.side_effect = fake_resolve + + receiver = Mock() + resolver = CachingHostnameResolver(reactor, cache_size=10) + resolver.resolveHostName(receiver, "example.com", portNumber=80) + + # The port requested on a cache miss is passed through unchanged, but it is + # not part of what gets cached. + resolved_ports = [ + call.args[0].port for call in receiver.addressResolved.call_args_list + ] + assert resolved_ports == [80, 80] + assert [address.port for address in dnscache["example.com"]] == [0, 0] + + +def test_caching_hostname_resolver_cache_hit_without_port(): + cached_addresses = [ + IPv4Address("TCP", "1.2.3.4", 0), + IPv6Address("TCP", "::1", 0), + ] + dnscache["example.com"] = cached_addresses + + receiver = Mock() + resolver = CachingHostnameResolver(Mock(), cache_size=10) + resolver.resolveHostName(receiver, "example.com") + + # Cached addresses already use the requested port, so they are reused as is. + resolved_addresses = [ + call.args[0] for call in receiver.addressResolved.call_args_list + ] + assert all( + resolved is cached + for resolved, cached in zip(resolved_addresses, cached_addresses, strict=True) + ) + + +def test_caching_hostname_resolver_cache_hit_uses_requested_port(): + dnscache["example.com"] = [ + IPv4Address("TCP", "1.2.3.4", 0), + IPv6Address("TCP", "::1", 0), + ] + + receiver = Mock() + resolver = CachingHostnameResolver(Mock(), cache_size=10) + resolver.resolveHostName(receiver, "example.com", portNumber=443) + + resolved_addresses = [ + call.args[0] for call in receiver.addressResolved.call_args_list + ] + assert resolved_addresses == [ + IPv4Address("TCP", "1.2.3.4", 443), + IPv6Address("TCP", "::1", 443), + ] + # The cached addresses must not be mutated in place. + assert [address.port for address in dnscache["example.com"]] == [0, 0] + + def test_caching_hostname_resolver_dnscache_disabled_rejects_storage(): def fake_resolve(receiver, *_): receiver.resolutionBegan(Mock()) - receiver.addressResolved(Mock()) + receiver.addressResolved(IPv4Address("TCP", "1.2.3.4", 80)) receiver.resolutionComplete() return receiver diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index da94a4e95..755f29959 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,30 +1,45 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import pytest from scrapy.robotstxt import ( ProtegoRobotParser, PythonRobotParser, RerpRobotParser, + RobotParser, decode_robotstxt, ) from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER from tests.utils.robotstxt import rerp_available +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + class BaseRobotParserTest: - def _setUp(self, parser_cls): + parser_cls: type[RobotParser] + + def _setUp(self, parser_cls: type[RobotParser]) -> None: self.parser_cls = parser_cls + def _parse(self, robotstxt_body: bytes) -> RobotParser: + # The parser backends only use the crawler to get the spider to log with. + return self.parser_cls.from_crawler(None, robotstxt_body) # type: ignore[arg-type] + def test_allowed(self): robotstxt_robotstxt_body = ( b"User-agent: * \nDisallow: /disallowed \nAllow: /allowed \nCrawl-delay: 10" ) - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://www.site.local/allowed", "*") assert not rp.allowed("https://www.site.local/disallowed", "*") - def test_allowed_wildcards(self): + def test_allowed_wildcards(self) -> None: robotstxt_robotstxt_body = b"""User-agent: first Disallow: /disallowed/*/end$ @@ -32,9 +47,7 @@ class BaseRobotParserTest: Allow: /*allowed Disallow: / """ - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://www.site.local/disallowed", "first") assert not rp.allowed("https://www.site.local/disallowed/xyz/end", "first") @@ -45,23 +58,19 @@ class BaseRobotParserTest: assert rp.allowed("https://www.site.local/is_still_allowed", "second") assert rp.allowed("https://www.site.local/is_allowed_too", "second") - def test_length_based_precedence(self): + def test_length_based_precedence(self) -> None: robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://www.site.local/page", "*") - def test_order_based_precedence(self): + def test_order_based_precedence(self) -> None: robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert not rp.allowed("https://www.site.local/page", "*") def test_empty_response(self): """empty response should equal 'allow all'""" - rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b"") + rp = self._parse(b"") assert rp.allowed("https://site.local/", "*") assert rp.allowed("https://site.local/", "chrome") assert rp.allowed("https://site.local/index.html", "*") @@ -70,14 +79,22 @@ class BaseRobotParserTest: def test_garbage_response(self): """garbage response should be discarded, equal 'allow all'""" robotstxt_robotstxt_body = b"GIF89a\xd3\x00\xfe\x00\xa2" - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://site.local/", "*") assert rp.allowed("https://site.local/", "chrome") assert rp.allowed("https://site.local/index.html", "*") assert rp.allowed("https://site.local/disallowed", "*") + def test_crawl_delay(self): + robotstxt_body = b"User-agent: *\nDisallow: /private\nCrawl-delay: 10\n" + rp = self._parse(robotstxt_body) + assert rp.crawl_delay("*") == 10.0 + + def test_crawl_delay_unset(self): + robotstxt_body = b"User-agent: *\nDisallow: /private\n" + rp = self._parse(robotstxt_body) + assert rp.crawl_delay("*") is None + def test_unicode_url_and_useragent(self): robotstxt_robotstxt_body = """ User-Agent: * @@ -89,9 +106,7 @@ class BaseRobotParserTest: User-Agent: UnicödeBöt Disallow: /some/randome/page.html""".encode() - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://site.local/", "*") assert not rp.allowed("https://site.local/admin/", "*") assert not rp.allowed("https://site.local/static/", "*") @@ -102,6 +117,20 @@ class BaseRobotParserTest: assert not rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt") +class TestRobotParser: + def test_crawl_delay_unsupported(self): + class AllowAllRobotParser(RobotParser): + @classmethod + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: + return cls() + + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: + return True + + rp = AllowAllRobotParser() + assert rp.crawl_delay("*") is None + + class TestDecodeRobotsTxt: def test_native_string_conversion(self): robotstxt_body = b"User-agent: *\nDisallow: /\n" @@ -135,21 +164,21 @@ class TestPythonRobotParser(BaseRobotParserTest): not STDLIB_IMPROVED_ROBOTFILEPARSER, reason="RobotFileParser from this Python version does not support length based directives precedence.", ) - def test_length_based_precedence(self): + def test_length_based_precedence(self) -> None: super().test_length_based_precedence() @pytest.mark.skipif( STDLIB_IMPROVED_ROBOTFILEPARSER, reason="RobotFileParser from this Python version does not support order based directives precedence.", ) - def test_order_based_precedence(self): + def test_order_based_precedence(self) -> None: super().test_order_based_precedence() @pytest.mark.skipif( not STDLIB_IMPROVED_ROBOTFILEPARSER, reason="RobotFileParser from this Python version does not support wildcards.", ) - def test_allowed_wildcards(self): + def test_allowed_wildcards(self) -> None: super().test_allowed_wildcards() @@ -158,7 +187,7 @@ class TestRerpRobotParser(BaseRobotParserTest): def setup_method(self): super()._setUp(RerpRobotParser) - def test_length_based_precedence(self): + def test_length_based_precedence(self) -> None: pytest.skip("Rerp does not support length based directives precedence.") @@ -166,5 +195,5 @@ class TestProtegoRobotParser(BaseRobotParserTest): def setup_method(self): super()._setUp(ProtegoRobotParser) - def test_order_based_precedence(self): + def test_order_based_precedence(self) -> None: pytest.skip("Protego does not support order based directives precedence.") diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 4fb594444..d7c21bdbd 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -3,7 +3,7 @@ from __future__ import annotations import warnings from abc import ABC, abstractmethod from contextlib import AbstractAsyncContextManager, asynccontextmanager -from typing import TYPE_CHECKING, Any, NamedTuple, cast +from typing import TYPE_CHECKING from unittest.mock import Mock import pytest @@ -15,43 +15,17 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.defer import ensure_awaitable -from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.downloader import MockDownloader if TYPE_CHECKING: from collections.abc import AsyncGenerator from pathlib import Path -class MockSlot(NamedTuple): - active: list[Any] - - -class MockDownloader: - def __init__(self) -> None: - self.slots: dict[str, MockSlot] = {} - - def get_slot_key(self, request: Request) -> str: - if Downloader.DOWNLOAD_SLOT in request.meta: - return cast("str", request.meta[Downloader.DOWNLOAD_SLOT]) - - return urlparse_cached(request).hostname or "" - - def increment(self, slot_key: str) -> None: - slot = self.slots.setdefault(slot_key, MockSlot(active=[])) - slot.active.append(1) - - def decrement(self, slot_key: str) -> None: - slot = self.slots[slot_key] - slot.active.pop() - - def close(self) -> None: - pass - - class MockCrawler(Crawler): def __init__(self, priority_queue_cls: str, jobdir: Path | None): settings = { diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 08acacae7..2c2d26d53 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -1,9 +1,10 @@ from __future__ import annotations +import logging +from typing import TYPE_CHECKING from urllib.parse import urljoin import pytest -from testfixtures import LogCapture from twisted.internet import defer from scrapy.core.scheduler import BaseScheduler @@ -12,8 +13,10 @@ from scrapy.spiders import Spider from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.request import fingerprint from scrapy.utils.test import get_crawler -from tests.mockserver.http import MockServer -from tests.utils.decorators import inline_callbacks_test +from tests.utils.decorators import coroutine_test, inline_callbacks_test + +if TYPE_CHECKING: + from tests.mockserver.http import MockServer PATHS = ["/a", "/b", "/c"] URLS = [urljoin("https://example.org", p) for p in PATHS] @@ -147,18 +150,19 @@ class TestSimpleScheduler(InterfaceCheckMixin): class TestMinimalSchedulerCrawl: scheduler_cls = MinimalScheduler - @inline_callbacks_test - def test_crawl(self): - with MockServer() as mockserver: - settings = { - "SCHEDULER": self.scheduler_cls, - } - with LogCapture() as log: - crawler = get_crawler(PathsSpider, settings) - yield crawler.crawl(mockserver) - for path in PATHS: - assert f"{{'path': '{path}'}}" in str(log) - assert f"'item_scraped_count': {len(PATHS)}" in str(log) + @coroutine_test + async def test_crawl( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + settings = { + "SCHEDULER": self.scheduler_cls, + } + with caplog.at_level(logging.DEBUG): + crawler = get_crawler(PathsSpider, settings) + await crawler.crawl_async(mockserver) + for path in PATHS: + assert f"{{'path': '{path}'}}" in caplog.text + assert f"'item_scraped_count': {len(PATHS)}" in caplog.text class TestSimpleSchedulerCrawl(TestMinimalSchedulerCrawl): diff --git a/tests/test_spider.py b/tests/test_spider.py index 7b2727370..38cb8da18 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,133 +1,33 @@ from __future__ import annotations -from typing import Any -from unittest import mock +from typing import TYPE_CHECKING import pytest -from testfixtures import LogCapture -from scrapy import signals -from scrapy.crawler import Crawler -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.http import Response, TextResponse, XmlResponse -from scrapy.settings import Settings +from scrapy.http import Request, Response, TextResponse, XmlResponse from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider -from scrapy.utils.test import get_crawler, get_reactor_settings from tests import get_testdata -from tests.utils.decorators import inline_callbacks_test +from tests.spiders import RawResponseSpider +from tests.utils.bases.spider import TestSpiderBase +from tests.utils.crawl import crawl_items +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from tests.mockserver.http import MockServer -class TestSpider: +class RawFeedSpider(RawResponseSpider): + content_type = "text/xml" + + async def start(self): + yield Request(self.raw_url) + + +class TestSpider(TestSpiderBase): spider_class = Spider - def test_base_spider(self): - spider = self.spider_class("example.com") - assert spider.name == "example.com" - assert spider.start_urls == [] - def test_spider_args(self): - """``__init__`` method arguments are assigned to spider attributes""" - spider = self.spider_class("example.com", foo="bar") - assert spider.foo == "bar" - - def test_spider_without_name(self): - """``__init__`` raises when the name is not provided.""" - msg = "must have a name" - with pytest.raises(ValueError, match=msg): - self.spider_class() - with pytest.raises(ValueError, match=msg): - self.spider_class(somearg="foo") - - def test_from_crawler_crawler_and_settings_population(self): - crawler = get_crawler() - spider = self.spider_class.from_crawler(crawler, "example.com") - assert hasattr(spider, "crawler") - assert spider.crawler is crawler - assert hasattr(spider, "settings") - assert spider.settings is crawler.settings - - def test_from_crawler_init_call(self): - with mock.patch.object( - self.spider_class, "__init__", return_value=None - ) as mock_init: - self.spider_class.from_crawler(get_crawler(), "example.com", foo="bar") - mock_init.assert_called_once_with("example.com", foo="bar") - - def test_closed_signal_call(self): - class TestSpider(self.spider_class): - closed_called = False - - def closed(self, reason): - self.closed_called = True - - crawler = get_crawler() - spider = TestSpider.from_crawler(crawler, "example.com") - crawler.signals.send_catch_log(signal=signals.spider_opened, spider=spider) - crawler.signals.send_catch_log( - signal=signals.spider_closed, spider=spider, reason=None - ) - assert spider.closed_called - - def test_update_settings(self): - spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = {"TEST1": "project", "TEST3": "project"} - self.spider_class.custom_settings = spider_settings - settings = Settings(project_settings, priority="project") - - self.spider_class.update_settings(settings) - assert settings.get("TEST1") == "spider" - assert settings.get("TEST2") == "spider" - assert settings.get("TEST3") == "project" - - @inline_callbacks_test - def test_settings_in_from_crawler(self): - spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = { - "TEST1": "project", - "TEST3": "project", - **get_reactor_settings(), - } - - class TestSpider(self.spider_class): - name = "test" - custom_settings = spider_settings - - @classmethod - def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any): - spider = super().from_crawler(crawler, *args, **kwargs) - spider.settings.set("TEST1", "spider_instance", priority="spider") - return spider - - crawler = Crawler(TestSpider, project_settings) - assert crawler.settings.get("TEST1") == "spider" - assert crawler.settings.get("TEST2") == "spider" - assert crawler.settings.get("TEST3") == "project" - yield crawler.crawl() - assert crawler.settings.get("TEST1") == "spider_instance" - - def test_logger(self): - spider = self.spider_class("example.com") - with LogCapture() as lc: - spider.logger.info("test log msg") - lc.check(("example.com", "INFO", "test log msg")) - - record = lc.records[0] - assert "spider" in record.__dict__ - assert record.spider is spider - - def test_log(self): - spider = self.spider_class("example.com") - with ( - mock.patch("scrapy.spiders.Spider.logger") as mock_logger, - pytest.warns( - ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated" - ), - ): - spider.log("test log msg", "INFO") - mock_logger.log.assert_called_once_with("INFO", "test log msg") - - -class TestXMLFeedSpider(TestSpider): +class TestXMLFeedSpider(TestSpiderBase): spider_class = XMLFeedSpider def test_register_namespace(self): @@ -175,8 +75,91 @@ class TestXMLFeedSpider(TestSpider): }, ], iterator + @coroutine_test + async def test_parse_node_uses_parse_item(self, mockserver: MockServer): + # parse_node falls back to parse_item for backward compatibility. + class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc] + itertag = "item" -class TestCSVFeedSpider(TestSpider): + def raw_body(self): + return "1" + + def parse_item(self, response, selector): + return {"id": selector.xpath("id/text()").get()} + + items, _ = await crawl_items(_Spider, mockserver) + assert items == [{"id": "1"}] + + @coroutine_test + async def test_parse_node_not_defined(self, mockserver: MockServer): + class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc] + itertag = "item" + + def raw_body(self): + return "1" + + items, crawler = await crawl_items(_Spider, mockserver) + assert items == [] + assert crawler.stats + assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1 + + @coroutine_test + async def test_html_iterator(self, mockserver: MockServer): + class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc] + iterator = "html" + itertag = "item" + content_type = "text/html" + + def raw_body(self): + return ( + "1" + "2" + ) + + def parse_node(self, response, selector): + return {"id": selector.xpath("id/text()").get()} + + items, _ = await crawl_items(_Spider, mockserver) + assert items == [{"id": "1"}, {"id": "2"}] + + @coroutine_test + async def test_unsupported_iterator(self, mockserver: MockServer): + class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc] + iterator = "unsupported" + + def raw_body(self): + return "" + + def parse_node(self, response, selector): + return {} + + items, crawler = await crawl_items(_Spider, mockserver) + assert items == [] + assert crawler.stats + assert crawler.stats.get_value("spider_exceptions/NotSupported") == 1 + + @pytest.mark.parametrize("feed_iterator", ["xml", "html"]) + @coroutine_test + async def test_non_text_response(self, feed_iterator: str, mockserver: MockServer): + # The xml and html iterators require a text response. + class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc] + content_type = "application/octet-stream" + iterator = feed_iterator + + def raw_body(self): + # A binary (non-text) body, so the response is a plain Response. + return "\x00\x01\x02\x03" + + def parse_node(self, response, selector): + return {} + + items, crawler = await crawl_items(_Spider, mockserver) + assert items == [] + assert crawler.stats + assert crawler.stats.get_value("spider_exceptions/ValueError") == 1 + + +class TestCSVFeedSpider(TestSpiderBase): spider_class = CSVFeedSpider def test_parse_rows(self): @@ -196,6 +179,36 @@ class TestCSVFeedSpider(TestSpider): assert rows[0] == {"id": "1", "name": "alpha", "value": "foobar"} assert len(rows) == 4 + @coroutine_test + async def test_parse(self, mockserver: MockServer): + class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc] + content_type = "text/csv" + delimiter = "," + quotechar = "'" + + def raw_body(self): + return get_testdata("feeds", "feed-sample6.csv").decode() + + def parse_row(self, response, row): + return row + + items, _ = await crawl_items(_Spider, mockserver) + assert items[0] == {"id": "1", "name": "alpha", "value": "foobar"} + assert len(items) == 4 + + @coroutine_test + async def test_parse_row_not_defined(self, mockserver: MockServer): + class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc] + content_type = "text/csv" + + def raw_body(self): + return "id\n1\n" + + items, crawler = await crawl_items(_Spider, mockserver) + assert items == [] + assert crawler.stats + assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1 + class TestNoParseMethodSpider: spider_class = Spider diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py index 63010e195..9d5f0548b 100644 --- a/tests/test_spider_crawl.py +++ b/tests/test_spider_crawl.py @@ -11,10 +11,11 @@ from scrapy.http import HtmlResponse, Request, TextResponse from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule, Spider from scrapy.utils.test import get_crawler -from tests.test_spider import TestSpider +from tests.utils.bases.spider import TestSpiderBase +from tests.utils.decorators import coroutine_test -class TestCrawlSpider(TestSpider): +class TestCrawlSpider(TestSpiderBase): test_body = b"""Page title

Item 12

@@ -293,6 +294,48 @@ class TestCrawlSpider(TestSpider): TextResponse(spider.start_urls, body=b""), None, None ) + @coroutine_test + async def test_parse_with_rules_without_callback(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(CrawlSpider): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(),) + + spider = _CrawlSpider.from_crawler(get_crawler(_CrawlSpider)) + results = [ + r async for r in spider.parse_with_rules(response, None, {}, follow=True) + ] + assert [r.url for r in results] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + + @coroutine_test + async def test_parse_with_rules_without_following(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + item = {"name": "item"} + + class _CrawlSpider(CrawlSpider): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(),) + + spider = _CrawlSpider.from_crawler(get_crawler(_CrawlSpider)) + results = [ + r + async for r in spider.parse_with_rules( + response, lambda response: [item], {}, follow=False + ) + ] + assert results == [item] + class TestDeprecation: def test_crawl_spider(self): diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py index 0af99ab6d..2f1ccab81 100644 --- a/tests/test_spider_sitemap.py +++ b/tests/test_spider_sitemap.py @@ -6,19 +6,40 @@ from datetime import datetime from io import BytesIO from logging import WARNING from pathlib import Path +from typing import TYPE_CHECKING import pytest -from testfixtures import LogCapture from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse from scrapy.spiders import SitemapSpider from scrapy.utils.test import get_crawler from tests import tests_datadir -from tests.test_spider import TestSpider +from tests.spiders import RawResponseSpider +from tests.utils.bases.spider import TestSpiderBase +from tests.utils.crawl import crawl_items from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + from tests.mockserver.http import MockServer -class TestSitemapSpider(TestSpider): + +class RawSitemapSpider(RawResponseSpider): + """Feeds :meth:`raw_body` to :class:`~scrapy.spiders.SitemapSpider` as a + sitemap, so that it is fetched and followed through a regular crawl. + + Subclasses build the document in :meth:`raw_body`, typically using + :attr:`mockserver` to point ```` entries at real endpoints. + """ + + content_type = "application/xml" + + async def start(self): + self.sitemap_urls = [self.raw_url] + async for request in super().start(): + yield request + + +class TestSitemapSpider(TestSpiderBase): spider_class = SitemapSpider BODY = b"SITEMAP" @@ -254,22 +275,63 @@ Sitemap: /sitemap-relative-url.xml urls = [req.url for req in spider._parse_sitemap(r)] assert urls == result - def test_parse_sitemap_empty_body(self): + @coroutine_test + async def test_sitemap_rules_with_callable(self, mockserver: MockServer): + # A sitemap_rules entry may hold a callable instead of a method name. + def parse_item(response): + yield {"url": response.url} + + class _Spider(RawSitemapSpider, self.spider_class): # type: ignore[name-defined,misc] + sitemap_rules = [("", parse_item)] + + def raw_body(self): + loc = self.mockserver.url("/text") + return ( + '' + '' + f"{loc}" + "" + ) + + items, _ = await crawl_items(_Spider, mockserver) + assert items == [{"url": mockserver.url("/text")}] + + @coroutine_test + async def test_sitemap_empty_loc(self, mockserver: MockServer): + class _Spider(RawSitemapSpider, self.spider_class): # type: ignore[name-defined,misc] + def parse(self, response): + yield {"url": response.url} + + def raw_body(self): + loc = self.mockserver.url("/text") + return ( + '' + '' + "" + f"{loc}" + "" + ) + + items, _ = await crawl_items(_Spider, mockserver) + assert items == [{"url": mockserver.url("/text")}] + + def test_parse_sitemap_empty_body(self, caplog: pytest.LogCaptureFixture) -> None: r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"") spider = self.spider_class("example.com") - with LogCapture() as lc: + caplog.clear() + with caplog.at_level(WARNING): results = list(spider._parse_sitemap(r)) assert not results - lc.check( + assert caplog.record_tuples == [ ( "scrapy.spiders.sitemap", - "WARNING", + WARNING, "Ignoring invalid sitemap: <200 http://www.example.com/sitemap.xml>", ) - ) + ] def test_parse_sitemap_not_sitemap(self): body = b""" @@ -342,7 +404,7 @@ Sitemap: /sitemap-relative-url.xml response = Response(url="https://example.com", body=body, request=request) assert spider._get_sitemap_body(response) is None - def test_download_warnsize_setting(self): + def test_download_warnsize_setting(self, caplog: pytest.LogCaptureFixture) -> None: settings = {"DOWNLOAD_WARNSIZE": 10_000_000} crawler = get_crawler(settings_dict=settings) spider = self.spider_class.from_crawler(crawler, "example.com") @@ -350,25 +412,26 @@ Sitemap: /sitemap-relative-url.xml body = body_path.read_bytes() request = Request(url="https://example.com") response = Response(url="https://example.com", body=body, request=request) - with LogCapture( - "scrapy.spiders.sitemap", propagate=False, level=WARNING - ) as log: + caplog.clear() + with caplog.at_level(WARNING, logger="scrapy.spiders.sitemap"): spider._get_sitemap_body(response) - log.check( + assert caplog.record_tuples == [ ( "scrapy.spiders.sitemap", - "WARNING", + WARNING, ( "<200 https://example.com> body size after decompression " "(11511612 B) is larger than the download warning size " "(10000000 B)." ), ), - ) + ] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_download_warnsize_spider_attr(self): - class DownloadWarnSizeSpider(self.spider_class): + def test_download_warnsize_spider_attr( + self, caplog: pytest.LogCaptureFixture + ) -> None: + class DownloadWarnSizeSpider(self.spider_class): # type: ignore[name-defined,misc] download_warnsize = 10_000_000 crawler = get_crawler() @@ -379,23 +442,24 @@ Sitemap: /sitemap-relative-url.xml url="https://example.com", meta={"download_warnsize": 10_000_000} ) response = Response(url="https://example.com", body=body, request=request) - with LogCapture( - "scrapy.spiders.sitemap", propagate=False, level=WARNING - ) as log: + caplog.clear() + with caplog.at_level(WARNING, logger="scrapy.spiders.sitemap"): spider._get_sitemap_body(response) - log.check( + assert caplog.record_tuples == [ ( "scrapy.spiders.sitemap", - "WARNING", + WARNING, ( "<200 https://example.com> body size after decompression " "(11511612 B) is larger than the download warning size " "(10000000 B)." ), ), - ) + ] - def test_download_warnsize_request_meta(self): + def test_download_warnsize_request_meta( + self, caplog: pytest.LogCaptureFixture + ) -> None: crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, "example.com") body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") @@ -404,21 +468,20 @@ Sitemap: /sitemap-relative-url.xml url="https://example.com", meta={"download_warnsize": 10_000_000} ) response = Response(url="https://example.com", body=body, request=request) - with LogCapture( - "scrapy.spiders.sitemap", propagate=False, level=WARNING - ) as log: + caplog.clear() + with caplog.at_level(WARNING, logger="scrapy.spiders.sitemap"): spider._get_sitemap_body(response) - log.check( + assert caplog.record_tuples == [ ( "scrapy.spiders.sitemap", - "WARNING", + WARNING, ( "<200 https://example.com> body size after decompression " "(11511612 B) is larger than the download warning size " "(10000000 B)." ), ), - ) + ] @coroutine_test async def test_sitemap_urls(self): diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index afdfaa322..dacc90b27 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -1,10 +1,16 @@ -from testfixtures import LogCapture +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING from scrapy import Request, Spider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + import pytest + class _BaseSpiderMiddleware: def __init__(self, crawler): @@ -250,113 +256,132 @@ class TestSpiderMiddleware: def teardown_class(cls): cls.mockserver.__exit__(None, None, None) - async def crawl_log(self, spider: type[Spider]) -> LogCapture: + async def crawl_log( + self, spider: type[Spider], caplog: pytest.LogCaptureFixture + ) -> str: crawler = get_crawler(spider) - with LogCapture() as log: + caplog.clear() + with caplog.at_level(logging.DEBUG): await crawler.crawl_async(mockserver=self.mockserver) - return log + return caplog.text @coroutine_test - async def test_recovery(self): + async def test_recovery(self, caplog: pytest.LogCaptureFixture) -> None: """ (0) Recover from an exception in a spider callback. The final item count should be 3 (one yielded from the callback method before the exception is raised, one directly from the recovery middleware and one from the spider when processing the request that was enqueued from the recovery middleware) """ - log = await self.crawl_log(RecoverySpider) - assert "Middleware: TabError exception caught" in str(log) - assert str(log).count("Middleware: TabError exception caught") == 1 - assert "'item_scraped_count': 3" in str(log) + log = await self.crawl_log(RecoverySpider, caplog) + assert "Middleware: TabError exception caught" in log + assert log.count("Middleware: TabError exception caught") == 1 + assert "'item_scraped_count': 3" in log @coroutine_test - async def test_recovery_asyncgen(self): + async def test_recovery_asyncgen(self, caplog: pytest.LogCaptureFixture) -> None: """ Same as test_recovery but with an async callback. """ - log = await self.crawl_log(RecoveryAsyncGenSpider) - assert "Middleware: TabError exception caught" in str(log) - assert str(log).count("Middleware: TabError exception caught") == 1 - assert "'item_scraped_count': 3" in str(log) + log = await self.crawl_log(RecoveryAsyncGenSpider, caplog) + assert "Middleware: TabError exception caught" in log + assert log.count("Middleware: TabError exception caught") == 1 + assert "'item_scraped_count': 3" in log @coroutine_test - async def test_process_spider_input_without_errback(self): + async def test_process_spider_input_without_errback( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ (1.1) An exception from the process_spider_input chain should be caught by the process_spider_exception chain from the start if the Request has no errback """ - log1 = await self.crawl_log(ProcessSpiderInputSpiderWithoutErrback) - assert "Middleware: will raise IndexError" in str(log1) - assert "Middleware: IndexError exception caught" in str(log1) + log1 = await self.crawl_log(ProcessSpiderInputSpiderWithoutErrback, caplog) + assert "Middleware: will raise IndexError" in log1 + assert "Middleware: IndexError exception caught" in log1 @coroutine_test - async def test_process_spider_input_with_errback(self): + async def test_process_spider_input_with_errback( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ (1.2) An exception from the process_spider_input chain should not be caught by the process_spider_exception chain if the Request has an errback """ - log1 = await self.crawl_log(ProcessSpiderInputSpiderWithErrback) - assert "Middleware: IndexError exception caught" not in str(log1) - assert "Middleware: will raise IndexError" in str(log1) - assert "Got a Failure on the Request errback" in str(log1) - assert "{'from': 'errback'}" in str(log1) - assert "{'from': 'callback'}" not in str(log1) - assert "'item_scraped_count': 1" in str(log1) + log1 = await self.crawl_log(ProcessSpiderInputSpiderWithErrback, caplog) + assert "Middleware: IndexError exception caught" not in log1 + assert "Middleware: will raise IndexError" in log1 + assert "Got a Failure on the Request errback" in log1 + assert "{'from': 'errback'}" in log1 + assert "{'from': 'callback'}" not in log1 + assert "'item_scraped_count': 1" in log1 @coroutine_test - async def test_generator_callback(self): + async def test_generator_callback(self, caplog: pytest.LogCaptureFixture) -> None: """ (2) An exception from a spider callback (returning a generator) should be caught by the process_spider_exception chain. Items yielded before the exception is raised should be processed normally. """ - log2 = await self.crawl_log(GeneratorCallbackSpider) - assert "Middleware: ImportError exception caught" in str(log2) - assert "'item_scraped_count': 2" in str(log2) + log2 = await self.crawl_log(GeneratorCallbackSpider, caplog) + assert "Middleware: ImportError exception caught" in log2 + assert "'item_scraped_count': 2" in log2 @coroutine_test - async def test_async_generator_callback(self): + async def test_async_generator_callback( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ Same as test_generator_callback but with an async callback. """ - log2 = await self.crawl_log(AsyncGeneratorCallbackSpider) - assert "Middleware: ImportError exception caught" in str(log2) - assert "'item_scraped_count': 2" in str(log2) + log2 = await self.crawl_log(AsyncGeneratorCallbackSpider, caplog) + assert "Middleware: ImportError exception caught" in log2 + assert "'item_scraped_count': 2" in log2 @coroutine_test - async def test_generator_callback_right_after_callback(self): + async def test_generator_callback_right_after_callback( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ (2.1) Special case of (2): Exceptions should be caught even if the middleware is placed right after the spider """ - log21 = await self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider) - assert "Middleware: ImportError exception caught" in str(log21) - assert "'item_scraped_count': 2" in str(log21) + log21 = await self.crawl_log( + GeneratorCallbackSpiderMiddlewareRightAfterSpider, caplog + ) + assert "Middleware: ImportError exception caught" in log21 + assert "'item_scraped_count': 2" in log21 @coroutine_test - async def test_not_a_generator_callback(self): + async def test_not_a_generator_callback( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ (3) An exception from a spider callback (returning a list) should be caught by the process_spider_exception chain. No items should be processed. """ - log3 = await self.crawl_log(NotGeneratorCallbackSpider) - assert "Middleware: ZeroDivisionError exception caught" in str(log3) - assert "item_scraped_count" not in str(log3) + log3 = await self.crawl_log(NotGeneratorCallbackSpider, caplog) + assert "Middleware: ZeroDivisionError exception caught" in log3 + assert "item_scraped_count" not in log3 @coroutine_test - async def test_not_a_generator_callback_right_after_callback(self): + async def test_not_a_generator_callback_right_after_callback( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ (3.1) Special case of (3): Exceptions should be caught even if the middleware is placed right after the spider """ log31 = await self.crawl_log( - NotGeneratorCallbackSpiderMiddlewareRightAfterSpider + NotGeneratorCallbackSpiderMiddlewareRightAfterSpider, caplog ) - assert "Middleware: ZeroDivisionError exception caught" in str(log31) - assert "item_scraped_count" not in str(log31) + assert "Middleware: ZeroDivisionError exception caught" in log31 + assert "item_scraped_count" not in log31 @coroutine_test - async def test_generator_output_chain(self): + async def test_generator_output_chain( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ (4) An exception from a middleware's process_spider_output method should be sent to the process_spider_exception method from the next middleware in the chain. @@ -365,23 +390,23 @@ class TestSpiderMiddleware: The final item count should be 2 (one from the spider callback and one from the process_spider_exception chain) """ - log4 = await self.crawl_log(GeneratorOutputChainSpider) - assert "'item_scraped_count': 2" in str(log4) + log4 = await self.crawl_log(GeneratorOutputChainSpider, caplog) + assert "'item_scraped_count': 2" in log4 assert ( "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught" - in str(log4) + in log4 ) assert ( "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught" - in str(log4) + in log4 ) assert ( "GeneratorFailMiddleware.process_spider_exception: LookupError caught" - not in str(log4) + not in log4 ) assert ( "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught" - not in str(log4) + not in log4 ) item_from_callback = { "processed": [ @@ -398,6 +423,6 @@ class TestSpiderMiddleware: "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output", ] } - assert str(item_from_callback) in str(log4) - assert str(item_recovered) in str(log4) - assert "parse-second-item" not in str(log4) + assert str(item_from_callback) in log4 + assert str(item_recovered) in log4 + assert "parse-second-item" not in log4 diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 67ce3a920..321139da3 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -5,7 +5,6 @@ import pytest from scrapy import Spider, signals from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler -from tests.test_spider_start import SLEEP_SECONDS from .utils import twisted_sleep from .utils.decorators import coroutine_test @@ -14,6 +13,8 @@ ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} ITEM_C = {"id": "c"} +SLEEP_SECONDS = 0.1 + class AsyncioSleepSpiderMiddleware: async def process_start(self, start): diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index fc4e1c487..1d36a66fc 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -1,16 +1,18 @@ +from __future__ import annotations + from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from tests.utils.decorators import coroutine_test -class TestAsyncgenUtils: - @coroutine_test - async def test_as_async_generator(self): - ag = as_async_generator(range(42)) - results = [i async for i in ag] - assert results == list(range(42)) +@coroutine_test +async def test_as_async_generator(): + ag = as_async_generator(range(42)) + results = [i async for i in ag] + assert results == list(range(42)) - @coroutine_test - async def test_collect_asyncgen(self): - ag = as_async_generator(range(42)) - results = await collect_asyncgen(ag) - assert results == list(range(42)) + +@coroutine_test +async def test_collect_asyncgen(): + ag = as_async_generator(range(42)) + results = await collect_asyncgen(ag) + assert results == list(range(42)) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 7528dc51a..4bd54acd4 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -12,7 +12,9 @@ from scrapy.utils.asyncgen import as_async_generator from scrapy.utils.asyncio import ( AsyncioLoopingCall, _parallel_asyncio, + call_later, is_asyncio_available, + sleep, ) from tests.utils.decorators import coroutine_test @@ -20,11 +22,19 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator -class TestAsyncio: - @coroutine_test - async def test_is_asyncio_available(self, reactor_pytest: str) -> None: - # the result should depend only on the pytest --reactor argument - assert is_asyncio_available() == (reactor_pytest != "default") +@coroutine_test +async def test_is_asyncio_available(reactor_pytest: str) -> None: + # the result should depend only on the pytest --reactor argument + assert is_asyncio_available() == (reactor_pytest != "default") + + +@coroutine_test +async def test_sleep() -> None: + events: list[str] = [] + call_later(0.05, events.append, "call_later") + await sleep(0.1) + events.append("sleep") + assert events == ["call_later", "sleep"] @pytest.mark.only_asyncio diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index ad9aa3dff..0dea8af6d 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -1,10 +1,38 @@ from __future__ import annotations +import subprocess +import sys from importlib.util import find_spec +from io import BytesIO +from typing import TYPE_CHECKING import pytest +from pexpect import EOF -from scrapy.utils.console import get_shell_embed_func +from scrapy.utils.console import get_shell_embed_func, start_python_console +from scrapy.utils.test import get_testenv + +if TYPE_CHECKING: + from pathlib import Path + +CONSOLE = """ +from scrapy.utils.console import start_python_console + +start_python_console(banner="SHELL-READY", shells=["ipython"]) +""" + +CONSOLE_IN_RUNNING_LOOP = """ +import asyncio + +from scrapy.utils.console import start_python_console + + +async def main(): + start_python_console(banner="SHELL-READY", shells=["ipython"]) + + +asyncio.run(main()) +""" def test_get_shell_embed_func(): @@ -59,3 +87,90 @@ def test_get_shell_embed_func_default(): else: expected = "_embed_standard_shell" assert shell.__name__ == expected + + +@pytest.mark.skipif(find_spec("IPython") is None, reason="IPython is not installed") +class TestIPythonShell: + """Starting an IPython shell, with and without an asyncio event loop already + running in the calling thread. The latter happens when inspect_response() is + called from a spider callback while using the asyncio reactor.""" + + @staticmethod + def _env(tmp_path: Path) -> dict[str, str]: + env = get_testenv() + # Keep IPython away from the profile and history of the user running the tests. + env["IPYTHONDIR"] = str(tmp_path) + return env + + def test_simple_prompt(self, tmp_path: Path) -> None: + """IPython falls back to its simple prompt, which needs no event loop, + when stdin is not a TTY.""" + env = self._env(tmp_path) + p = subprocess.run( + [sys.executable, "-c", CONSOLE_IN_RUNNING_LOOP], + check=False, + capture_output=True, + encoding="utf-8", + timeout=60, + env=env, + stdin=subprocess.DEVNULL, + ) + output = p.stdout + p.stderr + assert "SHELL-READY" in output + assert p.returncode == 0, output + + @pytest.mark.skipif( + sys.platform == "win32", reason="requires a POSIX pseudo-terminal" + ) + @pytest.mark.parametrize( + "script", + [CONSOLE, CONSOLE_IN_RUNNING_LOOP], + ids=["no_running_loop", "running_loop"], + ) + def test_tty(self, tmp_path: Path, script: str) -> None: + """IPython uses prompt_toolkit, which needs an event loop of its own, + when stdin is a TTY.""" + # pexpect only defines spawn, which needs a pseudo-terminal, on POSIX. + from pexpect import spawn # noqa: PLC0415 + + env = self._env(tmp_path) + env.pop("IPY_TEST_SIMPLE_PROMPT", None) + env["TERM"] = "xterm" + logfile = BytesIO() + p = spawn( + sys.executable, + ["-c", script], + env=env, + timeout=60, + ) + p.logfile_read = logfile + try: + # Wait for the prompt, which prompt_toolkit draws once it is done + # querying the terminal, before typing into it. + p.expect(r"In \[") + p.sendline("21*2") + p.expect_exact("42") + p.sendline("exit()") + p.expect(EOF) + finally: + p.close() + output = logfile.getvalue().decode() + assert "Traceback" not in output + assert p.exitstatus == 0, output + + +def test_start_python_console_exit(monkeypatch: pytest.MonkeyPatch) -> None: + def embed(namespace: dict[str, object], banner: str) -> None: + raise SystemExit + + monkeypatch.setattr( + "scrapy.utils.console.get_shell_embed_func", lambda shells: embed + ) + start_python_console() + + +def test_start_python_console_no_shell(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "scrapy.utils.console.get_shell_embed_func", lambda shells: None + ) + start_python_console() diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index fce9fc984..6b30744bb 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import warnings from typing import Any diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index fe1f60c7d..af203ca61 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,7 +1,9 @@ +from __future__ import annotations + import copy from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, MutableMapping -from typing import Any +from typing import Any, Generic, TypeVar import pytest @@ -16,11 +18,13 @@ from scrapy.utils.datatypes import ( ) from scrapy.utils.python import garbage_collect +_DictT = TypeVar("_DictT", bound="CaselessDict | CaseInsensitiveDict") -class TestCaseInsensitiveDictBase(ABC): + +class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]): @property @abstractmethod - def dict_class(self) -> type[MutableMapping[str, Any]]: + def dict_class(self) -> type[_DictT]: raise NotImplementedError def test_init_dict(self): @@ -36,17 +40,17 @@ class TestCaseInsensitiveDictBase(ABC): assert d["black"] == 3 def test_init_mapping(self): - class MyMapping(Mapping): - def __init__(self, **kwargs): + class MyMapping(Mapping[str, int]): + def __init__(self, **kwargs: int) -> None: self._d = kwargs - def __getitem__(self, key): + def __getitem__(self, key: str) -> int: return self._d[key] - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self._d) - def __len__(self): + def __len__(self) -> int: return len(self._d) seq = MyMapping(red=1, black=3) @@ -55,23 +59,23 @@ class TestCaseInsensitiveDictBase(ABC): assert d["black"] == 3 def test_init_mutable_mapping(self): - class MyMutableMapping(MutableMapping): - def __init__(self, **kwargs): + class MyMutableMapping(MutableMapping[str, int]): + def __init__(self, **kwargs: int) -> None: self._d = kwargs - def __getitem__(self, key): + def __getitem__(self, key: str) -> int: return self._d[key] - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: int) -> None: self._d[key] = value - def __delitem__(self, key): + def __delitem__(self, key: str) -> None: del self._d[key] - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self._d) - def __len__(self): + def __len__(self) -> int: return len(self._d) seq = MyMutableMapping(red=1, black=3) @@ -149,7 +153,7 @@ class TestCaseInsensitiveDictBase(ABC): d.pop("A") def test_normkey(self): - class MyDict(self.dict_class): + class MyDict(self.dict_class): # type: ignore[misc,name-defined] def _normkey(self, key): return key.title() @@ -160,7 +164,7 @@ class TestCaseInsensitiveDictBase(ABC): assert list(d.keys()) == ["Key-One"] def test_normvalue(self): - class MyDict(self.dict_class): + class MyDict(self.dict_class): # type: ignore[misc,name-defined] def _normvalue(self, value): if value is not None: return value + 1 @@ -204,9 +208,18 @@ class TestCaseInsensitiveDictBase(ABC): assert h1.get("header1") == h3.get("header1") assert h1.get("header1") == h3.get("HEADER1") + def test_copy_is_independent(self): + h1 = self.dict_class({"header1": "value1", "header2": "value2"}) + for h2 in (copy.copy(h1), h1.copy()): + del h2["header1"] + h2["header3"] = "value3" + assert "header1" in h1 + assert "header3" not in h1 + assert dict(h1) == {"header1": "value1", "header2": "value2"} -class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase): - dict_class = CaseInsensitiveDict # type: ignore[assignment] + +class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase[CaseInsensitiveDict]): + dict_class = CaseInsensitiveDict def test_repr(self): d1 = self.dict_class({"foo": "bar"}) @@ -220,9 +233,31 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase): assert isinstance(iterkeys, Iterator) assert list(iterkeys) == ["AsDf", "FoO"] + def test_copy_keeps_values(self): + class MyDict(self.dict_class): # type: ignore[misc,name-defined] + def _normvalue(self, value): + return value + 1 + + d = MyDict({"key": 1}) + for copied in (copy.copy(d), d.copy()): + assert copied["key"] == 2 + + def test_ior(self): + d = self.dict_class({"header1": "value1"}) + d |= {"HEADER1": "value2", "header2": "value3"} + assert len(d) == 2 + assert d["HeAdEr1"] == "value2" + assert d["HeAdEr2"] == "value3" + + def test_ior_mapping(self): + d = self.dict_class({"header1": "value1"}) + d |= self.dict_class({"HEADER1": "value2"}) + assert len(d) == 1 + assert d["HeAdEr1"] == "value2" + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestCaselessDict(TestCaseInsensitiveDictBase): +class TestCaselessDict(TestCaseInsensitiveDictBase[CaselessDict]): dict_class = CaselessDict def test_deprecation_message(self): @@ -288,7 +323,7 @@ class TestSequenceExclude: class TestLocalCache: def test_cache_with_limit(self): - cache = LocalCache(limit=2) + cache: LocalCache[str, int] = LocalCache(limit=2) cache["a"] = 1 cache["b"] = 2 cache["c"] = 3 @@ -301,7 +336,7 @@ class TestLocalCache: def test_cache_without_limit(self): maximum = 10**4 - cache = LocalCache() + cache: LocalCache[str, int] = LocalCache() for x in range(maximum): cache[str(x)] = x assert len(cache) == maximum @@ -310,7 +345,7 @@ class TestLocalCache: assert cache[str(x)] == x def test_cache_with_zero_limit(self): - cache = LocalCache(limit=0) + cache: LocalCache[str, int] = LocalCache(limit=0) cache["a"] = 1 cache["b"] = 2 cache["c"] = 3 @@ -322,7 +357,9 @@ class TestLocalCache: class TestLocalWeakReferencedCache: def test_cache_with_limit(self): - cache = LocalWeakReferencedCache(limit=2) + cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache( + limit=2 + ) r1 = Request("https://example.org") r2 = Request("https://example.com") r3 = Request("https://example.net") @@ -344,7 +381,7 @@ class TestLocalWeakReferencedCache: assert len(cache) == 1 def test_cache_non_weak_referenceable_objects(self): - cache = LocalWeakReferencedCache() + cache: LocalWeakReferencedCache[Any, int] = LocalWeakReferencedCache() k1 = None k2 = 1 k3 = [1, 2, 3] @@ -358,7 +395,7 @@ class TestLocalWeakReferencedCache: def test_cache_without_limit(self): maximum = 10**4 - cache = LocalWeakReferencedCache() + cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache() refs = [] for x in range(maximum): refs.append(Request(f"https://example.org/{x}")) diff --git a/tests/test_utils_decorators.py b/tests/test_utils_decorators.py index 9743e1a50..4c29d2917 100644 --- a/tests/test_utils_decorators.py +++ b/tests/test_utils_decorators.py @@ -1,6 +1,8 @@ from __future__ import annotations +import sys import warnings +from typing import TYPE_CHECKING, Any import pytest from twisted.internet.defer import Deferred @@ -10,11 +12,14 @@ from scrapy.utils.decorators import _warn_spider_arg, deprecated, inthread from scrapy.utils.defer import maybe_deferred_to_future from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Callable + class TestDeprecated: def test_warns_and_still_calls(self): @deprecated() - def add(a, b): + def add(a: int, b: int) -> int: return a + b with pytest.warns( @@ -26,7 +31,7 @@ class TestDeprecated: def test_use_instead_in_message(self): @deprecated(use_instead="other_function") - def old(): + def old() -> None: return None with pytest.warns( @@ -37,7 +42,7 @@ class TestDeprecated: def test_applied_without_parentheses(self): @deprecated - def square(x): + def square(x: int) -> int: return x * x with pytest.warns( @@ -65,7 +70,7 @@ class TestInthread: class TestWarnSpiderArg: def test_sync_warns_with_spider_arg(self): @_warn_spider_arg - def parse(response, spider=None): + def parse(response: str, spider: str | None = None) -> str: return response with pytest.warns( @@ -73,9 +78,34 @@ class TestWarnSpiderArg: ): assert parse("response", spider="spider") == "response" + @pytest.mark.skipif( + sys.version_info < (3, 14), + reason="annotations are only lazily evaluated since Python 3.14 (PEP 649)", + ) + def test_sync_warns_with_unresolvable_annotations(self): + # dont_inherit=True, or the module's future import stringizes the annotations + namespace: dict[str, Any] = {} + exec( # pylint: disable=exec-used + compile( + "def parse(response: OnlyAtTypeCheckingTime," + " spider: OnlyAtTypeCheckingTime | None = None): return response", + "", + "exec", + dont_inherit=True, + ), + namespace, + ) + parse_func: Callable[..., str] = namespace["parse"] + parse = _warn_spider_arg(parse_func) + + with pytest.warns( + ScrapyDeprecationWarning, match=r"Passing a 'spider' argument" + ): + assert parse("response", spider="spider") == "response" + def test_sync_no_warning_without_spider_arg(self): @_warn_spider_arg - def parse(response, spider=None): + def parse(response: str, spider: str | None = None) -> str: return response with warnings.catch_warnings(): @@ -85,7 +115,7 @@ class TestWarnSpiderArg: @coroutine_test async def test_async_warns_with_spider_arg(self): @_warn_spider_arg - async def parse(response, spider=None): + async def parse(response: str, spider: str | None = None) -> str: return response with pytest.warns( @@ -96,7 +126,9 @@ class TestWarnSpiderArg: @coroutine_test async def test_asyncgen_warns_with_spider_arg(self): @_warn_spider_arg - async def parse(response, spider=None): + async def parse( + response: str, spider: str | None = None + ) -> AsyncGenerator[str]: yield response with pytest.warns( diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 978c24f5a..175a4fe03 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -24,6 +24,8 @@ from tests.utils.decorators import coroutine_test, inline_callbacks_test if TYPE_CHECKING: from collections.abc import AsyncGenerator, Awaitable, Callable, Generator + from twisted.python.failure import Failure + @pytest.mark.requires_reactor # mustbe_deferred() requires a reactor @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") @@ -70,7 +72,7 @@ class TestIterErrback: def itergood() -> Generator[int, None, None]: yield from range(10) - errors = [] + errors: list[Failure] = [] out = list(iter_errback(itergood(), errors.append)) assert out == list(range(10)) assert not errors @@ -82,7 +84,7 @@ class TestIterErrback: 1 / 0 yield x - errors = [] + errors: list[Failure] = [] out = list(iter_errback(iterbad(), errors.append)) assert out == [0, 1, 2, 3, 4] assert len(errors) == 1 @@ -96,7 +98,7 @@ class TestAiterErrback: for x in range(10): yield x - errors = [] + errors: list[Failure] = [] out = await collect_asyncgen(aiter_errback(itergood(), errors.append)) assert out == list(range(10)) assert not errors @@ -109,7 +111,7 @@ class TestAiterErrback: 1 / 0 yield x - errors = [] + errors: list[Failure] = [] out = await collect_asyncgen(aiter_errback(iterbad(), errors.append)) assert out == [0, 1, 2, 3, 4] assert len(errors) == 1 @@ -202,7 +204,7 @@ class TestParallelAsync: for length in [20, 50, 100]: parallel_count = [0] max_parallel_count = [0] - results = [] + results: list[int] = [] ait = self.get_async_iterable(length) dl = parallel_async( ait, @@ -222,7 +224,7 @@ class TestParallelAsync: for length in [20, 50, 100]: parallel_count = [0] max_parallel_count = [0] - results = [] + results: list[int] = [] ait = self.get_async_iterable_with_delays(length) dl = parallel_async( ait, @@ -240,7 +242,7 @@ class TestParallelAsync: class TestDeferredFromCoro: def test_deferred(self): - d = Deferred() + d: Deferred[None] = Deferred() result = deferred_from_coro(d) assert isinstance(result, Deferred) assert result is d @@ -274,7 +276,7 @@ class TestDeferredFromCoro: @pytest.mark.only_asyncio @inline_callbacks_test def test_future(self): - future = Future() + future: Future[int] = Future() result = deferred_from_coro(future) assert isinstance(result, Deferred) future.set_result(42) @@ -324,7 +326,7 @@ class TestDeferredFFromCoroF: class TestDeferredToFuture: @coroutine_test async def test_deferred(self): - d = Deferred() + d: Deferred[int] = Deferred() result = deferred_to_future(d) assert isinstance(result, Future) d.callback(42) @@ -359,7 +361,7 @@ class TestDeferredToFuture: class TestMaybeDeferredToFutureAsyncio: @coroutine_test async def test_deferred(self): - d = Deferred() + d: Deferred[int] = Deferred() result = maybe_deferred_to_future(d) assert isinstance(result, Future) d.callback(42) @@ -394,7 +396,7 @@ class TestMaybeDeferredToFutureAsyncio: class TestMaybeDeferredToFutureNotAsyncio: @coroutine_test async def test_deferred(self): - d = Deferred() + d: Deferred[int] = Deferred() result = maybe_deferred_to_future(d) assert isinstance(result, Deferred) assert result is d diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 0706fec99..4c8585916 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import inspect import warnings from unittest import mock @@ -38,7 +40,7 @@ class TestWarnWhenSubclassed: ) with pytest.warns(MyWarning, match=msg) as w: - class UserClass(Deprecated): + class UserClass(Deprecated): # type: ignore[misc, valid-type] pass assert w[0].lineno == inspect.getsourcelines(UserClass)[1] @@ -57,7 +59,7 @@ class TestWarnWhenSubclassed: match=r"UserClass inherits from deprecated class bar\.OldClass, please inherit from foo\.NewClass", ): - class UserClass(Deprecated): + class UserClass(Deprecated): # type: ignore[misc, valid-type] pass with pytest.warns( @@ -76,7 +78,7 @@ class TestWarnWhenSubclassed: match="UserClass inherits from deprecated class", ): - class UserClass(Deprecated): + class UserClass(Deprecated): # type: ignore[misc, valid-type] pass with warnings.catch_warnings(): @@ -95,16 +97,16 @@ class TestWarnWhenSubclassed: match="UserClass inherits from deprecated class", ): - class UserClass(Deprecated): + class UserClass(Deprecated): # type: ignore[misc, valid-type] pass with warnings.catch_warnings(): warnings.simplefilter("error", MyWarning) - class FooClass(Deprecated): + class FooClass(Deprecated): # type: ignore[misc, valid-type] pass - class BarClass(Deprecated): + class BarClass(Deprecated): # type: ignore[misc, valid-type] pass def test_warning_on_instance(self): @@ -112,22 +114,20 @@ class TestWarnWhenSubclassed: "Deprecated", NewName, warn_category=MyWarning ) - with pytest.warns(MyWarning) as w: - _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) - - w = [x for x in w if x.category is MyWarning] + with pytest.warns( + MyWarning, + match=r"tests\.test_utils_deprecate\.Deprecated is deprecated, " + r"instantiate tests\.test_utils_deprecate\.NewName instead\.", + ) as w: + _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) # type: ignore[arg-type] assert len(w) == 1 - assert ( - str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, " - "instantiate tests.test_utils_deprecate.NewName instead." - ) assert w[0].lineno == lineno # ignore subclassing warnings with warnings.catch_warnings(): warnings.simplefilter("ignore", MyWarning) - class UserClass(Deprecated): + class UserClass(Deprecated): # type: ignore[misc, valid-type] pass with warnings.catch_warnings(): @@ -141,7 +141,7 @@ class TestWarnWhenSubclassed: match=r"UserClass2 inherits from deprecated class tests\.test_utils_deprecate\.Deprecated, please inherit from tests\.test_utils_deprecate\.NewName", ): - class UserClass2(Deprecated): + class UserClass2(Deprecated): # type: ignore[misc, valid-type] pass def test_issubclass(self): @@ -155,10 +155,10 @@ class TestWarnWhenSubclassed: class UpdatedUserClass1a(NewName): pass - class OutdatedUserClass1(DeprecatedName): + class OutdatedUserClass1(DeprecatedName): # type: ignore[misc, valid-type] pass - class OutdatedUserClass1a(DeprecatedName): + class OutdatedUserClass1a(DeprecatedName): # type: ignore[misc, valid-type] pass class UnrelatedClass: @@ -174,7 +174,7 @@ class TestWarnWhenSubclassed: assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1) with pytest.raises(TypeError): - issubclass(object(), DeprecatedName) + issubclass(object(), DeprecatedName) # type: ignore[arg-type] def test_isinstance(self): with warnings.catch_warnings(): @@ -187,10 +187,10 @@ class TestWarnWhenSubclassed: class UpdatedUserClass2a(NewName): pass - class OutdatedUserClass2(DeprecatedName): + class OutdatedUserClass2(DeprecatedName): # type: ignore[misc, valid-type] pass - class OutdatedUserClass2a(DeprecatedName): + class OutdatedUserClass2a(DeprecatedName): # type: ignore[misc, valid-type] pass class UnrelatedClass: @@ -211,7 +211,7 @@ class TestWarnWhenSubclassed: warnings.simplefilter("ignore", ScrapyDeprecationWarning) Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"}) - assert Deprecated.foo == "bar" + assert Deprecated.foo == "bar" # type: ignore[attr-defined] def test_deprecate_a_class_with_custom_metaclass(self): Meta1 = type("Meta1", (type,), {}) @@ -242,7 +242,7 @@ class TestWarnWhenSubclassed: match=r"UserClass inherits from deprecated class tests\.test_utils_deprecate\.AlsoDeprecated, please inherit from foo\.Bar", ): - class UserClass(AlsoDeprecated): + class UserClass(AlsoDeprecated): # type: ignore[misc, valid-type] pass def test_inspect_stack(self): diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index 9f9e24957..a87816c20 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -31,13 +31,13 @@ plain_string = "{'a': 1}" @mock.patch("sys.platform", "linux") @mock.patch("sys.stdout.isatty") -def test_pformat(isatty): +def test_pformat(isatty: mock.Mock) -> None: isatty.return_value = True assert pformat(value) in colorized_strings @mock.patch("sys.stdout.isatty") -def test_pformat_dont_colorize(isatty): +def test_pformat_dont_colorize(isatty: mock.Mock) -> None: isatty.return_value = True assert pformat(value, colorize=False) == plain_string @@ -49,7 +49,7 @@ def test_pformat_not_tty(): @mock.patch("sys.platform", "win32") @mock.patch("platform.version") @mock.patch("sys.stdout.isatty") -def test_pformat_old_windows(isatty, version): +def test_pformat_old_windows(isatty: mock.Mock, version: mock.Mock) -> None: isatty.return_value = True version.return_value = "10.0.14392" assert pformat(value) in colorized_strings @@ -59,7 +59,9 @@ def test_pformat_old_windows(isatty, version): @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") @mock.patch("platform.version") @mock.patch("sys.stdout.isatty") -def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing): +def test_pformat_windows_no_terminal_processing( + isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock +) -> None: isatty.return_value = True version.return_value = "10.0.14393" terminal_processing.return_value = False @@ -70,7 +72,9 @@ def test_pformat_windows_no_terminal_processing(isatty, version, terminal_proces @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") @mock.patch("platform.version") @mock.patch("sys.stdout.isatty") -def test_pformat_windows(isatty, version, terminal_processing): +def test_pformat_windows( + isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock +) -> None: isatty.return_value = True version.return_value = "10.0.14393" terminal_processing.return_value = True @@ -79,7 +83,7 @@ def test_pformat_windows(isatty, version, terminal_processing): @mock.patch("sys.platform", "linux") @mock.patch("sys.stdout.isatty") -def test_pformat_no_pygments(isatty): +def test_pformat_no_pygments(isatty: mock.Mock) -> None: isatty.return_value = True real_import = builtins.__import__ diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 06fdf9cba..75f8be6f6 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from gzip import BadGzipFile from pathlib import Path diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index 0eb330461..610c463ec 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from urllib.parse import urlparse from scrapy.http import Request diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index fe57492b6..7f5301387 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -8,7 +8,6 @@ from io import StringIO from typing import TYPE_CHECKING, Any import pytest -from testfixtures import LogCapture from twisted.python.failure import Failure from scrapy.utils.log import ( @@ -107,15 +106,15 @@ class TestLogCounterHandler: class TestStreamLogger: - def test_redirect(self): + def test_redirect(self, caplog: pytest.LogCaptureFixture) -> None: logger = logging.getLogger("test") logger.setLevel(logging.WARNING) old_stdout = sys.stdout sys.stdout = StreamLogger(logger, logging.ERROR) - with LogCapture() as log: - print("test log msg") - log.check(("test", "ERROR", "test log msg")) + caplog.clear() + print("test log msg") + assert caplog.record_tuples == [("test", logging.ERROR, "test log msg")] sys.stdout = old_stdout diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index ab6965ed5..0775ec608 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -20,150 +20,157 @@ from scrapy.utils.misc import ( ) -class TestUtilsMisc: - def test_load_object_class(self): - obj = load_object(Field) - assert obj is Field - obj = load_object("scrapy.item.Field") - assert obj is Field +def test_load_object_class() -> None: + obj = load_object(Field) + assert obj is Field + obj = load_object("scrapy.item.Field") + assert obj is Field - def test_load_object_function(self): - obj = load_object(load_object) - assert obj is load_object - obj = load_object("scrapy.utils.misc.load_object") - assert obj is load_object - def test_load_object_exceptions(self): - with pytest.raises(ImportError): - load_object("nomodule999.mod.function") - with pytest.raises(NameError): - load_object("scrapy.utils.misc.load_object999") - with pytest.raises(TypeError): - load_object({}) # type: ignore[arg-type] +def test_load_object_function() -> None: + obj = load_object(load_object) + assert obj is load_object + obj = load_object("scrapy.utils.misc.load_object") + assert obj is load_object - def test_walk_modules(self): - mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules") + +def test_load_object_exceptions() -> None: + with pytest.raises(ImportError): + load_object("nomodule999.mod.function") + with pytest.raises(NameError): + load_object("scrapy.utils.misc.load_object999") + with pytest.raises(TypeError): + load_object({}) # type: ignore[arg-type] + + +def test_walk_modules() -> None: + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules") + expected = [ + "tests.test_utils_misc.test_walk_modules", + "tests.test_utils_misc.test_walk_modules.mod", + "tests.test_utils_misc.test_walk_modules.mod.mod0", + "tests.test_utils_misc.test_walk_modules.mod1", + ] + assert {m.__name__ for m in mods} == set(expected) + + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod") + expected = [ + "tests.test_utils_misc.test_walk_modules.mod", + "tests.test_utils_misc.test_walk_modules.mod.mod0", + ] + assert {m.__name__ for m in mods} == set(expected) + + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1") + expected = [ + "tests.test_utils_misc.test_walk_modules.mod1", + ] + assert {m.__name__ for m in mods} == set(expected) + + with pytest.raises(ImportError): + for _ in walk_modules_iter("nomodule999"): + pass + with ( + pytest.raises(ImportError), + pytest.warns( + ScrapyDeprecationWarning, + match="The scrapy.utils.misc.walk_modules function is deprecated and will be " + "removed in a future version of Scrapy. " + "Use scrapy.utils.misc.walk_modules_iter instead.", + ), + ): + walk_modules("nomodule999") + + +def test_walk_modules_egg() -> None: + egg = str(Path(__file__).parent / "test.egg") + sys.path.append(egg) + try: + mods = walk_modules_iter("testegg") expected = [ - "tests.test_utils_misc.test_walk_modules", - "tests.test_utils_misc.test_walk_modules.mod", - "tests.test_utils_misc.test_walk_modules.mod.mod0", - "tests.test_utils_misc.test_walk_modules.mod1", + "testegg.spiders", + "testegg.spiders.a", + "testegg.spiders.b", + "testegg", ] assert {m.__name__ for m in mods} == set(expected) + finally: + sys.path.remove(egg) - mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod") - expected = [ - "tests.test_utils_misc.test_walk_modules.mod", - "tests.test_utils_misc.test_walk_modules.mod.mod0", - ] - assert {m.__name__ for m in mods} == set(expected) - mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1") - expected = [ - "tests.test_utils_misc.test_walk_modules.mod1", - ] - assert {m.__name__ for m in mods} == set(expected) +def test_arg_to_iter() -> None: + class TestItem(Item): + name = Field() - with pytest.raises(ImportError): - for _ in walk_modules_iter("nomodule999"): - pass - with ( - pytest.raises(ImportError), - pytest.warns( - ScrapyDeprecationWarning, - match="The scrapy.utils.misc.walk_modules function is deprecated and will be " - "removed in a future version of Scrapy. " - "Use scrapy.utils.misc.walk_modules_iter instead.", - ), - ): - walk_modules("nomodule999") + assert hasattr(arg_to_iter(None), "__iter__") + assert hasattr(arg_to_iter(100), "__iter__") + assert hasattr(arg_to_iter("lala"), "__iter__") + assert hasattr(arg_to_iter([1, 2, 3]), "__iter__") + assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__") - def test_walk_modules_egg(self): - egg = str(Path(__file__).parent / "test.egg") - sys.path.append(egg) - try: - mods = walk_modules_iter("testegg") - expected = [ - "testegg.spiders", - "testegg.spiders.a", - "testegg.spiders.b", - "testegg", - ] - assert {m.__name__ for m in mods} == set(expected) - finally: - sys.path.remove(egg) + assert not list(arg_to_iter(None)) + assert list(arg_to_iter("lala")) == ["lala"] + assert list(arg_to_iter(100)) == [100] + assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"] + assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3] + assert list(arg_to_iter({"a": 1})) == [{"a": 1}] + assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")] - def test_arg_to_iter(self): - class TestItem(Item): - name = Field() - assert hasattr(arg_to_iter(None), "__iter__") - assert hasattr(arg_to_iter(100), "__iter__") - assert hasattr(arg_to_iter("lala"), "__iter__") - assert hasattr(arg_to_iter([1, 2, 3]), "__iter__") - assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__") +def test_build_from_crawler() -> None: + crawler = mock.MagicMock(spec_set=["settings"]) + args = (True, 100.0) + kwargs = {"key": "val"} - assert not list(arg_to_iter(None)) - assert list(arg_to_iter("lala")) == ["lala"] - assert list(arg_to_iter(100)) == [100] - assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"] - assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3] - assert list(arg_to_iter({"a": 1})) == [{"a": 1}] - assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")] + def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None: + build_from_crawler(mock, crawler, *args, **kwargs) + if hasattr(mock, "from_crawler"): + mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) + assert mock.call_count == 0 + else: + mock.assert_called_once_with(*args, **kwargs) - def test_build_from_crawler(self): - crawler = mock.MagicMock(spec_set=["settings"]) - args = (True, 100.0) - kwargs = {"key": "val"} + # Check usage of correct constructor using 2 mocks: + # 1. with no alternative constructors + # 2. with from_crawler() constructor + spec_sets = ( + ["__qualname__"], + ["__qualname__", "from_crawler"], + ) + for specs in spec_sets: + m = mock.MagicMock(spec_set=specs) + _test_with_crawler(m, crawler) + m.reset_mock() - def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None: - build_from_crawler(mock, crawler, *args, **kwargs) - if hasattr(mock, "from_crawler"): - mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) - assert mock.call_count == 0 - else: - mock.assert_called_once_with(*args, **kwargs) + # Check adoption of crawler + m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"]) + m.from_crawler.return_value = None + with pytest.raises(TypeError): + build_from_crawler(m, crawler, *args, **kwargs) - # Check usage of correct constructor using 2 mocks: - # 1. with no alternative constructors - # 2. with from_crawler() constructor - spec_sets = ( - ["__qualname__"], - ["__qualname__", "from_crawler"], - ) - for specs in spec_sets: - m = mock.MagicMock(spec_set=specs) - _test_with_crawler(m, crawler) - m.reset_mock() - # Check adoption of crawler - m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"]) - m.from_crawler.return_value = None - with pytest.raises(TypeError): - build_from_crawler(m, crawler, *args, **kwargs) +def test_set_environ() -> None: + assert os.environ.get("some_test_environ") is None + with set_environ(some_test_environ="test_value"): + assert os.environ.get("some_test_environ") == "test_value" + assert os.environ.get("some_test_environ") is None - def test_set_environ(self): - assert os.environ.get("some_test_environ") is None - with set_environ(some_test_environ="test_value"): - assert os.environ.get("some_test_environ") == "test_value" - assert os.environ.get("some_test_environ") is None + os.environ["some_test_environ"] = "test" + assert os.environ.get("some_test_environ") == "test" + with set_environ(some_test_environ="test_value"): + assert os.environ.get("some_test_environ") == "test_value" + assert os.environ.get("some_test_environ") == "test" - os.environ["some_test_environ"] = "test" - assert os.environ.get("some_test_environ") == "test" - with set_environ(some_test_environ="test_value"): - assert os.environ.get("some_test_environ") == "test_value" - assert os.environ.get("some_test_environ") == "test" - def test_rel_has_nofollow(self): - assert rel_has_nofollow("ugc nofollow") is True - assert rel_has_nofollow("ugc,nofollow") is True - assert rel_has_nofollow("ugc") is False - assert rel_has_nofollow("nofollow") is True - assert rel_has_nofollow("nofollowfoo") is False - assert rel_has_nofollow("foonofollow") is False - assert rel_has_nofollow("ugc, , nofollow") is True - # rel attribute values are ASCII case-insensitive per the HTML spec - assert rel_has_nofollow("NoFollow") is True - assert rel_has_nofollow("NOFOLLOW") is True - assert rel_has_nofollow("UGC NoFollow") is True - assert rel_has_nofollow("ugc,NoFollow") is True +def test_rel_has_nofollow() -> None: + assert rel_has_nofollow("ugc nofollow") is True + assert rel_has_nofollow("ugc,nofollow") is True + assert rel_has_nofollow("ugc") is False + assert rel_has_nofollow("nofollow") is True + assert rel_has_nofollow("nofollowfoo") is False + assert rel_has_nofollow("foonofollow") is False + assert rel_has_nofollow("ugc, , nofollow") is True + # rel attribute values are ASCII case-insensitive per the HTML spec + assert rel_has_nofollow("NoFollow") is True + assert rel_has_nofollow("NOFOLLOW") is True + assert rel_has_nofollow("UGC NoFollow") is True + assert rel_has_nofollow("ugc,NoFollow") is True diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 1acc3aac2..7343a135a 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -1,5 +1,8 @@ +from __future__ import annotations + import warnings from functools import partial +from typing import TYPE_CHECKING, Any from unittest import mock import pytest @@ -9,6 +12,11 @@ from scrapy.utils.misc import ( warn_on_generator_with_return_value, ) +if TYPE_CHECKING: + from collections.abc import Generator + + from scrapy import Spider + def _indentation_error(*args, **kwargs): raise IndentationError @@ -35,244 +43,239 @@ https://example.org yield url -def generator_that_returns_stuff(): +def generator_that_returns_stuff() -> Generator[int, None, int]: yield 1 yield 2 return 3 -class TestUtilsMisc: - @pytest.fixture - def mock_spider(self): - class MockSettings: - def __init__(self, settings_dict=None): - self.settings_dict = settings_dict or { - "WARN_ON_GENERATOR_RETURN_VALUE": True - } +@pytest.fixture +def mock_spider() -> Spider: + class MockSettings: + def __init__(self, settings_dict: dict[str, Any] | None = None): + self.settings_dict = settings_dict or { + "WARN_ON_GENERATOR_RETURN_VALUE": True + } - def getbool(self, name, default=False): - return self.settings_dict.get(name, default) + def getbool(self, name, default=False): + return self.settings_dict.get(name, default) - class MockSpider: - def __init__(self): - self.settings = MockSettings() + class MockSpider: + def __init__(self) -> None: + self.settings = MockSettings() - return MockSpider() + return MockSpider() # type: ignore[return-value] - def test_generators_return_something(self, mock_spider): - def f1(): - yield 1 - return 2 - def g1(): - yield 1 - return "asdf" +def test_generators_return_something(mock_spider): + def f1(): + yield 1 + return 2 - def h1(): - yield 1 + def g1(): + yield 1 + return "asdf" - def helper(): - return 0 + def h1(): + yield 1 - yield helper() - return 2 + def helper() -> int: + return 0 - def i1(): - """ - docstring - """ - url = """ -https://example.org + yield helper() + return 2 + + def i1(): """ - yield url - return 1 - - assert is_generator_with_return_value(top_level_return_something) - assert is_generator_with_return_value(f1) - assert is_generator_with_return_value(g1) - assert is_generator_with_return_value(h1) - assert is_generator_with_return_value(i1) - - with pytest.warns( - UserWarning, - match='The "MockSpider.top_level_return_something" method is a generator', - ): - warn_on_generator_with_return_value(mock_spider, top_level_return_something) - with pytest.warns( - UserWarning, match='The "MockSpider.f1" method is a generator' - ): - warn_on_generator_with_return_value(mock_spider, f1) - with pytest.warns( - UserWarning, match='The "MockSpider.g1" method is a generator' - ): - warn_on_generator_with_return_value(mock_spider, g1) - with pytest.warns( - UserWarning, match='The "MockSpider.h1" method is a generator' - ): - warn_on_generator_with_return_value(mock_spider, h1) - with pytest.warns( - UserWarning, match='The "MockSpider.i1" method is a generator' - ): - warn_on_generator_with_return_value(mock_spider, i1) - - def test_generators_return_none(self, mock_spider): - def f2(): - yield 1 - - def g2(): - yield 1 - - def h2(): - yield 1 - - def i2(): - yield 1 - yield from generator_that_returns_stuff() - - def j2(): - yield 1 - - def helper(): - return 0 - - yield helper() - - def k2(): - """ - docstring - """ - url = """ -https://example.org + docstring """ - yield url - - def l2(): - return - - assert not is_generator_with_return_value(top_level_return_none) - assert not is_generator_with_return_value(f2) - assert not is_generator_with_return_value(g2) - assert not is_generator_with_return_value(h2) - assert not is_generator_with_return_value(i2) - assert not is_generator_with_return_value(j2) # not recursive - assert not is_generator_with_return_value(k2) # not recursive - assert not is_generator_with_return_value(l2) - - with warnings.catch_warnings(): - warnings.simplefilter("error", UserWarning) - warn_on_generator_with_return_value(mock_spider, top_level_return_none) - warn_on_generator_with_return_value(mock_spider, f2) - warn_on_generator_with_return_value(mock_spider, g2) - warn_on_generator_with_return_value(mock_spider, h2) - warn_on_generator_with_return_value(mock_spider, i2) - warn_on_generator_with_return_value(mock_spider, j2) - warn_on_generator_with_return_value(mock_spider, k2) - warn_on_generator_with_return_value(mock_spider, l2) - - def test_generators_return_none_with_decorator(self, mock_spider): - def decorator(func): - def inner_func(): - func() - - return inner_func - - @decorator - def f3(): - yield 1 - - @decorator - def g3(): - yield 1 - - @decorator - def h3(): - yield 1 - - @decorator - def i3(): - yield 1 - yield from generator_that_returns_stuff() - - @decorator - def j3(): - yield 1 - - def helper(): - return 0 - - yield helper() - - @decorator - def k3(): - """ - docstring - """ - url = """ + url = """ https://example.org + """ + yield url + return 1 + + assert is_generator_with_return_value(top_level_return_something) + assert is_generator_with_return_value(f1) + assert is_generator_with_return_value(g1) + assert is_generator_with_return_value(h1) + assert is_generator_with_return_value(i1) + + with pytest.warns( + UserWarning, + match='The "MockSpider.top_level_return_something" method is a generator', + ): + warn_on_generator_with_return_value(mock_spider, top_level_return_something) + with pytest.warns(UserWarning, match='The "MockSpider.f1" method is a generator'): + warn_on_generator_with_return_value(mock_spider, f1) + with pytest.warns(UserWarning, match='The "MockSpider.g1" method is a generator'): + warn_on_generator_with_return_value(mock_spider, g1) + with pytest.warns(UserWarning, match='The "MockSpider.h1" method is a generator'): + warn_on_generator_with_return_value(mock_spider, h1) + with pytest.warns(UserWarning, match='The "MockSpider.i1" method is a generator'): + warn_on_generator_with_return_value(mock_spider, i1) + + +def test_generators_return_none(mock_spider): + def f2(): + yield 1 + + def g2(): + yield 1 + + def h2(): + yield 1 + + def i2(): + yield 1 + yield from generator_that_returns_stuff() + + def j2(): + yield 1 + + def helper() -> int: + return 0 + + yield helper() + + def k2(): """ - yield url + docstring + """ + url = """ +https://example.org + """ + yield url - @decorator - def l3(): - return + def l2(): + return - assert not is_generator_with_return_value(top_level_return_none) - assert not is_generator_with_return_value(f3) - assert not is_generator_with_return_value(g3) - assert not is_generator_with_return_value(h3) - assert not is_generator_with_return_value(i3) - assert not is_generator_with_return_value(j3) # not recursive - assert not is_generator_with_return_value(k3) # not recursive - assert not is_generator_with_return_value(l3) + assert not is_generator_with_return_value(top_level_return_none) + assert not is_generator_with_return_value(f2) + assert not is_generator_with_return_value(g2) + assert not is_generator_with_return_value(h2) + assert not is_generator_with_return_value(i2) + assert not is_generator_with_return_value(j2) # not recursive + assert not is_generator_with_return_value(k2) # not recursive + assert not is_generator_with_return_value(l2) - with warnings.catch_warnings(): - warnings.simplefilter("error", UserWarning) - warn_on_generator_with_return_value(mock_spider, top_level_return_none) - warn_on_generator_with_return_value(mock_spider, f3) - warn_on_generator_with_return_value(mock_spider, g3) - warn_on_generator_with_return_value(mock_spider, h3) - warn_on_generator_with_return_value(mock_spider, i3) - warn_on_generator_with_return_value(mock_spider, j3) - warn_on_generator_with_return_value(mock_spider, k3) - warn_on_generator_with_return_value(mock_spider, l3) + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) + warn_on_generator_with_return_value(mock_spider, f2) + warn_on_generator_with_return_value(mock_spider, g2) + warn_on_generator_with_return_value(mock_spider, h2) + warn_on_generator_with_return_value(mock_spider, i2) + warn_on_generator_with_return_value(mock_spider, j2) + warn_on_generator_with_return_value(mock_spider, k2) + warn_on_generator_with_return_value(mock_spider, l2) - @mock.patch( - "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error - ) - def test_indentation_error(self, mock_spider): - with pytest.warns(UserWarning, match="Unable to determine"): - warn_on_generator_with_return_value(mock_spider, top_level_return_none) - def test_partial(self): - def cb(arg1, arg2): - yield {} +def test_generators_return_none_with_decorator(mock_spider): + def decorator(func): + def inner_func(): + func() - partial_cb = partial(cb, arg1=42) - assert not is_generator_with_return_value(partial_cb) + return inner_func - def test_warn_on_generator_with_return_value_settings_disabled(self): - class MockSettings: - def __init__(self, settings_dict=None): - self.settings_dict = settings_dict or {} + @decorator + def f3(): + yield 1 - def getbool(self, name, default=False): - return self.settings_dict.get(name, default) + @decorator + def g3(): + yield 1 - class MockSpider: - def __init__(self): - self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False}) + @decorator + def h3(): + yield 1 - spider = MockSpider() + @decorator + def i3(): + yield 1 + yield from generator_that_returns_stuff() - def gen_with_return(): - yield 1 - return "value" + @decorator + def j3(): + yield 1 - with warnings.catch_warnings(): - warnings.simplefilter("error", UserWarning) - warn_on_generator_with_return_value(spider, gen_with_return) + def helper() -> int: + return 0 - spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True + yield helper() - with pytest.warns(UserWarning, match="is a generator"): - warn_on_generator_with_return_value(spider, gen_with_return) + @decorator + def k3(): + """ + docstring + """ + url = """ +https://example.org + """ + yield url + + @decorator + def l3(): + return + + assert not is_generator_with_return_value(top_level_return_none) + assert not is_generator_with_return_value(f3) + assert not is_generator_with_return_value(g3) + assert not is_generator_with_return_value(h3) + assert not is_generator_with_return_value(i3) + assert not is_generator_with_return_value(j3) # not recursive + assert not is_generator_with_return_value(k3) # not recursive + assert not is_generator_with_return_value(l3) + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) + warn_on_generator_with_return_value(mock_spider, f3) + warn_on_generator_with_return_value(mock_spider, g3) + warn_on_generator_with_return_value(mock_spider, h3) + warn_on_generator_with_return_value(mock_spider, i3) + warn_on_generator_with_return_value(mock_spider, j3) + warn_on_generator_with_return_value(mock_spider, k3) + warn_on_generator_with_return_value(mock_spider, l3) + + +@mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error) +def test_indentation_error(mock_spider): + with pytest.warns(UserWarning, match="Unable to determine"): + warn_on_generator_with_return_value(mock_spider, top_level_return_none) + + +def test_partial() -> None: + def cb(arg1, arg2): + yield {} + + partial_cb = partial(cb, arg1=42) + assert not is_generator_with_return_value(partial_cb) + + +def test_warn_on_generator_with_return_value_settings_disabled() -> None: + class MockSettings: + def __init__(self, settings_dict: dict[str, Any] | None = None): + self.settings_dict = settings_dict or {} + + def getbool(self, name, default=False): + return self.settings_dict.get(name, default) + + class MockSpider: + def __init__(self) -> None: + self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False}) + + spider = MockSpider() + + def gen_with_return(): + yield 1 + return "value" + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + warn_on_generator_with_return_value(spider, gen_with_return) # type: ignore[arg-type] + + spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True + + with pytest.warns(UserWarning, match="is a generator"): + warn_on_generator_with_return_value(spider, gen_with_return) # type: ignore[arg-type] diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 5333a55cb..7eade6463 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -1,14 +1,20 @@ +from __future__ import annotations + import os from pathlib import Path +from typing import TYPE_CHECKING import pytest from scrapy.utils.misc import set_environ from scrapy.utils.project import data_path, get_project_settings +if TYPE_CHECKING: + from collections.abc import Generator + @pytest.fixture -def proj_path(tmp_path): +def proj_path(tmp_path: Path) -> Generator[Path]: prev_dir = Path.cwd() project_dir = tmp_path @@ -21,7 +27,7 @@ def proj_path(tmp_path): os.chdir(prev_dir) -def test_data_path_outside_project(): +def test_data_path_outside_project() -> None: assert str(Path(".scrapy", "somepath")) == data_path("somepath") abspath = str(Path(os.path.sep, "absolute", "path")) assert abspath == data_path(abspath) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 2e8047e2d..099b5ccc2 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -4,7 +4,7 @@ import functools import operator import platform import sys -from typing import TYPE_CHECKING, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar import pytest @@ -22,8 +22,7 @@ from scrapy.utils.python import ( from tests.utils.decorators import coroutine_test if TYPE_CHECKING: - from collections.abc import Iterable, Mapping - + from collections.abc import AsyncIterator, Iterable, Mapping _KT = TypeVar("_KT") _VT = TypeVar("_VT") @@ -31,22 +30,22 @@ _VT = TypeVar("_VT") class TestMutableAsyncChain: @staticmethod - async def g1(): + async def g1() -> AsyncIterator[int]: for i in range(3): yield i @staticmethod - async def g2(): + async def g2() -> AsyncIterator[int]: return yield @staticmethod - async def g3(): + async def g3() -> AsyncIterator[int]: for i in range(7, 10): yield i @staticmethod - async def g4(): + async def g4() -> AsyncIterator[int]: for i in range(3, 5): yield i 1 / 0 @@ -85,7 +84,7 @@ class TestToUnicode: def test_converting_a_strange_object_should_raise_type_error(self): with pytest.raises(TypeError): - to_unicode(423) + to_unicode(423) # type: ignore[arg-type] def test_errors_argument(self): assert to_unicode(b"a\xedb", "utf-8", errors="replace") == "a\ufffdb" @@ -103,7 +102,7 @@ class TestToBytes: def test_converting_a_strange_object_should_raise_type_error(self): with pytest.raises(TypeError): - to_bytes(pytest) + to_bytes(pytest) # type: ignore[arg-type] def test_errors_argument(self): assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b" @@ -112,10 +111,10 @@ class TestToBytes: def test_memoizemethod_noargs(): class A: @memoizemethod_noargs - def cached(self): + def cached(self) -> object: return object() - def noncached(self): + def noncached(self) -> object: return object() a = A() @@ -150,7 +149,7 @@ def test_get_func_args(): pass class A: - def __init__(self, a, b, c): + def __init__(self, a: int, b: int, c: int): pass def method(self, a, b, c): @@ -191,6 +190,25 @@ def test_get_func_args(): ] +@pytest.mark.skipif( + sys.version_info < (3, 14), + reason="annotations are only lazily evaluated since Python 3.14 (PEP 649)", +) +def test_get_func_args_unresolvable_annotations(): + # dont_inherit=True, or the module's future import stringizes the annotations + namespace: dict[str, Any] = {} + exec( # pylint: disable=exec-used + compile( + "def f(a: OnlyAtTypeCheckingTime, b: int = 1) -> OnlyAtTypeCheckingTime: pass", + "", + "exec", + dont_inherit=True, + ), + namespace, + ) + assert get_func_args(namespace["f"]) == ["a", "b"] + + @pytest.mark.parametrize( ("value", "expected"), [ diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py index 7d39a478e..44cb5c306 100644 --- a/tests/test_utils_reactor.py +++ b/tests/test_utils_reactor.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import pytest diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 1642a932b..935447bc4 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -2,7 +2,7 @@ from __future__ import annotations import json from hashlib import sha1 -from typing import Any +from typing import TYPE_CHECKING, Any, Protocol from weakref import WeakKeyDictionary import pytest @@ -17,6 +17,9 @@ from scrapy.utils.request import ( ) from scrapy.utils.test import get_crawler +if TYPE_CHECKING: + from collections.abc import Iterable + @pytest.mark.parametrize( ("r", "expected"), @@ -56,8 +59,18 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None: request_httprepr(r) +class _FingerprintFunction(Protocol): + def __call__( + self, + request: Request, + *, + include_headers: Iterable[bytes | str] | None = None, + keep_fragments: bool = False, + ) -> bytes: ... + + class TestFingerprint: - function: staticmethod[[Request], bytes] = staticmethod(fingerprint) + function: _FingerprintFunction = staticmethod(fingerprint) cache: ( WeakKeyDictionary[ Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes] @@ -261,6 +274,7 @@ class TestRequestFingerprinter: def test_fingerprint(self): crawler = get_crawler() request = Request("https://example.com") + assert crawler.request_fingerprinter assert crawler.request_fingerprinter.fingerprint(request) == fingerprint( request ) @@ -277,6 +291,7 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) + assert crawler.request_fingerprinter r1 = Request("http://www.example.com", headers={"X-ID": "1"}) fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com", headers={"X-ID": "2"}) @@ -285,9 +300,9 @@ class TestCustomRequestFingerprinter: def test_dont_canonicalize(self): class RequestFingerprinter: - cache = WeakKeyDictionary() + cache: WeakKeyDictionary[Request, bytes] = WeakKeyDictionary() - def fingerprint(self, request): + def fingerprint(self, request: Request) -> bytes: if request not in self.cache: fp = sha1() fp.update(to_bytes(request.url)) @@ -299,6 +314,7 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) + assert crawler.request_fingerprinter r1 = Request("http://www.example.com?a=1&a=2") fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com?a=2&a=1") @@ -317,6 +333,7 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) + assert crawler.request_fingerprinter r1 = Request("http://www.example.com") fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com", meta={"fingerprint": "a"}) @@ -348,6 +365,7 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) + assert crawler.request_fingerprinter request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) assert fingerprint == settings["FINGERPRINT"] @@ -457,3 +475,14 @@ class TestRequestToCurl: " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=1'" ) self._test_request(request_object, expected_curl_command) + + def test_request_to_curl_method(self) -> None: + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + 'curl -X POST https://www.httpbin.org/post --data-raw \'{"foo": "bar"}\'' + ) + assert request_object.to_curl() == expected_curl_command diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 146cdb802..608b2bbd9 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from pathlib import Path from time import process_time from urllib.parse import urlparse @@ -15,7 +17,7 @@ from scrapy.utils.response import ( ) -def _read_browser_output(burl: str): +def _read_browser_output(burl: str) -> bytes: path = urlparse(burl).path if not path or not Path(path).exists(): path = burl.replace("file://", "") @@ -224,7 +226,7 @@ def test_open_in_browser_redos_head(): (b"real", b"real"), ], ) -def test_remove_html_comments(input_body, output_body): +def test_remove_html_comments(input_body: bytes, output_body: bytes) -> None: assert _remove_html_comments(input_body) == output_body diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py index 2e6a790f8..2702c2cce 100644 --- a/tests/test_utils_serialize.py +++ b/tests/test_utils_serialize.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import dataclasses import datetime import json diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index dc615eca6..3109bc149 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -1,23 +1,22 @@ from __future__ import annotations import asyncio -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import pytest from pydispatch import dispatcher -from testfixtures import LogCapture from twisted.internet import defer from twisted.python.failure import Failure from scrapy.utils.asyncio import call_later -from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.defer import deferred_from_coro, ensure_awaitable from scrapy.utils.signal import ( send_catch_log, send_catch_log_async, send_catch_log_deferred, ) from scrapy.utils.test import get_from_asyncio_queue -from tests.utils.decorators import inline_callbacks_test +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from collections.abc import Callable @@ -27,25 +26,22 @@ class TestSendCatchLog: # whether the function being tested returns exceptions or failures returns_exceptions: bool = False - @inline_callbacks_test - def test_send_catch_log(self): + @coroutine_test + async def test_send_catch_log(self, caplog: pytest.LogCaptureFixture) -> None: test_signal = object() handlers_called: set[Callable[..., None]] = set() dispatcher.connect(self.error_handler, signal=test_signal) dispatcher.connect(self.ok_handler, signal=test_signal) - with LogCapture() as log: - result = yield defer.maybeDeferred( - self._get_result, - test_signal, - arg="test", - handlers_called=handlers_called, - ) + caplog.clear() + result = await ensure_awaitable( + self._get_result(test_signal, arg="test", handlers_called=handlers_called) + ) assert self.error_handler in handlers_called assert self.ok_handler in handlers_called - assert len(log.records) == 1 - record = log.records[0] + assert len(caplog.records) == 1 + record = caplog.records[0] assert "error_handler" in record.getMessage() assert record.levelname == "ERROR" assert result[0][0] == self.error_handler # pylint: disable=comparison-with-callable @@ -57,7 +53,7 @@ class TestSendCatchLog: dispatcher.disconnect(self.error_handler, signal=test_signal) dispatcher.disconnect(self.ok_handler, signal=test_signal) - def _get_result(self, signal, *a, **kw): + def _get_result(self, signal: Any, *a: Any, **kw: Any) -> Any: return send_catch_log(signal, *a, **kw) def error_handler(self, arg, handlers_called): @@ -72,7 +68,7 @@ class TestSendCatchLog: @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestSendCatchLogDeferred(TestSendCatchLog): - def _get_result(self, signal, *a, **kw): + def _get_result(self, signal: Any, *a: Any, **kw: Any) -> Any: return send_catch_log_deferred(signal, *a, **kw) @@ -137,14 +133,15 @@ class TestSendCatchLogAsyncAsyncio(TestSendCatchLogAsync): class TestSendCatchLog2: - def test_error_logged_if_deferred_not_supported(self): + def test_error_logged_if_deferred_not_supported( + self, caplog: pytest.LogCaptureFixture + ) -> None: def test_handler(): return defer.Deferred() test_signal = object() dispatcher.connect(test_handler, test_signal) - with LogCapture() as log: - send_catch_log(test_signal) - assert len(log.records) == 1 - assert "Cannot return deferreds from signal handler" in str(log) + send_catch_log(test_signal) + assert len(caplog.records) == 1 + assert "Cannot return deferreds from signal handler" in caplog.text dispatcher.disconnect(test_handler, test_signal) diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index ac57e1739..9f2dce6d6 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from scrapy.exceptions import ScrapyDeprecationWarning diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 4515ce36e..076edf3db 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -1,7 +1,14 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + from scrapy.utils.template import render_templatefile +if TYPE_CHECKING: + from pathlib import Path -def test_simple_render(tmp_path): + +def test_simple_render(tmp_path: Path) -> None: context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"} template = "from ${project_name}.spiders.${name} import ${classname}" rendered = "from proj.spiders.spi import TheSpider" diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 2334c76e9..5458aa603 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import sys from io import StringIO from unittest import mock @@ -46,13 +48,13 @@ Bar 1 oldest: 0s ago @mock.patch("sys.stdout", new_callable=StringIO) -def test_print_live_refs_empty(stdout): +def test_print_live_refs_empty(stdout: StringIO) -> None: trackref.print_live_refs() assert stdout.getvalue() == "Live References\n\n\n" @mock.patch("sys.stdout", new_callable=StringIO) -def test_print_live_refs_with_objects(stdout): +def test_print_live_refs_with_objects(stdout: StringIO) -> None: o1 = Foo() # noqa: F841 trackref.print_live_refs() assert ( diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 5b98131a1..d9d162c23 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from scrapy.linkextractors import IGNORED_EXTENSIONS @@ -191,7 +193,7 @@ def test_guess_scheme(url: str, expected: str): ), ], ) -def test_guess_scheme_skipped(url: str, expected: str, reason: str): +def test_guess_scheme_skipped(url: str, expected: str, reason: str) -> None: pytest.skip(reason) diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index b0632a7ea..b27c5ade7 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import os from pathlib import Path from typing import TYPE_CHECKING @@ -8,8 +7,6 @@ from typing import TYPE_CHECKING from twisted.internet.defer import Deferred from scrapy.settings import Settings, default_settings -from scrapy.utils.asyncio import is_asyncio_available -from scrapy.utils.defer import maybe_deferred_to_future if TYPE_CHECKING: from collections.abc import Callable @@ -23,13 +20,6 @@ def twisted_sleep(seconds: float): return d -async def async_sleep(seconds: float) -> None: - if is_asyncio_available(): - await asyncio.sleep(seconds) - else: - await maybe_deferred_to_future(twisted_sleep(seconds)) - - def get_script_run_env() -> dict[str, str]: """Return a OS environment dict suitable to run scripts shipped with tests.""" diff --git a/tests/utils/bases/__init__.py b/tests/utils/bases/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/utils/base_commands.py b/tests/utils/bases/commands.py similarity index 80% rename from tests/utils/base_commands.py rename to tests/utils/bases/commands.py index 594544c83..55ef686fa 100644 --- a/tests/utils/base_commands.py +++ b/tests/utils/bases/commands.py @@ -32,3 +32,9 @@ class TestProjectBase: proj_path = tmp_path / self.project_name copytree(_proj_path_cached, proj_path) return proj_path + + @staticmethod + def _append_settings(proj_mod_path: Path, text: str) -> None: + """Add text to the end of the project settings.py.""" + with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f: + f.write(text) diff --git a/tests/test_downloader_handlers_http_base.py b/tests/utils/bases/download_handlers_http.py similarity index 99% rename from tests/test_downloader_handlers_http_base.py rename to tests/utils/bases/download_handlers_http.py index 29e1d9757..e44f9bcb8 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/utils/bases/download_handlers_http.py @@ -1,5 +1,3 @@ -"""Base classes for HTTP download handler tests.""" - from __future__ import annotations import gzip diff --git a/tests/utils/bases/engine.py b/tests/utils/bases/engine.py new file mode 100644 index 000000000..1cabbb6ef --- /dev/null +++ b/tests/utils/bases/engine.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from itemadapter import ItemAdapter + +from scrapy import signals +from tests import get_testdata + +if TYPE_CHECKING: + from tests.utils.engine import CrawlerRun + + +class TestEngineBase: + @staticmethod + def _assert_visited_urls(run: CrawlerRun) -> None: + must_be_visited = [ + "/static/", + "/redirect", + "/redirected", + "/static/item1.html", + "/static/item2.html", + "/static/item999.html", + ] + urls_visited = {rp[0].url for rp in run.respplug} + urls_expected = {run.geturl(p) for p in must_be_visited} + assert urls_expected <= urls_visited, ( + f"URLs not visited: {list(urls_expected - urls_visited)}" + ) + + @staticmethod + def _assert_scheduled_requests(run: CrawlerRun, count: int) -> None: + assert len(run.reqplug) == count + + paths_expected = [ + "/static/item999.html", + "/static/item2.html", + "/static/item1.html", + ] + + urls_requested = {rq[0].url for rq in run.reqplug} + urls_expected = {run.geturl(p) for p in paths_expected} + assert urls_expected <= urls_requested + scheduled_requests_count = len(run.reqplug) + dropped_requests_count = len(run.reqdropped) + responses_count = len(run.respplug) + assert scheduled_requests_count == dropped_requests_count + responses_count + assert len(run.reqreached) == responses_count + + @staticmethod + def _assert_dropped_requests(run: CrawlerRun) -> None: + assert len(run.reqdropped) == 1 + + @staticmethod + def _assert_downloaded_responses(run: CrawlerRun, count: int) -> None: + # response tests + assert len(run.respplug) == count + assert len(run.reqreached) == count + + for response, _ in run.respplug: + if run.getpath(response.url) == "/static/item999.html": + assert response.status == 404 + if run.getpath(response.url) == "/redirect": + assert response.status == 302 + + @staticmethod + def _assert_items_error(run: CrawlerRun) -> None: + assert len(run.itemerror) == 2 + for item, response, spider, failure in run.itemerror: + assert failure.value.__class__ is ZeroDivisionError + assert spider == run.crawler.spider + + assert item["url"] == response.url + if "item1.html" in item["url"]: + assert item["name"] == "Item 1 name" + assert item["price"] == "100" + if "item2.html" in item["url"]: + assert item["name"] == "Item 2 name" + assert item["price"] == "200" + + @staticmethod + def _assert_scraped_items(run: CrawlerRun) -> None: + assert len(run.itemresp) == 2 + for item_, response in run.itemresp: + item = ItemAdapter(item_) + assert item["url"] == response.url + if "item1.html" in item["url"]: + assert item["name"] == "Item 1 name" + assert item["price"] == "100" + if "item2.html" in item["url"]: + assert item["name"] == "Item 2 name" + assert item["price"] == "200" + + @staticmethod + def _assert_headers_received(run: CrawlerRun) -> None: + for headers in run.headers.values(): + assert b"Server" in headers + assert headers[b"Server"] + assert b"TwistedWeb" in headers[b"Server"] + assert b"Date" in headers + assert b"Content-Type" in headers + + @staticmethod + def _assert_bytes_received(run: CrawlerRun) -> None: + assert len(run.bytes) == 9 + for request, data in run.bytes.items(): + joined_data = b"".join(data) + if run.getpath(request.url) == "/static/": + assert joined_data == get_testdata("test_site", "index.html") + elif run.getpath(request.url) == "/static/item1.html": + assert joined_data == get_testdata("test_site", "item1.html") + elif run.getpath(request.url) == "/static/item2.html": + assert joined_data == get_testdata("test_site", "item2.html") + elif run.getpath(request.url) == "/redirected": + assert joined_data == b"Redirected here" + elif run.getpath(request.url) == "/redirect": + assert ( + joined_data == b"\n\n" + b" \n" + b' \n' + b" \n" + b' \n' + b' click here\n' + b" \n" + b"\n" + ) + elif run.getpath(request.url) == "/static/item999.html": + assert ( + joined_data == b"\n\n" + b" 404 - No Such Resource\n" + b" \n" + b"

No Such Resource

\n" + b"

File not found.

\n" + b" \n" + b"\n" + ) + elif run.getpath(request.url) == "/numbers": + # signal was fired multiple times + assert len(data) > 1 + # bytes were received in order + numbers = [str(x).encode("utf8") for x in range(2**18)] + assert joined_data == b"".join(numbers) + + @staticmethod + def _assert_signals_caught(run: CrawlerRun) -> None: + assert signals.engine_started in run.signals_caught + assert signals.engine_stopped in run.signals_caught + assert signals.spider_opened in run.signals_caught + assert signals.spider_idle in run.signals_caught + assert signals.spider_closed in run.signals_caught + assert signals.headers_received in run.signals_caught + + assert {"spider": run.crawler.spider} == run.signals_caught[ + signals.spider_opened + ] + assert {"spider": run.crawler.spider} == run.signals_caught[signals.spider_idle] + assert { + "spider": run.crawler.spider, + "reason": "finished", + } == run.signals_caught[signals.spider_closed] diff --git a/tests/utils/bases/feedexport.py b/tests/utils/bases/feedexport.py new file mode 100644 index 000000000..f03ac1fd4 --- /dev/null +++ b/tests/utils/bases/feedexport.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import random +import shutil +import tempfile +from abc import ABC, abstractmethod +from pathlib import Path +from string import ascii_letters, digits +from typing import IO, TYPE_CHECKING, Any + +import scrapy +from scrapy import Spider +from tests.mockserver.http import MockServer + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable + + +class TestFeedExportBase(ABC): + mockserver: MockServer + + def _random_temp_filename(self, inter_dir="") -> Path: + chars = [random.choice(ascii_letters + digits) for _ in range(15)] + filename = "".join(chars) + return Path(self.temp_dir, inter_dir, filename) + + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() # pylint: disable=unnecessary-dunder-call + + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) + + def setup_method(self): + self.temp_dir = tempfile.mkdtemp() + + def teardown_method(self): + shutil.rmtree(self.temp_dir, ignore_errors=True) + + async def exported_data( + self, items: Iterable[Any], settings: dict[str, Any] + ) -> dict[str, Any]: + """ + Return exported data which a spider yielding ``items`` would return. + """ + + class TestSpider(scrapy.Spider): + name = "testspider" + + def parse(self, response): + yield from items + + return await self.run_and_export(TestSpider, settings) + + async def exported_no_data(self, settings: dict[str, Any]) -> dict[str, Any]: + """ + Return exported data which a spider yielding no ``items`` would return. + """ + + class TestSpider(scrapy.Spider): + name = "testspider" + + def parse(self, response): + pass + + return await self.run_and_export(TestSpider, settings) + + async def assertExported( + self, + items: Iterable[Any], + header: Iterable[str], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + await self.assertExportedCsv(items, header, rows, settings) + await self.assertExportedJsonLines(items, rows, settings) + await self.assertExportedXml(items, rows, settings) + await self.assertExportedPickle(items, rows, settings) + await self.assertExportedMarshal(items, rows, settings) + await self.assertExportedMultiple(items, rows, settings) + + async def assertExportedCsv( # noqa: B027 + self, + items: Iterable[Any], + header: Iterable[str], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedJsonLines( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedXml( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedMultiple( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedPickle( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedMarshal( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + @abstractmethod + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, Any]: + pass + + def _load_until_eof( + self, data: bytes, load_func: Callable[[IO[bytes]], Any] + ) -> list[Any]: + result: list[Any] = [] + with tempfile.TemporaryFile() as temp: + temp.write(data) + temp.seek(0) + while True: + try: + result.append(load_func(temp)) + except EOFError: + break + return result diff --git a/tests/utils/bases/http_request.py b/tests/utils/bases/http_request.py new file mode 100644 index 000000000..2b712f3ab --- /dev/null +++ b/tests/utils/bases/http_request.py @@ -0,0 +1,508 @@ +import warnings +from abc import ABC, abstractmethod +from typing import Any + +import pytest +from twisted.python.failure import Failure + +from scrapy.http import Headers, Request, Response +from scrapy.http.request import NO_CALLBACK +from scrapy.utils.request import request_to_curl + + +class TestRequestBase(ABC): + default_method = "GET" + default_headers: dict[bytes, list[bytes]] = {} + default_meta: dict[str, Any] = {} + + @property + @abstractmethod + def request_class(self) -> type[Request]: + raise NotImplementedError + + def test_init(self): + # Request requires url in the __init__ method + with pytest.raises(TypeError): + self.request_class() # type: ignore[call-arg] + + # url argument must be basestring + with pytest.raises(TypeError): + self.request_class(123) # type: ignore[arg-type] + + # priority argument must be an integer + with pytest.raises(TypeError, match="Request priority not an integer"): + self.request_class("http://www.example.com", priority="1") # type: ignore[arg-type] + + r = self.request_class("http://www.example.com") + assert isinstance(r.url, str) + assert r.url == "http://www.example.com" + assert r.method == self.default_method + + assert isinstance(r.headers, Headers) + assert r.headers == self.default_headers + assert r.meta == self.default_meta + + meta = {"lala": "lolo"} + headers = {b"caca": b"coco"} + r = self.request_class( + "http://www.example.com", meta=meta, headers=headers, body="a body" + ) + + assert r.meta is not meta + assert r.meta == meta + assert r.headers is not headers + assert r.headers[b"caca"] == b"coco" + + def test_url_scheme(self): + # This test passes by not raising any (ValueError) exception + self.request_class("http://example.org") + self.request_class("https://example.org") + self.request_class("s3://example.org") + self.request_class("ftp://example.org") + self.request_class("about:config") + self.request_class("data:,Hello%2C%20World!") + + def test_url_no_scheme(self): + msg = "Missing scheme in request url:" + with pytest.raises(ValueError, match=msg): + self.request_class("foo") + with pytest.raises(ValueError, match=msg): + self.request_class("/foo/") + with pytest.raises(ValueError, match=msg): + self.request_class("/foo:bar") + + def test_headers(self): + # Different ways of setting headers attribute + url = "http://www.scrapy.org" + headers = {b"Accept": "gzip", b"Custom-Header": "nothing to tell you"} + r = self.request_class(url=url, headers=headers) + p = self.request_class(url=url, headers=r.headers) + + assert r.headers == p.headers + assert r.headers is not headers + assert p.headers is not r.headers + + # headers must not be unicode + h = Headers({"key1": "val1", "key2": "val2"}) + h["newkey"] = "newval" + for k, v in h.items(): + assert isinstance(k, bytes) + for s in v: + assert isinstance(s, bytes) + + def test_eq(self): + url = "http://www.scrapy.org" + r1 = self.request_class(url=url) + r2 = self.request_class(url=url) + assert r1 != r2 + + set_ = set() + set_.add(r1) + set_.add(r2) + assert len(set_) == 2 + + def test_url(self): + r = self.request_class(url="http://www.scrapy.org/path") + assert r.url == "http://www.scrapy.org/path" + + def test_url_quoting(self): + r = self.request_class(url="http://www.scrapy.org/blank%20space") + assert r.url == "http://www.scrapy.org/blank%20space" + r = self.request_class(url="http://www.scrapy.org/blank space") + assert r.url == "http://www.scrapy.org/blank%20space" + + def test_url_encoding(self): + r = self.request_class(url="http://www.scrapy.org/price/£") + assert r.url == "http://www.scrapy.org/price/%C2%A3" + + def test_url_encoding_other(self): + # encoding affects only query part of URI, not path + # path part should always be UTF-8 encoded before percent-escaping + r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8") + assert r.url == "http://www.scrapy.org/price/%C2%A3" + + r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1") + assert r.url == "http://www.scrapy.org/price/%C2%A3" + + def test_url_encoding_query(self): + r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ") + assert r1.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" + + # should be same as above + r2 = self.request_class( + url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8" + ) + assert r2.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" + + def test_url_encoding_query_latin1(self): + # encoding is used for encoding query-string before percent-escaping; + # path is still UTF-8 encoded before percent-escaping + r3 = self.request_class( + url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1" + ) + assert r3.url == "http://www.scrapy.org/price/%C2%B5?currency=%A3" + + def test_url_encoding_nonutf8_untouched(self): + # percent-escaping sequences that do not match valid UTF-8 sequences + # should be kept untouched (just upper-cased perhaps) + # + # See https://datatracker.ietf.org/doc/html/rfc3987#section-3.2 + # + # "Conversions from URIs to IRIs MUST NOT use any character encoding + # other than UTF-8 in steps 3 and 4, even if it might be possible to + # guess from the context that another character encoding than UTF-8 was + # used in the URI. For example, the URI + # "http://www.example.org/r%E9sum%E9.html" might with some guessing be + # interpreted to contain two e-acute characters encoded as iso-8859-1. + # It must not be converted to an IRI containing these e-acute + # characters. Otherwise, in the future the IRI will be mapped to + # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different + # URI from "http://www.example.org/r%E9sum%E9.html". + r1 = self.request_class(url="http://www.scrapy.org/price/%a3") + assert r1.url == "http://www.scrapy.org/price/%a3" + + r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + assert r2.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" + + r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3") + assert r3.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" + + r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") + assert r4.url == "http://www.example.org/r%E9sum%E9.html" + + def test_url_verbatim(self): + r = self.request_class( + url="http://www.scrapy.org/price/£", + meta={"verbatim_url": True}, + ) + assert r.url == "http://www.scrapy.org/price/£" + + r = self.request_class( + url="http://www.scrapy.org/blank space", + meta={"verbatim_url": True}, + ) + assert r.url == "http://www.scrapy.org/blank space" + + def test_body(self): + r1 = self.request_class(url="http://www.example.com/") + assert r1.body == b"" + + r2 = self.request_class(url="http://www.example.com/", body=b"") + assert isinstance(r2.body, bytes) + assert r2.encoding == "utf-8" # default encoding + + r3 = self.request_class( + url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8" + ) + assert isinstance(r3.body, bytes) + assert r3.body == b"Price: \xc2\xa3100" + + r4 = self.request_class( + url="http://www.example.com/", body="Price: \xa3100", encoding="latin1" + ) + assert isinstance(r4.body, bytes) + assert r4.body == b"Price: \xa3100" + + def test_copy(self): + """Test Request copy""" + + def somecallback(response: Response) -> None: + pass + + def someerrback(failure: Failure) -> None: + pass + + r1 = self.request_class( + "http://www.example.com", + flags=["f1", "f2"], + callback=somecallback, + errback=someerrback, + ) + r1.meta["foo"] = "bar" + r1.cb_kwargs["key"] = "value" + r2 = r1.copy() + + # make sure callbaclks are copied + assert r1.callback is somecallback + assert r1.errback is someerrback + assert r2.callback is r1.callback + assert r2.errback is r1.errback + + # make sure flags list is shallow copied + assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" + assert r1.flags == r2.flags + + # make sure cb_kwargs dict is shallow copied + assert r1.cb_kwargs is not r2.cb_kwargs, ( + "cb_kwargs must be a shallow copy, not identical" + ) + assert r1.cb_kwargs == r2.cb_kwargs + + # make sure meta dict is shallow copied + assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" + assert r1.meta == r2.meta + + # make sure headers attribute is shallow copied + assert r1.headers is not r2.headers, ( + "headers must be a shallow copy, not identical" + ) + assert r1.headers == r2.headers + assert r1.encoding == r2.encoding + assert r1.dont_filter == r2.dont_filter + + # Request.body can be identical since it's an immutable object (str) + + def test_copy_inherited_classes(self): + """Test Request children copies preserve their class""" + + class CustomRequest(self.request_class): # type: ignore[misc,name-defined] + pass + + r1 = CustomRequest("http://www.example.com") + r2 = r1.copy() + + assert isinstance(r2, CustomRequest) + + def test_replace(self): + """Test Request.replace() method""" + r1 = self.request_class("http://www.example.com", method="GET") + hdrs = Headers(r1.headers) + hdrs[b"key"] = b"value" + r2 = r1.replace(method="POST", body="New body", headers=hdrs) + assert r1.url == r2.url + assert (r1.method, r2.method) == ("GET", "POST") + assert (r1.body, r2.body) == (b"", b"New body") + assert (r1.headers, r2.headers) == (self.default_headers, hdrs) + + # Empty attributes (which may fail if not compared properly) + r3 = self.request_class( + "http://www.example.com", meta={"a": 1}, dont_filter=True + ) + r4 = r3.replace( + url="http://www.example.com/2", body=b"", meta={}, dont_filter=False + ) + assert r4.url == "http://www.example.com/2" + assert r4.body == b"" + assert r4.meta == {} + assert r4.dont_filter is False + + # the cls argument allows changing the resulting class + custom_request_cls: type[Request] = type( + "CustomRequest", (self.request_class,), {} + ) + r5 = r1.replace(cls=custom_request_cls) + assert isinstance(r5, custom_request_cls) + assert r5.url == r1.url + + def test_method_always_str(self): + r = self.request_class("http://www.example.com", method="POST") + assert isinstance(r.method, str) + + def test_immutable_attributes(self): + r = self.request_class("http://example.com") + with pytest.raises(AttributeError): + r.url = "http://example2.com" # type: ignore[misc] + with pytest.raises(AttributeError): + r.body = "xxx" # type: ignore[misc,assignment] + + def test_callback_and_errback(self): + def a_callback(response: Response) -> None: + pass + + def an_errback(failure: Failure) -> None: + pass + + r1 = self.request_class("http://example.com") + assert r1.callback is None + assert r1.errback is None + + r2 = self.request_class("http://example.com", callback=a_callback) + assert r2.callback is a_callback + assert r2.errback is None + + r3 = self.request_class("http://example.com", errback=an_errback) + assert r3.callback is None + assert r3.errback is an_errback + + r4 = self.request_class( + url="http://example.com", + callback=a_callback, + errback=an_errback, + ) + assert r4.callback is a_callback + assert r4.errback is an_errback + + r5 = self.request_class( + url="http://example.com", + callback=NO_CALLBACK, + errback=NO_CALLBACK, + ) + assert r5.callback is NO_CALLBACK + assert r5.errback is NO_CALLBACK # type: ignore[comparison-overlap] + + def test_callback_and_errback_type(self): + with pytest.raises(TypeError): + self.request_class("http://example.com", callback="a_function") # type: ignore[arg-type] + with pytest.raises(TypeError): + self.request_class("http://example.com", errback="a_function") # type: ignore[arg-type] + with pytest.raises(TypeError): + self.request_class( + url="http://example.com", + callback="a_function", # type: ignore[arg-type] + errback="a_function", # type: ignore[arg-type] + ) + + def test_setters(self): + request = self.request_class("http://example.com") + + request.flags = ["f1"] + assert request.flags == ["f1"] + + request.cookies = {"sid": "1"} + assert request.cookies == {"sid": "1"} + + headers = Headers({b"X-Test": b"1"}) + request.headers = headers + assert request._headers is headers + request.headers = {b"A": b"b"} + assert isinstance(request.headers, Headers) + assert request._headers[b"A"] == b"b" + + def test_setter_mutable_lazy_loading(self): + """Mutable attributes are set internally to None only until they are + read, then they always return the same falsy instance of the + corresponding mutable structure. + + Setting them to None causes the next read to return a different object. + """ + + request = self.request_class("http://example.com") + + assert request._flags is None + assert request.flags == [] + assert request.flags is request.flags + assert request._flags == [] + original_flags = request.flags + request.flags = None + assert request._flags is None + assert request.flags == [] + assert request.flags is not original_flags + + assert request._cookies is None + assert request.cookies == {} + assert request.cookies is request.cookies + assert request._cookies == {} + original_cookies = request.cookies + request.cookies = None + assert request._cookies is None + assert request.cookies == {} + assert request.cookies is not original_cookies + + if self.default_headers: + assert request._headers == self.default_headers + assert request._headers is not self.default_headers + assert request.headers == self.default_headers + else: + assert request._headers is None + assert request.headers == {} + assert request.headers is request.headers + assert isinstance(request.headers, Headers) + assert isinstance(request._headers, Headers) + original_headers = request.headers + request.headers = None + assert request._headers is None + assert request.headers == {} + assert request._headers == {} + assert request.headers is not original_headers + + def test_no_callback(self): + with pytest.raises(RuntimeError): + NO_CALLBACK() + + def test_from_curl(self): + # Note: more curated tests regarding curl conversion are in + # `test_utils_curl.py` + curl_command = ( + "curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique" + "_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique" + "_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H" + " 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q" + "=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use" + "r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM" + "L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S" + "afari/537.36' -H 'Content-Type: application /x-www-form-urlencode" + "d' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=" + "0.9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0" + "' -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: kee" + "p-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith" + "%40example.org&size=small&topping=cheese&topping=onion&delivery=1" + "2%3A15&comments=' --compressed" + ) + r = self.request_class.from_curl(curl_command) + assert r.method == "POST" + assert r.url == "http://httpbin.org/post" + assert ( + r.body == b"custname=John+Smith&custtel=500&custemail=jsmith%40" + b"example.org&size=small&topping=cheese&topping=onion" + b"&delivery=12%3A15&comments=" + ) + assert r.cookies == { + "_gauges_unique_year": "1", + "_gauges_unique": "1", + "_gauges_unique_month": "1", + "_gauges_unique_hour": "1", + "_gauges_unique_day": "1", + } + assert r.headers == { + b"Origin": [b"http://httpbin.org"], + b"Accept-Encoding": [b"gzip, deflate"], + b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"], + b"Upgrade-Insecure-Requests": [b"1"], + b"User-Agent": [ + b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537." + b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202" + b".75 Chrome/62.0.3202.75 Safari/537.36" + ], + b"Content-Type": [b"application /x-www-form-urlencoded"], + b"Accept": [ + b"text/html,application/xhtml+xml,application/xml;q=0." + b"9,image/webp,image/apng,*/*;q=0.8" + ], + b"Cache-Control": [b"max-age=0"], + b"Referer": [b"http://httpbin.org/forms/post"], + b"Connection": [b"keep-alive"], + } + + def test_from_curl_with_kwargs(self): + r = self.request_class.from_curl( + 'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"} + ) + assert r.method == "POST" + assert r.meta == {"key": "value"} + + def test_from_curl_ignore_unknown_options(self): + # By default: it works and ignores the unknown options: --foo and -z + with warnings.catch_warnings(): # avoid warning when executing tests + warnings.filterwarnings( + "ignore", category=UserWarning, message="Unrecognized options:" + ) + r = self.request_class.from_curl( + 'curl -X DELETE "http://example.org" --foo -z', + ) + assert r.method == "DELETE" + + # If `ignore_unknown_options` is set to `False` it raises an error with + # the unknown options: --foo and -z + with pytest.raises(ValueError, match="Unrecognized options:"): + self.request_class.from_curl( + 'curl -X PATCH "http://example.org" --foo -z', + ignore_unknown_options=False, + ) + + def test_to_curl(self): + # Note: more curated tests regarding curl conversion are in + # `test_utils_request.py` + r = self.request_class( + "http://www.example.com/", method="POST", body=b"foo=bar" + ) + assert r.to_curl() == request_to_curl(r) diff --git a/tests/utils/bases/http_response.py b/tests/utils/bases/http_response.py new file mode 100644 index 000000000..78e14e7b0 --- /dev/null +++ b/tests/utils/bases/http_response.py @@ -0,0 +1,421 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +import pytest +from w3lib.encoding import resolve_encoding + +from scrapy.exceptions import NotSupported +from scrapy.http import Headers, Request, Response, TextResponse +from scrapy.link import Link +from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS +from tests import get_testdata + +if TYPE_CHECKING: + from collections.abc import Iterable + + from parsel import Selector + + +class TestResponseBase(ABC): + @property + @abstractmethod + def response_class(self) -> type[Response]: + raise NotImplementedError + + def test_init(self): + # Response requires url in the constructor + with pytest.raises(TypeError): + self.response_class() # type: ignore[call-arg] + assert isinstance( + self.response_class("http://example.com/"), self.response_class + ) + with pytest.raises(TypeError): + self.response_class(b"http://example.com") # type: ignore[arg-type] + with pytest.raises(TypeError): + self.response_class(url="http://example.com", body={}) # type: ignore[arg-type] + # body can be str or None + assert isinstance( + self.response_class("http://example.com/", body=b""), + self.response_class, + ) + assert isinstance( + self.response_class("http://example.com/", body=b"body"), + self.response_class, + ) + # test presence of all optional parameters + assert isinstance( + self.response_class( + "http://example.com/", body=b"", headers={}, status=200 + ), + self.response_class, + ) + + r = self.response_class("http://www.example.com") + assert isinstance(r.url, str) + assert r.url == "http://www.example.com" + assert r.status == 200 + + assert isinstance(r.headers, Headers) + assert not r.headers + + headers = {"foo": "bar"} + body = b"a body" + r = self.response_class("http://www.example.com", headers=headers, body=body) + + assert r.headers is not headers + assert r.headers[b"foo"] == b"bar" + + r = self.response_class("http://www.example.com", status=301) + assert r.status == 301 + r = self.response_class("http://www.example.com", status="301") # type: ignore[arg-type] + assert r.status == 301 + with pytest.raises(ValueError, match=r"invalid literal for int\(\)"): + self.response_class("http://example.com", status="lala200") # type: ignore[arg-type] + + def test_copy(self) -> None: + """Test Response copy""" + + r1 = self.response_class("http://www.example.com", body=b"Some body") + r1.flags.append("cached") + r2 = r1.copy() + + assert r1.status == r2.status + assert r1.body == r2.body + + # make sure flags list is shallow copied + assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" + assert r1.flags == r2.flags + + # make sure headers attribute is shallow copied + assert r1.headers is not r2.headers, ( + "headers must be a shallow copy, not identical" + ) + assert r1.headers == r2.headers + + def test_copy_meta(self): + req = Request("http://www.example.com") + req.meta["foo"] = "bar" + r1 = self.response_class( + "http://www.example.com", body=b"Some body", request=req + ) + assert r1.meta is req.meta + + def test_copy_cb_kwargs(self): + req = Request("http://www.example.com") + req.cb_kwargs["foo"] = "bar" + r1 = self.response_class( + "http://www.example.com", body=b"Some body", request=req + ) + assert r1.cb_kwargs is req.cb_kwargs + + def test_unavailable_meta(self): + r1 = self.response_class("http://www.example.com", body=b"Some body") + with pytest.raises(AttributeError, match=r"Response\.meta not available"): + r1.meta # pylint: disable=pointless-statement + + def test_unavailable_cb_kwargs(self): + r1 = self.response_class("http://www.example.com", body=b"Some body") + with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"): + r1.cb_kwargs # pylint: disable=pointless-statement + + def test_copy_inherited_classes(self): + """Test Response children copies preserve their class""" + + class CustomResponse(self.response_class): # type: ignore[misc,name-defined] + pass + + r1 = CustomResponse("http://www.example.com") + r2 = r1.copy() + + assert isinstance(r2, CustomResponse) + + def test_replace(self) -> None: + """Test Response.replace() method""" + hdrs = Headers({"key": "value"}) + r1 = self.response_class("http://www.example.com") + r2 = r1.replace(status=301, body=b"New body", headers=hdrs) + assert r1.body == b"" + assert r1.url == r2.url + assert (r1.status, r2.status) == (200, 301) + assert (r1.body, r2.body) == (b"", b"New body") + assert (r1.headers, r2.headers) == ({}, hdrs) + + # Empty attributes (which may fail if not compared properly) + r3 = self.response_class("http://www.example.com", flags=["cached"]) + r4 = r3.replace(body=b"", flags=[]) + assert r4.body == b"" + assert not r4.flags + + def _assert_response_values( + self, response: TextResponse, encoding: str, body: str | bytes + ) -> None: + if isinstance(body, str): + body_unicode = body + body_bytes = body.encode(encoding) + else: + body_unicode = body.decode(encoding) + body_bytes = body + + assert isinstance(response.body, bytes) + assert isinstance(response.text, str) + self._assert_response_encoding(response, encoding) + assert response.body == body_bytes + assert response.text == body_unicode + + def _assert_response_encoding(self, response: TextResponse, encoding: str) -> None: + assert response.encoding == resolve_encoding(encoding) + + def test_immutable_attributes(self): + r = self.response_class("http://example.com") + with pytest.raises(AttributeError): + r.url = "http://example2.com" # type: ignore[misc] + with pytest.raises(AttributeError): + r.body = "xxx" # type: ignore[misc,assignment] + + def test_setter_mutable_lazy_loading(self): + """Mutable attributes are set internally to None only until they are + read, then they always return the same falsy instance of the + corresponding mutable structure. + + Setting them to None causes the next read to return a different object. + """ + + response = self.response_class("http://example.com") + + response.request = Request("http://example.com") + + assert response._flags is None + assert response.flags == [] + assert response.flags is response.flags + assert response._flags == [] + original_flags = response.flags + response.flags = None + assert response._flags is None + assert response.flags == [] + assert response.flags is not original_flags + + assert response._headers is None + assert response.headers == {} + assert response.headers is response.headers + assert isinstance(response.headers, Headers) + assert isinstance(response._headers, Headers) + original_headers = response.headers + response.headers = None + assert response._headers is None + assert response.headers == {} + assert response._headers == {} + assert response.headers is not original_headers + + def test_setters(self): + response = self.response_class("http://example.com") + + response.flags = ["f1"] + assert response.flags == ["f1"] + + headers = Headers({b"X-Test": b"1"}) + response.headers = headers + assert response._headers is headers + response.headers = {b"A": b"b"} + assert isinstance(response.headers, Headers) + assert response._headers[b"A"] == b"b" + + def test_urljoin(self): + """Test urljoin shortcut (only for existence, since behavior equals urljoin)""" + joined = self.response_class("http://www.example.com").urljoin("/test") + absolute = "http://www.example.com/test" + assert joined == absolute + + def test_shortcut_attributes(self): + r = self.response_class("http://example.com", body=b"hello") + if self.response_class == Response: + msg = "Response content isn't text" + with pytest.raises(AttributeError, match=msg): + r.text # pylint: disable=pointless-statement + with pytest.raises(NotSupported, match=msg): + r.css("body") + with pytest.raises(NotSupported, match=msg): + r.xpath("//body") + with pytest.raises(NotSupported, match=msg): + r.jmespath("body") + else: + r.text # pylint: disable=pointless-statement + r.css("body") + r.xpath("//body") + + # Response.follow + + def test_follow_url_absolute(self): + self._assert_followed_url("http://foo.example.com", "http://foo.example.com") + + def test_follow_url_relative(self): + self._assert_followed_url("foo", "http://example.com/foo") + + def test_follow_link(self): + self._assert_followed_url( + Link("http://example.com/foo"), "http://example.com/foo" + ) + + def test_follow_None_url(self): + r = self.response_class("http://example.com") + with pytest.raises(ValueError, match="url can't be None"): + r.follow(None) # type: ignore[arg-type] + + def test_follow_None_encoding(self): + r = self.response_class("http://example.com") + with pytest.raises(ValueError, match="encoding can't be None"): + r.follow("foo", encoding=None) + + @pytest.mark.xfail( + not W3LIB_STRIPS_URLS, + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) + def test_follow_whitespace_url(self): + self._assert_followed_url("foo ", "http://example.com/foo") + + @pytest.mark.xfail( + not W3LIB_STRIPS_URLS, + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) + def test_follow_whitespace_link(self): + self._assert_followed_url( + Link("http://example.com/foo "), "http://example.com/foo" + ) + + def test_follow_flags(self): + res = self.response_class("http://example.com/") + fol = res.follow("http://example.com/", flags=["cached", "allowed"]) + assert fol.flags == ["cached", "allowed"] + + # Response.follow_all + + def test_follow_all_absolute(self): + url_list = [ + "http://example.org", + "http://www.example.org", + "http://example.com", + "http://www.example.com", + ] + self._assert_followed_all_urls(url_list, url_list) + + def test_follow_all_relative(self): + relative = ["foo", "bar", "foo/bar", "bar/foo"] + absolute = [ + "http://example.com/foo", + "http://example.com/bar", + "http://example.com/foo/bar", + "http://example.com/bar/foo", + ] + self._assert_followed_all_urls(relative, absolute) + + def test_follow_all_links(self): + absolute = [ + "http://example.com/foo", + "http://example.com/bar", + "http://example.com/foo/bar", + "http://example.com/bar/foo", + ] + links = map(Link, absolute) + self._assert_followed_all_urls(links, absolute) + + def test_follow_all_empty(self): + r = self.response_class("http://example.com") + assert not list(r.follow_all([])) + + def test_follow_all_invalid(self): + r = self.response_class("http://example.com") + if self.response_class == Response: + with pytest.raises(TypeError): + list(r.follow_all(urls=None)) # type: ignore[arg-type] + with pytest.raises(TypeError): + list(r.follow_all(urls=12345)) # type: ignore[arg-type] + with pytest.raises(ValueError, match="url can't be None"): + list(r.follow_all(urls=[None])) # type: ignore[list-item] + else: + with pytest.raises( + ValueError, match="Please supply exactly one of the following arguments" + ): + list(r.follow_all(urls=None)) # type: ignore[arg-type] + with pytest.raises(TypeError): + list(r.follow_all(urls=12345)) # type: ignore[arg-type] + with pytest.raises(ValueError, match="url can't be None"): + list(r.follow_all(urls=[None])) # type: ignore[list-item] + + @pytest.mark.xfail( + not W3LIB_STRIPS_URLS, + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) + def test_follow_all_whitespace(self): + relative = ["foo ", "bar ", "foo/bar ", "bar/foo "] + absolute = [ + "http://example.com/foo", + "http://example.com/bar", + "http://example.com/foo/bar", + "http://example.com/bar/foo", + ] + self._assert_followed_all_urls(relative, absolute) + + @pytest.mark.xfail( + not W3LIB_STRIPS_URLS, + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) + def test_follow_all_whitespace_links(self): + absolute = [ + "http://example.com/foo ", + "http://example.com/bar ", + "http://example.com/foo/bar ", + "http://example.com/bar/foo ", + ] + links = [Link(u) for u in absolute] + expected = [u.strip() for u in absolute] + self._assert_followed_all_urls(links, expected) + + def test_follow_all_flags(self): + re = self.response_class("http://www.example.com/") + urls = [ + "http://www.example.com/", + "http://www.example.com/2", + "http://www.example.com/foo", + ] + fol = re.follow_all(urls, flags=["cached", "allowed"]) + for req in fol: + assert req.flags == ["cached", "allowed"] + + def _assert_followed_url( + self, + follow_obj: str | Link | Selector, + target_url: str, + response: Response | None = None, + encoding: str | None = None, + ) -> None: + if response is None: + response = self._links_response() + req = response.follow(follow_obj) # type: ignore[arg-type] + assert req.url == target_url + if encoding is not None: + assert req.encoding == encoding + + def _assert_followed_all_urls( + self, + follow_obj: Iterable[str | Link], + target_urls: Iterable[str], + response: Response | None = None, + ) -> None: + if response is None: + response = self._links_response() + followed = response.follow_all(follow_obj) + for req, target in zip(followed, target_urls, strict=True): + assert req.url == target + + def _links_response(self) -> Response: + body = get_testdata("link_extractor", "linkextractor.html") + return self.response_class("http://example.com/index", body=body) + + def _links_response_no_href(self) -> Response: + body = get_testdata("link_extractor", "linkextractor_no_href.html") + return self.response_class("http://example.com/index", body=body) diff --git a/tests/utils/bases/redirect.py b/tests/utils/bases/redirect.py new file mode 100644 index 000000000..e77a30533 --- /dev/null +++ b/tests/utils/bases/redirect.py @@ -0,0 +1,988 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any + +import pytest + +from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware +from scrapy.exceptions import IgnoreRequest +from scrapy.http import Request, Response +from scrapy.utils.misc import set_environ +from scrapy.utils.test import get_crawler + + +class TestRedirectBase(ABC): + mwcls: type[Any] + mw: Any + reason: int | str + + @abstractmethod + def get_response( + self, request: Request, location: str, status: int = 302 + ) -> Response: + raise NotImplementedError + + def test_priority_adjust(self): + req = Request("http://a.example") + rsp = self.get_response(req, "http://a.example/redirected") + req2 = self.mw.process_response(req, rsp) + assert req2.priority > req.priority + + def test_dont_redirect(self): + url = "http://www.example.com/301" + url2 = "http://www.example.com/redirected" + req = Request(url, meta={"dont_redirect": True}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp) + assert isinstance(r, Response) + assert r is rsp + + # Test that it redirects when dont_redirect is False + req = Request(url, meta={"dont_redirect": False}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp) + assert isinstance(r, Request) + + def test_post(self): + url = "http://www.example.com/302" + url2 = "http://www.example.com/redirected2" + req = Request( + url, + method="POST", + body="test", + headers={"Content-Type": "text/plain", "Content-length": "4"}, + ) + rsp = self.get_response(req, url2) + + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == url2 + assert req2.method == "GET" + assert "Content-Type" not in req2.headers, ( + "Content-Type header must not be present in redirected request" + ) + assert "Content-Length" not in req2.headers, ( + "Content-Length header must not be present in redirected request" + ) + assert not req2.body, f"Redirected body must be empty, not '{req2.body!r}'" + + def test_max_redirect_times(self): + self.mw.max_redirect_times = 1 + req = Request("http://a.example/302") + rsp = self.get_response(req, "/redirected") + + req = self.mw.process_response(req, rsp) + assert isinstance(req, Request) + assert "redirect_times" in req.meta + assert req.meta["redirect_times"] == 1 + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp) + + def test_ttl(self): + self.mw.max_redirect_times = 100 + req = Request("http://a.example/302", meta={"redirect_ttl": 1}) + rsp = self.get_response(req, "/a") + + req = self.mw.process_response(req, rsp) + assert isinstance(req, Request) + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp) + + def test_redirect_urls(self): + req1 = Request("http://a.example/first") + rsp1 = self.get_response(req1, "/redirected") + req2 = self.mw.process_response(req1, rsp1) + rsp2 = self.get_response(req2, "/redirected2") + req3 = self.mw.process_response(req2, rsp2) + + assert req2.url == "http://a.example/redirected" + assert req2.meta["redirect_urls"] == ["http://a.example/first"] + assert req3.url == "http://a.example/redirected2" + assert req3.meta["redirect_urls"] == [ + "http://a.example/first", + "http://a.example/redirected", + ] + + def test_redirect_reasons(self): + req1 = Request("http://a.example/first") + rsp1 = self.get_response(req1, "/redirected1") + req2 = self.mw.process_response(req1, rsp1) + rsp2 = self.get_response(req2, "/redirected2") + req3 = self.mw.process_response(req2, rsp2) + assert req2.meta["redirect_reasons"] == [self.reason] + assert req3.meta["redirect_reasons"] == [self.reason, self.reason] + + def test_cross_origin_header_dropping(self): + safe_headers = {"A": "B"} + cookie_header = {"Cookie": "a=b"} + authorization_header = {"Authorization": "Bearer 123456"} + + original_request = Request( + "https://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers. + internal_response = self.get_response(original_request, "https://example.com/a") + internal_redirect_request = self.mw.process_response( + original_request, internal_response + ) + assert isinstance(internal_redirect_request, Request) + assert original_request.headers == internal_redirect_request.headers + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers also when the scheme is http. + http_request = Request( + "http://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + http_response = self.get_response(http_request, "http://example.com/a") + http_redirect_request = self.mw.process_response(http_request, http_response) + assert isinstance(http_redirect_request, Request) + assert http_request.headers == http_redirect_request.headers + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_explicit_port_response = self.get_response( + original_request, "https://example.com:443/a" + ) + to_explicit_port_redirect_request = self.mw.process_response( + original_request, to_explicit_port_response + ) + assert isinstance(to_explicit_port_redirect_request, Request) + assert original_request.headers == to_explicit_port_redirect_request.headers + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_implicit_port_response = self.get_response( + original_request, "https://example.com/a" + ) + to_implicit_port_redirect_request = self.mw.process_response( + original_request, to_implicit_port_response + ) + assert isinstance(to_implicit_port_redirect_request, Request) + assert original_request.headers == to_implicit_port_redirect_request.headers + + # A port change drops the Authorization header because the origin + # changes, but keeps the Cookie header because the domain remains the + # same. + different_port_response = self.get_response( + original_request, "https://example.com:8080/a" + ) + different_port_redirect_request = self.mw.process_response( + original_request, different_port_response + ) + assert isinstance(different_port_redirect_request, Request) + assert { + **safe_headers, + **cookie_header, + } == different_port_redirect_request.headers.to_unicode_dict() + + # A domain change drops both the Authorization and the Cookie header. + external_response = self.get_response(original_request, "https://example.org/a") + external_redirect_request = self.mw.process_response( + original_request, external_response + ) + assert isinstance(external_redirect_request, Request) + assert safe_headers == external_redirect_request.headers.to_unicode_dict() + + # A scheme upgrade (http → https) drops the Authorization header + # because the origin changes, but keeps the Cookie header because the + # domain remains the same. + upgrade_response = self.get_response(http_request, "https://example.com/a") + upgrade_redirect_request = self.mw.process_response( + http_request, upgrade_response + ) + assert isinstance(upgrade_redirect_request, Request) + assert { + **safe_headers, + **cookie_header, + } == upgrade_redirect_request.headers.to_unicode_dict() + + # A scheme downgrade (https → http) drops the Authorization header + # because the origin changes, and the Cookie header because its value + # cannot indicate whether the cookies were secure (HTTPS-only) or not. + # + # Note: If the Cookie header is set by the cookie management + # middleware, as recommended in the docs, the dropping of Cookie on + # scheme downgrade is not an issue, because the cookie management + # middleware will add again the Cookie header to the new request if + # appropriate. + downgrade_response = self.get_response(original_request, "http://example.com/a") + downgrade_redirect_request = self.mw.process_response( + original_request, downgrade_response + ) + assert isinstance(downgrade_redirect_request, Request) + assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict() + + def test_meta_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_http_to_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_to_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_proxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request2.meta["_auth_proxy"] == "https://b.example" + assert request2.meta["proxy"] == "https://b.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_proxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_unproxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request2.meta["_auth_proxy"] == "https://b.example" + assert request2.meta["proxy"] == "https://b.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_unproxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_proxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request1.meta["_auth_proxy"] == "https://b.example" + assert request1.meta["proxy"] == "https://b.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request3.meta["_auth_proxy"] == "https://b.example" + assert request3.meta["proxy"] == "https://b.example" + + def test_system_proxy_proxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request1.meta["_auth_proxy"] == "https://b.example" + assert request1.meta["proxy"] == "https://b.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request3.meta["_auth_proxy"] == "https://b.example" + assert request3.meta["proxy"] == "https://b.example" + + def test_system_proxy_unproxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_unproxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta diff --git a/tests/utils/bases/spider.py b/tests/utils/bases/spider.py new file mode 100644 index 000000000..799c1820d --- /dev/null +++ b/tests/utils/bases/spider.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import logging +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any +from unittest import mock + +import pytest + +from scrapy import signals +from scrapy.crawler import Crawler +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.settings import Settings +from scrapy.utils.test import get_crawler, get_reactor_settings +from tests.utils.decorators import inline_callbacks_test + +if TYPE_CHECKING: + from scrapy.spiders import Spider + + +class TestSpiderBase(ABC): + @property + @abstractmethod + def spider_class(self) -> type[Spider]: + raise NotImplementedError + + def test_base_spider(self): + spider = self.spider_class("example.com") + assert spider.name == "example.com" + assert spider.start_urls == [] + + def test_spider_args(self): + """``__init__`` method arguments are assigned to spider attributes""" + spider = self.spider_class("example.com", foo="bar") + assert spider.foo == "bar" + + def test_spider_without_name(self): + """``__init__`` raises when the name is not provided.""" + msg = "must have a name" + with pytest.raises(ValueError, match=msg): + self.spider_class() + with pytest.raises(ValueError, match=msg): + self.spider_class(somearg="foo") + + def test_from_crawler_crawler_and_settings_population(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + assert hasattr(spider, "crawler") + assert spider.crawler is crawler + assert hasattr(spider, "settings") + assert spider.settings is crawler.settings + + def test_from_crawler_init_call(self): + with mock.patch.object( + self.spider_class, "__init__", return_value=None + ) as mock_init: + self.spider_class.from_crawler(get_crawler(), "example.com", foo="bar") + mock_init.assert_called_once_with("example.com", foo="bar") + + def test_closed_signal_call(self): + class TestSpider(self.spider_class): + closed_called = False + + def closed(self, reason): + self.closed_called = True + + crawler = get_crawler() + spider = TestSpider.from_crawler(crawler, "example.com") + crawler.signals.send_catch_log(signal=signals.spider_opened, spider=spider) + crawler.signals.send_catch_log( + signal=signals.spider_closed, spider=spider, reason=None + ) + assert spider.closed_called + + def test_update_settings(self): + spider_settings = {"TEST1": "spider", "TEST2": "spider"} + project_settings = {"TEST1": "project", "TEST3": "project"} + self.spider_class.custom_settings = spider_settings + settings = Settings(project_settings, priority="project") + + self.spider_class.update_settings(settings) + assert settings.get("TEST1") == "spider" + assert settings.get("TEST2") == "spider" + assert settings.get("TEST3") == "project" + + @inline_callbacks_test + def test_settings_in_from_crawler(self): + spider_settings = {"TEST1": "spider", "TEST2": "spider"} + project_settings = { + "TEST1": "project", + "TEST3": "project", + **get_reactor_settings(), + } + + class TestSpider(self.spider_class): + name = "test" + custom_settings = spider_settings + + @classmethod + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("TEST1", "spider_instance", priority="spider") + return spider + + crawler = Crawler(TestSpider, project_settings) + assert crawler.settings.get("TEST1") == "spider" + assert crawler.settings.get("TEST2") == "spider" + assert crawler.settings.get("TEST3") == "project" + yield crawler.crawl() + assert crawler.settings.get("TEST1") == "spider_instance" + + def test_logger(self, caplog: pytest.LogCaptureFixture) -> None: + spider = self.spider_class("example.com") + caplog.clear() + with caplog.at_level(logging.INFO): + spider.logger.info("test log msg") + assert caplog.record_tuples == [("example.com", logging.INFO, "test log msg")] + + record = caplog.records[0] + assert getattr(record, "spider", None) is spider + + def test_log(self): + spider = self.spider_class("example.com") + with ( + mock.patch("scrapy.spiders.Spider.logger") as mock_logger, + pytest.warns( + ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated" + ), + ): + spider.log("test log msg", "INFO") + mock_logger.log.assert_called_once_with("INFO", "test log msg") diff --git a/tests/utils/cmdline.py b/tests/utils/cmdline.py index 62dff3d4c..095cb17a7 100644 --- a/tests/utils/cmdline.py +++ b/tests/utils/cmdline.py @@ -46,3 +46,16 @@ def write_recording_editor(editor: Path) -> None: open (its last argument) into the file given as its first argument.""" editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8") editor.chmod(0o755) + + +def write_recording_browser(browser: Path, recorded: Path) -> None: + """Create an executable browser script that writes the URL it is asked to + open into *recorded*. + + ``webbrowser`` only passes the URL to the command from the ``BROWSER`` + environment variable, hence the hardcoded output path. + """ + browser.write_text( + f'#!/bin/sh\nprintf "%s" "$1" > "{recorded}"\n', encoding="utf-8" + ) + browser.chmod(0o755) diff --git a/tests/utils/crawl.py b/tests/utils/crawl.py new file mode 100644 index 000000000..4631d909d --- /dev/null +++ b/tests/utils/crawl.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from scrapy import signals +from scrapy.utils.test import get_crawler + +if TYPE_CHECKING: + from scrapy.crawler import Crawler + from scrapy.spiders import Spider + from tests.mockserver.http import MockServer + + +async def crawl_items( + spider_cls: type[Spider], mockserver: MockServer, **kwargs: Any +) -> tuple[list[Any], Crawler]: + """Run *spider_cls* against *mockserver* and return the scraped items along + with the crawler, which gives tests access to the resulting stats.""" + items: list[Any] = [] + + def collect(item: Any) -> None: + items.append(item) + + crawler = get_crawler(spider_cls) + crawler.signals.connect(collect, signals.item_scraped) + await crawler.crawl_async(mockserver=mockserver, **kwargs) + return items, crawler diff --git a/tests/utils/downloader.py b/tests/utils/downloader.py new file mode 100644 index 000000000..f64c02332 --- /dev/null +++ b/tests/utils/downloader.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, NamedTuple, cast + +from scrapy.core.downloader import Downloader +from scrapy.utils.httpobj import urlparse_cached + +if TYPE_CHECKING: + from scrapy.http import Request + + +class MockSlot(NamedTuple): + active: list[Any] + + +class MockDownloader: + def __init__(self) -> None: + self.slots: dict[str, MockSlot] = {} + + def get_slot_key(self, request: Request) -> str: + if Downloader.DOWNLOAD_SLOT in request.meta: + return cast("str", request.meta[Downloader.DOWNLOAD_SLOT]) + + return urlparse_cached(request).hostname or "" + + def increment(self, slot_key: str) -> None: + slot = self.slots.setdefault(slot_key, MockSlot(active=[])) + slot.active.append(1) + + def decrement(self, slot_key: str) -> None: + slot = self.slots[slot_key] + slot.active.pop() + + def close(self) -> None: + pass diff --git a/tests/utils/engine.py b/tests/utils/engine.py new file mode 100644 index 000000000..c193d39a9 --- /dev/null +++ b/tests/utils/engine.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import re +from collections import defaultdict +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any +from urllib.parse import urlparse + +import attr +from itemadapter import ItemAdapter +from pydispatch import dispatcher +from twisted.internet import defer + +from scrapy import signals +from scrapy.http import Headers, Request, Response +from scrapy.item import Field, Item +from scrapy.linkextractors import LinkExtractor +from scrapy.spiders import Spider +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.signal import disconnect_all +from scrapy.utils.test import get_crawler + +if TYPE_CHECKING: + from twisted.python.failure import Failure + + from tests.mockserver.http import MockServer + + +class MyItem(Item): + name = Field() + url = Field() + price = Field() + + +@attr.s +class AttrsItem: + name = attr.ib(default="") + url = attr.ib(default="") + price = attr.ib(default=0) + + +@dataclass +class DataClassItem: + name: str = "" + url: str = "" + price: int = 0 + + +class MySpider(Spider): + name = "scrapytest.org" + + itemurl_re = re.compile(r"item\d+.html") + name_re = re.compile(r"

(.*?)

", re.MULTILINE) + price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE) + + item_cls: type = MyItem + + def parse(self, response): + xlink = LinkExtractor() + itemre = re.compile(self.itemurl_re) + for link in xlink.extract_links(response): + if itemre.search(link.url): + yield Request(url=link.url, callback=self.parse_item) + + def parse_item(self, response): + adapter = ItemAdapter(self.item_cls()) + m = self.name_re.search(response.text) + if m: + adapter["name"] = m.group(1) + adapter["url"] = response.url + m = self.price_re.search(response.text) + if m: + adapter["price"] = m.group(1) + return adapter.item + + +class DictItemsSpider(MySpider): + item_cls = dict + + +class AttrsItemsSpider(MySpider): + item_cls = AttrsItem + + +class DataClassItemsSpider(MySpider): + item_cls = DataClassItem + + +class CrawlerRun: + """A class to run the crawler and keep track of events occurred""" + + def __init__(self, spider_class: type[Spider]): + self.respplug: list[tuple[Response, Spider]] = [] + self.reqplug: list[tuple[Request, Spider]] = [] + self.reqdropped: list[tuple[Request, Spider]] = [] + self.reqreached: list[tuple[Request, Spider]] = [] + self.itemerror: list[tuple[Any, Response, Spider, Failure]] = [] + self.itemresp: list[tuple[Any, Response]] = [] + self.headers: dict[Request, Headers] = {} + self.bytes: defaultdict[Request, list[bytes]] = defaultdict(list) + self.signals_caught: dict[Any, dict[str, Any]] = {} + self.spider_class = spider_class + + async def run(self, mockserver: MockServer) -> None: + self.mockserver = mockserver + + start_urls = [ + self.geturl("/static/"), + self.geturl("/redirect"), + self.geturl("/redirect"), # duplicate + self.geturl("/numbers"), + ] + + for name, signal in vars(signals).items(): + if not name.startswith("_"): + dispatcher.connect(self.record_signal, signal) + + self.crawler = get_crawler(self.spider_class) + self.crawler.signals.connect(self.item_scraped, signals.item_scraped) + self.crawler.signals.connect(self.item_error, signals.item_error) + self.crawler.signals.connect(self.headers_received, signals.headers_received) + self.crawler.signals.connect(self.bytes_received, signals.bytes_received) + self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) + self.crawler.signals.connect(self.request_dropped, signals.request_dropped) + self.crawler.signals.connect( + self.request_reached, signals.request_reached_downloader + ) + self.crawler.signals.connect( + self.response_downloaded, signals.response_downloaded + ) + self.crawler.crawl(start_urls=start_urls) + + self.deferred: defer.Deferred[None] = defer.Deferred() + dispatcher.connect(self.stop, signals.engine_stopped) + await maybe_deferred_to_future(self.deferred) + + async def stop(self): + for name, signal in vars(signals).items(): + if not name.startswith("_"): + disconnect_all(signal) + self.deferred.callback(None) + await self.crawler.stop_async() + + def geturl(self, path: str) -> str: + return self.mockserver.url(path) + + def getpath(self, url: str) -> str: + u = urlparse(url) + return u.path + + def item_error( + self, item: Any, response: Response, spider: Spider, failure: Failure + ) -> None: + self.itemerror.append((item, response, spider, failure)) + + def item_scraped(self, item: Any, spider: Spider, response: Response) -> None: + self.itemresp.append((item, response)) + + def headers_received( + self, headers: Headers, body_length: int, request: Request, spider: Spider + ) -> None: + self.headers[request] = headers + + def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None: + self.bytes[request].append(data) + + def request_scheduled(self, request: Request, spider: Spider) -> None: + self.reqplug.append((request, spider)) + + def request_reached(self, request: Request, spider: Spider) -> None: + self.reqreached.append((request, spider)) + + def request_dropped(self, request: Request, spider: Spider) -> None: + self.reqdropped.append((request, spider)) + + def response_downloaded(self, response: Response, spider: Spider) -> None: + self.respplug.append((response, spider)) + + def record_signal(self, *args: Any, **kwargs: Any) -> None: + """Record a signal and its parameters""" + signalargs = kwargs.copy() + sig = signalargs.pop("signal") + signalargs.pop("sender", None) + self.signals_caught[sig] = signalargs diff --git a/tests/utils/media_pipelines.py b/tests/utils/media_pipelines.py new file mode 100644 index 000000000..7e013b797 --- /dev/null +++ b/tests/utils/media_pipelines.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from typing import Any + +from scrapy.http.request import NO_CALLBACK, Request +from scrapy.pipelines.media import MediaPipeline +from scrapy.utils.spider import DefaultSpider + +# required by persist_file() and stat_file(), but as some stores don't use the argument +# we can pass this singleton to keep type hints correct +DUMMY_SPIDER_INFO = MediaPipeline.SpiderInfo(DefaultSpider()) + + +async def mocked_download_func(request: Request) -> Any: + assert request.callback is NO_CALLBACK + response = request.meta.get("response") + if callable(response): + response = await response() + if isinstance(response, Exception): + raise response + return response diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing index a431091d5..4ff562dff 100644 --- a/tests_typing/test_http_request.mypy-testing +++ b/tests_typing/test_http_request.mypy-testing @@ -16,7 +16,7 @@ class MyRequest2(Request): @pytest.mark.mypy_testing def mypy_test_headers() -> None: - Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Mapping[bytes, Any] | Iterable[tuple[str | bytes, Any]] | None" Request("data:,", headers=None) Request("data:,", headers={}) Request("data:,", headers=[]) diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing index d497c2470..1c157328c 100644 --- a/tests_typing/test_http_response.mypy-testing +++ b/tests_typing/test_http_response.mypy-testing @@ -7,7 +7,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse @pytest.mark.mypy_testing def mypy_test_headers() -> None: - Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Mapping[bytes, Any] | Iterable[tuple[str | bytes, Any]] | None" Response("data:,", headers=None) Response("data:,", headers={}) Response("data:,", headers=[]) diff --git a/tox.ini b/tox.ini index d35fe91db..ac32064a6 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,8 @@ [tox] requires = - sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 + sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10 + tox-uv envlist = pre-commit pylint @@ -30,6 +31,7 @@ envlist = botocore pypy3 pypy3-extra-deps + benchmark minversion = 1.7.0 [test-requirements] @@ -44,7 +46,6 @@ deps = pytest-cov >= 7.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 - testfixtures pytest-twisted >= 1.14.3 [testenv] @@ -110,7 +111,8 @@ commands = pre-commit run {posargs:--all-files} [testenv:pylint] -basepython = python3 +# Some checks are Python-version-dependent, so pin the version used in CI. +basepython = python3.14 deps = {[testenv:extra-deps]deps} pylint==4.0.6 @@ -190,8 +192,8 @@ deps = brotlicffi==1.2.0.0; implementation_name == "pypy" google-cloud-storage==1.29.0 httpx2[http2,socks]==2.0.0 - ipython==7.1.0 - ptpython==2.0.1 + ipython==8.15.0 + ptpython==3.0.23 robotexclusionrulesparser==1.6.2 uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy" zstandard==0.16.0; implementation_name != "pypy" @@ -318,3 +320,20 @@ setenv = {[min]setenv} commands = pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore + + +# CPU benchmarks, tracked on CodSpeed. +# +# pytest-twisted is left out on purpose: benchmarked code must be callable +# synchronously, so tests/benchmarks drives the reactor itself. + +[testenv:benchmark] +basepython = python3.14 +deps = + pytest >= 8.4.1 + pytest-codspeed +passenv = + *codspeed* + *ci* +commands = + pytest {posargs:tests/benchmarks} --codspeed --codspeed-mode=simulation