diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..623d48cfa --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: monthly + groups: + github-actions: + patterns: + - "*" + cooldown: + default-days: 7 diff --git a/.github/workflows/auto-close-llm-pr.yml b/.github/workflows/auto-close-llm-pr.yml index 160b39488..15120b0d9 100644 --- a/.github/workflows/auto-close-llm-pr.yml +++ b/.github/workflows/auto-close-llm-pr.yml @@ -1,5 +1,7 @@ name: Auto-close LLM PRs -on: +# The workflow only reads the pull request body through the API, it never +# checks out or runs pull request code, so pull_request_target is safe here. +on: # zizmor: ignore[dangerous-triggers] pull_request_target: types: [opened] permissions: @@ -11,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check PR body and close if LLM-written - uses: actions/github-script@v6 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ed2388a59..331fade61 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,4 +1,8 @@ name: Checks + +permissions: + contents: read + on: push: branches: @@ -13,6 +17,10 @@ concurrency: jobs: checks: runs-on: ubuntu-latest + env: + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: @@ -38,21 +46,31 @@ jobs: TOXENV: twinecheck steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + docs/requirements.txt + pyproject.toml + tox.ini + - name: Run check env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - uses: pre-commit/action@v3.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 000000000..c50576ec7 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,59 @@ +--- +name: codspeed + +on: + push: + branches: + - master + pull_request: + paths: + - scrapy/** + - tests/benchmarks/** + - .github/workflows/codspeed.yml + - tox.ini + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: {} + +jobs: + benchmark: + runs-on: ubuntu-latest + env: + # Make uv use the interpreter that actions/setup-python installed + # instead of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system + permissions: + contents: read + id-token: write # OIDC authentication with CodSpeed + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python 3.14 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.14' + + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + + - name: Install dependencies + # tox must stay on PATH for the CodSpeed action to invoke it. + run: | + uv tool install --with tox-uv tox + tox -n -e benchmark + - name: Run benchmarks + uses: CodSpeedHQ/action@0ca9cbbf4623b599a6c3ed4fc8a922942705d9f1 # v5.0.2 + with: + mode: simulation + run: tox -e benchmark diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7779bbb6b..697647131 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,4 +1,8 @@ name: Publish + +permissions: + contents: read + on: push: tags: @@ -9,8 +13,28 @@ concurrency: cancel-in-progress: true jobs: + build: + name: Build distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + - run: | + python -m pip install --upgrade build + python -m build + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: python-package-distributions + path: dist/ + publish: name: Upload release to PyPI + needs: + - build runs-on: ubuntu-latest environment: name: pypi @@ -18,12 +42,9 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - python-version: "3.14" - - run: | - python -m pip install --upgrade build - python -m build + name: python-package-distributions + path: dist/ - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 0409b3ef2..9a09aa67d 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,4 +1,8 @@ name: macOS + +permissions: + contents: read + on: push: branches: @@ -12,39 +16,62 @@ concurrency: jobs: tests: + name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }}) runs-on: macos-latest env: - PYTEST_ADDOPTS: -n auto + PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - env: - - TOXENV: py include: - - python-version: '3.14' - env: - TOXENV: no-reactor + - python-version: "3.10" + env: + TOXENV: py + - python-version: "3.14" + env: + TOXENV: py + coverage: true + - python-version: "3.14" + env: + TOXENV: no-reactor steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + + - name: Install mitmproxy + env: + # mitmproxy needs a newer Python than the oldest matrix entries, so let + # uv download one where no system interpreter is new enough. + UV_PYTHON_PREFERENCE: system + run: uv tool install mitmproxy + - name: Run tests env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox - name: Upload coverage report - uses: codecov/codecov-action@v5 + if: ${{ matrix.coverage }} + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - name: Upload test results if: ${{ !cancelled() }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: report_type: test_results diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 15f25d9b8..cd726a2fe 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,4 +1,8 @@ name: Ubuntu + +permissions: + contents: read + on: push: branches: @@ -12,9 +16,13 @@ concurrency: jobs: tests: + name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }}) runs-on: ubuntu-latest env: - PYTEST_ADDOPTS: -n auto + PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: @@ -34,27 +42,25 @@ jobs: - python-version: "3.14" env: TOXENV: py + coverage: true - python-version: "3.14" env: TOXENV: default-reactor + coverage: true - python-version: "3.14" env: TOXENV: no-reactor - # pinned due to https://github.com/pypy/pypy/issues/5388 - - python-version: pypy3.11-7.3.20 - env: - TOXENV: pypy3 + coverage: true # min deps - python-version: "3.10.19" env: TOXENV: min + coverage: true - python-version: "3.10.19" env: TOXENV: min-default-reactor - - python-version: "3.10.19" - env: - TOXENV: min-no-reactor + coverage: true # pinned due to https://github.com/pypy/pypy/issues/5388 - python-version: pypy3.11-7.3.20 env: @@ -62,16 +68,20 @@ jobs: - python-version: "3.10.19" env: TOXENV: min-extra-deps + coverage: true - python-version: "3.10.19" env: TOXENV: min-botocore + coverage: true - python-version: "3.14" env: TOXENV: extra-deps + coverage: true - python-version: "3.14" env: TOXENV: no-reactor-extra-deps + coverage: true # pinned due to https://github.com/pypy/pypy/issues/5388 - python-version: pypy3.11-7.3.20 env: @@ -79,12 +89,15 @@ jobs: - python-version: "3.14" env: TOXENV: botocore + coverage: true steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} @@ -94,20 +107,32 @@ jobs: sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + - name: Install mitmproxy - run: pipx install mitmproxy + env: + # mitmproxy needs a newer Python than the oldest matrix entries, so let + # uv download one where no system interpreter is new enough. + UV_PYTHON_PREFERENCE: system + # mitmproxy has no PyPy wheels, so run it on CPython regardless of the + # interpreter under test. + run: uv tool install --python cpython mitmproxy - name: Run tests env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox - name: Upload coverage report - uses: codecov/codecov-action@v5 + if: ${{ matrix.coverage }} + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - name: Upload test results if: ${{ !cancelled() }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: report_type: test_results diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index f413782bc..254e9395e 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,4 +1,8 @@ name: Windows + +permissions: + contents: read + on: push: branches: @@ -12,9 +16,13 @@ concurrency: jobs: tests: + name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }}) runs-on: windows-latest env: - PYTEST_ADDOPTS: -n auto + PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: @@ -22,21 +30,10 @@ jobs: - python-version: "3.10" env: TOXENV: py - - python-version: "3.11" - env: - TOXENV: py - - python-version: "3.12" - env: - TOXENV: py - - python-version: "3.13" - env: - TOXENV: py - python-version: "3.14" env: TOXENV: py - - python-version: "3.14" - env: - TOXENV: default-reactor + coverage: true - python-version: "3.14" env: TOXENV: no-reactor @@ -54,24 +51,39 @@ jobs: TOXENV: extra-deps steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + + - name: Install mitmproxy + env: + # mitmproxy needs a newer Python than the oldest matrix entries, so let + # uv download one where no system interpreter is new enough. + UV_PYTHON_PREFERENCE: system + run: uv tool install mitmproxy + - name: Run tests env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox - name: Upload coverage report - uses: codecov/codecov-action@v5 + if: ${{ matrix.coverage }} + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - name: Upload test results if: ${{ !cancelled() }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: report_type: test_results diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 311df7052..c2cb5056d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,11 @@ repos: hooks: - id: sphinx-lint - repo: https://github.com/scrapy/sphinx-scrapy - rev: 0.8.8 + rev: 0.8.10 hooks: - id: sphinx-scrapy +- repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.28.0 + hooks: + - id: zizmor + args: [--no-progress, --fix] diff --git a/README.rst b/README.rst index 6235cb20c..651294add 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ :alt: Scrapy :width: 480px -|version| |python_version| |ubuntu| |macos| |windows| |coverage| |conda| |deepwiki| +|version| |python_version| |tests| |coverage| |conda| |deepwiki| .. |version| image:: https://img.shields.io/pypi/v/Scrapy.svg :target: https://pypi.org/pypi/Scrapy @@ -15,17 +15,9 @@ :target: https://pypi.org/pypi/Scrapy :alt: Supported Python Versions -.. |ubuntu| image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu - :alt: Ubuntu - -.. |macos| image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS - :alt: macOS - -.. |windows| image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows - :alt: Windows +.. |tests| image:: https://img.shields.io/github/check-runs/scrapy/scrapy/master?label=tests + :target: https://github.com/scrapy/scrapy/actions?query=branch%3Amaster + :alt: Tests .. |coverage| image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg :target: https://codecov.io/github/scrapy/scrapy?branch=master diff --git a/conftest.py b/conftest.py index 27c398792..5a535c168 100644 --- a/conftest.py +++ b/conftest.py @@ -54,6 +54,9 @@ if not H2_ENABLED: if find_spec("httpx2") is None and find_spec("httpx") is None: collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py") +if find_spec("pytest_codspeed") is None: + collect_ignore.append("tests/benchmarks") + def pytest_addoption(parser, pluginmanager): if pluginmanager.hasplugin("twisted"): diff --git a/docs/conf.py b/docs/conf.py index de722baac..1b41adaad 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -141,6 +141,8 @@ coverage_ignore_pyobjects = [ r"^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor", ] +# -- Options for the autodoc extension ---------------------------------------- +autodoc_member_order = "bysource" # -- Options for the InterSphinx extension ----------------------------------- # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration diff --git a/docs/faq.rst b/docs/faq.rst index 0446a6868..80658a5bf 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -97,7 +97,7 @@ handler documentation. How can I scrape an item with attributes in different pages? ------------------------------------------------------------ -See :ref:`topics-request-response-ref-request-callback-arguments`. +See :ref:`callback-data`. How can I simulate a user login in my spider? --------------------------------------------- @@ -220,21 +220,15 @@ the :ref:`topics-signals-ref` to know which ones. What does the response status code 999 mean? -------------------------------------------- -999 is a custom response status code used by Yahoo sites to throttle requests. +999 is a custom response status code used by some sites to throttle requests. Try slowing down the crawling speed by using a download delay of ``2`` (or -higher) in your spider: +higher) for the affected domains, with the :setting:`DOWNLOAD_SLOTS` setting: .. code-block:: python - from scrapy.spiders import CrawlSpider - - - class MySpider(CrawlSpider): - name = "myspider" - - download_delay = 2 - - # [ ... rest of the spider code ... ] + DOWNLOAD_SLOTS = { + "example.com": {"delay": 2}, + } Or by setting a global download delay in your project with the :setting:`DOWNLOAD_DELAY` setting. diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index c4e04364b..eaf492c95 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -769,7 +769,7 @@ crawlers on top of it. Also, a common pattern is to build an item with data from more than one page, using a :ref:`trick to pass additional data to the callbacks -`. +`. Using spider arguments diff --git a/docs/news.rst b/docs/news.rst index 8f8477eaa..670843e0e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1417,7 +1417,8 @@ Deprecations - ``download_warnsize`` (use :setting:`DOWNLOAD_WARNSIZE`) - - ``max_concurrent_requests`` (use :setting:`CONCURRENT_REQUESTS`) + - ``max_concurrent_requests`` (use + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) - ``user_agent`` (use :setting:`USER_AGENT`) diff --git a/docs/requirements.in b/docs/requirements.in index a1f3a7468..3783dd1dc 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -5,4 +5,4 @@ sphinx sphinx-notfound-page sphinx-rtd-theme sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10 diff --git a/docs/requirements.txt b/docs/requirements.txt index a5cbad302..0f5969401 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0 # via # -r docs/requirements.in # sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@fe176adc1a8577601bc3fa39b590ebed71a7e9b8 # via -r docs/requirements.in sphinx-sitemap==2.9.0 # via sphinx-scrapy diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 63c217e93..afccb491d 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -267,6 +267,7 @@ Here are some examples of APIs and patterns that need a replacement: Scrapy provides unified helpers for some of these examples: +.. autofunction:: scrapy.utils.asyncio.sleep .. autofunction:: scrapy.utils.asyncio.call_later .. autofunction:: scrapy.utils.asyncio.create_looping_call .. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 4f28019da..33289545c 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -106,10 +106,9 @@ delay of its download slot: Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True}) Note, however, that AutoThrottle still determines the starting delay of every -download slot by setting the ``download_delay`` attribute on the running -spider. If you want AutoThrottle not to impact a download slot at all, in -addition to setting this meta key in all requests that use that download slot, -you might want to set a custom value for the ``delay`` attribute of that +download slot. If you want AutoThrottle not to impact a download slot at all, +in addition to setting this meta key in all requests that use that download +slot, you might want to set a custom value for the ``delay`` attribute of that download slot, e.g. using :setting:`DOWNLOAD_SLOTS`. Settings diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index ee2c3a3cd..343193627 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -114,8 +114,8 @@ some usage help and the available commands:: scrapy [options] [args] Available commands: - crawl Run a spider fetch Fetch a URL using the Scrapy downloader + runspider Run a spider from a Python file, no project required [...] The first line will print the currently active project if you're inside a @@ -263,7 +263,9 @@ crawl * Syntax: ``scrapy crawl `` * Requires project: *yes* -Start crawling using a spider. +Start crawling using the spider with the given :attr:`~scrapy.Spider.name`, +which must be one of those that :command:`list` reports. To run a spider from a +file instead, use :command:`runspider`. Supported options: @@ -571,8 +573,9 @@ runspider * Syntax: ``scrapy runspider `` * Requires project: *no* -Run a spider self-contained in a Python file, without having to create a -project. +Run the spider defined in the given Python file, without requiring a project. + +Supported options: the same as :command:`crawl`. Example usage:: @@ -665,6 +668,8 @@ Example: COMMANDS_MODULE = "mybot.commands" +.. note:: This is a :ref:`pre-crawler setting `. + .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html Register commands via setup.py entry points diff --git a/docs/topics/components.rst b/docs/topics/components.rst index c0df86922..354375577 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -9,37 +9,22 @@ A Scrapy component is any class whose objects are built using That includes the classes that you may assign to the following settings: -- :setting:`ADDONS` - -- :setting:`TWISTED_DNS_RESOLVER` - -- :setting:`DOWNLOAD_HANDLERS` - -- :setting:`DOWNLOADER_MIDDLEWARES` - -- :setting:`DUPEFILTER_CLASS` - -- :setting:`EXTENSIONS` - -- :setting:`FEED_EXPORTERS` - -- :setting:`FEED_STORAGES` - -- :setting:`ITEM_PIPELINES` - -- :setting:`SCHEDULER` - -- :setting:`SCHEDULER_DISK_QUEUE` - -- :setting:`SCHEDULER_MEMORY_QUEUE` - -- :setting:`SCHEDULER_PRIORITY_QUEUE` - -- :setting:`SCHEDULER_START_DISK_QUEUE` - -- :setting:`SCHEDULER_START_MEMORY_QUEUE` - -- :setting:`SPIDER_MIDDLEWARES` +- :setting:`ADDONS` +- :setting:`DOWNLOAD_HANDLERS` +- :setting:`DOWNLOADER_MIDDLEWARES` +- :setting:`DUPEFILTER_CLASS` +- :setting:`EXTENSIONS` +- :setting:`FEED_EXPORTERS` +- :setting:`FEED_STORAGES` +- :setting:`ITEM_PIPELINES` +- :setting:`SCHEDULER` +- :setting:`SCHEDULER_DISK_QUEUE` +- :setting:`SCHEDULER_MEMORY_QUEUE` +- :setting:`SCHEDULER_PRIORITY_QUEUE` +- :setting:`SCHEDULER_START_DISK_QUEUE` +- :setting:`SCHEDULER_START_MEMORY_QUEUE` +- :setting:`SPIDER_MIDDLEWARES` +- :setting:`TWISTED_DNS_RESOLVER` Third-party Scrapy components may also let you define additional Scrapy components, usually configurable through :ref:`settings `, to diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 9dcd9d69c..b7ddb0a57 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -21,7 +21,9 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): .. versionadded:: 2.13 -- :class:`~scrapy.Request` callbacks. +- :class:`~scrapy.Request` :ref:`callbacks `, which may + also be defined as :term:`asynchronous generators `. - The :meth:`process_item` method of :ref:`item pipelines `. diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 34ab4f105..e0501c169 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -78,33 +78,15 @@ Writing your own download handler A download handler is a :ref:`component ` that defines the following API: -.. class:: SampleDownloadHandler - - .. attribute:: lazy - :type: bool - - If ``False``, the handler will be instantiated when Scrapy is - initialized. - - If ``True``, the handler will only be instantiated when the first - request handled by it needs to be downloaded. - - .. method:: download_request(request: Request) -> Response - :async: - - Download the given request and return a response. - - .. method:: close() -> None - :async: - - Clean up any resources used by the handler. +.. autoclass:: scrapy.core.downloader.handlers.DownloadHandlerProtocol + :members: An optional base class for custom handlers is provided: .. autoclass:: scrapy.core.downloader.handlers.base.BaseDownloadHandler :members: :undoc-members: - :member-order: bysource + :exclude-members: close, download_request, lazy .. _download-handlers-exceptions: diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 291855e6d..141de7a0d 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -567,6 +567,10 @@ defines the methods described below. Return response if present in cache, or ``None`` otherwise. + If this method raises an exception, e.g. because the cache entry is + corrupted, the middleware logs a warning and handles the request as a + cache miss. + :param spider: the spider which generated the request :type spider: :class:`~scrapy.Spider` object diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index ecd154122..c43b7e20f 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -211,6 +211,16 @@ BaseItemExporter - ``None`` (all fields [2]_, default) + Fields are exported in declaration order, i.e. the order in which + they are defined in the :ref:`item class `. For + :class:`dict` items, which have no declared fields, the key order of + each item is used instead. + + .. versionchanged:: VERSION + Fields of non-\ :class:`dict` items used to be exported in the + order in which they had been populated, except in + :class:`CsvItemExporter`, which has always used declaration order. + - A list of fields: .. code-block:: python diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 5598ab983..78b38cc3f 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -136,18 +136,10 @@ Core Stats extension Enable the collection of core statistics, provided the stats collection is enabled (see :ref:`topics-stats`). -The following stats are collected: - -* ``start_time``: start date/time of the crawl (:class:`~datetime.datetime`). -* ``finish_time``: end date/time of the crawl (:class:`~datetime.datetime`). -* ``elapsed_time_seconds``: total crawl duration in seconds (:class:`float`). -* ``finish_reason``: the closing reason string (e.g. ``"finished"``, - ``"closespider_timeout"``). -* ``item_scraped_count``: total number of items that passed all pipelines. -* ``item_dropped_count``: total number of items dropped by a pipeline. -* ``item_dropped_reasons_count/``: per-exception drop count - (e.g. ``item_dropped_reasons_count/DropItem``). -* ``response_received_count``: total number of HTTP responses received. +The following stats are collected: :stat:`elapsed_time_seconds`, +:stat:`finish_reason`, :stat:`finish_time`, :stat:`item_dropped_count`, +:stat:`item_dropped_reasons_count/{exception}`, :stat:`item_scraped_count`, +:stat:`response_received_count`, :stat:`start_time`. Log Count extension ~~~~~~~~~~~~~~~~~~~ @@ -190,7 +182,7 @@ Monitors the memory used by the Scrapy process that runs the spider and: 1. sends a :signal:`memusage_warning_reached` signal when it exceeds :setting:`MEMUSAGE_WARNING_MB` -2. closes the spider with the `"memusage_exceeded"` reason when it exceeds +2. closes the spider with the ``"memusage_exceeded"`` reason when it exceeds :setting:`MEMUSAGE_LIMIT_MB` This extension is enabled by the :setting:`MEMUSAGE_ENABLED` setting and @@ -214,7 +206,8 @@ An extension for debugging memory usage. It collects information about: * objects left alive that shouldn't. For more info, see :ref:`topics-leaks-trackrefs` To enable this extension, turn on the :setting:`MEMDEBUG_ENABLED` setting. The -info will be stored in the stats. +info will be stored in the :stat:`memdebug/gc_garbage_count` and +:stat:`memdebug/live_refs/{cls}` stats. .. _topics-extensions-ref-spiderstate: diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 66768c97b..2f686fd0f 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -218,12 +218,13 @@ passed through the following settings: .. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html -You can also define a custom ACL, custom endpoint, and region name for exported -feeds using these settings: +You can also define a custom ACL, custom endpoint, region name and connection +pool size for exported feeds using these settings: - :setting:`FEED_STORAGE_S3_ACL` - :setting:`AWS_ENDPOINT_URL` - :setting:`AWS_REGION_NAME` +- :setting:`AWS_MAX_POOL_CONNECTIONS` The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index c9916110d..c3043204b 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -83,6 +83,14 @@ stopping it cleanly. Forced, sudden or otherwise unclean shutdown can lead to data corruption in the job directory, which may prevent the spider from resuming correctly. +Scrapy version changes +---------------------- + +The contents of a job directory are an implementation detail of the Scrapy +version that wrote them. A job must be resumed with the same Scrapy version +that paused it; after upgrading or downgrading Scrapy, start a new job with a +new job directory. + Cookies expiration ------------------ @@ -96,9 +104,13 @@ Request serialization --------------------- For persistence to work, :class:`~scrapy.Request` objects must be -serializable with :mod:`pickle`, except for the ``callback`` and ``errback`` -values passed to their ``__init__`` method, which must be methods of the -running :class:`~scrapy.Spider` class. +serializable with :mod:`pickle`, except for the :ref:`callback +` and :ref:`errback +` values passed to their ``__init__`` +method, which must be methods of the running :class:`~scrapy.Spider` class. + +Requests that cannot be serialized are kept in memory only: they are still +sent, but they are lost when the crawl is paused. If you wish to log the requests that couldn't be serialized, you can set the :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 4ceb4732a..b16066d0c 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -268,6 +268,9 @@ For self-hosting you also might feel the need not to use SSL and not to verify S AWS_USE_SSL = False # or True (None by default) AWS_VERIFY = False # or True (None by default) +To reuse connections for as many files as you check or upload in parallel, set +:setting:`AWS_MAX_POOL_CONNECTIONS` accordingly. + .. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl .. _Minio: https://github.com/minio/minio .. _Zenko CloudServer: https://www.zenko.io/cloudserver/ diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 23738c98c..dfa1e21f6 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -533,8 +533,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites: * if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites directly * use a pool of rotating IPs. For example, the free `Tor project`_ or paid - services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a - super proxy that you can attach your own proxies to. + services like `ProxyMesh`_. * for HTTPS websites, if blocking appears related to TLS behavior, consider adjusting the :setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION` settings, since some websites may respond @@ -559,5 +558,4 @@ projects that detects common mistakes and anti-patterns. .. _ProxyMesh: https://proxymesh.com/ .. _Common Crawl: https://commoncrawl.org/ .. _testspiders: https://github.com/scrapinghub/testspiders -.. _scrapoxy: https://scrapoxy.io/ .. _Zyte API: https://docs.zyte.com/zyte-api/get-started.html diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 14c5c541f..a9e50095c 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -205,10 +205,11 @@ Request objects Request metadata can also be accessed through the :attr:`~scrapy.http.Response.meta` attribute of a response. - To pass data from one spider callback to another, consider using - :attr:`cb_kwargs` instead. However, request metadata may be the right - choice in certain scenarios, such as to maintain some debugging data - across all follow-up requests (e.g. the source URL). + To pass your own data from one spider callback to another, use + :attr:`cb_kwargs` instead, see :ref:`callback-data`. However, request + metadata may be the right choice in certain scenarios, such as to + maintain some debugging data across all follow-up requests (e.g. the + source URL). A common use of request metadata is to define request-specific parameters for Scrapy components (extensions, middlewares, etc.). For @@ -248,7 +249,7 @@ Request objects .. method:: Request.copy() Return a new Request which is a copy of this Request. See also: - :ref:`topics-request-response-ref-request-callback-arguments`. + :ref:`callback-data`. .. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs, cls]) @@ -256,10 +257,12 @@ Request objects given new values by whichever keyword arguments are specified. The :attr:`~scrapy.Request.cb_kwargs` and :attr:`~scrapy.Request.meta` attributes are shallow copied by default (unless new values are given as arguments). See also - :ref:`topics-request-response-ref-request-callback-arguments`. + :ref:`callback-data`. .. automethod:: from_curl + .. automethod:: to_curl + .. automethod:: to_dict @@ -345,160 +348,6 @@ Other functions related to requests .. autofunction:: scrapy.utils.httpobj.urlparse_cached -.. _topics-request-response-ref-request-callback-arguments: - -Passing additional data to callback functions ---------------------------------------------- - -The callback of a request is a function that will be called when the response -of that request is downloaded. The callback function will be called with the -downloaded :class:`Response` object as its first argument. - -Example: - -.. code-block:: python - - def parse_page1(self, response): - return scrapy.Request( - "http://www.example.com/some_page.html", callback=self.parse_page2 - ) - - - def parse_page2(self, response): - # this would log http://www.example.com/some_page.html - self.logger.info("Visited %s", response.url) - -In some cases you may be interested in passing arguments to those callback -functions so you can receive the arguments later, in the second callback. -The following example shows how to achieve this by using the -:attr:`.Request.cb_kwargs` attribute: - -.. code-block:: python - - def parse(self, response): - request = scrapy.Request( - "http://www.example.com/index.html", - callback=self.parse_page2, - cb_kwargs=dict(main_url=response.url), - ) - request.cb_kwargs["foo"] = "bar" # add more arguments for the callback - yield request - - - def parse_page2(self, response, main_url, foo): - yield dict( - main_url=main_url, - other_url=response.url, - foo=foo, - ) - -.. caution:: :attr:`.Request.cb_kwargs` was introduced in version ``1.7``. - Prior to that, using :attr:`.Request.meta` was recommended for passing - information around callbacks. After ``1.7``, :attr:`.Request.cb_kwargs` - became the preferred way for handling user information, leaving :attr:`.Request.meta` - for communication with components like middlewares and extensions. - -.. _topics-request-response-ref-errbacks: - -Using errbacks to catch exceptions in request processing --------------------------------------------------------- - -The errback of a request is a function that will be called when an exception -is raise while processing it. - -It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can -be used to track connection establishment timeouts, DNS errors etc. - -Here's an example spider logging all errors and catching some specific -errors if needed: - -.. code-block:: python - - import scrapy - - from scrapy.spidermiddlewares.httperror import HttpError - from twisted.internet.error import DNSLookupError - from twisted.internet.error import TimeoutError, TCPTimedOutError - - - class ErrbackSpider(scrapy.Spider): - name = "errback_example" - start_urls = [ - "http://www.httpbin.org/", # HTTP 200 expected - "http://www.httpbin.org/status/404", # Not found error - "http://www.httpbin.org/status/500", # server issue - "http://www.httpbin.org:12345/", # non-responding host, timeout expected - "https://example.invalid/", # DNS error expected - ] - - async def start(self): - for u in self.start_urls: - yield scrapy.Request( - u, - callback=self.parse_httpbin, - errback=self.errback_httpbin, - dont_filter=True, - ) - - def parse_httpbin(self, response): - self.logger.info(f"Got successful response from {response.url}") - # do something useful here... - - def errback_httpbin(self, failure): - # log all failures - self.logger.error(repr(failure)) - - # in case you want to do something special for some errors, - # you may need the failure's type: - - if failure.check(HttpError): - # these exceptions come from HttpError spider middleware - # you can get the non-200 response - response = failure.value.response - self.logger.error("HttpError on %s", response.url) - - elif failure.check(DNSLookupError): - # this is the original request - request = failure.request - self.logger.error("DNSLookupError on %s", request.url) - - elif failure.check(TimeoutError, TCPTimedOutError): - request = failure.request - self.logger.error("TimeoutError on %s", request.url) - - -.. _errback-cb_kwargs: - -Accessing additional data in errback functions ----------------------------------------------- - -In case of a failure to process the request, you may be interested in -accessing arguments to the callback functions so you can process further -based on the arguments in the errback. The following example shows how to -achieve this by using ``Failure.request.cb_kwargs``: - -.. code-block:: python - - def parse(self, response): - request = scrapy.Request( - "http://www.example.com/index.html", - callback=self.parse_page2, - errback=self.errback_page2, - cb_kwargs=dict(main_url=response.url), - ) - yield request - - - def parse_page2(self, response, main_url): - pass - - - def errback_page2(self, failure): - yield dict( - main_url=failure.request.cb_kwargs["main_url"], - ) - - .. _request-fingerprints: Request fingerprints @@ -700,6 +549,319 @@ The following built-in Scrapy components have such restrictions: 45-character-long keys must be supported. +.. _callbacks: + +Callbacks +========= + +A callback is a function that Scrapy calls with the :class:`Response` of a +:class:`~scrapy.Request` once that request has been downloaded, so that you can +extract data from that response and generate additional requests to continue +the crawl: + +.. code-block:: python + + from scrapy import Request, Spider + + + class BookSpider(Spider): + name = "books" + + async def start(self): + yield Request("https://books.toscrape.com/", callback=self.parse_home) + + def parse_home(self, response): + for url in response.css("h3 a::attr(href)").getall(): + yield Request(response.urljoin(url), callback=self.parse_book) + + def parse_book(self, response): + yield {"title": response.css("h1::text").get()} + +Requests may also define an :ref:`errback `, which Scrapy calls +instead of the callback when an exception is raised while processing the +request or its response, e.g. a connection error or, by default, a non-2xx +response. + + +.. _callback-assignment: + +Assigning a callback to a request +--------------------------------- + +To assign a callback to a request, use the ``callback`` parameter of +:class:`~scrapy.Request`, which sets the :attr:`.Request.callback` attribute: + +.. code-block:: python + + from scrapy import Request + + + def parse_home(response): ... + + + request = Request("https://books.toscrape.com/", callback=parse_home) + +Requests with no callback, i.e. with :attr:`~scrapy.Request.callback` set to +``None``, are handled by the :meth:`~scrapy.Spider.parse` method of the spider: + +.. code-block:: python + + request = Request("https://books.toscrape.com/") # Handled by parse() + +If a request is never meant to reach a spider callback, e.g. because a +:ref:`component ` sends it and handles its response itself, +assign the special :func:`~scrapy.http.request.NO_CALLBACK` value to it +instead, so that :ref:`downloader middlewares ` +can tell such requests apart. + +While :attr:`~scrapy.Request.callback` only accepts callables, some spider +classes let you also define a callback by name: both :attr:`CrawlSpider.rules +` and :attr:`SitemapSpider.sitemap_rules +` accept the name of a spider +method as a string. + + +.. _writing-callbacks: + +Writing a callback +------------------ + +Any callable can be a callback, as long as it takes the response as its first +positional parameter, and any :ref:`additional callback data ` +as keyword parameters. Spider methods are the most common choice, but plain +functions, lambda expressions and other callable objects work as well. + +.. note:: If you enable :ref:`job persistence ` through the + :setting:`JOBDIR` setting, callbacks must be methods of the running spider. + Requests with any other callback cannot be serialized, so they are kept in + memory only and lost when you pause the crawl. See + :ref:`request-serialization`. + +A callback can be: + +- A regular function: + + .. code-block:: python + + def parse(self, response): + return {"url": response.url} + +- A generator function: + + .. code-block:: python + + def parse(self, response): + yield {"url": response.url} + +- A coroutine function, i.e. defined with ``async def``: + + .. code-block:: python + + async def parse(self, response): + return {"url": response.url} + +- An asynchronous generator function: + + .. code-block:: python + + async def parse(self, response): + yield {"url": response.url} + +The last two allow using ``await``, ``async for`` and ``async with`` in your +callback. See :ref:`topics-coroutines`. + + +.. _callback-output: + +Callback output +--------------- + +A callback may return or yield any of the following: + +- ``None``, which does nothing. + + Callbacks that produce no output at all, e.g. callbacks that only log + information about the response, are perfectly valid. ``None`` values within + an iterable of callback output are ignored as well. + +- A :class:`~scrapy.Request` object, which Scrapy schedules, downloads and + eventually sends to its own callback. + +- An :ref:`item object `, which Scrapy sends to the + :ref:`item pipelines `. + + Any object that is neither ``None`` nor a :class:`~scrapy.Request` object + is treated as an item. + +- An iterable of any of the values above, e.g. a list or, more commonly, a + generator. + + :term:`Asynchronous iterables `, e.g. an + :term:`asynchronous generator`, are also supported. + +.. note:: When a callback *returns* an object, Scrapy iterates that object if + it supports iteration, except for :class:`dict`, :class:`~scrapy.Item`, + :class:`str` and :class:`bytes` objects, which are always handled as single + items. + +.. note:: In a generator callback, a ``return`` statement with a value does not + produce any output, since such a value is not part of what the generator + yields. Scrapy logs a warning when it detects such a callback, see + :setting:`WARN_ON_GENERATOR_RETURN_VALUE`. + +Before Scrapy acts on the output of a callback, that output goes through the +:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method +of your :ref:`spider middlewares `, which may modify +it or drop part of it. + +If a callback raises an exception, the :attr:`~scrapy.Request.errback` of the +request is *not* called. The exception goes through the +:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception` +method of your spider middlewares instead and, unless one of them handles it, +Scrapy logs it and sends the :signal:`spider_error` signal. + + +.. _callback-data: +.. _topics-request-response-ref-request-callback-arguments: + +Passing additional data to callback functions +--------------------------------------------- + +In some cases you may be interested in passing data to a callback in addition +to the response, e.g. data extracted from the response that triggered the +request. The following example shows how to achieve this by using the +:attr:`.Request.cb_kwargs` attribute: + +.. code-block:: python + + from scrapy import Request + + + def parse(self, response): + request = Request( + "http://www.example.com/index.html", + callback=self.parse_page2, + cb_kwargs=dict(main_url=response.url), + ) + request.cb_kwargs["foo"] = "bar" # add more arguments for the callback + yield request + + + def parse_page2(self, response, main_url, foo): + yield dict( + main_url=main_url, + other_url=response.url, + foo=foo, + ) + +:attr:`.Request.cb_kwargs` is the recommended way to pass your own data to a +callback. Use :attr:`.Request.meta` only for data aimed at :ref:`components +`, such as middlewares and extensions. + +.. _errbacks: +.. _topics-request-response-ref-errbacks: + +Errbacks +======== + +The errback of a request is a function that will be called when an exception +is raise while processing it. + +It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can +be used to track connection establishment timeouts, DNS errors etc. + +Here's an example spider logging all errors and catching some specific +errors if needed: + +.. code-block:: python + + from scrapy import Request, Spider + from scrapy.spidermiddlewares.httperror import HttpError + from twisted.internet.error import DNSLookupError + from twisted.internet.error import TimeoutError, TCPTimedOutError + + + class ErrbackSpider(Spider): + name = "errback_example" + start_urls = [ + "http://www.httpbin.org/", # HTTP 200 expected + "http://www.httpbin.org/status/404", # Not found error + "http://www.httpbin.org/status/500", # server issue + "http://www.httpbin.org:12345/", # non-responding host, timeout expected + "https://example.invalid/", # DNS error expected + ] + + async def start(self): + for u in self.start_urls: + yield Request( + u, + callback=self.parse_httpbin, + errback=self.errback_httpbin, + dont_filter=True, + ) + + def parse_httpbin(self, response): + self.logger.info(f"Got successful response from {response.url}") + # do something useful here... + + def errback_httpbin(self, failure): + # log all failures + self.logger.error(repr(failure)) + + # in case you want to do something special for some errors, + # you may need the failure's type: + + if failure.check(HttpError): + # these exceptions come from HttpError spider middleware + # you can get the non-200 response + response = failure.value.response + self.logger.error("HttpError on %s", response.url) + + elif failure.check(DNSLookupError): + # this is the original request + request = failure.request + self.logger.error("DNSLookupError on %s", request.url) + + elif failure.check(TimeoutError, TCPTimedOutError): + request = failure.request + self.logger.error("TimeoutError on %s", request.url) + + +.. _errback-cb_kwargs: + +Accessing additional data in errback functions +---------------------------------------------- + +In case of a failure to process the request, you may be interested in +accessing arguments to the callback functions so you can process further +based on the arguments in the errback. The following example shows how to +achieve this by using ``Failure.request.cb_kwargs``: + +.. code-block:: python + + from scrapy import Request + + + def parse(self, response): + request = Request( + "http://www.example.com/index.html", + callback=self.parse_page2, + errback=self.errback_page2, + cb_kwargs=dict(main_url=response.url), + ) + yield request + + + def parse_page2(self, response, main_url): + pass + + + def errback_page2(self, failure): + yield dict( + main_url=failure.request.cb_kwargs["main_url"], + ) + + .. _topics-request-meta: Request.meta special keys diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 1ae20645f..f113a490c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -305,10 +305,21 @@ These settings cannot be :ref:`set from a spider `. These settings are: -- :setting:`TWISTED_REACTOR_ENABLED` +- :setting:`ADDONS` +- :setting:`COMMANDS_MODULE` +- :setting:`FORCE_CRAWLER_PROCESS` - :setting:`SPIDER_LOADER_CLASS` and settings used by the corresponding spider loader class, e.g. :setting:`SPIDER_MODULES` and :setting:`SPIDER_LOADER_WARN_ONLY` for the default spider loader class. +- :setting:`TWISTED_REACTOR_ENABLED` + +:setting:`ADDONS` is a special case: it can be set from a spider, but the +``update_pre_crawler_settings()`` method of :ref:`add-ons ` +enabled that way is not called. + +:setting:`TWISTED_REACTOR` also acts as a pre-crawler setting when running a +:ref:`command that needs a CrawlerProcess `, +since its project-level value determines the crawler process class. .. _reactor-settings: @@ -409,6 +420,9 @@ Default: ``{}`` A dict containing paths to the add-ons enabled in your project and their priorities. For more information, see :ref:`topics-addons`. +.. note:: This is a :ref:`pre-crawler setting `, with a + caveat described in that section. + .. setting:: ASYNCIO_EVENT_LOOP ASYNCIO_EVENT_LOOP @@ -458,6 +472,26 @@ Default: ``None`` Endpoint URL used for S3-like storage, for example Minio or s3.scality. +.. setting:: AWS_MAX_POOL_CONNECTIONS + +AWS_MAX_POOL_CONNECTIONS +------------------------ + +.. versionadded:: VERSION + +Default: ``None`` + +Maximum number of connections that AWS clients, such as those of the +:ref:`S3 feed storage backend ` and of the +:ref:`S3 media pipeline storage backend `, keep in their +connection pool. + +If ``None``, the value of :setting:`REACTOR_THREADPOOL_MAXSIZE` is used. + +Values lower than the number of parallel AWS calls do not limit those calls, but +their connections are closed instead of reused, which hurts performance, and +``Connection pool is full, discarding connection`` warnings are logged. + .. setting:: AWS_REGION_NAME AWS_REGION_NAME @@ -541,7 +575,7 @@ CONCURRENT_REQUESTS Default: ``16`` The maximum number of concurrent (i.e. simultaneous) requests that will be -performed by the Scrapy downloader. +performed by the Scrapy downloader. Use ``0`` for no limit. .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN @@ -920,10 +954,6 @@ desired. .. _spider-download_delay-attribute: -.. note:: - - This delay can be set per spider using :attr:`download_delay` spider attribute. - It is possible to change this setting per domain by using :setting:`DOWNLOAD_SLOTS`. @@ -1383,6 +1413,8 @@ When :setting:`TWISTED_REACTOR_ENABLED` is set to ``False``, Set this to ``True`` if you want to set :setting:`TWISTED_REACTOR` to a non-default value in :ref:`per-spider settings `. +.. note:: This is a :ref:`pre-crawler setting `. + .. setting:: FTP_PASSIVE_MODE FTP_PASSIVE_MODE @@ -1849,7 +1881,8 @@ Default: ``False`` Setting to ``True`` will log debug information about the requests scheduler. This currently logs (only once) if the requests cannot be serialized to disk. -Stats counter (``scheduler/unserializable``) tracks the number of times this happens. +The :stat:`scheduler/unserializable` stat tracks the number of times this +happens. Example entry in logs:: diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 03996bee6..f7f9f5cca 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -504,6 +504,27 @@ headers_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.Spider` object +robots_parsed +~~~~~~~~~~~~~ + +.. signal:: robots_parsed +.. function:: robots_parsed(robotparser, request) + + .. versionadded:: VERSION + + Sent by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` after it + downloads and parses a :file:`robots.txt` file, for the host that *request* + targets. + + This signal supports :ref:`asynchronous handlers `. + + :param robotparser: the parser holding the parsed :file:`robots.txt` contents + :type robotparser: :class:`~scrapy.robotstxt.RobotParser` object + + :param request: the request that triggered the :file:`robots.txt` download + :type request: :class:`~scrapy.Request` object + Response signals ---------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 69ff08fa1..8fbf0c52d 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -4,43 +4,31 @@ Spiders ======= -Spiders are classes which define how a certain site (or a group of sites) will be -scraped, including how to perform the crawl (i.e. follow links) and how to -extract structured data from their pages (i.e. scraping items). In other words, -Spiders are the place where you define the custom behaviour for crawling and -parsing pages for a particular site (or, in some cases, a group of sites). +Spiders are classes that define how a site, or a group of sites, is scraped: +which requests to send, and how to parse their responses to extract data and to +send additional requests. -For spiders, the scraping cycle goes through something like this: +A crawl goes as follows: -1. You start by generating the initial requests to crawl the first URLs, and - specify a callback function to be called with the response downloaded from - those requests. +1. Scrapy iterates the :meth:`~scrapy.Spider.start` method of the spider to + get the initial requests. By default, that method yields a + :class:`~scrapy.Request` object for each URL in + :attr:`~scrapy.Spider.start_urls`, with :meth:`~scrapy.Spider.parse` as + :ref:`callback `. - The first requests to perform are obtained by iterating the - :meth:`~scrapy.Spider.start` method, which by default yields a - :class:`~scrapy.Request` object for each URL in the - :attr:`~scrapy.Spider.start_urls` spider attribute, with the - :attr:`~scrapy.Spider.parse` method set as :attr:`~scrapy.Request.callback` - function to handle each :class:`~scrapy.http.Response`. +2. Scrapy downloads each request and calls its callback with the resulting + :class:`~scrapy.http.Response`. -2. In the callback function, you parse the response (web page) and return - :ref:`item objects `, - :class:`~scrapy.Request` objects, or an iterable of these objects. - Those Requests will also contain a callback (maybe - the same) and will then be downloaded by Scrapy and then their - response handled by the specified callback. +3. Callbacks parse the response, typically using :ref:`topics-selectors`, and + return or yield :ref:`item objects ` with the extracted data + and :class:`~scrapy.Request` objects to continue the crawl, which go back + to step 2. See :ref:`callback-output`. -3. In callback functions, you parse the page contents, typically using - :ref:`topics-selectors` (but you can also use BeautifulSoup, lxml or whatever - mechanism you prefer) and generate items with the parsed data. +4. Items go through :ref:`item pipelines `, and are + usually stored through :ref:`topics-feed-exports`. -4. Finally, the items returned from the spider will be typically persisted to a - database (in some :ref:`Item Pipeline `) or written to - a file using :ref:`topics-feed-exports`. - -Even though this cycle applies (more or less) to any kind of spider, there are -different kinds of default spiders bundled into Scrapy for different purposes. -We will talk about those types here. +Scrapy includes different spider classes for different purposes, described +below. .. _topics-spiders-ref: @@ -191,22 +179,7 @@ scrapy.Spider .. automethod:: start - .. method:: parse(response) - - This is the default callback used by Scrapy to process downloaded - responses, when their requests don't specify a callback. - - The ``parse`` method is in charge of processing the response and returning - scraped data and/or more URLs to follow. Other Requests callbacks have - the same requirements as the :class:`~scrapy.Spider` class. - - This method, as well as any other Request callback, must return a - :class:`~scrapy.Request` object, an :ref:`item object `, an - iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects - `, or ``None``. - - :param response: the response to parse - :type response: :class:`~scrapy.http.Response` + .. automethod:: parse .. method:: closed(reason) diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 0cf4a72cc..c702cefe7 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -21,6 +21,8 @@ using the Stats Collector from. Another feature of the Stats Collector is that it's very efficient (when enabled) and extremely efficient (almost unnoticeable) when disabled. +See :ref:`topics-stats-reference` below for the stats that Scrapy sets. + .. _topics-stats-usecases: Common Stats Collector uses @@ -101,3 +103,642 @@ DummyStatsCollector ------------------- .. autoclass:: DummyStatsCollector + +.. _topics-stats-reference: + +Built-in stats reference +======================== + +Scrapy sets the following :ref:`stats `. Components other than +those built into Scrapy may set additional stats; see their documentation. + +Stat keys that contain a ``{placeholder}`` below stand for a family of stats, +one per actual value of the placeholder. + +.. note:: Most stats are set by a specific :ref:`component + `, and are only present if that component is enabled and + its code path is reached. A stat that is missing from + :meth:`~scrapy.statscollectors.StatsCollector.get_stats` output is + equivalent to a counter of 0. + +.. stat:: downloader/exception_count + +``downloader/exception_count`` + Number of exceptions raised while downloading requests. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/exception_type_count/{exception_type} + +``downloader/exception_type_count/{exception_type}`` + Number of exceptions raised while downloading requests, per exception type, + where ``{exception_type}`` is the import path of the exception class, e.g. + ``twisted.internet.error.DNSLookupError``. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/request_bytes + +``downloader/request_bytes`` + Total size, in bytes, of the requests sent, counting the request line, the + headers and the body. As with :stat:`downloader/request_count`, requests + served from the cache are also counted. + + It is an approximation, reconstructed from each :class:`~scrapy.Request` + object instead of measured on the wire, so it does not account for the + actual bytes that the :ref:`download handler + ` sends, e.g. transport-level overhead. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/request_count + +``downloader/request_count`` + Number of requests sent. + + Requests that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + serves from the cache are also counted, even though they are never sent, + because it handles requests after + :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/request_method_count/{method} + +``downloader/request_method_count/{method}`` + Number of requests sent, per HTTP method, e.g. ``GET`` or ``POST``. As with + :stat:`downloader/request_count`, requests served from the cache are also + counted. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/response_bytes + +``downloader/response_bytes`` + Total size, in bytes, of the responses received, counting the status line, + the headers and the body. It covers the same responses as + :stat:`downloader/response_count`. + + The body is counted as received, i.e. still compressed for responses that + used ``Content-Encoding``, because + :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` handles + responses before + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + decompresses them. See :stat:`httpcompression/response_bytes` for + decompressed sizes. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/response_count + +``downloader/response_count`` + Number of responses received. + + It counts responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + serves from the cache, even though they do not come from the network, and + responses that a downloader middleware consumes before they reach your + spider, e.g. redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware` + turns into new requests. Compare with :stat:`response_received_count`. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/response_status_count/{status_code} + +``downloader/response_status_count/{status_code}`` + Number of responses received, per HTTP status code, e.g. ``200`` or + ``404``. It covers the same responses as :stat:`downloader/response_count`. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: dupefilter/filtered + +``dupefilter/filtered`` + Number of requests dropped as duplicates. + + Set by :class:`~scrapy.dupefilters.RFPDupeFilter`. + +.. stat:: elapsed_time_seconds + +``elapsed_time_seconds`` + Time, as a :class:`float`, in seconds, between the :signal:`spider_opened` + and the :signal:`spider_closed` signals. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: feedexport/failed_count/{storage} + +``feedexport/failed_count/{storage}`` + Number of :ref:`feeds ` that could not be stored, per + :ref:`storage backend `, where ``{storage}`` + is the class name of the storage backend, e.g. ``FileFeedStorage``. + +.. stat:: feedexport/success_count/{storage} + +``feedexport/success_count/{storage}`` + Number of :ref:`feeds ` stored successfully, per + :ref:`storage backend `, where ``{storage}`` + is the class name of the storage backend, e.g. ``FileFeedStorage``. + +.. stat:: file_count + +``file_count`` + Number of files handled by the :ref:`media pipelines + `. + +.. stat:: file_status_count/{status} + +``file_status_count/{status}`` + Number of files handled by the :ref:`media pipelines + `, per status, where ``{status}`` is one of: + + - ``downloaded``: the file was downloaded. + + - ``cached``: the file came from the + :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + cache. + + - ``uptodate``: the file was already in the storage backend and had not + :ref:`expired `, so it was not downloaded again. + +.. stat:: finish_reason + +``finish_reason`` + String indicating why the crawl finished. It matches the *reason* argument + of the :signal:`spider_closed` signal. + + Scrapy uses the following reasons: + + - ``cancelled``: the spider was closed without a more specific reason, + e.g. because :exc:`~scrapy.exceptions.CloseSpider` was raised without + one. + + - ``closespider_errorcount``: see :setting:`CLOSESPIDER_ERRORCOUNT`. + + - ``closespider_itemcount``: see :setting:`CLOSESPIDER_ITEMCOUNT`. + + - ``closespider_pagecount``: see :setting:`CLOSESPIDER_PAGECOUNT`. + + - ``closespider_pagecount_no_item``: see + :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM`. + + - ``closespider_timeout``: see :setting:`CLOSESPIDER_TIMEOUT`. + + - ``closespider_timeout_no_item``: see + :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`. + + - ``finished``: the spider became idle with no pending requests, i.e. it + finished normally. + + - ``memusage_exceeded``: see :setting:`MEMUSAGE_LIMIT_MB`. + + - ``shutdown``: the crawl was interrupted, e.g. by a system signal such + as ``SIGINT`` (:kbd:`Ctrl-C`). + + Third-party components and your own code may use any other reason, e.g. by + raising :exc:`~scrapy.exceptions.CloseSpider` with it. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: finish_time + +``finish_time`` + Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when + the :signal:`spider_closed` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: httpcache/errorrecovery + +``httpcache/errorrecovery`` + Number of times that a stale cached response was used because downloading a + fresh response raised an exception. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/firsthand + +``httpcache/firsthand`` + Number of responses that were downloaded without a matching cache entry to + validate against, i.e. responses for requests counted in + :stat:`httpcache/miss`. + + It is lower than :stat:`httpcache/miss` when some of those requests yield + no response, either because they are dropped (see + :stat:`httpcache/ignore`) or because their download fails. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/hit + +``httpcache/hit`` + Number of requests served from the cache. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/ignore + +``httpcache/ignore`` + Number of requests dropped because they were not in the cache and + :setting:`HTTPCACHE_IGNORE_MISSING` is ``True``. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/invalidate + +``httpcache/invalidate`` + Number of times that a cached response failed validation and was replaced + with a freshly downloaded response. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/miss + +``httpcache/miss`` + Number of requests for which no cache entry could be read, either because + there was none or because reading it failed, in which case the request is + also counted in :stat:`httpcache/retrieve_error`. Those requests are + downloaded (see :stat:`httpcache/firsthand`), or dropped if + :setting:`HTTPCACHE_IGNORE_MISSING` is ``True`` (see + :stat:`httpcache/ignore`). + + Requests with a stale cache entry are not counted here; see + :stat:`httpcache/revalidate` and :stat:`httpcache/invalidate`. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/retrieve_error + +``httpcache/retrieve_error`` + Number of cache entries that could not be read, and hence were treated as + cache misses. Those requests are also counted in :stat:`httpcache/miss`. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/revalidate + +``httpcache/revalidate`` + Number of times that a cached response was successfully validated against + the target server, and hence used instead of the fresh response. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/store + +``httpcache/store`` + Number of responses stored in the cache. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/uncacheable + +``httpcache/uncacheable`` + Number of responses not stored in the cache because the + :setting:`HTTPCACHE_POLICY` did not allow it. + + Every response considered for caching is counted either here or in + :stat:`httpcache/store`, so ``httpcache/store + httpcache/uncacheable`` + equals ``httpcache/firsthand + httpcache/invalidate``. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcompression/response_bytes + +``httpcompression/response_bytes`` + Total size, in bytes, of decompressed response bodies, counting only the + body and only responses that were actually decompressed. Compare with + :stat:`downloader/response_bytes`. + + Set by + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`. + +.. stat:: httpcompression/response_count + +``httpcompression/response_count`` + Number of decompressed responses. + + Set by + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`. + +.. stat:: httperror/response_ignored_count + +``httperror/response_ignored_count`` + Number of responses dropped because of their HTTP status code. + + Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`. + +.. stat:: httperror/response_ignored_status_count/{status_code} + +``httperror/response_ignored_status_count/{status_code}`` + Number of responses dropped because of their HTTP status code, per HTTP + status code, e.g. ``404``. + + Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`. + +.. stat:: item_dropped_count + +``item_dropped_count`` + Number of items dropped by an :ref:`item pipeline + `, i.e. number of times that the + :signal:`item_dropped` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: item_dropped_reasons_count/{exception} + +``item_dropped_reasons_count/{exception}`` + Number of items dropped, per exception, where ``{exception}`` is the class + name of the exception that caused the item to be dropped. + + Only :exc:`~scrapy.exceptions.DropItem` and its subclasses drop items, and + each one is counted under its own class name, e.g. + ``item_dropped_reasons_count/DropItem`` for + :exc:`~scrapy.exceptions.DropItem` itself and + ``item_dropped_reasons_count/MyDropItem`` for a ``MyDropItem`` subclass of + it. Any other exception raised by an :ref:`item pipeline + ` triggers the :signal:`item_error` signal instead of + :signal:`item_dropped`, and is not counted here or in + :stat:`item_dropped_count`. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: item_scraped_count + +``item_scraped_count`` + Number of items that passed all :ref:`item pipelines + `, i.e. number of times that the + :signal:`item_scraped` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: items_per_minute + +``items_per_minute`` + Average number of items scraped per minute during the crawl. + + It is ``None`` if the crawl took less than a minute. + + Set by :class:`~scrapy.extensions.logstats.LogStats`. + +.. stat:: log_count/{level} + +``log_count/{level}`` + Number of log messages, per logging level name, e.g. ``INFO`` or + ``WARNING``. + + Only messages that the :setting:`LOG_LEVEL` setting allows are counted. + + Set by :class:`~scrapy.extensions.logcount.LogCount`. + +.. stat:: memdebug/gc_garbage_count + +``memdebug/gc_garbage_count`` + Number of objects in :data:`gc.garbage` when the spider is closed. + + Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires + :setting:`MEMDEBUG_ENABLED` to be ``True``. + +.. stat:: memdebug/live_refs/{cls} + +``memdebug/live_refs/{cls}`` + Number of live objects of class ``{cls}`` when the spider is closed, as + reported by :ref:`trackref `, e.g. + ``memdebug/live_refs/HtmlResponse``. + + Only set for classes with at least 1 live object. + + Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires + :setting:`MEMDEBUG_ENABLED` to be ``True``. + +.. stat:: memusage/limit_reached + +``memusage/limit_reached`` + ``1`` if memory usage exceeded :setting:`MEMUSAGE_LIMIT_MB`, which also + stops the crawl. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: memusage/max + +``memusage/max`` + Maximum peak memory usage, in bytes, observed during the crawl. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: memusage/startup + +``memusage/startup`` + Peak memory usage, in bytes, when the engine started. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: memusage/warning_reached + +``memusage/warning_reached`` + ``1`` if memory usage exceeded :setting:`MEMUSAGE_WARNING_MB`. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: offsite/domains + +``offsite/domains`` + Number of distinct domains for which at least 1 request was dropped for + being offsite. + + Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`. + +.. stat:: offsite/filtered + +``offsite/filtered`` + Number of requests dropped for being offsite. + + Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`. + +.. stat:: request_depth_count/{depth} + +``request_depth_count/{depth}`` + Number of requests scheduled at depth ``{depth}``, e.g. + ``request_depth_count/2``. + + Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`, which + requires :setting:`DEPTH_STATS_VERBOSE` to be ``True`` for this stat. + +.. stat:: request_depth_max + +``request_depth_max`` + Maximum depth reached. + + Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`. + +.. stat:: response_received_count + +``response_received_count`` + Number of responses received, i.e. number of times that the + :signal:`response_received` signal was sent. + + Unlike :stat:`downloader/response_count`, it does not count responses that + a downloader middleware consumes before they reach the engine, e.g. + redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware` + turns into new requests. Both count responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + serves from the cache. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: responses_per_minute + +``responses_per_minute`` + Average number of responses received per minute during the crawl. + + It is ``None`` if the crawl took less than a minute. + + Set by :class:`~scrapy.extensions.logstats.LogStats`. + +.. stat:: retry/count + +``retry/count`` + Number of requests retried. + + Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses. + +.. stat:: retry/max_reached + +``retry/max_reached`` + Number of requests that were not retried because they had already been + retried :setting:`RETRY_TIMES` times. + + Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses. + +.. stat:: retry/reason_count/{reason} + +``retry/reason_count/{reason}`` + Number of requests retried, per reason, e.g. + ``retry/reason_count/twisted.internet.error.TimeoutError`` or + ``retry/reason_count/504 Gateway Time-out``. + + Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses. + +.. note:: Code calling + :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` may pass a + custom *stats_base_key*, in which case ``retry`` is replaced with that key + in the 3 stats above. + +.. stat:: robotstxt/exception_count/{exception_type} + +``robotstxt/exception_count/{exception_type}`` + Number of exceptions raised while downloading ``robots.txt`` files, per + exception type, where ``{exception_type}`` is the string representation of + the exception class, e.g. ````. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/forbidden + +``robotstxt/forbidden`` + Number of requests dropped for being disallowed by ``robots.txt``. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/request_count + +``robotstxt/request_count`` + Number of ``robots.txt`` files requested, i.e. 1 per network location for + which at least 1 request was sent. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/response_count + +``robotstxt/response_count`` + Number of ``robots.txt`` responses received. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/response_status_count/{status_code} + +``robotstxt/response_status_count/{status_code}`` + Number of ``robots.txt`` responses received, per HTTP status code, e.g. + ``404``. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: scheduler/dequeued + +``scheduler/dequeued`` + Number of requests read from the :ref:`scheduler `. + +.. stat:: scheduler/dequeued/disk + +``scheduler/dequeued/disk`` + Number of requests read from the disk queue of the :ref:`scheduler + `. + +.. stat:: scheduler/dequeued/memory + +``scheduler/dequeued/memory`` + Number of requests read from the memory queue of the :ref:`scheduler + `. + +.. stat:: scheduler/enqueued + +``scheduler/enqueued`` + Number of requests stored into the :ref:`scheduler `. + +.. stat:: scheduler/enqueued/disk + +``scheduler/enqueued/disk`` + Number of requests stored into the disk queue of the :ref:`scheduler + `. + +.. stat:: scheduler/enqueued/memory + +``scheduler/enqueued/memory`` + Number of requests stored into the memory queue of the :ref:`scheduler + `. + +.. stat:: scheduler/unserializable + +``scheduler/unserializable`` + Number of requests that could not be stored into the disk queue of the + :ref:`scheduler ` because they could not be + :ref:`serialized `, and hence were stored into the + memory queue instead. + +.. stat:: spider_exceptions/count + +``spider_exceptions/count`` + Number of unhandled exceptions raised by spider callbacks. + + Set by the :ref:`scraper `. + +.. stat:: spider_exceptions/{exception} + +``spider_exceptions/{exception}`` + Number of unhandled exceptions raised by spider callbacks, per exception, + where ``{exception}`` is the class name of the exception, e.g. + ``spider_exceptions/ValueError``. + + Set by the :ref:`scraper `. + +.. stat:: start_time + +``start_time`` + Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when + the :signal:`spider_opened` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: urllength/request_ignored_count + +``urllength/request_ignored_count`` + Number of requests dropped for having a URL longer than + :setting:`URLLENGTH_LIMIT`. + + Set by :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`. diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 8e5001c1d..efb582254 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -16,23 +16,20 @@ class QPSSpider(Spider): name = "qps" benchurl = "http://localhost:8880/" - # Max concurrency is limited by global CONCURRENT_REQUESTS setting - max_concurrent_requests = 8 # Requests per second goal - qps = None # same as: 1 / download_delay - download_delay = None + qps = None # same as: 1 / DOWNLOAD_DELAY # time in seconds to delay server responses latency = None # number of slots to create slots = 1 - def __init__(self, *a, **kw): - super().__init__(*a, **kw) - if self.qps is not None: - self.qps = float(self.qps) - self.download_delay = 1 / self.qps - elif self.download_delay is not None: - self.download_delay = float(self.download_delay) + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + if spider.qps is not None: + spider.qps = float(spider.qps) + crawler.settings.set("DOWNLOAD_DELAY", 1 / spider.qps, priority="spider") + return spider async def start(self): url = self.benchurl diff --git a/pyproject.toml b/pyproject.toml index 85edad6fe..13267e427 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,8 +69,8 @@ brotli = [ gcs = ["google-cloud-storage>=1.29.0"] httpx = ["httpx2[http2,socks]>=2.0.0"] images = ["Pillow>=8.3.2"] -ipython = ["ipython>=7.1.0"] -ptpython = ["ptpython>=2.0.1"] +ipython = ["ipython>=8.15.0"] +ptpython = ["ptpython>=3.0.23"] robotparser = ["robotexclusionrulesparser>=1.6.2"] s3 = ["boto3>=1.20.0"] twisted-http2 = ["Twisted[http2]>=21.7.0"] @@ -118,23 +118,10 @@ allow_incomplete_defs = true # 59 errors # TODO [[tool.mypy.overrides]] module = [ - "tests.mockserver.*", "tests.spiders", "tests.test_closespider", "tests.test_cmdline", "tests.test_contracts", - "tests.test_core_downloader", - "tests.test_downloader_handler_twisted_ftp", - "tests.test_downloadermiddleware_cookies", - "tests.test_downloadermiddleware_httpcache", - "tests.test_downloadermiddleware_httpcompression", - "tests.test_downloadermiddleware_httpproxy", - "tests.test_downloadermiddleware_offsite", - "tests.test_downloadermiddleware_redirect", - "tests.test_downloadermiddleware_redirect_base", - "tests.test_downloadermiddleware_redirect_metarefresh", - "tests.test_downloadermiddleware_retry", - "tests.test_downloadermiddleware_robotstxt", "tests.test_downloaderslotssettings", "tests.test_dupefilters", "tests.test_engine_loop", @@ -145,12 +132,6 @@ module = [ "tests.test_feedexport_postprocess", "tests.test_feedexport_storages", "tests.test_feedexport_uri_params", - "tests.test_http2_client_protocol", - "tests.test_http_headers", - "tests.test_http_request", - "tests.test_http_request_form", - "tests.test_http_response", - "tests.test_http_response_text", "tests.test_item", "tests.test_linkextractors", "tests.test_loader", @@ -162,11 +143,6 @@ module = [ "tests.test_pipeline_media", "tests.test_pipelines", "tests.test_pqueues", - "tests.test_request_attribute_binding", - "tests.test_request_cb_kwargs", - "tests.test_request_dict", - "tests.test_request_left", - "tests.test_robotstxt_interface", "tests.test_scheduler_base", "tests.test_settings", "tests.test_spider", @@ -177,15 +153,6 @@ module = [ "tests.test_squeues", "tests.test_squeues_request", "tests.test_stats", - "tests.test_utils_datatypes", - "tests.test_utils_decorators", - "tests.test_utils_defer", - "tests.test_utils_deprecate", - "tests.test_utils_misc.test_return_with_argument_inside_generator", - "tests.test_utils_python", - "tests.test_utils_request", - "tests.utils.bases.http_request", - "tests.utils.bases.http_response", "tests.utils.bases.spider", ] check_untyped_defs = false diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 6c306afdb..e6d5ff96a 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -225,13 +225,11 @@ def _run_command(cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace) def _run_command_profiled( cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace ) -> None: - if opts.profile: - sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") + sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") loc = locals() p = cProfile.Profile() p.runctx("cmd.run(args, opts)", globals(), loc) - if opts.profile: - p.dump_stats(opts.profile) + p.dump_stats(opts.profile) if __name__ == "__main__": diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 866ba9f6b..4e086e057 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -16,7 +16,7 @@ class Command(BaseRunSpiderCommand): return "[options] " def short_desc(self) -> str: - return "Run a spider" + return "Run a spider of the current project, by name" def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) < 1: diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 4277232c3..52f9cd4b0 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -32,10 +32,7 @@ def sanitize_module_name(module_name: str) -> str: def extract_domain(url: str) -> str: """Extract domain name from URL string""" - o = urlparse(url) - if o.scheme == "" and o.netloc == "": - o = urlparse("//" + url.lstrip("/")) - return o.netloc + return urlparse(url).netloc def verify_url_scheme(url: str) -> str: diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 2ac65bf3f..51caed57f 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -41,7 +41,7 @@ class Command(BaseRunSpiderCommand): spider: Spider | None = None items: ClassVar[dict[int, list[Any]]] = {} requests: ClassVar[dict[int, list[Request]]] = {} - spidercls: type[Spider] | None + spidercls: type[Spider] | None = None first_response = None @@ -346,6 +346,8 @@ class Command(BaseRunSpiderCommand): self.first_response = response cb = self._get_callback(spider=spider, opts=opts, response=response) + assert response.request + response.request.callback = cb # parse items and requests depth: int = response.meta["_depth"] diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 0b9036457..9cdb393ab 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -38,7 +38,7 @@ class Command(BaseRunSpiderCommand): return "[options] " def short_desc(self) -> str: - return "Run a self-contained spider (without creating a project)" + return "Run a spider from a Python file, no project required" def long_desc(self) -> str: return "Run the spider defined in the given file" diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 7c0ee0eec..f9ee62838 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -27,7 +27,6 @@ from scrapy.utils.defer import ( deferred_from_coro, maybe_deferred_to_future, ) -from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: @@ -80,22 +79,6 @@ class Slot: ) -def _get_concurrency_delay( - concurrency: int, spider: Spider, settings: BaseSettings -) -> tuple[int, float]: - delay: float = settings.getfloat("DOWNLOAD_DELAY") - if hasattr(spider, "download_delay"): - delay = spider.download_delay - - if hasattr(spider, "max_concurrent_requests"): # pragma: no cover - warn_on_deprecated_spider_attribute( - "max_concurrent_requests", "CONCURRENT_REQUESTS" - ) - concurrency = spider.max_concurrent_requests - - return concurrency, delay - - class Downloader: DOWNLOAD_SLOT = "download_slot" _SLOT_GC_INTERVAL: float = 60.0 # seconds @@ -112,6 +95,9 @@ class Downloader: "CONCURRENT_REQUESTS_PER_DOMAIN" ) self.ip_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS_PER_IP") + # Default delay of new slots. AutoThrottle overrides it to apply + # AUTOTHROTTLE_START_DELAY. + self._delay: float = self.settings.getfloat("DOWNLOAD_DELAY") self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") self.middleware: DownloaderMiddlewareManager = ( DownloaderMiddlewareManager.from_crawler(crawler) @@ -138,7 +124,8 @@ class Downloader: self.active.remove(request) def needs_backout(self) -> bool: - return len(self.active) >= self.total_concurrency + # A total concurrency of 0 means no limit. + return 0 < self.total_concurrency <= len(self.active) @_warn_spider_arg def _get_slot( @@ -146,16 +133,11 @@ class Downloader: ) -> tuple[str, Slot]: key = self.get_slot_key(request) if key not in self.slots: - assert self.crawler.spider slot_settings = self.per_slot_settings.get(key, {}) - conc = self.ip_concurrency or self.domain_concurrency - conc, delay = _get_concurrency_delay( - conc, self.crawler.spider, self.settings - ) - conc, delay = ( - slot_settings.get("concurrency", conc), - slot_settings.get("delay", delay), + conc = slot_settings.get( + "concurrency", self.ip_concurrency or self.domain_concurrency ) + delay = slot_settings.get("delay", self._delay) randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index fb27cdb8b..84dc6216b 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -39,11 +39,23 @@ logger = logging.getLogger(__name__) class DownloadHandlerProtocol(Protocol): + """Interface that :ref:`download handlers ` must + implement. + + Besides implementing this protocol, the contract of a download handler + includes **never** calling :meth:`crawler.engine.download_async() + `. + """ + lazy: bool + """Whether to delay instantiation of the handler; see :ref:`lazy + `.""" - async def download_request(self, request: Request) -> Response: ... + async def download_request(self, request: Request) -> Response: + """Download *request* and return a response.""" - async def close(self) -> None: ... + async def close(self) -> None: + """Clean up any resources used by the handler.""" class DownloadHandlers: diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py index d5a4e9fcd..8bbffb233 100644 --- a/scrapy/core/downloader/handlers/_httpx.py +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -92,10 +92,11 @@ class HttpxDownloadHandler(_Base): self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings) self._bind_host: str | None = self._get_bind_address_host() self._limits: httpx.Limits = httpx.Limits( - # hard limit on simultaneous connections - max_connections=self._pool_size_total, + # hard limit on simultaneous connections (None for no limit, which + # is what a CONCURRENT_REQUESTS of 0 means) + max_connections=self._pool_size_total or None, # total number of idle connections in the pool (extra ones are closed) - max_keepalive_connections=self._pool_size_total, + max_keepalive_connections=self._pool_size_total or None, ) self._default_client: httpx.AsyncClient = self._make_client() diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 07ff4a74e..29b3e3c0f 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -126,5 +126,4 @@ class FTPDownloadHandler(BaseDownloadHandler): headers = {"local filename": protocol.filename or b"", "size": protocol.size} body = protocol.filename or protocol.body.read() respcls = responsetypes.from_args(url=request.url, body=body) - # hints for Headers-related types may need to be fixed to not use AnyStr - return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type] + return respcls(url=request.url, status=200, body=body, headers=headers) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 42c517222..a511b0c7b 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -366,8 +366,8 @@ class Scheduler(BaseScheduler): Unless the received request is filtered out by the Dupefilter, attempt to push it into the disk queue, falling back to pushing it into the memory queue. - Increment the appropriate stats, such as: ``scheduler/enqueued``, - ``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``. + Increment the appropriate stats, such as: :stat:`scheduler/enqueued`, + :stat:`scheduler/enqueued/disk`, :stat:`scheduler/enqueued/memory`. Return ``True`` if the request was stored successfully, ``False`` otherwise. """ @@ -390,8 +390,8 @@ class Scheduler(BaseScheduler): falling back to the disk queue if the memory queue is empty. Return ``None`` if there are no more enqueued requests. - Increment the appropriate stats, such as: ``scheduler/dequeued``, - ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``. + Increment the appropriate stats, such as: :stat:`scheduler/dequeued`, + :stat:`scheduler/dequeued/disk`, :stat:`scheduler/dequeued/memory`. """ request: Request | None = self.mqs.pop() assert self.stats is not None diff --git a/scrapy/crawler.py b/scrapy/crawler.py index c8d74fba7..e2f726519 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -100,6 +100,10 @@ class Crawler: return self.addons.load_settings(self.settings) + self._apply_deprecated_spider_attr("download_delay", "DOWNLOAD_DELAY") + self._apply_deprecated_spider_attr( + "max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN" + ) self.stats = load_object(self.settings["STATS_CLASS"])(self) lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) @@ -155,6 +159,30 @@ class Crawler: "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} ) + def _apply_deprecated_spider_attr(self, attr: str, setting: str) -> None: + """Bridge a deprecated spider attribute onto *setting*, warning about + the deprecation (and about being ignored when *setting* is already set + at spider or higher priority).""" + spider = self.spider if self.spider is not None else self.spidercls + if not hasattr(spider, attr): + return + if (self.settings.getpriority(setting) or 0) >= SETTINGS_PRIORITIES["spider"]: + warnings.warn( + f"The {attr!r} spider attribute is deprecated. It is also being " + f"ignored because {setting} is already set at spider or higher " + f"priority. Remove the {attr!r} attribute from your spider.", + category=ScrapyDeprecationWarning, + stacklevel=3, + ) + return + warnings.warn( + f"The {attr!r} spider attribute is deprecated. Use the {setting} " + f"setting instead.", + category=ScrapyDeprecationWarning, + stacklevel=3, + ) + self.settings.set(setting, getattr(spider, attr), priority="spider") + def _apply_reactorless_default_settings(self) -> None: """Change some setting defaults when not using a Twisted reactor. diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index c6c811809..e7ca0ac0e 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from email.utils import formatdate from typing import TYPE_CHECKING @@ -28,6 +29,9 @@ if TYPE_CHECKING: from scrapy.statscollectors import StatsCollector +logger = logging.getLogger(__name__) + + class HttpCacheMiddleware: DOWNLOAD_EXCEPTIONS = ( ConnectionDone, @@ -77,9 +81,20 @@ class HttpCacheMiddleware: return None # Look for cached response and check if expired - cachedresponse: Response | None = self.storage.retrieve_response( - self.crawler.spider, request - ) + cachedresponse: Response | None + try: + cachedresponse = self.storage.retrieve_response( + self.crawler.spider, request + ) + except Exception: + self.stats.inc_value("httpcache/retrieve_error") + logger.warning( + f"Could not read the cache entry for {request}, treating it as a " + f"cache miss.", + exc_info=True, + extra={"spider": self.crawler.spider}, + ) + cachedresponse = None if cachedresponse is None: self.stats.inc_value("httpcache/miss") if self.ignore_missing: diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index e1dcd90d5..f910d07c8 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -14,7 +14,7 @@ from typing import TYPE_CHECKING from scrapy.exceptions import NotConfigured from scrapy.utils.decorators import _warn_spider_arg -from scrapy.utils.misc import load_object +from scrapy.utils.misc import _load_objects from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message @@ -149,10 +149,7 @@ class RetryMiddleware: self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")} self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") self.give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"] - self.exceptions_to_retry = tuple( - load_object(x) if isinstance(x, str) else x - for x in settings.getlist("RETRY_EXCEPTIONS") - ) + self.exceptions_to_retry = _load_objects(settings.getlist("RETRY_EXCEPTIONS")) @classmethod def from_crawler(cls, crawler: Crawler) -> Self: diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7d0c17884..81a3a887f 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING from twisted.internet.defer import Deferred +from scrapy import signals from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK @@ -98,7 +99,7 @@ class RobotsTxtMiddleware: assert self.crawler.stats try: resp = await self.crawler.engine.download_async(robotsreq) - self._parse_robots(resp, netloc) + await self._parse_robots(resp, netloc, request) except Exception as e: if not isinstance(e, IgnoreRequest): logger.error( @@ -115,13 +116,20 @@ class RobotsTxtMiddleware: return await maybe_deferred_to_future(parser) return parser - def _parse_robots(self, response: Response, netloc: str) -> None: + async def _parse_robots( + self, response: Response, netloc: str, request: Request + ) -> None: assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/response_count") self.crawler.stats.inc_value( f"robotstxt/response_status_count/{response.status}" ) rp = self._parserimpl.from_crawler(self.crawler, response.body) + await self.crawler.signals.send_catch_log_async( + signal=signals.robots_parsed, + robotparser=rp, + request=request, + ) rp_dfd = self._parsers[netloc] assert isinstance(rp_dfd, Deferred) self._parsers[netloc] = rp diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 5a11df833..ea600d1a8 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -74,6 +74,17 @@ class BaseItemExporter(ABC): def finish_exporting(self) -> None: # noqa: B027 pass + @staticmethod + def _get_populated_field_names(adapter: ItemAdapter) -> Iterable[str]: + """Return the populated field names of *adapter*, in declaration order. + + Populated fields that are not declared, which some item types allow, + come last, in item order. + """ + populated = set(adapter.keys()) + declared = (name for name in adapter.field_names() if name in populated) + return dict.fromkeys([*declared, *adapter.keys()]) + def _get_serialized_fields( self, item: Any, default_value: Any = None, include_empty: bool | None = None ) -> Iterable[tuple[str, Any]]: @@ -86,7 +97,11 @@ class BaseItemExporter(ABC): include_empty = self.export_empty_fields if self.fields_to_export is None: - field_iter = item.field_names() if include_empty else item.keys() + field_iter = ( + item.field_names() + if include_empty + else self._get_populated_field_names(item) + ) elif isinstance(self.fields_to_export, Mapping): if include_empty: field_iter = self.fields_to_export.items() diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index a4362b182..9cb792e30 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -119,7 +119,7 @@ class CloseSpider: self.task = None if self.task_no_item: - if self.task_no_item.running: + if self.task_no_item.running: # pragma: no branch self.task_no_item.stop() self.task_no_item = None diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 678a29e2e..118462bc9 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -13,7 +13,7 @@ import re import sys import warnings from abc import ABC, abstractmethod -from collections.abc import Callable, Coroutine +from collections.abc import Callable from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile @@ -28,6 +28,7 @@ from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.asyncio import is_asyncio_available, run_in_thread +from scrapy.utils.boto import _get_max_pool_connections from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.defer import deferred_from_coro, ensure_awaitable from scrapy.utils.ftp import ftp_store_file @@ -213,11 +214,14 @@ class S3FeedStorage(BlockingFeedStorage): feed_options: dict[str, Any] | None = None, session_token: str | None = None, region_name: str | None = None, + max_pool_connections: int | None = None, ): try: import boto3.session # noqa: PLC0415 except ImportError: raise NotConfigured("missing boto3 library") from None + from botocore.config import Config # noqa: PLC0415 + u = urlparse(uri) assert u.hostname self.bucketname: str = u.hostname @@ -228,6 +232,7 @@ class S3FeedStorage(BlockingFeedStorage): self.acl: str | None = acl self.endpoint_url: str | None = endpoint_url self.region_name: str | None = region_name + self.max_pool_connections: int | None = max_pool_connections boto3_session = boto3.session.Session() self.s3_client = boto3_session.client( @@ -237,6 +242,11 @@ class S3FeedStorage(BlockingFeedStorage): aws_session_token=self.session_token, endpoint_url=self.endpoint_url, region_name=self.region_name, + config=( + Config(max_pool_connections=self.max_pool_connections) + if self.max_pool_connections is not None + else None + ), ) if feed_options and feed_options.get("overwrite", True) is False: @@ -262,6 +272,7 @@ class S3FeedStorage(BlockingFeedStorage): acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None, endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None, region_name=crawler.settings["AWS_REGION_NAME"] or None, + max_pool_connections=_get_max_pool_connections(crawler.settings), feed_options=feed_options, ) @@ -454,7 +465,7 @@ class FeedSlot: ) def finish_exporting(self) -> None: - if self._exporting: + if self._exporting: # pragma: no branch assert self.exporter self.exporter.finish_exporting() self._exporting = False @@ -475,7 +486,7 @@ class FeedExporter: self.feeds = {} self.slots: list[FeedSlot] = [] self.filters: dict[str, ItemFilter] = {} - self._pending_close_coros: list[Coroutine[Any, Any, None]] = [] + self._pending_close_tasks: list[asyncio.Task[None] | Deferred[None]] = [] if not self.settings["FEEDS"] and not self.settings["FEED_URI"]: raise NotConfigured @@ -539,23 +550,44 @@ class FeedExporter: ) async def close_spider(self, spider: Spider) -> None: - self._pending_close_coros.extend( - self._close_slot(slot, spider) for slot in self.slots - ) + for slot in self.slots: + self._schedule_slot_close(slot, spider) - if self._pending_close_coros: + if self._pending_close_tasks: # pragma: no branch if is_asyncio_available(): await asyncio.wait( - [asyncio.create_task(coro) for coro in self._pending_close_coros] + cast("list[asyncio.Task[None]]", list(self._pending_close_tasks)) ) else: await DeferredList( - deferred_from_coro(coro) for coro in self._pending_close_coros + cast("list[Deferred[None]]", list(self._pending_close_tasks)) ) # Send FEED_EXPORTER_CLOSED signal await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed) + def _schedule_slot_close( + self, slot: FeedSlot, spider: Spider + ) -> asyncio.Task[None] | Deferred[None]: + """Start closing the slot without waiting for it to finish, keeping + track of the pending work so that it can be awaited in + :meth:`close_spider` if it hasn't finished by then.""" + aw: asyncio.Task[None] | Deferred[None] + coro = self._close_slot(slot, spider) + if is_asyncio_available(): + aw = asyncio.create_task(coro) + self._pending_close_tasks.append(aw) + aw.add_done_callback(self._pending_close_tasks.remove) + else: + aw = deferred_from_coro(coro) + self._pending_close_tasks.append(aw) + aw.addBoth(self._untrack_pending_close_task, aw) + return aw + + def _untrack_pending_close_task(self, result: Any, aw: Deferred[None]) -> Any: + self._pending_close_tasks.remove(aw) + return result + @staticmethod def _get_file(slot_: FeedSlot) -> IO[bytes]: assert slot_.file @@ -652,7 +684,7 @@ class FeedExporter: uri_params = self._get_uri_params( spider, self.feeds[slot.uri_template]["uri_params"], slot ) - self._pending_close_coros.append(self._close_slot(slot, spider)) + self._schedule_slot_close(slot, spider) slots.append( self._start_new_batch( batch_id=slot.batch_id + 1, diff --git a/scrapy/extensions/logcount.py b/scrapy/extensions/logcount.py index e6d51a7d8..fcce64438 100644 --- a/scrapy/extensions/logcount.py +++ b/scrapy/extensions/logcount.py @@ -20,7 +20,7 @@ class LogCount: """Install a log handler that counts log messages by level. The handler installed is :class:`scrapy.utils.log.LogCounterHandler`. - The counts are stored in stats as ``log_count/``. + The counts are stored in the :stat:`log_count/{level}` stat. .. versionadded:: 2.14 """ diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 1444c8941..e0e289ce8 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -94,7 +94,7 @@ class MemoryUsage: def engine_stopped(self) -> None: for tsk in self.tasks: - if tsk.running: + if tsk.running: # pragma: no branch tsk.stop() def update(self) -> None: diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index cd35c8165..cbcc8b70e 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -38,7 +38,6 @@ class PeriodicLog: ): self.stats: StatsCollector = stats self.interval: float = interval - self.multiplier: float = 60.0 / self.interval self.task: AsyncioLoopingCall | LoopingCall | None = None self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) self.ext_stats_enabled: bool = bool(ext_stats) @@ -165,5 +164,5 @@ class PeriodicLog: def spider_closed(self, spider: Spider, reason: str) -> None: self.log() - if self.task and self.task.running: + if self.task and self.task.running: # pragma: no branch self.task.stop() diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 542ff1cdc..cde73f12e 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -43,18 +43,18 @@ class AutoThrottle: return cls(crawler) def _spider_opened(self, spider: Spider) -> None: - self.mindelay = self._min_delay(spider) - self.maxdelay = self._max_delay(spider) - spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined] + self.mindelay = self._min_delay() + self.maxdelay = self._max_delay() + assert self.crawler.engine + self.crawler.engine.downloader._delay = self._start_delay() - def _min_delay(self, spider: Spider) -> float: - s = self.crawler.settings - return getattr(spider, "download_delay", s.getfloat("DOWNLOAD_DELAY")) + def _min_delay(self) -> float: + return self.crawler.settings.getfloat("DOWNLOAD_DELAY") - def _max_delay(self, spider: Spider) -> float: + def _max_delay(self) -> float: return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY") - def _start_delay(self, spider: Spider) -> float: + def _start_delay(self) -> float: return max( self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") ) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 34d4ec6f2..b55ef6191 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, AnyStr, TypeAlias, cast +from typing import TYPE_CHECKING, Any, TypeAlias, cast from w3lib.http import headers_dict_to_raw @@ -25,14 +25,20 @@ class Headers(CaselessDict): def __init__( self, - seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + seq: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, encoding: str = "utf-8", ): self.encoding: str = encoding super().__init__(seq) def update( # type: ignore[override] - self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] + self, + seq: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]], ) -> None: seq = seq.items() if isinstance(seq, Mapping) else seq iseq: dict[bytes, list[bytes]] = {} @@ -40,7 +46,7 @@ class Headers(CaselessDict): iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v)) super().update(iseq) - def normkey(self, key: AnyStr) -> bytes: # type: ignore[override] + def normkey(self, key: str | bytes) -> bytes: """Normalize key to bytes""" return self._tobytes(key.title()) @@ -67,19 +73,19 @@ class Headers(CaselessDict): return str(x).encode(self.encoding) raise TypeError(f"Unsupported value type: {type(x)}") - def __getitem__(self, key: AnyStr) -> bytes | None: + def __getitem__(self, key: str | bytes) -> bytes | None: try: return cast("list[bytes]", super().__getitem__(key))[-1] except IndexError: return None - def get(self, key: AnyStr, def_val: Any = None) -> bytes | None: + def get(self, key: str | bytes, def_val: Any = None) -> bytes | None: try: return cast("list[bytes]", super().get(key, def_val))[-1] except IndexError: return None - def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]: + def getlist(self, key: str | bytes, def_val: Any = None) -> list[bytes]: try: return cast("list[bytes]", super().__getitem__(key)) except KeyError: @@ -87,15 +93,15 @@ class Headers(CaselessDict): return self.normvalue(def_val) return [] - def setlist(self, key: AnyStr, list_: Iterable[_RawValue]) -> None: + def setlist(self, key: str | bytes, list_: Iterable[_RawValue]) -> None: self[key] = list_ def setlistdefault( - self, key: AnyStr, default_list: Iterable[_RawValue] = () + self, key: str | bytes, default_list: Iterable[_RawValue] = () ) -> Any: return self.setdefault(key, default_list) - def appendlist(self, key: AnyStr, value: Iterable[_RawValue]) -> None: + def appendlist(self, key: str | bytes, value: Iterable[_RawValue]) -> None: lst = self.getlist(key) lst.extend(self.normvalue(value)) self[key] = lst diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 73c2e7dd4..7c53b6b48 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -11,7 +11,6 @@ import inspect from typing import ( TYPE_CHECKING, Any, - AnyStr, Concatenate, NoReturn, TypeAlias, @@ -51,7 +50,9 @@ class VerboseCookie(TypedDict): secure: NotRequired[bool] -CookiesT: TypeAlias = dict[str | bytes, str | bytes] | list[VerboseCookie] +CookiesT: TypeAlias = ( + dict[str | bytes, str | bytes | bool | float | int] | list[VerboseCookie] +) RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") @@ -125,7 +126,10 @@ class Request(object_ref): url: str, callback: CallbackT | None = None, method: str = "GET", - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes | str | None = None, cookies: CookiesT | None = None, meta: dict[str, Any] | None = None, @@ -167,7 +171,8 @@ class Request(object_ref): #: #: The callable must expect the response as its first parameter, and #: support any additional keyword arguments set through - #: :attr:`cb_kwargs`. + #: :attr:`cb_kwargs`. See :ref:`writing-callbacks` and + #: :ref:`callback-output`. #: #: In addition to an arbitrary callable, the following values are also #: supported: @@ -188,8 +193,7 @@ class Request(object_ref): #: raises exceptions for non-2xx responses by default, sending them #: to the :attr:`errback` instead. #: - #: .. seealso:: - #: :ref:`topics-request-response-ref-request-callback-arguments` + #: .. seealso:: :ref:`callbacks` self.callback: CallbackT | None = callback #: :class:`~collections.abc.Callable` to handle exceptions raised @@ -198,7 +202,7 @@ class Request(object_ref): #: The callable must expect a :exc:`~twisted.python.failure.Failure` as #: its first parameter. #: - #: .. seealso:: :ref:`topics-request-response-ref-errbacks` + #: .. seealso:: :ref:`errbacks` self.errback: Callable[[Failure], Any] | None = errback self._cookies: CookiesT | None = cookies or None @@ -310,7 +314,11 @@ class Request(object_ref): @headers.setter def headers( - self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None + self, + value: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None, ) -> None: if isinstance(value, Headers): self._headers = value @@ -381,6 +389,20 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) + def to_curl(self) -> str: + """Return a string with a `cURL `_ command equivalent + to this request. + + Inverse of :meth:`from_curl`. See also + :func:`scrapy.utils.request.request_to_curl`. + + .. versionadded:: VERSION + """ + # Imported here to avoid a circular import. + from scrapy.utils.request import request_to_curl # noqa: PLC0415 + + return request_to_curl(self) + def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]: """Return a dictionary containing the Request's data. diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index f1a8dbf3b..12745292b 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit from warnings import warn @@ -34,7 +34,7 @@ if TYPE_CHECKING: FormdataVType: TypeAlias = str | Iterable[str] FormdataKVType: TypeAlias = tuple[str, FormdataVType] -FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None +FormdataType: TypeAlias = Mapping[str, FormdataVType] | Iterable[FormdataKVType] | None class FormRequest(Request): @@ -100,7 +100,7 @@ class FormRequest(Request): super().__init__(*args, **kwargs) if formdata: - items = formdata.items() if isinstance(formdata, dict) else formdata + items = formdata.items() if isinstance(formdata, Mapping) else formdata form_query_str = _urlencode(items, self.encoding) if self.method == "POST": self.headers.setdefault( @@ -248,7 +248,7 @@ def _get_inputs( if clickable and clickable[0] not in formdata and clickable[0] is not None: values.append(clickable) - formdata_items = formdata.items() if isinstance(formdata, dict) else formdata + formdata_items = formdata.items() if isinstance(formdata, Mapping) else formdata values.extend((k, v) for k, v in formdata_items if v is not None) return values diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 09b1c8b32..f1db11488 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, overload +from typing import TYPE_CHECKING, Any, TypeVar, overload from urllib.parse import urljoin from scrapy.exceptions import NotSupported @@ -72,7 +72,10 @@ class Response(object_ref): self, url: str, status: int = 200, - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes = b"", flags: list[str] | None = None, request: Request | None = None, @@ -145,7 +148,11 @@ class Response(object_ref): @headers.setter def headers( - self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None + self, + value: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None, ) -> None: if isinstance(value, Headers): self._headers = value @@ -222,7 +229,10 @@ class Response(object_ref): url: str | Link, callback: CallbackT | None = None, method: str = "GET", - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes | str | None = None, cookies: CookiesT | None = None, meta: dict[str, Any] | None = None, @@ -272,7 +282,10 @@ class Response(object_ref): urls: Iterable[str | Link], callback: CallbackT | None = None, method: str = "GET", - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes | str | None = None, cookies: CookiesT | None = None, meta: dict[str, Any] | None = None, diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 6876e35e8..d01e23e47 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -9,7 +9,7 @@ from __future__ import annotations import json from contextlib import suppress -from typing import TYPE_CHECKING, Any, AnyStr, cast +from typing import TYPE_CHECKING, Any, cast from urllib.parse import urljoin import parsel @@ -170,7 +170,10 @@ class TextResponse(Response): url: str | Link | parsel.Selector, callback: CallbackT | None = None, method: str = "GET", - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes | str | None = None, cookies: CookiesT | None = None, meta: dict[str, Any] | None = None, @@ -223,7 +226,10 @@ class TextResponse(Response): urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None, callback: CallbackT | None = None, method: str = "GET", - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes | str | None = None, cookies: CookiesT | None = None, meta: dict[str, Any] | None = None, diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 8e8082332..55a3676e5 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -37,7 +37,7 @@ from scrapy.pipelines.media import ( _MediaRequestFiltered, ) from scrapy.utils.asyncio import run_in_thread -from scrapy.utils.boto import is_botocore_available +from scrapy.utils.boto import _get_max_pool_connections, is_botocore_available from scrapy.utils.datatypes import CaseInsensitiveDict from scrapy.utils.defer import deferred_from_coro, ensure_awaitable from scrapy.utils.ftp import ftp_store_file @@ -164,6 +164,9 @@ class S3FilesStore: AWS_REGION_NAME = None AWS_USE_SSL = None AWS_VERIFY = None + # Overridden from settings.AWS_MAX_POOL_CONNECTIONS in + # FilesPipeline.from_crawler(); None means the botocore default + AWS_MAX_POOL_CONNECTIONS: int | None = None POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler() HEADERS: ClassVar[dict[str, str]] = { @@ -174,7 +177,13 @@ class S3FilesStore: if not is_botocore_available(): raise NotConfigured("missing botocore library") import botocore.session # noqa: PLC0415 + from botocore.config import Config # noqa: PLC0415 + config = ( + Config(max_pool_connections=self.AWS_MAX_POOL_CONNECTIONS) + if self.AWS_MAX_POOL_CONNECTIONS is not None + else None + ) session = botocore.session.get_session() self.s3_client = session.create_client( "s3", @@ -185,6 +194,7 @@ class S3FilesStore: region_name=self.AWS_REGION_NAME, use_ssl=self.AWS_USE_SSL, verify=self.AWS_VERIFY, + config=config, ) if not uri.startswith("s3://"): raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'") @@ -522,6 +532,7 @@ class FilesPipeline(MediaPipeline): s3store.AWS_REGION_NAME = settings["AWS_REGION_NAME"] s3store.AWS_USE_SSL = settings["AWS_USE_SSL"] s3store.AWS_VERIFY = settings["AWS_VERIFY"] + s3store.AWS_MAX_POOL_CONNECTIONS = _get_max_pool_connections(settings) s3store.POLICY = settings["FILES_STORE_S3_ACL"] gcs_store: type[GCSFilesStore] = cast( diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 0c64ea5a5..b54011784 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -67,6 +67,15 @@ class RobotParser(metaclass=ABCMeta): :type user_agent: str or bytes """ + def crawl_delay(self, user_agent: str | bytes) -> float | None: + """Return the ``Crawl-delay`` directive for ``user_agent`` as a number + of seconds, or ``None`` if it is not set or the backend does not support + it. + + .. versionadded:: VERSION + """ + return None + class PythonRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -85,6 +94,10 @@ class PythonRobotParser(RobotParser): url = to_unicode(url) return self.rp.can_fetch(user_agent, url) + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) + class RerpRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -105,6 +118,10 @@ class RerpRobotParser(RobotParser): url = to_unicode(url) return cast("bool", self.rp.is_allowed(user_agent, url)) + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.get_crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) + class ProtegoRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -121,3 +138,7 @@ class ProtegoRobotParser(RobotParser): user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(url, user_agent) + + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index c4d88494b..61464107c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -28,6 +28,7 @@ __all__ = [ "AUTOTHROTTLE_TARGET_CONCURRENCY", "AWS_ACCESS_KEY_ID", "AWS_ENDPOINT_URL", + "AWS_MAX_POOL_CONNECTIONS", "AWS_REGION_NAME", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", @@ -229,6 +230,7 @@ AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 AWS_ACCESS_KEY_ID = None AWS_SECRET_ACCESS_KEY = None AWS_ENDPOINT_URL = None +AWS_MAX_POOL_CONNECTIONS = None AWS_REGION_NAME = None AWS_SESSION_TOKEN = None AWS_USE_SSL = None diff --git a/scrapy/signals.py b/scrapy/signals.py index 972f4fd60..3afeb6eab 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -21,6 +21,7 @@ response_received = object() response_downloaded = object() headers_received = object() bytes_received = object() +robots_parsed = object() item_scraped = object() item_dropped = object() item_error = object() diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 02dfa2ac6..6244e3264 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -143,6 +143,22 @@ class Spider(object_ref): else: def parse(self, response: Response, **kwargs: Any) -> Any: + """Process *response*, i.e. extract data from it and generate new + requests. + + This is the default :ref:`callback `: Scrapy uses + it for the response to any request that does not define a + :attr:`~scrapy.Request.callback`, such as the requests that + :meth:`start` yields by default. + + Any :attr:`~scrapy.Request.cb_kwargs` of the request are passed as + keyword parameters. + + Spiders must define this method, unless every request that they + send defines a callback. + + See :ref:`callback-output` about the supported return values. + """ raise NotImplementedError( f"{self.__class__.__name__}.parse callback is not defined" ) diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index 44604c0fe..7c7697f56 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -9,7 +9,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar from twisted.internet.defer import Deferred -from twisted.internet.task import LoopingCall +from twisted.internet.task import LoopingCall, deferLater from twisted.internet.threads import deferToThread from scrapy.utils.asyncgen import as_async_generator @@ -293,6 +293,24 @@ class CallLaterResult: self._delayed_call = None +async def sleep(seconds: float) -> None: + """Sleep for *seconds*. + + .. versionadded:: VERSION + + This uses either :func:`asyncio.sleep` or + :func:`~twisted.internet.task.deferLater`, depending on whether asyncio + support is available. + """ + if is_asyncio_available(): + await asyncio.sleep(seconds) + return + + from twisted.internet import reactor + + await deferLater(reactor, seconds) + + async def run_in_thread( func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs ) -> _T: diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 2a77ee2ac..76ee0e7ec 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,7 +1,22 @@ """Boto/botocore helpers""" +from __future__ import annotations + from importlib.util import find_spec +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from scrapy.settings import BaseSettings def is_botocore_available() -> bool: return find_spec("botocore") is not None + + +def _get_max_pool_connections(settings: BaseSettings) -> int: + """Return the maximum number of connections that AWS clients may keep in + their connection pool. + """ + return settings.getint("AWS_MAX_POOL_CONNECTIONS") or settings.getint( + "REACTOR_THREADPOOL_MAXSIZE" + ) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 31a4bb32f..23b4401e8 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,8 +1,9 @@ from __future__ import annotations +import asyncio import code from collections.abc import Callable -from functools import wraps +from functools import partial, wraps from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -16,16 +17,8 @@ def _embed_ipython_shell( namespace: dict[str, Any] | None = None, banner: str = "" ) -> EmbedFuncT: """Start an IPython Shell""" - try: - from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415 - from IPython.terminal.ipapp import load_default_config # noqa: PLC0415 - except ImportError: - from IPython.frontend.terminal.embed import ( # type: ignore[import-not-found,no-redef] # noqa: T100,PLC0415 - InteractiveShellEmbed, - ) - from IPython.frontend.terminal.ipapp import ( # type: ignore[import-not-found,no-redef] # noqa: PLC0415 - load_default_config, - ) + from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415 + from IPython.terminal.ipapp import load_default_config # noqa: PLC0415 @wraps(_embed_ipython_shell) def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None: @@ -38,6 +31,19 @@ def _embed_ipython_shell( shell = InteractiveShellEmbed.instance( banner1=banner, user_ns=namespace, config=config ) + # If an asyncio event loop is already running in this thread, e.g. when + # inspect_response() is called from a spider callback while using the + # asyncio reactor, prompt_toolkit cannot run its own event loop here, so + # ask it to run the prompt in a separate thread instead. pt_app is None + # when IPython falls back to its simple prompt, which needs no event loop. + # See https://github.com/scrapy/scrapy/issues/5447 + if (pt_app := getattr(shell, "pt_app", None)) is not None: + try: + asyncio.get_running_loop() + except RuntimeError: + pass + else: + pt_app.prompt = partial(pt_app.prompt, in_thread=True) shell() return wrapper diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index e761a2474..9a945c61c 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -11,12 +11,12 @@ import warnings import weakref from collections import OrderedDict from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, cast +from typing import TYPE_CHECKING, Any, TypeVar, cast from scrapy.exceptions import ScrapyDeprecationWarning if TYPE_CHECKING: - from collections.abc import Iterable, Sequence + from collections.abc import Container, Iterable # typing.Self requires Python 3.11 from typing_extensions import Self @@ -44,22 +44,25 @@ class CaselessDict(dict): # type: ignore[type-arg] def __init__( self, - seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + seq: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, ): super().__init__() if seq: self.update(seq) - def __getitem__(self, key: AnyStr) -> Any: + def __getitem__(self, key: str | bytes) -> Any: return dict.__getitem__(self, self.normkey(key)) - def __setitem__(self, key: AnyStr, value: Any) -> None: + def __setitem__(self, key: str | bytes, value: Any) -> None: dict.__setitem__(self, self.normkey(key), self.normvalue(value)) - def __delitem__(self, key: AnyStr) -> None: + def __delitem__(self, key: str | bytes) -> None: dict.__delitem__(self, self.normkey(key)) - def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] + def __contains__(self, key: str | bytes) -> bool: # type: ignore[override] return dict.__contains__(self, self.normkey(key)) has_key = __contains__ @@ -69,7 +72,7 @@ class CaselessDict(dict): # type: ignore[type-arg] copy = __copy__ - def normkey(self, key: AnyStr) -> AnyStr: + def normkey(self, key: str | bytes) -> str | bytes: """Method to normalize dictionary key access""" return key.lower() @@ -77,23 +80,28 @@ class CaselessDict(dict): # type: ignore[type-arg] """Method to normalize values prior to be set""" return value - def get(self, key: AnyStr, def_val: Any = None) -> Any: + def get(self, key: str | bytes, def_val: Any = None) -> Any: return dict.get(self, self.normkey(key), self.normvalue(def_val)) - def setdefault(self, key: AnyStr, def_val: Any = None) -> Any: + def setdefault(self, key: str | bytes, def_val: Any = None) -> Any: return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) # doesn't fully implement MutableMapping.update() - def update(self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]) -> None: # type: ignore[override] + def update( # type: ignore[override] + self, + seq: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]], + ) -> None: seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) super().update(iseq) @classmethod - def fromkeys(cls, keys: Iterable[AnyStr], value: Any = None) -> Self: # type: ignore[override] - return cls((k, value) for k in keys) # type: ignore[misc] + def fromkeys(cls, keys: Iterable[str | bytes], value: Any = None) -> Self: # type: ignore[override] + return cls((k, value) for k in keys) - def pop(self, key: AnyStr, *args: Any) -> Any: + def pop(self, key: str | bytes, *args: Any) -> Any: return dict.pop(self, self.normkey(key), *args) @@ -205,8 +213,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]): class SequenceExclude: """Object to test if an item is NOT within some sequence.""" - def __init__(self, seq: Sequence[Any]): - self.seq: Sequence[Any] = seq + def __init__(self, seq: Container[Any]): + self.seq: Container[Any] = seq def __contains__(self, item: Any) -> bool: return item not in self.seq diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index a5bb6fa24..2924c81f9 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -10,6 +10,7 @@ from twisted.internet.defer import Deferred, maybeDeferred from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncio import run_in_thread from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.python import _signature if TYPE_CHECKING: from collections.abc import AsyncGenerator, Callable, Coroutine @@ -19,9 +20,19 @@ _T = TypeVar("_T") _P = ParamSpec("_P") +@overload +def deprecated(use_instead: Callable[_P, _T]) -> Callable[_P, _T]: ... + + +@overload def deprecated( - use_instead: Any = None, -) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: + use_instead: str | None = None, +) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: ... + + +def deprecated( + use_instead: Callable[_P, _T] | str | None = None, +) -> Callable[_P, _T] | Callable[[Callable[_P, _T]], Callable[_P, _T]]: """This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted when the function is used.""" @@ -38,8 +49,9 @@ def deprecated( return wrapped if callable(use_instead): - deco = deco(use_instead) + func = use_instead use_instead = None + return deco(func) return deco @@ -98,7 +110,7 @@ def _warn_spider_arg( ): """Decorator to warn if a ``spider`` argument is passed to a function.""" - sig = inspect.signature(func) + sig = _signature(func) def check_args(*args: _P.args, **kwargs: _P.kwargs) -> None: bound = sig.bind(*args, **kwargs) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index d0259b634..7c6235f29 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -27,7 +27,7 @@ from twisted.internet.task import Cooperator from twisted.python import failure from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.asyncio import is_asyncio_available, sleep from scrapy.utils.python import global_object_name if TYPE_CHECKING: @@ -90,14 +90,7 @@ async def _defer_sleep_async() -> None: """Delay by _DEFER_DELAY so reactor has a chance to go through readers and writers before attending pending delayed calls, so do not set delay to zero. """ - if is_asyncio_available(): - await asyncio.sleep(_DEFER_DELAY) - else: - from twisted.internet import reactor - - d: Deferred[None] = Deferred() - reactor.callLater(_DEFER_DELAY, d.callback, None) - await d + await sleep(_DEFER_DELAY) def defer_result(result: Any) -> Deferred[Any]: # pragma: no cover diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 20e7cb381..57b526be6 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -90,6 +90,11 @@ def load_object(path: str | Callable[..., Any]) -> Any: return obj +def _load_objects(objects: Iterable[str | Callable[..., Any]]) -> tuple[Any, ...]: + """Resolve *objects* (objects or import paths) to a tuple of objects.""" + return tuple(load_object(obj) if isinstance(obj, str) else obj for obj in objects) + + def walk_modules_iter(path: str) -> Iterable[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 8a7517c1d..40fc05257 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -178,11 +178,30 @@ def binary_is_text(data: bytes) -> bool: return all(c not in _BINARYCHARS for c in data) +# PEP 649 (Python 3.14+) made annotation evaluation lazy, so inspect.signature() +# can raise NameError for names imported only under TYPE_CHECKING. We only need +# parameter names, kinds and defaults, so leave such annotations as ForwardRefs. +if sys.version_info >= (3, 14): + from annotationlib import Format + + def _signature(func: Callable[..., Any]) -> inspect.Signature: + return inspect.signature(func, annotation_format=Format.FORWARDREF) + +else: + + def _signature(func: Callable[..., Any]) -> inspect.Signature: + return inspect.signature(func) + + def get_func_args_dict( func: Callable[..., Any], stripself: bool = False ) -> Mapping[str, inspect.Parameter]: """Return the argument dict of a callable object. + Annotations are not evaluated, so on Python 3.14 and later the ``annotation`` + attribute of the returned parameters may be a ``ForwardRef`` instead of the + resolved type. + .. versionadded:: 2.14 """ if not callable(func): @@ -190,7 +209,7 @@ def get_func_args_dict( args: Mapping[str, inspect.Parameter] try: - sig = inspect.signature(func) + sig = _signature(func) except ValueError: return {} diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.py new file mode 100644 index 000000000..7b5ca0cb9 --- /dev/null +++ b/tests/benchmarks/__init__.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from scrapy.utils.test import get_crawler + +if TYPE_CHECKING: + from scrapy import Spider + from scrapy.crawler import Crawler + + +def crawl(spidercls: type[Spider], settings: dict[str, Any], **kwargs: Any) -> Crawler: + """Run a crawl to completion and return its crawler. + + Unlike the rest of the test suite, benchmarks run without ``pytest-twisted`` + and drive the reactor themselves, since the code being measured must be + callable synchronously by ``pytest-codspeed``. + """ + from twisted.internet import reactor + + crawler = get_crawler(spidercls, settings) + result: list[Any] = [] + crawler.crawl(**kwargs).addBoth(result.append) + while not result: + reactor.iterate(0.001) + if isinstance(result[0], BaseException): + raise result[0] + return crawler diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py new file mode 100644 index 000000000..55356083d --- /dev/null +++ b/tests/benchmarks/conftest.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from scrapy.utils.reactor import install_reactor + +if TYPE_CHECKING: + from collections.abc import Generator + + +@pytest.fixture(scope="session", autouse=True) +def running_reactor() -> Generator[None]: + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + + from twisted.internet import reactor + + # Marks the reactor as running without blocking, so that crawls can be + # driven with reactor.iterate(), see tests.benchmarks.crawl(). + reactor.startRunning(installSignalHandlers=False) + + yield + + reactor.stop() + # Lets the shutdown event triggers run, e.g. to join the thread pool. + reactor.iterate(0) diff --git a/tests/benchmarks/test_crawl.py b/tests/benchmarks/test_crawl.py new file mode 100644 index 000000000..0fdfe742b --- /dev/null +++ b/tests/benchmarks/test_crawl.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any +from urllib.parse import urlencode + +import pytest + +from scrapy import Field, Item, Request, Spider +from scrapy.linkextractors import LinkExtractor +from tests.benchmarks import crawl + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found] + + from scrapy.http import Response + from tests.mockserver.http import MockServer + +pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed") + +PAGES = 100 +LINKS_PER_PAGE = 5 + + +class _Page(Item): + url = Field() + anchors = Field() + + +class _FollowSpider(Spider): + name = "benchmark" + url: str + link_extractor = LinkExtractor() + + async def start(self) -> AsyncIterator[Any]: + yield Request(self.url, dont_filter=True) + + def parse(self, response: Response) -> Any: + yield _Page( + url=response.url, + anchors=response.css("a::text").getall(), + ) + for link in self.link_extractor.extract_links(response): # type: ignore[arg-type] + yield Request(link.url) + + +class _Pipeline: + def process_item(self, item: Any) -> Any: + return item + + +def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> None: + """Per-request overhead of a crawl over HTTP. + + The pages are small on purpose, so that the cost of parsing them stays + negligible next to the cost of moving requests and responses through the + engine, the middlewares and the download handler. + """ + query = urlencode({"total": PAGES, "show": LINKS_PER_PAGE, "order": "desc"}) + url = mockserver.url(f"/follow?{query}") + settings = {"ITEM_PIPELINES": {_Pipeline: 100}, "LOG_ENABLED": False} + + def run() -> None: + crawler = crawl(_FollowSpider, settings, url=url) + assert crawler.stats + assert crawler.stats.get_value("item_scraped_count") == PAGES + 1 + + benchmark(run) diff --git a/tests/mockserver/dns.py b/tests/mockserver/dns.py index e19a2e61a..2af018c66 100644 --- a/tests/mockserver/dns.py +++ b/tests/mockserver/dns.py @@ -2,6 +2,7 @@ from __future__ import annotations import sys from subprocess import PIPE, Popen +from typing import TYPE_CHECKING from twisted.internet import defer from twisted.names import dns, error @@ -9,39 +10,63 @@ from twisted.names.server import DNSServerFactory from tests.utils import get_script_run_env +if TYPE_CHECKING: + from collections.abc import Sequence + from types import TracebackType + + from twisted.internet.defer import Deferred + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + +_Answers = tuple[list[dns.RRHeader], list[dns.RRHeader], list[dns.RRHeader]] + class MockDNSResolver: """ Implements twisted.internet.interfaces.IResolver partially """ - def _resolve(self, name): + def _resolve(self, name: bytes) -> _Answers: record = dns.Record_A(address=b"127.0.0.1") - answer = dns.RRHeader(name=name, payload=record) + # zope.interface has no type hints, so mypy cannot tell that Record_A + # provides the IEncodableRecord interface. + answer = dns.RRHeader(name=name, payload=record) # type: ignore[arg-type] return [answer], [], [] - def query(self, query, timeout=None): + def query( + self, query: dns.Query, timeout: Sequence[int] | None = None + ) -> Deferred[_Answers]: if query.type == dns.A: return defer.succeed(self._resolve(query.name.name)) return defer.fail(error.DomainError()) - def lookupAllRecords(self, name, timeout=None): + def lookupAllRecords( + self, name: bytes, timeout: Sequence[int] | None = None + ) -> Deferred[_Answers]: return defer.succeed(self._resolve(name)) class MockDNSServer: - def __enter__(self): + def __enter__(self) -> Self: self.proc = Popen( [sys.executable, "-u", "-m", "tests.mockserver.dns"], stdout=PIPE, env=get_script_run_env(), text=True, ) + assert self.proc.stdout is not None self.host = "127.0.0.1" self.port = int(self.proc.stdout.readline().strip().split(":")[1]) return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: self.proc.kill() self.proc.communicate() @@ -54,7 +79,7 @@ def main() -> None: protocol = dns.DNSDatagramProtocol(controller=factory) listener = reactor.listenUDP(0, protocol) - def print_listening(): + def print_listening() -> None: host = listener.getHost() print(f"{host.host}:{host.port}") diff --git a/tests/mockserver/ftp.py b/tests/mockserver/ftp.py index 22efc966b..1edd64dda 100644 --- a/tests/mockserver/ftp.py +++ b/tests/mockserver/ftp.py @@ -7,6 +7,7 @@ from pathlib import Path from shutil import rmtree from subprocess import PIPE, Popen from tempfile import mkdtemp +from typing import TYPE_CHECKING from pyftpdlib.authorizers import DummyAuthorizer from pyftpdlib.handlers import FTPHandler @@ -14,6 +15,12 @@ from pyftpdlib.servers import FTPServer from tests.utils import get_script_run_env +if TYPE_CHECKING: + from types import TracebackType + + # typing.Self requires Python 3.11 + from typing_extensions import Self + class MockFTPServer: """Creates an FTP server on a random port with a default passwordless user @@ -26,7 +33,7 @@ class MockFTPServer: self.port: int | None = None self.path: Path | None = None - def __enter__(self): + def __enter__(self) -> Self: self.path = Path(mkdtemp()) self.proc = Popen( [sys.executable, "-u", "-m", "tests.mockserver.ftp", "-d", str(self.path)], @@ -34,6 +41,7 @@ class MockFTPServer: env=get_script_run_env(), text=True, ) + assert self.proc.stderr is not None for line in self.proc.stderr: if "starting FTP server" in line and ( m := re.search(r"starting FTP server on ([^ :]+):(\d+),", line) @@ -48,12 +56,18 @@ class MockFTPServer: ) return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: rmtree(str(self.path)) + assert self.proc is not None self.proc.kill() self.proc.communicate() - def url(self, path): + def url(self, path: str) -> str: return f"ftp://{self.host}:{self.port}/{path}" diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py index 7ad873c02..c4fd4464e 100644 --- a/tests/mockserver/http.py +++ b/tests/mockserver/http.py @@ -1,8 +1,8 @@ from __future__ import annotations from pathlib import Path +from typing import TYPE_CHECKING -from twisted.web import resource from twisted.web.static import Data, File from twisted.web.util import Redirect @@ -11,6 +11,7 @@ from tests import tests_datadir from .http_base import BaseMockServer, main_factory from .http_resources import ( ArbitraryLengthPayloadResource, + BaseResource, BrokenChunkedResource, BrokenDownloadResource, ChunkedResource, @@ -35,62 +36,68 @@ from .http_resources import ( SetCookie, Status, UriResource, + put_child, ) +if TYPE_CHECKING: + from twisted.web.server import Request -class Root(resource.Resource): - def __init__(self): + +class Root(BaseResource): + def __init__(self) -> None: super().__init__() - self.putChild(b"status", Status()) - self.putChild(b"follow", Follow()) - self.putChild(b"delay", Delay()) - self.putChild(b"partial", Partial()) - self.putChild(b"drop", Drop()) - self.putChild(b"raw", Raw()) - self.putChild(b"echo", Echo()) - self.putChild(b"payload", PayloadResource()) - self.putChild(b"alpayload", ArbitraryLengthPayloadResource()) - self.putChild(b"static", File(str(Path(tests_datadir, "test_site/")))) - self.putChild(b"redirect-to", RedirectTo()) - self.putChild(b"text", Data(b"Works", "text/plain")) - self.putChild( + put_child(self, b"status", Status()) + put_child(self, b"follow", Follow()) + put_child(self, b"delay", Delay()) + put_child(self, b"partial", Partial()) + put_child(self, b"drop", Drop()) + put_child(self, b"raw", Raw()) + put_child(self, b"echo", Echo()) + put_child(self, b"payload", PayloadResource()) + put_child(self, b"alpayload", ArbitraryLengthPayloadResource()) + put_child(self, b"static", File(str(Path(tests_datadir, "test_site/")))) + put_child(self, b"redirect-to", RedirectTo()) + put_child(self, b"text", Data(b"Works", "text/plain")) + put_child( + self, b"html", Data( b"

Works

World

", "text/html", ), ) - self.putChild( + put_child( + self, b"enc-gb18030", Data(b"

gb18030 encoding

", "text/html; charset=gb18030"), ) - self.putChild(b"redirect", Redirect(b"/redirected")) - self.putChild( - b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected") + put_child(self, b"redirect", Redirect(b"/redirected")) + put_child( + self, b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected") ) - self.putChild(b"redirected", Data(b"Redirected here", "text/plain")) + put_child(self, b"redirected", Data(b"Redirected here", "text/plain")) numbers = [str(x).encode("utf8") for x in range(2**18)] - self.putChild(b"numbers", Data(b"".join(numbers), "text/plain")) - self.putChild(b"wait", ForeverTakingResource()) - self.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) - self.putChild(b"host", HostHeaderResource()) - self.putChild(b"client-ip", ClientIPResource()) - self.putChild(b"broken", BrokenDownloadResource()) - self.putChild(b"chunked", ChunkedResource()) - self.putChild(b"broken-chunked", BrokenChunkedResource()) - self.putChild(b"contentlength", ContentLengthHeaderResource()) - self.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) - self.putChild(b"largechunkedfile", LargeChunkedFileResource()) - self.putChild(b"compress", Compress()) - self.putChild(b"duplicate-header", DuplicateHeaderResource()) - self.putChild(b"response-headers", ResponseHeadersResource()) - self.putChild(b"set-cookie", SetCookie()) - self.putChild(b"uri", UriResource()) + put_child(self, b"numbers", Data(b"".join(numbers), "text/plain")) + put_child(self, b"wait", ForeverTakingResource()) + put_child(self, b"hang-after-headers", ForeverTakingResource(write=True)) + put_child(self, b"host", HostHeaderResource()) + put_child(self, b"client-ip", ClientIPResource()) + put_child(self, b"broken", BrokenDownloadResource()) + put_child(self, b"chunked", ChunkedResource()) + put_child(self, b"broken-chunked", BrokenChunkedResource()) + put_child(self, b"contentlength", ContentLengthHeaderResource()) + put_child(self, b"nocontenttype", EmptyContentTypeHeaderResource()) + put_child(self, b"largechunkedfile", LargeChunkedFileResource()) + put_child(self, b"compress", Compress()) + put_child(self, b"duplicate-header", DuplicateHeaderResource()) + put_child(self, b"response-headers", ResponseHeadersResource()) + put_child(self, b"set-cookie", SetCookie()) + put_child(self, b"uri", UriResource()) - def getChild(self, path, request): + def getChild(self, path: bytes, request: Request) -> Root: return self - def render(self, request): + def render(self, request: Request) -> bytes: return b"Scrapy mock HTTP server\n" diff --git a/tests/mockserver/http_base.py b/tests/mockserver/http_base.py index 343c79781..5bc1252d6 100644 --- a/tests/mockserver/http_base.py +++ b/tests/mockserver/http_base.py @@ -17,6 +17,7 @@ from .utils import ssl_context_factory if TYPE_CHECKING: from collections.abc import Callable + from types import TracebackType from twisted.web import resource @@ -60,7 +61,12 @@ class BaseMockServer(ABC): self.https_port = https_parsed.port return self - def __exit__(self, exc_type, exc_value, traceback) -> None: + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: if self.proc: self.proc.kill() self.proc.communicate() @@ -135,7 +141,7 @@ def main_factory( context_factory = ssl_context_factory(**context_factory_kw) https_port = reactor.listenSSL(0, factory, context_factory) - def print_listening(): + def print_listening() -> None: if listen_http: http_host = http_port.getHost() http_address = f"http://{http_host.host}:{http_host.port}" diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.py index 98ac6cf6a..cb028bc10 100644 --- a/tests/mockserver/http_resources.py +++ b/tests/mockserver/http_resources.py @@ -3,7 +3,7 @@ from __future__ import annotations import gzip import json import random -from typing import TYPE_CHECKING, ParamSpec, TypeVar +from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar from urllib.parse import urlencode from twisted.internet.task import deferLater @@ -14,17 +14,24 @@ from twisted.web.util import Redirect, redirectTo from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Sequence from twisted.internet.defer import Deferred - from twisted.web.http import Request + from twisted.python.failure import Failure + from twisted.web.http import Request as HTTPRequest + from twisted.web.server import Request _T = TypeVar("_T") _P = ParamSpec("_P") -def getarg(request, name, default=None, type_=None): +def getarg( + request: Request, + name: bytes, + default: Any = None, + type_: Callable[[bytes], Any] | None = None, +) -> Any: if name in request.args: value = request.args[name][0] if type_ is not None: @@ -33,73 +40,91 @@ def getarg(request, name, default=None, type_=None): return default -def close_connection(request): +def close_connection(request: Request) -> None: # We have to force a disconnection for HTTP/1.1 clients. Otherwise # client keeps the connection open waiting for more data. request.channel.loseConnection() request.finish() +def put_child(parent: resource.Resource, path: bytes, child: resource.Resource) -> None: + # zope.interface has no type hints, so mypy cannot tell that Resource + # instances provide the IResource interface that putChild() expects. + parent.putChild(path, child) # type: ignore[arg-type] + + +class BaseResource(resource.Resource): + """Base class for mockserver resources, with type hints.""" + + # Only needed to give subclasses a typed __init__ to call. + def __init__(self) -> None: # pylint: disable=useless-parent-delegation + super().__init__() # type: ignore[no-untyped-call] + + # most of the following resources are copied from twisted.web.test.test_webclient -class ForeverTakingResource(resource.Resource): +class ForeverTakingResource(BaseResource): """ L{ForeverTakingResource} is a resource which never finishes responding to requests. """ - def __init__(self, write=False): - resource.Resource.__init__(self) + def __init__(self, write: bool = False): + super().__init__() self._write = write - def render(self, request): + def render(self, request: Request) -> int: if self._write: request.write(b"some bytes") return server.NOT_DONE_YET -class HostHeaderResource(resource.Resource): +class HostHeaderResource(BaseResource): """ A testing resource which renders itself as the value of the host header from the request. """ - def render(self, request): - return request.requestHeaders.getRawHeaders(b"host")[0] + def render(self, request: Request) -> bytes: + headers = request.requestHeaders.getRawHeaders(b"host") + assert headers + return headers[0] -class ClientIPResource(resource.Resource): +class ClientIPResource(BaseResource): """ A testing resource which renders itself as the request client IP address. """ - def render(self, request): + def render(self, request: Request) -> bytes: client_address = request.getClientAddress() if client_address is None or client_address.host is None: return b"" return to_bytes(client_address.host) -class PayloadResource(resource.Resource): +class PayloadResource(BaseResource): """ A testing resource which renders itself as the contents of the request body as long as the request body is 100 bytes long, otherwise which renders itself as C{"ERROR"}. """ - def render(self, request): - data = request.content.read() - contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0] - if len(data) != 100 or int(contentLength) != 100: + def render(self, request: Request) -> bytes: + assert request.content + data: bytes = request.content.read() + content_length = request.requestHeaders.getRawHeaders(b"content-length") + assert content_length + if len(data) != 100 or int(content_length[0]) != 100: return b"ERROR" return data -class LeafResource(resource.Resource): +class LeafResource(BaseResource): isLeaf = True def deferRequest( self, - request: Request, + request: HTTPRequest, delay: float, f: Callable[_P, _T], *a: _P.args, @@ -107,7 +132,7 @@ class LeafResource(resource.Resource): ) -> Deferred[_T]: from twisted.internet import reactor - def _cancelrequest(_): + def _cancelrequest(_: Failure) -> None: # silence CancelledError d.addErrback(lambda _: None) d.cancel() @@ -118,12 +143,13 @@ class LeafResource(resource.Resource): class Follow(LeafResource): - def render(self, request): + def render(self, request: Request) -> int: total = getarg(request, b"total", 100, type_=int) show = getarg(request, b"show", 1, type_=int) order = getarg(request, b"order", b"desc") maxlatency = getarg(request, b"maxlatency", 0, type_=float) n = getarg(request, b"n", total, type_=int) + nlist: Sequence[int] if order == b"rand": nlist = [random.randint(1, total) for _ in range(show)] else: # order == "desc" @@ -133,7 +159,7 @@ class Follow(LeafResource): self.deferRequest(request, lag, self.renderRequest, request, nlist) return NOT_DONE_YET - def renderRequest(self, request, nlist): + def renderRequest(self, request: Request, nlist: Sequence[int]) -> None: s = """ """ args = request.args.copy() for nl in nlist: @@ -146,45 +172,47 @@ class Follow(LeafResource): class Delay(LeafResource): - def render_GET(self, request): + def render_GET(self, request: Request) -> int: n = getarg(request, b"n", 1, type_=float) b = getarg(request, b"b", 1, type_=int) if b: # send headers now and delay body - request.write("") + request.write(b"") self.deferRequest(request, n, self._delayedRender, request, n) return NOT_DONE_YET - def _delayedRender(self, request, n): + def _delayedRender(self, request: Request, n: float) -> None: request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n")) request.finish() class Status(LeafResource): - def render_GET(self, request): + def render_GET(self, request: Request) -> bytes: n = getarg(request, b"n", 200, type_=int) request.setResponseCode(n) return b"" class Raw(LeafResource): - def render_GET(self, request): + def render_GET(self, request: Request) -> int: request.startedWriting = 1 self.deferRequest(request, 0, self._delayedRender, request) return NOT_DONE_YET render_POST = render_GET - def _delayedRender(self, request): + def _delayedRender(self, request: Request) -> None: raw = getarg(request, b"raw", b"HTTP 1.1 200 OK\n") request.startedWriting = 1 request.write(raw) + assert request.channel.transport is not None request.channel.transport.loseConnection() request.finish() class Echo(LeafResource): - def render_GET(self, request): + def render_GET(self, request: Request) -> bytes: + assert request.content output = { "headers": { to_unicode(k): [to_unicode(v) for v in vs] @@ -198,27 +226,29 @@ class Echo(LeafResource): class RedirectTo(LeafResource): - def render(self, request): + def render(self, request: Request) -> bytes: goto = getarg(request, b"goto", b"/") # we force the body content, otherwise Twisted redirectTo() # returns HTML with int: request.setHeader(b"Content-Length", b"1024") self.deferRequest(request, 0, self._delayedRender, request) return NOT_DONE_YET - def _delayedRender(self, request): + def _delayedRender(self, request: Request) -> None: request.write(b"partial content\n") request.finish() class Drop(Partial): - def _delayedRender(self, request): + def _delayedRender(self, request: Request) -> None: abort = getarg(request, b"abort", 0, type_=int) request.write(b"this connection will be dropped\n") tr = request.channel.transport @@ -233,8 +263,10 @@ class Drop(Partial): class ArbitraryLengthPayloadResource(LeafResource): - def render(self, request): - return request.content.read() + def render(self, request: Request) -> bytes: + assert request.content + data: bytes = request.content.read() + return data class NoMetaRefreshRedirect(Redirect): @@ -245,21 +277,23 @@ class NoMetaRefreshRedirect(Redirect): ) -class ContentLengthHeaderResource(resource.Resource): +class ContentLengthHeaderResource(BaseResource): """ A testing resource which renders itself as the value of the Content-Length header from the request. """ - def render(self, request): - return request.requestHeaders.getRawHeaders(b"content-length")[0] + def render(self, request: Request) -> bytes: + headers = request.requestHeaders.getRawHeaders(b"content-length") + assert headers + return headers[0] -class ChunkedResource(resource.Resource): - def render(self, request): +class ChunkedResource(BaseResource): + def render(self, request: Request) -> int: from twisted.internet import reactor - def response(): + def response() -> None: request.write(b"chunked ") request.write(b"content\n") request.finish() @@ -268,11 +302,11 @@ class ChunkedResource(resource.Resource): return server.NOT_DONE_YET -class BrokenChunkedResource(resource.Resource): - def render(self, request): +class BrokenChunkedResource(BaseResource): + def render(self, request: Request) -> int: from twisted.internet import reactor - def response(): + def response() -> None: request.write(b"chunked ") request.write(b"content\n") # Disable terminating chunk on finish. @@ -283,11 +317,11 @@ class BrokenChunkedResource(resource.Resource): return server.NOT_DONE_YET -class BrokenDownloadResource(resource.Resource): - def render(self, request): +class BrokenDownloadResource(BaseResource): + def render(self, request: Request) -> int: from twisted.internet import reactor - def response(): + def response() -> None: request.setHeader(b"Content-Length", b"20") request.write(b"partial") close_connection(request) @@ -296,22 +330,24 @@ class BrokenDownloadResource(resource.Resource): return server.NOT_DONE_YET -class EmptyContentTypeHeaderResource(resource.Resource): +class EmptyContentTypeHeaderResource(BaseResource): """ A testing resource which renders itself as the value of request body without content-type header in response. """ - def render(self, request): + def render(self, request: Request) -> bytes: + assert request.content request.setHeader("content-type", "") - return request.content.read() + data: bytes = request.content.read() + return data -class LargeChunkedFileResource(resource.Resource): - def render(self, request): +class LargeChunkedFileResource(BaseResource): + def render(self, request: Request) -> int: from twisted.internet import reactor - def response(): + def response() -> None: for _ in range(1024): request.write(b"x" * 1024) request.finish() @@ -320,43 +356,45 @@ class LargeChunkedFileResource(resource.Resource): return server.NOT_DONE_YET -class DuplicateHeaderResource(resource.Resource): - def render(self, request): +class DuplicateHeaderResource(BaseResource): + def render(self, request: Request) -> bytes: request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"]) return b"" -class UriResource(resource.Resource): +class UriResource(BaseResource): """Return the full uri that was requested""" - def getChild(self, path, request): + def getChild(self, path: bytes, request: Request) -> resource.Resource: return self - def render(self, request): + def render(self, request: Request) -> bytes | int: # Note: this is an ugly hack for CONNECT request timeout test. # Returning some data here fail SSL/TLS handshake # ToDo: implement proper HTTPS proxy tests, not faking them. if request.method != b"CONNECT": return request.uri + assert request.transport is not None request.transport.write(b"HTTP/1.1 200 Connection established\r\n\r\n") return NOT_DONE_YET -class ResponseHeadersResource(resource.Resource): +class ResponseHeadersResource(BaseResource): """Return a response with headers set from the JSON request body""" - def render(self, request): + def render(self, request: Request) -> bytes: + assert request.content body = json.loads(request.content.read().decode()) for header_name, header_value in body.items(): request.responseHeaders.setRawHeaders(header_name, [header_value]) return json.dumps(body).encode("utf-8") -class Compress(resource.Resource): +class Compress(BaseResource): """Compress the data sent in the request url params and set Content-Encoding header""" - def render(self, request): - data = request.args.get(b"data")[0] + def render(self, request: Request) -> bytes: + data = request.args[b"data"][0] accept_encoding_header = request.getHeader(b"accept-encoding") @@ -370,10 +408,10 @@ class Compress(resource.Resource): return b"Did not receive a valid accept-encoding header" -class SetCookie(resource.Resource): +class SetCookie(BaseResource): """Return a response with a Set-Cookie header for each request url parameter""" - def render(self, request): + def render(self, request: Request) -> bytes: for cookie_name, cookie_values in request.args.items(): for cookie_value in cookie_values: cookie = (cookie_name.decode() + "=" + cookie_value.decode()).encode() diff --git a/tests/mockserver/simple_https.py b/tests/mockserver/simple_https.py index fdea666e1..2a6cb6dd8 100644 --- a/tests/mockserver/simple_https.py +++ b/tests/mockserver/simple_https.py @@ -2,18 +2,23 @@ from __future__ import annotations -from twisted.web import resource +from typing import TYPE_CHECKING + from twisted.web.static import Data from .http_base import BaseMockServer, main_factory +from .http_resources import BaseResource, put_child + +if TYPE_CHECKING: + from twisted.web.server import Request -class Root(resource.Resource): - def __init__(self): - resource.Resource.__init__(self) - self.putChild(b"file", Data(b"0123456789", "text/plain")) +class Root(BaseResource): + def __init__(self) -> None: + super().__init__() + put_child(self, b"file", Data(b"0123456789", "text/plain")) - def getChild(self, path, request): + def getChild(self, path: bytes, request: Request) -> Root: return self @@ -29,7 +34,7 @@ class SimpleMockServer(BaseMockServer): cipher_string: str | None = None, tls_min_version: str | None = None, tls_max_version: str | None = None, - ): + ) -> None: super().__init__() self.keyfile = keyfile self.certfile = certfile diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py index 70c26e6d0..5306e3bf8 100644 --- a/tests/test_command_crawl.py +++ b/tests/test_command_crawl.py @@ -23,6 +23,18 @@ class TestCrawlCommand(TestProjectBase): _, _, stderr = self.crawl(code, proj_path, args=args) return stderr + def test_no_spider(self, proj_path: Path) -> None: + returncode, out, _ = proc("crawl", cwd=proj_path) + assert returncode == 2 + assert "Usage" in out + + def test_multiple_spiders(self, proj_path: Path) -> None: + returncode, _, err = proc("crawl", "myspider", "myspider2", cwd=proj_path) + assert returncode == 2 + assert ( + "running 'scrapy crawl' with more than one spider is not supported" in err + ) + def test_no_output(self, proj_path: Path) -> None: spider_code = """ import scrapy diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index d98dac968..c6a3afc91 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -2,13 +2,24 @@ from __future__ import annotations from typing import TYPE_CHECKING +import pytest + +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import proc if TYPE_CHECKING: + from pathlib import Path + from tests.mockserver.http import MockServer class TestFetchCommand: + @pytest.mark.parametrize("args", [(), ("not-a-url",), ("a:b", "c:d")]) + def test_bad_arguments(self, args: tuple[str, ...]) -> None: + returncode, out, _ = proc("fetch", *args) + assert returncode == 2 + assert "Usage" in out + def test_output(self, mockserver: MockServer) -> None: _, out, _ = proc("fetch", mockserver.url("/text")) assert out.strip() == "Works" @@ -36,3 +47,24 @@ class TestFetchCommand: "fetch", "-s", "TWISTED_REACTOR_ENABLED=False", mockserver.url("/text") ) assert out.strip() == "Works" + + +class TestFetchCommandWithSpider(TestProjectBase): + @pytest.fixture(autouse=True) + def create_files(self, proj_path: Path) -> None: + (proj_path / self.project_name / "spiders" / "myspider.py").write_text( + """ +import scrapy + +class MySpider(scrapy.Spider): + name = "myspider" + custom_settings = {"USER_AGENT": "myspider-user-agent"} +""", + encoding="utf-8", + ) + + def test_spider(self, proj_path: Path, mockserver: MockServer) -> None: + _, out, err = proc( + "fetch", "--spider", "myspider", mockserver.url("/echo"), cwd=proj_path + ) + assert "myspider-user-agent" in out, err diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py index 8bb6a2332..ddf25af4c 100644 --- a/tests/test_command_genspider.py +++ b/tests/test_command_genspider.py @@ -64,6 +64,24 @@ class TestGenspiderCommand(TestProjectBase): assert call("genspider", "--dump=basic", cwd=proj_path) == 0 assert call("genspider", "-d", "basic", cwd=proj_path) == 0 + @pytest.mark.parametrize( + "args", + [("--dump=nonexistent",), ("-t", "nonexistent", "test_name", "test.com")], + ) + def test_unknown_template(self, args: tuple[str, ...], proj_path: Path) -> None: + returncode, out, err = proc("genspider", *args, cwd=proj_path) + assert returncode == 0, err + assert "Unable to find template: nonexistent" in out + assert not (proj_path / self.project_name / "spiders" / "test_name.py").exists() + + def test_name_not_starting_with_a_letter(self, proj_path: Path) -> None: + """The module name, unlike the spider name, is prefixed with a letter.""" + _, out, err = proc("genspider", "1st_spider", "test.com", cwd=proj_path) + assert "Created spider '1st_spider'" in out, err + spider = proj_path / self.project_name / "spiders" / "a1st_spider.py" + assert spider.exists() + assert find_in_file(spider, r'name\s*=\s*"1st_spider"') is not None + @pytest.mark.skipif( sys.platform == "win32", reason="requires a POSIX shell editor script" ) @@ -87,7 +105,8 @@ class TestGenspiderCommand(TestProjectBase): ) def test_same_name_as_project(self, proj_path: Path) -> None: - assert call("genspider", self.project_name, cwd=proj_path) == 2 + _, out, err = proc("genspider", self.project_name, "test.com", cwd=proj_path) + assert "Cannot create a spider with the same name as your project" in out, err assert not ( proj_path / self.project_name / "spiders" / f"{self.project_name}.py" ).exists() diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 9b7131c7a..e434055b7 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse import re from typing import TYPE_CHECKING +from urllib.parse import urlparse import pytest @@ -126,6 +127,32 @@ class MySpider(scrapy.Spider): else: self.logger.debug('It Does Not Work :(') +class RetryRequestSpider(BaseSpider): + name = 'retry_request' + + def parse(self, response): + if response.meta.get('retried'): + yield {{'retried': True}} + return + response.meta['retried'] = True + yield response.request.replace(dont_filter=True) + +class CustomCallbackRetryRequestSpider(BaseSpider): + name = 'retry_request_custom_callback' + + def parse(self, response): + yield response.request.replace( + callback=self.parse_retry, + dont_filter=True, + ) + + def parse_retry(self, response): + if response.meta.get('retried'): + yield {{'retried_with_custom_callback': True}} + return + response.meta['retried'] = True + yield response.request.replace(dont_filter=True) + class MyGoodCrawlSpider(CrawlSpider): name = 'goodcrawl{self.spider_name}' @@ -381,6 +408,36 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "[{}, {'foo': 'bar'}]" in out + def test_retry_response_request( + self, proj_path: Path, mockserver: MockServer + ) -> None: + _, out, stderr = proc( + "parse", + "--spider", + "retry_request", + "-d", + "2", + mockserver.url("/html"), + cwd=proj_path, + ) + assert "RecursionError" not in stderr + assert "{'retried': True}" in out + + def test_retry_response_request_with_custom_callback( + self, proj_path: Path, mockserver: MockServer + ) -> None: + _, out, stderr = proc( + "parse", + "--spider", + "retry_request_custom_callback", + "-d", + "3", + mockserver.url("/html"), + cwd=proj_path, + ) + assert "RecursionError" not in stderr + assert "{'retried_with_custom_callback': True}" in out + def test_wrong_callback_passed( self, proj_path: Path, mockserver: MockServer ) -> None: @@ -496,6 +553,130 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} content = '[\n{},\n{"foo": "bar"}\n]' assert file_path.read_text(encoding="utf-8") == content + @pytest.mark.parametrize("args", [(), ("not-a-url",), ("a:b", "c:d")]) + def test_bad_arguments(self, args: tuple[str, ...], proj_path: Path) -> None: + returncode, out, _ = proc("parse", *args, cwd=proj_path) + assert returncode == 2 + assert "Usage" in out + + @pytest.mark.parametrize( + ("option", "message"), + [ + ("--meta", "Invalid -m/--meta value"), + ("-m", "Invalid -m/--meta value"), + ("--cbkwargs", "Invalid --cbkwargs value"), + ], + ) + def test_invalid_json( + self, option: str, message: str, proj_path: Path, mockserver: MockServer + ) -> None: + returncode, _, err = proc( + "parse", + "--spider", + self.spider_name, + option, + "{invalid", + mockserver.url("/html"), + cwd=proj_path, + ) + assert returncode == 2 + assert message in err + + def test_unknown_spider(self, proj_path: Path, mockserver: MockServer) -> None: + returncode, _, err = proc( + "parse", + "--spider", + "nonexistent", + mockserver.url("/html"), + cwd=proj_path, + ) + assert returncode == 0, err + assert "Unable to find spider: nonexistent" in err + + def test_spider_found_by_url(self, proj_path: Path, mockserver: MockServer) -> None: + """Without --spider, the spider is chosen based on the URL.""" + url = mockserver.url("/html") + # The spider name doubles as a domain of the spider, and it is matched + # against the netloc of the URL, hence the port. + (proj_path / self.project_name / "spiders" / "urlspider.py").write_text( + f""" +import scrapy + +class UrlSpider(scrapy.Spider): + name = "{urlparse(url).netloc}" + + def parse(self, response): + return [{{"found_by_url": True}}] +""", + encoding="utf-8", + ) + returncode, out, err = proc("parse", url, cwd=proj_path) + assert returncode == 0, err + assert "Unable to find spider for" not in err + assert "{'found_by_url': True}" in out + + def test_legacy_item_processor( + self, proj_path: Path, mockserver: MockServer + ) -> None: + """--pipelines supports an ITEM_PROCESSOR without process_item_async().""" + (proj_path / self.project_name / "legacy.py").write_text( + """ +import logging + +from twisted.internet.defer import succeed + + +class LegacyItemProcessor: + @classmethod + def from_crawler(cls, crawler): + return cls() + + def open_spider(self, spider): + return succeed(None) + + def close_spider(self, spider): + return succeed(None) + + def process_item(self, item, spider): + logging.info("Legacy item processor!") + return succeed(item) +""", + encoding="utf-8", + ) + _, _, stderr = proc( + "parse", + "--spider", + self.spider_name, + "--pipelines", + "-c", + "parse", + "-s", + f"ITEM_PROCESSOR={self.project_name}.legacy.LegacyItemProcessor", + mockserver.url("/html"), + cwd=proj_path, + ) + assert "INFO: Legacy item processor!" in stderr + + @pytest.mark.parametrize("verbose", [True, False]) + def test_no_items_no_links( + self, verbose: bool, proj_path: Path, mockserver: MockServer + ) -> None: + args = ["--verbose"] if verbose else [] + _, out, err = proc( + "parse", + "--spider", + self.spider_name, + "-c", + "parse", + "--noitems", + "--nolinks", + *args, + mockserver.url("/html"), + cwd=proj_path, + ) + assert "# Scraped Items" not in out, err + assert "# Requests" not in out + def test_parse_add_options(self): command = parse.Command() command.settings = Settings() diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index 2b410b5c6..11036eaeb 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -136,6 +136,12 @@ class MySpider(scrapy.Spider): log = self.get_log(tmp_path, "from scrapy.spiders import Spider\n") assert "No spider found in file" in log + @pytest.mark.parametrize("args", [(), ("a.py", "b.py")]) + def test_runspider_bad_arguments(self, args: tuple[str, ...]) -> None: + returncode, out, _ = proc("runspider", *args) + assert returncode == 2 + assert "Usage" in out + def test_runspider_file_not_found(self) -> None: _, _, log = proc("runspider", "some_non_existent_file") assert "File not found: some_non_existent_file" in log diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index f24200f53..29667a1ae 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -18,6 +18,7 @@ from scrapy.shell import Shell, inspect_response from scrapy.utils.reactor import _asyncio_reactor_path from scrapy.utils.test import get_crawler from tests import NON_EXISTING_RESOLVABLE, tests_datadir +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import proc from tests.utils.decorators import coroutine_test @@ -162,6 +163,33 @@ class TestShellCommand: assert ret == 0, out +class TestShellCommandWithSpider(TestProjectBase): + @pytest.fixture(autouse=True) + def create_files(self, proj_path: Path) -> None: + (proj_path / self.project_name / "spiders" / "myspider.py").write_text( + """ +import scrapy + +class MySpider(scrapy.Spider): + name = "myspider" +""", + encoding="utf-8", + ) + + def test_spider(self, proj_path: Path, mockserver: MockServer) -> None: + ret, out, err = proc( + "shell", + "--spider", + "myspider", + mockserver.url("/text"), + "-c", + "spider.name", + cwd=proj_path, + ) + assert ret == 0, err + assert out.strip() == "myspider" + + class TestInteractiveShell: def test_fetch(self, mockserver: MockServer) -> None: args = ( diff --git a/tests/test_commands.py b/tests/test_commands.py index 51f98db1b..f20ecc153 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3,23 +3,27 @@ from __future__ import annotations import argparse import json import sys -from io import StringIO +from pathlib import Path from typing import TYPE_CHECKING -from unittest import mock import pytest import scrapy -from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg -from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view +from scrapy.cmdline import _pop_command_name, execute +from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings from scrapy.utils.reactor import _asyncio_reactor_path from tests.utils.bases.commands import TestProjectBase -from tests.utils.cmdline import call, proc, write_recording_editor +from tests.utils.cmdline import ( + call, + proc, + write_recording_browser, + write_recording_editor, +) if TYPE_CHECKING: - from pathlib import Path + from tests.mockserver.http import MockServer class EmptyCommand(ScrapyCommand): @@ -109,6 +113,93 @@ class TestCommandSettings: ) +class TestGlobalOptions: + """Tests for the options that every command supports.""" + + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = "myspider" + + async def start(self): + self.logger.debug("It works!") + return + yield +""" + + @pytest.fixture + def spider_path(self, tmp_path: Path) -> Path: + path = tmp_path / "myspider.py" + path.write_text(self.spider_code, encoding="utf-8") + return path + + def test_invalid_set(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "-s", "FOO") + assert returncode == 2 + assert "Invalid -s value, use -s NAME=VALUE" in err + + def test_invalid_spider_argument(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "-a", "FOO") + assert returncode == 2 + assert "Invalid -a value, use -a NAME=VALUE" in err + + def test_logfile(self, tmp_path: Path, spider_path: Path) -> None: + logfile = tmp_path / "scrapy.log" + returncode, _, err = proc( + "runspider", str(spider_path), "--logfile", str(logfile) + ) + assert returncode == 0, err + assert "It works!" in logfile.read_text(encoding="utf-8") + assert "It works!" not in err + + def test_loglevel(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "--loglevel", "INFO") + assert returncode == 0, err + assert "It works!" not in err + assert "Spider closed (finished)" in err + + def test_nolog(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "--nolog") + assert returncode == 0, err + assert not err + + def test_pidfile(self, tmp_path: Path, spider_path: Path) -> None: + pidfile = tmp_path / "scrapy.pid" + returncode, _, err = proc( + "runspider", str(spider_path), "--pidfile", str(pidfile) + ) + assert returncode == 0, err + assert pidfile.read_text(encoding="utf-8").strip().isdigit() + + def test_pdb(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "--pdb") + assert returncode == 0, err + assert "It works!" in err + + +class TestSettingsCommand: + @pytest.mark.parametrize( + ("option", "setting", "expected"), + [ + ("--get", "BOT_NAME", "scrapybot"), + ("--getbool", "COOKIES_ENABLED", "True"), + ("--getint", "CONCURRENT_REQUESTS", "16"), + ("--getfloat", "DOWNLOAD_DELAY", "0.0"), + ("--getlist", "SPIDER_MODULES", "[]"), + ], + ) + def test_get(self, option: str, setting: str, expected: str) -> None: + returncode, out, err = proc("settings", option, setting) + assert returncode == 0, err + assert out.startswith(expected) + + def test_no_option(self) -> None: + returncode, out, err = proc("settings") + assert returncode == 0, err + assert not out + + class TestCommandCrawlerProcess(TestProjectBase): """Test that the command uses the expected kind of *CrawlerProcess and produces expected errors when needed.""" @@ -153,12 +244,6 @@ class MySpider(scrapy.Spider): self._append_settings(proj_mod_path, "LOG_LEVEL = 'DEBUG'\n") - @staticmethod - def _append_settings(proj_mod_path: Path, text: str) -> None: - """Add text to the end of the project settings.py.""" - with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f: - f.write(text) - @staticmethod def _replace_custom_settings( proj_mod_path: Path, spider_name: str, text: str @@ -347,23 +432,223 @@ class TestMiscCommands(TestProjectBase): subdir.mkdir(exist_ok=True) assert call("list", cwd=subdir) == 0 - def test_command_not_found(self) -> None: - na_msg = """ -The list command is not available from this location. -These commands are only available from within a project: check, crawl, edit, list, parse. -""" - not_found_msg = """ -Unknown command: abc -""" - params = [ - ("list", False, na_msg), - ("abc", False, not_found_msg), - ("abc", True, not_found_msg), - ] - for cmdname, inproject, message in params: - with mock.patch("sys.stdout", new=StringIO()) as out: - _print_unknown_command_msg(Settings(), cmdname, inproject) - assert out.getvalue().strip() == message.strip() + +class TestCommandListing(TestProjectBase): + """Tests for the command list that ``scrapy`` prints when called without a + command name.""" + + def test_outside_project(self) -> None: + returncode, out, err = proc() + assert returncode == 0, err + assert f"Scrapy {scrapy.__version__} - no active project" in out + assert "Available commands:" in out + assert "Create new project" in out + assert "More commands available when run from project directory" in out + assert 'Use "scrapy -h" to see more info about a command' in out + + def test_inside_project(self, proj_path: Path) -> None: + returncode, out, err = proc(cwd=proj_path) + assert returncode == 0, err + assert ( + f"Scrapy {scrapy.__version__} - active project: {self.project_name}" in out + ) + assert "List available spiders" in out + assert "More commands available when run from project directory" not in out + + +class TestUnknownCommand(TestProjectBase): + def test_outside_project(self) -> None: + returncode, out, err = proc("abc") + assert returncode == 2, err + assert f"Scrapy {scrapy.__version__} - no active project" in out + assert "Unknown command: abc" in out + assert 'Use "scrapy" to see available commands' in out + + def test_inside_project(self, proj_path: Path) -> None: + returncode, out, err = proc("abc", cwd=proj_path) + assert returncode == 2, err + assert ( + f"Scrapy {scrapy.__version__} - active project: {self.project_name}" in out + ) + assert "Unknown command: abc" in out + + def test_project_only_command_outside_project(self) -> None: + returncode, out, err = proc("list") + assert returncode == 2, err + assert "The list command is not available from this location." in out + assert ( + "These commands are only available from within a project: " + "check, crawl, edit, list, parse." in out + ) + + +class TestCommandsModule(TestProjectBase): + """Tests for commands defined in the module of the COMMANDS_MODULE setting.""" + + @pytest.fixture + def proj_path_with_commands(self, proj_path: Path) -> Path: + commands_path = proj_path / self.project_name / "commands" + commands_path.mkdir() + (commands_path / "__init__.py").touch() + (commands_path / "mycmd.py").write_text( + """ +from scrapy.commands import ScrapyCommand + + +class Command(ScrapyCommand): + requires_crawler_process = False + + def short_desc(self): + return "My custom command" + + def run(self, args, opts): + print("My custom command ran") +""", + encoding="utf-8", + ) + (commands_path / "helpcmd.py").write_text( + """ +from scrapy.commands import ScrapyCommand +from scrapy.exceptions import UsageError + + +class Command(ScrapyCommand): + requires_crawler_process = False + + def short_desc(self): + return "My command that asks for its help message" + + def run(self, args, opts): + raise UsageError +""", + encoding="utf-8", + ) + (commands_path / "silentcmd.py").write_text( + """ +from scrapy.commands import ScrapyCommand +from scrapy.exceptions import UsageError + + +class Command(ScrapyCommand): + requires_crawler_process = False + + def short_desc(self): + return "My command that fails silently" + + def run(self, args, opts): + raise UsageError(print_help=False) +""", + encoding="utf-8", + ) + self._append_settings( + proj_path / self.project_name, + f'\nCOMMANDS_MODULE = "{self.project_name}.commands"\n', + ) + return proj_path + + def test_listed(self, proj_path_with_commands: Path) -> None: + returncode, out, err = proc(cwd=proj_path_with_commands) + assert returncode == 0, err + assert "My custom command" in out + + def test_run(self, proj_path_with_commands: Path) -> None: + returncode, out, err = proc("mycmd", cwd=proj_path_with_commands) + assert returncode == 0, err + assert "My custom command ran" in out + + def test_usage_error(self, proj_path_with_commands: Path) -> None: + """A message-less UsageError makes the help message be printed.""" + returncode, out, err = proc("helpcmd", cwd=proj_path_with_commands) + assert returncode == 2, err + assert "scrapy helpcmd" in out + + def test_usage_error_without_help(self, proj_path_with_commands: Path) -> None: + """A message-less UsageError with print_help disabled prints nothing.""" + returncode, out, err = proc("silentcmd", cwd=proj_path_with_commands) + assert returncode == 2, err + assert not out + + +class TestEntryPointCommands: + """Tests for commands defined in the scrapy.commands entry point group.""" + + @staticmethod + def _write_dist(path: Path, entry_point: str) -> None: + """Write into *path* a package with a command and a function, and the + metadata of an installed distribution that declares *entry_point* in + the scrapy.commands entry point group. + + Since ``python -m scrapy.cmdline`` puts the current working directory + in the import path, running it with *path* as the working directory + makes Scrapy find that entry point. + """ + package_path = path / "mycmds" + package_path.mkdir() + (package_path / "__init__.py").touch() + (package_path / "mycmd.py").write_text( + """ +from scrapy.commands import ScrapyCommand + + +class Command(ScrapyCommand): + requires_crawler_process = False + + def short_desc(self): + return "My entry point command" + + def run(self, args, opts): + print("My entry point command ran") + + +def not_a_command(): + pass +""", + encoding="utf-8", + ) + dist_info_path = path / "mycmds-1.0.dist-info" + dist_info_path.mkdir() + (dist_info_path / "METADATA").write_text( + "Metadata-Version: 2.1\nName: mycmds\nVersion: 1.0\n", encoding="utf-8" + ) + (dist_info_path / "entry_points.txt").write_text( + f"[scrapy.commands]\n{entry_point}\n", encoding="utf-8" + ) + + def test_listed(self, tmp_path: Path) -> None: + self._write_dist(tmp_path, "mycmd = mycmds.mycmd:Command") + returncode, out, err = proc(cwd=tmp_path) + assert returncode == 0, err + assert "My entry point command" in out + + def test_run(self, tmp_path: Path) -> None: + self._write_dist(tmp_path, "mycmd = mycmds.mycmd:Command") + returncode, out, err = proc("mycmd", cwd=tmp_path) + assert returncode == 0, err + assert "My entry point command ran" in out + + def test_not_a_class(self, tmp_path: Path) -> None: + self._write_dist(tmp_path, "mycmd = mycmds.mycmd:not_a_command") + returncode, _, err = proc("version", cwd=tmp_path) + assert returncode == 1 + assert "ValueError: Invalid entry point mycmd" in err + + +class TestExecute: + """Tests for calls to scrapy.cmdline.execute() from Python code, which the + command line does not cover.""" + + def test_argv(self, capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit) as exc_info: + execute(["scrapy", "version"]) + assert exc_info.value.code == 0 + assert scrapy.__version__ in capsys.readouterr().out + + def test_settings(self, capsys: pytest.CaptureFixture[str]) -> None: + settings = Settings() + with pytest.raises(SystemExit) as exc_info: + execute(["scrapy", "settings", "--get", "BOT_NAME"], settings=settings) + assert exc_info.value.code == 0 + assert capsys.readouterr().out.strip() == "scrapybot" class TestBenchCommand: @@ -385,18 +670,31 @@ class TestBenchCommand: class TestViewCommand: - def test_methods(self) -> None: - command = view.Command() - command.settings = Settings() - parser = argparse.ArgumentParser( - prog="scrapy", - prefix_chars="-", - formatter_class=ScrapyHelpFormatter, - conflict_handler="resolve", + @pytest.mark.skipif( + sys.platform == "win32", reason="requires a POSIX shell browser script" + ) + def test_view( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mockserver: MockServer + ) -> None: + opened = tmp_path / "opened.txt" + browser = tmp_path / "fake-browser.sh" + write_recording_browser(browser, opened) + monkeypatch.setenv("BROWSER", str(browser)) + + returncode, _, err = proc("view", mockserver.url("/html"), cwd=tmp_path) + + assert returncode == 0, err + url = opened.read_text(encoding="utf-8") + assert url.startswith("file://") + body = Path(url.removeprefix("file://")).read_text(encoding="utf-8") + assert "

Works

" in body + + def test_non_text_response(self, mockserver: MockServer) -> None: + returncode, _, err = proc( + "view", mockserver.url("/static/files/images/scrapy.png") ) - command.add_options(parser) - assert command.short_desc() == "Open URL in browser, as seen by Scrapy" - assert "URL using the Scrapy downloader and show its" in command.long_desc() + assert returncode == 0, err + assert "Cannot view a non-text response." in err class TestEditCommand(TestProjectBase): @@ -423,6 +721,11 @@ class TestEditCommand(TestProjectBase): assert returncode == 1 assert "Spider not found: nonexistent" in err + def test_edit_no_spider(self, proj_path: Path) -> None: + returncode, out, _ = proc("edit", cwd=proj_path) + assert returncode == 2 + assert "Usage" in out + class TestHelpMessage(TestProjectBase): @pytest.mark.parametrize( diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index fdd5edc27..912c0450b 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -14,6 +14,7 @@ from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse +from scrapy import Request, Spider from scrapy.core.downloader import Downloader, Slot, tls from scrapy.core.downloader.contextfactory import ( _load_context_factory_from_settings, @@ -30,14 +31,17 @@ from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -from tests.mockserver.http_resources import PayloadResource +from tests.mockserver.http_resources import PayloadResource, put_child from tests.mockserver.utils import ssl_context_factory from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from twisted.internet.defer import Deferred + from twisted.internet.interfaces import IListeningPort from twisted.web.iweb import IBodyProducer + from scrapy.http import Response + class TestSlot: def test_repr(self): @@ -51,7 +55,7 @@ class TestContextFactoryBase: async def server_url(self, tmp_path): (tmp_path / "file").write_bytes(b"0123456789") r = static.File(str(tmp_path)) - r.putChild(b"payload", PayloadResource()) + put_child(r, b"payload", PayloadResource()) site = server.Site(r, timeout=None) port = self._listen(site) portno = port.getHost().port @@ -60,7 +64,7 @@ class TestContextFactoryBase: await port.stopListening() - def _listen(self, site): + def _listen(self, site: server.Site) -> IListeningPort: from twisted.internet import reactor return reactor.listenSSL( @@ -296,10 +300,30 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase): await self._assert_factory_works(server_url, client_context_factory) +@pytest.mark.parametrize( + ("concurrency", "active", "expected"), + [ + (2, 1, False), + (2, 2, True), + (0, 0, False), + (0, 2, False), + ], +) +def test_needs_backout(concurrency: int, active: int, expected: bool) -> None: + crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency}) + downloader = Downloader(crawler) + downloader.active = {Request(f"https://example.com/{i}") for i in range(active)} + assert downloader.needs_backout() is expected + downloader.close() + + @coroutine_test async def test_fetch_deprecated_spider_arg(): class CustomDownloader(Downloader): - def fetch(self, request, spider): # pylint: disable=signature-differs + # requiring the spider argument is what triggers the deprecation + def fetch( # type: ignore[override] # pylint: disable=signature-differs + self, request: Request, spider: Spider + ) -> Deferred[Response | Request]: return super().fetch(request, spider) crawler = get_crawler(DefaultSpider, {"DOWNLOADER": CustomDownloader}) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index b4f906e25..358f20ed7 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -74,6 +74,39 @@ class TestCrawler: assert not settings.frozen assert crawler.settings.frozen + @pytest.mark.parametrize( + ("attr", "setting"), + [ + ("download_delay", "DOWNLOAD_DELAY"), + ("max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN"), + ], + ) + def test_deprecated_spider_attr(self, attr: str, setting: str) -> None: + crawler = get_raw_crawler(type("_Spider", (DefaultSpider,), {attr: 2})) + with pytest.warns( + ScrapyDeprecationWarning, + match=f"The {attr!r} spider attribute is deprecated. Use the {setting} ", + ): + crawler._apply_settings() + assert crawler.settings.getint(setting) == 2 + + @pytest.mark.parametrize( + ("attr", "setting"), + [ + ("download_delay", "DOWNLOAD_DELAY"), + ("max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN"), + ], + ) + def test_deprecated_spider_attr_ignored(self, attr: str, setting: str) -> None: + crawler = get_raw_crawler(type("_Spider", (DefaultSpider,), {attr: 2})) + crawler.settings.set(setting, 3, priority="spider") + with pytest.warns( + ScrapyDeprecationWarning, + match=f"The {attr!r} spider attribute is deprecated. It is also being ", + ): + crawler._apply_settings() + assert crawler.settings.getint(setting) == 3 + def test_crawler_accepts_dict(self) -> None: crawler = get_crawler(DefaultSpider, {"foo": "bar"}) assert crawler.settings["foo"] == "bar" diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index fe2f83161..733b6797d 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -14,7 +14,8 @@ from packaging.version import parse as parse_version from pexpect.popen_spawn import PopenSpawn from w3lib import __version__ as w3lib_version -from tests.utils import async_sleep, get_script_run_env +from scrapy.utils.asyncio import sleep +from tests.utils import get_script_run_env from tests.utils.decorators import coroutine_test if TYPE_CHECKING: @@ -244,7 +245,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): p.kill(sig) p.expect_exact("shutting down gracefully") # sending the second signal too fast often causes problems - await async_sleep(0.01) + await sleep(0.01) p.kill(sig) p.expect_exact("forcing unclean shutdown") p.wait() # type: ignore[no-untyped-call] diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index 5ceb93382..976daacaf 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -15,6 +15,8 @@ from scrapy.core.downloader.handlers._httpx import ( HttpxDownloadHandler, ) from scrapy.exceptions import DownloadFailedError +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler from tests.utils.bases.download_handlers_http import ( TestHttpBase, TestHttpProxyBase, @@ -161,3 +163,15 @@ class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase): @pytest.mark.requires_internet class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase): pass + + +@pytest.mark.parametrize(("concurrency", "expected"), [(16, 16), (0, None)]) +@coroutine_test +async def test_pool_limits(concurrency: int, expected: int | None) -> None: + crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency}) + handler = build_from_crawler(HttpxDownloadHandler, crawler) + try: + assert handler._limits.max_connections == expected + assert handler._limits.max_keepalive_connections == expected + finally: + await handler.close() diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py index 489b70e74..14de97b21 100644 --- a/tests/test_downloader_handler_twisted_ftp.py +++ b/tests/test_downloader_handler_twisted_ftp.py @@ -156,14 +156,17 @@ class TestFTP(TestFTPBase): for filename, content in self.test_files: (userdir / filename).write_bytes(content) - def _get_factory(self, root): + def _get_factory(self, root: Path) -> FTPFactory: from twisted.protocols.ftp import FTPFactory, FTPRealm realm = FTPRealm(anonymousRoot=str(root), userHome=str(root)) - p = portal.Portal(realm) + # zope.interface has no type hints, so mypy cannot tell that these + # objects provide the interfaces that Portal expects. + p = portal.Portal(realm) # type: ignore[arg-type] users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() - users_checker.addUser(self.username, self.password) - p.registerChecker(users_checker, credentials.IUsernamePassword) + # the FTP protocol authenticates with str credentials + users_checker.addUser(self.username, self.password) # type: ignore[arg-type] + p.registerChecker(users_checker, credentials.IUsernamePassword) # type: ignore[arg-type] return FTPFactory(portal=p) @deferred_f_from_coro_f @@ -192,12 +195,17 @@ class TestAnonymousFTP(TestFTPBase): for filename, content in self.test_files: (root / filename).write_bytes(content) - def _get_factory(self, tmp_path): + def _get_factory(self, tmp_path: Path) -> FTPFactory: from twisted.protocols.ftp import FTPFactory, FTPRealm realm = FTPRealm(anonymousRoot=str(tmp_path)) - p = portal.Portal(realm) - p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous) + # zope.interface has no type hints, so mypy cannot tell that these + # objects provide the interfaces that Portal expects. + p = portal.Portal(realm) # type: ignore[arg-type] + p.registerChecker( + checkers.AllowAnonymousAccess(), # type: ignore[arg-type] + credentials.IAnonymous, + ) return FTPFactory(portal=p, userAnonymous=self.username) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 7ad103f27..8d999d952 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -1,5 +1,6 @@ import logging from collections.abc import Iterable +from typing import Any import pytest @@ -219,7 +220,7 @@ class TestCookiesMiddleware: def test_complex_cookies(self): # merge some cookies into jar - cookies = [ + cookies: list[VerboseCookie] = [ { "name": "C1", "value": "value1", @@ -483,13 +484,13 @@ class TestCookiesMiddleware: def _test_cookie_redirect( self, - source, - target, + source: str | dict[str, Any], + target: str | dict[str, Any], *, - cookies1, - cookies2, - ): - input_cookies = {"a": "b"} + cookies1: bool, + cookies2: bool, + ) -> None: + input_cookies: CookiesT = {"a": "b"} if not isinstance(source, dict): source = {"url": source} @@ -551,11 +552,11 @@ class TestCookiesMiddleware: def _test_cookie_header_redirect( self, - source, - target, + source: str | dict[str, Any], + target: str | dict[str, Any], *, - cookies2, - ): + cookies2: bool, + ) -> None: """Test the handling of a user-defined Cookie header when building a redirect follow-up request. @@ -623,14 +624,14 @@ class TestCookiesMiddleware: def _test_user_set_cookie_domain_followup( self, - url1, - url2, - domain, + url1: str, + url2: str, + domain: str, *, - cookies1, - cookies2, - ): - input_cookies = [ + cookies1: bool, + cookies2: bool, + ) -> None: + input_cookies: list[VerboseCookie] = [ { "name": "a", "value": "b", @@ -686,16 +687,16 @@ class TestCookiesMiddleware: def _test_server_set_cookie_domain_followup( self, - url1, - url2, - domain, + url1: str, + url2: str, + domain: str, *, - cookies, - ): + cookies: bool, + ) -> None: request1 = Request(url1) self.mw.process_request(request1) - input_cookies = [ + input_cookies: list[VerboseCookie] = [ { "name": "a", "value": "b", @@ -747,8 +748,14 @@ class TestCookiesMiddleware: ) def _test_cookie_redirect_scheme_change( - self, secure, from_scheme, to_scheme, cookies1, cookies2, cookies3 - ): + self, + secure: bool | object, + from_scheme: str, + to_scheme: str, + cookies1: bool, + cookies2: bool, + cookies3: bool, + ) -> None: """When a redirect causes the URL scheme to change from *from_scheme* to *to_scheme*, while domain and port remain the same, and given a cookie on the initial request with its secure attribute set to @@ -756,10 +763,11 @@ class TestCookiesMiddleware: initial request (*cookies1*), if it should be kept by the redirect middleware (*cookies2*), and if it should be present on the Cookie header in the redirected request (*cookie3*).""" - cookie_kwargs = {} + cookie: VerboseCookie = {"name": "a", "value": "b"} if secure is not UNSET: - cookie_kwargs["secure"] = secure - input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}] + assert isinstance(secure, bool) + cookie["secure"] = secure + input_cookies = [cookie] request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies) self.mw.process_request(request1) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 9d5d6874e..dc8228470 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -1,10 +1,12 @@ from __future__ import annotations import email.utils +import logging import shutil import tempfile import time from contextlib import contextmanager +from pathlib import Path from typing import TYPE_CHECKING, Any from unittest import mock @@ -12,6 +14,7 @@ import pytest from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware from scrapy.exceptions import IgnoreRequest +from scrapy.extensions.httpcache import DummyPolicy from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider from scrapy.utils.test import get_crawler @@ -22,6 +25,14 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler +class AlwaysStalePolicy(DummyPolicy): + """:class:`~scrapy.extensions.httpcache.DummyPolicy` that always + revalidates cached responses.""" + + def is_cached_response_fresh(self, cachedresponse, request): + return False + + class TestBase: """Base class with common setup and helper methods.""" @@ -83,16 +94,22 @@ class TestBase: finally: mw.spider_closed(crawler.spider) - def assertEqualResponse(self, response1, response2): + def assertEqualResponse(self, response1: Response, response2: Response) -> None: assert response1.url == response2.url assert response1.status == response2.status assert response1.headers == response2.headers assert response1.body == response2.body -class StorageTestMixin: +class StorageTestMixin(TestBase): """Mixin containing storage-specific test methods.""" + def _corrupt_cache_entry( + self, storage: Any, spider: Spider, request: Request + ) -> None: + """Make the cache entry of *request* unreadable for *storage*.""" + raise NotImplementedError + def test_storage(self): with self._storage(HTTPCACHE_EXPIRATION_SECS=1) as (storage, crawler): request2 = self.request.copy() @@ -115,6 +132,42 @@ class StorageTestMixin: with mock.patch("scrapy.extensions.httpcache.time", return_value=future): assert storage.retrieve_response(crawler.spider, self.request) + def test_corrupted_cache_entry_is_a_miss(self, caplog): + with self._middleware() as mw: + spider = mw.crawler.spider + assert spider + assert mw.crawler.stats + mw.storage.store_response(spider, self.request, self.response) + self._corrupt_cache_entry(mw.storage, spider, self.request) + + caplog.clear() + with caplog.at_level(logging.WARNING): + assert mw.process_request(self.request) is None + + assert "treating it as a cache miss" in caplog.text + assert mw.crawler.stats.get_value("httpcache/retrieve_error") == 1 + assert mw.crawler.stats.get_value("httpcache/miss") == 1 + + # Storing the response again replaces the corrupted cache entry. + mw.storage.store_response(spider, self.request, self.response) + self.assertEqualResponse( + self.response, mw.storage.retrieve_response(spider, self.request) + ) + + def test_corrupted_cache_entry_ignore_missing(self): + with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw: + spider = mw.crawler.spider + assert spider + assert mw.crawler.stats + mw.storage.store_response(spider, self.request, self.response) + self._corrupt_cache_entry(mw.storage, spider, self.request) + + with pytest.raises(IgnoreRequest): + mw.process_request(self.request) + + assert mw.crawler.stats.get_value("httpcache/retrieve_error") == 1 + assert mw.crawler.stats.get_value("httpcache/ignore") == 1 + def test_storage_no_content_type_header(self): """Test that the response body is used to get the right response class even if there is no Content-Type header""" @@ -131,7 +184,7 @@ class StorageTestMixin: self.assertEqualResponse(response, cached_response) -class PolicyTestMixin: +class PolicyTestMixin(TestBase): """Mixin containing policy-specific test methods.""" def test_dont_cache(self): @@ -242,6 +295,22 @@ class DummyPolicyTestMixin(PolicyTestMixin): self.assertEqualResponse(self.response, response) assert "cached" in response.flags + def test_revalidation_keeps_cached_response(self): + # The dummy policy considers every cached response valid, so a policy + # that subclasses it to force revalidation always gets the cached + # response back, whatever the new response is. + with self._middleware(HTTPCACHE_POLICY=AlwaysStalePolicy) as mw: + assert mw.process_request(self.request) is None + mw.process_response(self.request, self.response) + + assert mw.process_request(self.request) is None + fresh_response = self.response.replace(body=b"new body") + response = mw.process_response(self.request, fresh_response) + assert isinstance(response, Response) + self.assertEqualResponse(self.response, response) + assert "cached" in response.flags + assert mw.stats.get_value("httpcache/revalidate") == 1 + class RFC2616PolicyTestMixin(PolicyTestMixin): """Mixin containing RFC2616 policy specific test methods.""" @@ -249,12 +318,12 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): @staticmethod def _process_requestresponse( mw: HttpCacheMiddleware, request: Request, response: Response | None - ) -> Response | Request: - result = None + ) -> Response: + result: Request | Response | None = None try: result = mw.process_request(request) if result: - assert isinstance(result, (Request, Response)) + assert isinstance(result, Response) return result assert response is not None result = mw.process_response(request, response) @@ -282,6 +351,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): res2 = self._process_requestresponse(mw, req0, res0) assert "cached" not in res2.flags res3 = mw.process_request(req0) + assert isinstance(res3, Response) assert "cached" in res3.flags self.assertEqualResponse(res2, res3) # request with no-cache directive must not return cached response @@ -513,6 +583,53 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): else: assert "cached" in res5.flags + def test_middleware_ignore_schemes(self): + # file responses are not cached by default + req = Request("file:///tmp/t.txt") + res = Response(req.url, headers={"Expires": self.tomorrow}) + with self._middleware() as mw: + assert mw.process_request(req) is None + mw.process_response(req, res) + + assert mw.storage.retrieve_response(mw.crawler.spider, req) is None + assert mw.process_request(req) is None + + def test_max_stale_with_value(self): + # A response that expired one day ago. + headers = {"Date": self.yesterday, "Expires": self.yesterday} + with self._middleware() as mw: + req0 = Request("http://example.com") + res0 = Response(req0.url, headers=headers) + self._process_requestresponse(mw, req0, res0) + + # max-stale greater than the staleness of the cached response + req1 = req0.replace(headers={"Cache-Control": "max-stale=172800"}) + res1 = mw.process_request(req1) + assert isinstance(res1, Response) + assert "cached" in res1.flags + + # max-stale lower than the staleness of the cached response + req2 = req0.replace(headers={"Cache-Control": "max-stale=60"}) + assert mw.process_request(req2) is None + + # a non-integer max-stale value is ignored + req3 = req0.replace(headers={"Cache-Control": "max-stale=soon"}) + assert mw.process_request(req3) is None + + def test_response_dated_in_the_future(self): + # A Date header ahead of the local clock must not make the cached + # response look aged. + headers = {"Date": self.tomorrow, "Cache-Control": "max-age=10"} + with self._middleware() as mw: + req0 = Request("http://example.com") + res0 = Response(req0.url, headers=headers) + res1 = self._process_requestresponse(mw, req0, res0) + assert "cached" not in res1.flags + + res2 = self._process_requestresponse(mw, req0, None) + self.assertEqualResponse(res1, res2) + assert "cached" in res2.flags + def test_process_exception(self): with self._middleware() as mw: res0 = Response(self.request.url, headers={"Expires": self.yesterday}) @@ -523,6 +640,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): assert mw.process_request(req0) is None res1 = mw.process_exception(req0, e("foo")) # Use cached response as recovery + assert isinstance(res1, Response) assert "cached" in res1.flags self.assertEqualResponse(res0, res1) # Do not use cached response for unhandled exceptions @@ -556,29 +674,39 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): # Concrete test classes that combine storage and policy mixins -class TestFilesystemStorageWithDummyPolicy( - TestBase, StorageTestMixin, DummyPolicyTestMixin -): +class FilesystemStorageTestMixin(StorageTestMixin): storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" + + def _corrupt_cache_entry(self, storage, spider, request) -> None: + rpath = Path(storage._get_request_path(spider, request)) + (rpath / "response_body").unlink() + + +class DbmStorageTestMixin(StorageTestMixin): + storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" + + def _corrupt_cache_entry(self, storage, spider, request) -> None: + key = storage._fingerprinter.fingerprint(request).hex() + storage.db[f"{key}_data"] = b"not a pickle" + + +class TestFilesystemStorageWithDummyPolicy( + FilesystemStorageTestMixin, DummyPolicyTestMixin +): policy_class = "scrapy.extensions.httpcache.DummyPolicy" class TestFilesystemStorageWithRFC2616Policy( - TestBase, StorageTestMixin, RFC2616PolicyTestMixin + FilesystemStorageTestMixin, RFC2616PolicyTestMixin ): - storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" policy_class = "scrapy.extensions.httpcache.RFC2616Policy" -class TestDbmStorageWithDummyPolicy(TestBase, StorageTestMixin, DummyPolicyTestMixin): - storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" +class TestDbmStorageWithDummyPolicy(DbmStorageTestMixin, DummyPolicyTestMixin): policy_class = "scrapy.extensions.httpcache.DummyPolicy" -class TestDbmStorageWithRFC2616Policy( - TestBase, StorageTestMixin, RFC2616PolicyTestMixin -): - storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" +class TestDbmStorageWithRFC2616Policy(DbmStorageTestMixin, RFC2616PolicyTestMixin): policy_class = "scrapy.extensions.httpcache.RFC2616Policy" @@ -599,3 +727,8 @@ class TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPol def _get_settings(self, **new_settings) -> dict[str, Any]: new_settings.setdefault("HTTPCACHE_GZIP", True) return super()._get_settings(**new_settings) + + def _corrupt_cache_entry(self, storage, spider, request) -> None: + # A spider killed while writing a gzip file leaves it truncated. + body_path = Path(storage._get_request_path(spider, request), "response_body") + body_path.write_bytes(body_path.read_bytes()[:-5]) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 55f06b396..fa0707491 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -3,6 +3,7 @@ from importlib.util import find_spec from io import BytesIO from logging import WARNING from pathlib import Path +from typing import Any import pytest from w3lib.encoding import resolve_encoding @@ -15,6 +16,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWar from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider +from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler from tests import tests_datadir @@ -72,6 +74,7 @@ class TestHttpCompression: def setup_method(self): self.crawler = get_crawler(Spider) self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) + assert self.crawler.stats self.crawler.stats.open_spider() def _getresponse(self, coding: str) -> Response: @@ -96,7 +99,8 @@ class TestHttpCompression: ) return response - def assertStatsEqual(self, key, value): + def assertStatsEqual(self, key: str, value: Any) -> None: + assert self.crawler.stats assert self.crawler.stats.get_value(key) == value, str( self.crawler.stats.get_stats() ) @@ -145,6 +149,7 @@ class TestHttpCompression: def test_process_response_gzip(self): response = self._getresponse("gzip") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"gzip" @@ -159,6 +164,7 @@ class TestHttpCompression: _skip_if_no_br() response = self._getresponse("br") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"br" newresponse = self.mw.process_response(request, response) @@ -172,6 +178,7 @@ class TestHttpCompression: if find_spec("brotli") is not None or find_spec("brotlicffi") is not None: pytest.skip("Requires not having brotli support") response = self._getresponse("br") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"br" caplog.clear() @@ -201,6 +208,7 @@ class TestHttpCompression: if not check_key.startswith("zstd-"): continue response = self._getresponse(check_key) + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"zstd" newresponse = self.mw.process_response(request, response) @@ -216,6 +224,7 @@ class TestHttpCompression: if find_spec("zstandard") is not None: pytest.skip("Requires not having zstandard support") response = self._getresponse("zstd-static-content-size") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"zstd" caplog.clear() @@ -239,6 +248,7 @@ class TestHttpCompression: def test_process_response_rawdeflate(self): response = self._getresponse("rawdeflate") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"deflate" @@ -251,6 +261,7 @@ class TestHttpCompression: def test_process_response_zlibdelate(self): response = self._getresponse("zlibdeflate") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"deflate" @@ -275,6 +286,7 @@ class TestHttpCompression: def test_multipleencodings(self): response = self._getresponse("gzip") response.headers["Content-Encoding"] = ["uuencode", "gzip"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -282,6 +294,7 @@ class TestHttpCompression: def test_multi_compression_single_header(self): response = self._getresponse("gzip-deflate") + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -293,6 +306,7 @@ class TestHttpCompression: ) -> None: response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = [b"gzip, foo, deflate"] + assert response.request request = response.request caplog.clear() with caplog.at_level( @@ -315,6 +329,7 @@ class TestHttpCompression: def test_multi_compression_multiple_header(self): response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = ["gzip", "deflate"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -324,6 +339,7 @@ class TestHttpCompression: def test_multi_compression_multiple_header_invalid_compression(self): response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = ["gzip", "foo", "deflate"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -332,6 +348,7 @@ class TestHttpCompression: def test_multi_compression_single_and_multiple_header(self): response = self._getresponse("gzip-deflate-gzip") response.headers["Content-Encoding"] = ["gzip", "deflate, gzip"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -341,6 +358,7 @@ class TestHttpCompression: def test_multi_compression_single_and_multiple_header_invalid_compression(self): response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = ["gzip", "foo,deflate"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -397,9 +415,7 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) def test_process_response_no_content_type_header(self): - headers = { - "Content-Encoding": "identity", - } + headers = {b"Content-Encoding": b"identity"} plainbody = ( b"Some page" b'' @@ -414,6 +430,7 @@ class TestHttpCompression: newresponse = self.mw.process_response(request, response) assert isinstance(newresponse, respcls) + assert isinstance(newresponse, HtmlResponse) assert newresponse.body == plainbody assert newresponse.encoding == resolve_encoding("gb2312") self.assertStatsEqual("httpcompression/response_count", 1) @@ -422,6 +439,7 @@ class TestHttpCompression: def test_process_response_gzipped_contenttype(self): response = self._getresponse("gzip") response.headers["Content-Type"] = "application/gzip" + assert response.request request = response.request newresponse = self.mw.process_response(request, response) @@ -434,6 +452,7 @@ class TestHttpCompression: def test_process_response_gzip_app_octetstream_contenttype(self): response = self._getresponse("gzip") response.headers["Content-Type"] = "application/octet-stream" + assert response.request request = response.request newresponse = self.mw.process_response(request, response) @@ -446,6 +465,7 @@ class TestHttpCompression: def test_process_response_gzip_binary_octetstream_contenttype(self): response = self._getresponse("x-gzip") response.headers["Content-Type"] = "binary/octet-stream" + assert response.request request = response.request newresponse = self.mw.process_response(request, response) @@ -504,6 +524,7 @@ class TestHttpCompression: def test_process_response_head_request_no_decode_required(self): response = self._getresponse("gzip") response.headers["Content-Type"] = "application/gzip" + assert response.request request = response.request request.method = "HEAD" response = response.replace(body=None) @@ -513,7 +534,7 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_count", None) self.assertStatsEqual("httpcompression/response_bytes", None) - def _test_compression_bomb_setting(self, compression_id): + def _test_compression_bomb_setting(self, compression_id: str) -> None: settings = {"DOWNLOAD_MAXSIZE": 1_000_000} crawler = get_crawler(Spider, settings_dict=settings) spider = crawler._create_spider("scrapytest.org") @@ -521,9 +542,12 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B + assert response.request with pytest.raises(IgnoreRequest) as exc_info: mw.process_response(response.request, response) - assert exc_info.value.__cause__.decompressed_size < 1_100_000 + cause = exc_info.value.__cause__ + assert isinstance(cause, _DecompressionMaxSizeExceeded) + assert cause.decompressed_size < 1_100_000 def test_compression_bomb_setting_br(self): _skip_if_no_br() @@ -549,6 +573,7 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse("bomb-gzip") # 11_511_612 B + assert response.request caplog.clear() with ( caplog.at_level( @@ -565,7 +590,7 @@ class TestHttpCompression: ) ] - def _test_compression_bomb_spider_attr(self, compression_id): + def _test_compression_bomb_spider_attr(self, compression_id: str) -> None: class DownloadMaxSizeSpider(Spider): download_maxsize = 1_000_000 @@ -575,9 +600,12 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") + assert response.request with pytest.raises(IgnoreRequest) as exc_info: mw.process_response(response.request, response) - assert exc_info.value.__cause__.decompressed_size < 1_100_000 + cause = exc_info.value.__cause__ + assert isinstance(cause, _DecompressionMaxSizeExceeded) + assert cause.decompressed_size < 1_100_000 @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_compression_bomb_spider_attr_br(self): @@ -599,7 +627,7 @@ class TestHttpCompression: self._test_compression_bomb_spider_attr("zstd") - def _test_compression_bomb_request_meta(self, compression_id): + def _test_compression_bomb_request_meta(self, compression_id: str) -> None: crawler = get_crawler(Spider) spider = crawler._create_spider("scrapytest.org") mw = HttpCompressionMiddleware.from_crawler(crawler) @@ -607,9 +635,12 @@ class TestHttpCompression: response = self._getresponse(f"bomb-{compression_id}") response.meta["download_maxsize"] = 1_000_000 + assert response.request with pytest.raises(IgnoreRequest) as exc_info: mw.process_response(response.request, response) - assert exc_info.value.__cause__.decompressed_size < 1_100_000 + cause = exc_info.value.__cause__ + assert isinstance(cause, _DecompressionMaxSizeExceeded) + assert cause.decompressed_size < 1_100_000 def test_compression_bomb_request_meta_br(self): _skip_if_no_br() @@ -789,7 +820,7 @@ class TestHttpCompression: self._test_download_warnsize_request_meta(caplog, "zstd") - def _get_truncated_response(self, compression_id): + def _get_truncated_response(self, compression_id: str) -> Response: crawler = get_crawler(Spider) spider = crawler._create_spider("scrapytest.org") mw = HttpCompressionMiddleware.from_crawler(crawler) @@ -797,7 +828,10 @@ class TestHttpCompression: response = self._getresponse(compression_id) truncated_body = response.body[: len(response.body) // 2] response = response.replace(body=truncated_body) - return mw.process_response(response.request, response) + assert response.request + new_response = mw.process_response(response.request, response) + assert isinstance(new_response, Response) + return new_response def test_process_truncated_response_br(self): _skip_if_no_br() diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 7ed848764..54d4601a7 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -14,7 +14,8 @@ class TestHttpProxyMiddleware: self._oldenv = os.environ.copy() def teardown_method(self): - os.environ = self._oldenv + os.environ.clear() + os.environ.update(self._oldenv) def test_not_enabled(self): crawler = get_crawler(Spider, {"HTTPPROXY_ENABLED": False}) @@ -22,7 +23,8 @@ class TestHttpProxyMiddleware: HttpProxyMiddleware.from_crawler(crawler) def test_no_environment_proxies(self): - os.environ = {"dummy_proxy": "reset_env_and_do_not_raise"} + os.environ.clear() + os.environ["dummy_proxy"] = "reset_env_and_do_not_raise" mw = HttpProxyMiddleware() for url in ("http://e.com", "https://e.com", "file:///tmp/a"): diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index 78efb0191..cb17c2553 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -1,4 +1,5 @@ import re +from typing import Any import pytest @@ -53,7 +54,7 @@ def test_process_request_dont_filter(value, filtered): crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) - kwargs = {} + kwargs: dict[str, Any] = {} if value is not UNSET: kwargs["dont_filter"] = value request = Request("https://b.example", **kwargs) @@ -82,7 +83,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) - kwargs = {"meta": {}} + kwargs: dict[str, Any] = {"meta": {}} if allow_offsite is not UNSET: kwargs["meta"]["allow_offsite"] = allow_offsite if dont_filter is not UNSET: @@ -105,7 +106,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): ) def test_process_request_no_allowed_domains(value): crawler = get_crawler(Spider) - kwargs = {} + kwargs: dict[str, Any] = {} if value is not UNSET: kwargs["allowed_domains"] = value crawler.spider = crawler._create_spider(name="a", **kwargs) @@ -152,7 +153,7 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): mw.spider_opened(crawler.spider) request = Request(url) if allowed: - assert mw.request_scheduled(request, crawler.spider) is None + mw.request_scheduled(request, crawler.spider) else: with pytest.raises(IgnoreRequest): mw.request_scheduled(request, crawler.spider) @@ -172,7 +173,7 @@ def test_request_scheduled_dont_filter(value, filtered): crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) - kwargs = {} + kwargs: dict[str, Any] = {} if value is not UNSET: kwargs["dont_filter"] = value request = Request("https://b.example", **kwargs) @@ -180,7 +181,7 @@ def test_request_scheduled_dont_filter(value, filtered): with pytest.raises(IgnoreRequest): mw.request_scheduled(request, crawler.spider) else: - assert mw.request_scheduled(request, crawler.spider) is None + mw.request_scheduled(request, crawler.spider) @pytest.mark.parametrize( @@ -193,14 +194,14 @@ def test_request_scheduled_dont_filter(value, filtered): ) def test_request_scheduled_no_allowed_domains(value): crawler = get_crawler(Spider) - kwargs = {} + kwargs: dict[str, Any] = {} if value is not UNSET: kwargs["allowed_domains"] = value crawler.spider = crawler._create_spider(name="a", **kwargs) mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) request = Request("https://example.com") - assert mw.request_scheduled(request, crawler.spider) is None + mw.request_scheduled(request, crawler.spider) def test_request_scheduled_invalid_domains(): @@ -210,7 +211,7 @@ def test_request_scheduled_invalid_domains(): mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) request = Request("https://a.example") - assert mw.request_scheduled(request, crawler.spider) is None + mw.request_scheduled(request, crawler.spider) for letter in ("b", "c"): request = Request(f"https://{letter}.example") with pytest.raises(IgnoreRequest): @@ -227,6 +228,7 @@ def test_repeated_offsite_domain(): with pytest.raises(IgnoreRequest): mw.process_request(req1) assert "other.org" in mw.domains_seen + assert crawler.stats assert crawler.stats.get_value("offsite/domains") == 1 assert crawler.stats.get_value("offsite/filtered") == 1 with pytest.raises(IgnoreRequest): diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index ef2774a93..de97aaadb 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -309,7 +309,7 @@ class TestRedirectMiddleware(TestRedirectBase): url = "http://www.example.com/301" url2 = "http://www.example.com/redirected" - def _test_passthrough(req): + def _test_passthrough(req: Request) -> None: rsp = Response(url, headers={"Location": url2}, status=301, request=req) r = self.mw.process_response(req, rsp) assert r is rsp @@ -404,15 +404,17 @@ def test_response_referrer_policy(policy, source_url, target_url, expected_refer status=301, headers={"Location": target_url, **extra_headers}, ) - source_request = redirect_mw.process_response(source_request, response_redirect) - assert isinstance(source_request, Request) + target_request = redirect_mw.process_response(source_request, response_redirect) + assert isinstance(target_request, Request) - assert source_request.headers.get("Referer") == expected_referrer + assert target_request.headers.get("Referer") == expected_referrer def test_no_warning_when_referer_middleware_present(caplog): crawler = get_crawler() - crawler.get_spider_middleware = MagicMock(return_value=MagicMock()) + crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign] + return_value=MagicMock() + ) mw = build_from_crawler(RedirectMiddleware, crawler) caplog.clear() with caplog.at_level(logging.WARNING): @@ -426,7 +428,9 @@ def test_no_warning_when_referer_middleware_present(caplog): def test_warning_redirect_middleware(caplog): crawler = get_crawler() - crawler.get_spider_middleware = MagicMock(return_value=None) + crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign] + return_value=None + ) mw = build_from_crawler(RedirectMiddleware, crawler) with caplog.at_level(logging.WARNING): mw._engine_started() @@ -449,7 +453,9 @@ def test_warning_subclass(caplog): pass crawler = get_crawler() - crawler.get_spider_middleware = MagicMock(return_value=None) + crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign] + return_value=None + ) mw = build_from_crawler(MyRedirectMiddleware, crawler) with caplog.at_level(logging.WARNING): mw._engine_started() diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py index aeae759a0..83dc6825f 100644 --- a/tests/test_downloadermiddleware_redirect_metarefresh.py +++ b/tests/test_downloadermiddleware_redirect_metarefresh.py @@ -21,7 +21,7 @@ from tests.utils.redirect import ( ) -def meta_refresh_body(url, interval=5): +def meta_refresh_body(url: str, interval: int = 5) -> bytes: html = f"""""" return html.encode("utf-8") @@ -34,10 +34,14 @@ class TestMetaRefreshMiddleware(TestRedirectBase): crawler = get_crawler(Spider) self.mw = self.mwcls.from_crawler(crawler) - def _body(self, interval=5, url="http://example.org/newpage"): + def _body( + self, interval: int = 5, url: str = "http://example.org/newpage" + ) -> bytes: return meta_refresh_body(url, interval) - def get_response(self, request, location): + def get_response( + self, request: Request, location: str, status: int = 302 + ) -> Response: return HtmlResponse(request.url, body=self._body(url=location)) def test_meta_refresh(self): @@ -75,7 +79,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase): assert "Content-Length" not in req2.headers, ( "Content-Length header must not be present in redirected request" ) - assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" + assert not req2.body, f"Redirected body must be empty, not {req2.body!r}" def test_ignore_tags_default(self): req = Request(url="http://example.org") @@ -142,7 +146,9 @@ def test_meta_refresh_schemes(url, location, target): def test_warning_meta_refresh_middleware(caplog): crawler = get_crawler() - crawler.get_spider_middleware = MagicMock(return_value=None) + crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign] + return_value=None + ) mw = build_from_crawler(MetaRefreshMiddleware, crawler) with caplog.at_level(logging.WARNING): mw._engine_started() diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 410427b84..ab52590c7 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -31,6 +31,7 @@ class TestRetry: req = Request("http://www.scrapytest.org/503") rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) assert req2.priority < req.priority def test_404(self): @@ -53,9 +54,9 @@ class TestRetry: rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) # first retry - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 1 + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.meta["retry_times"] == 1 def test_dont_retry_exc(self): req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True}) @@ -68,18 +69,19 @@ class TestRetry: rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) # first retry - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 1 + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.meta["retry_times"] == 1 # second retry - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 2 + req3 = self.mw.process_response(req2, rsp) + assert isinstance(req3, Request) + assert req3.meta["retry_times"] == 2 # discard it - assert self.mw.process_response(req, rsp) is rsp + assert self.mw.process_response(req3, rsp) is rsp + assert self.crawler.stats assert self.crawler.stats.get_value("retry/max_reached") == 1 assert ( self.crawler.stats.get_value("retry/reason_count/503 Service Unavailable") @@ -131,6 +133,7 @@ class TestRetry: self._test_retry_exception(req, exc("foo")) stats = self.crawler.stats + assert stats assert stats.get_value("retry/max_reached") == len(exceptions) assert stats.get_value("retry/count") == len(exceptions) * 2 assert ( @@ -149,29 +152,30 @@ class TestRetry: req = Request(f"http://www.scrapytest.org/{exc.__name__}") self._test_retry_exception(req, exc("foo"), mw) - def _test_retry_exception(self, req, exception, mw=None): + def _test_retry_exception( + self, req: Request, exception: Exception, mw: RetryMiddleware | None = None + ) -> None: if mw is None: mw = self.mw # first retry - req = mw.process_exception(req, exception) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 1 + req2 = mw.process_exception(req, exception) + assert isinstance(req2, Request) + assert req2.meta["retry_times"] == 1 # second retry - req = mw.process_exception(req, exception) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 2 + req3 = mw.process_exception(req2, exception) + assert isinstance(req3, Request) + assert req3.meta["retry_times"] == 2 # discard it - req = mw.process_exception(req, exception) - assert req is None + assert mw.process_exception(req3, exception) is None class TestMaxRetryTimes: invalid_url = "http://www.scrapytest.org/invalid_url" - def get_middleware(self, settings=None): + def get_middleware(self, settings: dict[str, Any] | None = None) -> RetryMiddleware: crawler = get_crawler(DefaultSpider, settings or {}) crawler.spider = crawler._create_spider() return RetryMiddleware.from_crawler(crawler) @@ -275,20 +279,18 @@ class TestMaxRetryTimes: def _test_retry( self, - req, - exception, - max_retry_times, - middleware=None, - ): - middleware = middleware or self.mw - + req: Request, + exception: Exception, + max_retry_times: int, + middleware: RetryMiddleware, + ) -> None: for _ in range(max_retry_times): - req = middleware.process_exception(req, exception) - assert isinstance(req, Request) + result = middleware.process_exception(req, exception) + assert isinstance(result, Request) + req = result # discard it - req = middleware.process_exception(req, exception) - assert req is None + assert middleware.process_exception(req, exception) is None class TestGetRetryRequest: @@ -428,7 +430,7 @@ class TestGetRetryRequest: def test_no_spider(self): request = Request("https://example.com") with pytest.raises(TypeError): - get_retry_request(request) # pylint: disable=missing-kwoa + get_retry_request(request) # type: ignore[call-arg] # pylint: disable=missing-kwoa def test_max_retry_times_setting(self): max_retry_times = 0 @@ -471,6 +473,7 @@ class TestGetRetryRequest: request, spider=spider, ) + assert new_request assert new_request.priority == priority_adjust def test_priority_adjust_argument(self): @@ -482,6 +485,7 @@ class TestGetRetryRequest: spider=spider, priority_adjust=priority_adjust, ) + assert new_request assert new_request.priority == priority_adjust def test_log_extra_retry_success(self, caplog: pytest.LogCaptureFixture) -> None: @@ -732,6 +736,7 @@ class TestGetRetryRequest: reason=expected_reason, stats_base_key=stats_key, ) + assert spider.crawler.stats for stat in ( f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}", diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index f82041a62..793a2b5be 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,13 +1,13 @@ from __future__ import annotations import asyncio -from typing import TYPE_CHECKING from unittest import mock import pytest from twisted.internet.defer import Deferred, DeferredList from twisted.python import failure +from scrapy import signals from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware from scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse @@ -18,15 +18,13 @@ from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from tests.utils.decorators import coroutine_test from tests.utils.robotstxt import rerp_available -if TYPE_CHECKING: - from scrapy.crawler import Crawler - class TestRobotsTxtMiddleware: def setup_method(self) -> None: self.crawler: mock.MagicMock = mock.MagicMock() self.crawler.settings = Settings() self.crawler.engine.download_async = mock.AsyncMock() + self.crawler.signals.send_catch_log_async = mock.AsyncMock(return_value=[]) def teardown_method(self): del self.crawler @@ -37,7 +35,7 @@ class TestRobotsTxtMiddleware: with pytest.raises(NotConfigured): RobotsTxtMiddleware(self.crawler) - def _get_successful_crawler(self) -> Crawler: + def _get_successful_crawler(self) -> mock.MagicMock: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) ROBOTS = """ @@ -52,8 +50,8 @@ Disallow: /some/randome/page.html """.encode() response = TextResponse("http://site.local/robots.txt", body=ROBOTS) - async def return_response(request): - deferred = Deferred() + async def return_response(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.callback, response) return await maybe_deferred_to_future(deferred) @@ -74,6 +72,21 @@ Disallow: /some/randome/page.html Request("http://site.local/wiki/Käyttäjä:"), middleware ) + @coroutine_test + async def test_robotstxt_emits_robots_parsed_signal(self): + crawler = self._get_successful_crawler() + middleware = RobotsTxtMiddleware(crawler) + request = Request("http://site.local/allowed") + await self.assertNotIgnored(request, middleware) + calls = [ + kwargs + for _, kwargs in crawler.signals.send_catch_log_async.call_args_list + if kwargs.get("signal") is signals.robots_parsed + ] + assert len(calls) == 1 + assert calls[0]["request"] is request + assert calls[0]["robotparser"] is not None + @coroutine_test async def test_robotstxt_multiple_reqs(self) -> None: middleware = RobotsTxtMiddleware(self._get_successful_crawler()) @@ -113,15 +126,15 @@ Disallow: /some/randome/page.html Request("http://site.local/static/", meta=meta), middleware ) - def _get_garbage_crawler(self) -> Crawler: + def _get_garbage_crawler(self) -> mock.MagicMock: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) response = Response( "http://site.local/robots.txt", body=b"GIF89a\xd3\x00\xfe\x00\xa2" ) - async def return_response(request): - deferred = Deferred() + async def return_response(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.callback, response) return await maybe_deferred_to_future(deferred) @@ -137,13 +150,13 @@ Disallow: /some/randome/page.html await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware) await self.assertNotIgnored(Request("http://site.local/static/"), middleware) - def _get_emptybody_crawler(self) -> Crawler: + def _get_emptybody_crawler(self) -> mock.MagicMock: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) response = Response("http://site.local/robots.txt") - async def return_response(request): - deferred = Deferred() + async def return_response(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.callback, response) return await maybe_deferred_to_future(deferred) @@ -163,8 +176,8 @@ Disallow: /some/randome/page.html self.crawler.settings.set("ROBOTSTXT_OBEY", True) err = CannotResolveHostError("Robotstxt address not found") - async def return_failure(request): - deferred = Deferred() + async def return_failure(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.errback, failure.Failure(err)) return await maybe_deferred_to_future(deferred) @@ -191,8 +204,8 @@ Disallow: /some/randome/page.html async def test_ignore_robotstxt_request(self): self.crawler.settings.set("ROBOTSTXT_OBEY", True) - async def ignore_request(request): - deferred = Deferred() + async def ignore_request(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.errback, failure.Failure(IgnoreRequest())) return await maybe_deferred_to_future(deferred) @@ -219,7 +232,7 @@ Disallow: /some/randome/page.html @coroutine_test async def test_robotstxt_local_file(self): middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) - middleware.process_request_2 = mock.MagicMock() + middleware.process_request_2 = mock.MagicMock() # type: ignore[method-assign] await middleware.process_request(Request("data:text/plain,Hello World data")) assert not middleware.process_request_2.called diff --git a/tests/test_engine_download.py b/tests/test_engine_download.py index 962808d96..09b998f6a 100644 --- a/tests/test_engine_download.py +++ b/tests/test_engine_download.py @@ -116,6 +116,18 @@ class TestEngineDownloadAsync: engine._slot.add_request.assert_called_once_with(request) engine._slot.remove_request.assert_called_once_with(request) + @coroutine_test + async def test_download_async_fetch_needs_spider(self, engine): + engine._downloader_fetch_needs_spider = True + request = Request("http://example.com") + response = Response("http://example.com", body=b"test body") + engine.spider = Mock() + engine.downloader.fetch.return_value = defer.succeed(response) + + result = await self._download(engine, request) + assert result == response + engine.downloader.fetch.assert_called_once_with(request, engine.spider) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestEngineDownload(TestEngineDownloadAsync): diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index c15c396d3..14ec3d184 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -6,10 +6,9 @@ from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals from scrapy.core.scheduler import BaseScheduler -from scrapy.utils.asyncio import call_later +from scrapy.utils.asyncio import call_later, sleep from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer -from tests.utils import async_sleep from tests.utils.decorators import coroutine_test if TYPE_CHECKING: @@ -65,23 +64,23 @@ class TestMain: async def start(self): yield Request("data:,a") - await async_sleep(seconds) + await sleep(seconds) self.crawler.engine._slot.scheduler.pause() self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b")) # During this time, the scheduler reports having requests but # returns None. - await async_sleep(seconds) + await sleep(seconds) self.crawler.engine._slot.scheduler.unpause() # The scheduler request is processed. - await async_sleep(seconds) + await sleep(seconds) yield Request("data:,c") - await async_sleep(seconds) + await sleep(seconds) self.crawler.engine._slot.scheduler.pause() self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d")) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 3b997767e..b857728ba 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -118,6 +118,18 @@ class TestBaseItemExporter(ABC): ie = self._get_exporter(fields_to_export={"name": "名稱"}) assert list(ie._get_serialized_fields(self.i)) == [("名稱", "John\xa3")] + def test_field_order(self): + item = self.item_class(age="22", name="John\xa3") + ie = self._get_exporter() + assert [name for name, _ in ie._get_serialized_fields(item)] == ["name", "age"] + + def test_field_order_dict_item(self): + ie = self._get_exporter() + assert [name for name, _ in ie._get_serialized_fields({"age": "22"})] == ["age"] + assert [ + name for name, _ in ie._get_serialized_fields({"age": "22", "name": "John"}) + ] == ["age", "name"] + def test_field_custom_serializer(self): i = self.custom_field_item_class(name="John\xa3", age="22") a = ItemAdapter(i) diff --git a/tests/test_extension_memusage.py b/tests/test_extension_memusage.py index a474725d8..76e8ca5d6 100644 --- a/tests/test_extension_memusage.py +++ b/tests/test_extension_memusage.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging import sys +from typing import TYPE_CHECKING import pytest @@ -13,8 +14,12 @@ from scrapy.extensions.memusage import MemoryUsage from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.utils import OneShotLoop +from tests.utils.cmdline import proc from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + from tests.mockserver.http import MockServer + # MemoryUsage relies on the stdlib 'resource' module (not available on Windows) pytestmark = pytest.mark.skipif( sys.platform.startswith("win"), @@ -25,6 +30,14 @@ pytestmark = pytest.mark.skipif( MB = 1024 * 1024 +class TwoShotLoop(OneShotLoop): + """Like :class:`OneShotLoop`, but runs the check twice.""" + + def start(self, interval: float, now: bool = True) -> None: + super().start(interval, now=now) + self.func() + + class _LoopSpider(Spider): name = "loop-data-spider" @@ -50,6 +63,49 @@ def test_memusage_disabled() -> None: MemoryUsage.from_crawler(get_crawler(settings_dict=settings)) +def test_memusage_limit_stops_crawler_without_spider(mockserver: MockServer) -> None: + # The Scrapy shell starts the engine without opening a spider, so the + # whole crawler is stopped instead of a spider being closed. + _, out, err = proc( + "shell", + mockserver.url("/text"), + "-c", + "response.status", + "--set", + "MEMUSAGE_LIMIT_MB=1", + ) + assert "Memory usage exceeded 1MiB" in err + assert "200" in out + + +@coroutine_test +async def test_memusage_below_thresholds_logs_peak( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + settings = { + "MEMUSAGE_LIMIT_MB": 100, + "MEMUSAGE_WARNING_MB": 50, + "MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01, + "TELNETCONSOLE_ENABLED": False, + "LOG_LEVEL": "INFO", + } + + monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop) + monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda _: 25 * MB) + + crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings) + + with caplog.at_level(logging.INFO, logger="scrapy.extensions.memusage"): + await crawler.crawl_async(url="data:,", loops=1) + + assert crawler.stats + assert crawler.stats.get_value("memusage/limit_reached") is None + assert crawler.stats.get_value("memusage/warning_reached") is None + assert crawler.stats.get_value("memusage/max") == 25 * MB + assert crawler.stats.get_value("finish_reason") == "finished" + assert any("Peak memory usage is 25MiB" in r.getMessage() for r in caplog.records) + + @coroutine_test async def test_memusage_limit_closes_spider_with_reason_and_error_log( caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch @@ -92,8 +148,9 @@ async def test_memusage_warning_logs_but_allows_normal_finish( "LOG_LEVEL": "INFO", } - # Avoid background LoopingCall that can log after the test finishes. - monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop) + # Avoid background LoopingCall that can log after the test finishes; check + # twice, since the warning is only meant to be reported once. + monkeypatch.setattr(memusage_mod, "create_looping_call", TwoShotLoop) monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda self: 75 * MB) crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings) @@ -112,4 +169,7 @@ async def test_memusage_warning_logs_but_allows_normal_finish( assert crawler.stats assert crawler.stats.get_value("memusage/warning_reached") == 1 assert crawler.stats.get_value("finish_reason") == "finished" - assert any("memory usage reached" in r.getMessage().lower() for r in caplog.records) + warnings_logged = [ + r for r in caplog.records if "memory usage reached" in r.getMessage().lower() + ] + assert len(warnings_logged) == 1 diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index ffe7a0dc7..723fb6e17 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -1,8 +1,13 @@ from __future__ import annotations import datetime +import json +import logging from typing import TYPE_CHECKING, Any +import pytest + +from scrapy.exceptions import NotConfigured from scrapy.extensions.periodic_log import PeriodicLog from scrapy.utils.test import get_crawler @@ -86,6 +91,14 @@ class TestPeriodicLog: assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60}) assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60}) + def test_no_interval(self): + with pytest.raises(NotConfigured): + extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 0}) + + def test_nothing_enabled(self): + with pytest.raises(NotConfigured): + extension({"LOGSTATS_INTERVAL": 60}) + @coroutine_test async def test_log_delta(self): def emulate( @@ -212,3 +225,26 @@ class TestPeriodicLog: {"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}}, lambda k, v: "downloader/" in k and "bytes" not in k, ) + + @coroutine_test + async def test_log_timing(self, caplog: pytest.LogCaptureFixture) -> None: + settings = { + "EXTENSIONS": {"scrapy.extensions.periodic_log.PeriodicLog": 0}, + "PERIODIC_LOG_TIMING_ENABLED": True, + "LOGSTATS_INTERVAL": 30, + } + crawler = get_crawler(MetaSpider, settings) + with caplog.at_level(logging.INFO, logger="scrapy.extensions.periodic_log"): + await crawler.crawl_async() + + records = [ + r for r in caplog.records if r.name == "scrapy.extensions.periodic_log" + ] + assert records, "PeriodicLog logged nothing" + # Only the timing section is enabled, and it is logged on spider close. + data = json.loads(records[-1].getMessage()) + assert list(data) == ["time"] + assert data["time"]["log_interval"] == 30 + assert data["time"]["log_interval_real"] >= 0 + assert data["time"]["elapsed"] >= 0 + assert data["time"]["start_time"] <= data["time"]["utcnow"] diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 20c801558..fca0e3153 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -7,7 +7,8 @@ import pytest from twisted.conch.telnet import ITelnetProtocol from twisted.cred import credentials -from scrapy.extensions.telnet import TelnetConsole +from scrapy import Spider +from scrapy.extensions.telnet import TelnetConsole, update_telnet_vars from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler from tests.utils.decorators import coroutine_test @@ -16,16 +17,20 @@ if TYPE_CHECKING: from collections.abc import Generator from scrapy.crawler import Crawler + from scrapy.http import Response pytestmark = pytest.mark.requires_reactor # TelnetConsole requires a reactor -def _get_crawler(settings_dict: dict[str, Any] | None = None) -> Crawler: +def _get_crawler( + spidercls: type[Spider] | None = None, + settings_dict: dict[str, Any] | None = None, +) -> Crawler: settings = { "TELNETCONSOLE_ENABLED": True, **(settings_dict or {}), } - return get_crawler(settings_dict=settings) + return get_crawler(spidercls, settings_dict=settings) @contextmanager @@ -84,3 +89,47 @@ def test_invalid_reversed_portrange() -> None: console = TelnetConsole(_get_crawler(settings_dict=settings)) with pytest.raises(ValueError, match=r"invalid portrange: \[2, 1\]"): console.start_listening() + + +@coroutine_test +async def test_telnet_vars() -> None: + """Log into the console of a running crawl, which is when the telnet + variables are built.""" + received: list[dict[str, Any]] = [] + + def on_update_telnet_vars(telnet_vars: dict[str, Any]) -> None: + received.append(telnet_vars) + + class TelnetSpider(Spider): + name = "telnet" + start_urls = ["data:,"] + + async def parse(self, response: Response) -> None: + assert self.crawler.extensions + console = next( + ext + for ext in self.crawler.extensions.middlewares + if isinstance(ext, TelnetConsole) + ) + creds = credentials.UsernamePassword( + console.username.encode("utf8"), console.password.encode("utf8") + ) + portal = console.protocol().protocolArgs[0] + await maybe_deferred_to_future(portal.login(creds, None, ITelnetProtocol)) + + crawler = _get_crawler(TelnetSpider) + crawler.signals.connect(on_update_telnet_vars, signal=update_telnet_vars) + await crawler.crawl_async() + + assert len(received) == 1 + telnet_vars = received[0] + assert telnet_vars["crawler"] is crawler + assert telnet_vars["engine"] is crawler.engine + assert telnet_vars["spider"] is crawler.spider + assert telnet_vars["extensions"] is crawler.extensions + assert telnet_vars["stats"] is crawler.stats + assert telnet_vars["settings"] is crawler.settings + assert callable(telnet_vars["est"]) + assert callable(telnet_vars["p"]) + assert callable(telnet_vars["prefs"]) + assert "telnetconsole.html" in telnet_vars["help"] diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index 4874f284a..2c718d95f 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -3,7 +3,7 @@ from unittest.mock import Mock import pytest -from scrapy import Request, Spider +from scrapy import Request from scrapy.exceptions import NotConfigured from scrapy.extensions.throttle import AutoThrottle from scrapy.http.response import Response @@ -25,6 +25,13 @@ def get_crawler(settings=None, spidercls=None): return _get_crawler(settings_dict=settings, spidercls=spidercls) +def _mock_downloader(crawler): + """Give *crawler* a mock engine, whose downloader AutoThrottle reads.""" + crawler.engine = Mock() + crawler.engine.downloader.slots = {} + return crawler.engine.downloader + + @pytest.mark.parametrize( ("value", "expected"), [ @@ -60,29 +67,21 @@ def test_target_concurrency_invalid(value): @pytest.mark.parametrize( - ("spider", "setting", "expected"), + ("setting", "expected"), [ - (UNSET, UNSET, DOWNLOAD_DELAY), - (1.0, UNSET, 1.0), - (UNSET, 1.0, 1.0), - (1.0, 2.0, 1.0), - (3.0, 2.0, 3.0), + (UNSET, DOWNLOAD_DELAY), + (1.0, 1.0), ], ) -def test_mindelay_definition(spider, setting, expected): +def test_mindelay_definition(setting, expected): settings = {} if setting is not UNSET: settings["DOWNLOAD_DELAY"] = setting - class _TestSpider(Spider): - name = "test" - - if spider is not UNSET: - _TestSpider.download_delay = spider - - crawler = get_crawler(settings, _TestSpider) + crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - at._spider_opened(_TestSpider()) + _mock_downloader(crawler) + at._spider_opened(DefaultSpider()) assert at.mindelay == expected @@ -99,58 +98,43 @@ def test_maxdelay_definition(value, expected): settings["AUTOTHROTTLE_MAX_DELAY"] = value crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) + _mock_downloader(crawler) at._spider_opened(DefaultSpider()) assert at.maxdelay == expected @pytest.mark.parametrize( - ("min_spider", "min_setting", "start_setting", "expected"), + ("min_setting", "start_setting", "expected"), [ - (UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), - (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), - (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), - (UNSET, AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY), - (UNSET, AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), - (UNSET, UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0), - (UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 1.0), - ( - AUTOTHROTTLE_START_DELAY + 1.0, - AUTOTHROTTLE_START_DELAY + 2.0, - UNSET, - AUTOTHROTTLE_START_DELAY + 1.0, - ), + (UNSET, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), + (UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0), + (UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 1.0), ( AUTOTHROTTLE_START_DELAY + 2.0, - UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 2.0, ), ( AUTOTHROTTLE_START_DELAY + 1.0, - UNSET, AUTOTHROTTLE_START_DELAY + 2.0, AUTOTHROTTLE_START_DELAY + 2.0, ), ], ) -def test_startdelay_definition(min_spider, min_setting, start_setting, expected): +def test_startdelay_definition(min_setting, start_setting, expected): settings = {} if min_setting is not UNSET: settings["DOWNLOAD_DELAY"] = min_setting if start_setting is not UNSET: settings["AUTOTHROTTLE_START_DELAY"] = start_setting - class _TestSpider(Spider): - name = "test" - - if min_spider is not UNSET: - _TestSpider.download_delay = min_spider - - crawler = get_crawler(settings, _TestSpider) + crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - spider = _TestSpider() - at._spider_opened(spider) - assert spider.download_delay == expected + downloader = _mock_downloader(crawler) + at._spider_opened(DefaultSpider()) + assert downloader._delay == expected @pytest.mark.parametrize( @@ -174,15 +158,13 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) def test_skipped(meta, slot): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) request = Request("https://example.com", meta=meta) - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} if slot is not None: - crawler.engine.downloader.slots[slot] = object() + downloader.slots[slot] = object() at._adjust_delay = None # Raise exception if called. at._response_downloaded(None, request, spider) @@ -204,18 +186,16 @@ def test_adjustment(download_latency, target_concurrency, slot_delay, expected): settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) response = Response(request.url) - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} slot = Mock() slot.delay = slot_delay - crawler.engine.downloader.slots["foo"] = slot + downloader.slots["foo"] = slot at._response_downloaded(response, request, spider) @@ -240,18 +220,16 @@ def test_adjustment_limits(mindelay, maxdelay, expected): } crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) response = Response(request.url) - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} slot = Mock() slot.delay = slot_delay - crawler.engine.downloader.slots["foo"] = slot + downloader.slots["foo"] = slot at._response_downloaded(response, request, spider) @@ -272,18 +250,16 @@ def test_adjustment_bad_response( settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) response = Response(request.url, status=400) - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} slot = Mock() slot.delay = slot_delay - crawler.engine.downloader.slots["foo"] = slot + downloader.slots["foo"] = slot at._response_downloaded(response, request, spider) @@ -294,19 +270,17 @@ def test_debug(caplog): settings = {"AUTOTHROTTLE_DEBUG": True} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": 1.0, "download_slot": "foo"} request = Request("https://example.com", meta=meta) response = Response(request.url, body=b"foo") - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} slot = Mock() slot.delay = 2.0 slot.transferring = (None, None) - crawler.engine.downloader.slots["foo"] = slot + downloader.slots["foo"] = slot caplog.clear() with caplog.at_level(INFO): @@ -324,19 +298,17 @@ def test_debug(caplog): def test_debug_disabled(caplog): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": 1.0, "download_slot": "foo"} request = Request("https://example.com", meta=meta) response = Response(request.url, body=b"foo") - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} slot = Mock() slot.delay = 2.0 slot.transferring = (None, None) - crawler.engine.downloader.slots["foo"] = slot + downloader.slots["foo"] = slot caplog.clear() with caplog.at_level(INFO): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 92c414bef..40a763efd 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -24,6 +24,7 @@ from scrapy.extensions.feedexport import ( FeedExporter, FeedSlot, FileFeedStorage, + ItemFilter, apply_uri_params, ) from scrapy.utils.python import to_unicode @@ -675,14 +676,14 @@ class TestFeedExport(TestFeedExportBase): formats = { "csv": b"foo,egg,baz\r\nbar1,spam1,\r\n", - "json": b'[\n{"hello": "world2", "foo": "bar2"}\n]', + "json": b'[\n{"foo": "bar2", "hello": "world2"}\n]', "jsonlines": ( - b'{"foo": "bar1", "egg": "spam1"}\n{"hello": "world2", "foo": "bar2"}\n' + b'{"foo": "bar1", "egg": "spam1"}\n{"foo": "bar2", "hello": "world2"}\n' ), "xml": ( b'\n\n' - b"bar1spam1\n" - b"world2bar2\nworld3" + b"bar1spam1\n" + b"bar2world2\nworld3" b"spam3\n" ), } @@ -740,8 +741,8 @@ class TestFeedExport(TestFeedExportBase): "json": b'[\n{"foo": "bar1", "egg": "spam1"}\n]', "xml": ( b'\n\n' - b"bar1spam1\n" - b"world2bar2\n" + b"bar1spam1\n" + b"bar2world2\n" ), "jsonlines": b'{"foo": "bar1", "egg": "spam1"}\n', } @@ -1289,6 +1290,13 @@ class TestFeedExporterSignals: assert self.feed_exporter_closed_received +class TestItemFilter: + def test_no_feed_options(self): + item_filter = ItemFilter(None) + assert item_filter.item_classes == () + assert item_filter.accepts(MyItem({"foo": "bar"})) + + class TestFeedExportInit: def test_unsupported_storage(self): settings = { @@ -1300,6 +1308,24 @@ class TestFeedExportInit: with pytest.raises(NotConfigured): FeedExporter.from_crawler(crawler) + def test_disabled_storage(self, caplog: pytest.LogCaptureFixture): + class DisabledFeedStorage: + def __init__(self, uri, *, feed_options=None): + raise NotConfigured("not today") + + settings = { + "FEED_STORAGES": {"disabled": DisabledFeedStorage}, + "FEEDS": { + "disabled://uri": {}, + }, + } + crawler = get_crawler(settings_dict=settings) + with caplog.at_level(logging.ERROR), pytest.raises(NotConfigured): + FeedExporter.from_crawler(crawler) + assert ( + "Disabled feed storage scheme: disabled. Reason: not today" in caplog.text + ) + def test_unsupported_format(self): settings = { "FEEDS": { diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py index 80ff6229b..4b0962c43 100644 --- a/tests/test_feedexport_batch.py +++ b/tests/test_feedexport_batch.py @@ -210,6 +210,47 @@ class TestBatchDeliveries(TestFeedExportBase): header = MyItem.fields.keys() await self.assertExported(items, header, rows, settings=settings) + @coroutine_test + async def test_batch_delivered_when_full(self): + """Full batches must be finalized and delivered as soon as they are + full, instead of when the spider closes.""" + dir_path = self._random_temp_filename() + batch1_path = Path(dir_path, "1.json") + mockserver_url = self.mockserver.url("/") + batch1_contents: list[bytes | None] = [] + + class TestSpider(scrapy.Spider): + name = "testspider" + start_urls = [mockserver_url] + + def parse(self, response): + yield {"foo": "bar1"} + yield {"foo": "bar2"} + yield scrapy.Request( + mockserver_url, callback=self.parse2, dont_filter=True + ) + + def parse2(self, response): + # the first batch was full after the second item, so it must + # have been delivered by now + batch1_contents.append( + batch1_path.read_bytes() if batch1_path.exists() else None + ) + yield {"foo": "bar3"} + + settings = { + "FEEDS": { + build_url(dir_path / "%(batch_id)d.json"): {"format": "json"}, + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 2, + } + crawler = get_crawler(TestSpider, settings) + await crawler.crawl_async() + + assert batch1_contents, "the second request was not processed" + assert batch1_contents[0] is not None, "batch 1 was not stored during the crawl" + assert json.loads(batch1_contents[0]) == [{"foo": "bar1"}, {"foo": "bar2"}] + def test_wrong_path(self): """If path is without %(batch_time)s and %(batch_id) an exception must be raised""" settings = { diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py index f120ce36f..36d8586ce 100644 --- a/tests/test_feedexport_postprocess.py +++ b/tests/test_feedexport_postprocess.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any import pytest +from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.test import get_crawler from tests.utils.bases.feedexport import TestFeedExportBase from tests.utils.decorators import coroutine_test @@ -87,6 +88,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data_stream.seek(0) return data_stream.read() + def test_tell_reports_target_file_position(self): + """Exporters that wrap the file they get, e.g. through + :class:`io.TextIOWrapper`, need it to report a position.""" + file = BytesIO() + manager = PostProcessingManager([self.MyPlugin1], file, {}) + assert manager.tell() == 0 + manager.write(b"foo") + assert manager.tell() == file.tell() == 3 + @coroutine_test async def test_gzip_plugin(self): filename = self._named_tempfile("gzip_file") diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py index 66488540f..4d28872b7 100644 --- a/tests/test_feedexport_storages.py +++ b/tests/test_feedexport_storages.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging import os import string +import sys import tempfile from io import BytesIO from pathlib import Path @@ -14,6 +15,7 @@ import pytest from w3lib.url import path_to_file_uri import scrapy +from scrapy.exceptions import NotConfigured from scrapy.extensions.feedexport import ( BlockingFeedStorage, FileFeedStorage, @@ -166,6 +168,12 @@ class TestFTPFeedStorage: st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {}) assert st.password == string.punctuation + def test_uri_without_hostname(self): + with pytest.raises( + ValueError, match="Got a storage URI without a hostname: ftp:///some_path" + ): + FTPFeedStorage("ftp:///some_path") + class MyBlockingFeedStorage(BlockingFeedStorage): def _store_in_thread(self, file: IO[bytes]) -> None: @@ -205,6 +213,13 @@ class TestBlockingFeedStorage: b.open(spider=spider) +def test_s3_without_boto3(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "boto3", None) + monkeypatch.setitem(sys.modules, "boto3.session", None) + with pytest.raises(NotConfigured, match="missing boto3 library"): + S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") + + @pytest.mark.requires_boto3 class TestS3FeedStorage: def test_parse_credentials(self): @@ -381,6 +396,41 @@ class TestS3FeedStorage: assert storage.region_name == region_name assert storage.s3_client._client_config.region_name == region_name + def test_init_without_max_pool_connections(self) -> None: + storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") + assert storage.max_pool_connections is None + config: Any = storage.s3_client.meta.config + assert config.max_pool_connections == 10 + + def test_init_with_max_pool_connections(self) -> None: + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + max_pool_connections=30, + ) + assert storage.max_pool_connections == 30 + config: Any = storage.s3_client.meta.config + assert config.max_pool_connections == 30 + + @pytest.mark.parametrize( + ("settings", "expected"), + [ + ({}, 10), + ({"REACTOR_THREADPOOL_MAXSIZE": 20}, 20), + ({"AWS_MAX_POOL_CONNECTIONS": 30}, 30), + ({"AWS_MAX_POOL_CONNECTIONS": 30, "REACTOR_THREADPOOL_MAXSIZE": 20}, 30), + ], + ) + def test_from_crawler_max_pool_connections( + self, settings: dict[str, Any], expected: int + ) -> None: + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") + assert storage.max_pool_connections == expected + config: Any = storage.s3_client.meta.config + assert config.max_pool_connections == expected + @coroutine_test async def test_store_without_acl(self): storage = S3FeedStorage( diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 3c1347fd3..b8586d1ca 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -30,7 +30,7 @@ from scrapy.utils.defer import ( deferred_from_coro, maybe_deferred_to_future, ) -from tests.mockserver.http_resources import LeafResource, Status +from tests.mockserver.http_resources import LeafResource, Status, put_child from tests.mockserver.utils import ssl_context_factory if TYPE_CHECKING: @@ -199,18 +199,18 @@ class TestHttps2ClientProtocol: @pytest.fixture def site(self, tmp_path): r = File(str(tmp_path)) - r.putChild(b"get-data-html-small", GetDataHtmlSmall()) - r.putChild(b"get-data-html-large", GetDataHtmlLarge()) + put_child(r, b"get-data-html-small", GetDataHtmlSmall()) + put_child(r, b"get-data-html-large", GetDataHtmlLarge()) - r.putChild(b"post-data-json-small", PostDataJsonSmall()) - r.putChild(b"post-data-json-large", PostDataJsonLarge()) + put_child(r, b"post-data-json-small", PostDataJsonSmall()) + put_child(r, b"post-data-json-large", PostDataJsonLarge()) - r.putChild(b"dataloss", Dataloss()) - r.putChild(b"no-content-length-header", NoContentLengthHeader()) - r.putChild(b"status", Status()) - r.putChild(b"query-params", QueryParams()) - r.putChild(b"timeout", TimeoutResponse()) - r.putChild(b"request-headers", RequestHeaders()) + put_child(r, b"dataloss", Dataloss()) + put_child(r, b"no-content-length-header", NoContentLengthHeader()) + put_child(r, b"status", Status()) + put_child(r, b"query-params", QueryParams()) + put_child(r, b"timeout", TimeoutResponse()) + put_child(r, b"request-headers", RequestHeaders()) return Site(r, timeout=None) @async_yield_fixture # type: ignore[untyped-decorator] diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index aff3562e3..e7ed17615 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -6,9 +6,6 @@ from scrapy.http import Headers class TestHeaders: - def assertSortedEqual(self, first, second, msg=None): - assert sorted(first) == sorted(second), msg - def test_basics(self): h = Headers({"Content-Type": "text/html", "Content-Length": 1234}) assert h["Content-Type"] @@ -39,7 +36,7 @@ class TestHeaders: assert h["X-Forwarded-For"] == b"ip2" assert h.get("X-Forwarded-For") == b"ip2" assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"] - assert h.getlist("X-Forwarded-For") is not hlist + assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap] def test_multivalue_for_one_header(self): h = Headers((("a", "b"), ("a", "c"))) @@ -49,19 +46,19 @@ class TestHeaders: def test_encode_utf8(self): h = Headers({"key": "\xa3"}, encoding="utf-8") - key, val = dict(h).popitem() + key, val = dict(h.items()).popitem() assert isinstance(key, bytes), key assert isinstance(val[0], bytes), val[0] assert val[0] == b"\xc2\xa3" def test_encode_latin1(self): h = Headers({"key": "\xa3"}, encoding="latin1") - _, val = dict(h).popitem() + _, val = dict(h.items()).popitem() assert val[0] == b"\xa3" def test_encode_multiple(self): h = Headers({"key": ["\xa3"]}, encoding="utf-8") - _, val = dict(h).popitem() + _, val = dict(h.items()).popitem() assert val[0] == b"\xc2\xa3" def test_delete_and_contains(self): @@ -75,7 +72,7 @@ class TestHeaders: h = Headers() hlist = ["ip1", "ip2"] olist = h.setdefault("X-Forwarded-For", hlist) - assert h.getlist("X-Forwarded-For") is not hlist + assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap] assert h.getlist("X-Forwarded-For") is olist h = Headers() @@ -87,16 +84,16 @@ class TestHeaders: idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]} h = Headers(idict) - assert dict(h) == { + assert dict(h.items()) == { b"Content-Type": [b"text/html"], b"X-Forwarded-For": [b"ip1", b"ip2"], } - self.assertSortedEqual(h.keys(), [b"X-Forwarded-For", b"Content-Type"]) - self.assertSortedEqual( - h.items(), - [(b"X-Forwarded-For", [b"ip1", b"ip2"]), (b"Content-Type", [b"text/html"])], - ) - self.assertSortedEqual(h.values(), [b"ip2", b"text/html"]) + assert sorted(h.keys()) == [b"Content-Type", b"X-Forwarded-For"] + assert sorted(h.items()) == [ + (b"Content-Type", [b"text/html"]), + (b"X-Forwarded-For", [b"ip1", b"ip2"]), + ] + assert set(h.values()) == {b"ip2", b"text/html"} def test_update(self): h = Headers() @@ -162,4 +159,4 @@ class TestHeaders: with pytest.raises(TypeError, match="Unsupported value type"): Headers().setdefault("foo", object()) with pytest.raises(TypeError, match="Unsupported value type"): - Headers().setlist("foo", [object()]) + Headers().setlist("foo", [object()]) # type: ignore[list-item] diff --git a/tests/test_http_request.py b/tests/test_http_request.py index e58ae8f39..b9cec93a1 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,4 +1,5 @@ import xmlrpc.client +from typing import Any import pytest @@ -17,7 +18,7 @@ class TestXmlRpcRequest(TestRequestBase): default_method = "POST" default_headers = {b"Content-Type": [b"text/xml"]} - def _test_request(self, **kwargs): + def _test_request(self, **kwargs: Any) -> None: r = self.request_class("http://scrapytest.org/rpc2", **kwargs) assert r.headers[b"Content-Type"] == b"text/xml" assert r.body == to_bytes( diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py index 5e965e8dc..cb18a0b03 100644 --- a/tests/test_http_request_form.py +++ b/tests/test_http_request_form.py @@ -2,6 +2,7 @@ from __future__ import annotations import re import warnings +from typing import TYPE_CHECKING, Any from urllib.parse import parse_qs, unquote_to_bytes import pytest @@ -12,20 +13,32 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode from tests.utils.bases.http_request import TestRequestBase +if TYPE_CHECKING: + from scrapy import Request -def _buildresponse(body, **kwargs): + +def _buildresponse(body: bytes | str, **kwargs: Any) -> HtmlResponse: kwargs.setdefault("body", body) kwargs.setdefault("url", "http://example.com") kwargs.setdefault("encoding", "utf-8") return HtmlResponse(**kwargs) -def _qs(req, encoding="utf-8", to_unicode=False): - qs = req.body if req.method == "POST" else req.url.partition("?")[2] - uqs = unquote_to_bytes(qs) - if to_unicode: - uqs = uqs.decode(encoding) - return parse_qs(uqs, True) +def _query_string(req: Request) -> bytes: + return req.body if req.method == "POST" else req.url.partition("?")[2].encode() + + +def _qs(req: Request) -> dict[bytes, list[bytes]]: + return parse_qs(unquote_to_bytes(_query_string(req)), True) + + +def _qs_unicode(req: Request, encoding: str = "utf-8") -> dict[str, list[str]]: + qs = unquote_to_bytes(_query_string(req)).decode(encoding) + return parse_qs(qs, True) + + +def _assert_query_equal(first: bytes, second: bytes) -> None: + assert sorted(to_unicode(first).split("&")) == sorted(to_unicode(second).split("&")) # FormRequest.from_response() is deprecated in favor of form2request, so the @@ -34,11 +47,6 @@ def _qs(req, encoding="utf-8", to_unicode=False): class TestFormRequest(TestRequestBase): request_class = FormRequest - def assertQueryEqual(self, first, second, msg=None): - first = to_unicode(first).split("&") - second = to_unicode(second).split("&") - assert sorted(first) == sorted(second), msg - def test_init_not_deprecated(self): # Building a request directly from form data is not deprecated. with warnings.catch_warnings(): @@ -75,20 +83,22 @@ class TestFormRequest(TestRequestBase): assert fs[b"b"] == [b"2"] assert fs.get(b"c") is None - data = {"a": "1", "b": "2"} + mapping = {"a": "1", "b": "2"} fs = _qs( - self.request_class("http://www.example.com/", method="GET", formdata=data) + self.request_class( + "http://www.example.com/", method="GET", formdata=mapping + ) ) assert fs[b"a"] == [b"1"] assert fs[b"b"] == [b"2"] def test_default_encoding_bytes(self): # using default encoding (utf-8) - data = {b"one": b"two", b"price": b"\xc2\xa3 100"} + data: dict[Any, Any] = {b"one": b"two", b"price": b"\xc2\xa3 100"} r2 = self.request_class("http://www.example.com", formdata=data) assert r2.method == "POST" assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two") + _assert_query_equal(r2.body, b"price=%C2%A3+100&one=two") assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_default_encoding_textual_data(self): @@ -97,26 +107,26 @@ class TestFormRequest(TestRequestBase): r2 = self.request_class("http://www.example.com", formdata=data) assert r2.method == "POST" assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") + _assert_query_equal(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_default_encoding_mixed_data(self): # using default encoding (utf-8) - data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} + data: dict[Any, Any] = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} r2 = self.request_class("http://www.example.com", formdata=data) assert r2.method == "POST" assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") + _assert_query_equal(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_custom_encoding_bytes(self): - data = {b"\xb5 one": b"two", b"price": b"\xa3 100"} + data: dict[Any, Any] = {b"\xb5 one": b"two", b"price": b"\xa3 100"} r2 = self.request_class( "http://www.example.com", formdata=data, encoding="latin1" ) assert r2.method == "POST" assert r2.encoding == "latin1" - self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two") + _assert_query_equal(r2.body, b"price=%A3+100&%B5+one=two") assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_custom_encoding_textual_data(self): @@ -131,7 +141,7 @@ class TestFormRequest(TestRequestBase): # using multiples values for a single key data = {"price": "\xa3 100", "colours": ["red", "blue", "green"]} r3 = self.request_class("http://www.example.com", formdata=data) - self.assertQueryEqual( + _assert_query_equal( r3.body, b"colours=red&colours=blue&colours=green&price=%C2%A3+100" ) @@ -173,7 +183,7 @@ class TestFormRequest(TestRequestBase): assert req.method == "POST" assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True) + fs = _qs_unicode(req) assert set(fs["test £"]) == {"val1", "val2"} assert set(fs["one"]) == {"two", "three"} assert fs["test2"] == ["xxx µ"] @@ -196,7 +206,7 @@ class TestFormRequest(TestRequestBase): assert req.method == "POST" assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True, encoding="latin1") + fs = _qs_unicode(req, encoding="latin1") assert set(fs["test £"]) == {"val1", "val2"} assert set(fs["one"]) == {"two", "three"} assert fs["test2"] == ["xxx µ"] @@ -218,7 +228,7 @@ class TestFormRequest(TestRequestBase): assert req.method == "POST" assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True) + fs = _qs_unicode(req) assert set(fs["test £"]) == {"val1", "val2"} assert set(fs["one"]) == {"two", "three"} assert fs["test2"] == ["xxx µ"] @@ -305,7 +315,10 @@ class TestFormRequest(TestRequestBase): """ ) - req = self.request_class.from_response(response, formdata={"two": None}) + req = self.request_class.from_response( + response, + formdata={"two": None}, # type: ignore[arg-type] + ) fs = _qs(req) assert fs[b"one"] == [b"1"] assert b"two" not in fs @@ -450,7 +463,7 @@ class TestFormRequest(TestRequestBase): req = self.request_class.from_response( response, clickdata={"name": "price in \u00a3"} ) - fs = _qs(req, to_unicode=True) + fs = _qs_unicode(req) assert fs["price in \u00a3"] def test_from_response_unicode_clickdata_latin1(self): @@ -466,7 +479,7 @@ class TestFormRequest(TestRequestBase): req = self.request_class.from_response( response, clickdata={"name": "price in \u00a5"} ) - fs = _qs(req, to_unicode=True, encoding="latin1") + fs = _qs_unicode(req, encoding="latin1") assert fs["price in \u00a5"] def test_from_response_multiple_forms_clickdata(self): @@ -737,7 +750,7 @@ class TestFormRequest(TestRequestBase): """ ) req = self.request_class.from_response(res) - fs = _qs(req, to_unicode=True) + fs = _qs_unicode(req) assert fs == {"i1": ["i1v2"], "i2": ["i2v1"], "i4": ["i4v2", "i4v3"]} def test_from_response_radio(self): @@ -1022,7 +1035,7 @@ class TestFormRequest(TestRequestBase): with pytest.raises( ValueError, match="formdata should be a dict or iterable of tuples" ): - FormRequest.from_response(response, formdata=123) + FormRequest.from_response(response, formdata=123) # type: ignore[arg-type] def test_form_response_with_custom_invalid_formdata_value_error(self): """Test that a ValueError is raised for fault-inducing iterable formdata input""" @@ -1037,7 +1050,7 @@ class TestFormRequest(TestRequestBase): with pytest.raises( ValueError, match="formdata should be a dict or iterable of tuples" ): - FormRequest.from_response(response, formdata=("a",)) + FormRequest.from_response(response, formdata=("a",)) # type: ignore[arg-type] def test_get_form_with_xpath_no_form_parent(self): """Test that _get_from raised a ValueError when an XPath selects an element diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py index 04315ad89..efa63e049 100644 --- a/tests/test_http_response_text.py +++ b/tests/test_http_response_text.py @@ -1,6 +1,7 @@ from __future__ import annotations import codecs +from typing import cast from unittest import mock import pytest @@ -14,6 +15,12 @@ from tests.utils.bases.http_response import TestResponseBase class TestTextResponse(TestResponseBase): response_class = TextResponse + def _links_response(self) -> TextResponse: + return cast("TextResponse", super()._links_response()) + + def _links_response_no_href(self) -> TextResponse: + return cast("TextResponse", super()._links_response_no_href()) + def test_follow_None_encoding(self): # unlike the base Response, TextResponse.follow() falls back to the # response encoding when encoding is None instead of raising @@ -21,7 +28,7 @@ class TestTextResponse(TestResponseBase): req = r.follow("foo", encoding=None) assert req.encoding == "cp1252" - def test_replace(self): + def test_replace(self) -> None: super().test_replace() r1 = self.response_class( "http://www.example.com", body="hello", encoding="cp852" @@ -344,7 +351,7 @@ class TestTextResponse(TestResponseBase): def test_follow_selector_list(self): resp = self._links_response() with pytest.raises(ValueError, match="SelectorList"): - resp.follow(resp.css("a")) + resp.follow(resp.css("a")) # type: ignore[arg-type] def test_follow_selector_invalid(self): resp = self._links_response() @@ -616,7 +623,7 @@ class CustomResponse(TextResponse): class TestCustomResponse(TestTextResponse): response_class = CustomResponse - def test_copy(self): + def test_copy(self) -> None: super().test_copy() r1 = self.response_class( url="https://example.org", @@ -632,7 +639,7 @@ class TestCustomResponse(TestTextResponse): assert r1.lost == "lost" assert r2.lost is None - def test_replace(self): + def test_replace(self) -> None: super().test_replace() r1 = self.response_class( url="https://example.org", diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4f6fa21a0..4e7fb118b 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -33,7 +33,7 @@ from scrapy.pipelines.files import ( GCSFilesStore, S3FilesStore, ) -from scrapy.pipelines.media import MediaPipeline, _MediaRequestFiltered +from scrapy.pipelines.media import _MediaRequestFiltered from scrapy.settings import Settings from scrapy.utils.asyncio import call_later from scrapy.utils.defer import maybe_deferred_to_future @@ -43,11 +43,7 @@ from tests.mockserver.ftp import MockFTPServer from tests.utils.decorators import coroutine_test, inline_callbacks_test from .utils.cloud import mock_google_cloud_storage -from .utils.media_pipelines import mocked_download_func - -# required by persist_file() and stat_file(), but as some stores don't use the argument -# we can pass this singleton to keep type hints correct -DUMMY_SPIDER_INFO = MediaPipeline.SpiderInfo(DefaultSpider()) +from .utils.media_pipelines import DUMMY_SPIDER_INFO, mocked_download_func def get_ftp_content_and_delete( @@ -94,16 +90,19 @@ class DeferredFSFilesStore(FSFilesStore): class TestFilesPipeline: def setup_method(self): self.tempdir = mkdtemp() - settings_dict = {"FILES_STORE": self.tempdir} - crawler = get_crawler(DefaultSpider, settings_dict=settings_dict) - crawler.spider = crawler._create_spider() - crawler.engine = MagicMock(download_async=mocked_download_func) - self.pipeline = FilesPipeline.from_crawler(crawler) - self.pipeline.open_spider() + self.pipeline = self._create_pipeline(FilesPipeline) def teardown_method(self): rmtree(self.tempdir) + def _create_pipeline(self, pipeline_cls: type[FilesPipeline]) -> FilesPipeline: + crawler = get_crawler(DefaultSpider, {"FILES_STORE": self.tempdir}) + crawler.spider = crawler._create_spider() + crawler.engine = MagicMock(download_async=mocked_download_func) + pipeline = pipeline_cls.from_crawler(crawler) + pipeline.open_spider() + return pipeline + def test_file_path_query_parameters(self): file_path = self.pipeline.file_path @@ -254,6 +253,107 @@ class TestFilesPipeline: assert result["files"][0]["checksum"] != "abc" assert result["files"][0]["status"] == "cached" + @coroutine_test + async def test_file_stat_without_last_modified(self) -> None: + """A stat result without a last modification time forces a download.""" + item_url = "http://example.com/file4.pdf" + item = _create_item_with_files(item_url) + with ( + mock.patch.object(FilesPipeline, "inc_stats", return_value=True), + mock.patch.object( + FSFilesStore, "stat_file", return_value={"checksum": "abc"} + ), + mock.patch.object( + FilesPipeline, + "get_media_requests", + return_value=[_prepare_request_object(item_url)], + ), + ): + result = await self.pipeline.process_item(item) + assert result["files"][0]["checksum"] != "abc" + assert result["files"][0]["status"] == "downloaded" + + @coroutine_test + async def test_file_empty_content(self, caplog: pytest.LogCaptureFixture) -> None: + item_url = "http://example.com/empty.pdf" + item = _create_item_with_files(item_url) + request = Request( + item_url, meta={"response": Response(item_url, status=200, body=b"")} + ) + with ( + caplog.at_level(logging.WARNING), + mock.patch.object( + FilesPipeline, "get_media_requests", return_value=[request] + ), + ): + result = await self.pipeline.process_item(item) + assert result["files"] == [] + assert "File (empty-content): Empty file from" in caplog.text + + @coroutine_test + async def test_file_downloaded_file_exception( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """A FileException from file_downloaded() is logged as a warning and + kept as is.""" + + class FailingFilesPipeline(FilesPipeline): + def file_downloaded(self, response, request, info, *, item=None): + raise FileException("boom") + + item_url = "http://example.com/file5.pdf" + item = _create_item_with_files(item_url) + pipeline = self._create_pipeline(FailingFilesPipeline) + with ( + caplog.at_level(logging.WARNING), + mock.patch.object( + FilesPipeline, + "get_media_requests", + return_value=[_prepare_request_object(item_url)], + ), + ): + result = await pipeline.process_item(item) + assert result["files"] == [] + records = [ + r for r in caplog.records if "Error processing file" in r.getMessage() + ] + assert len(records) == 1 + assert records[0].levelname == "WARNING" + assert "boom" in records[0].getMessage() + + @coroutine_test + async def test_file_downloaded_unknown_error( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Any other exception from file_downloaded() is logged as an error and + reported as a FileException.""" + + class FailingFilesPipeline(FilesPipeline): + def file_downloaded(self, response, request, info, *, item=None): + raise RuntimeError("boom") + + item_url = "http://example.com/file6.pdf" + item = _create_item_with_files(item_url) + pipeline = self._create_pipeline(FailingFilesPipeline) + with ( + caplog.at_level(logging.WARNING), + mock.patch.object( + FilesPipeline, + "get_media_requests", + return_value=[_prepare_request_object(item_url)], + ), + ): + result = await pipeline.process_item(item) + assert result["files"] == [] + records = [ + r for r in caplog.records if "Error processing file" in r.getMessage() + ] + assert len(records) == 1 + assert records[0].levelname == "ERROR" + exc_info = records[0].exc_info + assert exc_info is not None + assert exc_info[0] is RuntimeError + @coroutine_test async def test_async_store(self) -> None: """Test that async persist_file() works and is awaited.""" @@ -648,9 +748,24 @@ class TestFilesPipelineCustomSettings: request = Request("http://example.com/image01.jpg") assert pipeline.file_path(request) == Path("subdir/image01.jpg") - def test_files_store_constructor_with_pathlike_object(self, tmp_path): - fs_store = FSFilesStore(tmp_path) - assert fs_store.basedir == str(tmp_path) + +class TestFSFilesStore: + def test_constructor_with_pathlike_object(self, tmp_path: Path) -> None: + assert FSFilesStore(tmp_path).basedir == str(tmp_path) + + def test_constructor_with_uri(self, tmp_path: Path) -> None: + assert FSFilesStore(f"file://{tmp_path}").basedir == str(tmp_path) + + def test_stat_file(self, tmp_path: Path) -> None: + store = FSFilesStore(tmp_path) + store.persist_file("full/filename", BytesIO(b"data"), DUMMY_SPIDER_INFO) + stat = store.stat_file("full/filename", DUMMY_SPIDER_INFO) + assert stat["checksum"] == "8d777f385d3dfec8815d20f7496026dc" + assert stat["last_modified"] == pytest.approx(time.time(), abs=60) + + def test_stat_missing_file(self, tmp_path: Path) -> None: + store = FSFilesStore(tmp_path) + assert store.stat_file("full/filename", DUMMY_SPIDER_INFO) == {} @pytest.mark.requires_botocore @@ -695,6 +810,59 @@ class TestS3FilesStore: # The call to read does not happen with Stubber assert buffer.method_calls == [mock.call.seek(0)] + @inline_callbacks_test + def test_persist_without_headers(self): + """Without custom headers only the default ones are sent.""" + bucket = "mybucket" + key = "export.csv" + buffer = mock.MagicMock() + + store = S3FilesStore(f"s3://{bucket}/{key}") + from botocore.stub import Stubber # noqa: PLC0415 + + with Stubber(store.s3_client) as stub: + stub.add_response( + "put_object", + expected_params={ + "ACL": S3FilesStore.POLICY, + "Body": buffer, + "Bucket": bucket, + "CacheControl": S3FilesStore.HEADERS["Cache-Control"], + "Key": key, + "Metadata": {}, + }, + service_response={}, + ) + + yield store.persist_file("", buffer, info=DUMMY_SPIDER_INFO) + + stub.assert_no_pending_responses() + + def test_missing_botocore(self): + with ( + mock.patch( + "scrapy.pipelines.files.is_botocore_available", return_value=False + ), + pytest.raises(NotConfigured, match="missing botocore library"), + ): + S3FilesStore("s3://mybucket/key") + + def test_wrong_uri_scheme(self): + with pytest.raises( + ValueError, + match=re.escape( + "Incorrect URI scheme in ftp://mybucket/key, expected 's3'" + ), + ): + S3FilesStore("ftp://mybucket/key") + + def test_unsupported_header(self): + store = S3FilesStore("s3://mybucket/key") + with pytest.raises( + TypeError, match='Header "X-Custom" is not supported by botocore' + ): + store._headers_to_botocore_kwargs({"X-Custom": "value"}) + @inline_callbacks_test def test_stat(self): bucket = "mybucket" @@ -727,6 +895,33 @@ class TestS3FilesStore: stub.assert_no_pending_responses() + def test_default_max_pool_connections(self) -> None: + store = S3FilesStore("s3://mybucket/prefix/") + config: Any = store.s3_client.meta.config + assert config.max_pool_connections == 10 + + @pytest.mark.parametrize( + ("settings", "expected"), + [ + ({}, 10), + ({"REACTOR_THREADPOOL_MAXSIZE": 20}, 20), + ({"AWS_MAX_POOL_CONNECTIONS": 30}, 30), + ({"AWS_MAX_POOL_CONNECTIONS": 30, "REACTOR_THREADPOOL_MAXSIZE": 20}, 30), + ], + ) + def test_max_pool_connections( + self, monkeypatch: pytest.MonkeyPatch, settings: dict[str, Any], expected: int + ) -> None: + # restores the value that FilesPipeline.from_crawler() sets on the class + monkeypatch.setattr(S3FilesStore, "AWS_MAX_POOL_CONNECTIONS", None) + crawler = get_crawler( + settings_dict={"FILES_STORE": "s3://mybucket/prefix/", **settings} + ) + store = FilesPipeline.from_crawler(crawler).store + assert isinstance(store, S3FilesStore) + config: Any = store.s3_client.meta.config + assert config.max_pool_connections == expected + class TestGCSFilesStore: @staticmethod @@ -901,6 +1096,28 @@ class TestFTPFileStore: ) assert data == content + @inline_callbacks_test + def test_persist_active_mode(self, monkeypatch: pytest.MonkeyPatch): + data = b"active mode" + path = "full/filename" + monkeypatch.setattr(FTPFilesStore, "FTP_USERNAME", "anonymous") + monkeypatch.setattr(FTPFilesStore, "FTP_PASSWORD", "guest") + monkeypatch.setattr(FTPFilesStore, "USE_ACTIVE_MODE", True) + with MockFTPServer() as ftp_server: + store = FTPFilesStore(ftp_server.url("/")) + yield store.persist_file(path, BytesIO(data), info=DUMMY_SPIDER_INFO) + stat = yield store.stat_file(path, info=DUMMY_SPIDER_INFO) + assert stat["checksum"] == "ff1575649a39a27c13faa0d37c84bab3" + + def test_wrong_uri_scheme(self): + with pytest.raises( + ValueError, + match=re.escape( + "Incorrect URI scheme in http://example.com/, expected 'ftp'" + ), + ): + FTPFilesStore("http://example.com/") + class ItemWithFiles(Item): file_urls = Field() diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 1b73dd157..19e61579f 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -3,20 +3,26 @@ from __future__ import annotations import dataclasses import io import random +import sys from abc import ABC, abstractmethod +from pathlib import Path from shutil import rmtree from tempfile import mkdtemp +from types import SimpleNamespace from typing import Any import attr import pytest from itemadapter import ItemAdapter +from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.item import Field, Item -from scrapy.pipelines.files import GCSFilesStore, S3FilesStore +from scrapy.pipelines.files import GCSFilesStore, S3FilesStore, _md5sum from scrapy.pipelines.images import ImageException, ImagesPipeline from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test +from tests.utils.media_pipelines import DUMMY_SPIDER_INFO try: from PIL import Image @@ -40,6 +46,11 @@ class TestImagesPipeline: def teardown_method(self): rmtree(self.tempdir) + def test_missing_pillow(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "PIL", None) + with pytest.raises(NotConfigured, match="requires installing Pillow"): + ImagesPipeline(self.tempdir, crawler=get_crawler()) + def test_file_path(self): file_path = self.pipeline.file_path assert ( @@ -197,6 +208,25 @@ class TestImagesPipeline: assert path == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" assert new_im.getpixel((0, 0)) == (255, 0, 0) + @coroutine_test + async def test_image_downloaded(self) -> None: + """The image and its thumbnails are stored, and the checksum of the + full-size image is returned.""" + self.pipeline.thumbs = {"small": (20, 20)} + _, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0)) + url = "https://dev.mydeco.com/mydeco.gif" + response = Response(url=url, body=buf.getvalue()) + + checksum = await self.pipeline.image_downloaded( + response, Request(url=url), DUMMY_SPIDER_INFO + ) + + buf.seek(0) + assert checksum == _md5sum(buf) + name = "3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" + assert Path(self.tempdir, "full", name).read_bytes() == buf.getvalue() + assert Path(self.tempdir, "thumbs", "small", name).exists() + def test_convert_image(self): SIZE = (100, 100) # straight forward case: RGB and JPEG @@ -230,6 +260,24 @@ class TestImagesPipeline: assert converted.mode == "RGB" assert converted.getcolors() == [(10000, (205, 230, 255))] + def test_convert_image_legacy_resampling_filter( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Pillow older than 9.1.0 has Image.ANTIALIAS instead of + Image.Resampling.LANCZOS.""" + # Image.LANCZOS is the only spelling that exists in every supported + # Pillow version, but Pillow defines it dynamically, hence the ignore. + monkeypatch.setattr( + self.pipeline, + "_Image", + SimpleNamespace(ANTIALIAS=Image.LANCZOS), # type: ignore[attr-defined] + ) + im, buf = _create_image("JPEG", "RGB", (100, 100), (0, 127, 255)) + + thumbnail, _ = self.pipeline.convert_image(im, size=(10, 25), response_body=buf) + + assert thumbnail.size == (10, 10) + @pytest.mark.parametrize( "bad_type", [ @@ -581,7 +629,7 @@ class TestImagesPipelineCustomSettings: GCSFilesStore.POLICY = old_policy -def _create_image(format_, *a, **kw): +def _create_image(format_: str, *a: Any, **kw: Any) -> tuple[Image.Image, io.BytesIO]: buf = io.BytesIO() Image.new(*a, **kw).save(buf, format_) buf.seek(0) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 23c19e4be..ba1c18006 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -319,6 +319,42 @@ class TestMediaPipeline(TestBaseMediaPipeline): assert self.fingerprint(req1) == self.fingerprint(req2) assert new_item["results"] == [(True, {})] + @coroutine_test + async def test_failures_are_cached_across_multiple_items(self): + self.pipe.LOG_FAILED_RESULTS = False + exc = Exception("foo") + req1 = Request("http://url1", meta={"response": exc}) + new_item = await self.pipe.process_item({"requests": req1}) + assert new_item["results"][0][1].value is exc + + # rsp2 is ignored, the cached failure must be reused because request + # fingerprints are the same + req2 = Request( + req1.url, meta={"response": Response("http://donot.download.me")} + ) + new_item = await self.pipe.process_item({"requests": req2}) + assert new_item["results"][0][0] is False + assert new_item["results"][0][1].value is exc + assert self.pipe._mockcalled.count("media_to_download") == 1 + + @coroutine_test + async def test_cached_failure_calls_errback(self): + """The errback of a request is called for a cached failure as well.""" + self.pipe.LOG_FAILED_RESULTS = False + exc = Exception("foo") + await self.pipe.process_item( + {"requests": Request("http://url1", meta={"response": exc})} + ) + + def errback(failure): + self.pipe._mockcalled.append("request_errback") + return {"recovered": failure.value} + + req = Request("http://url1", errback=errback) + new_item = await self.pipe.process_item({"requests": req}) + assert new_item["results"] == [(True, {"recovered": exc})] + assert self.pipe._mockcalled.count("request_errback") == 1 + @coroutine_test async def test_results_are_cached_for_requests_of_single_item(self): rsp1 = Response("http://url1") @@ -472,6 +508,30 @@ class TestBuildFromCrawler: assert pipe._from_crawler_called +class MediaFailedNonePipeline(MockedMediaPipeline): + def media_failed(self, failure, request, info): + self._mockcalled.append("media_failed") + + +class TestMediaFailedNone(TestBaseMediaPipeline): + """Test what happens when media_failed() neither raises an exception nor + returns a failure.""" + + pipeline_class = MediaFailedNonePipeline + + @coroutine_test + async def test_result_none(self): + req = Request("http://url1", meta={"response": Exception("foo")}) + new_item = await self.pipe.process_item({"requests": req}) + assert new_item["results"] == [(True, None)] + assert self.pipe._mockcalled == [ + "get_media_requests", + "media_to_download", + "media_failed", + "item_completed", + ] + + class MediaFailedFailurePipeline(MockedMediaPipeline): def media_failed(self, failure, request, info): self._mockcalled.append("media_failed") diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index a624d2097..a2a4e8fdb 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -85,6 +85,7 @@ class TestCrawl: url = self.mockserver.url("/status?n=200") crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) response = crawler.spider.meta["responses"][0] assert response.request.url == url @@ -94,6 +95,7 @@ class TestCrawl: url = self.mockserver.url(f"/status?n={status}") crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) failure = crawler.spider.meta["failure"] response = failure.value.response assert failure.request.url == url @@ -111,6 +113,7 @@ class TestCrawl: }, ) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) failure = crawler.spider.meta["failure"] assert failure.request.url == url assert isinstance(failure.value, ZeroDivisionError) @@ -178,6 +181,7 @@ class TestCrawl: }, ) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) response = crawler.spider.meta["responses"][0] assert response.body == b"Caught ZeroDivisionError" assert response.request.url == OVERRIDDEN_URL @@ -201,6 +205,7 @@ class TestCrawl: }, ) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) response = crawler.spider.meta["responses"][0] assert response.body == b"Caught ZeroDivisionError" assert response.request.url == url diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index b88893b2b..6c26aa878 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -49,6 +49,7 @@ class InjectArgumentsSpiderMiddleware: async for element in result: if ( isinstance(element, Request) + and element.callback and element.callback.__name__ == "parse_spider_mw_2" ): element.cb_kwargs["from_process_spider_output"] = True @@ -68,7 +69,12 @@ class KeywordArgumentsSpider(MockServerSpider): checks: list[bool] = [] + def _inc_checks(self, count: int = 1) -> None: + assert self.crawler.stats + self.crawler.stats.inc_value("boolean_checks", count) + async def start(self): + assert self.mockserver data = {"key": "value", "number": 123, "callback": "some_callback"} yield Request(self.mockserver.url("/first"), self.parse_first, cb_kwargs=data) yield Request( @@ -89,9 +95,10 @@ class KeywordArgumentsSpider(MockServerSpider): yield Request(self.mockserver.url("/spider_mw"), self.parse_spider_mw) def parse_first(self, response, key, number): + assert self.mockserver self.checks.append(key == "value") self.checks.append(number == 123) - self.crawler.stats.inc_value("boolean_checks", 2) + self._inc_checks(2) yield response.follow( self.mockserver.url("/two"), self.parse_second, @@ -100,28 +107,28 @@ class KeywordArgumentsSpider(MockServerSpider): def parse_second(self, response, new_key): self.checks.append(new_key == "new_value") - self.crawler.stats.inc_value("boolean_checks") + self._inc_checks() def parse_general(self, response, **kwargs): if response.url.endswith("/general_with"): self.checks.append(kwargs["key"] == "value") self.checks.append(kwargs["number"] == 123) self.checks.append(kwargs["callback"] == "some_callback") - self.crawler.stats.inc_value("boolean_checks", 3) + self._inc_checks(3) elif response.url.endswith("/general_without"): self.checks.append(kwargs == {}) - self.crawler.stats.inc_value("boolean_checks") + self._inc_checks() def parse_no_kwargs(self, response): self.checks.append(response.url.endswith("/no_kwargs")) - self.crawler.stats.inc_value("boolean_checks") + self._inc_checks() def parse_default(self, response, key, number=None, default=99): self.checks.append(response.url.endswith("/default")) self.checks.append(key == "value") self.checks.append(number == 123) self.checks.append(default == 99) - self.crawler.stats.inc_value("boolean_checks", 4) + self._inc_checks(4) def parse_takes_less(self, response, key, callback): """ @@ -140,17 +147,18 @@ class KeywordArgumentsSpider(MockServerSpider): ): self.checks.append(bool(from_process_request)) self.checks.append(bool(from_process_response)) - self.crawler.stats.inc_value("boolean_checks", 2) + self._inc_checks(2) def parse_spider_mw(self, response, from_process_spider_input, from_process_start): + assert self.mockserver self.checks.append(bool(from_process_spider_input)) self.checks.append(bool(from_process_start)) - self.crawler.stats.inc_value("boolean_checks", 2) + self._inc_checks(2) return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2) def parse_spider_mw_2(self, response, from_process_spider_output): self.checks.append(bool(from_process_spider_output)) - self.crawler.stats.inc_value("boolean_checks", 1) + self._inc_checks() class TestCallbackKeywordArguments: diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 78ff18b15..c7596e45d 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -1,7 +1,10 @@ +from typing import Any + import pytest +from twisted.python.failure import Failure from scrapy import Request, Spider -from scrapy.http import JsonRequest +from scrapy.http import JsonRequest, Response from scrapy.utils.request import request_from_dict @@ -10,7 +13,7 @@ class CustomRequest(Request): class TestRequestSerialization: - def setup_method(self): + def setup_method(self) -> None: self.spider = MethodsSpider() def test_basic(self): @@ -42,12 +45,14 @@ class TestRequestSerialization: r = Request("http://www.example.com", body=b"\xc2\xa3") self._assert_serializes_ok(r) - def _assert_serializes_ok(self, request, spider=None): + def _assert_serializes_ok( + self, request: Request, spider: Spider | None = None + ) -> None: d = request.to_dict(spider=spider) request2 = request_from_dict(d, spider=spider) self._assert_same_request(request, request2) - def _assert_same_request(self, r1, r2): + def _assert_same_request(self, r1: Request, r2: Request) -> None: assert r1.__class__ == r2.__class__ assert r1.url == r2.url assert r1.callback == r2.callback @@ -64,6 +69,7 @@ class TestRequestSerialization: assert r1.dont_filter == r2.dont_filter assert r1.flags == r2.flags if isinstance(r1, JsonRequest): + assert isinstance(r2, JsonRequest) assert r1.dumps_kwargs == r2.dumps_kwargs def test_request_class(self): @@ -83,8 +89,8 @@ class TestRequestSerialization: def test_reference_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider.parse_item_reference, - errback=self.spider.handle_error_reference, + callback=self.spider.parse_item_reference, # type: ignore[arg-type,misc] + errback=self.spider.handle_error_reference, # type: ignore[arg-type,misc] ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) @@ -94,8 +100,8 @@ class TestRequestSerialization: def test_private_reference_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._MethodsSpider__parse_item_reference, - errback=self.spider._MethodsSpider__handle_error_reference, + callback=self.spider._MethodsSpider__parse_item_reference, # type: ignore[attr-defined] + errback=self.spider._MethodsSpider__handle_error_reference, # type: ignore[attr-defined] ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) @@ -105,7 +111,7 @@ class TestRequestSerialization: def test_private_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._MethodsSpider__parse_item_private, + callback=self.spider._MethodsSpider__parse_item_private, # type: ignore[attr-defined] errback=self.spider.handle_error, ) self._assert_serializes_ok(r, spider=self.spider) @@ -113,7 +119,7 @@ class TestRequestSerialization: def test_mixin_private_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._SpiderMixin__mixin_callback, + callback=self.spider._SpiderMixin__mixin_callback, # type: ignore[attr-defined] errback=self.spider.handle_error, ) self._assert_serializes_ok(r, spider=self.spider) @@ -127,7 +133,7 @@ class TestRequestSerialization: self._assert_serializes_ok(r, spider=self.spider) def test_unserializable_callback1(self): - r = Request("http://www.example.com", callback=lambda x: x) + r = Request("http://www.example.com", callback=lambda x: x) # type: ignore[misc] with pytest.raises( ValueError, match="is not an instance method in: None: pass spider = MySpider() r = Request("http://www.example.com", callback=spider.parse) - spider.parse = None + spider.parse = None # type: ignore[method-assign,assignment] with pytest.raises(ValueError, match="is not an instance method in: None: pass class SpiderDelegation: - def delegated_callback(self, response): + def delegated_callback(self, response: Response) -> None: pass -def parse_item(response): +def parse_item(response: Response) -> None: pass -def handle_error(failure): +def handle_error(failure: Failure) -> None: pass -def private_parse_item(response): +def private_parse_item(response: Response) -> None: pass -def private_handle_error(failure): +def private_handle_error(failure: Failure) -> None: pass @@ -197,15 +205,17 @@ class MethodsSpider(Spider, SpiderMixin): __parse_item_reference = private_parse_item __handle_error_reference = private_handle_error - def __init__(self, **kwargs): + def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) self.delegated_callback = SpiderDelegation().delegated_callback - def parse_item(self, response): + def parse_item(self, response: Response) -> None: pass - def handle_error(self, failure): + def handle_error(self, failure: Failure) -> None: pass - def __parse_item_private(self, response): # pylint: disable=unused-private-member + def __parse_item_private( # pylint: disable=unused-private-member + self, response: Response + ) -> None: pass diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 726e0573a..46a16ad1e 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -1,57 +1,62 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + from scrapy.signals import request_left_downloader from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from tests.mockserver.http import MockServer from tests.utils.decorators import inline_callbacks_test +if TYPE_CHECKING: + from scrapy import Request + from scrapy.crawler import Crawler + from tests.mockserver.http import MockServer + class SignalCatcherSpider(Spider): name = "signal_catcher" - def __init__(self, crawler, url, *args, **kwargs): + def __init__(self, crawler: Crawler, url: str, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) crawler.signals.connect(self.on_request_left, signal=request_left_downloader) self.caught_times = 0 self.start_urls = [url] @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler( + cls, crawler: Crawler, *args: Any, **kwargs: Any + ) -> SignalCatcherSpider: return cls(crawler, *args, **kwargs) - def on_request_left(self, request, spider): + def on_request_left(self, request: Request, spider: Spider) -> None: self.caught_times += 1 class TestCatching: - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - @inline_callbacks_test - def test_success(self): + def test_success(self, mockserver: MockServer): crawler = get_crawler(SignalCatcherSpider) - yield crawler.crawl(self.mockserver.url("/status?n=200")) + yield crawler.crawl(mockserver.url("/status?n=200")) + assert isinstance(crawler.spider, SignalCatcherSpider) assert crawler.spider.caught_times == 1 @inline_callbacks_test - def test_timeout(self): + def test_timeout(self, mockserver: MockServer): crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1}) - yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) + yield crawler.crawl(mockserver.url("/delay?n=0.2")) + assert isinstance(crawler.spider, SignalCatcherSpider) assert crawler.spider.caught_times == 1 @inline_callbacks_test - def test_disconnect(self): + def test_disconnect(self, mockserver: MockServer): crawler = get_crawler(SignalCatcherSpider) - yield crawler.crawl(self.mockserver.url("/drop")) + yield crawler.crawl(mockserver.url("/drop")) + assert isinstance(crawler.spider, SignalCatcherSpider) assert crawler.spider.caught_times == 1 @inline_callbacks_test def test_noconnect(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl("http://thereisdefinetelynosuchdomain.com") + assert isinstance(crawler.spider, SignalCatcherSpider) assert crawler.spider.caught_times == 1 diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index da94a4e95..755f29959 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,30 +1,45 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import pytest from scrapy.robotstxt import ( ProtegoRobotParser, PythonRobotParser, RerpRobotParser, + RobotParser, decode_robotstxt, ) from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER from tests.utils.robotstxt import rerp_available +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + class BaseRobotParserTest: - def _setUp(self, parser_cls): + parser_cls: type[RobotParser] + + def _setUp(self, parser_cls: type[RobotParser]) -> None: self.parser_cls = parser_cls + def _parse(self, robotstxt_body: bytes) -> RobotParser: + # The parser backends only use the crawler to get the spider to log with. + return self.parser_cls.from_crawler(None, robotstxt_body) # type: ignore[arg-type] + def test_allowed(self): robotstxt_robotstxt_body = ( b"User-agent: * \nDisallow: /disallowed \nAllow: /allowed \nCrawl-delay: 10" ) - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://www.site.local/allowed", "*") assert not rp.allowed("https://www.site.local/disallowed", "*") - def test_allowed_wildcards(self): + def test_allowed_wildcards(self) -> None: robotstxt_robotstxt_body = b"""User-agent: first Disallow: /disallowed/*/end$ @@ -32,9 +47,7 @@ class BaseRobotParserTest: Allow: /*allowed Disallow: / """ - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://www.site.local/disallowed", "first") assert not rp.allowed("https://www.site.local/disallowed/xyz/end", "first") @@ -45,23 +58,19 @@ class BaseRobotParserTest: assert rp.allowed("https://www.site.local/is_still_allowed", "second") assert rp.allowed("https://www.site.local/is_allowed_too", "second") - def test_length_based_precedence(self): + def test_length_based_precedence(self) -> None: robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://www.site.local/page", "*") - def test_order_based_precedence(self): + def test_order_based_precedence(self) -> None: robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert not rp.allowed("https://www.site.local/page", "*") def test_empty_response(self): """empty response should equal 'allow all'""" - rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b"") + rp = self._parse(b"") assert rp.allowed("https://site.local/", "*") assert rp.allowed("https://site.local/", "chrome") assert rp.allowed("https://site.local/index.html", "*") @@ -70,14 +79,22 @@ class BaseRobotParserTest: def test_garbage_response(self): """garbage response should be discarded, equal 'allow all'""" robotstxt_robotstxt_body = b"GIF89a\xd3\x00\xfe\x00\xa2" - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://site.local/", "*") assert rp.allowed("https://site.local/", "chrome") assert rp.allowed("https://site.local/index.html", "*") assert rp.allowed("https://site.local/disallowed", "*") + def test_crawl_delay(self): + robotstxt_body = b"User-agent: *\nDisallow: /private\nCrawl-delay: 10\n" + rp = self._parse(robotstxt_body) + assert rp.crawl_delay("*") == 10.0 + + def test_crawl_delay_unset(self): + robotstxt_body = b"User-agent: *\nDisallow: /private\n" + rp = self._parse(robotstxt_body) + assert rp.crawl_delay("*") is None + def test_unicode_url_and_useragent(self): robotstxt_robotstxt_body = """ User-Agent: * @@ -89,9 +106,7 @@ class BaseRobotParserTest: User-Agent: UnicödeBöt Disallow: /some/randome/page.html""".encode() - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://site.local/", "*") assert not rp.allowed("https://site.local/admin/", "*") assert not rp.allowed("https://site.local/static/", "*") @@ -102,6 +117,20 @@ class BaseRobotParserTest: assert not rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt") +class TestRobotParser: + def test_crawl_delay_unsupported(self): + class AllowAllRobotParser(RobotParser): + @classmethod + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: + return cls() + + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: + return True + + rp = AllowAllRobotParser() + assert rp.crawl_delay("*") is None + + class TestDecodeRobotsTxt: def test_native_string_conversion(self): robotstxt_body = b"User-agent: *\nDisallow: /\n" @@ -135,21 +164,21 @@ class TestPythonRobotParser(BaseRobotParserTest): not STDLIB_IMPROVED_ROBOTFILEPARSER, reason="RobotFileParser from this Python version does not support length based directives precedence.", ) - def test_length_based_precedence(self): + def test_length_based_precedence(self) -> None: super().test_length_based_precedence() @pytest.mark.skipif( STDLIB_IMPROVED_ROBOTFILEPARSER, reason="RobotFileParser from this Python version does not support order based directives precedence.", ) - def test_order_based_precedence(self): + def test_order_based_precedence(self) -> None: super().test_order_based_precedence() @pytest.mark.skipif( not STDLIB_IMPROVED_ROBOTFILEPARSER, reason="RobotFileParser from this Python version does not support wildcards.", ) - def test_allowed_wildcards(self): + def test_allowed_wildcards(self) -> None: super().test_allowed_wildcards() @@ -158,7 +187,7 @@ class TestRerpRobotParser(BaseRobotParserTest): def setup_method(self): super()._setUp(RerpRobotParser) - def test_length_based_precedence(self): + def test_length_based_precedence(self) -> None: pytest.skip("Rerp does not support length based directives precedence.") @@ -166,5 +195,5 @@ class TestProtegoRobotParser(BaseRobotParserTest): def setup_method(self): super()._setUp(ProtegoRobotParser) - def test_order_based_precedence(self): + def test_order_based_precedence(self) -> None: pytest.skip("Protego does not support order based directives precedence.") diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index fc4e1c487..1d36a66fc 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -1,16 +1,18 @@ +from __future__ import annotations + from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from tests.utils.decorators import coroutine_test -class TestAsyncgenUtils: - @coroutine_test - async def test_as_async_generator(self): - ag = as_async_generator(range(42)) - results = [i async for i in ag] - assert results == list(range(42)) +@coroutine_test +async def test_as_async_generator(): + ag = as_async_generator(range(42)) + results = [i async for i in ag] + assert results == list(range(42)) - @coroutine_test - async def test_collect_asyncgen(self): - ag = as_async_generator(range(42)) - results = await collect_asyncgen(ag) - assert results == list(range(42)) + +@coroutine_test +async def test_collect_asyncgen(): + ag = as_async_generator(range(42)) + results = await collect_asyncgen(ag) + assert results == list(range(42)) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 7528dc51a..4bd54acd4 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -12,7 +12,9 @@ from scrapy.utils.asyncgen import as_async_generator from scrapy.utils.asyncio import ( AsyncioLoopingCall, _parallel_asyncio, + call_later, is_asyncio_available, + sleep, ) from tests.utils.decorators import coroutine_test @@ -20,11 +22,19 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator -class TestAsyncio: - @coroutine_test - async def test_is_asyncio_available(self, reactor_pytest: str) -> None: - # the result should depend only on the pytest --reactor argument - assert is_asyncio_available() == (reactor_pytest != "default") +@coroutine_test +async def test_is_asyncio_available(reactor_pytest: str) -> None: + # the result should depend only on the pytest --reactor argument + assert is_asyncio_available() == (reactor_pytest != "default") + + +@coroutine_test +async def test_sleep() -> None: + events: list[str] = [] + call_later(0.05, events.append, "call_later") + await sleep(0.1) + events.append("sleep") + assert events == ["call_later", "sleep"] @pytest.mark.only_asyncio diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index ad9aa3dff..0dea8af6d 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -1,10 +1,38 @@ from __future__ import annotations +import subprocess +import sys from importlib.util import find_spec +from io import BytesIO +from typing import TYPE_CHECKING import pytest +from pexpect import EOF -from scrapy.utils.console import get_shell_embed_func +from scrapy.utils.console import get_shell_embed_func, start_python_console +from scrapy.utils.test import get_testenv + +if TYPE_CHECKING: + from pathlib import Path + +CONSOLE = """ +from scrapy.utils.console import start_python_console + +start_python_console(banner="SHELL-READY", shells=["ipython"]) +""" + +CONSOLE_IN_RUNNING_LOOP = """ +import asyncio + +from scrapy.utils.console import start_python_console + + +async def main(): + start_python_console(banner="SHELL-READY", shells=["ipython"]) + + +asyncio.run(main()) +""" def test_get_shell_embed_func(): @@ -59,3 +87,90 @@ def test_get_shell_embed_func_default(): else: expected = "_embed_standard_shell" assert shell.__name__ == expected + + +@pytest.mark.skipif(find_spec("IPython") is None, reason="IPython is not installed") +class TestIPythonShell: + """Starting an IPython shell, with and without an asyncio event loop already + running in the calling thread. The latter happens when inspect_response() is + called from a spider callback while using the asyncio reactor.""" + + @staticmethod + def _env(tmp_path: Path) -> dict[str, str]: + env = get_testenv() + # Keep IPython away from the profile and history of the user running the tests. + env["IPYTHONDIR"] = str(tmp_path) + return env + + def test_simple_prompt(self, tmp_path: Path) -> None: + """IPython falls back to its simple prompt, which needs no event loop, + when stdin is not a TTY.""" + env = self._env(tmp_path) + p = subprocess.run( + [sys.executable, "-c", CONSOLE_IN_RUNNING_LOOP], + check=False, + capture_output=True, + encoding="utf-8", + timeout=60, + env=env, + stdin=subprocess.DEVNULL, + ) + output = p.stdout + p.stderr + assert "SHELL-READY" in output + assert p.returncode == 0, output + + @pytest.mark.skipif( + sys.platform == "win32", reason="requires a POSIX pseudo-terminal" + ) + @pytest.mark.parametrize( + "script", + [CONSOLE, CONSOLE_IN_RUNNING_LOOP], + ids=["no_running_loop", "running_loop"], + ) + def test_tty(self, tmp_path: Path, script: str) -> None: + """IPython uses prompt_toolkit, which needs an event loop of its own, + when stdin is a TTY.""" + # pexpect only defines spawn, which needs a pseudo-terminal, on POSIX. + from pexpect import spawn # noqa: PLC0415 + + env = self._env(tmp_path) + env.pop("IPY_TEST_SIMPLE_PROMPT", None) + env["TERM"] = "xterm" + logfile = BytesIO() + p = spawn( + sys.executable, + ["-c", script], + env=env, + timeout=60, + ) + p.logfile_read = logfile + try: + # Wait for the prompt, which prompt_toolkit draws once it is done + # querying the terminal, before typing into it. + p.expect(r"In \[") + p.sendline("21*2") + p.expect_exact("42") + p.sendline("exit()") + p.expect(EOF) + finally: + p.close() + output = logfile.getvalue().decode() + assert "Traceback" not in output + assert p.exitstatus == 0, output + + +def test_start_python_console_exit(monkeypatch: pytest.MonkeyPatch) -> None: + def embed(namespace: dict[str, object], banner: str) -> None: + raise SystemExit + + monkeypatch.setattr( + "scrapy.utils.console.get_shell_embed_func", lambda shells: embed + ) + start_python_console() + + +def test_start_python_console_no_shell(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "scrapy.utils.console.get_shell_embed_func", lambda shells: None + ) + start_python_console() diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index fce9fc984..6b30744bb 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import warnings from typing import Any diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index f43d20e69..af203ca61 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,7 +1,9 @@ +from __future__ import annotations + import copy from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, MutableMapping -from typing import Any +from typing import Any, Generic, TypeVar import pytest @@ -16,11 +18,13 @@ from scrapy.utils.datatypes import ( ) from scrapy.utils.python import garbage_collect +_DictT = TypeVar("_DictT", bound="CaselessDict | CaseInsensitiveDict") -class TestCaseInsensitiveDictBase(ABC): + +class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]): @property @abstractmethod - def dict_class(self) -> type[MutableMapping[str, Any]]: + def dict_class(self) -> type[_DictT]: raise NotImplementedError def test_init_dict(self): @@ -36,17 +40,17 @@ class TestCaseInsensitiveDictBase(ABC): assert d["black"] == 3 def test_init_mapping(self): - class MyMapping(Mapping): - def __init__(self, **kwargs): + class MyMapping(Mapping[str, int]): + def __init__(self, **kwargs: int) -> None: self._d = kwargs - def __getitem__(self, key): + def __getitem__(self, key: str) -> int: return self._d[key] - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self._d) - def __len__(self): + def __len__(self) -> int: return len(self._d) seq = MyMapping(red=1, black=3) @@ -55,23 +59,23 @@ class TestCaseInsensitiveDictBase(ABC): assert d["black"] == 3 def test_init_mutable_mapping(self): - class MyMutableMapping(MutableMapping): - def __init__(self, **kwargs): + class MyMutableMapping(MutableMapping[str, int]): + def __init__(self, **kwargs: int) -> None: self._d = kwargs - def __getitem__(self, key): + def __getitem__(self, key: str) -> int: return self._d[key] - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: int) -> None: self._d[key] = value - def __delitem__(self, key): + def __delitem__(self, key: str) -> None: del self._d[key] - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self._d) - def __len__(self): + def __len__(self) -> int: return len(self._d) seq = MyMutableMapping(red=1, black=3) @@ -149,7 +153,7 @@ class TestCaseInsensitiveDictBase(ABC): d.pop("A") def test_normkey(self): - class MyDict(self.dict_class): + class MyDict(self.dict_class): # type: ignore[misc,name-defined] def _normkey(self, key): return key.title() @@ -160,7 +164,7 @@ class TestCaseInsensitiveDictBase(ABC): assert list(d.keys()) == ["Key-One"] def test_normvalue(self): - class MyDict(self.dict_class): + class MyDict(self.dict_class): # type: ignore[misc,name-defined] def _normvalue(self, value): if value is not None: return value + 1 @@ -214,8 +218,8 @@ class TestCaseInsensitiveDictBase(ABC): assert dict(h1) == {"header1": "value1", "header2": "value2"} -class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase): - dict_class = CaseInsensitiveDict # type: ignore[assignment] +class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase[CaseInsensitiveDict]): + dict_class = CaseInsensitiveDict def test_repr(self): d1 = self.dict_class({"foo": "bar"}) @@ -230,7 +234,7 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase): assert list(iterkeys) == ["AsDf", "FoO"] def test_copy_keeps_values(self): - class MyDict(self.dict_class): + class MyDict(self.dict_class): # type: ignore[misc,name-defined] def _normvalue(self, value): return value + 1 @@ -253,7 +257,7 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase): @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestCaselessDict(TestCaseInsensitiveDictBase): +class TestCaselessDict(TestCaseInsensitiveDictBase[CaselessDict]): dict_class = CaselessDict def test_deprecation_message(self): @@ -319,7 +323,7 @@ class TestSequenceExclude: class TestLocalCache: def test_cache_with_limit(self): - cache = LocalCache(limit=2) + cache: LocalCache[str, int] = LocalCache(limit=2) cache["a"] = 1 cache["b"] = 2 cache["c"] = 3 @@ -332,7 +336,7 @@ class TestLocalCache: def test_cache_without_limit(self): maximum = 10**4 - cache = LocalCache() + cache: LocalCache[str, int] = LocalCache() for x in range(maximum): cache[str(x)] = x assert len(cache) == maximum @@ -341,7 +345,7 @@ class TestLocalCache: assert cache[str(x)] == x def test_cache_with_zero_limit(self): - cache = LocalCache(limit=0) + cache: LocalCache[str, int] = LocalCache(limit=0) cache["a"] = 1 cache["b"] = 2 cache["c"] = 3 @@ -353,7 +357,9 @@ class TestLocalCache: class TestLocalWeakReferencedCache: def test_cache_with_limit(self): - cache = LocalWeakReferencedCache(limit=2) + cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache( + limit=2 + ) r1 = Request("https://example.org") r2 = Request("https://example.com") r3 = Request("https://example.net") @@ -375,7 +381,7 @@ class TestLocalWeakReferencedCache: assert len(cache) == 1 def test_cache_non_weak_referenceable_objects(self): - cache = LocalWeakReferencedCache() + cache: LocalWeakReferencedCache[Any, int] = LocalWeakReferencedCache() k1 = None k2 = 1 k3 = [1, 2, 3] @@ -389,7 +395,7 @@ class TestLocalWeakReferencedCache: def test_cache_without_limit(self): maximum = 10**4 - cache = LocalWeakReferencedCache() + cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache() refs = [] for x in range(maximum): refs.append(Request(f"https://example.org/{x}")) diff --git a/tests/test_utils_decorators.py b/tests/test_utils_decorators.py index 9743e1a50..4c29d2917 100644 --- a/tests/test_utils_decorators.py +++ b/tests/test_utils_decorators.py @@ -1,6 +1,8 @@ from __future__ import annotations +import sys import warnings +from typing import TYPE_CHECKING, Any import pytest from twisted.internet.defer import Deferred @@ -10,11 +12,14 @@ from scrapy.utils.decorators import _warn_spider_arg, deprecated, inthread from scrapy.utils.defer import maybe_deferred_to_future from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Callable + class TestDeprecated: def test_warns_and_still_calls(self): @deprecated() - def add(a, b): + def add(a: int, b: int) -> int: return a + b with pytest.warns( @@ -26,7 +31,7 @@ class TestDeprecated: def test_use_instead_in_message(self): @deprecated(use_instead="other_function") - def old(): + def old() -> None: return None with pytest.warns( @@ -37,7 +42,7 @@ class TestDeprecated: def test_applied_without_parentheses(self): @deprecated - def square(x): + def square(x: int) -> int: return x * x with pytest.warns( @@ -65,7 +70,7 @@ class TestInthread: class TestWarnSpiderArg: def test_sync_warns_with_spider_arg(self): @_warn_spider_arg - def parse(response, spider=None): + def parse(response: str, spider: str | None = None) -> str: return response with pytest.warns( @@ -73,9 +78,34 @@ class TestWarnSpiderArg: ): assert parse("response", spider="spider") == "response" + @pytest.mark.skipif( + sys.version_info < (3, 14), + reason="annotations are only lazily evaluated since Python 3.14 (PEP 649)", + ) + def test_sync_warns_with_unresolvable_annotations(self): + # dont_inherit=True, or the module's future import stringizes the annotations + namespace: dict[str, Any] = {} + exec( # pylint: disable=exec-used + compile( + "def parse(response: OnlyAtTypeCheckingTime," + " spider: OnlyAtTypeCheckingTime | None = None): return response", + "", + "exec", + dont_inherit=True, + ), + namespace, + ) + parse_func: Callable[..., str] = namespace["parse"] + parse = _warn_spider_arg(parse_func) + + with pytest.warns( + ScrapyDeprecationWarning, match=r"Passing a 'spider' argument" + ): + assert parse("response", spider="spider") == "response" + def test_sync_no_warning_without_spider_arg(self): @_warn_spider_arg - def parse(response, spider=None): + def parse(response: str, spider: str | None = None) -> str: return response with warnings.catch_warnings(): @@ -85,7 +115,7 @@ class TestWarnSpiderArg: @coroutine_test async def test_async_warns_with_spider_arg(self): @_warn_spider_arg - async def parse(response, spider=None): + async def parse(response: str, spider: str | None = None) -> str: return response with pytest.warns( @@ -96,7 +126,9 @@ class TestWarnSpiderArg: @coroutine_test async def test_asyncgen_warns_with_spider_arg(self): @_warn_spider_arg - async def parse(response, spider=None): + async def parse( + response: str, spider: str | None = None + ) -> AsyncGenerator[str]: yield response with pytest.warns( diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 978c24f5a..175a4fe03 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -24,6 +24,8 @@ from tests.utils.decorators import coroutine_test, inline_callbacks_test if TYPE_CHECKING: from collections.abc import AsyncGenerator, Awaitable, Callable, Generator + from twisted.python.failure import Failure + @pytest.mark.requires_reactor # mustbe_deferred() requires a reactor @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") @@ -70,7 +72,7 @@ class TestIterErrback: def itergood() -> Generator[int, None, None]: yield from range(10) - errors = [] + errors: list[Failure] = [] out = list(iter_errback(itergood(), errors.append)) assert out == list(range(10)) assert not errors @@ -82,7 +84,7 @@ class TestIterErrback: 1 / 0 yield x - errors = [] + errors: list[Failure] = [] out = list(iter_errback(iterbad(), errors.append)) assert out == [0, 1, 2, 3, 4] assert len(errors) == 1 @@ -96,7 +98,7 @@ class TestAiterErrback: for x in range(10): yield x - errors = [] + errors: list[Failure] = [] out = await collect_asyncgen(aiter_errback(itergood(), errors.append)) assert out == list(range(10)) assert not errors @@ -109,7 +111,7 @@ class TestAiterErrback: 1 / 0 yield x - errors = [] + errors: list[Failure] = [] out = await collect_asyncgen(aiter_errback(iterbad(), errors.append)) assert out == [0, 1, 2, 3, 4] assert len(errors) == 1 @@ -202,7 +204,7 @@ class TestParallelAsync: for length in [20, 50, 100]: parallel_count = [0] max_parallel_count = [0] - results = [] + results: list[int] = [] ait = self.get_async_iterable(length) dl = parallel_async( ait, @@ -222,7 +224,7 @@ class TestParallelAsync: for length in [20, 50, 100]: parallel_count = [0] max_parallel_count = [0] - results = [] + results: list[int] = [] ait = self.get_async_iterable_with_delays(length) dl = parallel_async( ait, @@ -240,7 +242,7 @@ class TestParallelAsync: class TestDeferredFromCoro: def test_deferred(self): - d = Deferred() + d: Deferred[None] = Deferred() result = deferred_from_coro(d) assert isinstance(result, Deferred) assert result is d @@ -274,7 +276,7 @@ class TestDeferredFromCoro: @pytest.mark.only_asyncio @inline_callbacks_test def test_future(self): - future = Future() + future: Future[int] = Future() result = deferred_from_coro(future) assert isinstance(result, Deferred) future.set_result(42) @@ -324,7 +326,7 @@ class TestDeferredFFromCoroF: class TestDeferredToFuture: @coroutine_test async def test_deferred(self): - d = Deferred() + d: Deferred[int] = Deferred() result = deferred_to_future(d) assert isinstance(result, Future) d.callback(42) @@ -359,7 +361,7 @@ class TestDeferredToFuture: class TestMaybeDeferredToFutureAsyncio: @coroutine_test async def test_deferred(self): - d = Deferred() + d: Deferred[int] = Deferred() result = maybe_deferred_to_future(d) assert isinstance(result, Future) d.callback(42) @@ -394,7 +396,7 @@ class TestMaybeDeferredToFutureAsyncio: class TestMaybeDeferredToFutureNotAsyncio: @coroutine_test async def test_deferred(self): - d = Deferred() + d: Deferred[int] = Deferred() result = maybe_deferred_to_future(d) assert isinstance(result, Deferred) assert result is d diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 0706fec99..4c8585916 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import inspect import warnings from unittest import mock @@ -38,7 +40,7 @@ class TestWarnWhenSubclassed: ) with pytest.warns(MyWarning, match=msg) as w: - class UserClass(Deprecated): + class UserClass(Deprecated): # type: ignore[misc, valid-type] pass assert w[0].lineno == inspect.getsourcelines(UserClass)[1] @@ -57,7 +59,7 @@ class TestWarnWhenSubclassed: match=r"UserClass inherits from deprecated class bar\.OldClass, please inherit from foo\.NewClass", ): - class UserClass(Deprecated): + class UserClass(Deprecated): # type: ignore[misc, valid-type] pass with pytest.warns( @@ -76,7 +78,7 @@ class TestWarnWhenSubclassed: match="UserClass inherits from deprecated class", ): - class UserClass(Deprecated): + class UserClass(Deprecated): # type: ignore[misc, valid-type] pass with warnings.catch_warnings(): @@ -95,16 +97,16 @@ class TestWarnWhenSubclassed: match="UserClass inherits from deprecated class", ): - class UserClass(Deprecated): + class UserClass(Deprecated): # type: ignore[misc, valid-type] pass with warnings.catch_warnings(): warnings.simplefilter("error", MyWarning) - class FooClass(Deprecated): + class FooClass(Deprecated): # type: ignore[misc, valid-type] pass - class BarClass(Deprecated): + class BarClass(Deprecated): # type: ignore[misc, valid-type] pass def test_warning_on_instance(self): @@ -112,22 +114,20 @@ class TestWarnWhenSubclassed: "Deprecated", NewName, warn_category=MyWarning ) - with pytest.warns(MyWarning) as w: - _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) - - w = [x for x in w if x.category is MyWarning] + with pytest.warns( + MyWarning, + match=r"tests\.test_utils_deprecate\.Deprecated is deprecated, " + r"instantiate tests\.test_utils_deprecate\.NewName instead\.", + ) as w: + _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) # type: ignore[arg-type] assert len(w) == 1 - assert ( - str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, " - "instantiate tests.test_utils_deprecate.NewName instead." - ) assert w[0].lineno == lineno # ignore subclassing warnings with warnings.catch_warnings(): warnings.simplefilter("ignore", MyWarning) - class UserClass(Deprecated): + class UserClass(Deprecated): # type: ignore[misc, valid-type] pass with warnings.catch_warnings(): @@ -141,7 +141,7 @@ class TestWarnWhenSubclassed: match=r"UserClass2 inherits from deprecated class tests\.test_utils_deprecate\.Deprecated, please inherit from tests\.test_utils_deprecate\.NewName", ): - class UserClass2(Deprecated): + class UserClass2(Deprecated): # type: ignore[misc, valid-type] pass def test_issubclass(self): @@ -155,10 +155,10 @@ class TestWarnWhenSubclassed: class UpdatedUserClass1a(NewName): pass - class OutdatedUserClass1(DeprecatedName): + class OutdatedUserClass1(DeprecatedName): # type: ignore[misc, valid-type] pass - class OutdatedUserClass1a(DeprecatedName): + class OutdatedUserClass1a(DeprecatedName): # type: ignore[misc, valid-type] pass class UnrelatedClass: @@ -174,7 +174,7 @@ class TestWarnWhenSubclassed: assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1) with pytest.raises(TypeError): - issubclass(object(), DeprecatedName) + issubclass(object(), DeprecatedName) # type: ignore[arg-type] def test_isinstance(self): with warnings.catch_warnings(): @@ -187,10 +187,10 @@ class TestWarnWhenSubclassed: class UpdatedUserClass2a(NewName): pass - class OutdatedUserClass2(DeprecatedName): + class OutdatedUserClass2(DeprecatedName): # type: ignore[misc, valid-type] pass - class OutdatedUserClass2a(DeprecatedName): + class OutdatedUserClass2a(DeprecatedName): # type: ignore[misc, valid-type] pass class UnrelatedClass: @@ -211,7 +211,7 @@ class TestWarnWhenSubclassed: warnings.simplefilter("ignore", ScrapyDeprecationWarning) Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"}) - assert Deprecated.foo == "bar" + assert Deprecated.foo == "bar" # type: ignore[attr-defined] def test_deprecate_a_class_with_custom_metaclass(self): Meta1 = type("Meta1", (type,), {}) @@ -242,7 +242,7 @@ class TestWarnWhenSubclassed: match=r"UserClass inherits from deprecated class tests\.test_utils_deprecate\.AlsoDeprecated, please inherit from foo\.Bar", ): - class UserClass(AlsoDeprecated): + class UserClass(AlsoDeprecated): # type: ignore[misc, valid-type] pass def test_inspect_stack(self): diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index 9f9e24957..a87816c20 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -31,13 +31,13 @@ plain_string = "{'a': 1}" @mock.patch("sys.platform", "linux") @mock.patch("sys.stdout.isatty") -def test_pformat(isatty): +def test_pformat(isatty: mock.Mock) -> None: isatty.return_value = True assert pformat(value) in colorized_strings @mock.patch("sys.stdout.isatty") -def test_pformat_dont_colorize(isatty): +def test_pformat_dont_colorize(isatty: mock.Mock) -> None: isatty.return_value = True assert pformat(value, colorize=False) == plain_string @@ -49,7 +49,7 @@ def test_pformat_not_tty(): @mock.patch("sys.platform", "win32") @mock.patch("platform.version") @mock.patch("sys.stdout.isatty") -def test_pformat_old_windows(isatty, version): +def test_pformat_old_windows(isatty: mock.Mock, version: mock.Mock) -> None: isatty.return_value = True version.return_value = "10.0.14392" assert pformat(value) in colorized_strings @@ -59,7 +59,9 @@ def test_pformat_old_windows(isatty, version): @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") @mock.patch("platform.version") @mock.patch("sys.stdout.isatty") -def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing): +def test_pformat_windows_no_terminal_processing( + isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock +) -> None: isatty.return_value = True version.return_value = "10.0.14393" terminal_processing.return_value = False @@ -70,7 +72,9 @@ def test_pformat_windows_no_terminal_processing(isatty, version, terminal_proces @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") @mock.patch("platform.version") @mock.patch("sys.stdout.isatty") -def test_pformat_windows(isatty, version, terminal_processing): +def test_pformat_windows( + isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock +) -> None: isatty.return_value = True version.return_value = "10.0.14393" terminal_processing.return_value = True @@ -79,7 +83,7 @@ def test_pformat_windows(isatty, version, terminal_processing): @mock.patch("sys.platform", "linux") @mock.patch("sys.stdout.isatty") -def test_pformat_no_pygments(isatty): +def test_pformat_no_pygments(isatty: mock.Mock) -> None: isatty.return_value = True real_import = builtins.__import__ diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 06fdf9cba..75f8be6f6 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from gzip import BadGzipFile from pathlib import Path diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index 0eb330461..610c463ec 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from urllib.parse import urlparse from scrapy.http import Request diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index ab6965ed5..0775ec608 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -20,150 +20,157 @@ from scrapy.utils.misc import ( ) -class TestUtilsMisc: - def test_load_object_class(self): - obj = load_object(Field) - assert obj is Field - obj = load_object("scrapy.item.Field") - assert obj is Field +def test_load_object_class() -> None: + obj = load_object(Field) + assert obj is Field + obj = load_object("scrapy.item.Field") + assert obj is Field - def test_load_object_function(self): - obj = load_object(load_object) - assert obj is load_object - obj = load_object("scrapy.utils.misc.load_object") - assert obj is load_object - def test_load_object_exceptions(self): - with pytest.raises(ImportError): - load_object("nomodule999.mod.function") - with pytest.raises(NameError): - load_object("scrapy.utils.misc.load_object999") - with pytest.raises(TypeError): - load_object({}) # type: ignore[arg-type] +def test_load_object_function() -> None: + obj = load_object(load_object) + assert obj is load_object + obj = load_object("scrapy.utils.misc.load_object") + assert obj is load_object - def test_walk_modules(self): - mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules") + +def test_load_object_exceptions() -> None: + with pytest.raises(ImportError): + load_object("nomodule999.mod.function") + with pytest.raises(NameError): + load_object("scrapy.utils.misc.load_object999") + with pytest.raises(TypeError): + load_object({}) # type: ignore[arg-type] + + +def test_walk_modules() -> None: + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules") + expected = [ + "tests.test_utils_misc.test_walk_modules", + "tests.test_utils_misc.test_walk_modules.mod", + "tests.test_utils_misc.test_walk_modules.mod.mod0", + "tests.test_utils_misc.test_walk_modules.mod1", + ] + assert {m.__name__ for m in mods} == set(expected) + + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod") + expected = [ + "tests.test_utils_misc.test_walk_modules.mod", + "tests.test_utils_misc.test_walk_modules.mod.mod0", + ] + assert {m.__name__ for m in mods} == set(expected) + + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1") + expected = [ + "tests.test_utils_misc.test_walk_modules.mod1", + ] + assert {m.__name__ for m in mods} == set(expected) + + with pytest.raises(ImportError): + for _ in walk_modules_iter("nomodule999"): + pass + with ( + pytest.raises(ImportError), + pytest.warns( + ScrapyDeprecationWarning, + match="The scrapy.utils.misc.walk_modules function is deprecated and will be " + "removed in a future version of Scrapy. " + "Use scrapy.utils.misc.walk_modules_iter instead.", + ), + ): + walk_modules("nomodule999") + + +def test_walk_modules_egg() -> None: + egg = str(Path(__file__).parent / "test.egg") + sys.path.append(egg) + try: + mods = walk_modules_iter("testegg") expected = [ - "tests.test_utils_misc.test_walk_modules", - "tests.test_utils_misc.test_walk_modules.mod", - "tests.test_utils_misc.test_walk_modules.mod.mod0", - "tests.test_utils_misc.test_walk_modules.mod1", + "testegg.spiders", + "testegg.spiders.a", + "testegg.spiders.b", + "testegg", ] assert {m.__name__ for m in mods} == set(expected) + finally: + sys.path.remove(egg) - mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod") - expected = [ - "tests.test_utils_misc.test_walk_modules.mod", - "tests.test_utils_misc.test_walk_modules.mod.mod0", - ] - assert {m.__name__ for m in mods} == set(expected) - mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1") - expected = [ - "tests.test_utils_misc.test_walk_modules.mod1", - ] - assert {m.__name__ for m in mods} == set(expected) +def test_arg_to_iter() -> None: + class TestItem(Item): + name = Field() - with pytest.raises(ImportError): - for _ in walk_modules_iter("nomodule999"): - pass - with ( - pytest.raises(ImportError), - pytest.warns( - ScrapyDeprecationWarning, - match="The scrapy.utils.misc.walk_modules function is deprecated and will be " - "removed in a future version of Scrapy. " - "Use scrapy.utils.misc.walk_modules_iter instead.", - ), - ): - walk_modules("nomodule999") + assert hasattr(arg_to_iter(None), "__iter__") + assert hasattr(arg_to_iter(100), "__iter__") + assert hasattr(arg_to_iter("lala"), "__iter__") + assert hasattr(arg_to_iter([1, 2, 3]), "__iter__") + assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__") - def test_walk_modules_egg(self): - egg = str(Path(__file__).parent / "test.egg") - sys.path.append(egg) - try: - mods = walk_modules_iter("testegg") - expected = [ - "testegg.spiders", - "testegg.spiders.a", - "testegg.spiders.b", - "testegg", - ] - assert {m.__name__ for m in mods} == set(expected) - finally: - sys.path.remove(egg) + assert not list(arg_to_iter(None)) + assert list(arg_to_iter("lala")) == ["lala"] + assert list(arg_to_iter(100)) == [100] + assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"] + assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3] + assert list(arg_to_iter({"a": 1})) == [{"a": 1}] + assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")] - def test_arg_to_iter(self): - class TestItem(Item): - name = Field() - assert hasattr(arg_to_iter(None), "__iter__") - assert hasattr(arg_to_iter(100), "__iter__") - assert hasattr(arg_to_iter("lala"), "__iter__") - assert hasattr(arg_to_iter([1, 2, 3]), "__iter__") - assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__") +def test_build_from_crawler() -> None: + crawler = mock.MagicMock(spec_set=["settings"]) + args = (True, 100.0) + kwargs = {"key": "val"} - assert not list(arg_to_iter(None)) - assert list(arg_to_iter("lala")) == ["lala"] - assert list(arg_to_iter(100)) == [100] - assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"] - assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3] - assert list(arg_to_iter({"a": 1})) == [{"a": 1}] - assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")] + def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None: + build_from_crawler(mock, crawler, *args, **kwargs) + if hasattr(mock, "from_crawler"): + mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) + assert mock.call_count == 0 + else: + mock.assert_called_once_with(*args, **kwargs) - def test_build_from_crawler(self): - crawler = mock.MagicMock(spec_set=["settings"]) - args = (True, 100.0) - kwargs = {"key": "val"} + # Check usage of correct constructor using 2 mocks: + # 1. with no alternative constructors + # 2. with from_crawler() constructor + spec_sets = ( + ["__qualname__"], + ["__qualname__", "from_crawler"], + ) + for specs in spec_sets: + m = mock.MagicMock(spec_set=specs) + _test_with_crawler(m, crawler) + m.reset_mock() - def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None: - build_from_crawler(mock, crawler, *args, **kwargs) - if hasattr(mock, "from_crawler"): - mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) - assert mock.call_count == 0 - else: - mock.assert_called_once_with(*args, **kwargs) + # Check adoption of crawler + m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"]) + m.from_crawler.return_value = None + with pytest.raises(TypeError): + build_from_crawler(m, crawler, *args, **kwargs) - # Check usage of correct constructor using 2 mocks: - # 1. with no alternative constructors - # 2. with from_crawler() constructor - spec_sets = ( - ["__qualname__"], - ["__qualname__", "from_crawler"], - ) - for specs in spec_sets: - m = mock.MagicMock(spec_set=specs) - _test_with_crawler(m, crawler) - m.reset_mock() - # Check adoption of crawler - m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"]) - m.from_crawler.return_value = None - with pytest.raises(TypeError): - build_from_crawler(m, crawler, *args, **kwargs) +def test_set_environ() -> None: + assert os.environ.get("some_test_environ") is None + with set_environ(some_test_environ="test_value"): + assert os.environ.get("some_test_environ") == "test_value" + assert os.environ.get("some_test_environ") is None - def test_set_environ(self): - assert os.environ.get("some_test_environ") is None - with set_environ(some_test_environ="test_value"): - assert os.environ.get("some_test_environ") == "test_value" - assert os.environ.get("some_test_environ") is None + os.environ["some_test_environ"] = "test" + assert os.environ.get("some_test_environ") == "test" + with set_environ(some_test_environ="test_value"): + assert os.environ.get("some_test_environ") == "test_value" + assert os.environ.get("some_test_environ") == "test" - os.environ["some_test_environ"] = "test" - assert os.environ.get("some_test_environ") == "test" - with set_environ(some_test_environ="test_value"): - assert os.environ.get("some_test_environ") == "test_value" - assert os.environ.get("some_test_environ") == "test" - def test_rel_has_nofollow(self): - assert rel_has_nofollow("ugc nofollow") is True - assert rel_has_nofollow("ugc,nofollow") is True - assert rel_has_nofollow("ugc") is False - assert rel_has_nofollow("nofollow") is True - assert rel_has_nofollow("nofollowfoo") is False - assert rel_has_nofollow("foonofollow") is False - assert rel_has_nofollow("ugc, , nofollow") is True - # rel attribute values are ASCII case-insensitive per the HTML spec - assert rel_has_nofollow("NoFollow") is True - assert rel_has_nofollow("NOFOLLOW") is True - assert rel_has_nofollow("UGC NoFollow") is True - assert rel_has_nofollow("ugc,NoFollow") is True +def test_rel_has_nofollow() -> None: + assert rel_has_nofollow("ugc nofollow") is True + assert rel_has_nofollow("ugc,nofollow") is True + assert rel_has_nofollow("ugc") is False + assert rel_has_nofollow("nofollow") is True + assert rel_has_nofollow("nofollowfoo") is False + assert rel_has_nofollow("foonofollow") is False + assert rel_has_nofollow("ugc, , nofollow") is True + # rel attribute values are ASCII case-insensitive per the HTML spec + assert rel_has_nofollow("NoFollow") is True + assert rel_has_nofollow("NOFOLLOW") is True + assert rel_has_nofollow("UGC NoFollow") is True + assert rel_has_nofollow("ugc,NoFollow") is True diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 1acc3aac2..7343a135a 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -1,5 +1,8 @@ +from __future__ import annotations + import warnings from functools import partial +from typing import TYPE_CHECKING, Any from unittest import mock import pytest @@ -9,6 +12,11 @@ from scrapy.utils.misc import ( warn_on_generator_with_return_value, ) +if TYPE_CHECKING: + from collections.abc import Generator + + from scrapy import Spider + def _indentation_error(*args, **kwargs): raise IndentationError @@ -35,244 +43,239 @@ https://example.org yield url -def generator_that_returns_stuff(): +def generator_that_returns_stuff() -> Generator[int, None, int]: yield 1 yield 2 return 3 -class TestUtilsMisc: - @pytest.fixture - def mock_spider(self): - class MockSettings: - def __init__(self, settings_dict=None): - self.settings_dict = settings_dict or { - "WARN_ON_GENERATOR_RETURN_VALUE": True - } +@pytest.fixture +def mock_spider() -> Spider: + class MockSettings: + def __init__(self, settings_dict: dict[str, Any] | None = None): + self.settings_dict = settings_dict or { + "WARN_ON_GENERATOR_RETURN_VALUE": True + } - def getbool(self, name, default=False): - return self.settings_dict.get(name, default) + def getbool(self, name, default=False): + return self.settings_dict.get(name, default) - class MockSpider: - def __init__(self): - self.settings = MockSettings() + class MockSpider: + def __init__(self) -> None: + self.settings = MockSettings() - return MockSpider() + return MockSpider() # type: ignore[return-value] - def test_generators_return_something(self, mock_spider): - def f1(): - yield 1 - return 2 - def g1(): - yield 1 - return "asdf" +def test_generators_return_something(mock_spider): + def f1(): + yield 1 + return 2 - def h1(): - yield 1 + def g1(): + yield 1 + return "asdf" - def helper(): - return 0 + def h1(): + yield 1 - yield helper() - return 2 + def helper() -> int: + return 0 - def i1(): - """ - docstring - """ - url = """ -https://example.org + yield helper() + return 2 + + def i1(): """ - yield url - return 1 - - assert is_generator_with_return_value(top_level_return_something) - assert is_generator_with_return_value(f1) - assert is_generator_with_return_value(g1) - assert is_generator_with_return_value(h1) - assert is_generator_with_return_value(i1) - - with pytest.warns( - UserWarning, - match='The "MockSpider.top_level_return_something" method is a generator', - ): - warn_on_generator_with_return_value(mock_spider, top_level_return_something) - with pytest.warns( - UserWarning, match='The "MockSpider.f1" method is a generator' - ): - warn_on_generator_with_return_value(mock_spider, f1) - with pytest.warns( - UserWarning, match='The "MockSpider.g1" method is a generator' - ): - warn_on_generator_with_return_value(mock_spider, g1) - with pytest.warns( - UserWarning, match='The "MockSpider.h1" method is a generator' - ): - warn_on_generator_with_return_value(mock_spider, h1) - with pytest.warns( - UserWarning, match='The "MockSpider.i1" method is a generator' - ): - warn_on_generator_with_return_value(mock_spider, i1) - - def test_generators_return_none(self, mock_spider): - def f2(): - yield 1 - - def g2(): - yield 1 - - def h2(): - yield 1 - - def i2(): - yield 1 - yield from generator_that_returns_stuff() - - def j2(): - yield 1 - - def helper(): - return 0 - - yield helper() - - def k2(): - """ - docstring - """ - url = """ -https://example.org + docstring """ - yield url - - def l2(): - return - - assert not is_generator_with_return_value(top_level_return_none) - assert not is_generator_with_return_value(f2) - assert not is_generator_with_return_value(g2) - assert not is_generator_with_return_value(h2) - assert not is_generator_with_return_value(i2) - assert not is_generator_with_return_value(j2) # not recursive - assert not is_generator_with_return_value(k2) # not recursive - assert not is_generator_with_return_value(l2) - - with warnings.catch_warnings(): - warnings.simplefilter("error", UserWarning) - warn_on_generator_with_return_value(mock_spider, top_level_return_none) - warn_on_generator_with_return_value(mock_spider, f2) - warn_on_generator_with_return_value(mock_spider, g2) - warn_on_generator_with_return_value(mock_spider, h2) - warn_on_generator_with_return_value(mock_spider, i2) - warn_on_generator_with_return_value(mock_spider, j2) - warn_on_generator_with_return_value(mock_spider, k2) - warn_on_generator_with_return_value(mock_spider, l2) - - def test_generators_return_none_with_decorator(self, mock_spider): - def decorator(func): - def inner_func(): - func() - - return inner_func - - @decorator - def f3(): - yield 1 - - @decorator - def g3(): - yield 1 - - @decorator - def h3(): - yield 1 - - @decorator - def i3(): - yield 1 - yield from generator_that_returns_stuff() - - @decorator - def j3(): - yield 1 - - def helper(): - return 0 - - yield helper() - - @decorator - def k3(): - """ - docstring - """ - url = """ + url = """ https://example.org + """ + yield url + return 1 + + assert is_generator_with_return_value(top_level_return_something) + assert is_generator_with_return_value(f1) + assert is_generator_with_return_value(g1) + assert is_generator_with_return_value(h1) + assert is_generator_with_return_value(i1) + + with pytest.warns( + UserWarning, + match='The "MockSpider.top_level_return_something" method is a generator', + ): + warn_on_generator_with_return_value(mock_spider, top_level_return_something) + with pytest.warns(UserWarning, match='The "MockSpider.f1" method is a generator'): + warn_on_generator_with_return_value(mock_spider, f1) + with pytest.warns(UserWarning, match='The "MockSpider.g1" method is a generator'): + warn_on_generator_with_return_value(mock_spider, g1) + with pytest.warns(UserWarning, match='The "MockSpider.h1" method is a generator'): + warn_on_generator_with_return_value(mock_spider, h1) + with pytest.warns(UserWarning, match='The "MockSpider.i1" method is a generator'): + warn_on_generator_with_return_value(mock_spider, i1) + + +def test_generators_return_none(mock_spider): + def f2(): + yield 1 + + def g2(): + yield 1 + + def h2(): + yield 1 + + def i2(): + yield 1 + yield from generator_that_returns_stuff() + + def j2(): + yield 1 + + def helper() -> int: + return 0 + + yield helper() + + def k2(): """ - yield url + docstring + """ + url = """ +https://example.org + """ + yield url - @decorator - def l3(): - return + def l2(): + return - assert not is_generator_with_return_value(top_level_return_none) - assert not is_generator_with_return_value(f3) - assert not is_generator_with_return_value(g3) - assert not is_generator_with_return_value(h3) - assert not is_generator_with_return_value(i3) - assert not is_generator_with_return_value(j3) # not recursive - assert not is_generator_with_return_value(k3) # not recursive - assert not is_generator_with_return_value(l3) + assert not is_generator_with_return_value(top_level_return_none) + assert not is_generator_with_return_value(f2) + assert not is_generator_with_return_value(g2) + assert not is_generator_with_return_value(h2) + assert not is_generator_with_return_value(i2) + assert not is_generator_with_return_value(j2) # not recursive + assert not is_generator_with_return_value(k2) # not recursive + assert not is_generator_with_return_value(l2) - with warnings.catch_warnings(): - warnings.simplefilter("error", UserWarning) - warn_on_generator_with_return_value(mock_spider, top_level_return_none) - warn_on_generator_with_return_value(mock_spider, f3) - warn_on_generator_with_return_value(mock_spider, g3) - warn_on_generator_with_return_value(mock_spider, h3) - warn_on_generator_with_return_value(mock_spider, i3) - warn_on_generator_with_return_value(mock_spider, j3) - warn_on_generator_with_return_value(mock_spider, k3) - warn_on_generator_with_return_value(mock_spider, l3) + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) + warn_on_generator_with_return_value(mock_spider, f2) + warn_on_generator_with_return_value(mock_spider, g2) + warn_on_generator_with_return_value(mock_spider, h2) + warn_on_generator_with_return_value(mock_spider, i2) + warn_on_generator_with_return_value(mock_spider, j2) + warn_on_generator_with_return_value(mock_spider, k2) + warn_on_generator_with_return_value(mock_spider, l2) - @mock.patch( - "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error - ) - def test_indentation_error(self, mock_spider): - with pytest.warns(UserWarning, match="Unable to determine"): - warn_on_generator_with_return_value(mock_spider, top_level_return_none) - def test_partial(self): - def cb(arg1, arg2): - yield {} +def test_generators_return_none_with_decorator(mock_spider): + def decorator(func): + def inner_func(): + func() - partial_cb = partial(cb, arg1=42) - assert not is_generator_with_return_value(partial_cb) + return inner_func - def test_warn_on_generator_with_return_value_settings_disabled(self): - class MockSettings: - def __init__(self, settings_dict=None): - self.settings_dict = settings_dict or {} + @decorator + def f3(): + yield 1 - def getbool(self, name, default=False): - return self.settings_dict.get(name, default) + @decorator + def g3(): + yield 1 - class MockSpider: - def __init__(self): - self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False}) + @decorator + def h3(): + yield 1 - spider = MockSpider() + @decorator + def i3(): + yield 1 + yield from generator_that_returns_stuff() - def gen_with_return(): - yield 1 - return "value" + @decorator + def j3(): + yield 1 - with warnings.catch_warnings(): - warnings.simplefilter("error", UserWarning) - warn_on_generator_with_return_value(spider, gen_with_return) + def helper() -> int: + return 0 - spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True + yield helper() - with pytest.warns(UserWarning, match="is a generator"): - warn_on_generator_with_return_value(spider, gen_with_return) + @decorator + def k3(): + """ + docstring + """ + url = """ +https://example.org + """ + yield url + + @decorator + def l3(): + return + + assert not is_generator_with_return_value(top_level_return_none) + assert not is_generator_with_return_value(f3) + assert not is_generator_with_return_value(g3) + assert not is_generator_with_return_value(h3) + assert not is_generator_with_return_value(i3) + assert not is_generator_with_return_value(j3) # not recursive + assert not is_generator_with_return_value(k3) # not recursive + assert not is_generator_with_return_value(l3) + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) + warn_on_generator_with_return_value(mock_spider, f3) + warn_on_generator_with_return_value(mock_spider, g3) + warn_on_generator_with_return_value(mock_spider, h3) + warn_on_generator_with_return_value(mock_spider, i3) + warn_on_generator_with_return_value(mock_spider, j3) + warn_on_generator_with_return_value(mock_spider, k3) + warn_on_generator_with_return_value(mock_spider, l3) + + +@mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error) +def test_indentation_error(mock_spider): + with pytest.warns(UserWarning, match="Unable to determine"): + warn_on_generator_with_return_value(mock_spider, top_level_return_none) + + +def test_partial() -> None: + def cb(arg1, arg2): + yield {} + + partial_cb = partial(cb, arg1=42) + assert not is_generator_with_return_value(partial_cb) + + +def test_warn_on_generator_with_return_value_settings_disabled() -> None: + class MockSettings: + def __init__(self, settings_dict: dict[str, Any] | None = None): + self.settings_dict = settings_dict or {} + + def getbool(self, name, default=False): + return self.settings_dict.get(name, default) + + class MockSpider: + def __init__(self) -> None: + self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False}) + + spider = MockSpider() + + def gen_with_return(): + yield 1 + return "value" + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + warn_on_generator_with_return_value(spider, gen_with_return) # type: ignore[arg-type] + + spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True + + with pytest.warns(UserWarning, match="is a generator"): + warn_on_generator_with_return_value(spider, gen_with_return) # type: ignore[arg-type] diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 5333a55cb..7eade6463 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -1,14 +1,20 @@ +from __future__ import annotations + import os from pathlib import Path +from typing import TYPE_CHECKING import pytest from scrapy.utils.misc import set_environ from scrapy.utils.project import data_path, get_project_settings +if TYPE_CHECKING: + from collections.abc import Generator + @pytest.fixture -def proj_path(tmp_path): +def proj_path(tmp_path: Path) -> Generator[Path]: prev_dir = Path.cwd() project_dir = tmp_path @@ -21,7 +27,7 @@ def proj_path(tmp_path): os.chdir(prev_dir) -def test_data_path_outside_project(): +def test_data_path_outside_project() -> None: assert str(Path(".scrapy", "somepath")) == data_path("somepath") abspath = str(Path(os.path.sep, "absolute", "path")) assert abspath == data_path(abspath) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 2e8047e2d..099b5ccc2 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -4,7 +4,7 @@ import functools import operator import platform import sys -from typing import TYPE_CHECKING, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar import pytest @@ -22,8 +22,7 @@ from scrapy.utils.python import ( from tests.utils.decorators import coroutine_test if TYPE_CHECKING: - from collections.abc import Iterable, Mapping - + from collections.abc import AsyncIterator, Iterable, Mapping _KT = TypeVar("_KT") _VT = TypeVar("_VT") @@ -31,22 +30,22 @@ _VT = TypeVar("_VT") class TestMutableAsyncChain: @staticmethod - async def g1(): + async def g1() -> AsyncIterator[int]: for i in range(3): yield i @staticmethod - async def g2(): + async def g2() -> AsyncIterator[int]: return yield @staticmethod - async def g3(): + async def g3() -> AsyncIterator[int]: for i in range(7, 10): yield i @staticmethod - async def g4(): + async def g4() -> AsyncIterator[int]: for i in range(3, 5): yield i 1 / 0 @@ -85,7 +84,7 @@ class TestToUnicode: def test_converting_a_strange_object_should_raise_type_error(self): with pytest.raises(TypeError): - to_unicode(423) + to_unicode(423) # type: ignore[arg-type] def test_errors_argument(self): assert to_unicode(b"a\xedb", "utf-8", errors="replace") == "a\ufffdb" @@ -103,7 +102,7 @@ class TestToBytes: def test_converting_a_strange_object_should_raise_type_error(self): with pytest.raises(TypeError): - to_bytes(pytest) + to_bytes(pytest) # type: ignore[arg-type] def test_errors_argument(self): assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b" @@ -112,10 +111,10 @@ class TestToBytes: def test_memoizemethod_noargs(): class A: @memoizemethod_noargs - def cached(self): + def cached(self) -> object: return object() - def noncached(self): + def noncached(self) -> object: return object() a = A() @@ -150,7 +149,7 @@ def test_get_func_args(): pass class A: - def __init__(self, a, b, c): + def __init__(self, a: int, b: int, c: int): pass def method(self, a, b, c): @@ -191,6 +190,25 @@ def test_get_func_args(): ] +@pytest.mark.skipif( + sys.version_info < (3, 14), + reason="annotations are only lazily evaluated since Python 3.14 (PEP 649)", +) +def test_get_func_args_unresolvable_annotations(): + # dont_inherit=True, or the module's future import stringizes the annotations + namespace: dict[str, Any] = {} + exec( # pylint: disable=exec-used + compile( + "def f(a: OnlyAtTypeCheckingTime, b: int = 1) -> OnlyAtTypeCheckingTime: pass", + "", + "exec", + dont_inherit=True, + ), + namespace, + ) + assert get_func_args(namespace["f"]) == ["a", "b"] + + @pytest.mark.parametrize( ("value", "expected"), [ diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py index 7d39a478e..44cb5c306 100644 --- a/tests/test_utils_reactor.py +++ b/tests/test_utils_reactor.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import pytest diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 1642a932b..935447bc4 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -2,7 +2,7 @@ from __future__ import annotations import json from hashlib import sha1 -from typing import Any +from typing import TYPE_CHECKING, Any, Protocol from weakref import WeakKeyDictionary import pytest @@ -17,6 +17,9 @@ from scrapy.utils.request import ( ) from scrapy.utils.test import get_crawler +if TYPE_CHECKING: + from collections.abc import Iterable + @pytest.mark.parametrize( ("r", "expected"), @@ -56,8 +59,18 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None: request_httprepr(r) +class _FingerprintFunction(Protocol): + def __call__( + self, + request: Request, + *, + include_headers: Iterable[bytes | str] | None = None, + keep_fragments: bool = False, + ) -> bytes: ... + + class TestFingerprint: - function: staticmethod[[Request], bytes] = staticmethod(fingerprint) + function: _FingerprintFunction = staticmethod(fingerprint) cache: ( WeakKeyDictionary[ Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes] @@ -261,6 +274,7 @@ class TestRequestFingerprinter: def test_fingerprint(self): crawler = get_crawler() request = Request("https://example.com") + assert crawler.request_fingerprinter assert crawler.request_fingerprinter.fingerprint(request) == fingerprint( request ) @@ -277,6 +291,7 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) + assert crawler.request_fingerprinter r1 = Request("http://www.example.com", headers={"X-ID": "1"}) fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com", headers={"X-ID": "2"}) @@ -285,9 +300,9 @@ class TestCustomRequestFingerprinter: def test_dont_canonicalize(self): class RequestFingerprinter: - cache = WeakKeyDictionary() + cache: WeakKeyDictionary[Request, bytes] = WeakKeyDictionary() - def fingerprint(self, request): + def fingerprint(self, request: Request) -> bytes: if request not in self.cache: fp = sha1() fp.update(to_bytes(request.url)) @@ -299,6 +314,7 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) + assert crawler.request_fingerprinter r1 = Request("http://www.example.com?a=1&a=2") fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com?a=2&a=1") @@ -317,6 +333,7 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) + assert crawler.request_fingerprinter r1 = Request("http://www.example.com") fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com", meta={"fingerprint": "a"}) @@ -348,6 +365,7 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) + assert crawler.request_fingerprinter request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) assert fingerprint == settings["FINGERPRINT"] @@ -457,3 +475,14 @@ class TestRequestToCurl: " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=1'" ) self._test_request(request_object, expected_curl_command) + + def test_request_to_curl_method(self) -> None: + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + 'curl -X POST https://www.httpbin.org/post --data-raw \'{"foo": "bar"}\'' + ) + assert request_object.to_curl() == expected_curl_command diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 146cdb802..608b2bbd9 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from pathlib import Path from time import process_time from urllib.parse import urlparse @@ -15,7 +17,7 @@ from scrapy.utils.response import ( ) -def _read_browser_output(burl: str): +def _read_browser_output(burl: str) -> bytes: path = urlparse(burl).path if not path or not Path(path).exists(): path = burl.replace("file://", "") @@ -224,7 +226,7 @@ def test_open_in_browser_redos_head(): (b"real", b"real"), ], ) -def test_remove_html_comments(input_body, output_body): +def test_remove_html_comments(input_body: bytes, output_body: bytes) -> None: assert _remove_html_comments(input_body) == output_body diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py index 2e6a790f8..2702c2cce 100644 --- a/tests/test_utils_serialize.py +++ b/tests/test_utils_serialize.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import dataclasses import datetime import json diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index dbb9caf2a..3109bc149 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -21,9 +21,6 @@ from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from collections.abc import Callable -if TYPE_CHECKING: - from collections.abc import Callable - class TestSendCatchLog: # whether the function being tested returns exceptions or failures diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index ac57e1739..9f2dce6d6 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from scrapy.exceptions import ScrapyDeprecationWarning diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 4515ce36e..076edf3db 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -1,7 +1,14 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + from scrapy.utils.template import render_templatefile +if TYPE_CHECKING: + from pathlib import Path -def test_simple_render(tmp_path): + +def test_simple_render(tmp_path: Path) -> None: context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"} template = "from ${project_name}.spiders.${name} import ${classname}" rendered = "from proj.spiders.spi import TheSpider" diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 2334c76e9..5458aa603 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import sys from io import StringIO from unittest import mock @@ -46,13 +48,13 @@ Bar 1 oldest: 0s ago @mock.patch("sys.stdout", new_callable=StringIO) -def test_print_live_refs_empty(stdout): +def test_print_live_refs_empty(stdout: StringIO) -> None: trackref.print_live_refs() assert stdout.getvalue() == "Live References\n\n\n" @mock.patch("sys.stdout", new_callable=StringIO) -def test_print_live_refs_with_objects(stdout): +def test_print_live_refs_with_objects(stdout: StringIO) -> None: o1 = Foo() # noqa: F841 trackref.print_live_refs() assert ( diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 5b98131a1..d9d162c23 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from scrapy.linkextractors import IGNORED_EXTENSIONS @@ -191,7 +193,7 @@ def test_guess_scheme(url: str, expected: str): ), ], ) -def test_guess_scheme_skipped(url: str, expected: str, reason: str): +def test_guess_scheme_skipped(url: str, expected: str, reason: str) -> None: pytest.skip(reason) diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index b0632a7ea..b27c5ade7 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import os from pathlib import Path from typing import TYPE_CHECKING @@ -8,8 +7,6 @@ from typing import TYPE_CHECKING from twisted.internet.defer import Deferred from scrapy.settings import Settings, default_settings -from scrapy.utils.asyncio import is_asyncio_available -from scrapy.utils.defer import maybe_deferred_to_future if TYPE_CHECKING: from collections.abc import Callable @@ -23,13 +20,6 @@ def twisted_sleep(seconds: float): return d -async def async_sleep(seconds: float) -> None: - if is_asyncio_available(): - await asyncio.sleep(seconds) - else: - await maybe_deferred_to_future(twisted_sleep(seconds)) - - def get_script_run_env() -> dict[str, str]: """Return a OS environment dict suitable to run scripts shipped with tests.""" diff --git a/tests/utils/bases/commands.py b/tests/utils/bases/commands.py index 594544c83..55ef686fa 100644 --- a/tests/utils/bases/commands.py +++ b/tests/utils/bases/commands.py @@ -32,3 +32,9 @@ class TestProjectBase: proj_path = tmp_path / self.project_name copytree(_proj_path_cached, proj_path) return proj_path + + @staticmethod + def _append_settings(proj_mod_path: Path, text: str) -> None: + """Add text to the end of the project settings.py.""" + with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f: + f.write(text) diff --git a/tests/utils/bases/http_request.py b/tests/utils/bases/http_request.py index c255b5e4c..2b712f3ab 100644 --- a/tests/utils/bases/http_request.py +++ b/tests/utils/bases/http_request.py @@ -3,9 +3,11 @@ from abc import ABC, abstractmethod from typing import Any import pytest +from twisted.python.failure import Failure -from scrapy.http import Headers, Request +from scrapy.http import Headers, Request, Response from scrapy.http.request import NO_CALLBACK +from scrapy.utils.request import request_to_curl class TestRequestBase(ABC): @@ -21,15 +23,15 @@ class TestRequestBase(ABC): def test_init(self): # Request requires url in the __init__ method with pytest.raises(TypeError): - self.request_class() + self.request_class() # type: ignore[call-arg] # url argument must be basestring with pytest.raises(TypeError): - self.request_class(123) + self.request_class(123) # type: ignore[arg-type] # priority argument must be an integer with pytest.raises(TypeError, match="Request priority not an integer"): - self.request_class("http://www.example.com", priority="1") + self.request_class("http://www.example.com", priority="1") # type: ignore[arg-type] r = self.request_class("http://www.example.com") assert isinstance(r.url, str) @@ -204,14 +206,17 @@ class TestRequestBase(ABC): def test_copy(self): """Test Request copy""" - def somecallback(): + def somecallback(response: Response) -> None: + pass + + def someerrback(failure: Failure) -> None: pass r1 = self.request_class( "http://www.example.com", flags=["f1", "f2"], callback=somecallback, - errback=somecallback, + errback=someerrback, ) r1.meta["foo"] = "bar" r1.cb_kwargs["key"] = "value" @@ -219,7 +224,7 @@ class TestRequestBase(ABC): # make sure callbaclks are copied assert r1.callback is somecallback - assert r1.errback is somecallback + assert r1.errback is someerrback assert r2.callback is r1.callback assert r2.errback is r1.errback @@ -250,7 +255,7 @@ class TestRequestBase(ABC): def test_copy_inherited_classes(self): """Test Request children copies preserve their class""" - class CustomRequest(self.request_class): + class CustomRequest(self.request_class): # type: ignore[misc,name-defined] pass r1 = CustomRequest("http://www.example.com") @@ -282,7 +287,9 @@ class TestRequestBase(ABC): assert r4.dont_filter is False # the cls argument allows changing the resulting class - custom_request_cls = type("CustomRequest", (self.request_class,), {}) + custom_request_cls: type[Request] = type( + "CustomRequest", (self.request_class,), {} + ) r5 = r1.replace(cls=custom_request_cls) assert isinstance(r5, custom_request_cls) assert r5.url == r1.url @@ -294,33 +301,36 @@ class TestRequestBase(ABC): def test_immutable_attributes(self): r = self.request_class("http://example.com") with pytest.raises(AttributeError): - r.url = "http://example2.com" + r.url = "http://example2.com" # type: ignore[misc] with pytest.raises(AttributeError): - r.body = "xxx" + r.body = "xxx" # type: ignore[misc,assignment] def test_callback_and_errback(self): - def a_function(): + def a_callback(response: Response) -> None: + pass + + def an_errback(failure: Failure) -> None: pass r1 = self.request_class("http://example.com") assert r1.callback is None assert r1.errback is None - r2 = self.request_class("http://example.com", callback=a_function) - assert r2.callback is a_function + r2 = self.request_class("http://example.com", callback=a_callback) + assert r2.callback is a_callback assert r2.errback is None - r3 = self.request_class("http://example.com", errback=a_function) + r3 = self.request_class("http://example.com", errback=an_errback) assert r3.callback is None - assert r3.errback is a_function + assert r3.errback is an_errback r4 = self.request_class( url="http://example.com", - callback=a_function, - errback=a_function, + callback=a_callback, + errback=an_errback, ) - assert r4.callback is a_function - assert r4.errback is a_function + assert r4.callback is a_callback + assert r4.errback is an_errback r5 = self.request_class( url="http://example.com", @@ -328,18 +338,18 @@ class TestRequestBase(ABC): errback=NO_CALLBACK, ) assert r5.callback is NO_CALLBACK - assert r5.errback is NO_CALLBACK + assert r5.errback is NO_CALLBACK # type: ignore[comparison-overlap] def test_callback_and_errback_type(self): with pytest.raises(TypeError): - self.request_class("http://example.com", callback="a_function") + self.request_class("http://example.com", callback="a_function") # type: ignore[arg-type] with pytest.raises(TypeError): - self.request_class("http://example.com", errback="a_function") + self.request_class("http://example.com", errback="a_function") # type: ignore[arg-type] with pytest.raises(TypeError): self.request_class( url="http://example.com", - callback="a_function", - errback="a_function", + callback="a_function", # type: ignore[arg-type] + errback="a_function", # type: ignore[arg-type] ) def test_setters(self): @@ -488,3 +498,11 @@ class TestRequestBase(ABC): 'curl -X PATCH "http://example.org" --foo -z', ignore_unknown_options=False, ) + + def test_to_curl(self): + # Note: more curated tests regarding curl conversion are in + # `test_utils_request.py` + r = self.request_class( + "http://www.example.com/", method="POST", body=b"foo=bar" + ) + assert r.to_curl() == request_to_curl(r) diff --git a/tests/utils/bases/http_response.py b/tests/utils/bases/http_response.py index 2fbf6527a..78e14e7b0 100644 --- a/tests/utils/bases/http_response.py +++ b/tests/utils/bases/http_response.py @@ -7,7 +7,7 @@ import pytest from w3lib.encoding import resolve_encoding from scrapy.exceptions import NotSupported -from scrapy.http import Headers, Request, Response +from scrapy.http import Headers, Request, Response, TextResponse from scrapy.link import Link from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS from tests import get_testdata @@ -15,6 +15,8 @@ from tests import get_testdata if TYPE_CHECKING: from collections.abc import Iterable + from parsel import Selector + class TestResponseBase(ABC): @property @@ -25,14 +27,14 @@ class TestResponseBase(ABC): def test_init(self): # Response requires url in the constructor with pytest.raises(TypeError): - self.response_class() + self.response_class() # type: ignore[call-arg] assert isinstance( self.response_class("http://example.com/"), self.response_class ) with pytest.raises(TypeError): - self.response_class(b"http://example.com") + self.response_class(b"http://example.com") # type: ignore[arg-type] with pytest.raises(TypeError): - self.response_class(url="http://example.com", body={}) + self.response_class(url="http://example.com", body={}) # type: ignore[arg-type] # body can be str or None assert isinstance( self.response_class("http://example.com/", body=b""), @@ -67,12 +69,12 @@ class TestResponseBase(ABC): r = self.response_class("http://www.example.com", status=301) assert r.status == 301 - r = self.response_class("http://www.example.com", status="301") + r = self.response_class("http://www.example.com", status="301") # type: ignore[arg-type] assert r.status == 301 with pytest.raises(ValueError, match=r"invalid literal for int\(\)"): - self.response_class("http://example.com", status="lala200") + self.response_class("http://example.com", status="lala200") # type: ignore[arg-type] - def test_copy(self): + def test_copy(self) -> None: """Test Response copy""" r1 = self.response_class("http://www.example.com", body=b"Some body") @@ -121,7 +123,7 @@ class TestResponseBase(ABC): def test_copy_inherited_classes(self): """Test Response children copies preserve their class""" - class CustomResponse(self.response_class): + class CustomResponse(self.response_class): # type: ignore[misc,name-defined] pass r1 = CustomResponse("http://www.example.com") @@ -129,7 +131,7 @@ class TestResponseBase(ABC): assert isinstance(r2, CustomResponse) - def test_replace(self): + def test_replace(self) -> None: """Test Response.replace() method""" hdrs = Headers({"key": "value"}) r1 = self.response_class("http://www.example.com") @@ -146,7 +148,9 @@ class TestResponseBase(ABC): assert r4.body == b"" assert not r4.flags - def _assert_response_values(self, response, encoding, body): + def _assert_response_values( + self, response: TextResponse, encoding: str, body: str | bytes + ) -> None: if isinstance(body, str): body_unicode = body body_bytes = body.encode(encoding) @@ -160,15 +164,15 @@ class TestResponseBase(ABC): assert response.body == body_bytes assert response.text == body_unicode - def _assert_response_encoding(self, response, encoding): + def _assert_response_encoding(self, response: TextResponse, encoding: str) -> None: assert response.encoding == resolve_encoding(encoding) def test_immutable_attributes(self): r = self.response_class("http://example.com") with pytest.raises(AttributeError): - r.url = "http://example2.com" + r.url = "http://example2.com" # type: ignore[misc] with pytest.raises(AttributeError): - r.body = "xxx" + r.body = "xxx" # type: ignore[misc,assignment] def test_setter_mutable_lazy_loading(self): """Mutable attributes are set internally to None only until they are @@ -256,7 +260,7 @@ class TestResponseBase(ABC): def test_follow_None_url(self): r = self.response_class("http://example.com") with pytest.raises(ValueError, match="url can't be None"): - r.follow(None) + r.follow(None) # type: ignore[arg-type] def test_follow_None_encoding(self): r = self.response_class("http://example.com") @@ -325,20 +329,20 @@ class TestResponseBase(ABC): r = self.response_class("http://example.com") if self.response_class == Response: with pytest.raises(TypeError): - list(r.follow_all(urls=None)) + list(r.follow_all(urls=None)) # type: ignore[arg-type] with pytest.raises(TypeError): - list(r.follow_all(urls=12345)) + list(r.follow_all(urls=12345)) # type: ignore[arg-type] with pytest.raises(ValueError, match="url can't be None"): - list(r.follow_all(urls=[None])) + list(r.follow_all(urls=[None])) # type: ignore[list-item] else: with pytest.raises( ValueError, match="Please supply exactly one of the following arguments" ): - list(r.follow_all(urls=None)) + list(r.follow_all(urls=None)) # type: ignore[arg-type] with pytest.raises(TypeError): - list(r.follow_all(urls=12345)) + list(r.follow_all(urls=12345)) # type: ignore[arg-type] with pytest.raises(ValueError, match="url can't be None"): - list(r.follow_all(urls=[None])) + list(r.follow_all(urls=[None])) # type: ignore[list-item] @pytest.mark.xfail( not W3LIB_STRIPS_URLS, @@ -384,14 +388,14 @@ class TestResponseBase(ABC): def _assert_followed_url( self, - follow_obj: str | Link, + follow_obj: str | Link | Selector, target_url: str, response: Response | None = None, encoding: str | None = None, ) -> None: if response is None: response = self._links_response() - req = response.follow(follow_obj) + req = response.follow(follow_obj) # type: ignore[arg-type] assert req.url == target_url if encoding is not None: assert req.encoding == encoding diff --git a/tests/utils/cmdline.py b/tests/utils/cmdline.py index 62dff3d4c..095cb17a7 100644 --- a/tests/utils/cmdline.py +++ b/tests/utils/cmdline.py @@ -46,3 +46,16 @@ def write_recording_editor(editor: Path) -> None: open (its last argument) into the file given as its first argument.""" editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8") editor.chmod(0o755) + + +def write_recording_browser(browser: Path, recorded: Path) -> None: + """Create an executable browser script that writes the URL it is asked to + open into *recorded*. + + ``webbrowser`` only passes the URL to the command from the ``BROWSER`` + environment variable, hence the hardcoded output path. + """ + browser.write_text( + f'#!/bin/sh\nprintf "%s" "$1" > "{recorded}"\n', encoding="utf-8" + ) + browser.chmod(0o755) diff --git a/tests/utils/media_pipelines.py b/tests/utils/media_pipelines.py index 283c95940..7e013b797 100644 --- a/tests/utils/media_pipelines.py +++ b/tests/utils/media_pipelines.py @@ -3,6 +3,12 @@ from __future__ import annotations from typing import Any from scrapy.http.request import NO_CALLBACK, Request +from scrapy.pipelines.media import MediaPipeline +from scrapy.utils.spider import DefaultSpider + +# required by persist_file() and stat_file(), but as some stores don't use the argument +# we can pass this singleton to keep type hints correct +DUMMY_SPIDER_INFO = MediaPipeline.SpiderInfo(DefaultSpider()) async def mocked_download_func(request: Request) -> Any: diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing index a431091d5..4ff562dff 100644 --- a/tests_typing/test_http_request.mypy-testing +++ b/tests_typing/test_http_request.mypy-testing @@ -16,7 +16,7 @@ class MyRequest2(Request): @pytest.mark.mypy_testing def mypy_test_headers() -> None: - Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Mapping[bytes, Any] | Iterable[tuple[str | bytes, Any]] | None" Request("data:,", headers=None) Request("data:,", headers={}) Request("data:,", headers=[]) diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing index d497c2470..1c157328c 100644 --- a/tests_typing/test_http_response.mypy-testing +++ b/tests_typing/test_http_response.mypy-testing @@ -7,7 +7,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse @pytest.mark.mypy_testing def mypy_test_headers() -> None: - Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Mapping[bytes, Any] | Iterable[tuple[str | bytes, Any]] | None" Response("data:,", headers=None) Response("data:,", headers={}) Response("data:,", headers=[]) diff --git a/tox.ini b/tox.ini index e10a7cc0c..ac32064a6 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,8 @@ [tox] requires = - sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 + sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10 + tox-uv envlist = pre-commit pylint @@ -30,6 +31,7 @@ envlist = botocore pypy3 pypy3-extra-deps + benchmark minversion = 1.7.0 [test-requirements] @@ -109,7 +111,8 @@ commands = pre-commit run {posargs:--all-files} [testenv:pylint] -basepython = python3 +# Some checks are Python-version-dependent, so pin the version used in CI. +basepython = python3.14 deps = {[testenv:extra-deps]deps} pylint==4.0.6 @@ -189,8 +192,8 @@ deps = brotlicffi==1.2.0.0; implementation_name == "pypy" google-cloud-storage==1.29.0 httpx2[http2,socks]==2.0.0 - ipython==7.1.0 - ptpython==2.0.1 + ipython==8.15.0 + ptpython==3.0.23 robotexclusionrulesparser==1.6.2 uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy" zstandard==0.16.0; implementation_name != "pypy" @@ -317,3 +320,20 @@ setenv = {[min]setenv} commands = pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore + + +# CPU benchmarks, tracked on CodSpeed. +# +# pytest-twisted is left out on purpose: benchmarked code must be callable +# synchronously, so tests/benchmarks drives the reactor itself. + +[testenv:benchmark] +basepython = python3.14 +deps = + pytest >= 8.4.1 + pytest-codspeed +passenv = + *codspeed* + *ci* +commands = + pytest {posargs:tests/benchmarks} --codspeed --codspeed-mode=simulation