RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into faster-shutdown-2

This commit is contained in:
Adrian Chaves 2026-07-31 17:13:27 +02:00
parent 62a2753c65 1f03fbc17e
commit 4a77c20c7b
124 changed files with 4109 additions and 926 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
.github/workflows/codspeed.yml vendored Normal file
View File

@ -0,0 +1,45 @@
---
name: codspeed
on:
push:
branches:
- master
pull_request:
paths:
- scrapy/**
- tests/benchmarks/**
- .github/workflows/codspeed.yml
- tox.ini
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
permissions: {}
jobs:
benchmark:
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write # OIDC authentication with CodSpeed
steps:
- uses: actions/checkout@v6
- name: Set up Python 3.14
uses: actions/setup-python@v6
with:
python-version: '3.14'
- name: Install dependencies
run: |
pip install --upgrade pip
pip install --upgrade tox
tox -n -e benchmark
- name: Run benchmarks
uses: CodSpeedHQ/action@v4
with:
mode: simulation
run: tox -e benchmark

9
.github/workflows/tests-macos.yml vendored
View File

@ -15,14 +15,18 @@ jobs:
runs-on: macos-latest
timeout-minutes: 60
env:
PYTEST_ADDOPTS: -n auto
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
python-version: ["3.10", "3.11", "3.12", "3.13"]
env:
- TOXENV: py
include:
- python-version: '3.14'
env:
TOXENV: py
coverage: true
- python-version: '3.14'
env:
TOXENV: no-reactor
@ -42,6 +46,7 @@ jobs:
tox
- name: Upload coverage report
if: ${{ matrix.coverage }}
uses: codecov/codecov-action@v5
- name: Upload test results

14
.github/workflows/tests-ubuntu.yml vendored
View File

@ -15,7 +15,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 60
env:
PYTEST_ADDOPTS: -n auto
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
strategy:
fail-fast: false
matrix:
@ -35,12 +35,15 @@ jobs:
- python-version: "3.14"
env:
TOXENV: py
coverage: true
- python-version: "3.14"
env:
TOXENV: default-reactor
coverage: true
- python-version: "3.14"
env:
TOXENV: no-reactor
coverage: true
# pinned due to https://github.com/pypy/pypy/issues/5388
- python-version: pypy3.11-7.3.20
env:
@ -50,12 +53,15 @@ jobs:
- python-version: "3.10.19"
env:
TOXENV: min
coverage: true
- python-version: "3.10.19"
env:
TOXENV: min-default-reactor
coverage: true
- python-version: "3.10.19"
env:
TOXENV: min-no-reactor
coverage: true
# pinned due to https://github.com/pypy/pypy/issues/5388
- python-version: pypy3.11-7.3.20
env:
@ -63,16 +69,20 @@ jobs:
- python-version: "3.10.19"
env:
TOXENV: min-extra-deps
coverage: true
- python-version: "3.10.19"
env:
TOXENV: min-botocore
coverage: true
- python-version: "3.14"
env:
TOXENV: extra-deps
coverage: true
- python-version: "3.14"
env:
TOXENV: no-reactor-extra-deps
coverage: true
# pinned due to https://github.com/pypy/pypy/issues/5388
- python-version: pypy3.11-7.3.20
env:
@ -80,6 +90,7 @@ jobs:
- python-version: "3.14"
env:
TOXENV: botocore
coverage: true
steps:
- uses: actions/checkout@v6
@ -105,6 +116,7 @@ jobs:
tox
- name: Upload coverage report
if: ${{ matrix.coverage }}
uses: codecov/codecov-action@v5
- name: Upload test results

4
.github/workflows/tests-windows.yml vendored
View File

@ -15,7 +15,7 @@ jobs:
runs-on: windows-latest
timeout-minutes: 60
env:
PYTEST_ADDOPTS: -n auto
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
strategy:
fail-fast: false
matrix:
@ -35,6 +35,7 @@ jobs:
- python-version: "3.14"
env:
TOXENV: py
coverage: true
- python-version: "3.14"
env:
TOXENV: default-reactor
@ -69,6 +70,7 @@ jobs:
tox
- name: Upload coverage report
if: ${{ matrix.coverage }}
uses: codecov/codecov-action@v5
- name: Upload test results

2
.pre-commit-config.yaml
View File

@ -27,6 +27,6 @@ repos:
hooks:
- id: sphinx-lint
- repo: https://github.com/scrapy/sphinx-scrapy
rev: 0.8.8
rev: 0.8.9
hooks:
- id: sphinx-scrapy

16
README.rst
View File

@ -5,7 +5,7 @@
:alt: Scrapy
:width: 480px
|version| |python_version| |ubuntu| |macos| |windows| |coverage| |conda| |deepwiki|
|version| |python_version| |tests| |coverage| |conda| |deepwiki|
.. |version| image:: https://img.shields.io/pypi/v/Scrapy.svg
:target: https://pypi.org/pypi/Scrapy
@ -15,17 +15,9 @@
:target: https://pypi.org/pypi/Scrapy
:alt: Supported Python Versions
.. |ubuntu| image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu
:alt: Ubuntu
.. |macos| image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
:alt: macOS
.. |windows| image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows
:alt: Windows
.. |tests| image:: https://img.shields.io/github/check-runs/scrapy/scrapy/master?label=tests
:target: https://github.com/scrapy/scrapy/actions?query=branch%3Amaster
:alt: Tests
.. |coverage| image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg
:target: https://codecov.io/github/scrapy/scrapy?branch=master

3
conftest.py
View File

@ -54,6 +54,9 @@ if not H2_ENABLED:
if find_spec("httpx2") is None and find_spec("httpx") is None:
collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")
if find_spec("pytest_codspeed") is None:
collect_ignore.append("tests/benchmarks")
def pytest_addoption(parser, pluginmanager):
if pluginmanager.hasplugin("twisted"):

2
docs/faq.rst
View File

@ -97,7 +97,7 @@ handler documentation.
How can I scrape an item with attributes in different pages?
------------------------------------------------------------
See :ref:`topics-request-response-ref-request-callback-arguments`.
See :ref:`callback-data`.
How can I simulate a user login in my spider?
---------------------------------------------

2
docs/intro/tutorial.rst
View File

@ -769,7 +769,7 @@ crawlers on top of it.
Also, a common pattern is to build an item with data from more than one page,
using a :ref:`trick to pass additional data to the callbacks
<topics-request-response-ref-request-callback-arguments>`.
<callback-data>`.
Using spider arguments

2
docs/requirements.in
View File

@ -5,4 +5,4 @@ sphinx
sphinx-notfound-page
sphinx-rtd-theme
sphinx-rtd-dark-mode
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.9

2
docs/requirements.txt
View File

@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0
# via
# -r docs/requirements.in
# sphinx-rtd-dark-mode
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@912ed0507405e16ac60a47dd08195a1cd0ced984
# via -r docs/requirements.in
sphinx-sitemap==2.9.0
# via sphinx-scrapy

1
docs/topics/asyncio.rst
View File

@ -267,6 +267,7 @@ Here are some examples of APIs and patterns that need a replacement:
Scrapy provides unified helpers for some of these examples:
.. autofunction:: scrapy.utils.asyncio.sleep
.. autofunction:: scrapy.utils.asyncio.call_later
.. autofunction:: scrapy.utils.asyncio.create_looping_call
.. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall

13
docs/topics/commands.rst
View File

@ -114,8 +114,8 @@ some usage help and the available commands::
scrapy <command> [options] [args]
Available commands:
crawl Run a spider
fetch Fetch a URL using the Scrapy downloader
runspider Run a spider from a Python file, no project required
[...]
The first line will print the currently active project if you're inside a
@ -263,7 +263,9 @@ crawl
* Syntax: ``scrapy crawl <spider>``
* Requires project: *yes*
Start crawling using a spider.
Start crawling using the spider with the given :attr:`~scrapy.Spider.name`,
which must be one of those that :command:`list` reports. To run a spider from a
file instead, use :command:`runspider`.
Supported options:
@ -571,8 +573,9 @@ runspider
* Syntax: ``scrapy runspider <spider_file.py>``
* Requires project: *no*
Run a spider self-contained in a Python file, without having to create a
project.
Run the spider defined in the given Python file, without requiring a project.
Supported options: the same as :command:`crawl`.
Example usage::
@ -665,6 +668,8 @@ Example:
COMMANDS_MODULE = "mybot.commands"
.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
Register commands via setup.py entry points

4
docs/topics/coroutines.rst
View File

@ -21,7 +21,9 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
.. versionadded:: 2.13
- :class:`~scrapy.Request` callbacks.
- :class:`~scrapy.Request` :ref:`callbacks <callbacks>`, which may
also be defined as :term:`asynchronous generators <asynchronous
generator>`.
- The :meth:`process_item` method of
:ref:`item pipelines <topics-item-pipeline>`.

4
docs/topics/downloader-middleware.rst
View File

@ -564,6 +564,10 @@ defines the methods described below.
Return response if present in cache, or ``None`` otherwise.
If this method raises an exception, e.g. because the cache entry is
corrupted, the middleware logs a warning and handles the request as a
cache miss.
:param spider: the spider which generated the request
:type spider: :class:`~scrapy.Spider` object

10
docs/topics/exporters.rst
View File

@ -211,6 +211,16 @@ BaseItemExporter
- ``None`` (all fields [2]_, default)
Fields are exported in declaration order, i.e. the order in which
they are defined in the :ref:`item class <item-types>`. For
:class:`dict` items, which have no declared fields, the key order of
each item is used instead.
.. versionchanged:: VERSION
Fields of non-\ :class:`dict` items used to be exported in the
order in which they had been populated, except in
:class:`CsvItemExporter`, which has always used declaration order.
- A list of fields:
.. code-block:: python

21
docs/topics/extensions.rst
View File

@ -136,18 +136,10 @@ Core Stats extension
Enable the collection of core statistics, provided the stats collection is
enabled (see :ref:`topics-stats`).
The following stats are collected:
* ``start_time``: start date/time of the crawl (:class:`~datetime.datetime`).
* ``finish_time``: end date/time of the crawl (:class:`~datetime.datetime`).
* ``elapsed_time_seconds``: total crawl duration in seconds (:class:`float`).
* ``finish_reason``: the closing reason string (e.g. ``"finished"``,
``"closespider_timeout"``).
* ``item_scraped_count``: total number of items that passed all pipelines.
* ``item_dropped_count``: total number of items dropped by a pipeline.
* ``item_dropped_reasons_count/<ExceptionName>``: per-exception drop count
(e.g. ``item_dropped_reasons_count/DropItem``).
* ``response_received_count``: total number of HTTP responses received.
The following stats are collected: :stat:`elapsed_time_seconds`,
:stat:`finish_reason`, :stat:`finish_time`, :stat:`item_dropped_count`,
:stat:`item_dropped_reasons_count/{exception}`, :stat:`item_scraped_count`,
:stat:`response_received_count`, :stat:`start_time`.
Log Count extension
~~~~~~~~~~~~~~~~~~~
@ -190,7 +182,7 @@ Monitors the memory used by the Scrapy process that runs the spider and:
1. sends a :signal:`memusage_warning_reached` signal when it exceeds
:setting:`MEMUSAGE_WARNING_MB`
2. closes the spider with the `"memusage_exceeded"` reason when it exceeds
2. closes the spider with the ``"memusage_exceeded"`` reason when it exceeds
:setting:`MEMUSAGE_LIMIT_MB`
This extension is enabled by the :setting:`MEMUSAGE_ENABLED` setting and
@ -214,7 +206,8 @@ An extension for debugging memory usage. It collects information about:
* objects left alive that shouldn't. For more info, see :ref:`topics-leaks-trackrefs`
To enable this extension, turn on the :setting:`MEMDEBUG_ENABLED` setting. The
info will be stored in the stats.
info will be stored in the :stat:`memdebug/gc_garbage_count` and
:stat:`memdebug/live_refs/{cls}` stats.
.. _topics-extensions-ref-spiderstate:

5
docs/topics/feed-exports.rst
View File

@ -218,12 +218,13 @@ passed through the following settings:
.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html
You can also define a custom ACL, custom endpoint, and region name for exported
feeds using these settings:
You can also define a custom ACL, custom endpoint, region name and connection
pool size for exported feeds using these settings:
- :setting:`FEED_STORAGE_S3_ACL`
- :setting:`AWS_ENDPOINT_URL`
- :setting:`AWS_REGION_NAME`
- :setting:`AWS_MAX_POOL_CONNECTIONS`
The default value for the ``overwrite`` key in the :setting:`FEEDS` for this
storage backend is: ``True``.

10
docs/topics/jobs.rst
View File

@ -96,9 +96,13 @@ Request serialization
---------------------
For persistence to work, :class:`~scrapy.Request` objects must be
serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
values passed to their ``__init__`` method, which must be methods of the
running :class:`~scrapy.Spider` class.
serializable with :mod:`pickle`, except for the :ref:`callback
<callbacks>` and :ref:`errback
<errbacks>` values passed to their ``__init__``
method, which must be methods of the running :class:`~scrapy.Spider` class.
Requests that cannot be serialized are kept in memory only: they are still
sent, but they are lost when the crawl is paused.
If you wish to log the requests that couldn't be serialized, you can set the
:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.

3
docs/topics/media-pipeline.rst
View File

@ -268,6 +268,9 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
AWS_USE_SSL = False # or True (None by default)
AWS_VERIFY = False # or True (None by default)
To reuse connections for as many files as you check or upload in parallel, set
:setting:`AWS_MAX_POOL_CONNECTIONS` accordingly.
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
.. _Minio: https://github.com/minio/minio
.. _Zenko CloudServer: https://www.zenko.io/cloudserver/

4
docs/topics/practices.rst
View File

@ -533,8 +533,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
* if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites
directly
* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
super proxy that you can attach your own proxies to.
services like `ProxyMesh`_.
* for HTTPS websites, if blocking appears related to TLS behavior, consider
adjusting the :setting:`DOWNLOAD_TLS_MIN_VERSION` and
:setting:`DOWNLOAD_TLS_MAX_VERSION` settings, since some websites may respond
@ -559,5 +558,4 @@ projects that detects common mistakes and anti-patterns.
.. _ProxyMesh: https://proxymesh.com/
.. _Common Crawl: https://commoncrawl.org/
.. _testspiders: https://github.com/scrapinghub/testspiders
.. _scrapoxy: https://scrapoxy.io/
.. _Zyte API: https://docs.zyte.com/zyte-api/get-started.html

482
docs/topics/request-response.rst
View File

@ -205,10 +205,11 @@ Request objects
Request metadata can also be accessed through the
:attr:`~scrapy.http.Response.meta` attribute of a response.
To pass data from one spider callback to another, consider using
:attr:`cb_kwargs` instead. However, request metadata may be the right
choice in certain scenarios, such as to maintain some debugging data
across all follow-up requests (e.g. the source URL).
To pass your own data from one spider callback to another, use
:attr:`cb_kwargs` instead, see :ref:`callback-data`. However, request
metadata may be the right choice in certain scenarios, such as to
maintain some debugging data across all follow-up requests (e.g. the
source URL).
A common use of request metadata is to define request-specific
parameters for Scrapy components (extensions, middlewares, etc.). For
@ -248,7 +249,7 @@ Request objects
.. method:: Request.copy()
Return a new Request which is a copy of this Request. See also:
:ref:`topics-request-response-ref-request-callback-arguments`.
:ref:`callback-data`.
.. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs, cls])
@ -256,10 +257,12 @@ Request objects
given new values by whichever keyword arguments are specified. The
:attr:`~scrapy.Request.cb_kwargs` and :attr:`~scrapy.Request.meta` attributes are shallow
copied by default (unless new values are given as arguments). See also
:ref:`topics-request-response-ref-request-callback-arguments`.
:ref:`callback-data`.
.. automethod:: from_curl
.. automethod:: to_curl
.. automethod:: to_dict
@ -345,160 +348,6 @@ Other functions related to requests
.. autofunction:: scrapy.utils.httpobj.urlparse_cached
.. _topics-request-response-ref-request-callback-arguments:
Passing additional data to callback functions
---------------------------------------------
The callback of a request is a function that will be called when the response
of that request is downloaded. The callback function will be called with the
downloaded :class:`Response` object as its first argument.
Example:
.. code-block:: python
def parse_page1(self, response):
return scrapy.Request(
"http://www.example.com/some_page.html", callback=self.parse_page2
)
def parse_page2(self, response):
# this would log http://www.example.com/some_page.html
self.logger.info("Visited %s", response.url)
In some cases you may be interested in passing arguments to those callback
functions so you can receive the arguments later, in the second callback.
The following example shows how to achieve this by using the
:attr:`.Request.cb_kwargs` attribute:
.. code-block:: python
def parse(self, response):
request = scrapy.Request(
"http://www.example.com/index.html",
callback=self.parse_page2,
cb_kwargs=dict(main_url=response.url),
)
request.cb_kwargs["foo"] = "bar" # add more arguments for the callback
yield request
def parse_page2(self, response, main_url, foo):
yield dict(
main_url=main_url,
other_url=response.url,
foo=foo,
)
.. caution:: :attr:`.Request.cb_kwargs` was introduced in version ``1.7``.
Prior to that, using :attr:`.Request.meta` was recommended for passing
information around callbacks. After ``1.7``, :attr:`.Request.cb_kwargs`
became the preferred way for handling user information, leaving :attr:`.Request.meta`
for communication with components like middlewares and extensions.
.. _topics-request-response-ref-errbacks:
Using errbacks to catch exceptions in request processing
--------------------------------------------------------
The errback of a request is a function that will be called when an exception
is raise while processing it.
It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
be used to track connection establishment timeouts, DNS errors etc.
Here's an example spider logging all errors and catching some specific
errors if needed:
.. code-block:: python
import scrapy
from scrapy.spidermiddlewares.httperror import HttpError
from twisted.internet.error import DNSLookupError
from twisted.internet.error import TimeoutError, TCPTimedOutError
class ErrbackSpider(scrapy.Spider):
name = "errback_example"
start_urls = [
"http://www.httpbin.org/", # HTTP 200 expected
"http://www.httpbin.org/status/404", # Not found error
"http://www.httpbin.org/status/500", # server issue
"http://www.httpbin.org:12345/", # non-responding host, timeout expected
"https://example.invalid/", # DNS error expected
]
async def start(self):
for u in self.start_urls:
yield scrapy.Request(
u,
callback=self.parse_httpbin,
errback=self.errback_httpbin,
dont_filter=True,
)
def parse_httpbin(self, response):
self.logger.info(f"Got successful response from {response.url}")
# do something useful here...
def errback_httpbin(self, failure):
# log all failures
self.logger.error(repr(failure))
# in case you want to do something special for some errors,
# you may need the failure's type:
if failure.check(HttpError):
# these exceptions come from HttpError spider middleware
# you can get the non-200 response
response = failure.value.response
self.logger.error("HttpError on %s", response.url)
elif failure.check(DNSLookupError):
# this is the original request
request = failure.request
self.logger.error("DNSLookupError on %s", request.url)
elif failure.check(TimeoutError, TCPTimedOutError):
request = failure.request
self.logger.error("TimeoutError on %s", request.url)
.. _errback-cb_kwargs:
Accessing additional data in errback functions
----------------------------------------------
In case of a failure to process the request, you may be interested in
accessing arguments to the callback functions so you can process further
based on the arguments in the errback. The following example shows how to
achieve this by using ``Failure.request.cb_kwargs``:
.. code-block:: python
def parse(self, response):
request = scrapy.Request(
"http://www.example.com/index.html",
callback=self.parse_page2,
errback=self.errback_page2,
cb_kwargs=dict(main_url=response.url),
)
yield request
def parse_page2(self, response, main_url):
pass
def errback_page2(self, failure):
yield dict(
main_url=failure.request.cb_kwargs["main_url"],
)
.. _request-fingerprints:
Request fingerprints
@ -700,6 +549,319 @@ The following built-in Scrapy components have such restrictions:
45-character-long keys must be supported.
.. _callbacks:
Callbacks
=========
A callback is a function that Scrapy calls with the :class:`Response` of a
:class:`~scrapy.Request` once that request has been downloaded, so that you can
extract data from that response and generate additional requests to continue
the crawl:
.. code-block:: python
from scrapy import Request, Spider
class BookSpider(Spider):
name = "books"
async def start(self):
yield Request("https://books.toscrape.com/", callback=self.parse_home)
def parse_home(self, response):
for url in response.css("h3 a::attr(href)").getall():
yield Request(response.urljoin(url), callback=self.parse_book)
def parse_book(self, response):
yield {"title": response.css("h1::text").get()}
Requests may also define an :ref:`errback <errbacks>`, which Scrapy calls
instead of the callback when an exception is raised while processing the
request or its response, e.g. a connection error or, by default, a non-2xx
response.
.. _callback-assignment:
Assigning a callback to a request
---------------------------------
To assign a callback to a request, use the ``callback`` parameter of
:class:`~scrapy.Request`, which sets the :attr:`.Request.callback` attribute:
.. code-block:: python
from scrapy import Request
def parse_home(response): ...
request = Request("https://books.toscrape.com/", callback=parse_home)
Requests with no callback, i.e. with :attr:`~scrapy.Request.callback` set to
``None``, are handled by the :meth:`~scrapy.Spider.parse` method of the spider:
.. code-block:: python
request = Request("https://books.toscrape.com/") # Handled by parse()
If a request is never meant to reach a spider callback, e.g. because a
:ref:`component <topics-components>` sends it and handles its response itself,
assign the special :func:`~scrapy.http.request.NO_CALLBACK` value to it
instead, so that :ref:`downloader middlewares <topics-downloader-middleware>`
can tell such requests apart.
While :attr:`~scrapy.Request.callback` only accepts callables, some spider
classes let you also define a callback by name: both :attr:`CrawlSpider.rules
<scrapy.spiders.CrawlSpider.rules>` and :attr:`SitemapSpider.sitemap_rules
<scrapy.spiders.SitemapSpider.sitemap_rules>` accept the name of a spider
method as a string.
.. _writing-callbacks:
Writing a callback
------------------
Any callable can be a callback, as long as it takes the response as its first
positional parameter, and any :ref:`additional callback data <callback-data>`
as keyword parameters. Spider methods are the most common choice, but plain
functions, lambda expressions and other callable objects work as well.
.. note:: If you enable :ref:`job persistence <topics-jobs>` through the
:setting:`JOBDIR` setting, callbacks must be methods of the running spider.
Requests with any other callback cannot be serialized, so they are kept in
memory only and lost when you pause the crawl. See
:ref:`request-serialization`.
A callback can be:
- A regular function:
.. code-block:: python
def parse(self, response):
return {"url": response.url}
- A generator function:
.. code-block:: python
def parse(self, response):
yield {"url": response.url}
- A coroutine function, i.e. defined with ``async def``:
.. code-block:: python
async def parse(self, response):
return {"url": response.url}
- An asynchronous generator function:
.. code-block:: python
async def parse(self, response):
yield {"url": response.url}
The last two allow using ``await``, ``async for`` and ``async with`` in your
callback. See :ref:`topics-coroutines`.
.. _callback-output:
Callback output
---------------
A callback may return or yield any of the following:
- ``None``, which does nothing.
Callbacks that produce no output at all, e.g. callbacks that only log
information about the response, are perfectly valid. ``None`` values within
an iterable of callback output are ignored as well.
- A :class:`~scrapy.Request` object, which Scrapy schedules, downloads and
eventually sends to its own callback.
- An :ref:`item object <topics-items>`, which Scrapy sends to the
:ref:`item pipelines <topics-item-pipeline>`.
Any object that is neither ``None`` nor a :class:`~scrapy.Request` object
is treated as an item.
- An iterable of any of the values above, e.g. a list or, more commonly, a
generator.
:term:`Asynchronous iterables <asynchronous iterable>`, e.g. an
:term:`asynchronous generator`, are also supported.
.. note:: When a callback *returns* an object, Scrapy iterates that object if
it supports iteration, except for :class:`dict`, :class:`~scrapy.Item`,
:class:`str` and :class:`bytes` objects, which are always handled as single
items.
.. note:: In a generator callback, a ``return`` statement with a value does not
produce any output, since such a value is not part of what the generator
yields. Scrapy logs a warning when it detects such a callback, see
:setting:`WARN_ON_GENERATOR_RETURN_VALUE`.
Before Scrapy acts on the output of a callback, that output goes through the
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method
of your :ref:`spider middlewares <topics-spider-middleware>`, which may modify
it or drop part of it.
If a callback raises an exception, the :attr:`~scrapy.Request.errback` of the
request is *not* called. The exception goes through the
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception`
method of your spider middlewares instead and, unless one of them handles it,
Scrapy logs it and sends the :signal:`spider_error` signal.
.. _callback-data:
.. _topics-request-response-ref-request-callback-arguments:
Passing additional data to callback functions
---------------------------------------------
In some cases you may be interested in passing data to a callback in addition
to the response, e.g. data extracted from the response that triggered the
request. The following example shows how to achieve this by using the
:attr:`.Request.cb_kwargs` attribute:
.. code-block:: python
from scrapy import Request
def parse(self, response):
request = Request(
"http://www.example.com/index.html",
callback=self.parse_page2,
cb_kwargs=dict(main_url=response.url),
)
request.cb_kwargs["foo"] = "bar" # add more arguments for the callback
yield request
def parse_page2(self, response, main_url, foo):
yield dict(
main_url=main_url,
other_url=response.url,
foo=foo,
)
:attr:`.Request.cb_kwargs` is the recommended way to pass your own data to a
callback. Use :attr:`.Request.meta` only for data aimed at :ref:`components
<topics-components>`, such as middlewares and extensions.
.. _errbacks:
.. _topics-request-response-ref-errbacks:
Errbacks
========
The errback of a request is a function that will be called when an exception
is raise while processing it.
It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
be used to track connection establishment timeouts, DNS errors etc.
Here's an example spider logging all errors and catching some specific
errors if needed:
.. code-block:: python
from scrapy import Request, Spider
from scrapy.spidermiddlewares.httperror import HttpError
from twisted.internet.error import DNSLookupError
from twisted.internet.error import TimeoutError, TCPTimedOutError
class ErrbackSpider(Spider):
name = "errback_example"
start_urls = [
"http://www.httpbin.org/", # HTTP 200 expected
"http://www.httpbin.org/status/404", # Not found error
"http://www.httpbin.org/status/500", # server issue
"http://www.httpbin.org:12345/", # non-responding host, timeout expected
"https://example.invalid/", # DNS error expected
]
async def start(self):
for u in self.start_urls:
yield Request(
u,
callback=self.parse_httpbin,
errback=self.errback_httpbin,
dont_filter=True,
)
def parse_httpbin(self, response):
self.logger.info(f"Got successful response from {response.url}")
# do something useful here...
def errback_httpbin(self, failure):
# log all failures
self.logger.error(repr(failure))
# in case you want to do something special for some errors,
# you may need the failure's type:
if failure.check(HttpError):
# these exceptions come from HttpError spider middleware
# you can get the non-200 response
response = failure.value.response
self.logger.error("HttpError on %s", response.url)
elif failure.check(DNSLookupError):
# this is the original request
request = failure.request
self.logger.error("DNSLookupError on %s", request.url)
elif failure.check(TimeoutError, TCPTimedOutError):
request = failure.request
self.logger.error("TimeoutError on %s", request.url)
.. _errback-cb_kwargs:
Accessing additional data in errback functions
----------------------------------------------
In case of a failure to process the request, you may be interested in
accessing arguments to the callback functions so you can process further
based on the arguments in the errback. The following example shows how to
achieve this by using ``Failure.request.cb_kwargs``:
.. code-block:: python
from scrapy import Request
def parse(self, response):
request = Request(
"http://www.example.com/index.html",
callback=self.parse_page2,
errback=self.errback_page2,
cb_kwargs=dict(main_url=response.url),
)
yield request
def parse_page2(self, response, main_url):
pass
def errback_page2(self, failure):
yield dict(
main_url=failure.request.cb_kwargs["main_url"],
)
.. _topics-request-meta:
Request.meta special keys

41
docs/topics/settings.rst
View File

@ -305,10 +305,21 @@ These settings cannot be :ref:`set from a spider <spider-settings>`.
These settings are:
- :setting:`TWISTED_REACTOR_ENABLED`
- :setting:`ADDONS`
- :setting:`COMMANDS_MODULE`
- :setting:`FORCE_CRAWLER_PROCESS`
- :setting:`SPIDER_LOADER_CLASS` and settings used by the corresponding
spider loader class, e.g. :setting:`SPIDER_MODULES` and
:setting:`SPIDER_LOADER_WARN_ONLY` for the default spider loader class.
- :setting:`TWISTED_REACTOR_ENABLED`
:setting:`ADDONS` is a special case: it can be set from a spider, but the
``update_pre_crawler_settings()`` method of :ref:`add-ons <topics-addons>`
enabled that way is not called.
:setting:`TWISTED_REACTOR` also acts as a pre-crawler setting when running a
:ref:`command that needs a CrawlerProcess <topics-commands-crawlerprocess>`,
since its project-level value determines the crawler process class.
.. _reactor-settings:
@ -409,6 +420,9 @@ Default: ``{}``
A dict containing paths to the add-ons enabled in your project and their
priorities. For more information, see :ref:`topics-addons`.
.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`, with a
caveat described in that section.
.. setting:: ASYNCIO_EVENT_LOOP
ASYNCIO_EVENT_LOOP
@ -458,6 +472,26 @@ Default: ``None``
Endpoint URL used for S3-like storage, for example Minio or s3.scality.
.. setting:: AWS_MAX_POOL_CONNECTIONS
AWS_MAX_POOL_CONNECTIONS
------------------------
.. versionadded:: VERSION
Default: ``None``
Maximum number of connections that AWS clients, such as those of the
:ref:`S3 feed storage backend <topics-feed-storage-s3>` and of the
:ref:`S3 media pipeline storage backend <media-pipelines-s3>`, keep in their
connection pool.
If ``None``, the value of :setting:`REACTOR_THREADPOOL_MAXSIZE` is used.
Values lower than the number of parallel AWS calls do not limit those calls, but
their connections are closed instead of reused, which hurts performance, and
``Connection pool is full, discarding connection`` warnings are logged.
.. setting:: AWS_REGION_NAME
AWS_REGION_NAME
@ -1382,6 +1416,8 @@ When :setting:`TWISTED_REACTOR_ENABLED` is set to ``False``,
Set this to ``True`` if you want to set :setting:`TWISTED_REACTOR` to a
non-default value in :ref:`per-spider settings <spider-settings>`.
.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
.. setting:: FTP_PASSIVE_MODE
FTP_PASSIVE_MODE
@ -1835,7 +1871,8 @@ Default: ``False``
Setting to ``True`` will log debug information about the requests scheduler.
This currently logs (only once) if the requests cannot be serialized to disk.
Stats counter (``scheduler/unserializable``) tracks the number of times this happens.
The :stat:`scheduler/unserializable` stat tracks the number of times this
happens.
Example entry in logs::

21
docs/topics/signals.rst
View File

@ -504,6 +504,27 @@ headers_received
:param spider: the spider associated with the response
:type spider: :class:`~scrapy.Spider` object
robots_parsed
~~~~~~~~~~~~~
.. signal:: robots_parsed
.. function:: robots_parsed(robotparser, request)
.. versionadded:: VERSION
Sent by
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` after it
downloads and parses a :file:`robots.txt` file, for the host that *request*
targets.
This signal supports :ref:`asynchronous handlers <signal-deferred>`.
:param robotparser: the parser holding the parsed :file:`robots.txt` contents
:type robotparser: :class:`~scrapy.robotstxt.RobotParser` object
:param request: the request that triggered the :file:`robots.txt` download
:type request: :class:`~scrapy.Request` object
Response signals
----------------

67
docs/topics/spiders.rst
View File

@ -4,43 +4,31 @@
Spiders
=======
Spiders are classes which define how a certain site (or a group of sites) will be
scraped, including how to perform the crawl (i.e. follow links) and how to
extract structured data from their pages (i.e. scraping items). In other words,
Spiders are the place where you define the custom behaviour for crawling and
parsing pages for a particular site (or, in some cases, a group of sites).
Spiders are classes that define how a site, or a group of sites, is scraped:
which requests to send, and how to parse their responses to extract data and to
send additional requests.
For spiders, the scraping cycle goes through something like this:
A crawl goes as follows:
1. You start by generating the initial requests to crawl the first URLs, and
specify a callback function to be called with the response downloaded from
those requests.
1. Scrapy iterates the :meth:`~scrapy.Spider.start` method of the spider to
get the initial requests. By default, that method yields a
:class:`~scrapy.Request` object for each URL in
:attr:`~scrapy.Spider.start_urls`, with :meth:`~scrapy.Spider.parse` as
:ref:`callback <callbacks>`.
The first requests to perform are obtained by iterating the
:meth:`~scrapy.Spider.start` method, which by default yields a
:class:`~scrapy.Request` object for each URL in the
:attr:`~scrapy.Spider.start_urls` spider attribute, with the
:attr:`~scrapy.Spider.parse` method set as :attr:`~scrapy.Request.callback`
function to handle each :class:`~scrapy.http.Response`.
2. Scrapy downloads each request and calls its callback with the resulting
:class:`~scrapy.http.Response`.
2. In the callback function, you parse the response (web page) and return
:ref:`item objects <topics-items>`,
:class:`~scrapy.Request` objects, or an iterable of these objects.
Those Requests will also contain a callback (maybe
the same) and will then be downloaded by Scrapy and then their
response handled by the specified callback.
3. Callbacks parse the response, typically using :ref:`topics-selectors`, and
return or yield :ref:`item objects <topics-items>` with the extracted data
and :class:`~scrapy.Request` objects to continue the crawl, which go back
to step 2. See :ref:`callback-output`.
3. In callback functions, you parse the page contents, typically using
:ref:`topics-selectors` (but you can also use BeautifulSoup, lxml or whatever
mechanism you prefer) and generate items with the parsed data.
4. Items go through :ref:`item pipelines <topics-item-pipeline>`, and are
usually stored through :ref:`topics-feed-exports`.
4. Finally, the items returned from the spider will be typically persisted to a
database (in some :ref:`Item Pipeline <topics-item-pipeline>`) or written to
a file using :ref:`topics-feed-exports`.
Even though this cycle applies (more or less) to any kind of spider, there are
different kinds of default spiders bundled into Scrapy for different purposes.
We will talk about those types here.
Scrapy includes different spider classes for different purposes, described
below.
.. _topics-spiders-ref:
@ -191,22 +179,7 @@ scrapy.Spider
.. automethod:: start
.. method:: parse(response)
This is the default callback used by Scrapy to process downloaded
responses, when their requests don't specify a callback.
The ``parse`` method is in charge of processing the response and returning
scraped data and/or more URLs to follow. Other Requests callbacks have
the same requirements as the :class:`~scrapy.Spider` class.
This method, as well as any other Request callback, must return a
:class:`~scrapy.Request` object, an :ref:`item object <topics-items>`, an
iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects
<topics-items>`, or ``None``.
:param response: the response to parse
:type response: :class:`~scrapy.http.Response`
.. automethod:: parse
.. method:: closed(reason)

641
docs/topics/stats.rst
View File

@ -21,6 +21,8 @@ using the Stats Collector from.
Another feature of the Stats Collector is that it's very efficient (when
enabled) and extremely efficient (almost unnoticeable) when disabled.
See :ref:`topics-stats-reference` below for the stats that Scrapy sets.
.. _topics-stats-usecases:
Common Stats Collector uses
@ -101,3 +103,642 @@ DummyStatsCollector
-------------------
.. autoclass:: DummyStatsCollector
.. _topics-stats-reference:
Built-in stats reference
========================
Scrapy sets the following :ref:`stats <topics-stats>`. Components other than
those built into Scrapy may set additional stats; see their documentation.
Stat keys that contain a ``{placeholder}`` below stand for a family of stats,
one per actual value of the placeholder.
.. note:: Most stats are set by a specific :ref:`component
<topics-components>`, and are only present if that component is enabled and
its code path is reached. A stat that is missing from
:meth:`~scrapy.statscollectors.StatsCollector.get_stats` output is
equivalent to a counter of 0.
.. stat:: downloader/exception_count
``downloader/exception_count``
Number of exceptions raised while downloading requests.
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
.. stat:: downloader/exception_type_count/{exception_type}
``downloader/exception_type_count/{exception_type}``
Number of exceptions raised while downloading requests, per exception type,
where ``{exception_type}`` is the import path of the exception class, e.g.
``twisted.internet.error.DNSLookupError``.
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
.. stat:: downloader/request_bytes
``downloader/request_bytes``
Total size, in bytes, of the requests sent, counting the request line, the
headers and the body. As with :stat:`downloader/request_count`, requests
served from the cache are also counted.
It is an approximation, reconstructed from each :class:`~scrapy.Request`
object instead of measured on the wire, so it does not account for the
actual bytes that the :ref:`download handler
<topics-download-handlers>` sends, e.g. transport-level overhead.
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
.. stat:: downloader/request_count
``downloader/request_count``
Number of requests sent.
Requests that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
serves from the cache are also counted, even though they are never sent,
because it handles requests after
:class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
.. stat:: downloader/request_method_count/{method}
``downloader/request_method_count/{method}``
Number of requests sent, per HTTP method, e.g. ``GET`` or ``POST``. As with
:stat:`downloader/request_count`, requests served from the cache are also
counted.
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
.. stat:: downloader/response_bytes
``downloader/response_bytes``
Total size, in bytes, of the responses received, counting the status line,
the headers and the body. It covers the same responses as
:stat:`downloader/response_count`.
The body is counted as received, i.e. still compressed for responses that
used ``Content-Encoding``, because
:class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` handles
responses before
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
decompresses them. See :stat:`httpcompression/response_bytes` for
decompressed sizes.
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
.. stat:: downloader/response_count
``downloader/response_count``
Number of responses received.
It counts responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
serves from the cache, even though they do not come from the network, and
responses that a downloader middleware consumes before they reach your
spider, e.g. redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware`
turns into new requests. Compare with :stat:`response_received_count`.
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
.. stat:: downloader/response_status_count/{status_code}
``downloader/response_status_count/{status_code}``
Number of responses received, per HTTP status code, e.g. ``200`` or
``404``. It covers the same responses as :stat:`downloader/response_count`.
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
.. stat:: dupefilter/filtered
``dupefilter/filtered``
Number of requests dropped as duplicates.
Set by :class:`~scrapy.dupefilters.RFPDupeFilter`.
.. stat:: elapsed_time_seconds
``elapsed_time_seconds``
Time, as a :class:`float`, in seconds, between the :signal:`spider_opened`
and the :signal:`spider_closed` signals.
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
.. stat:: feedexport/failed_count/{storage}
``feedexport/failed_count/{storage}``
Number of :ref:`feeds <topics-feed-exports>` that could not be stored, per
:ref:`storage backend <topics-feed-storage-backends>`, where ``{storage}``
is the class name of the storage backend, e.g. ``FileFeedStorage``.
.. stat:: feedexport/success_count/{storage}
``feedexport/success_count/{storage}``
Number of :ref:`feeds <topics-feed-exports>` stored successfully, per
:ref:`storage backend <topics-feed-storage-backends>`, where ``{storage}``
is the class name of the storage backend, e.g. ``FileFeedStorage``.
.. stat:: file_count
``file_count``
Number of files handled by the :ref:`media pipelines
<topics-media-pipeline>`.
.. stat:: file_status_count/{status}
``file_status_count/{status}``
Number of files handled by the :ref:`media pipelines
<topics-media-pipeline>`, per status, where ``{status}`` is one of:
- ``downloaded``: the file was downloaded.
- ``cached``: the file came from the
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
cache.
- ``uptodate``: the file was already in the storage backend and had not
:ref:`expired <file-expiration>`, so it was not downloaded again.
.. stat:: finish_reason
``finish_reason``
String indicating why the crawl finished. It matches the *reason* argument
of the :signal:`spider_closed` signal.
Scrapy uses the following reasons:
- ``cancelled``: the spider was closed without a more specific reason,
e.g. because :exc:`~scrapy.exceptions.CloseSpider` was raised without
one.
- ``closespider_errorcount``: see :setting:`CLOSESPIDER_ERRORCOUNT`.
- ``closespider_itemcount``: see :setting:`CLOSESPIDER_ITEMCOUNT`.
- ``closespider_pagecount``: see :setting:`CLOSESPIDER_PAGECOUNT`.
- ``closespider_pagecount_no_item``: see
:setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM`.
- ``closespider_timeout``: see :setting:`CLOSESPIDER_TIMEOUT`.
- ``closespider_timeout_no_item``: see
:setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`.
- ``finished``: the spider became idle with no pending requests, i.e. it
finished normally.
- ``memusage_exceeded``: see :setting:`MEMUSAGE_LIMIT_MB`.
- ``shutdown``: the crawl was interrupted, e.g. by a system signal such
as ``SIGINT`` (:kbd:`Ctrl-C`).
Third-party components and your own code may use any other reason, e.g. by
raising :exc:`~scrapy.exceptions.CloseSpider` with it.
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
.. stat:: finish_time
``finish_time``
Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when
the :signal:`spider_closed` signal was sent.
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
.. stat:: httpcache/errorrecovery
``httpcache/errorrecovery``
Number of times that a stale cached response was used because downloading a
fresh response raised an exception.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/firsthand
``httpcache/firsthand``
Number of responses that were downloaded without a matching cache entry to
validate against, i.e. responses for requests counted in
:stat:`httpcache/miss`.
It is lower than :stat:`httpcache/miss` when some of those requests yield
no response, either because they are dropped (see
:stat:`httpcache/ignore`) or because their download fails.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/hit
``httpcache/hit``
Number of requests served from the cache.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/ignore
``httpcache/ignore``
Number of requests dropped because they were not in the cache and
:setting:`HTTPCACHE_IGNORE_MISSING` is ``True``.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/invalidate
``httpcache/invalidate``
Number of times that a cached response failed validation and was replaced
with a freshly downloaded response.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/miss
``httpcache/miss``
Number of requests for which no cache entry could be read, either because
there was none or because reading it failed, in which case the request is
also counted in :stat:`httpcache/retrieve_error`. Those requests are
downloaded (see :stat:`httpcache/firsthand`), or dropped if
:setting:`HTTPCACHE_IGNORE_MISSING` is ``True`` (see
:stat:`httpcache/ignore`).
Requests with a stale cache entry are not counted here; see
:stat:`httpcache/revalidate` and :stat:`httpcache/invalidate`.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/retrieve_error
``httpcache/retrieve_error``
Number of cache entries that could not be read, and hence were treated as
cache misses. Those requests are also counted in :stat:`httpcache/miss`.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/revalidate
``httpcache/revalidate``
Number of times that a cached response was successfully validated against
the target server, and hence used instead of the fresh response.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/store
``httpcache/store``
Number of responses stored in the cache.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/uncacheable
``httpcache/uncacheable``
Number of responses not stored in the cache because the
:setting:`HTTPCACHE_POLICY` did not allow it.
Every response considered for caching is counted either here or in
:stat:`httpcache/store`, so ``httpcache/store + httpcache/uncacheable``
equals ``httpcache/firsthand + httpcache/invalidate``.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcompression/response_bytes
``httpcompression/response_bytes``
Total size, in bytes, of decompressed response bodies, counting only the
body and only responses that were actually decompressed. Compare with
:stat:`downloader/response_bytes`.
Set by
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`.
.. stat:: httpcompression/response_count
``httpcompression/response_count``
Number of decompressed responses.
Set by
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`.
.. stat:: httperror/response_ignored_count
``httperror/response_ignored_count``
Number of responses dropped because of their HTTP status code.
Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`.
.. stat:: httperror/response_ignored_status_count/{status_code}
``httperror/response_ignored_status_count/{status_code}``
Number of responses dropped because of their HTTP status code, per HTTP
status code, e.g. ``404``.
Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`.
.. stat:: item_dropped_count
``item_dropped_count``
Number of items dropped by an :ref:`item pipeline
<topics-item-pipeline>`, i.e. number of times that the
:signal:`item_dropped` signal was sent.
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
.. stat:: item_dropped_reasons_count/{exception}
``item_dropped_reasons_count/{exception}``
Number of items dropped, per exception, where ``{exception}`` is the class
name of the exception that caused the item to be dropped.
Only :exc:`~scrapy.exceptions.DropItem` and its subclasses drop items, and
each one is counted under its own class name, e.g.
``item_dropped_reasons_count/DropItem`` for
:exc:`~scrapy.exceptions.DropItem` itself and
``item_dropped_reasons_count/MyDropItem`` for a ``MyDropItem`` subclass of
it. Any other exception raised by an :ref:`item pipeline
<topics-item-pipeline>` triggers the :signal:`item_error` signal instead of
:signal:`item_dropped`, and is not counted here or in
:stat:`item_dropped_count`.
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
.. stat:: item_scraped_count
``item_scraped_count``
Number of items that passed all :ref:`item pipelines
<topics-item-pipeline>`, i.e. number of times that the
:signal:`item_scraped` signal was sent.
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
.. stat:: items_per_minute
``items_per_minute``
Average number of items scraped per minute during the crawl.
It is ``None`` if the crawl took less than a minute.
Set by :class:`~scrapy.extensions.logstats.LogStats`.
.. stat:: log_count/{level}
``log_count/{level}``
Number of log messages, per logging level name, e.g. ``INFO`` or
``WARNING``.
Only messages that the :setting:`LOG_LEVEL` setting allows are counted.
Set by :class:`~scrapy.extensions.logcount.LogCount`.
.. stat:: memdebug/gc_garbage_count
``memdebug/gc_garbage_count``
Number of objects in :data:`gc.garbage` when the spider is closed.
Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires
:setting:`MEMDEBUG_ENABLED` to be ``True``.
.. stat:: memdebug/live_refs/{cls}
``memdebug/live_refs/{cls}``
Number of live objects of class ``{cls}`` when the spider is closed, as
reported by :ref:`trackref <topics-leaks-trackrefs>`, e.g.
``memdebug/live_refs/HtmlResponse``.
Only set for classes with at least 1 live object.
Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires
:setting:`MEMDEBUG_ENABLED` to be ``True``.
.. stat:: memusage/limit_reached
``memusage/limit_reached``
``1`` if memory usage exceeded :setting:`MEMUSAGE_LIMIT_MB`, which also
stops the crawl.
Set by :class:`~scrapy.extensions.memusage.MemoryUsage`.
.. stat:: memusage/max
``memusage/max``
Maximum peak memory usage, in bytes, observed during the crawl.
Set by :class:`~scrapy.extensions.memusage.MemoryUsage`.
.. stat:: memusage/startup
``memusage/startup``
Peak memory usage, in bytes, when the engine started.
Set by :class:`~scrapy.extensions.memusage.MemoryUsage`.
.. stat:: memusage/warning_reached
``memusage/warning_reached``
``1`` if memory usage exceeded :setting:`MEMUSAGE_WARNING_MB`.
Set by :class:`~scrapy.extensions.memusage.MemoryUsage`.
.. stat:: offsite/domains
``offsite/domains``
Number of distinct domains for which at least 1 request was dropped for
being offsite.
Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`.
.. stat:: offsite/filtered
``offsite/filtered``
Number of requests dropped for being offsite.
Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`.
.. stat:: request_depth_count/{depth}
``request_depth_count/{depth}``
Number of requests scheduled at depth ``{depth}``, e.g.
``request_depth_count/2``.
Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`, which
requires :setting:`DEPTH_STATS_VERBOSE` to be ``True`` for this stat.
.. stat:: request_depth_max
``request_depth_max``
Maximum depth reached.
Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`.
.. stat:: response_received_count
``response_received_count``
Number of responses received, i.e. number of times that the
:signal:`response_received` signal was sent.
Unlike :stat:`downloader/response_count`, it does not count responses that
a downloader middleware consumes before they reach the engine, e.g.
redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware`
turns into new requests. Both count responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
serves from the cache.
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
.. stat:: responses_per_minute
``responses_per_minute``
Average number of responses received per minute during the crawl.
It is ``None`` if the crawl took less than a minute.
Set by :class:`~scrapy.extensions.logstats.LogStats`.
.. stat:: retry/count
``retry/count``
Number of requests retried.
Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses.
.. stat:: retry/max_reached
``retry/max_reached``
Number of requests that were not retried because they had already been
retried :setting:`RETRY_TIMES` times.
Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses.
.. stat:: retry/reason_count/{reason}
``retry/reason_count/{reason}``
Number of requests retried, per reason, e.g.
``retry/reason_count/twisted.internet.error.TimeoutError`` or
``retry/reason_count/504 Gateway Time-out``.
Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses.
.. note:: Code calling
:func:`~scrapy.downloadermiddlewares.retry.get_retry_request` may pass a
custom *stats_base_key*, in which case ``retry`` is replaced with that key
in the 3 stats above.
.. stat:: robotstxt/exception_count/{exception_type}
``robotstxt/exception_count/{exception_type}``
Number of exceptions raised while downloading ``robots.txt`` files, per
exception type, where ``{exception_type}`` is the string representation of
the exception class, e.g. ``<class
'twisted.internet.error.DNSLookupError'>``.
Set by
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
.. stat:: robotstxt/forbidden
``robotstxt/forbidden``
Number of requests dropped for being disallowed by ``robots.txt``.
Set by
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
.. stat:: robotstxt/request_count
``robotstxt/request_count``
Number of ``robots.txt`` files requested, i.e. 1 per network location for
which at least 1 request was sent.
Set by
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
.. stat:: robotstxt/response_count
``robotstxt/response_count``
Number of ``robots.txt`` responses received.
Set by
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
.. stat:: robotstxt/response_status_count/{status_code}
``robotstxt/response_status_count/{status_code}``
Number of ``robots.txt`` responses received, per HTTP status code, e.g.
``404``.
Set by
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
.. stat:: scheduler/dequeued
``scheduler/dequeued``
Number of requests read from the :ref:`scheduler <topics-scheduler>`.
.. stat:: scheduler/dequeued/disk
``scheduler/dequeued/disk``
Number of requests read from the disk queue of the :ref:`scheduler
<topics-scheduler>`.
.. stat:: scheduler/dequeued/memory
``scheduler/dequeued/memory``
Number of requests read from the memory queue of the :ref:`scheduler
<topics-scheduler>`.
.. stat:: scheduler/enqueued
``scheduler/enqueued``
Number of requests stored into the :ref:`scheduler <topics-scheduler>`.
.. stat:: scheduler/enqueued/disk
``scheduler/enqueued/disk``
Number of requests stored into the disk queue of the :ref:`scheduler
<topics-scheduler>`.
.. stat:: scheduler/enqueued/memory
``scheduler/enqueued/memory``
Number of requests stored into the memory queue of the :ref:`scheduler
<topics-scheduler>`.
.. stat:: scheduler/unserializable
``scheduler/unserializable``
Number of requests that could not be stored into the disk queue of the
:ref:`scheduler <topics-scheduler>` because they could not be
:ref:`serialized <request-serialization>`, and hence were stored into the
memory queue instead.
.. stat:: spider_exceptions/count
``spider_exceptions/count``
Number of unhandled exceptions raised by spider callbacks.
Set by the :ref:`scraper <topics-architecture>`.
.. stat:: spider_exceptions/{exception}
``spider_exceptions/{exception}``
Number of unhandled exceptions raised by spider callbacks, per exception,
where ``{exception}`` is the class name of the exception, e.g.
``spider_exceptions/ValueError``.
Set by the :ref:`scraper <topics-architecture>`.
.. stat:: start_time
``start_time``
Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when
the :signal:`spider_opened` signal was sent.
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
.. stat:: urllength/request_ignored_count
``urllength/request_ignored_count``
Number of requests dropped for having a URL longer than
:setting:`URLLENGTH_LIMIT`.
Set by :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`.

7
pyproject.toml
View File

@ -178,13 +178,6 @@ module = [
"tests.test_squeues",
"tests.test_squeues_request",
"tests.test_stats",
"tests.test_utils_datatypes",
"tests.test_utils_decorators",
"tests.test_utils_defer",
"tests.test_utils_deprecate",
"tests.test_utils_misc.test_return_with_argument_inside_generator",
"tests.test_utils_python",
"tests.test_utils_request",
"tests.utils.bases.http_request",
"tests.utils.bases.http_response",
"tests.utils.bases.spider",

6
scrapy/cmdline.py
View File

@ -225,13 +225,11 @@ def _run_command(cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace)
def _run_command_profiled(
cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace
) -> None:
if opts.profile:
sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
loc = locals()
p = cProfile.Profile()
p.runctx("cmd.run(args, opts)", globals(), loc)
if opts.profile:
p.dump_stats(opts.profile)
p.dump_stats(opts.profile)
if __name__ == "__main__":

2
scrapy/commands/crawl.py
View File

@ -16,7 +16,7 @@ class Command(BaseRunSpiderCommand):
return "[options] <spider>"
def short_desc(self) -> str:
return "Run a spider"
return "Run a spider of the current project, by name"
def run(self, args: list[str], opts: argparse.Namespace) -> None:
if len(args) < 1:

5
scrapy/commands/genspider.py
View File

@ -32,10 +32,7 @@ def sanitize_module_name(module_name: str) -> str:
def extract_domain(url: str) -> str:
"""Extract domain name from URL string"""
o = urlparse(url)
if o.scheme == "" and o.netloc == "":
o = urlparse("//" + url.lstrip("/"))
return o.netloc
return urlparse(url).netloc
def verify_url_scheme(url: str) -> str:

4
scrapy/commands/parse.py
View File

@ -41,7 +41,7 @@ class Command(BaseRunSpiderCommand):
spider: Spider | None = None
items: ClassVar[dict[int, list[Any]]] = {}
requests: ClassVar[dict[int, list[Request]]] = {}
spidercls: type[Spider] | None
spidercls: type[Spider] | None = None
first_response = None
@ -346,6 +346,8 @@ class Command(BaseRunSpiderCommand):
self.first_response = response
cb = self._get_callback(spider=spider, opts=opts, response=response)
assert response.request
response.request.callback = cb
# parse items and requests
depth: int = response.meta["_depth"]

2
scrapy/commands/runspider.py
View File

@ -38,7 +38,7 @@ class Command(BaseRunSpiderCommand):
return "[options] <spider_file>"
def short_desc(self) -> str:
return "Run a self-contained spider (without creating a project)"
return "Run a spider from a Python file, no project required"
def long_desc(self) -> str:
return "Run the spider defined in the given file"

3
scrapy/core/downloader/handlers/ftp.py
View File

@ -126,5 +126,4 @@ class FTPDownloadHandler(BaseDownloadHandler):
headers = {"local filename": protocol.filename or b"", "size": protocol.size}
body = protocol.filename or protocol.body.read()
respcls = responsetypes.from_args(url=request.url, body=body)
# hints for Headers-related types may need to be fixed to not use AnyStr
return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type]
return respcls(url=request.url, status=200, body=body, headers=headers)

8
scrapy/core/scheduler.py
View File

@ -366,8 +366,8 @@ class Scheduler(BaseScheduler):
Unless the received request is filtered out by the Dupefilter, attempt to push
it into the disk queue, falling back to pushing it into the memory queue.
Increment the appropriate stats, such as: ``scheduler/enqueued``,
``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``.
Increment the appropriate stats, such as: :stat:`scheduler/enqueued`,
:stat:`scheduler/enqueued/disk`, :stat:`scheduler/enqueued/memory`.
Return ``True`` if the request was stored successfully, ``False`` otherwise.
"""
@ -390,8 +390,8 @@ class Scheduler(BaseScheduler):
falling back to the disk queue if the memory queue is empty.
Return ``None`` if there are no more enqueued requests.
Increment the appropriate stats, such as: ``scheduler/dequeued``,
``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``.
Increment the appropriate stats, such as: :stat:`scheduler/dequeued`,
:stat:`scheduler/dequeued/disk`, :stat:`scheduler/dequeued/memory`.
"""
request: Request | None = self.mqs.pop()
assert self.stats is not None

21
scrapy/downloadermiddlewares/httpcache.py
View File

@ -1,5 +1,6 @@
from __future__ import annotations
import logging
from email.utils import formatdate
from typing import TYPE_CHECKING
@ -28,6 +29,9 @@ if TYPE_CHECKING:
from scrapy.statscollectors import StatsCollector
logger = logging.getLogger(__name__)
class HttpCacheMiddleware:
DOWNLOAD_EXCEPTIONS = (
ConnectionDone,
@ -77,9 +81,20 @@ class HttpCacheMiddleware:
return None
# Look for cached response and check if expired
cachedresponse: Response | None = self.storage.retrieve_response(
self.crawler.spider, request
)
cachedresponse: Response | None
try:
cachedresponse = self.storage.retrieve_response(
self.crawler.spider, request
)
except Exception:
self.stats.inc_value("httpcache/retrieve_error")
logger.warning(
f"Could not read the cache entry for {request}, treating it as a "
f"cache miss.",
exc_info=True,
extra={"spider": self.crawler.spider},
)
cachedresponse = None
if cachedresponse is None:
self.stats.inc_value("httpcache/miss")
if self.ignore_missing:

12
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -11,6 +11,7 @@ from typing import TYPE_CHECKING
from twisted.internet.defer import Deferred
from scrapy import signals
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
@ -98,7 +99,7 @@ class RobotsTxtMiddleware:
assert self.crawler.stats
try:
resp = await self.crawler.engine.download_async(robotsreq)
self._parse_robots(resp, netloc)
await self._parse_robots(resp, netloc, request)
except Exception as e:
if not isinstance(e, IgnoreRequest):
logger.error(
@ -115,13 +116,20 @@ class RobotsTxtMiddleware:
return await maybe_deferred_to_future(parser)
return parser
def _parse_robots(self, response: Response, netloc: str) -> None:
async def _parse_robots(
self, response: Response, netloc: str, request: Request
) -> None:
assert self.crawler.stats
self.crawler.stats.inc_value("robotstxt/response_count")
self.crawler.stats.inc_value(
f"robotstxt/response_status_count/{response.status}"
)
rp = self._parserimpl.from_crawler(self.crawler, response.body)
await self.crawler.signals.send_catch_log_async(
signal=signals.robots_parsed,
robotparser=rp,
request=request,
)
rp_dfd = self._parsers[netloc]
assert isinstance(rp_dfd, Deferred)
self._parsers[netloc] = rp

17
scrapy/exporters.py
View File

@ -74,6 +74,17 @@ class BaseItemExporter(ABC):
def finish_exporting(self) -> None: # noqa: B027
pass
@staticmethod
def _get_populated_field_names(adapter: ItemAdapter) -> Iterable[str]:
"""Return the populated field names of *adapter*, in declaration order.
Populated fields that are not declared, which some item types allow,
come last, in item order.
"""
populated = set(adapter.keys())
declared = (name for name in adapter.field_names() if name in populated)
return dict.fromkeys([*declared, *adapter.keys()])
def _get_serialized_fields(
self, item: Any, default_value: Any = None, include_empty: bool | None = None
) -> Iterable[tuple[str, Any]]:
@ -86,7 +97,11 @@ class BaseItemExporter(ABC):
include_empty = self.export_empty_fields
if self.fields_to_export is None:
field_iter = item.field_names() if include_empty else item.keys()
field_iter = (
item.field_names()
if include_empty
else self._get_populated_field_names(item)
)
elif isinstance(self.fields_to_export, Mapping):
if include_empty:
field_iter = self.fields_to_export.items()

2
scrapy/extensions/closespider.py
View File

@ -119,7 +119,7 @@ class CloseSpider:
self.task = None
if self.task_no_item:
if self.task_no_item.running:
if self.task_no_item.running: # pragma: no branch
self.task_no_item.stop()
self.task_no_item = None

52
scrapy/extensions/feedexport.py
View File

@ -13,7 +13,7 @@ import re
import sys
import warnings
from abc import ABC, abstractmethod
from collections.abc import Callable, Coroutine
from collections.abc import Callable
from datetime import datetime, timezone
from pathlib import Path, PureWindowsPath
from tempfile import NamedTemporaryFile
@ -28,6 +28,7 @@ from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.extensions.postprocessing import PostProcessingManager
from scrapy.utils.asyncio import is_asyncio_available, run_in_thread
from scrapy.utils.boto import _get_max_pool_connections
from scrapy.utils.conf import feed_complete_default_values_from_settings
from scrapy.utils.defer import deferred_from_coro, ensure_awaitable
from scrapy.utils.ftp import ftp_store_file
@ -213,11 +214,14 @@ class S3FeedStorage(BlockingFeedStorage):
feed_options: dict[str, Any] | None = None,
session_token: str | None = None,
region_name: str | None = None,
max_pool_connections: int | None = None,
):
try:
import boto3.session # noqa: PLC0415
except ImportError:
raise NotConfigured("missing boto3 library") from None
from botocore.config import Config # noqa: PLC0415
u = urlparse(uri)
assert u.hostname
self.bucketname: str = u.hostname
@ -228,6 +232,7 @@ class S3FeedStorage(BlockingFeedStorage):
self.acl: str | None = acl
self.endpoint_url: str | None = endpoint_url
self.region_name: str | None = region_name
self.max_pool_connections: int | None = max_pool_connections
boto3_session = boto3.session.Session()
self.s3_client = boto3_session.client(
@ -237,6 +242,11 @@ class S3FeedStorage(BlockingFeedStorage):
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
region_name=self.region_name,
config=(
Config(max_pool_connections=self.max_pool_connections)
if self.max_pool_connections is not None
else None
),
)
if feed_options and feed_options.get("overwrite", True) is False:
@ -262,6 +272,7 @@ class S3FeedStorage(BlockingFeedStorage):
acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None,
endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None,
region_name=crawler.settings["AWS_REGION_NAME"] or None,
max_pool_connections=_get_max_pool_connections(crawler.settings),
feed_options=feed_options,
)
@ -454,7 +465,7 @@ class FeedSlot:
)
def finish_exporting(self) -> None:
if self._exporting:
if self._exporting: # pragma: no branch
assert self.exporter
self.exporter.finish_exporting()
self._exporting = False
@ -475,7 +486,7 @@ class FeedExporter:
self.feeds = {}
self.slots: list[FeedSlot] = []
self.filters: dict[str, ItemFilter] = {}
self._pending_close_coros: list[Coroutine[Any, Any, None]] = []
self._pending_close_tasks: list[asyncio.Task[None] | Deferred[None]] = []
if not self.settings["FEEDS"] and not self.settings["FEED_URI"]:
raise NotConfigured
@ -539,23 +550,44 @@ class FeedExporter:
)
async def close_spider(self, spider: Spider) -> None:
self._pending_close_coros.extend(
self._close_slot(slot, spider) for slot in self.slots
)
for slot in self.slots:
self._schedule_slot_close(slot, spider)
if self._pending_close_coros:
if self._pending_close_tasks: # pragma: no branch
if is_asyncio_available():
await asyncio.wait(
[asyncio.create_task(coro) for coro in self._pending_close_coros]
cast("list[asyncio.Task[None]]", list(self._pending_close_tasks))
)
else:
await DeferredList(
deferred_from_coro(coro) for coro in self._pending_close_coros
cast("list[Deferred[None]]", list(self._pending_close_tasks))
)
# Send FEED_EXPORTER_CLOSED signal
await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed)
def _schedule_slot_close(
self, slot: FeedSlot, spider: Spider
) -> asyncio.Task[None] | Deferred[None]:
"""Start closing the slot without waiting for it to finish, keeping
track of the pending work so that it can be awaited in
:meth:`close_spider` if it hasn't finished by then."""
aw: asyncio.Task[None] | Deferred[None]
coro = self._close_slot(slot, spider)
if is_asyncio_available():
aw = asyncio.create_task(coro)
self._pending_close_tasks.append(aw)
aw.add_done_callback(self._pending_close_tasks.remove)
else:
aw = deferred_from_coro(coro)
self._pending_close_tasks.append(aw)
aw.addBoth(self._untrack_pending_close_task, aw)
return aw
def _untrack_pending_close_task(self, result: Any, aw: Deferred[None]) -> Any:
self._pending_close_tasks.remove(aw)
return result
@staticmethod
def _get_file(slot_: FeedSlot) -> IO[bytes]:
assert slot_.file
@ -652,7 +684,7 @@ class FeedExporter:
uri_params = self._get_uri_params(
spider, self.feeds[slot.uri_template]["uri_params"], slot
)
self._pending_close_coros.append(self._close_slot(slot, spider))
self._schedule_slot_close(slot, spider)
slots.append(
self._start_new_batch(
batch_id=slot.batch_id + 1,

2
scrapy/extensions/logcount.py
View File

@ -20,7 +20,7 @@ class LogCount:
"""Install a log handler that counts log messages by level.
The handler installed is :class:`scrapy.utils.log.LogCounterHandler`.
The counts are stored in stats as ``log_count/<level>``.
The counts are stored in the :stat:`log_count/{level}` stat.
.. versionadded:: 2.14
"""

2
scrapy/extensions/memusage.py
View File

@ -94,7 +94,7 @@ class MemoryUsage:
def engine_stopped(self) -> None:
for tsk in self.tasks:
if tsk.running:
if tsk.running: # pragma: no branch
tsk.stop()
def update(self) -> None:

3
scrapy/extensions/periodic_log.py
View File

@ -38,7 +38,6 @@ class PeriodicLog:
):
self.stats: StatsCollector = stats
self.interval: float = interval
self.multiplier: float = 60.0 / self.interval
self.task: AsyncioLoopingCall | LoopingCall | None = None
self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4)
self.ext_stats_enabled: bool = bool(ext_stats)
@ -165,5 +164,5 @@ class PeriodicLog:
def spider_closed(self, spider: Spider, reason: str) -> None:
self.log()
if self.task and self.task.running:
if self.task and self.task.running: # pragma: no branch
self.task.stop()

26
scrapy/http/headers.py
View File

@ -1,7 +1,7 @@
from __future__ import annotations
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, AnyStr, TypeAlias, cast
from typing import TYPE_CHECKING, Any, TypeAlias, cast
from w3lib.http import headers_dict_to_raw
@ -25,14 +25,20 @@ class Headers(CaselessDict):
def __init__(
self,
seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
seq: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None = None,
encoding: str = "utf-8",
):
self.encoding: str = encoding
super().__init__(seq)
def update( # type: ignore[override]
self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]
self,
seq: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]],
) -> None:
seq = seq.items() if isinstance(seq, Mapping) else seq
iseq: dict[bytes, list[bytes]] = {}
@ -40,7 +46,7 @@ class Headers(CaselessDict):
iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
super().update(iseq)
def normkey(self, key: AnyStr) -> bytes: # type: ignore[override]
def normkey(self, key: str | bytes) -> bytes:
"""Normalize key to bytes"""
return self._tobytes(key.title())
@ -67,19 +73,19 @@ class Headers(CaselessDict):
return str(x).encode(self.encoding)
raise TypeError(f"Unsupported value type: {type(x)}")
def __getitem__(self, key: AnyStr) -> bytes | None:
def __getitem__(self, key: str | bytes) -> bytes | None:
try:
return cast("list[bytes]", super().__getitem__(key))[-1]
except IndexError:
return None
def get(self, key: AnyStr, def_val: Any = None) -> bytes | None:
def get(self, key: str | bytes, def_val: Any = None) -> bytes | None:
try:
return cast("list[bytes]", super().get(key, def_val))[-1]
except IndexError:
return None
def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]:
def getlist(self, key: str | bytes, def_val: Any = None) -> list[bytes]:
try:
return cast("list[bytes]", super().__getitem__(key))
except KeyError:
@ -87,15 +93,15 @@ class Headers(CaselessDict):
return self.normvalue(def_val)
return []
def setlist(self, key: AnyStr, list_: Iterable[_RawValue]) -> None:
def setlist(self, key: str | bytes, list_: Iterable[_RawValue]) -> None:
self[key] = list_
def setlistdefault(
self, key: AnyStr, default_list: Iterable[_RawValue] = ()
self, key: str | bytes, default_list: Iterable[_RawValue] = ()
) -> Any:
return self.setdefault(key, default_list)
def appendlist(self, key: AnyStr, value: Iterable[_RawValue]) -> None:
def appendlist(self, key: str | bytes, value: Iterable[_RawValue]) -> None:
lst = self.getlist(key)
lst.extend(self.normvalue(value))
self[key] = lst

34
scrapy/http/request/__init__.py
View File

@ -11,7 +11,6 @@ import inspect
from typing import (
TYPE_CHECKING,
Any,
AnyStr,
Concatenate,
NoReturn,
TypeAlias,
@ -125,7 +124,10 @@ class Request(object_ref):
url: str,
callback: CallbackT | None = None,
method: str = "GET",
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
headers: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,
@ -167,7 +169,8 @@ class Request(object_ref):
#:
#: The callable must expect the response as its first parameter, and
#: support any additional keyword arguments set through
#: :attr:`cb_kwargs`.
#: :attr:`cb_kwargs`. See :ref:`writing-callbacks` and
#: :ref:`callback-output`.
#:
#: In addition to an arbitrary callable, the following values are also
#: supported:
@ -188,8 +191,7 @@ class Request(object_ref):
#: raises exceptions for non-2xx responses by default, sending them
#: to the :attr:`errback` instead.
#:
#: .. seealso::
#: :ref:`topics-request-response-ref-request-callback-arguments`
#: .. seealso:: :ref:`callbacks`
self.callback: CallbackT | None = callback
#: :class:`~collections.abc.Callable` to handle exceptions raised
@ -198,7 +200,7 @@ class Request(object_ref):
#: The callable must expect a :exc:`~twisted.python.failure.Failure` as
#: its first parameter.
#:
#: .. seealso:: :ref:`topics-request-response-ref-errbacks`
#: .. seealso:: :ref:`errbacks`
self.errback: Callable[[Failure], Any] | None = errback
self._cookies: CookiesT | None = cookies or None
@ -310,7 +312,11 @@ class Request(object_ref):
@headers.setter
def headers(
self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None
self,
value: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None,
) -> None:
if isinstance(value, Headers):
self._headers = value
@ -381,6 +387,20 @@ class Request(object_ref):
request_kwargs.update(kwargs)
return cls(**request_kwargs)
def to_curl(self) -> str:
"""Return a string with a `cURL <https://curl.se/>`_ command equivalent
to this request.
Inverse of :meth:`from_curl`. See also
:func:`scrapy.utils.request.request_to_curl`.
.. versionadded:: VERSION
"""
# Imported here to avoid a circular import.
from scrapy.utils.request import request_to_curl # noqa: PLC0415
return request_to_curl(self)
def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]:
"""Return a dictionary containing the Request's data.

23
scrapy/http/response/__init__.py
View File

@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst
from __future__ import annotations
from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, overload
from typing import TYPE_CHECKING, Any, TypeVar, overload
from urllib.parse import urljoin
from scrapy.exceptions import NotSupported
@ -72,7 +72,10 @@ class Response(object_ref):
self,
url: str,
status: int = 200,
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
headers: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None = None,
body: bytes = b"",
flags: list[str] | None = None,
request: Request | None = None,
@ -145,7 +148,11 @@ class Response(object_ref):
@headers.setter
def headers(
self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None
self,
value: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None,
) -> None:
if isinstance(value, Headers):
self._headers = value
@ -222,7 +229,10 @@ class Response(object_ref):
url: str | Link,
callback: CallbackT | None = None,
method: str = "GET",
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
headers: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,
@ -272,7 +282,10 @@ class Response(object_ref):
urls: Iterable[str | Link],
callback: CallbackT | None = None,
method: str = "GET",
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
headers: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,

12
scrapy/http/response/text.py
View File

@ -9,7 +9,7 @@ from __future__ import annotations
import json
from contextlib import suppress
from typing import TYPE_CHECKING, Any, AnyStr, cast
from typing import TYPE_CHECKING, Any, cast
from urllib.parse import urljoin
import parsel
@ -170,7 +170,10 @@ class TextResponse(Response):
url: str | Link | parsel.Selector,
callback: CallbackT | None = None,
method: str = "GET",
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
headers: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,
@ -223,7 +226,10 @@ class TextResponse(Response):
urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None,
callback: CallbackT | None = None,
method: str = "GET",
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
headers: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,

2
scrapy/mail.py
View File

@ -2,6 +2,8 @@
Mail sending helpers
"""
# pragma: no file cover
from __future__ import annotations
import logging

13
scrapy/pipelines/files.py
View File

@ -37,7 +37,7 @@ from scrapy.pipelines.media import (
_MediaRequestFiltered,
)
from scrapy.utils.asyncio import run_in_thread
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.boto import _get_max_pool_connections, is_botocore_available
from scrapy.utils.datatypes import CaseInsensitiveDict
from scrapy.utils.defer import deferred_from_coro, ensure_awaitable
from scrapy.utils.ftp import ftp_store_file
@ -164,6 +164,9 @@ class S3FilesStore:
AWS_REGION_NAME = None
AWS_USE_SSL = None
AWS_VERIFY = None
# Overridden from settings.AWS_MAX_POOL_CONNECTIONS in
# FilesPipeline.from_crawler(); None means the botocore default
AWS_MAX_POOL_CONNECTIONS: int | None = None
POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler()
HEADERS: ClassVar[dict[str, str]] = {
@ -174,7 +177,13 @@ class S3FilesStore:
if not is_botocore_available():
raise NotConfigured("missing botocore library")
import botocore.session # noqa: PLC0415
from botocore.config import Config # noqa: PLC0415
config = (
Config(max_pool_connections=self.AWS_MAX_POOL_CONNECTIONS)
if self.AWS_MAX_POOL_CONNECTIONS is not None
else None
)
session = botocore.session.get_session()
self.s3_client = session.create_client(
"s3",
@ -185,6 +194,7 @@ class S3FilesStore:
region_name=self.AWS_REGION_NAME,
use_ssl=self.AWS_USE_SSL,
verify=self.AWS_VERIFY,
config=config,
)
if not uri.startswith("s3://"):
raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
@ -522,6 +532,7 @@ class FilesPipeline(MediaPipeline):
s3store.AWS_REGION_NAME = settings["AWS_REGION_NAME"]
s3store.AWS_USE_SSL = settings["AWS_USE_SSL"]
s3store.AWS_VERIFY = settings["AWS_VERIFY"]
s3store.AWS_MAX_POOL_CONNECTIONS = _get_max_pool_connections(settings)
s3store.POLICY = settings["FILES_STORE_S3_ACL"]
gcs_store: type[GCSFilesStore] = cast(

21
scrapy/robotstxt.py
View File

@ -67,6 +67,15 @@ class RobotParser(metaclass=ABCMeta):
:type user_agent: str or bytes
"""
def crawl_delay(self, user_agent: str | bytes) -> float | None:
"""Return the ``Crawl-delay`` directive for ``user_agent`` as a number
of seconds, or ``None`` if it is not set or the backend does not support
it.
.. versionadded:: VERSION
"""
return None
class PythonRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
@ -85,6 +94,10 @@ class PythonRobotParser(RobotParser):
url = to_unicode(url)
return self.rp.can_fetch(user_agent, url)
def crawl_delay(self, user_agent: str | bytes) -> float | None:
delay = self.rp.crawl_delay(to_unicode(user_agent))
return None if delay is None else float(delay)
class RerpRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
@ -105,6 +118,10 @@ class RerpRobotParser(RobotParser):
url = to_unicode(url)
return cast("bool", self.rp.is_allowed(user_agent, url))
def crawl_delay(self, user_agent: str | bytes) -> float | None:
delay = self.rp.get_crawl_delay(to_unicode(user_agent))
return None if delay is None else float(delay)
class ProtegoRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
@ -121,3 +138,7 @@ class ProtegoRobotParser(RobotParser):
user_agent = to_unicode(user_agent)
url = to_unicode(url)
return self.rp.can_fetch(url, user_agent)
def crawl_delay(self, user_agent: str | bytes) -> float | None:
delay = self.rp.crawl_delay(to_unicode(user_agent))
return None if delay is None else float(delay)

2
scrapy/settings/default_settings.py
View File

@ -28,6 +28,7 @@ __all__ = [
"AUTOTHROTTLE_TARGET_CONCURRENCY",
"AWS_ACCESS_KEY_ID",
"AWS_ENDPOINT_URL",
"AWS_MAX_POOL_CONNECTIONS",
"AWS_REGION_NAME",
"AWS_SECRET_ACCESS_KEY",
"AWS_SESSION_TOKEN",
@ -229,6 +230,7 @@ AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
AWS_ACCESS_KEY_ID = None
AWS_SECRET_ACCESS_KEY = None
AWS_ENDPOINT_URL = None
AWS_MAX_POOL_CONNECTIONS = None
AWS_REGION_NAME = None
AWS_SESSION_TOKEN = None
AWS_USE_SSL = None

1
scrapy/signals.py
View File

@ -21,6 +21,7 @@ response_received = object()
response_downloaded = object()
headers_received = object()
bytes_received = object()
robots_parsed = object()
item_scraped = object()
item_dropped = object()
item_error = object()

16
scrapy/spiders/__init__.py
View File

@ -143,6 +143,22 @@ class Spider(object_ref):
else:
def parse(self, response: Response, **kwargs: Any) -> Any:
"""Process *response*, i.e. extract data from it and generate new
requests.
This is the default :ref:`callback <callbacks>`: Scrapy uses
it for the response to any request that does not define a
:attr:`~scrapy.Request.callback`, such as the requests that
:meth:`start` yields by default.
Any :attr:`~scrapy.Request.cb_kwargs` of the request are passed as
keyword parameters.
Spiders must define this method, unless every request that they
send defines a callback.
See :ref:`callback-output` about the supported return values.
"""
raise NotImplementedError(
f"{self.__class__.__name__}.parse callback is not defined"
)

11
scrapy/spiders/feed.py
View File

@ -9,7 +9,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any
from scrapy.exceptions import NotConfigured, NotSupported
from scrapy.exceptions import NotSupported
from scrapy.http import Response, TextResponse
from scrapy.selector import Selector
from scrapy.spiders import Spider
@ -76,11 +76,6 @@ class XMLFeedSpider(Spider):
yield from self.process_results(response, ret)
def _parse(self, response: Response, **kwargs: Any) -> Any:
if not hasattr(self, "parse_node"):
raise NotConfigured(
"You must define parse_node method in order to scrape this XML feed"
)
response = self.adapt_response(response)
nodes: Iterable[Selector]
if self.iterator == "iternodes":
@ -158,9 +153,5 @@ class CSVFeedSpider(Spider):
yield from self.process_results(response, ret)
def _parse(self, response: Response, **kwargs: Any) -> Any:
if not hasattr(self, "parse_row"):
raise NotConfigured(
"You must define parse_row method in order to scrape this CSV feed"
)
response = self.adapt_response(response)
return self.parse_rows(response)

20
scrapy/utils/asyncio.py
View File

@ -9,7 +9,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar
from twisted.internet.defer import Deferred
from twisted.internet.task import LoopingCall
from twisted.internet.task import LoopingCall, deferLater
from twisted.internet.threads import deferToThread
from scrapy.utils.asyncgen import as_async_generator
@ -293,6 +293,24 @@ class CallLaterResult:
self._delayed_call = None
async def sleep(seconds: float) -> None:
"""Sleep for *seconds*.
.. versionadded:: VERSION
This uses either :func:`asyncio.sleep` or
:func:`~twisted.internet.task.deferLater`, depending on whether asyncio
support is available.
"""
if is_asyncio_available():
await asyncio.sleep(seconds)
return
from twisted.internet import reactor
await deferLater(reactor, seconds)
async def run_in_thread(
func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs
) -> _T:

15
scrapy/utils/boto.py
View File

@ -1,7 +1,22 @@
"""Boto/botocore helpers"""
from __future__ import annotations
from importlib.util import find_spec
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from scrapy.settings import BaseSettings
def is_botocore_available() -> bool:
return find_spec("botocore") is not None
def _get_max_pool_connections(settings: BaseSettings) -> int:
"""Return the maximum number of connections that AWS clients may keep in
their connection pool.
"""
return settings.getint("AWS_MAX_POOL_CONNECTIONS") or settings.getint(
"REACTOR_THREADPOOL_MAXSIZE"
)

56
scrapy/utils/datatypes.py
View File

@ -11,12 +11,12 @@ import warnings
import weakref
from collections import OrderedDict
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, cast
from typing import TYPE_CHECKING, Any, TypeVar, cast
from scrapy.exceptions import ScrapyDeprecationWarning
if TYPE_CHECKING:
from collections.abc import Iterable, Sequence
from collections.abc import Container, Iterable
# typing.Self requires Python 3.11
from typing_extensions import Self
@ -44,22 +44,25 @@ class CaselessDict(dict): # type: ignore[type-arg]
def __init__(
self,
seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
seq: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]]
| None = None,
):
super().__init__()
if seq:
self.update(seq)
def __getitem__(self, key: AnyStr) -> Any:
def __getitem__(self, key: str | bytes) -> Any:
return dict.__getitem__(self, self.normkey(key))
def __setitem__(self, key: AnyStr, value: Any) -> None:
def __setitem__(self, key: str | bytes, value: Any) -> None:
dict.__setitem__(self, self.normkey(key), self.normvalue(value))
def __delitem__(self, key: AnyStr) -> None:
def __delitem__(self, key: str | bytes) -> None:
dict.__delitem__(self, self.normkey(key))
def __contains__(self, key: AnyStr) -> bool: # type: ignore[override]
def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]
return dict.__contains__(self, self.normkey(key))
has_key = __contains__
@ -69,7 +72,7 @@ class CaselessDict(dict): # type: ignore[type-arg]
copy = __copy__
def normkey(self, key: AnyStr) -> AnyStr:
def normkey(self, key: str | bytes) -> str | bytes:
"""Method to normalize dictionary key access"""
return key.lower()
@ -77,23 +80,28 @@ class CaselessDict(dict): # type: ignore[type-arg]
"""Method to normalize values prior to be set"""
return value
def get(self, key: AnyStr, def_val: Any = None) -> Any:
def get(self, key: str | bytes, def_val: Any = None) -> Any:
return dict.get(self, self.normkey(key), self.normvalue(def_val))
def setdefault(self, key: AnyStr, def_val: Any = None) -> Any:
def setdefault(self, key: str | bytes, def_val: Any = None) -> Any:
return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))
# doesn't fully implement MutableMapping.update()
def update(self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]) -> None: # type: ignore[override]
def update( # type: ignore[override]
self,
seq: Mapping[str, Any]
| Mapping[bytes, Any]
| Iterable[tuple[str | bytes, Any]],
) -> None:
seq = seq.items() if isinstance(seq, Mapping) else seq
iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
super().update(iseq)
@classmethod
def fromkeys(cls, keys: Iterable[AnyStr], value: Any = None) -> Self: # type: ignore[override]
return cls((k, value) for k in keys) # type: ignore[misc]
def fromkeys(cls, keys: Iterable[str | bytes], value: Any = None) -> Self: # type: ignore[override]
return cls((k, value) for k in keys)
def pop(self, key: AnyStr, *args: Any) -> Any:
def pop(self, key: str | bytes, *args: Any) -> Any:
return dict.pop(self, self.normkey(key), *args)
@ -132,6 +140,22 @@ class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]):
def __repr__(self) -> str:
return f"<{self.__class__.__name__}: {super().__repr__()}>"
# UserDict.copy() shallow-copies the instance, which would share self._keys
# between the copy and the original.
def __copy__(self) -> Self:
new = self.__class__()
new.data = self.data.copy()
new._keys = self._keys.copy()
return new
copy = __copy__
# UserDict.__ior__ updates self.data directly, which would leave self._keys
# out of date.
def __ior__(self, other: Any) -> Self: # type: ignore[override,misc]
self.update(other)
return self
def _normkey(self, key: str | bytes) -> str | bytes:
return key
@ -189,8 +213,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]):
class SequenceExclude:
"""Object to test if an item is NOT within some sequence."""
def __init__(self, seq: Sequence[Any]):
self.seq: Sequence[Any] = seq
def __init__(self, seq: Container[Any]):
self.seq: Container[Any] = seq
def __contains__(self, item: Any) -> bool:
return item not in self.seq

20
scrapy/utils/decorators.py
View File

@ -10,6 +10,7 @@ from twisted.internet.defer import Deferred, maybeDeferred
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.asyncio import run_in_thread
from scrapy.utils.defer import deferred_from_coro
from scrapy.utils.python import _signature
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Callable, Coroutine
@ -19,9 +20,19 @@ _T = TypeVar("_T")
_P = ParamSpec("_P")
@overload
def deprecated(use_instead: Callable[_P, _T]) -> Callable[_P, _T]: ...
@overload
def deprecated(
use_instead: Any = None,
) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]:
use_instead: str | None = None,
) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: ...
def deprecated(
use_instead: Callable[_P, _T] | str | None = None,
) -> Callable[_P, _T] | Callable[[Callable[_P, _T]], Callable[_P, _T]]:
"""This is a decorator which can be used to mark functions
as deprecated. It will result in a warning being emitted
when the function is used."""
@ -38,8 +49,9 @@ def deprecated(
return wrapped
if callable(use_instead):
deco = deco(use_instead)
func = use_instead
use_instead = None
return deco(func)
return deco
@ -98,7 +110,7 @@ def _warn_spider_arg(
):
"""Decorator to warn if a ``spider`` argument is passed to a function."""
sig = inspect.signature(func)
sig = _signature(func)
def check_args(*args: _P.args, **kwargs: _P.kwargs) -> None:
bound = sig.bind(*args, **kwargs)

11
scrapy/utils/defer.py
View File

@ -27,7 +27,7 @@ from twisted.internet.task import Cooperator
from twisted.python import failure
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.asyncio import is_asyncio_available
from scrapy.utils.asyncio import is_asyncio_available, sleep
from scrapy.utils.python import global_object_name
if TYPE_CHECKING:
@ -90,14 +90,7 @@ async def _defer_sleep_async() -> None:
"""Delay by _DEFER_DELAY so reactor has a chance to go through readers and writers
before attending pending delayed calls, so do not set delay to zero.
"""
if is_asyncio_available():
await asyncio.sleep(_DEFER_DELAY)
else:
from twisted.internet import reactor
d: Deferred[None] = Deferred()
reactor.callLater(_DEFER_DELAY, d.callback, None)
await d
await sleep(_DEFER_DELAY)
def defer_result(result: Any) -> Deferred[Any]: # pragma: no cover

21
scrapy/utils/python.py
View File

@ -178,11 +178,30 @@ def binary_is_text(data: bytes) -> bool:
return all(c not in _BINARYCHARS for c in data)
# PEP 649 (Python 3.14+) made annotation evaluation lazy, so inspect.signature()
# can raise NameError for names imported only under TYPE_CHECKING. We only need
# parameter names, kinds and defaults, so leave such annotations as ForwardRefs.
if sys.version_info >= (3, 14):
from annotationlib import Format
def _signature(func: Callable[..., Any]) -> inspect.Signature:
return inspect.signature(func, annotation_format=Format.FORWARDREF)
else:
def _signature(func: Callable[..., Any]) -> inspect.Signature:
return inspect.signature(func)
def get_func_args_dict(
func: Callable[..., Any], stripself: bool = False
) -> Mapping[str, inspect.Parameter]:
"""Return the argument dict of a callable object.
Annotations are not evaluated, so on Python 3.14 and later the ``annotation``
attribute of the returned parameters may be a ``ForwardRef`` instead of the
resolved type.
.. versionadded:: 2.14
"""
if not callable(func):
@ -190,7 +209,7 @@ def get_func_args_dict(
args: Mapping[str, inspect.Parameter]
try:
sig = inspect.signature(func)
sig = _signature(func)
except ValueError:
return {}

28
tests/benchmarks/__init__.py Normal file
View File

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
from scrapy import Spider
from scrapy.crawler import Crawler
def crawl(spidercls: type[Spider], settings: dict[str, Any], **kwargs: Any) -> Crawler:
"""Run a crawl to completion and return its crawler.
Unlike the rest of the test suite, benchmarks run without ``pytest-twisted``
and drive the reactor themselves, since the code being measured must be
callable synchronously by ``pytest-codspeed``.
"""
from twisted.internet import reactor
crawler = get_crawler(spidercls, settings)
result: list[Any] = []
crawler.crawl(**kwargs).addBoth(result.append)
while not result:
reactor.iterate(0.001)
if isinstance(result[0], BaseException):
raise result[0]
return crawler

27
tests/benchmarks/conftest.py Normal file
View File

@ -0,0 +1,27 @@
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from scrapy.utils.reactor import install_reactor
if TYPE_CHECKING:
from collections.abc import Generator
@pytest.fixture(scope="session", autouse=True)
def running_reactor() -> Generator[None]:
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
from twisted.internet import reactor
# Marks the reactor as running without blocking, so that crawls can be
# driven with reactor.iterate(), see tests.benchmarks.crawl().
reactor.startRunning(installSignalHandlers=False)
yield
reactor.stop()
# Lets the shutdown event triggers run, e.g. to join the thread pool.
reactor.iterate(0)

69
tests/benchmarks/test_crawl.py Normal file
View File

@ -0,0 +1,69 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from urllib.parse import urlencode
import pytest
from scrapy import Field, Item, Request, Spider
from scrapy.linkextractors import LinkExtractor
from tests.benchmarks import crawl
if TYPE_CHECKING:
from collections.abc import AsyncIterator
from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found]
from scrapy.http import Response
from tests.mockserver.http import MockServer
pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed")
PAGES = 100
LINKS_PER_PAGE = 5
class _Page(Item):
url = Field()
anchors = Field()
class _FollowSpider(Spider):
name = "benchmark"
url: str
link_extractor = LinkExtractor()
async def start(self) -> AsyncIterator[Any]:
yield Request(self.url, dont_filter=True)
def parse(self, response: Response) -> Any:
yield _Page(
url=response.url,
anchors=response.css("a::text").getall(),
)
for link in self.link_extractor.extract_links(response): # type: ignore[arg-type]
yield Request(link.url)
class _Pipeline:
def process_item(self, item: Any) -> Any:
return item
def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> None:
"""Per-request overhead of a crawl over HTTP.
The pages are small on purpose, so that the cost of parsing them stays
negligible next to the cost of moving requests and responses through the
engine, the middlewares and the download handler.
"""
query = urlencode({"total": PAGES, "show": LINKS_PER_PAGE, "order": "desc"})
url = mockserver.url(f"/follow?{query}")
settings = {"ITEM_PIPELINES": {_Pipeline: 100}, "LOG_ENABLED": False}
def run() -> None:
crawler = crawl(_FollowSpider, settings, url=url)
assert crawler.stats
assert crawler.stats.get_value("item_scraped_count") == PAGES + 1
benchmark(run)

46
tests/spiders.py
View File

@ -38,6 +38,35 @@ class MockServerSpider(Spider):
self.is_secure = is_secure
class RawResponseSpider(MockServerSpider):
"""Base class for spiders that fetch a response built by the test itself.
Subclasses return the body from :meth:`raw_body` and request
:attr:`raw_url`, which the mock server answers with that body verbatim
under :attr:`content_type`. This lets tests reach parsing code that only
a specific kind of response triggers while still going through a regular
crawl, instead of calling internal parsing methods directly.
"""
name = "raw_response"
content_type = "text/plain"
def raw_body(self) -> str:
raise NotImplementedError
@property
def raw_url(self) -> str:
assert self.mockserver
raw = (
"HTTP/1.1 200 OK\r\n"
f"Content-Type: {self.content_type}\r\n"
"Connection: close\r\n"
"\r\n"
f"{self.raw_body()}"
)
return self.mockserver.url("/raw?" + urlencode({"raw": raw}))
class MetaSpider(MockServerSpider):
name = "meta"
@ -496,6 +525,23 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
self.logger.info("[errback] status %i", failure.value.response.status)
class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod):
name = "crawl_spider_without_errback"
async def start(self):
test_body = b"""
<html>
<head><title>Page title</title></head>
<body>
<p><a href="/status?n=200">Item 200</a></p> <!-- callback -->
<p><a href="/status?n=404">Item 404</a></p> <!-- failure, no errback -->
</body>
</html>
"""
url = self.mockserver.url("/alpayload")
yield Request(url, method="POST", body=test_body)
class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParseMethod):
name = "crawl_spider_with_process_request_cb_kwargs"
rules = (

12
tests/test_command_crawl.py
View File

@ -23,6 +23,18 @@ class TestCrawlCommand(TestProjectBase):
_, _, stderr = self.crawl(code, proj_path, args=args)
return stderr
def test_no_spider(self, proj_path: Path) -> None:
returncode, out, _ = proc("crawl", cwd=proj_path)
assert returncode == 2
assert "Usage" in out
def test_multiple_spiders(self, proj_path: Path) -> None:
returncode, _, err = proc("crawl", "myspider", "myspider2", cwd=proj_path)
assert returncode == 2
assert (
"running 'scrapy crawl' with more than one spider is not supported" in err
)
def test_no_output(self, proj_path: Path) -> None:
spider_code = """
import scrapy

32
tests/test_command_fetch.py
View File

@ -2,13 +2,24 @@ from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from tests.utils.bases.commands import TestProjectBase
from tests.utils.cmdline import proc
if TYPE_CHECKING:
from pathlib import Path
from tests.mockserver.http import MockServer
class TestFetchCommand:
@pytest.mark.parametrize("args", [(), ("not-a-url",), ("a:b", "c:d")])
def test_bad_arguments(self, args: tuple[str, ...]) -> None:
returncode, out, _ = proc("fetch", *args)
assert returncode == 2
assert "Usage" in out
def test_output(self, mockserver: MockServer) -> None:
_, out, _ = proc("fetch", mockserver.url("/text"))
assert out.strip() == "Works"
@ -36,3 +47,24 @@ class TestFetchCommand:
"fetch", "-s", "TWISTED_REACTOR_ENABLED=False", mockserver.url("/text")
)
assert out.strip() == "Works"
class TestFetchCommandWithSpider(TestProjectBase):
@pytest.fixture(autouse=True)
def create_files(self, proj_path: Path) -> None:
(proj_path / self.project_name / "spiders" / "myspider.py").write_text(
"""
import scrapy
class MySpider(scrapy.Spider):
name = "myspider"
custom_settings = {"USER_AGENT": "myspider-user-agent"}
""",
encoding="utf-8",
)
def test_spider(self, proj_path: Path, mockserver: MockServer) -> None:
_, out, err = proc(
"fetch", "--spider", "myspider", mockserver.url("/echo"), cwd=proj_path
)
assert "myspider-user-agent" in out, err

21
tests/test_command_genspider.py
View File

@ -64,6 +64,24 @@ class TestGenspiderCommand(TestProjectBase):
assert call("genspider", "--dump=basic", cwd=proj_path) == 0
assert call("genspider", "-d", "basic", cwd=proj_path) == 0
@pytest.mark.parametrize(
"args",
[("--dump=nonexistent",), ("-t", "nonexistent", "test_name", "test.com")],
)
def test_unknown_template(self, args: tuple[str, ...], proj_path: Path) -> None:
returncode, out, err = proc("genspider", *args, cwd=proj_path)
assert returncode == 0, err
assert "Unable to find template: nonexistent" in out
assert not (proj_path / self.project_name / "spiders" / "test_name.py").exists()
def test_name_not_starting_with_a_letter(self, proj_path: Path) -> None:
"""The module name, unlike the spider name, is prefixed with a letter."""
_, out, err = proc("genspider", "1st_spider", "test.com", cwd=proj_path)
assert "Created spider '1st_spider'" in out, err
spider = proj_path / self.project_name / "spiders" / "a1st_spider.py"
assert spider.exists()
assert find_in_file(spider, r'name\s*=\s*"1st_spider"') is not None
@pytest.mark.skipif(
sys.platform == "win32", reason="requires a POSIX shell editor script"
)
@ -87,7 +105,8 @@ class TestGenspiderCommand(TestProjectBase):
)
def test_same_name_as_project(self, proj_path: Path) -> None:
assert call("genspider", self.project_name, cwd=proj_path) == 2
_, out, err = proc("genspider", self.project_name, "test.com", cwd=proj_path)
assert "Cannot create a spider with the same name as your project" in out, err
assert not (
proj_path / self.project_name / "spiders" / f"{self.project_name}.py"
).exists()

181
tests/test_command_parse.py
View File

@ -3,6 +3,7 @@ from __future__ import annotations
import argparse
import re
from typing import TYPE_CHECKING
from urllib.parse import urlparse
import pytest
@ -126,6 +127,32 @@ class MySpider(scrapy.Spider):
else:
self.logger.debug('It Does Not Work :(')
class RetryRequestSpider(BaseSpider):
name = 'retry_request'
def parse(self, response):
if response.meta.get('retried'):
yield {{'retried': True}}
return
response.meta['retried'] = True
yield response.request.replace(dont_filter=True)
class CustomCallbackRetryRequestSpider(BaseSpider):
name = 'retry_request_custom_callback'
def parse(self, response):
yield response.request.replace(
callback=self.parse_retry,
dont_filter=True,
)
def parse_retry(self, response):
if response.meta.get('retried'):
yield {{'retried_with_custom_callback': True}}
return
response.meta['retried'] = True
yield response.request.replace(dont_filter=True)
class MyGoodCrawlSpider(CrawlSpider):
name = 'goodcrawl{self.spider_name}'
@ -381,6 +408,36 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
)
assert "[{}, {'foo': 'bar'}]" in out
def test_retry_response_request(
self, proj_path: Path, mockserver: MockServer
) -> None:
_, out, stderr = proc(
"parse",
"--spider",
"retry_request",
"-d",
"2",
mockserver.url("/html"),
cwd=proj_path,
)
assert "RecursionError" not in stderr
assert "{'retried': True}" in out
def test_retry_response_request_with_custom_callback(
self, proj_path: Path, mockserver: MockServer
) -> None:
_, out, stderr = proc(
"parse",
"--spider",
"retry_request_custom_callback",
"-d",
"3",
mockserver.url("/html"),
cwd=proj_path,
)
assert "RecursionError" not in stderr
assert "{'retried_with_custom_callback': True}" in out
def test_wrong_callback_passed(
self, proj_path: Path, mockserver: MockServer
) -> None:
@ -496,6 +553,130 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
content = '[\n{},\n{"foo": "bar"}\n]'
assert file_path.read_text(encoding="utf-8") == content
@pytest.mark.parametrize("args", [(), ("not-a-url",), ("a:b", "c:d")])
def test_bad_arguments(self, args: tuple[str, ...], proj_path: Path) -> None:
returncode, out, _ = proc("parse", *args, cwd=proj_path)
assert returncode == 2
assert "Usage" in out
@pytest.mark.parametrize(
("option", "message"),
[
("--meta", "Invalid -m/--meta value"),
("-m", "Invalid -m/--meta value"),
("--cbkwargs", "Invalid --cbkwargs value"),
],
)
def test_invalid_json(
self, option: str, message: str, proj_path: Path, mockserver: MockServer
) -> None:
returncode, _, err = proc(
"parse",
"--spider",
self.spider_name,
option,
"{invalid",
mockserver.url("/html"),
cwd=proj_path,
)
assert returncode == 2
assert message in err
def test_unknown_spider(self, proj_path: Path, mockserver: MockServer) -> None:
returncode, _, err = proc(
"parse",
"--spider",
"nonexistent",
mockserver.url("/html"),
cwd=proj_path,
)
assert returncode == 0, err
assert "Unable to find spider: nonexistent" in err
def test_spider_found_by_url(self, proj_path: Path, mockserver: MockServer) -> None:
"""Without --spider, the spider is chosen based on the URL."""
url = mockserver.url("/html")
# The spider name doubles as a domain of the spider, and it is matched
# against the netloc of the URL, hence the port.
(proj_path / self.project_name / "spiders" / "urlspider.py").write_text(
f"""
import scrapy
class UrlSpider(scrapy.Spider):
name = "{urlparse(url).netloc}"
def parse(self, response):
return [{{"found_by_url": True}}]
""",
encoding="utf-8",
)
returncode, out, err = proc("parse", url, cwd=proj_path)
assert returncode == 0, err
assert "Unable to find spider for" not in err
assert "{'found_by_url': True}" in out
def test_legacy_item_processor(
self, proj_path: Path, mockserver: MockServer
) -> None:
"""--pipelines supports an ITEM_PROCESSOR without process_item_async()."""
(proj_path / self.project_name / "legacy.py").write_text(
"""
import logging
from twisted.internet.defer import succeed
class LegacyItemProcessor:
@classmethod
def from_crawler(cls, crawler):
return cls()
def open_spider(self, spider):
return succeed(None)
def close_spider(self, spider):
return succeed(None)
def process_item(self, item, spider):
logging.info("Legacy item processor!")
return succeed(item)
""",
encoding="utf-8",
)
_, _, stderr = proc(
"parse",
"--spider",
self.spider_name,
"--pipelines",
"-c",
"parse",
"-s",
f"ITEM_PROCESSOR={self.project_name}.legacy.LegacyItemProcessor",
mockserver.url("/html"),
cwd=proj_path,
)
assert "INFO: Legacy item processor!" in stderr
@pytest.mark.parametrize("verbose", [True, False])
def test_no_items_no_links(
self, verbose: bool, proj_path: Path, mockserver: MockServer
) -> None:
args = ["--verbose"] if verbose else []
_, out, err = proc(
"parse",
"--spider",
self.spider_name,
"-c",
"parse",
"--noitems",
"--nolinks",
*args,
mockserver.url("/html"),
cwd=proj_path,
)
assert "# Scraped Items" not in out, err
assert "# Requests" not in out
def test_parse_add_options(self):
command = parse.Command()
command.settings = Settings()

6
tests/test_command_runspider.py
View File

@ -136,6 +136,12 @@ class MySpider(scrapy.Spider):
log = self.get_log(tmp_path, "from scrapy.spiders import Spider\n")
assert "No spider found in file" in log
@pytest.mark.parametrize("args", [(), ("a.py", "b.py")])
def test_runspider_bad_arguments(self, args: tuple[str, ...]) -> None:
returncode, out, _ = proc("runspider", *args)
assert returncode == 2
assert "Usage" in out
def test_runspider_file_not_found(self) -> None:
_, _, log = proc("runspider", "some_non_existent_file")
assert "File not found: some_non_existent_file" in log

28
tests/test_command_shell.py
View File

@ -18,6 +18,7 @@ from scrapy.shell import Shell, inspect_response
from scrapy.utils.reactor import _asyncio_reactor_path
from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE, tests_datadir
from tests.utils.bases.commands import TestProjectBase
from tests.utils.cmdline import proc
from tests.utils.decorators import coroutine_test
@ -162,6 +163,33 @@ class TestShellCommand:
assert ret == 0, out
class TestShellCommandWithSpider(TestProjectBase):
@pytest.fixture(autouse=True)
def create_files(self, proj_path: Path) -> None:
(proj_path / self.project_name / "spiders" / "myspider.py").write_text(
"""
import scrapy
class MySpider(scrapy.Spider):
name = "myspider"
""",
encoding="utf-8",
)
def test_spider(self, proj_path: Path, mockserver: MockServer) -> None:
ret, out, err = proc(
"shell",
"--spider",
"myspider",
mockserver.url("/text"),
"-c",
"spider.name",
cwd=proj_path,
)
assert ret == 0, err
assert out.strip() == "myspider"
class TestInteractiveShell:
def test_fetch(self, mockserver: MockServer) -> None:
args = (

383
tests/test_commands.py
View File

@ -3,23 +3,27 @@ from __future__ import annotations
import argparse
import json
import sys
from io import StringIO
from pathlib import Path
from typing import TYPE_CHECKING
from unittest import mock
import pytest
import scrapy
from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view
from scrapy.cmdline import _pop_command_name, execute
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings
from scrapy.utils.reactor import _asyncio_reactor_path
from tests.utils.bases.commands import TestProjectBase
from tests.utils.cmdline import call, proc, write_recording_editor
from tests.utils.cmdline import (
call,
proc,
write_recording_browser,
write_recording_editor,
)
if TYPE_CHECKING:
from pathlib import Path
from tests.mockserver.http import MockServer
class EmptyCommand(ScrapyCommand):
@ -109,6 +113,93 @@ class TestCommandSettings:
)
class TestGlobalOptions:
"""Tests for the options that every command supports."""
spider_code = """
import scrapy
class MySpider(scrapy.Spider):
name = "myspider"
async def start(self):
self.logger.debug("It works!")
return
yield
"""
@pytest.fixture
def spider_path(self, tmp_path: Path) -> Path:
path = tmp_path / "myspider.py"
path.write_text(self.spider_code, encoding="utf-8")
return path
def test_invalid_set(self, spider_path: Path) -> None:
returncode, _, err = proc("runspider", str(spider_path), "-s", "FOO")
assert returncode == 2
assert "Invalid -s value, use -s NAME=VALUE" in err
def test_invalid_spider_argument(self, spider_path: Path) -> None:
returncode, _, err = proc("runspider", str(spider_path), "-a", "FOO")
assert returncode == 2
assert "Invalid -a value, use -a NAME=VALUE" in err
def test_logfile(self, tmp_path: Path, spider_path: Path) -> None:
logfile = tmp_path / "scrapy.log"
returncode, _, err = proc(
"runspider", str(spider_path), "--logfile", str(logfile)
)
assert returncode == 0, err
assert "It works!" in logfile.read_text(encoding="utf-8")
assert "It works!" not in err
def test_loglevel(self, spider_path: Path) -> None:
returncode, _, err = proc("runspider", str(spider_path), "--loglevel", "INFO")
assert returncode == 0, err
assert "It works!" not in err
assert "Spider closed (finished)" in err
def test_nolog(self, spider_path: Path) -> None:
returncode, _, err = proc("runspider", str(spider_path), "--nolog")
assert returncode == 0, err
assert not err
def test_pidfile(self, tmp_path: Path, spider_path: Path) -> None:
pidfile = tmp_path / "scrapy.pid"
returncode, _, err = proc(
"runspider", str(spider_path), "--pidfile", str(pidfile)
)
assert returncode == 0, err
assert pidfile.read_text(encoding="utf-8").strip().isdigit()
def test_pdb(self, spider_path: Path) -> None:
returncode, _, err = proc("runspider", str(spider_path), "--pdb")
assert returncode == 0, err
assert "It works!" in err
class TestSettingsCommand:
@pytest.mark.parametrize(
("option", "setting", "expected"),
[
("--get", "BOT_NAME", "scrapybot"),
("--getbool", "COOKIES_ENABLED", "True"),
("--getint", "CONCURRENT_REQUESTS", "16"),
("--getfloat", "DOWNLOAD_DELAY", "0.0"),
("--getlist", "SPIDER_MODULES", "[]"),
],
)
def test_get(self, option: str, setting: str, expected: str) -> None:
returncode, out, err = proc("settings", option, setting)
assert returncode == 0, err
assert out.startswith(expected)
def test_no_option(self) -> None:
returncode, out, err = proc("settings")
assert returncode == 0, err
assert not out
class TestCommandCrawlerProcess(TestProjectBase):
"""Test that the command uses the expected kind of *CrawlerProcess
and produces expected errors when needed."""
@ -153,12 +244,6 @@ class MySpider(scrapy.Spider):
self._append_settings(proj_mod_path, "LOG_LEVEL = 'DEBUG'\n")
@staticmethod
def _append_settings(proj_mod_path: Path, text: str) -> None:
"""Add text to the end of the project settings.py."""
with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f:
f.write(text)
@staticmethod
def _replace_custom_settings(
proj_mod_path: Path, spider_name: str, text: str
@ -347,23 +432,223 @@ class TestMiscCommands(TestProjectBase):
subdir.mkdir(exist_ok=True)
assert call("list", cwd=subdir) == 0
def test_command_not_found(self) -> None:
na_msg = """
The list command is not available from this location.
These commands are only available from within a project: check, crawl, edit, list, parse.
"""
not_found_msg = """
Unknown command: abc
"""
params = [
("list", False, na_msg),
("abc", False, not_found_msg),
("abc", True, not_found_msg),
]
for cmdname, inproject, message in params:
with mock.patch("sys.stdout", new=StringIO()) as out:
_print_unknown_command_msg(Settings(), cmdname, inproject)
assert out.getvalue().strip() == message.strip()
class TestCommandListing(TestProjectBase):
"""Tests for the command list that ``scrapy`` prints when called without a
command name."""
def test_outside_project(self) -> None:
returncode, out, err = proc()
assert returncode == 0, err
assert f"Scrapy {scrapy.__version__} - no active project" in out
assert "Available commands:" in out
assert "Create new project" in out
assert "More commands available when run from project directory" in out
assert 'Use "scrapy <command> -h" to see more info about a command' in out
def test_inside_project(self, proj_path: Path) -> None:
returncode, out, err = proc(cwd=proj_path)
assert returncode == 0, err
assert (
f"Scrapy {scrapy.__version__} - active project: {self.project_name}" in out
)
assert "List available spiders" in out
assert "More commands available when run from project directory" not in out
class TestUnknownCommand(TestProjectBase):
def test_outside_project(self) -> None:
returncode, out, err = proc("abc")
assert returncode == 2, err
assert f"Scrapy {scrapy.__version__} - no active project" in out
assert "Unknown command: abc" in out
assert 'Use "scrapy" to see available commands' in out
def test_inside_project(self, proj_path: Path) -> None:
returncode, out, err = proc("abc", cwd=proj_path)
assert returncode == 2, err
assert (
f"Scrapy {scrapy.__version__} - active project: {self.project_name}" in out
)
assert "Unknown command: abc" in out
def test_project_only_command_outside_project(self) -> None:
returncode, out, err = proc("list")
assert returncode == 2, err
assert "The list command is not available from this location." in out
assert (
"These commands are only available from within a project: "
"check, crawl, edit, list, parse." in out
)
class TestCommandsModule(TestProjectBase):
"""Tests for commands defined in the module of the COMMANDS_MODULE setting."""
@pytest.fixture
def proj_path_with_commands(self, proj_path: Path) -> Path:
commands_path = proj_path / self.project_name / "commands"
commands_path.mkdir()
(commands_path / "__init__.py").touch()
(commands_path / "mycmd.py").write_text(
"""
from scrapy.commands import ScrapyCommand
class Command(ScrapyCommand):
requires_crawler_process = False
def short_desc(self):
return "My custom command"
def run(self, args, opts):
print("My custom command ran")
""",
encoding="utf-8",
)
(commands_path / "helpcmd.py").write_text(
"""
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
class Command(ScrapyCommand):
requires_crawler_process = False
def short_desc(self):
return "My command that asks for its help message"
def run(self, args, opts):
raise UsageError
""",
encoding="utf-8",
)
(commands_path / "silentcmd.py").write_text(
"""
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
class Command(ScrapyCommand):
requires_crawler_process = False
def short_desc(self):
return "My command that fails silently"
def run(self, args, opts):
raise UsageError(print_help=False)
""",
encoding="utf-8",
)
self._append_settings(
proj_path / self.project_name,
f'\nCOMMANDS_MODULE = "{self.project_name}.commands"\n',
)
return proj_path
def test_listed(self, proj_path_with_commands: Path) -> None:
returncode, out, err = proc(cwd=proj_path_with_commands)
assert returncode == 0, err
assert "My custom command" in out
def test_run(self, proj_path_with_commands: Path) -> None:
returncode, out, err = proc("mycmd", cwd=proj_path_with_commands)
assert returncode == 0, err
assert "My custom command ran" in out
def test_usage_error(self, proj_path_with_commands: Path) -> None:
"""A message-less UsageError makes the help message be printed."""
returncode, out, err = proc("helpcmd", cwd=proj_path_with_commands)
assert returncode == 2, err
assert "scrapy helpcmd" in out
def test_usage_error_without_help(self, proj_path_with_commands: Path) -> None:
"""A message-less UsageError with print_help disabled prints nothing."""
returncode, out, err = proc("silentcmd", cwd=proj_path_with_commands)
assert returncode == 2, err
assert not out
class TestEntryPointCommands:
"""Tests for commands defined in the scrapy.commands entry point group."""
@staticmethod
def _write_dist(path: Path, entry_point: str) -> None:
"""Write into *path* a package with a command and a function, and the
metadata of an installed distribution that declares *entry_point* in
the scrapy.commands entry point group.
Since ``python -m scrapy.cmdline`` puts the current working directory
in the import path, running it with *path* as the working directory
makes Scrapy find that entry point.
"""
package_path = path / "mycmds"
package_path.mkdir()
(package_path / "__init__.py").touch()
(package_path / "mycmd.py").write_text(
"""
from scrapy.commands import ScrapyCommand
class Command(ScrapyCommand):
requires_crawler_process = False
def short_desc(self):
return "My entry point command"
def run(self, args, opts):
print("My entry point command ran")
def not_a_command():
pass
""",
encoding="utf-8",
)
dist_info_path = path / "mycmds-1.0.dist-info"
dist_info_path.mkdir()
(dist_info_path / "METADATA").write_text(
"Metadata-Version: 2.1\nName: mycmds\nVersion: 1.0\n", encoding="utf-8"
)
(dist_info_path / "entry_points.txt").write_text(
f"[scrapy.commands]\n{entry_point}\n", encoding="utf-8"
)
def test_listed(self, tmp_path: Path) -> None:
self._write_dist(tmp_path, "mycmd = mycmds.mycmd:Command")
returncode, out, err = proc(cwd=tmp_path)
assert returncode == 0, err
assert "My entry point command" in out
def test_run(self, tmp_path: Path) -> None:
self._write_dist(tmp_path, "mycmd = mycmds.mycmd:Command")
returncode, out, err = proc("mycmd", cwd=tmp_path)
assert returncode == 0, err
assert "My entry point command ran" in out
def test_not_a_class(self, tmp_path: Path) -> None:
self._write_dist(tmp_path, "mycmd = mycmds.mycmd:not_a_command")
returncode, _, err = proc("version", cwd=tmp_path)
assert returncode == 1
assert "ValueError: Invalid entry point mycmd" in err
class TestExecute:
"""Tests for calls to scrapy.cmdline.execute() from Python code, which the
command line does not cover."""
def test_argv(self, capsys: pytest.CaptureFixture[str]) -> None:
with pytest.raises(SystemExit) as exc_info:
execute(["scrapy", "version"])
assert exc_info.value.code == 0
assert scrapy.__version__ in capsys.readouterr().out
def test_settings(self, capsys: pytest.CaptureFixture[str]) -> None:
settings = Settings()
with pytest.raises(SystemExit) as exc_info:
execute(["scrapy", "settings", "--get", "BOT_NAME"], settings=settings)
assert exc_info.value.code == 0
assert capsys.readouterr().out.strip() == "scrapybot"
class TestBenchCommand:
@ -385,18 +670,31 @@ class TestBenchCommand:
class TestViewCommand:
def test_methods(self) -> None:
command = view.Command()
command.settings = Settings()
parser = argparse.ArgumentParser(
prog="scrapy",
prefix_chars="-",
formatter_class=ScrapyHelpFormatter,
conflict_handler="resolve",
@pytest.mark.skipif(
sys.platform == "win32", reason="requires a POSIX shell browser script"
)
def test_view(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mockserver: MockServer
) -> None:
opened = tmp_path / "opened.txt"
browser = tmp_path / "fake-browser.sh"
write_recording_browser(browser, opened)
monkeypatch.setenv("BROWSER", str(browser))
returncode, _, err = proc("view", mockserver.url("/html"), cwd=tmp_path)
assert returncode == 0, err
url = opened.read_text(encoding="utf-8")
assert url.startswith("file://")
body = Path(url.removeprefix("file://")).read_text(encoding="utf-8")
assert "<p class='one'>Works</p>" in body
def test_non_text_response(self, mockserver: MockServer) -> None:
returncode, _, err = proc(
"view", mockserver.url("/static/files/images/scrapy.png")
)
command.add_options(parser)
assert command.short_desc() == "Open URL in browser, as seen by Scrapy"
assert "URL using the Scrapy downloader and show its" in command.long_desc()
assert returncode == 0, err
assert "Cannot view a non-text response." in err
class TestEditCommand(TestProjectBase):
@ -423,6 +721,11 @@ class TestEditCommand(TestProjectBase):
assert returncode == 1
assert "Spider not found: nonexistent" in err
def test_edit_no_spider(self, proj_path: Path) -> None:
returncode, out, _ = proc("edit", cwd=proj_path)
assert returncode == 2
assert "Usage" in out
class TestHelpMessage(TestProjectBase):
@pytest.mark.parametrize(

16
tests/test_crawl.py
View File

@ -41,6 +41,7 @@ from tests.spiders import (
CrawlSpiderWithAsyncCallback,
CrawlSpiderWithAsyncGeneratorCallback,
CrawlSpiderWithErrback,
CrawlSpiderWithoutErrback,
CrawlSpiderWithParseMethod,
CrawlSpiderWithProcessRequestCallbackKeywordArguments,
DelaySpider,
@ -505,6 +506,21 @@ class TestCrawlSpider:
assert "[errback] status 500" in caplog.text
assert "[errback] status 501" in caplog.text
@coroutine_test
async def test_crawlspider_without_errback(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
) -> None:
crawler = get_crawler(CrawlSpiderWithoutErrback)
with caplog.at_level(logging.INFO):
await crawler.crawl_async(mockserver=mockserver)
# The failing request (404) is followed by a rule without an errback,
# so the failure is dropped silently and the crawl finishes normally.
assert "[parse] status 200 (foo: None)" in caplog.text
assert "[errback]" not in caplog.text
assert crawler.stats
assert crawler.stats.get_value("downloader/response_status_count/404") == 1
@coroutine_test
async def test_crawlspider_process_request_cb_kwargs(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer

7
tests/test_crawler_subprocess.py
View File

@ -14,7 +14,8 @@ from packaging.version import parse as parse_version
from pexpect.popen_spawn import PopenSpawn
from w3lib import __version__ as w3lib_version
from tests.utils import async_sleep, get_script_run_env
from scrapy.utils.asyncio import sleep
from tests.utils import get_script_run_env
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
@ -244,10 +245,10 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
p.kill(sig)
p.expect_exact("shutting down gracefully")
# sending a new signal too fast often causes problems
await async_sleep(0.01)
await sleep(0.01)
p.kill(sig)
p.expect_exact("dropping downloader requests")
await async_sleep(0.01)
await sleep(0.01)
p.kill(sig)
p.expect_exact("forcing unclean shutdown", timeout=20)
p.wait() # type: ignore[no-untyped-call]

148
tests/test_downloadermiddleware_httpcache.py
View File

@ -1,10 +1,12 @@
from __future__ import annotations
import email.utils
import logging
import shutil
import tempfile
import time
from contextlib import contextmanager
from pathlib import Path
from typing import TYPE_CHECKING, Any
from unittest import mock
@ -12,6 +14,7 @@ import pytest
from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware
from scrapy.exceptions import IgnoreRequest
from scrapy.extensions.httpcache import DummyPolicy
from scrapy.http import HtmlResponse, Request, Response
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
@ -22,6 +25,14 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
class AlwaysStalePolicy(DummyPolicy):
""":class:`~scrapy.extensions.httpcache.DummyPolicy` that always
revalidates cached responses."""
def is_cached_response_fresh(self, cachedresponse, request):
return False
class TestBase:
"""Base class with common setup and helper methods."""
@ -93,6 +104,12 @@ class TestBase:
class StorageTestMixin:
"""Mixin containing storage-specific test methods."""
def _corrupt_cache_entry(
self, storage: Any, spider: Spider, request: Request
) -> None:
"""Make the cache entry of *request* unreadable for *storage*."""
raise NotImplementedError
def test_storage(self):
with self._storage(HTTPCACHE_EXPIRATION_SECS=1) as (storage, crawler):
request2 = self.request.copy()
@ -115,6 +132,38 @@ class StorageTestMixin:
with mock.patch("scrapy.extensions.httpcache.time", return_value=future):
assert storage.retrieve_response(crawler.spider, self.request)
def test_corrupted_cache_entry_is_a_miss(self, caplog):
with self._middleware() as mw:
spider = mw.crawler.spider
mw.storage.store_response(spider, self.request, self.response)
self._corrupt_cache_entry(mw.storage, spider, self.request)
caplog.clear()
with caplog.at_level(logging.WARNING):
assert mw.process_request(self.request) is None
assert "treating it as a cache miss" in caplog.text
assert mw.crawler.stats.get_value("httpcache/retrieve_error") == 1
assert mw.crawler.stats.get_value("httpcache/miss") == 1
# Storing the response again replaces the corrupted cache entry.
mw.storage.store_response(spider, self.request, self.response)
self.assertEqualResponse(
self.response, mw.storage.retrieve_response(spider, self.request)
)
def test_corrupted_cache_entry_ignore_missing(self):
with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:
spider = mw.crawler.spider
mw.storage.store_response(spider, self.request, self.response)
self._corrupt_cache_entry(mw.storage, spider, self.request)
with pytest.raises(IgnoreRequest):
mw.process_request(self.request)
assert mw.crawler.stats.get_value("httpcache/retrieve_error") == 1
assert mw.crawler.stats.get_value("httpcache/ignore") == 1
def test_storage_no_content_type_header(self):
"""Test that the response body is used to get the right response class
even if there is no Content-Type header"""
@ -242,6 +291,21 @@ class DummyPolicyTestMixin(PolicyTestMixin):
self.assertEqualResponse(self.response, response)
assert "cached" in response.flags
def test_revalidation_keeps_cached_response(self):
# The dummy policy considers every cached response valid, so a policy
# that subclasses it to force revalidation always gets the cached
# response back, whatever the new response is.
with self._middleware(HTTPCACHE_POLICY=AlwaysStalePolicy) as mw:
assert mw.process_request(self.request) is None
mw.process_response(self.request, self.response)
assert mw.process_request(self.request) is None
fresh_response = self.response.replace(body=b"new body")
response = mw.process_response(self.request, fresh_response)
self.assertEqualResponse(self.response, response)
assert "cached" in response.flags
assert mw.stats.get_value("httpcache/revalidate") == 1
class RFC2616PolicyTestMixin(PolicyTestMixin):
"""Mixin containing RFC2616 policy specific test methods."""
@ -513,6 +577,53 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
else:
assert "cached" in res5.flags
def test_middleware_ignore_schemes(self):
# file responses are not cached by default
req = Request("file:///tmp/t.txt")
res = Response(req.url, headers={"Expires": self.tomorrow})
with self._middleware() as mw:
assert mw.process_request(req) is None
mw.process_response(req, res)
assert mw.storage.retrieve_response(mw.crawler.spider, req) is None
assert mw.process_request(req) is None
def test_max_stale_with_value(self):
# A response that expired one day ago.
headers = {"Date": self.yesterday, "Expires": self.yesterday}
with self._middleware() as mw:
req0 = Request("http://example.com")
res0 = Response(req0.url, headers=headers)
self._process_requestresponse(mw, req0, res0)
# max-stale greater than the staleness of the cached response
req1 = req0.replace(headers={"Cache-Control": "max-stale=172800"})
res1 = mw.process_request(req1)
assert isinstance(res1, Response)
assert "cached" in res1.flags
# max-stale lower than the staleness of the cached response
req2 = req0.replace(headers={"Cache-Control": "max-stale=60"})
assert mw.process_request(req2) is None
# a non-integer max-stale value is ignored
req3 = req0.replace(headers={"Cache-Control": "max-stale=soon"})
assert mw.process_request(req3) is None
def test_response_dated_in_the_future(self):
# A Date header ahead of the local clock must not make the cached
# response look aged.
headers = {"Date": self.tomorrow, "Cache-Control": "max-age=10"}
with self._middleware() as mw:
req0 = Request("http://example.com")
res0 = Response(req0.url, headers=headers)
res1 = self._process_requestresponse(mw, req0, res0)
assert "cached" not in res1.flags
res2 = self._process_requestresponse(mw, req0, None)
self.assertEqualResponse(res1, res2)
assert "cached" in res2.flags
def test_process_exception(self):
with self._middleware() as mw:
res0 = Response(self.request.url, headers={"Expires": self.yesterday})
@ -556,29 +667,43 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
# Concrete test classes that combine storage and policy mixins
class TestFilesystemStorageWithDummyPolicy(
TestBase, StorageTestMixin, DummyPolicyTestMixin
):
class FilesystemStorageTestMixin(StorageTestMixin):
storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"
def _corrupt_cache_entry(self, storage, spider, request) -> None:
rpath = Path(storage._get_request_path(spider, request))
(rpath / "response_body").unlink()
class DbmStorageTestMixin(StorageTestMixin):
storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
def _corrupt_cache_entry(self, storage, spider, request) -> None:
key = storage._fingerprinter.fingerprint(request).hex()
storage.db[f"{key}_data"] = b"not a pickle"
class TestFilesystemStorageWithDummyPolicy(
TestBase, FilesystemStorageTestMixin, DummyPolicyTestMixin
):
policy_class = "scrapy.extensions.httpcache.DummyPolicy"
class TestFilesystemStorageWithRFC2616Policy(
TestBase, StorageTestMixin, RFC2616PolicyTestMixin
TestBase, FilesystemStorageTestMixin, RFC2616PolicyTestMixin
):
storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"
policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
class TestDbmStorageWithDummyPolicy(TestBase, StorageTestMixin, DummyPolicyTestMixin):
storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
class TestDbmStorageWithDummyPolicy(
TestBase, DbmStorageTestMixin, DummyPolicyTestMixin
):
policy_class = "scrapy.extensions.httpcache.DummyPolicy"
class TestDbmStorageWithRFC2616Policy(
TestBase, StorageTestMixin, RFC2616PolicyTestMixin
TestBase, DbmStorageTestMixin, RFC2616PolicyTestMixin
):
storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
@ -599,3 +724,8 @@ class TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPol
def _get_settings(self, **new_settings) -> dict[str, Any]:
new_settings.setdefault("HTTPCACHE_GZIP", True)
return super()._get_settings(**new_settings)
def _corrupt_cache_entry(self, storage, spider, request) -> None:
# A spider killed while writing a gzip file leaves it truncated.
body_path = Path(storage._get_request_path(spider, request), "response_body")
body_path.write_bytes(body_path.read_bytes()[:-5])

17
tests/test_downloadermiddleware_robotstxt.py
View File

@ -8,6 +8,7 @@ import pytest
from twisted.internet.defer import Deferred, DeferredList
from twisted.python import failure
from scrapy import signals
from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware
from scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfigured
from scrapy.http import Request, Response, TextResponse
@ -27,6 +28,7 @@ class TestRobotsTxtMiddleware:
self.crawler: mock.MagicMock = mock.MagicMock()
self.crawler.settings = Settings()
self.crawler.engine.download_async = mock.AsyncMock()
self.crawler.signals.send_catch_log_async = mock.AsyncMock(return_value=[])
def teardown_method(self):
del self.crawler
@ -74,6 +76,21 @@ Disallow: /some/randome/page.html
Request("http://site.local/wiki/Käyttäjä:"), middleware
)
@coroutine_test
async def test_robotstxt_emits_robots_parsed_signal(self):
crawler = self._get_successful_crawler()
middleware = RobotsTxtMiddleware(crawler)
request = Request("http://site.local/allowed")
await self.assertNotIgnored(request, middleware)
calls = [
kwargs
for _, kwargs in crawler.signals.send_catch_log_async.call_args_list
if kwargs.get("signal") is signals.robots_parsed
]
assert len(calls) == 1
assert calls[0]["request"] is request
assert calls[0]["robotparser"] is not None
@coroutine_test
async def test_robotstxt_multiple_reqs(self) -> None:
middleware = RobotsTxtMiddleware(self._get_successful_crawler())

4
tests/test_engine.py
View File

@ -17,6 +17,7 @@ from scrapy.core.scheduler import BaseScheduler
from scrapy.exceptions import CloseSpider, DownloadCancelledError, IgnoreRequest
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.asyncio import sleep
from scrapy.utils.defer import (
_schedule_coro,
deferred_from_coro,
@ -24,7 +25,6 @@ from scrapy.utils.defer import (
)
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.utils import async_sleep
from tests.utils.bases.engine import TestEngineBase
from tests.utils.decorators import coroutine_test, inline_callbacks_test
from tests.utils.engine import (
@ -165,7 +165,7 @@ class TestEngine(TestEngineBase):
engine, "close_spider_async", new_callable=AsyncMock
) as close:
stop_dfd = deferred_from_coro(engine.stop_async(mode="fast"))
await async_sleep(0)
await sleep(0)
close.assert_called_once_with(reason="shutdown", mode="fast")
assert not stop_dfd.called

11
tests/test_engine_loop.py
View File

@ -6,10 +6,9 @@ from typing import TYPE_CHECKING, Any
from scrapy import Request, Spider, signals
from scrapy.core.scheduler import BaseScheduler
from scrapy.utils.asyncio import call_later
from scrapy.utils.asyncio import call_later, sleep
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.utils import async_sleep
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
@ -65,23 +64,23 @@ class TestMain:
async def start(self):
yield Request("data:,a")
await async_sleep(seconds)
await sleep(seconds)
self.crawler.engine._slot.scheduler.pause()
self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b"))
# During this time, the scheduler reports having requests but
# returns None.
await async_sleep(seconds)
await sleep(seconds)
self.crawler.engine._slot.scheduler.unpause()
# The scheduler request is processed.
await async_sleep(seconds)
await sleep(seconds)
yield Request("data:,c")
await async_sleep(seconds)
await sleep(seconds)
self.crawler.engine._slot.scheduler.pause()
self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d"))

12
tests/test_exporters.py
View File

@ -118,6 +118,18 @@ class TestBaseItemExporter(ABC):
ie = self._get_exporter(fields_to_export={"name": "名稱"})
assert list(ie._get_serialized_fields(self.i)) == [("名稱", "John\xa3")]
def test_field_order(self):
item = self.item_class(age="22", name="John\xa3")
ie = self._get_exporter()
assert [name for name, _ in ie._get_serialized_fields(item)] == ["name", "age"]
def test_field_order_dict_item(self):
ie = self._get_exporter()
assert [name for name, _ in ie._get_serialized_fields({"age": "22"})] == ["age"]
assert [
name for name, _ in ie._get_serialized_fields({"age": "22", "name": "John"})
] == ["age", "name"]
def test_field_custom_serializer(self):
i = self.custom_field_item_class(name="John\xa3", age="22")
a = ItemAdapter(i)

66
tests/test_extension_memusage.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import logging
import sys
from typing import TYPE_CHECKING
import pytest
@ -13,8 +14,12 @@ from scrapy.extensions.memusage import MemoryUsage
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
from tests.utils import OneShotLoop
from tests.utils.cmdline import proc
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from tests.mockserver.http import MockServer
# MemoryUsage relies on the stdlib 'resource' module (not available on Windows)
pytestmark = pytest.mark.skipif(
sys.platform.startswith("win"),
@ -25,6 +30,14 @@ pytestmark = pytest.mark.skipif(
MB = 1024 * 1024
class TwoShotLoop(OneShotLoop):
"""Like :class:`OneShotLoop`, but runs the check twice."""
def start(self, interval: float, now: bool = True) -> None:
super().start(interval, now=now)
self.func()
class _LoopSpider(Spider):
name = "loop-data-spider"
@ -50,6 +63,49 @@ def test_memusage_disabled() -> None:
MemoryUsage.from_crawler(get_crawler(settings_dict=settings))
def test_memusage_limit_stops_crawler_without_spider(mockserver: MockServer) -> None:
# The Scrapy shell starts the engine without opening a spider, so the
# whole crawler is stopped instead of a spider being closed.
_, out, err = proc(
"shell",
mockserver.url("/text"),
"-c",
"response.status",
"--set",
"MEMUSAGE_LIMIT_MB=1",
)
assert "Memory usage exceeded 1MiB" in err
assert "200" in out
@coroutine_test
async def test_memusage_below_thresholds_logs_peak(
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
) -> None:
settings = {
"MEMUSAGE_LIMIT_MB": 100,
"MEMUSAGE_WARNING_MB": 50,
"MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01,
"TELNETCONSOLE_ENABLED": False,
"LOG_LEVEL": "INFO",
}
monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop)
monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda _: 25 * MB)
crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings)
with caplog.at_level(logging.INFO, logger="scrapy.extensions.memusage"):
await crawler.crawl_async(url="data:,", loops=1)
assert crawler.stats
assert crawler.stats.get_value("memusage/limit_reached") is None
assert crawler.stats.get_value("memusage/warning_reached") is None
assert crawler.stats.get_value("memusage/max") == 25 * MB
assert crawler.stats.get_value("finish_reason") == "finished"
assert any("Peak memory usage is 25MiB" in r.getMessage() for r in caplog.records)
@coroutine_test
async def test_memusage_limit_closes_spider_with_reason_and_error_log(
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
@ -92,8 +148,9 @@ async def test_memusage_warning_logs_but_allows_normal_finish(
"LOG_LEVEL": "INFO",
}
# Avoid background LoopingCall that can log after the test finishes.
monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop)
# Avoid background LoopingCall that can log after the test finishes; check
# twice, since the warning is only meant to be reported once.
monkeypatch.setattr(memusage_mod, "create_looping_call", TwoShotLoop)
monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda self: 75 * MB)
crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings)
@ -112,4 +169,7 @@ async def test_memusage_warning_logs_but_allows_normal_finish(
assert crawler.stats
assert crawler.stats.get_value("memusage/warning_reached") == 1
assert crawler.stats.get_value("finish_reason") == "finished"
assert any("memory usage reached" in r.getMessage().lower() for r in caplog.records)
warnings_logged = [
r for r in caplog.records if "memory usage reached" in r.getMessage().lower()
]
assert len(warnings_logged) == 1

36
tests/test_extension_periodic_log.py
View File

@ -1,8 +1,13 @@
from __future__ import annotations
import datetime
import json
import logging
from typing import TYPE_CHECKING, Any
import pytest
from scrapy.exceptions import NotConfigured
from scrapy.extensions.periodic_log import PeriodicLog
from scrapy.utils.test import get_crawler
@ -86,6 +91,14 @@ class TestPeriodicLog:
assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60})
assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60})
def test_no_interval(self):
with pytest.raises(NotConfigured):
extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 0})
def test_nothing_enabled(self):
with pytest.raises(NotConfigured):
extension({"LOGSTATS_INTERVAL": 60})
@coroutine_test
async def test_log_delta(self):
def emulate(
@ -212,3 +225,26 @@ class TestPeriodicLog:
{"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}},
lambda k, v: "downloader/" in k and "bytes" not in k,
)
@coroutine_test
async def test_log_timing(self, caplog: pytest.LogCaptureFixture) -> None:
settings = {
"EXTENSIONS": {"scrapy.extensions.periodic_log.PeriodicLog": 0},
"PERIODIC_LOG_TIMING_ENABLED": True,
"LOGSTATS_INTERVAL": 30,
}
crawler = get_crawler(MetaSpider, settings)
with caplog.at_level(logging.INFO, logger="scrapy.extensions.periodic_log"):
await crawler.crawl_async()
records = [
r for r in caplog.records if r.name == "scrapy.extensions.periodic_log"
]
assert records, "PeriodicLog logged nothing"
# Only the timing section is enabled, and it is logged on spider close.
data = json.loads(records[-1].getMessage())
assert list(data) == ["time"]
assert data["time"]["log_interval"] == 30
assert data["time"]["log_interval_real"] >= 0
assert data["time"]["elapsed"] >= 0
assert data["time"]["start_time"] <= data["time"]["utcnow"]

55
tests/test_extension_telnet.py
View File

@ -7,7 +7,8 @@ import pytest
from twisted.conch.telnet import ITelnetProtocol
from twisted.cred import credentials
from scrapy.extensions.telnet import TelnetConsole
from scrapy import Spider
from scrapy.extensions.telnet import TelnetConsole, update_telnet_vars
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
@ -16,16 +17,20 @@ if TYPE_CHECKING:
from collections.abc import Generator
from scrapy.crawler import Crawler
from scrapy.http import Response
pytestmark = pytest.mark.requires_reactor # TelnetConsole requires a reactor
def _get_crawler(settings_dict: dict[str, Any] | None = None) -> Crawler:
def _get_crawler(
spidercls: type[Spider] | None = None,
settings_dict: dict[str, Any] | None = None,
) -> Crawler:
settings = {
"TELNETCONSOLE_ENABLED": True,
**(settings_dict or {}),
}
return get_crawler(settings_dict=settings)
return get_crawler(spidercls, settings_dict=settings)
@contextmanager
@ -84,3 +89,47 @@ def test_invalid_reversed_portrange() -> None:
console = TelnetConsole(_get_crawler(settings_dict=settings))
with pytest.raises(ValueError, match=r"invalid portrange: \[2, 1\]"):
console.start_listening()
@coroutine_test
async def test_telnet_vars() -> None:
"""Log into the console of a running crawl, which is when the telnet
variables are built."""
received: list[dict[str, Any]] = []
def on_update_telnet_vars(telnet_vars: dict[str, Any]) -> None:
received.append(telnet_vars)
class TelnetSpider(Spider):
name = "telnet"
start_urls = ["data:,"]
async def parse(self, response: Response) -> None:
assert self.crawler.extensions
console = next(
ext
for ext in self.crawler.extensions.middlewares
if isinstance(ext, TelnetConsole)
)
creds = credentials.UsernamePassword(
console.username.encode("utf8"), console.password.encode("utf8")
)
portal = console.protocol().protocolArgs[0]
await maybe_deferred_to_future(portal.login(creds, None, ITelnetProtocol))
crawler = _get_crawler(TelnetSpider)
crawler.signals.connect(on_update_telnet_vars, signal=update_telnet_vars)
await crawler.crawl_async()
assert len(received) == 1
telnet_vars = received[0]
assert telnet_vars["crawler"] is crawler
assert telnet_vars["engine"] is crawler.engine
assert telnet_vars["spider"] is crawler.spider
assert telnet_vars["extensions"] is crawler.extensions
assert telnet_vars["stats"] is crawler.stats
assert telnet_vars["settings"] is crawler.settings
assert callable(telnet_vars["est"])
assert callable(telnet_vars["p"])
assert callable(telnet_vars["prefs"])
assert "telnetconsole.html" in telnet_vars["help"]

38
tests/test_feedexport.py
View File

@ -24,6 +24,7 @@ from scrapy.extensions.feedexport import (
FeedExporter,
FeedSlot,
FileFeedStorage,
ItemFilter,
apply_uri_params,
)
from scrapy.utils.python import to_unicode
@ -675,14 +676,14 @@ class TestFeedExport(TestFeedExportBase):
formats = {
"csv": b"foo,egg,baz\r\nbar1,spam1,\r\n",
"json": b'[\n{"hello": "world2", "foo": "bar2"}\n]',
"json": b'[\n{"foo": "bar2", "hello": "world2"}\n]',
"jsonlines": (
b'{"foo": "bar1", "egg": "spam1"}\n{"hello": "world2", "foo": "bar2"}\n'
b'{"foo": "bar1", "egg": "spam1"}\n{"foo": "bar2", "hello": "world2"}\n'
),
"xml": (
b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'
b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>"
b"world2</hello><foo>bar2</foo></item>\n<item><hello>world3"
b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><foo>"
b"bar2</foo><hello>world2</hello></item>\n<item><hello>world3"
b"</hello><egg>spam3</egg></item>\n</items>"
),
}
@ -740,8 +741,8 @@ class TestFeedExport(TestFeedExportBase):
"json": b'[\n{"foo": "bar1", "egg": "spam1"}\n]',
"xml": (
b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'
b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>"
b"world2</hello><foo>bar2</foo></item>\n</items>"
b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><foo>"
b"bar2</foo><hello>world2</hello></item>\n</items>"
),
"jsonlines": b'{"foo": "bar1", "egg": "spam1"}\n',
}
@ -1289,6 +1290,13 @@ class TestFeedExporterSignals:
assert self.feed_exporter_closed_received
class TestItemFilter:
def test_no_feed_options(self):
item_filter = ItemFilter(None)
assert item_filter.item_classes == ()
assert item_filter.accepts(MyItem({"foo": "bar"}))
class TestFeedExportInit:
def test_unsupported_storage(self):
settings = {
@ -1300,6 +1308,24 @@ class TestFeedExportInit:
with pytest.raises(NotConfigured):
FeedExporter.from_crawler(crawler)
def test_disabled_storage(self, caplog: pytest.LogCaptureFixture):
class DisabledFeedStorage:
def __init__(self, uri, *, feed_options=None):
raise NotConfigured("not today")
settings = {
"FEED_STORAGES": {"disabled": DisabledFeedStorage},
"FEEDS": {
"disabled://uri": {},
},
}
crawler = get_crawler(settings_dict=settings)
with caplog.at_level(logging.ERROR), pytest.raises(NotConfigured):
FeedExporter.from_crawler(crawler)
assert (
"Disabled feed storage scheme: disabled. Reason: not today" in caplog.text
)
def test_unsupported_format(self):
settings = {
"FEEDS": {

41
tests/test_feedexport_batch.py
View File

@ -210,6 +210,47 @@ class TestBatchDeliveries(TestFeedExportBase):
header = MyItem.fields.keys()
await self.assertExported(items, header, rows, settings=settings)
@coroutine_test
async def test_batch_delivered_when_full(self):
"""Full batches must be finalized and delivered as soon as they are
full, instead of when the spider closes."""
dir_path = self._random_temp_filename()
batch1_path = Path(dir_path, "1.json")
mockserver_url = self.mockserver.url("/")
batch1_contents: list[bytes | None] = []
class TestSpider(scrapy.Spider):
name = "testspider"
start_urls = [mockserver_url]
def parse(self, response):
yield {"foo": "bar1"}
yield {"foo": "bar2"}
yield scrapy.Request(
mockserver_url, callback=self.parse2, dont_filter=True
)
def parse2(self, response):
# the first batch was full after the second item, so it must
# have been delivered by now
batch1_contents.append(
batch1_path.read_bytes() if batch1_path.exists() else None
)
yield {"foo": "bar3"}
settings = {
"FEEDS": {
build_url(dir_path / "%(batch_id)d.json"): {"format": "json"},
},
"FEED_EXPORT_BATCH_ITEM_COUNT": 2,
}
crawler = get_crawler(TestSpider, settings)
await crawler.crawl_async()
assert batch1_contents, "the second request was not processed"
assert batch1_contents[0] is not None, "batch 1 was not stored during the crawl"
assert json.loads(batch1_contents[0]) == [{"foo": "bar1"}, {"foo": "bar2"}]
def test_wrong_path(self):
"""If path is without %(batch_time)s and %(batch_id) an exception must be raised"""
settings = {

10
tests/test_feedexport_postprocess.py
View File

@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any
import pytest
from scrapy.extensions.postprocessing import PostProcessingManager
from scrapy.utils.test import get_crawler
from tests.utils.bases.feedexport import TestFeedExportBase
from tests.utils.decorators import coroutine_test
@ -87,6 +88,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
data_stream.seek(0)
return data_stream.read()
def test_tell_reports_target_file_position(self):
"""Exporters that wrap the file they get, e.g. through
:class:`io.TextIOWrapper`, need it to report a position."""
file = BytesIO()
manager = PostProcessingManager([self.MyPlugin1], file, {})
assert manager.tell() == 0
manager.write(b"foo")
assert manager.tell() == file.tell() == 3
@coroutine_test
async def test_gzip_plugin(self):
filename = self._named_tempfile("gzip_file")

50
tests/test_feedexport_storages.py
View File

@ -3,6 +3,7 @@ from __future__ import annotations
import logging
import os
import string
import sys
import tempfile
from io import BytesIO
from pathlib import Path
@ -14,6 +15,7 @@ import pytest
from w3lib.url import path_to_file_uri
import scrapy
from scrapy.exceptions import NotConfigured
from scrapy.extensions.feedexport import (
BlockingFeedStorage,
FileFeedStorage,
@ -166,6 +168,12 @@ class TestFTPFeedStorage:
st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {})
assert st.password == string.punctuation
def test_uri_without_hostname(self):
with pytest.raises(
ValueError, match="Got a storage URI without a hostname: ftp:///some_path"
):
FTPFeedStorage("ftp:///some_path")
class MyBlockingFeedStorage(BlockingFeedStorage):
def _store_in_thread(self, file: IO[bytes]) -> None:
@ -205,6 +213,13 @@ class TestBlockingFeedStorage:
b.open(spider=spider)
def test_s3_without_boto3(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(sys.modules, "boto3", None)
monkeypatch.setitem(sys.modules, "boto3.session", None)
with pytest.raises(NotConfigured, match="missing boto3 library"):
S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key")
@pytest.mark.requires_boto3
class TestS3FeedStorage:
def test_parse_credentials(self):
@ -381,6 +396,41 @@ class TestS3FeedStorage:
assert storage.region_name == region_name
assert storage.s3_client._client_config.region_name == region_name
def test_init_without_max_pool_connections(self) -> None:
storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key")
assert storage.max_pool_connections is None
config: Any = storage.s3_client.meta.config
assert config.max_pool_connections == 10
def test_init_with_max_pool_connections(self) -> None:
storage = S3FeedStorage(
"s3://mybucket/export.csv",
"access_key",
"secret_key",
max_pool_connections=30,
)
assert storage.max_pool_connections == 30
config: Any = storage.s3_client.meta.config
assert config.max_pool_connections == 30
@pytest.mark.parametrize(
("settings", "expected"),
[
({}, 10),
({"REACTOR_THREADPOOL_MAXSIZE": 20}, 20),
({"AWS_MAX_POOL_CONNECTIONS": 30}, 30),
({"AWS_MAX_POOL_CONNECTIONS": 30, "REACTOR_THREADPOOL_MAXSIZE": 20}, 30),
],
)
def test_from_crawler_max_pool_connections(
self, settings: dict[str, Any], expected: int
) -> None:
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv")
assert storage.max_pool_connections == expected
config: Any = storage.s3_client.meta.config
assert config.max_pool_connections == expected
@coroutine_test
async def test_store_without_acl(self):
storage = S3FeedStorage(

247
tests/test_pipeline_files.py
View File

@ -33,7 +33,7 @@ from scrapy.pipelines.files import (
GCSFilesStore,
S3FilesStore,
)
from scrapy.pipelines.media import MediaPipeline, _MediaRequestFiltered
from scrapy.pipelines.media import _MediaRequestFiltered
from scrapy.settings import Settings
from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import maybe_deferred_to_future
@ -43,11 +43,7 @@ from tests.mockserver.ftp import MockFTPServer
from tests.utils.decorators import coroutine_test, inline_callbacks_test
from .utils.cloud import mock_google_cloud_storage
from .utils.media_pipelines import mocked_download_func
# required by persist_file() and stat_file(), but as some stores don't use the argument
# we can pass this singleton to keep type hints correct
DUMMY_SPIDER_INFO = MediaPipeline.SpiderInfo(DefaultSpider())
from .utils.media_pipelines import DUMMY_SPIDER_INFO, mocked_download_func
def get_ftp_content_and_delete(
@ -94,16 +90,19 @@ class DeferredFSFilesStore(FSFilesStore):
class TestFilesPipeline:
def setup_method(self):
self.tempdir = mkdtemp()
settings_dict = {"FILES_STORE": self.tempdir}
crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)
crawler.spider = crawler._create_spider()
crawler.engine = MagicMock(download_async=mocked_download_func)
self.pipeline = FilesPipeline.from_crawler(crawler)
self.pipeline.open_spider()
self.pipeline = self._create_pipeline(FilesPipeline)
def teardown_method(self):
rmtree(self.tempdir)
def _create_pipeline(self, pipeline_cls: type[FilesPipeline]) -> FilesPipeline:
crawler = get_crawler(DefaultSpider, {"FILES_STORE": self.tempdir})
crawler.spider = crawler._create_spider()
crawler.engine = MagicMock(download_async=mocked_download_func)
pipeline = pipeline_cls.from_crawler(crawler)
pipeline.open_spider()
return pipeline
def test_file_path_query_parameters(self):
file_path = self.pipeline.file_path
@ -254,6 +253,107 @@ class TestFilesPipeline:
assert result["files"][0]["checksum"] != "abc"
assert result["files"][0]["status"] == "cached"
@coroutine_test
async def test_file_stat_without_last_modified(self) -> None:
"""A stat result without a last modification time forces a download."""
item_url = "http://example.com/file4.pdf"
item = _create_item_with_files(item_url)
with (
mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
mock.patch.object(
FSFilesStore, "stat_file", return_value={"checksum": "abc"}
),
mock.patch.object(
FilesPipeline,
"get_media_requests",
return_value=[_prepare_request_object(item_url)],
),
):
result = await self.pipeline.process_item(item)
assert result["files"][0]["checksum"] != "abc"
assert result["files"][0]["status"] == "downloaded"
@coroutine_test
async def test_file_empty_content(self, caplog: pytest.LogCaptureFixture) -> None:
item_url = "http://example.com/empty.pdf"
item = _create_item_with_files(item_url)
request = Request(
item_url, meta={"response": Response(item_url, status=200, body=b"")}
)
with (
caplog.at_level(logging.WARNING),
mock.patch.object(
FilesPipeline, "get_media_requests", return_value=[request]
),
):
result = await self.pipeline.process_item(item)
assert result["files"] == []
assert "File (empty-content): Empty file from" in caplog.text
@coroutine_test
async def test_file_downloaded_file_exception(
self, caplog: pytest.LogCaptureFixture
) -> None:
"""A FileException from file_downloaded() is logged as a warning and
kept as is."""
class FailingFilesPipeline(FilesPipeline):
def file_downloaded(self, response, request, info, *, item=None):
raise FileException("boom")
item_url = "http://example.com/file5.pdf"
item = _create_item_with_files(item_url)
pipeline = self._create_pipeline(FailingFilesPipeline)
with (
caplog.at_level(logging.WARNING),
mock.patch.object(
FilesPipeline,
"get_media_requests",
return_value=[_prepare_request_object(item_url)],
),
):
result = await pipeline.process_item(item)
assert result["files"] == []
records = [
r for r in caplog.records if "Error processing file" in r.getMessage()
]
assert len(records) == 1
assert records[0].levelname == "WARNING"
assert "boom" in records[0].getMessage()
@coroutine_test
async def test_file_downloaded_unknown_error(
self, caplog: pytest.LogCaptureFixture
) -> None:
"""Any other exception from file_downloaded() is logged as an error and
reported as a FileException."""
class FailingFilesPipeline(FilesPipeline):
def file_downloaded(self, response, request, info, *, item=None):
raise RuntimeError("boom")
item_url = "http://example.com/file6.pdf"
item = _create_item_with_files(item_url)
pipeline = self._create_pipeline(FailingFilesPipeline)
with (
caplog.at_level(logging.WARNING),
mock.patch.object(
FilesPipeline,
"get_media_requests",
return_value=[_prepare_request_object(item_url)],
),
):
result = await pipeline.process_item(item)
assert result["files"] == []
records = [
r for r in caplog.records if "Error processing file" in r.getMessage()
]
assert len(records) == 1
assert records[0].levelname == "ERROR"
exc_info = records[0].exc_info
assert exc_info is not None
assert exc_info[0] is RuntimeError
@coroutine_test
async def test_async_store(self) -> None:
"""Test that async persist_file() works and is awaited."""
@ -648,9 +748,24 @@ class TestFilesPipelineCustomSettings:
request = Request("http://example.com/image01.jpg")
assert pipeline.file_path(request) == Path("subdir/image01.jpg")
def test_files_store_constructor_with_pathlike_object(self, tmp_path):
fs_store = FSFilesStore(tmp_path)
assert fs_store.basedir == str(tmp_path)
class TestFSFilesStore:
def test_constructor_with_pathlike_object(self, tmp_path: Path) -> None:
assert FSFilesStore(tmp_path).basedir == str(tmp_path)
def test_constructor_with_uri(self, tmp_path: Path) -> None:
assert FSFilesStore(f"file://{tmp_path}").basedir == str(tmp_path)
def test_stat_file(self, tmp_path: Path) -> None:
store = FSFilesStore(tmp_path)
store.persist_file("full/filename", BytesIO(b"data"), DUMMY_SPIDER_INFO)
stat = store.stat_file("full/filename", DUMMY_SPIDER_INFO)
assert stat["checksum"] == "8d777f385d3dfec8815d20f7496026dc"
assert stat["last_modified"] == pytest.approx(time.time(), abs=60)
def test_stat_missing_file(self, tmp_path: Path) -> None:
store = FSFilesStore(tmp_path)
assert store.stat_file("full/filename", DUMMY_SPIDER_INFO) == {}
@pytest.mark.requires_botocore
@ -695,6 +810,59 @@ class TestS3FilesStore:
# The call to read does not happen with Stubber
assert buffer.method_calls == [mock.call.seek(0)]
@inline_callbacks_test
def test_persist_without_headers(self):
"""Without custom headers only the default ones are sent."""
bucket = "mybucket"
key = "export.csv"
buffer = mock.MagicMock()
store = S3FilesStore(f"s3://{bucket}/{key}")
from botocore.stub import Stubber # noqa: PLC0415
with Stubber(store.s3_client) as stub:
stub.add_response(
"put_object",
expected_params={
"ACL": S3FilesStore.POLICY,
"Body": buffer,
"Bucket": bucket,
"CacheControl": S3FilesStore.HEADERS["Cache-Control"],
"Key": key,
"Metadata": {},
},
service_response={},
)
yield store.persist_file("", buffer, info=DUMMY_SPIDER_INFO)
stub.assert_no_pending_responses()
def test_missing_botocore(self):
with (
mock.patch(
"scrapy.pipelines.files.is_botocore_available", return_value=False
),
pytest.raises(NotConfigured, match="missing botocore library"),
):
S3FilesStore("s3://mybucket/key")
def test_wrong_uri_scheme(self):
with pytest.raises(
ValueError,
match=re.escape(
"Incorrect URI scheme in ftp://mybucket/key, expected 's3'"
),
):
S3FilesStore("ftp://mybucket/key")
def test_unsupported_header(self):
store = S3FilesStore("s3://mybucket/key")
with pytest.raises(
TypeError, match='Header "X-Custom" is not supported by botocore'
):
store._headers_to_botocore_kwargs({"X-Custom": "value"})
@inline_callbacks_test
def test_stat(self):
bucket = "mybucket"
@ -727,6 +895,33 @@ class TestS3FilesStore:
stub.assert_no_pending_responses()
def test_default_max_pool_connections(self) -> None:
store = S3FilesStore("s3://mybucket/prefix/")
config: Any = store.s3_client.meta.config
assert config.max_pool_connections == 10
@pytest.mark.parametrize(
("settings", "expected"),
[
({}, 10),
({"REACTOR_THREADPOOL_MAXSIZE": 20}, 20),
({"AWS_MAX_POOL_CONNECTIONS": 30}, 30),
({"AWS_MAX_POOL_CONNECTIONS": 30, "REACTOR_THREADPOOL_MAXSIZE": 20}, 30),
],
)
def test_max_pool_connections(
self, monkeypatch: pytest.MonkeyPatch, settings: dict[str, Any], expected: int
) -> None:
# restores the value that FilesPipeline.from_crawler() sets on the class
monkeypatch.setattr(S3FilesStore, "AWS_MAX_POOL_CONNECTIONS", None)
crawler = get_crawler(
settings_dict={"FILES_STORE": "s3://mybucket/prefix/", **settings}
)
store = FilesPipeline.from_crawler(crawler).store
assert isinstance(store, S3FilesStore)
config: Any = store.s3_client.meta.config
assert config.max_pool_connections == expected
class TestGCSFilesStore:
@staticmethod
@ -901,6 +1096,28 @@ class TestFTPFileStore:
)
assert data == content
@inline_callbacks_test
def test_persist_active_mode(self, monkeypatch: pytest.MonkeyPatch):
data = b"active mode"
path = "full/filename"
monkeypatch.setattr(FTPFilesStore, "FTP_USERNAME", "anonymous")
monkeypatch.setattr(FTPFilesStore, "FTP_PASSWORD", "guest")
monkeypatch.setattr(FTPFilesStore, "USE_ACTIVE_MODE", True)
with MockFTPServer() as ftp_server:
store = FTPFilesStore(ftp_server.url("/"))
yield store.persist_file(path, BytesIO(data), info=DUMMY_SPIDER_INFO)
stat = yield store.stat_file(path, info=DUMMY_SPIDER_INFO)
assert stat["checksum"] == "ff1575649a39a27c13faa0d37c84bab3"
def test_wrong_uri_scheme(self):
with pytest.raises(
ValueError,
match=re.escape(
"Incorrect URI scheme in http://example.com/, expected 'ftp'"
),
):
FTPFilesStore("http://example.com/")
class ItemWithFiles(Item):
file_urls = Field()

52
tests/test_pipeline_images.py
View File

@ -3,20 +3,26 @@ from __future__ import annotations
import dataclasses
import io
import random
import sys
from abc import ABC, abstractmethod
from pathlib import Path
from shutil import rmtree
from tempfile import mkdtemp
from types import SimpleNamespace
from typing import Any
import attr
import pytest
from itemadapter import ItemAdapter
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.item import Field, Item
from scrapy.pipelines.files import GCSFilesStore, S3FilesStore
from scrapy.pipelines.files import GCSFilesStore, S3FilesStore, _md5sum
from scrapy.pipelines.images import ImageException, ImagesPipeline
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
from tests.utils.media_pipelines import DUMMY_SPIDER_INFO
try:
from PIL import Image
@ -40,6 +46,11 @@ class TestImagesPipeline:
def teardown_method(self):
rmtree(self.tempdir)
def test_missing_pillow(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(sys.modules, "PIL", None)
with pytest.raises(NotConfigured, match="requires installing Pillow"):
ImagesPipeline(self.tempdir, crawler=get_crawler())
def test_file_path(self):
file_path = self.pipeline.file_path
assert (
@ -197,6 +208,25 @@ class TestImagesPipeline:
assert path == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"
assert new_im.getpixel((0, 0)) == (255, 0, 0)
@coroutine_test
async def test_image_downloaded(self) -> None:
"""The image and its thumbnails are stored, and the checksum of the
full-size image is returned."""
self.pipeline.thumbs = {"small": (20, 20)}
_, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
url = "https://dev.mydeco.com/mydeco.gif"
response = Response(url=url, body=buf.getvalue())
checksum = await self.pipeline.image_downloaded(
response, Request(url=url), DUMMY_SPIDER_INFO
)
buf.seek(0)
assert checksum == _md5sum(buf)
name = "3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"
assert Path(self.tempdir, "full", name).read_bytes() == buf.getvalue()
assert Path(self.tempdir, "thumbs", "small", name).exists()
def test_convert_image(self):
SIZE = (100, 100)
# straight forward case: RGB and JPEG
@ -230,6 +260,24 @@ class TestImagesPipeline:
assert converted.mode == "RGB"
assert converted.getcolors() == [(10000, (205, 230, 255))]
def test_convert_image_legacy_resampling_filter(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Pillow older than 9.1.0 has Image.ANTIALIAS instead of
Image.Resampling.LANCZOS."""
# Image.LANCZOS is the only spelling that exists in every supported
# Pillow version, but Pillow defines it dynamically, hence the ignore.
monkeypatch.setattr(
self.pipeline,
"_Image",
SimpleNamespace(ANTIALIAS=Image.LANCZOS), # type: ignore[attr-defined]
)
im, buf = _create_image("JPEG", "RGB", (100, 100), (0, 127, 255))
thumbnail, _ = self.pipeline.convert_image(im, size=(10, 25), response_body=buf)
assert thumbnail.size == (10, 10)
@pytest.mark.parametrize(
"bad_type",
[
@ -581,7 +629,7 @@ class TestImagesPipelineCustomSettings:
GCSFilesStore.POLICY = old_policy
def _create_image(format_, *a, **kw):
def _create_image(format_: str, *a: Any, **kw: Any) -> tuple[Image.Image, io.BytesIO]:
buf = io.BytesIO()
Image.new(*a, **kw).save(buf, format_)
buf.seek(0)

60
tests/test_pipeline_media.py
View File

@ -319,6 +319,42 @@ class TestMediaPipeline(TestBaseMediaPipeline):
assert self.fingerprint(req1) == self.fingerprint(req2)
assert new_item["results"] == [(True, {})]
@coroutine_test
async def test_failures_are_cached_across_multiple_items(self):
self.pipe.LOG_FAILED_RESULTS = False
exc = Exception("foo")
req1 = Request("http://url1", meta={"response": exc})
new_item = await self.pipe.process_item({"requests": req1})
assert new_item["results"][0][1].value is exc
# rsp2 is ignored, the cached failure must be reused because request
# fingerprints are the same
req2 = Request(
req1.url, meta={"response": Response("http://donot.download.me")}
)
new_item = await self.pipe.process_item({"requests": req2})
assert new_item["results"][0][0] is False
assert new_item["results"][0][1].value is exc
assert self.pipe._mockcalled.count("media_to_download") == 1
@coroutine_test
async def test_cached_failure_calls_errback(self):
"""The errback of a request is called for a cached failure as well."""
self.pipe.LOG_FAILED_RESULTS = False
exc = Exception("foo")
await self.pipe.process_item(
{"requests": Request("http://url1", meta={"response": exc})}
)
def errback(failure):
self.pipe._mockcalled.append("request_errback")
return {"recovered": failure.value}
req = Request("http://url1", errback=errback)
new_item = await self.pipe.process_item({"requests": req})
assert new_item["results"] == [(True, {"recovered": exc})]
assert self.pipe._mockcalled.count("request_errback") == 1
@coroutine_test
async def test_results_are_cached_for_requests_of_single_item(self):
rsp1 = Response("http://url1")
@ -472,6 +508,30 @@ class TestBuildFromCrawler:
assert pipe._from_crawler_called
class MediaFailedNonePipeline(MockedMediaPipeline):
def media_failed(self, failure, request, info):
self._mockcalled.append("media_failed")
class TestMediaFailedNone(TestBaseMediaPipeline):
"""Test what happens when media_failed() neither raises an exception nor
returns a failure."""
pipeline_class = MediaFailedNonePipeline
@coroutine_test
async def test_result_none(self):
req = Request("http://url1", meta={"response": Exception("foo")})
new_item = await self.pipe.process_item({"requests": req})
assert new_item["results"] == [(True, None)]
assert self.pipe._mockcalled == [
"get_media_requests",
"media_to_download",
"media_failed",
"item_completed",
]
class MediaFailedFailurePipeline(MockedMediaPipeline):
def media_failed(self, failure, request, info):
self._mockcalled.append("media_failed")

27
tests/test_robotstxt_interface.py
View File

@ -4,6 +4,7 @@ from scrapy.robotstxt import (
ProtegoRobotParser,
PythonRobotParser,
RerpRobotParser,
RobotParser,
decode_robotstxt,
)
from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER
@ -78,6 +79,16 @@ class BaseRobotParserTest:
assert rp.allowed("https://site.local/index.html", "*")
assert rp.allowed("https://site.local/disallowed", "*")
def test_crawl_delay(self):
robotstxt_body = b"User-agent: *\nDisallow: /private\nCrawl-delay: 10\n"
rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)
assert rp.crawl_delay("*") == 10.0
def test_crawl_delay_unset(self):
robotstxt_body = b"User-agent: *\nDisallow: /private\n"
rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)
assert rp.crawl_delay("*") is None
def test_unicode_url_and_useragent(self):
robotstxt_robotstxt_body = """
User-Agent: *
@ -102,6 +113,22 @@ class BaseRobotParserTest:
assert not rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt")
class TestRobotParser:
def test_crawl_delay_unsupported(self):
class AllowAllRobotParser(RobotParser):
@classmethod
def from_crawler(cls, crawler, robotstxt_body):
return cls()
def allowed(self, url, user_agent):
return True
rp = AllowAllRobotParser.from_crawler(
crawler=None, robotstxt_body=b"User-agent: *\nCrawl-delay: 10\n"
)
assert rp.crawl_delay("*") is None
class TestDecodeRobotsTxt:
def test_native_string_conversion(self):
robotstxt_body = b"User-agent: *\nDisallow: /\n"

130
tests/test_spider.py
View File

@ -1,11 +1,26 @@
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from scrapy.http import Response, TextResponse, XmlResponse
from scrapy.http import Request, Response, TextResponse, XmlResponse
from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider
from tests import get_testdata
from tests.spiders import RawResponseSpider
from tests.utils.bases.spider import TestSpiderBase
from tests.utils.crawl import crawl_items
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from tests.mockserver.http import MockServer
class RawFeedSpider(RawResponseSpider):
content_type = "text/xml"
async def start(self):
yield Request(self.raw_url)
class TestSpider(TestSpiderBase):
@ -60,6 +75,89 @@ class TestXMLFeedSpider(TestSpiderBase):
},
], iterator
@coroutine_test
async def test_parse_node_uses_parse_item(self, mockserver: MockServer):
# parse_node falls back to parse_item for backward compatibility.
class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc]
itertag = "item"
def raw_body(self):
return "<items><item><id>1</id></item></items>"
def parse_item(self, response, selector):
return {"id": selector.xpath("id/text()").get()}
items, _ = await crawl_items(_Spider, mockserver)
assert items == [{"id": "1"}]
@coroutine_test
async def test_parse_node_not_defined(self, mockserver: MockServer):
class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc]
itertag = "item"
def raw_body(self):
return "<items><item><id>1</id></item></items>"
items, crawler = await crawl_items(_Spider, mockserver)
assert items == []
assert crawler.stats
assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1
@coroutine_test
async def test_html_iterator(self, mockserver: MockServer):
class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc]
iterator = "html"
itertag = "item"
content_type = "text/html"
def raw_body(self):
return (
"<html><body><item><id>1</id></item>"
"<item><id>2</id></item></body></html>"
)
def parse_node(self, response, selector):
return {"id": selector.xpath("id/text()").get()}
items, _ = await crawl_items(_Spider, mockserver)
assert items == [{"id": "1"}, {"id": "2"}]
@coroutine_test
async def test_unsupported_iterator(self, mockserver: MockServer):
class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc]
iterator = "unsupported"
def raw_body(self):
return "<items><item/></items>"
def parse_node(self, response, selector):
return {}
items, crawler = await crawl_items(_Spider, mockserver)
assert items == []
assert crawler.stats
assert crawler.stats.get_value("spider_exceptions/NotSupported") == 1
@pytest.mark.parametrize("feed_iterator", ["xml", "html"])
@coroutine_test
async def test_non_text_response(self, feed_iterator: str, mockserver: MockServer):
# The xml and html iterators require a text response.
class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc]
content_type = "application/octet-stream"
iterator = feed_iterator
def raw_body(self):
# A binary (non-text) body, so the response is a plain Response.
return "\x00\x01\x02\x03"
def parse_node(self, response, selector):
return {}
items, crawler = await crawl_items(_Spider, mockserver)
assert items == []
assert crawler.stats
assert crawler.stats.get_value("spider_exceptions/ValueError") == 1
class TestCSVFeedSpider(TestSpiderBase):
spider_class = CSVFeedSpider
@ -81,6 +179,36 @@ class TestCSVFeedSpider(TestSpiderBase):
assert rows[0] == {"id": "1", "name": "alpha", "value": "foobar"}
assert len(rows) == 4
@coroutine_test
async def test_parse(self, mockserver: MockServer):
class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc]
content_type = "text/csv"
delimiter = ","
quotechar = "'"
def raw_body(self):
return get_testdata("feeds", "feed-sample6.csv").decode()
def parse_row(self, response, row):
return row
items, _ = await crawl_items(_Spider, mockserver)
assert items[0] == {"id": "1", "name": "alpha", "value": "foobar"}
assert len(items) == 4
@coroutine_test
async def test_parse_row_not_defined(self, mockserver: MockServer):
class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc]
content_type = "text/csv"
def raw_body(self):
return "id\n1\n"
items, crawler = await crawl_items(_Spider, mockserver)
assert items == []
assert crawler.stats
assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1
class TestNoParseMethodSpider:
spider_class = Spider

43
tests/test_spider_crawl.py
View File

@ -12,6 +12,7 @@ from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule, Spider
from scrapy.utils.test import get_crawler
from tests.utils.bases.spider import TestSpiderBase
from tests.utils.decorators import coroutine_test
class TestCrawlSpider(TestSpiderBase):
@ -293,6 +294,48 @@ class TestCrawlSpider(TestSpiderBase):
TextResponse(spider.start_urls, body=b""), None, None
)
@coroutine_test
async def test_parse_with_rules_without_callback(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(CrawlSpider):
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(),)
spider = _CrawlSpider.from_crawler(get_crawler(_CrawlSpider))
results = [
r async for r in spider.parse_with_rules(response, None, {}, follow=True)
]
assert [r.url for r in results] == [
"http://example.org/somepage/item/12.html",
"http://example.org/about.html",
"http://example.org/nofollow.html",
]
@coroutine_test
async def test_parse_with_rules_without_following(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
item = {"name": "item"}
class _CrawlSpider(CrawlSpider):
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(),)
spider = _CrawlSpider.from_crawler(get_crawler(_CrawlSpider))
results = [
r
async for r in spider.parse_with_rules(
response, lambda response: [item], {}, follow=False
)
]
assert results == [item]
class TestDeprecation:
def test_crawl_spider(self):

62
tests/test_spider_sitemap.py
View File

@ -6,6 +6,7 @@ from datetime import datetime
from io import BytesIO
from logging import WARNING
from pathlib import Path
from typing import TYPE_CHECKING
import pytest
@ -13,9 +14,30 @@ from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlRespon
from scrapy.spiders import SitemapSpider
from scrapy.utils.test import get_crawler
from tests import tests_datadir
from tests.spiders import RawResponseSpider
from tests.utils.bases.spider import TestSpiderBase
from tests.utils.crawl import crawl_items
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from tests.mockserver.http import MockServer
class RawSitemapSpider(RawResponseSpider):
"""Feeds :meth:`raw_body` to :class:`~scrapy.spiders.SitemapSpider` as a
sitemap, so that it is fetched and followed through a regular crawl.
Subclasses build the document in :meth:`raw_body`, typically using
:attr:`mockserver` to point ``<loc>`` entries at real endpoints.
"""
content_type = "application/xml"
async def start(self):
self.sitemap_urls = [self.raw_url]
async for request in super().start():
yield request
class TestSitemapSpider(TestSpiderBase):
spider_class = SitemapSpider
@ -253,6 +275,46 @@ Sitemap: /sitemap-relative-url.xml
urls = [req.url for req in spider._parse_sitemap(r)]
assert urls == result
@coroutine_test
async def test_sitemap_rules_with_callable(self, mockserver: MockServer):
# A sitemap_rules entry may hold a callable instead of a method name.
def parse_item(response):
yield {"url": response.url}
class _Spider(RawSitemapSpider, self.spider_class): # type: ignore[name-defined,misc]
sitemap_rules = [("", parse_item)]
def raw_body(self):
loc = self.mockserver.url("/text")
return (
'<?xml version="1.0" encoding="UTF-8"?>'
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
f"<url><loc>{loc}</loc></url>"
"</urlset>"
)
items, _ = await crawl_items(_Spider, mockserver)
assert items == [{"url": mockserver.url("/text")}]
@coroutine_test
async def test_sitemap_empty_loc(self, mockserver: MockServer):
class _Spider(RawSitemapSpider, self.spider_class): # type: ignore[name-defined,misc]
def parse(self, response):
yield {"url": response.url}
def raw_body(self):
loc = self.mockserver.url("/text")
return (
'<?xml version="1.0" encoding="UTF-8"?>'
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
"<url><loc></loc></url>"
f"<url><loc>{loc}</loc></url>"
"</urlset>"
)
items, _ = await crawl_items(_Spider, mockserver)
assert items == [{"url": mockserver.url("/text")}]
def test_parse_sitemap_empty_body(self, caplog: pytest.LogCaptureFixture) -> None:
r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"")
spider = self.spider_class("example.com")

24
tests/test_utils_asyncgen.py
View File

@ -1,16 +1,18 @@
from __future__ import annotations
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from tests.utils.decorators import coroutine_test
class TestAsyncgenUtils:
@coroutine_test
async def test_as_async_generator(self):
ag = as_async_generator(range(42))
results = [i async for i in ag]
assert results == list(range(42))
@coroutine_test
async def test_as_async_generator():
ag = as_async_generator(range(42))
results = [i async for i in ag]
assert results == list(range(42))
@coroutine_test
async def test_collect_asyncgen(self):
ag = as_async_generator(range(42))
results = await collect_asyncgen(ag)
assert results == list(range(42))
@coroutine_test
async def test_collect_asyncgen():
ag = as_async_generator(range(42))
results = await collect_asyncgen(ag)
assert results == list(range(42))

20
tests/test_utils_asyncio.py
View File

@ -12,7 +12,9 @@ from scrapy.utils.asyncgen import as_async_generator
from scrapy.utils.asyncio import (
AsyncioLoopingCall,
_parallel_asyncio,
call_later,
is_asyncio_available,
sleep,
)
from tests.utils.decorators import coroutine_test
@ -20,11 +22,19 @@ if TYPE_CHECKING:
from collections.abc import AsyncGenerator
class TestAsyncio:
@coroutine_test
async def test_is_asyncio_available(self, reactor_pytest: str) -> None:
# the result should depend only on the pytest --reactor argument
assert is_asyncio_available() == (reactor_pytest != "default")
@coroutine_test
async def test_is_asyncio_available(reactor_pytest: str) -> None:
# the result should depend only on the pytest --reactor argument
assert is_asyncio_available() == (reactor_pytest != "default")
@coroutine_test
async def test_sleep() -> None:
events: list[str] = []
call_later(0.05, events.append, "call_later")
await sleep(0.1)
events.append("sleep")
assert events == ["call_later", "sleep"]
@pytest.mark.only_asyncio

19
tests/test_utils_console.py
View File

@ -4,7 +4,7 @@ from importlib.util import find_spec
import pytest
from scrapy.utils.console import get_shell_embed_func
from scrapy.utils.console import get_shell_embed_func, start_python_console
def test_get_shell_embed_func():
@ -59,3 +59,20 @@ def test_get_shell_embed_func_default():
else:
expected = "_embed_standard_shell"
assert shell.__name__ == expected
def test_start_python_console_exit(monkeypatch: pytest.MonkeyPatch) -> None:
def embed(namespace: dict[str, object], banner: str) -> None:
raise SystemExit
monkeypatch.setattr(
"scrapy.utils.console.get_shell_embed_func", lambda shells: embed
)
start_python_console()
def test_start_python_console_no_shell(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
"scrapy.utils.console.get_shell_embed_func", lambda shells: None
)
start_python_console()

2
tests/test_utils_curl.py
View File

@ -1,3 +1,5 @@
from __future__ import annotations
import warnings
from typing import Any

89
tests/test_utils_datatypes.py
View File

@ -1,7 +1,9 @@
from __future__ import annotations
import copy
from abc import ABC, abstractmethod
from collections.abc import Iterator, Mapping, MutableMapping
from typing import Any
from typing import Any, Generic, TypeVar
import pytest
@ -16,11 +18,13 @@ from scrapy.utils.datatypes import (
)
from scrapy.utils.python import garbage_collect
_DictT = TypeVar("_DictT", bound="CaselessDict | CaseInsensitiveDict")
class TestCaseInsensitiveDictBase(ABC):
class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]):
@property
@abstractmethod
def dict_class(self) -> type[MutableMapping[str, Any]]:
def dict_class(self) -> type[_DictT]:
raise NotImplementedError
def test_init_dict(self):
@ -36,17 +40,17 @@ class TestCaseInsensitiveDictBase(ABC):
assert d["black"] == 3
def test_init_mapping(self):
class MyMapping(Mapping):
def __init__(self, **kwargs):
class MyMapping(Mapping[str, int]):
def __init__(self, **kwargs: int) -> None:
self._d = kwargs
def __getitem__(self, key):
def __getitem__(self, key: str) -> int:
return self._d[key]
def __iter__(self):
def __iter__(self) -> Iterator[str]:
return iter(self._d)
def __len__(self):
def __len__(self) -> int:
return len(self._d)
seq = MyMapping(red=1, black=3)
@ -55,23 +59,23 @@ class TestCaseInsensitiveDictBase(ABC):
assert d["black"] == 3
def test_init_mutable_mapping(self):
class MyMutableMapping(MutableMapping):
def __init__(self, **kwargs):
class MyMutableMapping(MutableMapping[str, int]):
def __init__(self, **kwargs: int) -> None:
self._d = kwargs
def __getitem__(self, key):
def __getitem__(self, key: str) -> int:
return self._d[key]
def __setitem__(self, key, value):
def __setitem__(self, key: str, value: int) -> None:
self._d[key] = value
def __delitem__(self, key):
def __delitem__(self, key: str) -> None:
del self._d[key]
def __iter__(self):
def __iter__(self) -> Iterator[str]:
return iter(self._d)
def __len__(self):
def __len__(self) -> int:
return len(self._d)
seq = MyMutableMapping(red=1, black=3)
@ -149,7 +153,7 @@ class TestCaseInsensitiveDictBase(ABC):
d.pop("A")
def test_normkey(self):
class MyDict(self.dict_class):
class MyDict(self.dict_class): # type: ignore[misc,name-defined]
def _normkey(self, key):
return key.title()
@ -160,7 +164,7 @@ class TestCaseInsensitiveDictBase(ABC):
assert list(d.keys()) == ["Key-One"]
def test_normvalue(self):
class MyDict(self.dict_class):
class MyDict(self.dict_class): # type: ignore[misc,name-defined]
def _normvalue(self, value):
if value is not None:
return value + 1
@ -204,9 +208,18 @@ class TestCaseInsensitiveDictBase(ABC):
assert h1.get("header1") == h3.get("header1")
assert h1.get("header1") == h3.get("HEADER1")
def test_copy_is_independent(self):
h1 = self.dict_class({"header1": "value1", "header2": "value2"})
for h2 in (copy.copy(h1), h1.copy()):
del h2["header1"]
h2["header3"] = "value3"
assert "header1" in h1
assert "header3" not in h1
assert dict(h1) == {"header1": "value1", "header2": "value2"}
class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
dict_class = CaseInsensitiveDict # type: ignore[assignment]
class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase[CaseInsensitiveDict]):
dict_class = CaseInsensitiveDict
def test_repr(self):
d1 = self.dict_class({"foo": "bar"})
@ -220,9 +233,31 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
assert isinstance(iterkeys, Iterator)
assert list(iterkeys) == ["AsDf", "FoO"]
def test_copy_keeps_values(self):
class MyDict(self.dict_class): # type: ignore[misc,name-defined]
def _normvalue(self, value):
return value + 1
d = MyDict({"key": 1})
for copied in (copy.copy(d), d.copy()):
assert copied["key"] == 2
def test_ior(self):
d = self.dict_class({"header1": "value1"})
d |= {"HEADER1": "value2", "header2": "value3"}
assert len(d) == 2
assert d["HeAdEr1"] == "value2"
assert d["HeAdEr2"] == "value3"
def test_ior_mapping(self):
d = self.dict_class({"header1": "value1"})
d |= self.dict_class({"HEADER1": "value2"})
assert len(d) == 1
assert d["HeAdEr1"] == "value2"
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
class TestCaselessDict(TestCaseInsensitiveDictBase):
class TestCaselessDict(TestCaseInsensitiveDictBase[CaselessDict]):
dict_class = CaselessDict
def test_deprecation_message(self):
@ -288,7 +323,7 @@ class TestSequenceExclude:
class TestLocalCache:
def test_cache_with_limit(self):
cache = LocalCache(limit=2)
cache: LocalCache[str, int] = LocalCache(limit=2)
cache["a"] = 1
cache["b"] = 2
cache["c"] = 3
@ -301,7 +336,7 @@ class TestLocalCache:
def test_cache_without_limit(self):
maximum = 10**4
cache = LocalCache()
cache: LocalCache[str, int] = LocalCache()
for x in range(maximum):
cache[str(x)] = x
assert len(cache) == maximum
@ -310,7 +345,7 @@ class TestLocalCache:
assert cache[str(x)] == x
def test_cache_with_zero_limit(self):
cache = LocalCache(limit=0)
cache: LocalCache[str, int] = LocalCache(limit=0)
cache["a"] = 1
cache["b"] = 2
cache["c"] = 3
@ -322,7 +357,9 @@ class TestLocalCache:
class TestLocalWeakReferencedCache:
def test_cache_with_limit(self):
cache = LocalWeakReferencedCache(limit=2)
cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache(
limit=2
)
r1 = Request("https://example.org")
r2 = Request("https://example.com")
r3 = Request("https://example.net")
@ -344,7 +381,7 @@ class TestLocalWeakReferencedCache:
assert len(cache) == 1
def test_cache_non_weak_referenceable_objects(self):
cache = LocalWeakReferencedCache()
cache: LocalWeakReferencedCache[Any, int] = LocalWeakReferencedCache()
k1 = None
k2 = 1
k3 = [1, 2, 3]
@ -358,7 +395,7 @@ class TestLocalWeakReferencedCache:
def test_cache_without_limit(self):
maximum = 10**4
cache = LocalWeakReferencedCache()
cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache()
refs = []
for x in range(maximum):
refs.append(Request(f"https://example.org/{x}"))

46
tests/test_utils_decorators.py
View File

@ -1,6 +1,8 @@
from __future__ import annotations
import sys
import warnings
from typing import TYPE_CHECKING, Any
import pytest
from twisted.internet.defer import Deferred
@ -10,11 +12,14 @@ from scrapy.utils.decorators import _warn_spider_arg, deprecated, inthread
from scrapy.utils.defer import maybe_deferred_to_future
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Callable
class TestDeprecated:
def test_warns_and_still_calls(self):
@deprecated()
def add(a, b):
def add(a: int, b: int) -> int:
return a + b
with pytest.warns(
@ -26,7 +31,7 @@ class TestDeprecated:
def test_use_instead_in_message(self):
@deprecated(use_instead="other_function")
def old():
def old() -> None:
return None
with pytest.warns(
@ -37,7 +42,7 @@ class TestDeprecated:
def test_applied_without_parentheses(self):
@deprecated
def square(x):
def square(x: int) -> int:
return x * x
with pytest.warns(
@ -65,7 +70,7 @@ class TestInthread:
class TestWarnSpiderArg:
def test_sync_warns_with_spider_arg(self):
@_warn_spider_arg
def parse(response, spider=None):
def parse(response: str, spider: str | None = None) -> str:
return response
with pytest.warns(
@ -73,9 +78,34 @@ class TestWarnSpiderArg:
):
assert parse("response", spider="spider") == "response"
@pytest.mark.skipif(
sys.version_info < (3, 14),
reason="annotations are only lazily evaluated since Python 3.14 (PEP 649)",
)
def test_sync_warns_with_unresolvable_annotations(self):
# dont_inherit=True, or the module's future import stringizes the annotations
namespace: dict[str, Any] = {}
exec( # pylint: disable=exec-used
compile(
"def parse(response: OnlyAtTypeCheckingTime,"
" spider: OnlyAtTypeCheckingTime | None = None): return response",
"<test>",
"exec",
dont_inherit=True,
),
namespace,
)
parse_func: Callable[..., str] = namespace["parse"]
parse = _warn_spider_arg(parse_func)
with pytest.warns(
ScrapyDeprecationWarning, match=r"Passing a 'spider' argument"
):
assert parse("response", spider="spider") == "response"
def test_sync_no_warning_without_spider_arg(self):
@_warn_spider_arg
def parse(response, spider=None):
def parse(response: str, spider: str | None = None) -> str:
return response
with warnings.catch_warnings():
@ -85,7 +115,7 @@ class TestWarnSpiderArg:
@coroutine_test
async def test_async_warns_with_spider_arg(self):
@_warn_spider_arg
async def parse(response, spider=None):
async def parse(response: str, spider: str | None = None) -> str:
return response
with pytest.warns(
@ -96,7 +126,9 @@ class TestWarnSpiderArg:
@coroutine_test
async def test_asyncgen_warns_with_spider_arg(self):
@_warn_spider_arg
async def parse(response, spider=None):
async def parse(
response: str, spider: str | None = None
) -> AsyncGenerator[str]:
yield response
with pytest.warns(

24
tests/test_utils_defer.py
View File

@ -24,6 +24,8 @@ from tests.utils.decorators import coroutine_test, inline_callbacks_test
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Awaitable, Callable, Generator
from twisted.python.failure import Failure
@pytest.mark.requires_reactor # mustbe_deferred() requires a reactor
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
@ -70,7 +72,7 @@ class TestIterErrback:
def itergood() -> Generator[int, None, None]:
yield from range(10)
errors = []
errors: list[Failure] = []
out = list(iter_errback(itergood(), errors.append))
assert out == list(range(10))
assert not errors
@ -82,7 +84,7 @@ class TestIterErrback:
1 / 0
yield x
errors = []
errors: list[Failure] = []
out = list(iter_errback(iterbad(), errors.append))
assert out == [0, 1, 2, 3, 4]
assert len(errors) == 1
@ -96,7 +98,7 @@ class TestAiterErrback:
for x in range(10):
yield x
errors = []
errors: list[Failure] = []
out = await collect_asyncgen(aiter_errback(itergood(), errors.append))
assert out == list(range(10))
assert not errors
@ -109,7 +111,7 @@ class TestAiterErrback:
1 / 0
yield x
errors = []
errors: list[Failure] = []
out = await collect_asyncgen(aiter_errback(iterbad(), errors.append))
assert out == [0, 1, 2, 3, 4]
assert len(errors) == 1
@ -202,7 +204,7 @@ class TestParallelAsync:
for length in [20, 50, 100]:
parallel_count = [0]
max_parallel_count = [0]
results = []
results: list[int] = []
ait = self.get_async_iterable(length)
dl = parallel_async(
ait,
@ -222,7 +224,7 @@ class TestParallelAsync:
for length in [20, 50, 100]:
parallel_count = [0]
max_parallel_count = [0]
results = []
results: list[int] = []
ait = self.get_async_iterable_with_delays(length)
dl = parallel_async(
ait,
@ -240,7 +242,7 @@ class TestParallelAsync:
class TestDeferredFromCoro:
def test_deferred(self):
d = Deferred()
d: Deferred[None] = Deferred()
result = deferred_from_coro(d)
assert isinstance(result, Deferred)
assert result is d
@ -274,7 +276,7 @@ class TestDeferredFromCoro:
@pytest.mark.only_asyncio
@inline_callbacks_test
def test_future(self):
future = Future()
future: Future[int] = Future()
result = deferred_from_coro(future)
assert isinstance(result, Deferred)
future.set_result(42)
@ -324,7 +326,7 @@ class TestDeferredFFromCoroF:
class TestDeferredToFuture:
@coroutine_test
async def test_deferred(self):
d = Deferred()
d: Deferred[int] = Deferred()
result = deferred_to_future(d)
assert isinstance(result, Future)
d.callback(42)
@ -359,7 +361,7 @@ class TestDeferredToFuture:
class TestMaybeDeferredToFutureAsyncio:
@coroutine_test
async def test_deferred(self):
d = Deferred()
d: Deferred[int] = Deferred()
result = maybe_deferred_to_future(d)
assert isinstance(result, Future)
d.callback(42)
@ -394,7 +396,7 @@ class TestMaybeDeferredToFutureAsyncio:
class TestMaybeDeferredToFutureNotAsyncio:
@coroutine_test
async def test_deferred(self):
d = Deferred()
d: Deferred[int] = Deferred()
result = maybe_deferred_to_future(d)
assert isinstance(result, Deferred)
assert result is d

46
tests/test_utils_deprecate.py
View File

@ -1,3 +1,5 @@
from __future__ import annotations
import inspect
import warnings
from unittest import mock
@ -38,7 +40,7 @@ class TestWarnWhenSubclassed:
)
with pytest.warns(MyWarning, match=msg) as w:
class UserClass(Deprecated):
class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
assert w[0].lineno == inspect.getsourcelines(UserClass)[1]
@ -57,7 +59,7 @@ class TestWarnWhenSubclassed:
match=r"UserClass inherits from deprecated class bar\.OldClass, please inherit from foo\.NewClass",
):
class UserClass(Deprecated):
class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
with pytest.warns(
@ -76,7 +78,7 @@ class TestWarnWhenSubclassed:
match="UserClass inherits from deprecated class",
):
class UserClass(Deprecated):
class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
with warnings.catch_warnings():
@ -95,16 +97,16 @@ class TestWarnWhenSubclassed:
match="UserClass inherits from deprecated class",
):
class UserClass(Deprecated):
class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
with warnings.catch_warnings():
warnings.simplefilter("error", MyWarning)
class FooClass(Deprecated):
class FooClass(Deprecated): # type: ignore[misc, valid-type]
pass
class BarClass(Deprecated):
class BarClass(Deprecated): # type: ignore[misc, valid-type]
pass
def test_warning_on_instance(self):
@ -112,22 +114,20 @@ class TestWarnWhenSubclassed:
"Deprecated", NewName, warn_category=MyWarning
)
with pytest.warns(MyWarning) as w:
_, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())
w = [x for x in w if x.category is MyWarning]
with pytest.warns(
MyWarning,
match=r"tests\.test_utils_deprecate\.Deprecated is deprecated, "
r"instantiate tests\.test_utils_deprecate\.NewName instead\.",
) as w:
_, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) # type: ignore[arg-type]
assert len(w) == 1
assert (
str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, "
"instantiate tests.test_utils_deprecate.NewName instead."
)
assert w[0].lineno == lineno
# ignore subclassing warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore", MyWarning)
class UserClass(Deprecated):
class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
with warnings.catch_warnings():
@ -141,7 +141,7 @@ class TestWarnWhenSubclassed:
match=r"UserClass2 inherits from deprecated class tests\.test_utils_deprecate\.Deprecated, please inherit from tests\.test_utils_deprecate\.NewName",
):
class UserClass2(Deprecated):
class UserClass2(Deprecated): # type: ignore[misc, valid-type]
pass
def test_issubclass(self):
@ -155,10 +155,10 @@ class TestWarnWhenSubclassed:
class UpdatedUserClass1a(NewName):
pass
class OutdatedUserClass1(DeprecatedName):
class OutdatedUserClass1(DeprecatedName): # type: ignore[misc, valid-type]
pass
class OutdatedUserClass1a(DeprecatedName):
class OutdatedUserClass1a(DeprecatedName): # type: ignore[misc, valid-type]
pass
class UnrelatedClass:
@ -174,7 +174,7 @@ class TestWarnWhenSubclassed:
assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1)
with pytest.raises(TypeError):
issubclass(object(), DeprecatedName)
issubclass(object(), DeprecatedName) # type: ignore[arg-type]
def test_isinstance(self):
with warnings.catch_warnings():
@ -187,10 +187,10 @@ class TestWarnWhenSubclassed:
class UpdatedUserClass2a(NewName):
pass
class OutdatedUserClass2(DeprecatedName):
class OutdatedUserClass2(DeprecatedName): # type: ignore[misc, valid-type]
pass
class OutdatedUserClass2a(DeprecatedName):
class OutdatedUserClass2a(DeprecatedName): # type: ignore[misc, valid-type]
pass
class UnrelatedClass:
@ -211,7 +211,7 @@ class TestWarnWhenSubclassed:
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"})
assert Deprecated.foo == "bar"
assert Deprecated.foo == "bar" # type: ignore[attr-defined]
def test_deprecate_a_class_with_custom_metaclass(self):
Meta1 = type("Meta1", (type,), {})
@ -242,7 +242,7 @@ class TestWarnWhenSubclassed:
match=r"UserClass inherits from deprecated class tests\.test_utils_deprecate\.AlsoDeprecated, please inherit from foo\.Bar",
):
class UserClass(AlsoDeprecated):
class UserClass(AlsoDeprecated): # type: ignore[misc, valid-type]
pass
def test_inspect_stack(self):

16
tests/test_utils_display.py
View File

@ -31,13 +31,13 @@ plain_string = "{'a': 1}"
@mock.patch("sys.platform", "linux")
@mock.patch("sys.stdout.isatty")
def test_pformat(isatty):
def test_pformat(isatty: mock.Mock) -> None:
isatty.return_value = True
assert pformat(value) in colorized_strings
@mock.patch("sys.stdout.isatty")
def test_pformat_dont_colorize(isatty):
def test_pformat_dont_colorize(isatty: mock.Mock) -> None:
isatty.return_value = True
assert pformat(value, colorize=False) == plain_string
@ -49,7 +49,7 @@ def test_pformat_not_tty():
@mock.patch("sys.platform", "win32")
@mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
def test_pformat_old_windows(isatty, version):
def test_pformat_old_windows(isatty: mock.Mock, version: mock.Mock) -> None:
isatty.return_value = True
version.return_value = "10.0.14392"
assert pformat(value) in colorized_strings
@ -59,7 +59,9 @@ def test_pformat_old_windows(isatty, version):
@mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
@mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing):
def test_pformat_windows_no_terminal_processing(
isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock
) -> None:
isatty.return_value = True
version.return_value = "10.0.14393"
terminal_processing.return_value = False
@ -70,7 +72,9 @@ def test_pformat_windows_no_terminal_processing(isatty, version, terminal_proces
@mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
@mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
def test_pformat_windows(isatty, version, terminal_processing):
def test_pformat_windows(
isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock
) -> None:
isatty.return_value = True
version.return_value = "10.0.14393"
terminal_processing.return_value = True
@ -79,7 +83,7 @@ def test_pformat_windows(isatty, version, terminal_processing):
@mock.patch("sys.platform", "linux")
@mock.patch("sys.stdout.isatty")
def test_pformat_no_pygments(isatty):
def test_pformat_no_pygments(isatty: mock.Mock) -> None:
isatty.return_value = True
real_import = builtins.__import__

Some files were not shown because too many files have changed in this diff Show More